xref: /illumos-gate/usr/src/uts/common/vm/vm_page.c (revision 66582b606a8194f7f3ba5b3a3a6dca5b0d346361)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2015, Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
24  * Copyright (c) 2015, 2016 by Delphix. All rights reserved.
25  */
26 
27 /*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989  AT&T	*/
28 /*	  All Rights Reserved  	*/
29 
30 /*
31  * University Copyright- Copyright (c) 1982, 1986, 1988
32  * The Regents of the University of California
33  * All Rights Reserved
34  *
35  * University Acknowledgment- Portions of this document are derived from
36  * software developed by the University of California, Berkeley, and its
37  * contributors.
38  */
39 
40 /*
41  * VM - physical page management.
42  */
43 
44 #include <sys/types.h>
45 #include <sys/t_lock.h>
46 #include <sys/param.h>
47 #include <sys/systm.h>
48 #include <sys/errno.h>
49 #include <sys/time.h>
50 #include <sys/vnode.h>
51 #include <sys/vm.h>
52 #include <sys/vtrace.h>
53 #include <sys/swap.h>
54 #include <sys/cmn_err.h>
55 #include <sys/tuneable.h>
56 #include <sys/sysmacros.h>
57 #include <sys/cpuvar.h>
58 #include <sys/callb.h>
59 #include <sys/debug.h>
60 #include <sys/tnf_probe.h>
61 #include <sys/condvar_impl.h>
62 #include <sys/mem_config.h>
63 #include <sys/mem_cage.h>
64 #include <sys/kmem.h>
65 #include <sys/atomic.h>
66 #include <sys/strlog.h>
67 #include <sys/mman.h>
68 #include <sys/ontrap.h>
69 #include <sys/lgrp.h>
70 #include <sys/vfs.h>
71 
72 #include <vm/hat.h>
73 #include <vm/anon.h>
74 #include <vm/page.h>
75 #include <vm/seg.h>
76 #include <vm/pvn.h>
77 #include <vm/seg_kmem.h>
78 #include <vm/vm_dep.h>
79 #include <sys/vm_usage.h>
80 #include <fs/fs_subr.h>
81 #include <sys/ddi.h>
82 #include <sys/modctl.h>
83 
84 static pgcnt_t max_page_get;	/* max page_get request size in pages */
85 pgcnt_t total_pages = 0;	/* total number of pages (used by /proc) */
86 
87 /*
88  * freemem_lock protects all freemem variables:
89  * availrmem. Also this lock protects the globals which track the
90  * availrmem changes for accurate kernel footprint calculation.
91  * See below for an explanation of these
92  * globals.
93  */
94 kmutex_t freemem_lock;
95 pgcnt_t availrmem;
96 pgcnt_t availrmem_initial;
97 
98 /*
99  * These globals track availrmem changes to get a more accurate
100  * estimate of tke kernel size. Historically pp_kernel is used for
101  * kernel size and is based on availrmem. But availrmem is adjusted for
102  * locked pages in the system not just for kernel locked pages.
103  * These new counters will track the pages locked through segvn and
104  * by explicit user locking.
105  *
106  * pages_locked : How many pages are locked because of user specified
107  * locking through mlock or plock.
108  *
109  * pages_useclaim,pages_claimed : These two variables track the
110  * claim adjustments because of the protection changes on a segvn segment.
111  *
112  * All these globals are protected by the same lock which protects availrmem.
113  */
114 pgcnt_t pages_locked = 0;
115 pgcnt_t pages_useclaim = 0;
116 pgcnt_t pages_claimed = 0;
117 
118 
119 /*
120  * new_freemem_lock protects freemem, freemem_wait & freemem_cv.
121  */
122 static kmutex_t	new_freemem_lock;
123 static uint_t	freemem_wait;	/* someone waiting for freemem */
124 static kcondvar_t freemem_cv;
125 
126 /*
127  * The logical page free list is maintained as two lists, the 'free'
128  * and the 'cache' lists.
129  * The free list contains those pages that should be reused first.
130  *
131  * The implementation of the lists is machine dependent.
132  * page_get_freelist(), page_get_cachelist(),
133  * page_list_sub(), and page_list_add()
134  * form the interface to the machine dependent implementation.
135  *
136  * Pages with p_free set are on the cache list.
137  * Pages with p_free and p_age set are on the free list,
138  *
139  * A page may be locked while on either list.
140  */
141 
142 /*
143  * free list accounting stuff.
144  *
145  *
146  * Spread out the value for the number of pages on the
147  * page free and page cache lists.  If there is just one
148  * value, then it must be under just one lock.
149  * The lock contention and cache traffic are a real bother.
150  *
151  * When we acquire and then drop a single pcf lock
152  * we can start in the middle of the array of pcf structures.
153  * If we acquire more than one pcf lock at a time, we need to
154  * start at the front to avoid deadlocking.
155  *
156  * pcf_count holds the number of pages in each pool.
157  *
158  * pcf_block is set when page_create_get_something() has asked the
159  * PSM page freelist and page cachelist routines without specifying
160  * a color and nothing came back.  This is used to block anything
161  * else from moving pages from one list to the other while the
162  * lists are searched again.  If a page is freeed while pcf_block is
163  * set, then pcf_reserve is incremented.  pcgs_unblock() takes care
164  * of clearning pcf_block, doing the wakeups, etc.
165  */
166 
167 #define	MAX_PCF_FANOUT NCPU
168 static uint_t pcf_fanout = 1; /* Will get changed at boot time */
169 static uint_t pcf_fanout_mask = 0;
170 
171 struct pcf {
172 	kmutex_t	pcf_lock;	/* protects the structure */
173 	uint_t		pcf_count;	/* page count */
174 	uint_t		pcf_wait;	/* number of waiters */
175 	uint_t		pcf_block; 	/* pcgs flag to page_free() */
176 	uint_t		pcf_reserve; 	/* pages freed after pcf_block set */
177 	uint_t		pcf_fill[10];	/* to line up on the caches */
178 };
179 
180 /*
181  * PCF_INDEX hash needs to be dynamic (every so often the hash changes where
182  * it will hash the cpu to).  This is done to prevent a drain condition
183  * from happening.  This drain condition will occur when pcf_count decrement
184  * occurs on cpu A and the increment of pcf_count always occurs on cpu B.  An
185  * example of this shows up with device interrupts.  The dma buffer is allocated
186  * by the cpu requesting the IO thus the pcf_count is decremented based on that.
187  * When the memory is returned by the interrupt thread, the pcf_count will be
188  * incremented based on the cpu servicing the interrupt.
189  */
190 static struct pcf pcf[MAX_PCF_FANOUT];
191 #define	PCF_INDEX() ((int)(((long)CPU->cpu_seqid) + \
192 	(randtick() >> 24)) & (pcf_fanout_mask))
193 
194 static int pcf_decrement_bucket(pgcnt_t);
195 static int pcf_decrement_multiple(pgcnt_t *, pgcnt_t, int);
196 
197 kmutex_t	pcgs_lock;		/* serializes page_create_get_ */
198 kmutex_t	pcgs_cagelock;		/* serializes NOSLEEP cage allocs */
199 kmutex_t	pcgs_wait_lock;		/* used for delay in pcgs */
200 static kcondvar_t	pcgs_cv;	/* cv for delay in pcgs */
201 
202 #ifdef VM_STATS
203 
204 /*
205  * No locks, but so what, they are only statistics.
206  */
207 
208 static struct page_tcnt {
209 	int	pc_free_cache;		/* free's into cache list */
210 	int	pc_free_dontneed;	/* free's with dontneed */
211 	int	pc_free_pageout;	/* free's from pageout */
212 	int	pc_free_free;		/* free's into free list */
213 	int	pc_free_pages;		/* free's into large page free list */
214 	int	pc_destroy_pages;	/* large page destroy's */
215 	int	pc_get_cache;		/* get's from cache list */
216 	int	pc_get_free;		/* get's from free list */
217 	int	pc_reclaim;		/* reclaim's */
218 	int	pc_abortfree;		/* abort's of free pages */
219 	int	pc_find_hit;		/* find's that find page */
220 	int	pc_find_miss;		/* find's that don't find page */
221 	int	pc_destroy_free;	/* # of free pages destroyed */
222 #define	PC_HASH_CNT	(4*PAGE_HASHAVELEN)
223 	int	pc_find_hashlen[PC_HASH_CNT+1];
224 	int	pc_addclaim_pages;
225 	int	pc_subclaim_pages;
226 	int	pc_free_replacement_page[2];
227 	int	pc_try_demote_pages[6];
228 	int	pc_demote_pages[2];
229 } pagecnt;
230 
231 uint_t	hashin_count;
232 uint_t	hashin_not_held;
233 uint_t	hashin_already;
234 
235 uint_t	hashout_count;
236 uint_t	hashout_not_held;
237 
238 uint_t	page_create_count;
239 uint_t	page_create_not_enough;
240 uint_t	page_create_not_enough_again;
241 uint_t	page_create_zero;
242 uint_t	page_create_hashout;
243 uint_t	page_create_page_lock_failed;
244 uint_t	page_create_trylock_failed;
245 uint_t	page_create_found_one;
246 uint_t	page_create_hashin_failed;
247 uint_t	page_create_dropped_phm;
248 
249 uint_t	page_create_new;
250 uint_t	page_create_exists;
251 uint_t	page_create_putbacks;
252 uint_t	page_create_overshoot;
253 
254 uint_t	page_reclaim_zero;
255 uint_t	page_reclaim_zero_locked;
256 
257 uint_t	page_rename_exists;
258 uint_t	page_rename_count;
259 
260 uint_t	page_lookup_cnt[20];
261 uint_t	page_lookup_nowait_cnt[10];
262 uint_t	page_find_cnt;
263 uint_t	page_exists_cnt;
264 uint_t	page_exists_forreal_cnt;
265 uint_t	page_lookup_dev_cnt;
266 uint_t	get_cachelist_cnt;
267 uint_t	page_create_cnt[10];
268 uint_t	alloc_pages[9];
269 uint_t	page_exphcontg[19];
270 uint_t  page_create_large_cnt[10];
271 
272 #endif
273 
274 static inline page_t *
275 page_hash_search(ulong_t index, vnode_t *vnode, u_offset_t off)
276 {
277 	uint_t mylen = 0;
278 	page_t *page;
279 
280 	for (page = page_hash[index]; page; page = page->p_hash, mylen++)
281 		if (page->p_vnode == vnode && page->p_offset == off)
282 			break;
283 
284 #ifdef	VM_STATS
285 	if (page != NULL)
286 		pagecnt.pc_find_hit++;
287 	else
288 		pagecnt.pc_find_miss++;
289 
290 	pagecnt.pc_find_hashlen[MIN(mylen, PC_HASH_CNT)]++;
291 #endif
292 
293 	return (page);
294 }
295 
296 
297 #ifdef DEBUG
298 #define	MEMSEG_SEARCH_STATS
299 #endif
300 
301 #ifdef MEMSEG_SEARCH_STATS
302 struct memseg_stats {
303     uint_t nsearch;
304     uint_t nlastwon;
305     uint_t nhashwon;
306     uint_t nnotfound;
307 } memseg_stats;
308 
309 #define	MEMSEG_STAT_INCR(v) \
310 	atomic_inc_32(&memseg_stats.v)
311 #else
312 #define	MEMSEG_STAT_INCR(x)
313 #endif
314 
315 struct memseg *memsegs;		/* list of memory segments */
316 
317 /*
318  * /etc/system tunable to control large page allocation hueristic.
319  *
320  * Setting to LPAP_LOCAL will heavily prefer the local lgroup over remote lgroup
321  * for large page allocation requests.  If a large page is not readily
322  * avaliable on the local freelists we will go through additional effort
323  * to create a large page, potentially moving smaller pages around to coalesce
324  * larger pages in the local lgroup.
325  * Default value of LPAP_DEFAULT will go to remote freelists if large pages
326  * are not readily available in the local lgroup.
327  */
328 enum lpap {
329 	LPAP_DEFAULT,	/* default large page allocation policy */
330 	LPAP_LOCAL	/* local large page allocation policy */
331 };
332 
333 enum lpap lpg_alloc_prefer = LPAP_DEFAULT;
334 
335 static void page_init_mem_config(void);
336 static int page_do_hashin(page_t *, vnode_t *, u_offset_t);
337 static void page_do_hashout(page_t *);
338 static void page_capture_init();
339 int page_capture_take_action(page_t *, uint_t, void *);
340 
341 static void page_demote_vp_pages(page_t *);
342 
343 
344 void
345 pcf_init(void)
346 {
347 	if (boot_ncpus != -1) {
348 		pcf_fanout = boot_ncpus;
349 	} else {
350 		pcf_fanout = max_ncpus;
351 	}
352 #ifdef sun4v
353 	/*
354 	 * Force at least 4 buckets if possible for sun4v.
355 	 */
356 	pcf_fanout = MAX(pcf_fanout, 4);
357 #endif /* sun4v */
358 
359 	/*
360 	 * Round up to the nearest power of 2.
361 	 */
362 	pcf_fanout = MIN(pcf_fanout, MAX_PCF_FANOUT);
363 	if (!ISP2(pcf_fanout)) {
364 		pcf_fanout = 1 << highbit(pcf_fanout);
365 
366 		if (pcf_fanout > MAX_PCF_FANOUT) {
367 			pcf_fanout = 1 << (highbit(MAX_PCF_FANOUT) - 1);
368 		}
369 	}
370 	pcf_fanout_mask = pcf_fanout - 1;
371 }
372 
373 /*
374  * vm subsystem related initialization
375  */
376 void
377 vm_init(void)
378 {
379 	boolean_t callb_vm_cpr(void *, int);
380 
381 	(void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm");
382 	page_init_mem_config();
383 	page_retire_init();
384 	vm_usage_init();
385 	page_capture_init();
386 }
387 
388 /*
389  * This function is called at startup and when memory is added or deleted.
390  */
391 void
392 init_pages_pp_maximum()
393 {
394 	static pgcnt_t p_min;
395 	static pgcnt_t pages_pp_maximum_startup;
396 	static pgcnt_t avrmem_delta;
397 	static int init_done;
398 	static int user_set;	/* true if set in /etc/system */
399 
400 	if (init_done == 0) {
401 
402 		/* If the user specified a value, save it */
403 		if (pages_pp_maximum != 0) {
404 			user_set = 1;
405 			pages_pp_maximum_startup = pages_pp_maximum;
406 		}
407 
408 		/*
409 		 * Setting of pages_pp_maximum is based first time
410 		 * on the value of availrmem just after the start-up
411 		 * allocations. To preserve this relationship at run
412 		 * time, use a delta from availrmem_initial.
413 		 */
414 		ASSERT(availrmem_initial >= availrmem);
415 		avrmem_delta = availrmem_initial - availrmem;
416 
417 		/* The allowable floor of pages_pp_maximum */
418 		p_min = tune.t_minarmem + 100;
419 
420 		/* Make sure we don't come through here again. */
421 		init_done = 1;
422 	}
423 	/*
424 	 * Determine pages_pp_maximum, the number of currently available
425 	 * pages (availrmem) that can't be `locked'. If not set by
426 	 * the user, we set it to 4% of the currently available memory
427 	 * plus 4MB.
428 	 * But we also insist that it be greater than tune.t_minarmem;
429 	 * otherwise a process could lock down a lot of memory, get swapped
430 	 * out, and never have enough to get swapped back in.
431 	 */
432 	if (user_set)
433 		pages_pp_maximum = pages_pp_maximum_startup;
434 	else
435 		pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25)
436 		    + btop(4 * 1024 * 1024);
437 
438 	if (pages_pp_maximum <= p_min) {
439 		pages_pp_maximum = p_min;
440 	}
441 }
442 
443 void
444 set_max_page_get(pgcnt_t target_total_pages)
445 {
446 	max_page_get = target_total_pages / 2;
447 }
448 
449 static pgcnt_t pending_delete;
450 
451 /*ARGSUSED*/
452 static void
453 page_mem_config_post_add(
454 	void *arg,
455 	pgcnt_t delta_pages)
456 {
457 	set_max_page_get(total_pages - pending_delete);
458 	init_pages_pp_maximum();
459 }
460 
461 /*ARGSUSED*/
462 static int
463 page_mem_config_pre_del(
464 	void *arg,
465 	pgcnt_t delta_pages)
466 {
467 	pgcnt_t nv;
468 
469 	nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages);
470 	set_max_page_get(total_pages - nv);
471 	return (0);
472 }
473 
474 /*ARGSUSED*/
475 static void
476 page_mem_config_post_del(
477 	void *arg,
478 	pgcnt_t delta_pages,
479 	int cancelled)
480 {
481 	pgcnt_t nv;
482 
483 	nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages);
484 	set_max_page_get(total_pages - nv);
485 	if (!cancelled)
486 		init_pages_pp_maximum();
487 }
488 
489 static kphysm_setup_vector_t page_mem_config_vec = {
490 	KPHYSM_SETUP_VECTOR_VERSION,
491 	page_mem_config_post_add,
492 	page_mem_config_pre_del,
493 	page_mem_config_post_del,
494 };
495 
496 static void
497 page_init_mem_config(void)
498 {
499 	int ret;
500 
501 	ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL);
502 	ASSERT(ret == 0);
503 }
504 
505 /*
506  * Evenly spread out the PCF counters for large free pages
507  */
508 static void
509 page_free_large_ctr(pgcnt_t npages)
510 {
511 	static struct pcf	*p = pcf;
512 	pgcnt_t			lump;
513 
514 	freemem += npages;
515 
516 	lump = roundup(npages, pcf_fanout) / pcf_fanout;
517 
518 	while (npages > 0) {
519 
520 		ASSERT(!p->pcf_block);
521 
522 		if (lump < npages) {
523 			p->pcf_count += (uint_t)lump;
524 			npages -= lump;
525 		} else {
526 			p->pcf_count += (uint_t)npages;
527 			npages = 0;
528 		}
529 
530 		ASSERT(!p->pcf_wait);
531 
532 		if (++p > &pcf[pcf_fanout - 1])
533 			p = pcf;
534 	}
535 
536 	ASSERT(npages == 0);
537 }
538 
539 /*
540  * Add a physical chunk of memory to the system free lists during startup.
541  * Platform specific startup() allocates the memory for the page structs.
542  *
543  * num	- number of page structures
544  * base - page number (pfn) to be associated with the first page.
545  *
546  * Since we are doing this during startup (ie. single threaded), we will
547  * use shortcut routines to avoid any locking overhead while putting all
548  * these pages on the freelists.
549  *
550  * NOTE: Any changes performed to page_free(), must also be performed to
551  *	 add_physmem() since this is how we initialize all page_t's at
552  *	 boot time.
553  */
554 void
555 add_physmem(
556 	page_t	*pp,
557 	pgcnt_t	num,
558 	pfn_t	pnum)
559 {
560 	page_t	*root = NULL;
561 	uint_t	szc = page_num_pagesizes() - 1;
562 	pgcnt_t	large = page_get_pagecnt(szc);
563 	pgcnt_t	cnt = 0;
564 
565 	TRACE_2(TR_FAC_VM, TR_PAGE_INIT,
566 	    "add_physmem:pp %p num %lu", pp, num);
567 
568 	/*
569 	 * Arbitrarily limit the max page_get request
570 	 * to 1/2 of the page structs we have.
571 	 */
572 	total_pages += num;
573 	set_max_page_get(total_pages);
574 
575 	PLCNT_MODIFY_MAX(pnum, (long)num);
576 
577 	/*
578 	 * The physical space for the pages array
579 	 * representing ram pages has already been
580 	 * allocated.  Here we initialize each lock
581 	 * in the page structure, and put each on
582 	 * the free list
583 	 */
584 	for (; num; pp++, pnum++, num--) {
585 
586 		/*
587 		 * this needs to fill in the page number
588 		 * and do any other arch specific initialization
589 		 */
590 		add_physmem_cb(pp, pnum);
591 
592 		pp->p_lckcnt = 0;
593 		pp->p_cowcnt = 0;
594 		pp->p_slckcnt = 0;
595 
596 		/*
597 		 * Initialize the page lock as unlocked, since nobody
598 		 * can see or access this page yet.
599 		 */
600 		pp->p_selock = 0;
601 
602 		/*
603 		 * Initialize IO lock
604 		 */
605 		page_iolock_init(pp);
606 
607 		/*
608 		 * initialize other fields in the page_t
609 		 */
610 		PP_SETFREE(pp);
611 		page_clr_all_props(pp);
612 		PP_SETAGED(pp);
613 		pp->p_offset = (u_offset_t)-1;
614 		pp->p_next = pp;
615 		pp->p_prev = pp;
616 
617 		/*
618 		 * Simple case: System doesn't support large pages.
619 		 */
620 		if (szc == 0) {
621 			pp->p_szc = 0;
622 			page_free_at_startup(pp);
623 			continue;
624 		}
625 
626 		/*
627 		 * Handle unaligned pages, we collect them up onto
628 		 * the root page until we have a full large page.
629 		 */
630 		if (!IS_P2ALIGNED(pnum, large)) {
631 
632 			/*
633 			 * If not in a large page,
634 			 * just free as small page.
635 			 */
636 			if (root == NULL) {
637 				pp->p_szc = 0;
638 				page_free_at_startup(pp);
639 				continue;
640 			}
641 
642 			/*
643 			 * Link a constituent page into the large page.
644 			 */
645 			pp->p_szc = szc;
646 			page_list_concat(&root, &pp);
647 
648 			/*
649 			 * When large page is fully formed, free it.
650 			 */
651 			if (++cnt == large) {
652 				page_free_large_ctr(cnt);
653 				page_list_add_pages(root, PG_LIST_ISINIT);
654 				root = NULL;
655 				cnt = 0;
656 			}
657 			continue;
658 		}
659 
660 		/*
661 		 * At this point we have a page number which
662 		 * is aligned. We assert that we aren't already
663 		 * in a different large page.
664 		 */
665 		ASSERT(IS_P2ALIGNED(pnum, large));
666 		ASSERT(root == NULL && cnt == 0);
667 
668 		/*
669 		 * If insufficient number of pages left to form
670 		 * a large page, just free the small page.
671 		 */
672 		if (num < large) {
673 			pp->p_szc = 0;
674 			page_free_at_startup(pp);
675 			continue;
676 		}
677 
678 		/*
679 		 * Otherwise start a new large page.
680 		 */
681 		pp->p_szc = szc;
682 		cnt++;
683 		root = pp;
684 	}
685 	ASSERT(root == NULL && cnt == 0);
686 }
687 
688 /*
689  * Find a page representing the specified [vp, offset].
690  * If we find the page but it is intransit coming in,
691  * it will have an "exclusive" lock and we wait for
692  * the i/o to complete.  A page found on the free list
693  * is always reclaimed and then locked.  On success, the page
694  * is locked, its data is valid and it isn't on the free
695  * list, while a NULL is returned if the page doesn't exist.
696  */
697 page_t *
698 page_lookup(vnode_t *vp, u_offset_t off, se_t se)
699 {
700 	return (page_lookup_create(vp, off, se, NULL, NULL, 0));
701 }
702 
703 /*
704  * Find a page representing the specified [vp, offset].
705  * We either return the one we found or, if passed in,
706  * create one with identity of [vp, offset] of the
707  * pre-allocated page. If we find existing page but it is
708  * intransit coming in, it will have an "exclusive" lock
709  * and we wait for the i/o to complete.  A page found on
710  * the free list is always reclaimed and then locked.
711  * On success, the page is locked, its data is valid and
712  * it isn't on the free list, while a NULL is returned
713  * if the page doesn't exist and newpp is NULL;
714  */
715 page_t *
716 page_lookup_create(
717 	vnode_t *vp,
718 	u_offset_t off,
719 	se_t se,
720 	page_t *newpp,
721 	spgcnt_t *nrelocp,
722 	int flags)
723 {
724 	page_t		*pp;
725 	kmutex_t	*phm;
726 	ulong_t		index;
727 	uint_t		hash_locked;
728 	uint_t		es;
729 
730 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
731 	VM_STAT_ADD(page_lookup_cnt[0]);
732 	ASSERT(newpp ? PAGE_EXCL(newpp) : 1);
733 
734 	/*
735 	 * Acquire the appropriate page hash lock since
736 	 * we have to search the hash list.  Pages that
737 	 * hash to this list can't change identity while
738 	 * this lock is held.
739 	 */
740 	hash_locked = 0;
741 	index = PAGE_HASH_FUNC(vp, off);
742 	phm = NULL;
743 top:
744 	pp = page_hash_search(index, vp, off);
745 	if (pp != NULL) {
746 		VM_STAT_ADD(page_lookup_cnt[1]);
747 		es = (newpp != NULL) ? 1 : 0;
748 		es |= flags;
749 		if (!hash_locked) {
750 			VM_STAT_ADD(page_lookup_cnt[2]);
751 			if (!page_try_reclaim_lock(pp, se, es)) {
752 				/*
753 				 * On a miss, acquire the phm.  Then
754 				 * next time, page_lock() will be called,
755 				 * causing a wait if the page is busy.
756 				 * just looping with page_trylock() would
757 				 * get pretty boring.
758 				 */
759 				VM_STAT_ADD(page_lookup_cnt[3]);
760 				phm = PAGE_HASH_MUTEX(index);
761 				mutex_enter(phm);
762 				hash_locked = 1;
763 				goto top;
764 			}
765 		} else {
766 			VM_STAT_ADD(page_lookup_cnt[4]);
767 			if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) {
768 				VM_STAT_ADD(page_lookup_cnt[5]);
769 				goto top;
770 			}
771 		}
772 
773 		/*
774 		 * Since `pp' is locked it can not change identity now.
775 		 * Reconfirm we locked the correct page.
776 		 *
777 		 * Both the p_vnode and p_offset *must* be cast volatile
778 		 * to force a reload of their values: The page_hash_search
779 		 * function will have stuffed p_vnode and p_offset into
780 		 * registers before calling page_trylock(); another thread,
781 		 * actually holding the hash lock, could have changed the
782 		 * page's identity in memory, but our registers would not
783 		 * be changed, fooling the reconfirmation.  If the hash
784 		 * lock was held during the search, the casting would
785 		 * not be needed.
786 		 */
787 		VM_STAT_ADD(page_lookup_cnt[6]);
788 		if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
789 		    ((volatile u_offset_t)(pp->p_offset) != off)) {
790 			VM_STAT_ADD(page_lookup_cnt[7]);
791 			if (hash_locked) {
792 				panic("page_lookup_create: lost page %p",
793 				    (void *)pp);
794 				/*NOTREACHED*/
795 			}
796 			page_unlock(pp);
797 			phm = PAGE_HASH_MUTEX(index);
798 			mutex_enter(phm);
799 			hash_locked = 1;
800 			goto top;
801 		}
802 
803 		/*
804 		 * If page_trylock() was called, then pp may still be on
805 		 * the cachelist (can't be on the free list, it would not
806 		 * have been found in the search).  If it is on the
807 		 * cachelist it must be pulled now. To pull the page from
808 		 * the cachelist, it must be exclusively locked.
809 		 *
810 		 * The other big difference between page_trylock() and
811 		 * page_lock(), is that page_lock() will pull the
812 		 * page from whatever free list (the cache list in this
813 		 * case) the page is on.  If page_trylock() was used
814 		 * above, then we have to do the reclaim ourselves.
815 		 */
816 		if ((!hash_locked) && (PP_ISFREE(pp))) {
817 			ASSERT(PP_ISAGED(pp) == 0);
818 			VM_STAT_ADD(page_lookup_cnt[8]);
819 
820 			/*
821 			 * page_relcaim will insure that we
822 			 * have this page exclusively
823 			 */
824 
825 			if (!page_reclaim(pp, NULL)) {
826 				/*
827 				 * Page_reclaim dropped whatever lock
828 				 * we held.
829 				 */
830 				VM_STAT_ADD(page_lookup_cnt[9]);
831 				phm = PAGE_HASH_MUTEX(index);
832 				mutex_enter(phm);
833 				hash_locked = 1;
834 				goto top;
835 			} else if (se == SE_SHARED && newpp == NULL) {
836 				VM_STAT_ADD(page_lookup_cnt[10]);
837 				page_downgrade(pp);
838 			}
839 		}
840 
841 		if (hash_locked) {
842 			mutex_exit(phm);
843 		}
844 
845 		if (newpp != NULL && pp->p_szc < newpp->p_szc &&
846 		    PAGE_EXCL(pp) && nrelocp != NULL) {
847 			ASSERT(nrelocp != NULL);
848 			(void) page_relocate(&pp, &newpp, 1, 1, nrelocp,
849 			    NULL);
850 			if (*nrelocp > 0) {
851 				VM_STAT_COND_ADD(*nrelocp == 1,
852 				    page_lookup_cnt[11]);
853 				VM_STAT_COND_ADD(*nrelocp > 1,
854 				    page_lookup_cnt[12]);
855 				pp = newpp;
856 				se = SE_EXCL;
857 			} else {
858 				if (se == SE_SHARED) {
859 					page_downgrade(pp);
860 				}
861 				VM_STAT_ADD(page_lookup_cnt[13]);
862 			}
863 		} else if (newpp != NULL && nrelocp != NULL) {
864 			if (PAGE_EXCL(pp) && se == SE_SHARED) {
865 				page_downgrade(pp);
866 			}
867 			VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc,
868 			    page_lookup_cnt[14]);
869 			VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc,
870 			    page_lookup_cnt[15]);
871 			VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc,
872 			    page_lookup_cnt[16]);
873 		} else if (newpp != NULL && PAGE_EXCL(pp)) {
874 			se = SE_EXCL;
875 		}
876 	} else if (!hash_locked) {
877 		VM_STAT_ADD(page_lookup_cnt[17]);
878 		phm = PAGE_HASH_MUTEX(index);
879 		mutex_enter(phm);
880 		hash_locked = 1;
881 		goto top;
882 	} else if (newpp != NULL) {
883 		/*
884 		 * If we have a preallocated page then
885 		 * insert it now and basically behave like
886 		 * page_create.
887 		 */
888 		VM_STAT_ADD(page_lookup_cnt[18]);
889 		/*
890 		 * Since we hold the page hash mutex and
891 		 * just searched for this page, page_hashin
892 		 * had better not fail.  If it does, that
893 		 * means some thread did not follow the
894 		 * page hash mutex rules.  Panic now and
895 		 * get it over with.  As usual, go down
896 		 * holding all the locks.
897 		 */
898 		ASSERT(MUTEX_HELD(phm));
899 		if (!page_hashin(newpp, vp, off, phm)) {
900 			ASSERT(MUTEX_HELD(phm));
901 			panic("page_lookup_create: hashin failed %p %p %llx %p",
902 			    (void *)newpp, (void *)vp, off, (void *)phm);
903 			/*NOTREACHED*/
904 		}
905 		ASSERT(MUTEX_HELD(phm));
906 		mutex_exit(phm);
907 		phm = NULL;
908 		page_set_props(newpp, P_REF);
909 		page_io_lock(newpp);
910 		pp = newpp;
911 		se = SE_EXCL;
912 	} else {
913 		VM_STAT_ADD(page_lookup_cnt[19]);
914 		mutex_exit(phm);
915 	}
916 
917 	ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
918 
919 	ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1);
920 
921 	return (pp);
922 }
923 
924 /*
925  * Search the hash list for the page representing the
926  * specified [vp, offset] and return it locked.  Skip
927  * free pages and pages that cannot be locked as requested.
928  * Used while attempting to kluster pages.
929  */
930 page_t *
931 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se)
932 {
933 	page_t		*pp;
934 	kmutex_t	*phm;
935 	ulong_t		index;
936 	uint_t		locked;
937 
938 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
939 	VM_STAT_ADD(page_lookup_nowait_cnt[0]);
940 
941 	index = PAGE_HASH_FUNC(vp, off);
942 	pp = page_hash_search(index, vp, off);
943 	locked = 0;
944 	if (pp == NULL) {
945 top:
946 		VM_STAT_ADD(page_lookup_nowait_cnt[1]);
947 		locked = 1;
948 		phm = PAGE_HASH_MUTEX(index);
949 		mutex_enter(phm);
950 		pp = page_hash_search(index, vp, off);
951 	}
952 
953 	if (pp == NULL || PP_ISFREE(pp)) {
954 		VM_STAT_ADD(page_lookup_nowait_cnt[2]);
955 		pp = NULL;
956 	} else {
957 		if (!page_trylock(pp, se)) {
958 			VM_STAT_ADD(page_lookup_nowait_cnt[3]);
959 			pp = NULL;
960 		} else {
961 			VM_STAT_ADD(page_lookup_nowait_cnt[4]);
962 			/*
963 			 * See the comment in page_lookup()
964 			 */
965 			if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
966 			    ((u_offset_t)(pp->p_offset) != off)) {
967 				VM_STAT_ADD(page_lookup_nowait_cnt[5]);
968 				if (locked) {
969 					panic("page_lookup_nowait %p",
970 					    (void *)pp);
971 					/*NOTREACHED*/
972 				}
973 				page_unlock(pp);
974 				goto top;
975 			}
976 			if (PP_ISFREE(pp)) {
977 				VM_STAT_ADD(page_lookup_nowait_cnt[6]);
978 				page_unlock(pp);
979 				pp = NULL;
980 			}
981 		}
982 	}
983 	if (locked) {
984 		VM_STAT_ADD(page_lookup_nowait_cnt[7]);
985 		mutex_exit(phm);
986 	}
987 
988 	ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
989 
990 	return (pp);
991 }
992 
993 /*
994  * Search the hash list for a page with the specified [vp, off]
995  * that is known to exist and is already locked.  This routine
996  * is typically used by segment SOFTUNLOCK routines.
997  */
998 page_t *
999 page_find(vnode_t *vp, u_offset_t off)
1000 {
1001 	page_t		*pp;
1002 	kmutex_t	*phm;
1003 	ulong_t		index;
1004 
1005 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1006 	VM_STAT_ADD(page_find_cnt);
1007 
1008 	index = PAGE_HASH_FUNC(vp, off);
1009 	phm = PAGE_HASH_MUTEX(index);
1010 
1011 	mutex_enter(phm);
1012 	pp = page_hash_search(index, vp, off);
1013 	mutex_exit(phm);
1014 
1015 	ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr);
1016 	return (pp);
1017 }
1018 
1019 /*
1020  * Determine whether a page with the specified [vp, off]
1021  * currently exists in the system.  Obviously this should
1022  * only be considered as a hint since nothing prevents the
1023  * page from disappearing or appearing immediately after
1024  * the return from this routine. Subsequently, we don't
1025  * even bother to lock the list.
1026  */
1027 page_t *
1028 page_exists(vnode_t *vp, u_offset_t off)
1029 {
1030 	ulong_t		index;
1031 
1032 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1033 	VM_STAT_ADD(page_exists_cnt);
1034 
1035 	index = PAGE_HASH_FUNC(vp, off);
1036 
1037 	return (page_hash_search(index, vp, off));
1038 }
1039 
1040 /*
1041  * Determine if physically contiguous pages exist for [vp, off] - [vp, off +
1042  * page_size(szc)) range.  if they exist and ppa is not NULL fill ppa array
1043  * with these pages locked SHARED. If necessary reclaim pages from
1044  * freelist. Return 1 if contiguous pages exist and 0 otherwise.
1045  *
1046  * If we fail to lock pages still return 1 if pages exist and contiguous.
1047  * But in this case return value is just a hint. ppa array won't be filled.
1048  * Caller should initialize ppa[0] as NULL to distinguish return value.
1049  *
1050  * Returns 0 if pages don't exist or not physically contiguous.
1051  *
1052  * This routine doesn't work for anonymous(swapfs) pages.
1053  */
1054 int
1055 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[])
1056 {
1057 	pgcnt_t pages;
1058 	pfn_t pfn;
1059 	page_t *rootpp;
1060 	pgcnt_t i;
1061 	pgcnt_t j;
1062 	u_offset_t save_off = off;
1063 	ulong_t index;
1064 	kmutex_t *phm;
1065 	page_t *pp;
1066 	uint_t pszc;
1067 	int loopcnt = 0;
1068 
1069 	ASSERT(szc != 0);
1070 	ASSERT(vp != NULL);
1071 	ASSERT(!IS_SWAPFSVP(vp));
1072 	ASSERT(!VN_ISKAS(vp));
1073 
1074 again:
1075 	if (++loopcnt > 3) {
1076 		VM_STAT_ADD(page_exphcontg[0]);
1077 		return (0);
1078 	}
1079 
1080 	index = PAGE_HASH_FUNC(vp, off);
1081 	phm = PAGE_HASH_MUTEX(index);
1082 
1083 	mutex_enter(phm);
1084 	pp = page_hash_search(index, vp, off);
1085 	mutex_exit(phm);
1086 
1087 	VM_STAT_ADD(page_exphcontg[1]);
1088 
1089 	if (pp == NULL) {
1090 		VM_STAT_ADD(page_exphcontg[2]);
1091 		return (0);
1092 	}
1093 
1094 	pages = page_get_pagecnt(szc);
1095 	rootpp = pp;
1096 	pfn = rootpp->p_pagenum;
1097 
1098 	if ((pszc = pp->p_szc) >= szc && ppa != NULL) {
1099 		VM_STAT_ADD(page_exphcontg[3]);
1100 		if (!page_trylock(pp, SE_SHARED)) {
1101 			VM_STAT_ADD(page_exphcontg[4]);
1102 			return (1);
1103 		}
1104 		/*
1105 		 * Also check whether p_pagenum was modified by DR.
1106 		 */
1107 		if (pp->p_szc != pszc || pp->p_vnode != vp ||
1108 		    pp->p_offset != off || pp->p_pagenum != pfn) {
1109 			VM_STAT_ADD(page_exphcontg[5]);
1110 			page_unlock(pp);
1111 			off = save_off;
1112 			goto again;
1113 		}
1114 		/*
1115 		 * szc was non zero and vnode and offset matched after we
1116 		 * locked the page it means it can't become free on us.
1117 		 */
1118 		ASSERT(!PP_ISFREE(pp));
1119 		if (!IS_P2ALIGNED(pfn, pages)) {
1120 			page_unlock(pp);
1121 			return (0);
1122 		}
1123 		ppa[0] = pp;
1124 		pp++;
1125 		off += PAGESIZE;
1126 		pfn++;
1127 		for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
1128 			if (!page_trylock(pp, SE_SHARED)) {
1129 				VM_STAT_ADD(page_exphcontg[6]);
1130 				pp--;
1131 				while (i-- > 0) {
1132 					page_unlock(pp);
1133 					pp--;
1134 				}
1135 				ppa[0] = NULL;
1136 				return (1);
1137 			}
1138 			if (pp->p_szc != pszc) {
1139 				VM_STAT_ADD(page_exphcontg[7]);
1140 				page_unlock(pp);
1141 				pp--;
1142 				while (i-- > 0) {
1143 					page_unlock(pp);
1144 					pp--;
1145 				}
1146 				ppa[0] = NULL;
1147 				off = save_off;
1148 				goto again;
1149 			}
1150 			/*
1151 			 * szc the same as for previous already locked pages
1152 			 * with right identity. Since this page had correct
1153 			 * szc after we locked it can't get freed or destroyed
1154 			 * and therefore must have the expected identity.
1155 			 */
1156 			ASSERT(!PP_ISFREE(pp));
1157 			if (pp->p_vnode != vp ||
1158 			    pp->p_offset != off) {
1159 				panic("page_exists_physcontig: "
1160 				    "large page identity doesn't match");
1161 			}
1162 			ppa[i] = pp;
1163 			ASSERT(pp->p_pagenum == pfn);
1164 		}
1165 		VM_STAT_ADD(page_exphcontg[8]);
1166 		ppa[pages] = NULL;
1167 		return (1);
1168 	} else if (pszc >= szc) {
1169 		VM_STAT_ADD(page_exphcontg[9]);
1170 		if (!IS_P2ALIGNED(pfn, pages)) {
1171 			return (0);
1172 		}
1173 		return (1);
1174 	}
1175 
1176 	if (!IS_P2ALIGNED(pfn, pages)) {
1177 		VM_STAT_ADD(page_exphcontg[10]);
1178 		return (0);
1179 	}
1180 
1181 	if (page_numtomemseg_nolock(pfn) !=
1182 	    page_numtomemseg_nolock(pfn + pages - 1)) {
1183 		VM_STAT_ADD(page_exphcontg[11]);
1184 		return (0);
1185 	}
1186 
1187 	/*
1188 	 * We loop up 4 times across pages to promote page size.
1189 	 * We're extra cautious to promote page size atomically with respect
1190 	 * to everybody else.  But we can probably optimize into 1 loop if
1191 	 * this becomes an issue.
1192 	 */
1193 
1194 	for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
1195 		if (!page_trylock(pp, SE_EXCL)) {
1196 			VM_STAT_ADD(page_exphcontg[12]);
1197 			break;
1198 		}
1199 		/*
1200 		 * Check whether p_pagenum was modified by DR.
1201 		 */
1202 		if (pp->p_pagenum != pfn) {
1203 			page_unlock(pp);
1204 			break;
1205 		}
1206 		if (pp->p_vnode != vp ||
1207 		    pp->p_offset != off) {
1208 			VM_STAT_ADD(page_exphcontg[13]);
1209 			page_unlock(pp);
1210 			break;
1211 		}
1212 		if (pp->p_szc >= szc) {
1213 			ASSERT(i == 0);
1214 			page_unlock(pp);
1215 			off = save_off;
1216 			goto again;
1217 		}
1218 	}
1219 
1220 	if (i != pages) {
1221 		VM_STAT_ADD(page_exphcontg[14]);
1222 		--pp;
1223 		while (i-- > 0) {
1224 			page_unlock(pp);
1225 			--pp;
1226 		}
1227 		return (0);
1228 	}
1229 
1230 	pp = rootpp;
1231 	for (i = 0; i < pages; i++, pp++) {
1232 		if (PP_ISFREE(pp)) {
1233 			VM_STAT_ADD(page_exphcontg[15]);
1234 			ASSERT(!PP_ISAGED(pp));
1235 			ASSERT(pp->p_szc == 0);
1236 			if (!page_reclaim(pp, NULL)) {
1237 				break;
1238 			}
1239 		} else {
1240 			ASSERT(pp->p_szc < szc);
1241 			VM_STAT_ADD(page_exphcontg[16]);
1242 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
1243 		}
1244 	}
1245 	if (i < pages) {
1246 		VM_STAT_ADD(page_exphcontg[17]);
1247 		/*
1248 		 * page_reclaim failed because we were out of memory.
1249 		 * drop the rest of the locks and return because this page
1250 		 * must be already reallocated anyway.
1251 		 */
1252 		pp = rootpp;
1253 		for (j = 0; j < pages; j++, pp++) {
1254 			if (j != i) {
1255 				page_unlock(pp);
1256 			}
1257 		}
1258 		return (0);
1259 	}
1260 
1261 	off = save_off;
1262 	pp = rootpp;
1263 	for (i = 0; i < pages; i++, pp++, off += PAGESIZE) {
1264 		ASSERT(PAGE_EXCL(pp));
1265 		ASSERT(!PP_ISFREE(pp));
1266 		ASSERT(!hat_page_is_mapped(pp));
1267 		ASSERT(pp->p_vnode == vp);
1268 		ASSERT(pp->p_offset == off);
1269 		pp->p_szc = szc;
1270 	}
1271 	pp = rootpp;
1272 	for (i = 0; i < pages; i++, pp++) {
1273 		if (ppa == NULL) {
1274 			page_unlock(pp);
1275 		} else {
1276 			ppa[i] = pp;
1277 			page_downgrade(ppa[i]);
1278 		}
1279 	}
1280 	if (ppa != NULL) {
1281 		ppa[pages] = NULL;
1282 	}
1283 	VM_STAT_ADD(page_exphcontg[18]);
1284 	ASSERT(vp->v_pages != NULL);
1285 	return (1);
1286 }
1287 
1288 /*
1289  * Determine whether a page with the specified [vp, off]
1290  * currently exists in the system and if so return its
1291  * size code. Obviously this should only be considered as
1292  * a hint since nothing prevents the page from disappearing
1293  * or appearing immediately after the return from this routine.
1294  */
1295 int
1296 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc)
1297 {
1298 	page_t		*pp;
1299 	kmutex_t	*phm;
1300 	ulong_t		index;
1301 	int		rc = 0;
1302 
1303 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1304 	ASSERT(szc != NULL);
1305 	VM_STAT_ADD(page_exists_forreal_cnt);
1306 
1307 	index = PAGE_HASH_FUNC(vp, off);
1308 	phm = PAGE_HASH_MUTEX(index);
1309 
1310 	mutex_enter(phm);
1311 	pp = page_hash_search(index, vp, off);
1312 	if (pp != NULL) {
1313 		*szc = pp->p_szc;
1314 		rc = 1;
1315 	}
1316 	mutex_exit(phm);
1317 	return (rc);
1318 }
1319 
1320 /* wakeup threads waiting for pages in page_create_get_something() */
1321 void
1322 wakeup_pcgs(void)
1323 {
1324 	if (!CV_HAS_WAITERS(&pcgs_cv))
1325 		return;
1326 	cv_broadcast(&pcgs_cv);
1327 }
1328 
1329 /*
1330  * 'freemem' is used all over the kernel as an indication of how many
1331  * pages are free (either on the cache list or on the free page list)
1332  * in the system.  In very few places is a really accurate 'freemem'
1333  * needed.  To avoid contention of the lock protecting a the
1334  * single freemem, it was spread out into NCPU buckets.  Set_freemem
1335  * sets freemem to the total of all NCPU buckets.  It is called from
1336  * clock() on each TICK.
1337  */
1338 void
1339 set_freemem()
1340 {
1341 	struct pcf	*p;
1342 	ulong_t		t;
1343 	uint_t		i;
1344 
1345 	t = 0;
1346 	p = pcf;
1347 	for (i = 0;  i < pcf_fanout; i++) {
1348 		t += p->pcf_count;
1349 		p++;
1350 	}
1351 	freemem = t;
1352 
1353 	/*
1354 	 * Don't worry about grabbing mutex.  It's not that
1355 	 * critical if we miss a tick or two.  This is
1356 	 * where we wakeup possible delayers in
1357 	 * page_create_get_something().
1358 	 */
1359 	wakeup_pcgs();
1360 }
1361 
1362 ulong_t
1363 get_freemem()
1364 {
1365 	struct pcf	*p;
1366 	ulong_t		t;
1367 	uint_t		i;
1368 
1369 	t = 0;
1370 	p = pcf;
1371 	for (i = 0; i < pcf_fanout; i++) {
1372 		t += p->pcf_count;
1373 		p++;
1374 	}
1375 	/*
1376 	 * We just calculated it, might as well set it.
1377 	 */
1378 	freemem = t;
1379 	return (t);
1380 }
1381 
1382 /*
1383  * Acquire all of the page cache & free (pcf) locks.
1384  */
1385 void
1386 pcf_acquire_all()
1387 {
1388 	struct pcf	*p;
1389 	uint_t		i;
1390 
1391 	p = pcf;
1392 	for (i = 0; i < pcf_fanout; i++) {
1393 		mutex_enter(&p->pcf_lock);
1394 		p++;
1395 	}
1396 }
1397 
1398 /*
1399  * Release all the pcf_locks.
1400  */
1401 void
1402 pcf_release_all()
1403 {
1404 	struct pcf	*p;
1405 	uint_t		i;
1406 
1407 	p = pcf;
1408 	for (i = 0; i < pcf_fanout; i++) {
1409 		mutex_exit(&p->pcf_lock);
1410 		p++;
1411 	}
1412 }
1413 
1414 /*
1415  * Inform the VM system that we need some pages freed up.
1416  * Calls must be symmetric, e.g.:
1417  *
1418  *	page_needfree(100);
1419  *	wait a bit;
1420  *	page_needfree(-100);
1421  */
1422 void
1423 page_needfree(spgcnt_t npages)
1424 {
1425 	mutex_enter(&new_freemem_lock);
1426 	needfree += npages;
1427 	mutex_exit(&new_freemem_lock);
1428 }
1429 
1430 /*
1431  * Throttle for page_create(): try to prevent freemem from dropping
1432  * below throttlefree.  We can't provide a 100% guarantee because
1433  * KM_NOSLEEP allocations, page_reclaim(), and various other things
1434  * nibble away at the freelist.  However, we can block all PG_WAIT
1435  * allocations until memory becomes available.  The motivation is
1436  * that several things can fall apart when there's no free memory:
1437  *
1438  * (1) If pageout() needs memory to push a page, the system deadlocks.
1439  *
1440  * (2) By (broken) specification, timeout(9F) can neither fail nor
1441  *     block, so it has no choice but to panic the system if it
1442  *     cannot allocate a callout structure.
1443  *
1444  * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block;
1445  *     it panics if it cannot allocate a callback structure.
1446  *
1447  * (4) Untold numbers of third-party drivers have not yet been hardened
1448  *     against KM_NOSLEEP and/or allocb() failures; they simply assume
1449  *     success and panic the system with a data fault on failure.
1450  *     (The long-term solution to this particular problem is to ship
1451  *     hostile fault-injecting DEBUG kernels with the DDK.)
1452  *
1453  * It is theoretically impossible to guarantee success of non-blocking
1454  * allocations, but in practice, this throttle is very hard to break.
1455  */
1456 static int
1457 page_create_throttle(pgcnt_t npages, int flags)
1458 {
1459 	ulong_t	fm;
1460 	uint_t	i;
1461 	pgcnt_t tf;	/* effective value of throttlefree */
1462 
1463 	/*
1464 	 * Normal priority allocations.
1465 	 */
1466 	if ((flags & (PG_WAIT | PG_NORMALPRI)) == PG_NORMALPRI) {
1467 		ASSERT(!(flags & (PG_PANIC | PG_PUSHPAGE)));
1468 		return (freemem >= npages + throttlefree);
1469 	}
1470 
1471 	/*
1472 	 * Never deny pages when:
1473 	 * - it's a thread that cannot block [NOMEMWAIT()]
1474 	 * - the allocation cannot block and must not fail
1475 	 * - the allocation cannot block and is pageout dispensated
1476 	 */
1477 	if (NOMEMWAIT() ||
1478 	    ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) ||
1479 	    ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE))
1480 		return (1);
1481 
1482 	/*
1483 	 * If the allocation can't block, we look favorably upon it
1484 	 * unless we're below pageout_reserve.  In that case we fail
1485 	 * the allocation because we want to make sure there are a few
1486 	 * pages available for pageout.
1487 	 */
1488 	if ((flags & PG_WAIT) == 0)
1489 		return (freemem >= npages + pageout_reserve);
1490 
1491 	/* Calculate the effective throttlefree value */
1492 	tf = throttlefree -
1493 	    ((flags & PG_PUSHPAGE) ? pageout_reserve : 0);
1494 
1495 	cv_signal(&proc_pageout->p_cv);
1496 
1497 	for (;;) {
1498 		fm = 0;
1499 		pcf_acquire_all();
1500 		mutex_enter(&new_freemem_lock);
1501 		for (i = 0; i < pcf_fanout; i++) {
1502 			fm += pcf[i].pcf_count;
1503 			pcf[i].pcf_wait++;
1504 			mutex_exit(&pcf[i].pcf_lock);
1505 		}
1506 		freemem = fm;
1507 		if (freemem >= npages + tf) {
1508 			mutex_exit(&new_freemem_lock);
1509 			break;
1510 		}
1511 		needfree += npages;
1512 		freemem_wait++;
1513 		cv_wait(&freemem_cv, &new_freemem_lock);
1514 		freemem_wait--;
1515 		needfree -= npages;
1516 		mutex_exit(&new_freemem_lock);
1517 	}
1518 	return (1);
1519 }
1520 
1521 /*
1522  * page_create_wait() is called to either coalesce pages from the
1523  * different pcf buckets or to wait because there simply are not
1524  * enough pages to satisfy the caller's request.
1525  *
1526  * Sadly, this is called from platform/vm/vm_machdep.c
1527  */
1528 int
1529 page_create_wait(pgcnt_t npages, uint_t flags)
1530 {
1531 	pgcnt_t		total;
1532 	uint_t		i;
1533 	struct pcf	*p;
1534 
1535 	/*
1536 	 * Wait until there are enough free pages to satisfy our
1537 	 * entire request.
1538 	 * We set needfree += npages before prodding pageout, to make sure
1539 	 * it does real work when npages > lotsfree > freemem.
1540 	 */
1541 	VM_STAT_ADD(page_create_not_enough);
1542 
1543 	ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1);
1544 checkagain:
1545 	if ((flags & PG_NORELOC) &&
1546 	    kcage_freemem < kcage_throttlefree + npages)
1547 		(void) kcage_create_throttle(npages, flags);
1548 
1549 	if (freemem < npages + throttlefree)
1550 		if (!page_create_throttle(npages, flags))
1551 			return (0);
1552 
1553 	if (pcf_decrement_bucket(npages) ||
1554 	    pcf_decrement_multiple(&total, npages, 0))
1555 		return (1);
1556 
1557 	/*
1558 	 * All of the pcf locks are held, there are not enough pages
1559 	 * to satisfy the request (npages < total).
1560 	 * Be sure to acquire the new_freemem_lock before dropping
1561 	 * the pcf locks.  This prevents dropping wakeups in page_free().
1562 	 * The order is always pcf_lock then new_freemem_lock.
1563 	 *
1564 	 * Since we hold all the pcf locks, it is a good time to set freemem.
1565 	 *
1566 	 * If the caller does not want to wait, return now.
1567 	 * Else turn the pageout daemon loose to find something
1568 	 * and wait till it does.
1569 	 *
1570 	 */
1571 	freemem = total;
1572 
1573 	if ((flags & PG_WAIT) == 0) {
1574 		pcf_release_all();
1575 
1576 		TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM,
1577 		"page_create_nomem:npages %ld freemem %ld", npages, freemem);
1578 		return (0);
1579 	}
1580 
1581 	ASSERT(proc_pageout != NULL);
1582 	cv_signal(&proc_pageout->p_cv);
1583 
1584 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START,
1585 	    "page_create_sleep_start: freemem %ld needfree %ld",
1586 	    freemem, needfree);
1587 
1588 	/*
1589 	 * We are going to wait.
1590 	 * We currently hold all of the pcf_locks,
1591 	 * get the new_freemem_lock (it protects freemem_wait),
1592 	 * before dropping the pcf_locks.
1593 	 */
1594 	mutex_enter(&new_freemem_lock);
1595 
1596 	p = pcf;
1597 	for (i = 0; i < pcf_fanout; i++) {
1598 		p->pcf_wait++;
1599 		mutex_exit(&p->pcf_lock);
1600 		p++;
1601 	}
1602 
1603 	needfree += npages;
1604 	freemem_wait++;
1605 
1606 	cv_wait(&freemem_cv, &new_freemem_lock);
1607 
1608 	freemem_wait--;
1609 	needfree -= npages;
1610 
1611 	mutex_exit(&new_freemem_lock);
1612 
1613 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END,
1614 	    "page_create_sleep_end: freemem %ld needfree %ld",
1615 	    freemem, needfree);
1616 
1617 	VM_STAT_ADD(page_create_not_enough_again);
1618 	goto checkagain;
1619 }
1620 /*
1621  * A routine to do the opposite of page_create_wait().
1622  */
1623 void
1624 page_create_putback(spgcnt_t npages)
1625 {
1626 	struct pcf	*p;
1627 	pgcnt_t		lump;
1628 	uint_t		*which;
1629 
1630 	/*
1631 	 * When a contiguous lump is broken up, we have to
1632 	 * deal with lots of pages (min 64) so lets spread
1633 	 * the wealth around.
1634 	 */
1635 	lump = roundup(npages, pcf_fanout) / pcf_fanout;
1636 	freemem += npages;
1637 
1638 	for (p = pcf; (npages > 0) && (p < &pcf[pcf_fanout]); p++) {
1639 		which = &p->pcf_count;
1640 
1641 		mutex_enter(&p->pcf_lock);
1642 
1643 		if (p->pcf_block) {
1644 			which = &p->pcf_reserve;
1645 		}
1646 
1647 		if (lump < npages) {
1648 			*which += (uint_t)lump;
1649 			npages -= lump;
1650 		} else {
1651 			*which += (uint_t)npages;
1652 			npages = 0;
1653 		}
1654 
1655 		if (p->pcf_wait) {
1656 			mutex_enter(&new_freemem_lock);
1657 			/*
1658 			 * Check to see if some other thread
1659 			 * is actually waiting.  Another bucket
1660 			 * may have woken it up by now.  If there
1661 			 * are no waiters, then set our pcf_wait
1662 			 * count to zero to avoid coming in here
1663 			 * next time.
1664 			 */
1665 			if (freemem_wait) {
1666 				if (npages > 1) {
1667 					cv_broadcast(&freemem_cv);
1668 				} else {
1669 					cv_signal(&freemem_cv);
1670 				}
1671 				p->pcf_wait--;
1672 			} else {
1673 				p->pcf_wait = 0;
1674 			}
1675 			mutex_exit(&new_freemem_lock);
1676 		}
1677 		mutex_exit(&p->pcf_lock);
1678 	}
1679 	ASSERT(npages == 0);
1680 }
1681 
1682 /*
1683  * A helper routine for page_create_get_something.
1684  * The indenting got to deep down there.
1685  * Unblock the pcf counters.  Any pages freed after
1686  * pcf_block got set are moved to pcf_count and
1687  * wakeups (cv_broadcast() or cv_signal()) are done as needed.
1688  */
1689 static void
1690 pcgs_unblock(void)
1691 {
1692 	int		i;
1693 	struct pcf	*p;
1694 
1695 	/* Update freemem while we're here. */
1696 	freemem = 0;
1697 	p = pcf;
1698 	for (i = 0; i < pcf_fanout; i++) {
1699 		mutex_enter(&p->pcf_lock);
1700 		ASSERT(p->pcf_count == 0);
1701 		p->pcf_count = p->pcf_reserve;
1702 		p->pcf_block = 0;
1703 		freemem += p->pcf_count;
1704 		if (p->pcf_wait) {
1705 			mutex_enter(&new_freemem_lock);
1706 			if (freemem_wait) {
1707 				if (p->pcf_reserve > 1) {
1708 					cv_broadcast(&freemem_cv);
1709 					p->pcf_wait = 0;
1710 				} else {
1711 					cv_signal(&freemem_cv);
1712 					p->pcf_wait--;
1713 				}
1714 			} else {
1715 				p->pcf_wait = 0;
1716 			}
1717 			mutex_exit(&new_freemem_lock);
1718 		}
1719 		p->pcf_reserve = 0;
1720 		mutex_exit(&p->pcf_lock);
1721 		p++;
1722 	}
1723 }
1724 
1725 /*
1726  * Called from page_create_va() when both the cache and free lists
1727  * have been checked once.
1728  *
1729  * Either returns a page or panics since the accounting was done
1730  * way before we got here.
1731  *
1732  * We don't come here often, so leave the accounting on permanently.
1733  */
1734 
1735 #define	MAX_PCGS	100
1736 
1737 #ifdef	DEBUG
1738 #define	PCGS_TRIES	100
1739 #else	/* DEBUG */
1740 #define	PCGS_TRIES	10
1741 #endif	/* DEBUG */
1742 
1743 #ifdef	VM_STATS
1744 uint_t	pcgs_counts[PCGS_TRIES];
1745 uint_t	pcgs_too_many;
1746 uint_t	pcgs_entered;
1747 uint_t	pcgs_entered_noreloc;
1748 uint_t	pcgs_locked;
1749 uint_t	pcgs_cagelocked;
1750 #endif	/* VM_STATS */
1751 
1752 static page_t *
1753 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg,
1754     caddr_t vaddr, uint_t flags)
1755 {
1756 	uint_t		count;
1757 	page_t		*pp;
1758 	uint_t		locked, i;
1759 	struct	pcf	*p;
1760 	lgrp_t		*lgrp;
1761 	int		cagelocked = 0;
1762 
1763 	VM_STAT_ADD(pcgs_entered);
1764 
1765 	/*
1766 	 * Tap any reserve freelists: if we fail now, we'll die
1767 	 * since the page(s) we're looking for have already been
1768 	 * accounted for.
1769 	 */
1770 	flags |= PG_PANIC;
1771 
1772 	if ((flags & PG_NORELOC) != 0) {
1773 		VM_STAT_ADD(pcgs_entered_noreloc);
1774 		/*
1775 		 * Requests for free pages from critical threads
1776 		 * such as pageout still won't throttle here, but
1777 		 * we must try again, to give the cageout thread
1778 		 * another chance to catch up. Since we already
1779 		 * accounted for the pages, we had better get them
1780 		 * this time.
1781 		 *
1782 		 * N.B. All non-critical threads acquire the pcgs_cagelock
1783 		 * to serialize access to the freelists. This implements a
1784 		 * turnstile-type synchornization to avoid starvation of
1785 		 * critical requests for PG_NORELOC memory by non-critical
1786 		 * threads: all non-critical threads must acquire a 'ticket'
1787 		 * before passing through, which entails making sure
1788 		 * kcage_freemem won't fall below minfree prior to grabbing
1789 		 * pages from the freelists.
1790 		 */
1791 		if (kcage_create_throttle(1, flags) == KCT_NONCRIT) {
1792 			mutex_enter(&pcgs_cagelock);
1793 			cagelocked = 1;
1794 			VM_STAT_ADD(pcgs_cagelocked);
1795 		}
1796 	}
1797 
1798 	/*
1799 	 * Time to get serious.
1800 	 * We failed to get a `correctly colored' page from both the
1801 	 * free and cache lists.
1802 	 * We escalate in stage.
1803 	 *
1804 	 * First try both lists without worring about color.
1805 	 *
1806 	 * Then, grab all page accounting locks (ie. pcf[]) and
1807 	 * steal any pages that they have and set the pcf_block flag to
1808 	 * stop deletions from the lists.  This will help because
1809 	 * a page can get added to the free list while we are looking
1810 	 * at the cache list, then another page could be added to the cache
1811 	 * list allowing the page on the free list to be removed as we
1812 	 * move from looking at the cache list to the free list. This
1813 	 * could happen over and over. We would never find the page
1814 	 * we have accounted for.
1815 	 *
1816 	 * Noreloc pages are a subset of the global (relocatable) page pool.
1817 	 * They are not tracked separately in the pcf bins, so it is
1818 	 * impossible to know when doing pcf accounting if the available
1819 	 * page(s) are noreloc pages or not. When looking for a noreloc page
1820 	 * it is quite easy to end up here even if the global (relocatable)
1821 	 * page pool has plenty of free pages but the noreloc pool is empty.
1822 	 *
1823 	 * When the noreloc pool is empty (or low), additional noreloc pages
1824 	 * are created by converting pages from the global page pool. This
1825 	 * process will stall during pcf accounting if the pcf bins are
1826 	 * already locked. Such is the case when a noreloc allocation is
1827 	 * looping here in page_create_get_something waiting for more noreloc
1828 	 * pages to appear.
1829 	 *
1830 	 * Short of adding a new field to the pcf bins to accurately track
1831 	 * the number of free noreloc pages, we instead do not grab the
1832 	 * pcgs_lock, do not set the pcf blocks and do not timeout when
1833 	 * allocating a noreloc page. This allows noreloc allocations to
1834 	 * loop without blocking global page pool allocations.
1835 	 *
1836 	 * NOTE: the behaviour of page_create_get_something has not changed
1837 	 * for the case of global page pool allocations.
1838 	 */
1839 
1840 	flags &= ~PG_MATCH_COLOR;
1841 	locked = 0;
1842 #if defined(__i386) || defined(__amd64)
1843 	flags = page_create_update_flags_x86(flags);
1844 #endif
1845 
1846 	lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
1847 
1848 	for (count = 0; kcage_on || count < MAX_PCGS; count++) {
1849 		pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
1850 		    flags, lgrp);
1851 		if (pp == NULL) {
1852 			pp = page_get_cachelist(vp, off, seg, vaddr,
1853 			    flags, lgrp);
1854 		}
1855 		if (pp == NULL) {
1856 			/*
1857 			 * Serialize.  Don't fight with other pcgs().
1858 			 */
1859 			if (!locked && (!kcage_on || !(flags & PG_NORELOC))) {
1860 				mutex_enter(&pcgs_lock);
1861 				VM_STAT_ADD(pcgs_locked);
1862 				locked = 1;
1863 				p = pcf;
1864 				for (i = 0; i < pcf_fanout; i++) {
1865 					mutex_enter(&p->pcf_lock);
1866 					ASSERT(p->pcf_block == 0);
1867 					p->pcf_block = 1;
1868 					p->pcf_reserve = p->pcf_count;
1869 					p->pcf_count = 0;
1870 					mutex_exit(&p->pcf_lock);
1871 					p++;
1872 				}
1873 				freemem = 0;
1874 			}
1875 
1876 			if (count) {
1877 				/*
1878 				 * Since page_free() puts pages on
1879 				 * a list then accounts for it, we
1880 				 * just have to wait for page_free()
1881 				 * to unlock any page it was working
1882 				 * with. The page_lock()-page_reclaim()
1883 				 * path falls in the same boat.
1884 				 *
1885 				 * We don't need to check on the
1886 				 * PG_WAIT flag, we have already
1887 				 * accounted for the page we are
1888 				 * looking for in page_create_va().
1889 				 *
1890 				 * We just wait a moment to let any
1891 				 * locked pages on the lists free up,
1892 				 * then continue around and try again.
1893 				 *
1894 				 * Will be awakened by set_freemem().
1895 				 */
1896 				mutex_enter(&pcgs_wait_lock);
1897 				cv_wait(&pcgs_cv, &pcgs_wait_lock);
1898 				mutex_exit(&pcgs_wait_lock);
1899 			}
1900 		} else {
1901 #ifdef VM_STATS
1902 			if (count >= PCGS_TRIES) {
1903 				VM_STAT_ADD(pcgs_too_many);
1904 			} else {
1905 				VM_STAT_ADD(pcgs_counts[count]);
1906 			}
1907 #endif
1908 			if (locked) {
1909 				pcgs_unblock();
1910 				mutex_exit(&pcgs_lock);
1911 			}
1912 			if (cagelocked)
1913 				mutex_exit(&pcgs_cagelock);
1914 			return (pp);
1915 		}
1916 	}
1917 	/*
1918 	 * we go down holding the pcf locks.
1919 	 */
1920 	panic("no %spage found %d",
1921 	    ((flags & PG_NORELOC) ? "non-reloc " : ""), count);
1922 	/*NOTREACHED*/
1923 }
1924 
1925 /*
1926  * Create enough pages for "bytes" worth of data starting at
1927  * "off" in "vp".
1928  *
1929  *	Where flag must be one of:
1930  *
1931  *		PG_EXCL:	Exclusive create (fail if any page already
1932  *				exists in the page cache) which does not
1933  *				wait for memory to become available.
1934  *
1935  *		PG_WAIT:	Non-exclusive create which can wait for
1936  *				memory to become available.
1937  *
1938  *		PG_PHYSCONTIG:	Allocate physically contiguous pages.
1939  *				(Not Supported)
1940  *
1941  * A doubly linked list of pages is returned to the caller.  Each page
1942  * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock)
1943  * lock.
1944  *
1945  * Unable to change the parameters to page_create() in a minor release,
1946  * we renamed page_create() to page_create_va(), changed all known calls
1947  * from page_create() to page_create_va(), and created this wrapper.
1948  *
1949  * Upon a major release, we should break compatibility by deleting this
1950  * wrapper, and replacing all the strings "page_create_va", with "page_create".
1951  *
1952  * NOTE: There is a copy of this interface as page_create_io() in
1953  *	 i86/vm/vm_machdep.c. Any bugs fixed here should be applied
1954  *	 there.
1955  */
1956 page_t *
1957 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags)
1958 {
1959 	caddr_t random_vaddr;
1960 	struct seg kseg;
1961 
1962 #ifdef DEBUG
1963 	cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p",
1964 	    (void *)caller());
1965 #endif
1966 
1967 	random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^
1968 	    (uintptr_t)(off >> PAGESHIFT));
1969 	kseg.s_as = &kas;
1970 
1971 	return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr));
1972 }
1973 
1974 #ifdef DEBUG
1975 uint32_t pg_alloc_pgs_mtbf = 0;
1976 #endif
1977 
1978 /*
1979  * Used for large page support. It will attempt to allocate
1980  * a large page(s) off the freelist.
1981  *
1982  * Returns non zero on failure.
1983  */
1984 int
1985 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr,
1986     page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz, int pgflags)
1987 {
1988 	pgcnt_t		npgs, curnpgs, totpgs;
1989 	size_t		pgsz;
1990 	page_t		*pplist = NULL, *pp;
1991 	int		err = 0;
1992 	lgrp_t		*lgrp;
1993 
1994 	ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1));
1995 	ASSERT(pgflags == 0 || pgflags == PG_LOCAL);
1996 
1997 	/*
1998 	 * Check if system heavily prefers local large pages over remote
1999 	 * on systems with multiple lgroups.
2000 	 */
2001 	if (lpg_alloc_prefer == LPAP_LOCAL && nlgrps > 1) {
2002 		pgflags = PG_LOCAL;
2003 	}
2004 
2005 	VM_STAT_ADD(alloc_pages[0]);
2006 
2007 #ifdef DEBUG
2008 	if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) {
2009 		return (ENOMEM);
2010 	}
2011 #endif
2012 
2013 	/*
2014 	 * One must be NULL but not both.
2015 	 * And one must be non NULL but not both.
2016 	 */
2017 	ASSERT(basepp != NULL || ppa != NULL);
2018 	ASSERT(basepp == NULL || ppa == NULL);
2019 
2020 #if defined(__i386) || defined(__amd64)
2021 	while (page_chk_freelist(szc) == 0) {
2022 		VM_STAT_ADD(alloc_pages[8]);
2023 		if (anypgsz == 0 || --szc == 0)
2024 			return (ENOMEM);
2025 	}
2026 #endif
2027 
2028 	pgsz = page_get_pagesize(szc);
2029 	totpgs = curnpgs = npgs = pgsz >> PAGESHIFT;
2030 
2031 	ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0);
2032 
2033 	(void) page_create_wait(npgs, PG_WAIT);
2034 
2035 	while (npgs && szc) {
2036 		lgrp = lgrp_mem_choose(seg, addr, pgsz);
2037 		if (pgflags == PG_LOCAL) {
2038 			pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2039 			    pgflags, lgrp);
2040 			if (pp == NULL) {
2041 				pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2042 				    0, lgrp);
2043 			}
2044 		} else {
2045 			pp = page_get_freelist(vp, 0, seg, addr, pgsz,
2046 			    0, lgrp);
2047 		}
2048 		if (pp != NULL) {
2049 			VM_STAT_ADD(alloc_pages[1]);
2050 			page_list_concat(&pplist, &pp);
2051 			ASSERT(npgs >= curnpgs);
2052 			npgs -= curnpgs;
2053 		} else if (anypgsz) {
2054 			VM_STAT_ADD(alloc_pages[2]);
2055 			szc--;
2056 			pgsz = page_get_pagesize(szc);
2057 			curnpgs = pgsz >> PAGESHIFT;
2058 		} else {
2059 			VM_STAT_ADD(alloc_pages[3]);
2060 			ASSERT(npgs == totpgs);
2061 			page_create_putback(npgs);
2062 			return (ENOMEM);
2063 		}
2064 	}
2065 	if (szc == 0) {
2066 		VM_STAT_ADD(alloc_pages[4]);
2067 		ASSERT(npgs != 0);
2068 		page_create_putback(npgs);
2069 		err = ENOMEM;
2070 	} else if (basepp != NULL) {
2071 		ASSERT(npgs == 0);
2072 		ASSERT(ppa == NULL);
2073 		*basepp = pplist;
2074 	}
2075 
2076 	npgs = totpgs - npgs;
2077 	pp = pplist;
2078 
2079 	/*
2080 	 * Clear the free and age bits. Also if we were passed in a ppa then
2081 	 * fill it in with all the constituent pages from the large page. But
2082 	 * if we failed to allocate all the pages just free what we got.
2083 	 */
2084 	while (npgs != 0) {
2085 		ASSERT(PP_ISFREE(pp));
2086 		ASSERT(PP_ISAGED(pp));
2087 		if (ppa != NULL || err != 0) {
2088 			if (err == 0) {
2089 				VM_STAT_ADD(alloc_pages[5]);
2090 				PP_CLRFREE(pp);
2091 				PP_CLRAGED(pp);
2092 				page_sub(&pplist, pp);
2093 				*ppa++ = pp;
2094 				npgs--;
2095 			} else {
2096 				VM_STAT_ADD(alloc_pages[6]);
2097 				ASSERT(pp->p_szc != 0);
2098 				curnpgs = page_get_pagecnt(pp->p_szc);
2099 				page_list_break(&pp, &pplist, curnpgs);
2100 				page_list_add_pages(pp, 0);
2101 				page_create_putback(curnpgs);
2102 				ASSERT(npgs >= curnpgs);
2103 				npgs -= curnpgs;
2104 			}
2105 			pp = pplist;
2106 		} else {
2107 			VM_STAT_ADD(alloc_pages[7]);
2108 			PP_CLRFREE(pp);
2109 			PP_CLRAGED(pp);
2110 			pp = pp->p_next;
2111 			npgs--;
2112 		}
2113 	}
2114 	return (err);
2115 }
2116 
2117 /*
2118  * Get a single large page off of the freelists, and set it up for use.
2119  * Number of bytes requested must be a supported page size.
2120  *
2121  * Note that this call may fail even if there is sufficient
2122  * memory available or PG_WAIT is set, so the caller must
2123  * be willing to fallback on page_create_va(), block and retry,
2124  * or fail the requester.
2125  */
2126 page_t *
2127 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
2128     struct seg *seg, caddr_t vaddr, void *arg)
2129 {
2130 	pgcnt_t		npages;
2131 	page_t		*pp;
2132 	page_t		*rootpp;
2133 	lgrp_t		*lgrp;
2134 	lgrp_id_t	*lgrpid = (lgrp_id_t *)arg;
2135 
2136 	ASSERT(vp != NULL);
2137 
2138 	ASSERT((flags & ~(PG_EXCL | PG_WAIT |
2139 	    PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0);
2140 	/* but no others */
2141 
2142 	ASSERT((flags & PG_EXCL) == PG_EXCL);
2143 
2144 	npages = btop(bytes);
2145 
2146 	if (!kcage_on || panicstr) {
2147 		/*
2148 		 * Cage is OFF, or we are single threaded in
2149 		 * panic, so make everything a RELOC request.
2150 		 */
2151 		flags &= ~PG_NORELOC;
2152 	}
2153 
2154 	/*
2155 	 * Make sure there's adequate physical memory available.
2156 	 * Note: PG_WAIT is ignored here.
2157 	 */
2158 	if (freemem <= throttlefree + npages) {
2159 		VM_STAT_ADD(page_create_large_cnt[1]);
2160 		return (NULL);
2161 	}
2162 
2163 	/*
2164 	 * If cage is on, dampen draw from cage when available
2165 	 * cage space is low.
2166 	 */
2167 	if ((flags & (PG_NORELOC | PG_WAIT)) ==  (PG_NORELOC | PG_WAIT) &&
2168 	    kcage_freemem < kcage_throttlefree + npages) {
2169 
2170 		/*
2171 		 * The cage is on, the caller wants PG_NORELOC
2172 		 * pages and available cage memory is very low.
2173 		 * Call kcage_create_throttle() to attempt to
2174 		 * control demand on the cage.
2175 		 */
2176 		if (kcage_create_throttle(npages, flags) == KCT_FAILURE) {
2177 			VM_STAT_ADD(page_create_large_cnt[2]);
2178 			return (NULL);
2179 		}
2180 	}
2181 
2182 	if (!pcf_decrement_bucket(npages) &&
2183 	    !pcf_decrement_multiple(NULL, npages, 1)) {
2184 		VM_STAT_ADD(page_create_large_cnt[4]);
2185 		return (NULL);
2186 	}
2187 
2188 	/*
2189 	 * This is where this function behaves fundamentally differently
2190 	 * than page_create_va(); since we're intending to map the page
2191 	 * with a single TTE, we have to get it as a physically contiguous
2192 	 * hardware pagesize chunk.  If we can't, we fail.
2193 	 */
2194 	if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max &&
2195 	    LGRP_EXISTS(lgrp_table[*lgrpid]))
2196 		lgrp = lgrp_table[*lgrpid];
2197 	else
2198 		lgrp = lgrp_mem_choose(seg, vaddr, bytes);
2199 
2200 	if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr,
2201 	    bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) {
2202 		page_create_putback(npages);
2203 		VM_STAT_ADD(page_create_large_cnt[5]);
2204 		return (NULL);
2205 	}
2206 
2207 	/*
2208 	 * if we got the page with the wrong mtype give it back this is a
2209 	 * workaround for CR 6249718. When CR 6249718 is fixed we never get
2210 	 * inside "if" and the workaround becomes just a nop
2211 	 */
2212 	if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) {
2213 		page_list_add_pages(rootpp, 0);
2214 		page_create_putback(npages);
2215 		VM_STAT_ADD(page_create_large_cnt[6]);
2216 		return (NULL);
2217 	}
2218 
2219 	/*
2220 	 * If satisfying this request has left us with too little
2221 	 * memory, start the wheels turning to get some back.  The
2222 	 * first clause of the test prevents waking up the pageout
2223 	 * daemon in situations where it would decide that there's
2224 	 * nothing to do.
2225 	 */
2226 	if (nscan < desscan && freemem < minfree) {
2227 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
2228 		    "pageout_cv_signal:freemem %ld", freemem);
2229 		cv_signal(&proc_pageout->p_cv);
2230 	}
2231 
2232 	pp = rootpp;
2233 	while (npages--) {
2234 		ASSERT(PAGE_EXCL(pp));
2235 		ASSERT(pp->p_vnode == NULL);
2236 		ASSERT(!hat_page_is_mapped(pp));
2237 		PP_CLRFREE(pp);
2238 		PP_CLRAGED(pp);
2239 		if (!page_hashin(pp, vp, off, NULL))
2240 			panic("page_create_large: hashin failed: page %p",
2241 			    (void *)pp);
2242 		page_io_lock(pp);
2243 		off += PAGESIZE;
2244 		pp = pp->p_next;
2245 	}
2246 
2247 	VM_STAT_ADD(page_create_large_cnt[0]);
2248 	return (rootpp);
2249 }
2250 
2251 page_t *
2252 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
2253     struct seg *seg, caddr_t vaddr)
2254 {
2255 	page_t		*plist = NULL;
2256 	pgcnt_t		npages;
2257 	pgcnt_t		found_on_free = 0;
2258 	pgcnt_t		pages_req;
2259 	page_t		*npp = NULL;
2260 	struct pcf	*p;
2261 	lgrp_t		*lgrp;
2262 
2263 	TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
2264 	    "page_create_start:vp %p off %llx bytes %lu flags %x",
2265 	    vp, off, bytes, flags);
2266 
2267 	ASSERT(bytes != 0 && vp != NULL);
2268 
2269 	if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) {
2270 		panic("page_create: invalid flags");
2271 		/*NOTREACHED*/
2272 	}
2273 	ASSERT((flags & ~(PG_EXCL | PG_WAIT |
2274 	    PG_NORELOC | PG_PANIC | PG_PUSHPAGE | PG_NORMALPRI)) == 0);
2275 	    /* but no others */
2276 
2277 	pages_req = npages = btopr(bytes);
2278 	/*
2279 	 * Try to see whether request is too large to *ever* be
2280 	 * satisfied, in order to prevent deadlock.  We arbitrarily
2281 	 * decide to limit maximum size requests to max_page_get.
2282 	 */
2283 	if (npages >= max_page_get) {
2284 		if ((flags & PG_WAIT) == 0) {
2285 			TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG,
2286 			    "page_create_toobig:vp %p off %llx npages "
2287 			    "%lu max_page_get %lu",
2288 			    vp, off, npages, max_page_get);
2289 			return (NULL);
2290 		} else {
2291 			cmn_err(CE_WARN,
2292 			    "Request for too much kernel memory "
2293 			    "(%lu bytes), will hang forever", bytes);
2294 			for (;;)
2295 				delay(1000000000);
2296 		}
2297 	}
2298 
2299 	if (!kcage_on || panicstr) {
2300 		/*
2301 		 * Cage is OFF, or we are single threaded in
2302 		 * panic, so make everything a RELOC request.
2303 		 */
2304 		flags &= ~PG_NORELOC;
2305 	}
2306 
2307 	if (freemem <= throttlefree + npages)
2308 		if (!page_create_throttle(npages, flags))
2309 			return (NULL);
2310 
2311 	/*
2312 	 * If cage is on, dampen draw from cage when available
2313 	 * cage space is low.
2314 	 */
2315 	if ((flags & PG_NORELOC) &&
2316 	    kcage_freemem < kcage_throttlefree + npages) {
2317 
2318 		/*
2319 		 * The cage is on, the caller wants PG_NORELOC
2320 		 * pages and available cage memory is very low.
2321 		 * Call kcage_create_throttle() to attempt to
2322 		 * control demand on the cage.
2323 		 */
2324 		if (kcage_create_throttle(npages, flags) == KCT_FAILURE)
2325 			return (NULL);
2326 	}
2327 
2328 	VM_STAT_ADD(page_create_cnt[0]);
2329 
2330 	if (!pcf_decrement_bucket(npages)) {
2331 		/*
2332 		 * Have to look harder.  If npages is greater than
2333 		 * one, then we might have to coalesce the counters.
2334 		 *
2335 		 * Go wait.  We come back having accounted
2336 		 * for the memory.
2337 		 */
2338 		VM_STAT_ADD(page_create_cnt[1]);
2339 		if (!page_create_wait(npages, flags)) {
2340 			VM_STAT_ADD(page_create_cnt[2]);
2341 			return (NULL);
2342 		}
2343 	}
2344 
2345 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
2346 	    "page_create_success:vp %p off %llx", vp, off);
2347 
2348 	/*
2349 	 * If satisfying this request has left us with too little
2350 	 * memory, start the wheels turning to get some back.  The
2351 	 * first clause of the test prevents waking up the pageout
2352 	 * daemon in situations where it would decide that there's
2353 	 * nothing to do.
2354 	 */
2355 	if (nscan < desscan && freemem < minfree) {
2356 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
2357 		    "pageout_cv_signal:freemem %ld", freemem);
2358 		cv_signal(&proc_pageout->p_cv);
2359 	}
2360 
2361 	/*
2362 	 * Loop around collecting the requested number of pages.
2363 	 * Most of the time, we have to `create' a new page. With
2364 	 * this in mind, pull the page off the free list before
2365 	 * getting the hash lock.  This will minimize the hash
2366 	 * lock hold time, nesting, and the like.  If it turns
2367 	 * out we don't need the page, we put it back at the end.
2368 	 */
2369 	while (npages--) {
2370 		page_t		*pp;
2371 		kmutex_t	*phm = NULL;
2372 		ulong_t		index;
2373 
2374 		index = PAGE_HASH_FUNC(vp, off);
2375 top:
2376 		ASSERT(phm == NULL);
2377 		ASSERT(index == PAGE_HASH_FUNC(vp, off));
2378 		ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
2379 
2380 		if (npp == NULL) {
2381 			/*
2382 			 * Try to get a page from the freelist (ie,
2383 			 * a page with no [vp, off] tag).  If that
2384 			 * fails, use the cachelist.
2385 			 *
2386 			 * During the first attempt at both the free
2387 			 * and cache lists we try for the correct color.
2388 			 */
2389 			/*
2390 			 * XXXX-how do we deal with virtual indexed
2391 			 * caches and and colors?
2392 			 */
2393 			VM_STAT_ADD(page_create_cnt[4]);
2394 			/*
2395 			 * Get lgroup to allocate next page of shared memory
2396 			 * from and use it to specify where to allocate
2397 			 * the physical memory
2398 			 */
2399 			lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
2400 			npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
2401 			    flags | PG_MATCH_COLOR, lgrp);
2402 			if (npp == NULL) {
2403 				npp = page_get_cachelist(vp, off, seg,
2404 				    vaddr, flags | PG_MATCH_COLOR, lgrp);
2405 				if (npp == NULL) {
2406 					npp = page_create_get_something(vp,
2407 					    off, seg, vaddr,
2408 					    flags & ~PG_MATCH_COLOR);
2409 				}
2410 
2411 				if (PP_ISAGED(npp) == 0) {
2412 					/*
2413 					 * Since this page came from the
2414 					 * cachelist, we must destroy the
2415 					 * old vnode association.
2416 					 */
2417 					page_hashout(npp, NULL);
2418 				}
2419 			}
2420 		}
2421 
2422 		/*
2423 		 * We own this page!
2424 		 */
2425 		ASSERT(PAGE_EXCL(npp));
2426 		ASSERT(npp->p_vnode == NULL);
2427 		ASSERT(!hat_page_is_mapped(npp));
2428 		PP_CLRFREE(npp);
2429 		PP_CLRAGED(npp);
2430 
2431 		/*
2432 		 * Here we have a page in our hot little mits and are
2433 		 * just waiting to stuff it on the appropriate lists.
2434 		 * Get the mutex and check to see if it really does
2435 		 * not exist.
2436 		 */
2437 		phm = PAGE_HASH_MUTEX(index);
2438 		mutex_enter(phm);
2439 		pp = page_hash_search(index, vp, off);
2440 		if (pp == NULL) {
2441 			VM_STAT_ADD(page_create_new);
2442 			pp = npp;
2443 			npp = NULL;
2444 			if (!page_hashin(pp, vp, off, phm)) {
2445 				/*
2446 				 * Since we hold the page hash mutex and
2447 				 * just searched for this page, page_hashin
2448 				 * had better not fail.  If it does, that
2449 				 * means somethread did not follow the
2450 				 * page hash mutex rules.  Panic now and
2451 				 * get it over with.  As usual, go down
2452 				 * holding all the locks.
2453 				 */
2454 				ASSERT(MUTEX_HELD(phm));
2455 				panic("page_create: "
2456 				    "hashin failed %p %p %llx %p",
2457 				    (void *)pp, (void *)vp, off, (void *)phm);
2458 				/*NOTREACHED*/
2459 			}
2460 			ASSERT(MUTEX_HELD(phm));
2461 			mutex_exit(phm);
2462 			phm = NULL;
2463 
2464 			/*
2465 			 * Hat layer locking need not be done to set
2466 			 * the following bits since the page is not hashed
2467 			 * and was on the free list (i.e., had no mappings).
2468 			 *
2469 			 * Set the reference bit to protect
2470 			 * against immediate pageout
2471 			 *
2472 			 * XXXmh modify freelist code to set reference
2473 			 * bit so we don't have to do it here.
2474 			 */
2475 			page_set_props(pp, P_REF);
2476 			found_on_free++;
2477 		} else {
2478 			VM_STAT_ADD(page_create_exists);
2479 			if (flags & PG_EXCL) {
2480 				/*
2481 				 * Found an existing page, and the caller
2482 				 * wanted all new pages.  Undo all of the work
2483 				 * we have done.
2484 				 */
2485 				mutex_exit(phm);
2486 				phm = NULL;
2487 				while (plist != NULL) {
2488 					pp = plist;
2489 					page_sub(&plist, pp);
2490 					page_io_unlock(pp);
2491 					/* large pages should not end up here */
2492 					ASSERT(pp->p_szc == 0);
2493 					/*LINTED: constant in conditional ctx*/
2494 					VN_DISPOSE(pp, B_INVAL, 0, kcred);
2495 				}
2496 				VM_STAT_ADD(page_create_found_one);
2497 				goto fail;
2498 			}
2499 			ASSERT(flags & PG_WAIT);
2500 			if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) {
2501 				/*
2502 				 * Start all over again if we blocked trying
2503 				 * to lock the page.
2504 				 */
2505 				mutex_exit(phm);
2506 				VM_STAT_ADD(page_create_page_lock_failed);
2507 				phm = NULL;
2508 				goto top;
2509 			}
2510 			mutex_exit(phm);
2511 			phm = NULL;
2512 
2513 			if (PP_ISFREE(pp)) {
2514 				ASSERT(PP_ISAGED(pp) == 0);
2515 				VM_STAT_ADD(pagecnt.pc_get_cache);
2516 				page_list_sub(pp, PG_CACHE_LIST);
2517 				PP_CLRFREE(pp);
2518 				found_on_free++;
2519 			}
2520 		}
2521 
2522 		/*
2523 		 * Got a page!  It is locked.  Acquire the i/o
2524 		 * lock since we are going to use the p_next and
2525 		 * p_prev fields to link the requested pages together.
2526 		 */
2527 		page_io_lock(pp);
2528 		page_add(&plist, pp);
2529 		plist = plist->p_next;
2530 		off += PAGESIZE;
2531 		vaddr += PAGESIZE;
2532 	}
2533 
2534 	ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1);
2535 fail:
2536 	if (npp != NULL) {
2537 		/*
2538 		 * Did not need this page after all.
2539 		 * Put it back on the free list.
2540 		 */
2541 		VM_STAT_ADD(page_create_putbacks);
2542 		PP_SETFREE(npp);
2543 		PP_SETAGED(npp);
2544 		npp->p_offset = (u_offset_t)-1;
2545 		page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
2546 		page_unlock(npp);
2547 
2548 	}
2549 
2550 	ASSERT(pages_req >= found_on_free);
2551 
2552 	{
2553 		uint_t overshoot = (uint_t)(pages_req - found_on_free);
2554 
2555 		if (overshoot) {
2556 			VM_STAT_ADD(page_create_overshoot);
2557 			p = &pcf[PCF_INDEX()];
2558 			mutex_enter(&p->pcf_lock);
2559 			if (p->pcf_block) {
2560 				p->pcf_reserve += overshoot;
2561 			} else {
2562 				p->pcf_count += overshoot;
2563 				if (p->pcf_wait) {
2564 					mutex_enter(&new_freemem_lock);
2565 					if (freemem_wait) {
2566 						cv_signal(&freemem_cv);
2567 						p->pcf_wait--;
2568 					} else {
2569 						p->pcf_wait = 0;
2570 					}
2571 					mutex_exit(&new_freemem_lock);
2572 				}
2573 			}
2574 			mutex_exit(&p->pcf_lock);
2575 			/* freemem is approximate, so this test OK */
2576 			if (!p->pcf_block)
2577 				freemem += overshoot;
2578 		}
2579 	}
2580 
2581 	return (plist);
2582 }
2583 
2584 /*
2585  * One or more constituent pages of this large page has been marked
2586  * toxic. Simply demote the large page to PAGESIZE pages and let
2587  * page_free() handle it. This routine should only be called by
2588  * large page free routines (page_free_pages() and page_destroy_pages().
2589  * All pages are locked SE_EXCL and have already been marked free.
2590  */
2591 static void
2592 page_free_toxic_pages(page_t *rootpp)
2593 {
2594 	page_t	*tpp;
2595 	pgcnt_t	i, pgcnt = page_get_pagecnt(rootpp->p_szc);
2596 	uint_t	szc = rootpp->p_szc;
2597 
2598 	for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) {
2599 		ASSERT(tpp->p_szc == szc);
2600 		ASSERT((PAGE_EXCL(tpp) &&
2601 		    !page_iolock_assert(tpp)) || panicstr);
2602 		tpp->p_szc = 0;
2603 	}
2604 
2605 	while (rootpp != NULL) {
2606 		tpp = rootpp;
2607 		page_sub(&rootpp, tpp);
2608 		ASSERT(PP_ISFREE(tpp));
2609 		PP_CLRFREE(tpp);
2610 		page_free(tpp, 1);
2611 	}
2612 }
2613 
2614 /*
2615  * Put page on the "free" list.
2616  * The free list is really two lists maintained by
2617  * the PSM of whatever machine we happen to be on.
2618  */
2619 void
2620 page_free(page_t *pp, int dontneed)
2621 {
2622 	struct pcf	*p;
2623 	uint_t		pcf_index;
2624 
2625 	ASSERT((PAGE_EXCL(pp) &&
2626 	    !page_iolock_assert(pp)) || panicstr);
2627 
2628 	if (PP_ISFREE(pp)) {
2629 		panic("page_free: page %p is free", (void *)pp);
2630 	}
2631 
2632 	if (pp->p_szc != 0) {
2633 		if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
2634 		    PP_ISKAS(pp)) {
2635 			panic("page_free: anon or kernel "
2636 			    "or no vnode large page %p", (void *)pp);
2637 		}
2638 		page_demote_vp_pages(pp);
2639 		ASSERT(pp->p_szc == 0);
2640 	}
2641 
2642 	/*
2643 	 * The page_struct_lock need not be acquired to examine these
2644 	 * fields since the page has an "exclusive" lock.
2645 	 */
2646 	if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
2647 	    pp->p_slckcnt != 0) {
2648 		panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d "
2649 		    "slckcnt = %d", (void *)pp, page_pptonum(pp), pp->p_lckcnt,
2650 		    pp->p_cowcnt, pp->p_slckcnt);
2651 		/*NOTREACHED*/
2652 	}
2653 
2654 	ASSERT(!hat_page_getshare(pp));
2655 
2656 	PP_SETFREE(pp);
2657 	ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) ||
2658 	    !hat_ismod(pp));
2659 	page_clr_all_props(pp);
2660 	ASSERT(!hat_page_getshare(pp));
2661 
2662 	/*
2663 	 * Now we add the page to the head of the free list.
2664 	 * But if this page is associated with a paged vnode
2665 	 * then we adjust the head forward so that the page is
2666 	 * effectively at the end of the list.
2667 	 */
2668 	if (pp->p_vnode == NULL) {
2669 		/*
2670 		 * Page has no identity, put it on the free list.
2671 		 */
2672 		PP_SETAGED(pp);
2673 		pp->p_offset = (u_offset_t)-1;
2674 		page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
2675 		VM_STAT_ADD(pagecnt.pc_free_free);
2676 		TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
2677 		    "page_free_free:pp %p", pp);
2678 	} else {
2679 		PP_CLRAGED(pp);
2680 
2681 		if (!dontneed) {
2682 			/* move it to the tail of the list */
2683 			page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL);
2684 
2685 			VM_STAT_ADD(pagecnt.pc_free_cache);
2686 			TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL,
2687 			    "page_free_cache_tail:pp %p", pp);
2688 		} else {
2689 			page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD);
2690 
2691 			VM_STAT_ADD(pagecnt.pc_free_dontneed);
2692 			TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD,
2693 			    "page_free_cache_head:pp %p", pp);
2694 		}
2695 	}
2696 	page_unlock(pp);
2697 
2698 	/*
2699 	 * Now do the `freemem' accounting.
2700 	 */
2701 	pcf_index = PCF_INDEX();
2702 	p = &pcf[pcf_index];
2703 
2704 	mutex_enter(&p->pcf_lock);
2705 	if (p->pcf_block) {
2706 		p->pcf_reserve += 1;
2707 	} else {
2708 		p->pcf_count += 1;
2709 		if (p->pcf_wait) {
2710 			mutex_enter(&new_freemem_lock);
2711 			/*
2712 			 * Check to see if some other thread
2713 			 * is actually waiting.  Another bucket
2714 			 * may have woken it up by now.  If there
2715 			 * are no waiters, then set our pcf_wait
2716 			 * count to zero to avoid coming in here
2717 			 * next time.  Also, since only one page
2718 			 * was put on the free list, just wake
2719 			 * up one waiter.
2720 			 */
2721 			if (freemem_wait) {
2722 				cv_signal(&freemem_cv);
2723 				p->pcf_wait--;
2724 			} else {
2725 				p->pcf_wait = 0;
2726 			}
2727 			mutex_exit(&new_freemem_lock);
2728 		}
2729 	}
2730 	mutex_exit(&p->pcf_lock);
2731 
2732 	/* freemem is approximate, so this test OK */
2733 	if (!p->pcf_block)
2734 		freemem += 1;
2735 }
2736 
2737 /*
2738  * Put page on the "free" list during intial startup.
2739  * This happens during initial single threaded execution.
2740  */
2741 void
2742 page_free_at_startup(page_t *pp)
2743 {
2744 	struct pcf	*p;
2745 	uint_t		pcf_index;
2746 
2747 	page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT);
2748 	VM_STAT_ADD(pagecnt.pc_free_free);
2749 
2750 	/*
2751 	 * Now do the `freemem' accounting.
2752 	 */
2753 	pcf_index = PCF_INDEX();
2754 	p = &pcf[pcf_index];
2755 
2756 	ASSERT(p->pcf_block == 0);
2757 	ASSERT(p->pcf_wait == 0);
2758 	p->pcf_count += 1;
2759 
2760 	/* freemem is approximate, so this is OK */
2761 	freemem += 1;
2762 }
2763 
2764 void
2765 page_free_pages(page_t *pp)
2766 {
2767 	page_t	*tpp, *rootpp = NULL;
2768 	pgcnt_t	pgcnt = page_get_pagecnt(pp->p_szc);
2769 	pgcnt_t	i;
2770 	uint_t	szc = pp->p_szc;
2771 
2772 	VM_STAT_ADD(pagecnt.pc_free_pages);
2773 	TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
2774 	    "page_free_free:pp %p", pp);
2775 
2776 	ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
2777 	if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
2778 		panic("page_free_pages: not root page %p", (void *)pp);
2779 		/*NOTREACHED*/
2780 	}
2781 
2782 	for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
2783 		ASSERT((PAGE_EXCL(tpp) &&
2784 		    !page_iolock_assert(tpp)) || panicstr);
2785 		if (PP_ISFREE(tpp)) {
2786 			panic("page_free_pages: page %p is free", (void *)tpp);
2787 			/*NOTREACHED*/
2788 		}
2789 		if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 ||
2790 		    tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) {
2791 			panic("page_free_pages %p", (void *)tpp);
2792 			/*NOTREACHED*/
2793 		}
2794 
2795 		ASSERT(!hat_page_getshare(tpp));
2796 		ASSERT(tpp->p_vnode == NULL);
2797 		ASSERT(tpp->p_szc == szc);
2798 
2799 		PP_SETFREE(tpp);
2800 		page_clr_all_props(tpp);
2801 		PP_SETAGED(tpp);
2802 		tpp->p_offset = (u_offset_t)-1;
2803 		ASSERT(tpp->p_next == tpp);
2804 		ASSERT(tpp->p_prev == tpp);
2805 		page_list_concat(&rootpp, &tpp);
2806 	}
2807 	ASSERT(rootpp == pp);
2808 
2809 	page_list_add_pages(rootpp, 0);
2810 	page_create_putback(pgcnt);
2811 }
2812 
2813 int free_pages = 1;
2814 
2815 /*
2816  * This routine attempts to return pages to the cachelist via page_release().
2817  * It does not *have* to be successful in all cases, since the pageout scanner
2818  * will catch any pages it misses.  It does need to be fast and not introduce
2819  * too much overhead.
2820  *
2821  * If a page isn't found on the unlocked sweep of the page_hash bucket, we
2822  * don't lock and retry.  This is ok, since the page scanner will eventually
2823  * find any page we miss in free_vp_pages().
2824  */
2825 void
2826 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len)
2827 {
2828 	page_t *pp;
2829 	u_offset_t eoff;
2830 	extern int swap_in_range(vnode_t *, u_offset_t, size_t);
2831 
2832 	eoff = off + len;
2833 
2834 	if (free_pages == 0)
2835 		return;
2836 	if (swap_in_range(vp, off, len))
2837 		return;
2838 
2839 	for (; off < eoff; off += PAGESIZE) {
2840 
2841 		/*
2842 		 * find the page using a fast, but inexact search. It'll be OK
2843 		 * if a few pages slip through the cracks here.
2844 		 */
2845 		pp = page_exists(vp, off);
2846 
2847 		/*
2848 		 * If we didn't find the page (it may not exist), the page
2849 		 * is free, looks still in use (shared), or we can't lock it,
2850 		 * just give up.
2851 		 */
2852 		if (pp == NULL ||
2853 		    PP_ISFREE(pp) ||
2854 		    page_share_cnt(pp) > 0 ||
2855 		    !page_trylock(pp, SE_EXCL))
2856 			continue;
2857 
2858 		/*
2859 		 * Once we have locked pp, verify that it's still the
2860 		 * correct page and not already free
2861 		 */
2862 		ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL));
2863 		if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) {
2864 			page_unlock(pp);
2865 			continue;
2866 		}
2867 
2868 		/*
2869 		 * try to release the page...
2870 		 */
2871 		(void) page_release(pp, 1);
2872 	}
2873 }
2874 
2875 /*
2876  * Reclaim the given page from the free list.
2877  * If pp is part of a large pages, only the given constituent page is reclaimed
2878  * and the large page it belonged to will be demoted.  This can only happen
2879  * if the page is not on the cachelist.
2880  *
2881  * Returns 1 on success or 0 on failure.
2882  *
2883  * The page is unlocked if it can't be reclaimed (when freemem == 0).
2884  * If `lock' is non-null, it will be dropped and re-acquired if
2885  * the routine must wait while freemem is 0.
2886  *
2887  * As it turns out, boot_getpages() does this.  It picks a page,
2888  * based on where OBP mapped in some address, gets its pfn, searches
2889  * the memsegs, locks the page, then pulls it off the free list!
2890  */
2891 int
2892 page_reclaim(page_t *pp, kmutex_t *lock)
2893 {
2894 	struct pcf	*p;
2895 	struct cpu	*cpup;
2896 	int		enough;
2897 	uint_t		i;
2898 
2899 	ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1);
2900 	ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp));
2901 
2902 	/*
2903 	 * If `freemem' is 0, we cannot reclaim this page from the
2904 	 * freelist, so release every lock we might hold: the page,
2905 	 * and the `lock' before blocking.
2906 	 *
2907 	 * The only way `freemem' can become 0 while there are pages
2908 	 * marked free (have their p->p_free bit set) is when the
2909 	 * system is low on memory and doing a page_create().  In
2910 	 * order to guarantee that once page_create() starts acquiring
2911 	 * pages it will be able to get all that it needs since `freemem'
2912 	 * was decreased by the requested amount.  So, we need to release
2913 	 * this page, and let page_create() have it.
2914 	 *
2915 	 * Since `freemem' being zero is not supposed to happen, just
2916 	 * use the usual hash stuff as a starting point.  If that bucket
2917 	 * is empty, then assume the worst, and start at the beginning
2918 	 * of the pcf array.  If we always start at the beginning
2919 	 * when acquiring more than one pcf lock, there won't be any
2920 	 * deadlock problems.
2921 	 */
2922 
2923 	/* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */
2924 
2925 	if (freemem <= throttlefree && !page_create_throttle(1l, 0)) {
2926 		pcf_acquire_all();
2927 		goto page_reclaim_nomem;
2928 	}
2929 
2930 	enough = pcf_decrement_bucket(1);
2931 
2932 	if (!enough) {
2933 		VM_STAT_ADD(page_reclaim_zero);
2934 		/*
2935 		 * Check again. Its possible that some other thread
2936 		 * could have been right behind us, and added one
2937 		 * to a list somewhere.  Acquire each of the pcf locks
2938 		 * until we find a page.
2939 		 */
2940 		p = pcf;
2941 		for (i = 0; i < pcf_fanout; i++) {
2942 			mutex_enter(&p->pcf_lock);
2943 			if (p->pcf_count >= 1) {
2944 				p->pcf_count -= 1;
2945 				/*
2946 				 * freemem is not protected by any lock. Thus,
2947 				 * we cannot have any assertion containing
2948 				 * freemem here.
2949 				 */
2950 				freemem -= 1;
2951 				enough = 1;
2952 				break;
2953 			}
2954 			p++;
2955 		}
2956 
2957 		if (!enough) {
2958 page_reclaim_nomem:
2959 			/*
2960 			 * We really can't have page `pp'.
2961 			 * Time for the no-memory dance with
2962 			 * page_free().  This is just like
2963 			 * page_create_wait().  Plus the added
2964 			 * attraction of releasing whatever mutex
2965 			 * we held when we were called with in `lock'.
2966 			 * Page_unlock() will wakeup any thread
2967 			 * waiting around for this page.
2968 			 */
2969 			if (lock) {
2970 				VM_STAT_ADD(page_reclaim_zero_locked);
2971 				mutex_exit(lock);
2972 			}
2973 			page_unlock(pp);
2974 
2975 			/*
2976 			 * get this before we drop all the pcf locks.
2977 			 */
2978 			mutex_enter(&new_freemem_lock);
2979 
2980 			p = pcf;
2981 			for (i = 0; i < pcf_fanout; i++) {
2982 				p->pcf_wait++;
2983 				mutex_exit(&p->pcf_lock);
2984 				p++;
2985 			}
2986 
2987 			freemem_wait++;
2988 			cv_wait(&freemem_cv, &new_freemem_lock);
2989 			freemem_wait--;
2990 
2991 			mutex_exit(&new_freemem_lock);
2992 
2993 			if (lock) {
2994 				mutex_enter(lock);
2995 			}
2996 			return (0);
2997 		}
2998 
2999 		/*
3000 		 * The pcf accounting has been done,
3001 		 * though none of the pcf_wait flags have been set,
3002 		 * drop the locks and continue on.
3003 		 */
3004 		while (p >= pcf) {
3005 			mutex_exit(&p->pcf_lock);
3006 			p--;
3007 		}
3008 	}
3009 
3010 
3011 	VM_STAT_ADD(pagecnt.pc_reclaim);
3012 
3013 	/*
3014 	 * page_list_sub will handle the case where pp is a large page.
3015 	 * It's possible that the page was promoted while on the freelist
3016 	 */
3017 	if (PP_ISAGED(pp)) {
3018 		page_list_sub(pp, PG_FREE_LIST);
3019 		TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE,
3020 		    "page_reclaim_free:pp %p", pp);
3021 	} else {
3022 		page_list_sub(pp, PG_CACHE_LIST);
3023 		TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE,
3024 		    "page_reclaim_cache:pp %p", pp);
3025 	}
3026 
3027 	/*
3028 	 * clear the p_free & p_age bits since this page is no longer
3029 	 * on the free list.  Notice that there was a brief time where
3030 	 * a page is marked as free, but is not on the list.
3031 	 *
3032 	 * Set the reference bit to protect against immediate pageout.
3033 	 */
3034 	PP_CLRFREE(pp);
3035 	PP_CLRAGED(pp);
3036 	page_set_props(pp, P_REF);
3037 
3038 	CPU_STATS_ENTER_K();
3039 	cpup = CPU;	/* get cpup now that CPU cannot change */
3040 	CPU_STATS_ADDQ(cpup, vm, pgrec, 1);
3041 	CPU_STATS_ADDQ(cpup, vm, pgfrec, 1);
3042 	CPU_STATS_EXIT_K();
3043 	ASSERT(pp->p_szc == 0);
3044 
3045 	return (1);
3046 }
3047 
3048 /*
3049  * Destroy identity of the page and put it back on
3050  * the page free list.  Assumes that the caller has
3051  * acquired the "exclusive" lock on the page.
3052  */
3053 void
3054 page_destroy(page_t *pp, int dontfree)
3055 {
3056 	ASSERT((PAGE_EXCL(pp) &&
3057 	    !page_iolock_assert(pp)) || panicstr);
3058 	ASSERT(pp->p_slckcnt == 0 || panicstr);
3059 
3060 	if (pp->p_szc != 0) {
3061 		if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
3062 		    PP_ISKAS(pp)) {
3063 			panic("page_destroy: anon or kernel or no vnode "
3064 			    "large page %p", (void *)pp);
3065 		}
3066 		page_demote_vp_pages(pp);
3067 		ASSERT(pp->p_szc == 0);
3068 	}
3069 
3070 	TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp);
3071 
3072 	/*
3073 	 * Unload translations, if any, then hash out the
3074 	 * page to erase its identity.
3075 	 */
3076 	(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
3077 	page_hashout(pp, NULL);
3078 
3079 	if (!dontfree) {
3080 		/*
3081 		 * Acquire the "freemem_lock" for availrmem.
3082 		 * The page_struct_lock need not be acquired for lckcnt
3083 		 * and cowcnt since the page has an "exclusive" lock.
3084 		 * We are doing a modified version of page_pp_unlock here.
3085 		 */
3086 		if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) {
3087 			mutex_enter(&freemem_lock);
3088 			if (pp->p_lckcnt != 0) {
3089 				availrmem++;
3090 				pages_locked--;
3091 				pp->p_lckcnt = 0;
3092 			}
3093 			if (pp->p_cowcnt != 0) {
3094 				availrmem += pp->p_cowcnt;
3095 				pages_locked -= pp->p_cowcnt;
3096 				pp->p_cowcnt = 0;
3097 			}
3098 			mutex_exit(&freemem_lock);
3099 		}
3100 		/*
3101 		 * Put the page on the "free" list.
3102 		 */
3103 		page_free(pp, 0);
3104 	}
3105 }
3106 
3107 void
3108 page_destroy_pages(page_t *pp)
3109 {
3110 
3111 	page_t	*tpp, *rootpp = NULL;
3112 	pgcnt_t	pgcnt = page_get_pagecnt(pp->p_szc);
3113 	pgcnt_t	i, pglcks = 0;
3114 	uint_t	szc = pp->p_szc;
3115 
3116 	ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
3117 
3118 	VM_STAT_ADD(pagecnt.pc_destroy_pages);
3119 
3120 	TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp);
3121 
3122 	if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
3123 		panic("page_destroy_pages: not root page %p", (void *)pp);
3124 		/*NOTREACHED*/
3125 	}
3126 
3127 	for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
3128 		ASSERT((PAGE_EXCL(tpp) &&
3129 		    !page_iolock_assert(tpp)) || panicstr);
3130 		ASSERT(tpp->p_slckcnt == 0 || panicstr);
3131 		(void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
3132 		page_hashout(tpp, NULL);
3133 		ASSERT(tpp->p_offset == (u_offset_t)-1);
3134 		if (tpp->p_lckcnt != 0) {
3135 			pglcks++;
3136 			tpp->p_lckcnt = 0;
3137 		} else if (tpp->p_cowcnt != 0) {
3138 			pglcks += tpp->p_cowcnt;
3139 			tpp->p_cowcnt = 0;
3140 		}
3141 		ASSERT(!hat_page_getshare(tpp));
3142 		ASSERT(tpp->p_vnode == NULL);
3143 		ASSERT(tpp->p_szc == szc);
3144 
3145 		PP_SETFREE(tpp);
3146 		page_clr_all_props(tpp);
3147 		PP_SETAGED(tpp);
3148 		ASSERT(tpp->p_next == tpp);
3149 		ASSERT(tpp->p_prev == tpp);
3150 		page_list_concat(&rootpp, &tpp);
3151 	}
3152 
3153 	ASSERT(rootpp == pp);
3154 	if (pglcks != 0) {
3155 		mutex_enter(&freemem_lock);
3156 		availrmem += pglcks;
3157 		mutex_exit(&freemem_lock);
3158 	}
3159 
3160 	page_list_add_pages(rootpp, 0);
3161 	page_create_putback(pgcnt);
3162 }
3163 
3164 /*
3165  * Similar to page_destroy(), but destroys pages which are
3166  * locked and known to be on the page free list.  Since
3167  * the page is known to be free and locked, no one can access
3168  * it.
3169  *
3170  * Also, the number of free pages does not change.
3171  */
3172 void
3173 page_destroy_free(page_t *pp)
3174 {
3175 	ASSERT(PAGE_EXCL(pp));
3176 	ASSERT(PP_ISFREE(pp));
3177 	ASSERT(pp->p_vnode);
3178 	ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0);
3179 	ASSERT(!hat_page_is_mapped(pp));
3180 	ASSERT(PP_ISAGED(pp) == 0);
3181 	ASSERT(pp->p_szc == 0);
3182 
3183 	VM_STAT_ADD(pagecnt.pc_destroy_free);
3184 	page_list_sub(pp, PG_CACHE_LIST);
3185 
3186 	page_hashout(pp, NULL);
3187 	ASSERT(pp->p_vnode == NULL);
3188 	ASSERT(pp->p_offset == (u_offset_t)-1);
3189 	ASSERT(pp->p_hash == NULL);
3190 
3191 	PP_SETAGED(pp);
3192 	page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
3193 	page_unlock(pp);
3194 
3195 	mutex_enter(&new_freemem_lock);
3196 	if (freemem_wait) {
3197 		cv_signal(&freemem_cv);
3198 	}
3199 	mutex_exit(&new_freemem_lock);
3200 }
3201 
3202 /*
3203  * Rename the page "opp" to have an identity specified
3204  * by [vp, off].  If a page already exists with this name
3205  * it is locked and destroyed.  Note that the page's
3206  * translations are not unloaded during the rename.
3207  *
3208  * This routine is used by the anon layer to "steal" the
3209  * original page and is not unlike destroying a page and
3210  * creating a new page using the same page frame.
3211  *
3212  * XXX -- Could deadlock if caller 1 tries to rename A to B while
3213  * caller 2 tries to rename B to A.
3214  */
3215 void
3216 page_rename(page_t *opp, vnode_t *vp, u_offset_t off)
3217 {
3218 	page_t		*pp;
3219 	int		olckcnt = 0;
3220 	int		ocowcnt = 0;
3221 	kmutex_t	*phm;
3222 	ulong_t		index;
3223 
3224 	ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp));
3225 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3226 	ASSERT(PP_ISFREE(opp) == 0);
3227 
3228 	VM_STAT_ADD(page_rename_count);
3229 
3230 	TRACE_3(TR_FAC_VM, TR_PAGE_RENAME,
3231 	    "page rename:pp %p vp %p off %llx", opp, vp, off);
3232 
3233 	/*
3234 	 * CacheFS may call page_rename for a large NFS page
3235 	 * when both CacheFS and NFS mount points are used
3236 	 * by applications. Demote this large page before
3237 	 * renaming it, to ensure that there are no "partial"
3238 	 * large pages left lying around.
3239 	 */
3240 	if (opp->p_szc != 0) {
3241 		vnode_t *ovp = opp->p_vnode;
3242 		ASSERT(ovp != NULL);
3243 		ASSERT(!IS_SWAPFSVP(ovp));
3244 		ASSERT(!VN_ISKAS(ovp));
3245 		page_demote_vp_pages(opp);
3246 		ASSERT(opp->p_szc == 0);
3247 	}
3248 
3249 	page_hashout(opp, NULL);
3250 	PP_CLRAGED(opp);
3251 
3252 	/*
3253 	 * Acquire the appropriate page hash lock, since
3254 	 * we're going to rename the page.
3255 	 */
3256 	index = PAGE_HASH_FUNC(vp, off);
3257 	phm = PAGE_HASH_MUTEX(index);
3258 	mutex_enter(phm);
3259 top:
3260 	/*
3261 	 * Look for an existing page with this name and destroy it if found.
3262 	 * By holding the page hash lock all the way to the page_hashin()
3263 	 * call, we are assured that no page can be created with this
3264 	 * identity.  In the case when the phm lock is dropped to undo any
3265 	 * hat layer mappings, the existing page is held with an "exclusive"
3266 	 * lock, again preventing another page from being created with
3267 	 * this identity.
3268 	 */
3269 	pp = page_hash_search(index, vp, off);
3270 	if (pp != NULL) {
3271 		VM_STAT_ADD(page_rename_exists);
3272 
3273 		/*
3274 		 * As it turns out, this is one of only two places where
3275 		 * page_lock() needs to hold the passed in lock in the
3276 		 * successful case.  In all of the others, the lock could
3277 		 * be dropped as soon as the attempt is made to lock
3278 		 * the page.  It is tempting to add yet another arguement,
3279 		 * PL_KEEP or PL_DROP, to let page_lock know what to do.
3280 		 */
3281 		if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) {
3282 			/*
3283 			 * Went to sleep because the page could not
3284 			 * be locked.  We were woken up when the page
3285 			 * was unlocked, or when the page was destroyed.
3286 			 * In either case, `phm' was dropped while we
3287 			 * slept.  Hence we should not just roar through
3288 			 * this loop.
3289 			 */
3290 			goto top;
3291 		}
3292 
3293 		/*
3294 		 * If an existing page is a large page, then demote
3295 		 * it to ensure that no "partial" large pages are
3296 		 * "created" after page_rename. An existing page
3297 		 * can be a CacheFS page, and can't belong to swapfs.
3298 		 */
3299 		if (hat_page_is_mapped(pp)) {
3300 			/*
3301 			 * Unload translations.  Since we hold the
3302 			 * exclusive lock on this page, the page
3303 			 * can not be changed while we drop phm.
3304 			 * This is also not a lock protocol violation,
3305 			 * but rather the proper way to do things.
3306 			 */
3307 			mutex_exit(phm);
3308 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
3309 			if (pp->p_szc != 0) {
3310 				ASSERT(!IS_SWAPFSVP(vp));
3311 				ASSERT(!VN_ISKAS(vp));
3312 				page_demote_vp_pages(pp);
3313 				ASSERT(pp->p_szc == 0);
3314 			}
3315 			mutex_enter(phm);
3316 		} else if (pp->p_szc != 0) {
3317 			ASSERT(!IS_SWAPFSVP(vp));
3318 			ASSERT(!VN_ISKAS(vp));
3319 			mutex_exit(phm);
3320 			page_demote_vp_pages(pp);
3321 			ASSERT(pp->p_szc == 0);
3322 			mutex_enter(phm);
3323 		}
3324 		page_hashout(pp, phm);
3325 	}
3326 	/*
3327 	 * Hash in the page with the new identity.
3328 	 */
3329 	if (!page_hashin(opp, vp, off, phm)) {
3330 		/*
3331 		 * We were holding phm while we searched for [vp, off]
3332 		 * and only dropped phm if we found and locked a page.
3333 		 * If we can't create this page now, then some thing
3334 		 * is really broken.
3335 		 */
3336 		panic("page_rename: Can't hash in page: %p", (void *)pp);
3337 		/*NOTREACHED*/
3338 	}
3339 
3340 	ASSERT(MUTEX_HELD(phm));
3341 	mutex_exit(phm);
3342 
3343 	/*
3344 	 * Now that we have dropped phm, lets get around to finishing up
3345 	 * with pp.
3346 	 */
3347 	if (pp != NULL) {
3348 		ASSERT(!hat_page_is_mapped(pp));
3349 		/* for now large pages should not end up here */
3350 		ASSERT(pp->p_szc == 0);
3351 		/*
3352 		 * Save the locks for transfer to the new page and then
3353 		 * clear them so page_free doesn't think they're important.
3354 		 * The page_struct_lock need not be acquired for lckcnt and
3355 		 * cowcnt since the page has an "exclusive" lock.
3356 		 */
3357 		olckcnt = pp->p_lckcnt;
3358 		ocowcnt = pp->p_cowcnt;
3359 		pp->p_lckcnt = pp->p_cowcnt = 0;
3360 
3361 		/*
3362 		 * Put the page on the "free" list after we drop
3363 		 * the lock.  The less work under the lock the better.
3364 		 */
3365 		/*LINTED: constant in conditional context*/
3366 		VN_DISPOSE(pp, B_FREE, 0, kcred);
3367 	}
3368 
3369 	/*
3370 	 * Transfer the lock count from the old page (if any).
3371 	 * The page_struct_lock need not be acquired for lckcnt and
3372 	 * cowcnt since the page has an "exclusive" lock.
3373 	 */
3374 	opp->p_lckcnt += olckcnt;
3375 	opp->p_cowcnt += ocowcnt;
3376 }
3377 
3378 /*
3379  * low level routine to add page `pp' to the hash and vp chains for [vp, offset]
3380  *
3381  * Pages are normally inserted at the start of a vnode's v_pages list.
3382  * If the vnode is VMODSORT and the page is modified, it goes at the end.
3383  * This can happen when a modified page is relocated for DR.
3384  *
3385  * Returns 1 on success and 0 on failure.
3386  */
3387 static int
3388 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset)
3389 {
3390 	page_t		**listp;
3391 	page_t		*tp;
3392 	ulong_t		index;
3393 
3394 	ASSERT(PAGE_EXCL(pp));
3395 	ASSERT(vp != NULL);
3396 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
3397 
3398 	/*
3399 	 * Be sure to set these up before the page is inserted on the hash
3400 	 * list.  As soon as the page is placed on the list some other
3401 	 * thread might get confused and wonder how this page could
3402 	 * possibly hash to this list.
3403 	 */
3404 	pp->p_vnode = vp;
3405 	pp->p_offset = offset;
3406 
3407 	/*
3408 	 * record if this page is on a swap vnode
3409 	 */
3410 	if ((vp->v_flag & VISSWAP) != 0)
3411 		PP_SETSWAP(pp);
3412 
3413 	index = PAGE_HASH_FUNC(vp, offset);
3414 	ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index)));
3415 	listp = &page_hash[index];
3416 
3417 	/*
3418 	 * If this page is already hashed in, fail this attempt to add it.
3419 	 */
3420 	for (tp = *listp; tp != NULL; tp = tp->p_hash) {
3421 		if (tp->p_vnode == vp && tp->p_offset == offset) {
3422 			pp->p_vnode = NULL;
3423 			pp->p_offset = (u_offset_t)(-1);
3424 			return (0);
3425 		}
3426 	}
3427 	pp->p_hash = *listp;
3428 	*listp = pp;
3429 
3430 	/*
3431 	 * Add the page to the vnode's list of pages
3432 	 */
3433 	if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp))
3434 		listp = &vp->v_pages->p_vpprev->p_vpnext;
3435 	else
3436 		listp = &vp->v_pages;
3437 
3438 	page_vpadd(listp, pp);
3439 
3440 	return (1);
3441 }
3442 
3443 /*
3444  * Add page `pp' to both the hash and vp chains for [vp, offset].
3445  *
3446  * Returns 1 on success and 0 on failure.
3447  * If hold is passed in, it is not dropped.
3448  */
3449 int
3450 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold)
3451 {
3452 	kmutex_t	*phm = NULL;
3453 	kmutex_t	*vphm;
3454 	int		rc;
3455 
3456 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3457 	ASSERT(pp->p_fsdata == 0 || panicstr);
3458 
3459 	TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN,
3460 	    "page_hashin:pp %p vp %p offset %llx",
3461 	    pp, vp, offset);
3462 
3463 	VM_STAT_ADD(hashin_count);
3464 
3465 	if (hold != NULL)
3466 		phm = hold;
3467 	else {
3468 		VM_STAT_ADD(hashin_not_held);
3469 		phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset));
3470 		mutex_enter(phm);
3471 	}
3472 
3473 	vphm = page_vnode_mutex(vp);
3474 	mutex_enter(vphm);
3475 	rc = page_do_hashin(pp, vp, offset);
3476 	mutex_exit(vphm);
3477 	if (hold == NULL)
3478 		mutex_exit(phm);
3479 	if (rc == 0)
3480 		VM_STAT_ADD(hashin_already);
3481 	return (rc);
3482 }
3483 
3484 /*
3485  * Remove page ``pp'' from the hash and vp chains and remove vp association.
3486  * All mutexes must be held
3487  */
3488 static void
3489 page_do_hashout(page_t *pp)
3490 {
3491 	page_t	**hpp;
3492 	page_t	*hp;
3493 	vnode_t	*vp = pp->p_vnode;
3494 
3495 	ASSERT(vp != NULL);
3496 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
3497 
3498 	/*
3499 	 * First, take pp off of its hash chain.
3500 	 */
3501 	hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)];
3502 
3503 	for (;;) {
3504 		hp = *hpp;
3505 		if (hp == pp)
3506 			break;
3507 		if (hp == NULL) {
3508 			panic("page_do_hashout");
3509 			/*NOTREACHED*/
3510 		}
3511 		hpp = &hp->p_hash;
3512 	}
3513 	*hpp = pp->p_hash;
3514 
3515 	/*
3516 	 * Now remove it from its associated vnode.
3517 	 */
3518 	if (vp->v_pages)
3519 		page_vpsub(&vp->v_pages, pp);
3520 
3521 	pp->p_hash = NULL;
3522 	page_clr_all_props(pp);
3523 	PP_CLRSWAP(pp);
3524 	pp->p_vnode = NULL;
3525 	pp->p_offset = (u_offset_t)-1;
3526 	pp->p_fsdata = 0;
3527 }
3528 
3529 /*
3530  * Remove page ``pp'' from the hash and vp chains and remove vp association.
3531  *
3532  * When `phm' is non-NULL it contains the address of the mutex protecting the
3533  * hash list pp is on.  It is not dropped.
3534  */
3535 void
3536 page_hashout(page_t *pp, kmutex_t *phm)
3537 {
3538 	vnode_t		*vp;
3539 	ulong_t		index;
3540 	kmutex_t	*nphm;
3541 	kmutex_t	*vphm;
3542 	kmutex_t	*sep;
3543 
3544 	ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1);
3545 	ASSERT(pp->p_vnode != NULL);
3546 	ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr);
3547 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode)));
3548 
3549 	vp = pp->p_vnode;
3550 
3551 	TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT,
3552 	    "page_hashout:pp %p vp %p", pp, vp);
3553 
3554 	/* Kernel probe */
3555 	TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */,
3556 	    tnf_opaque, vnode, vp,
3557 	    tnf_offset, offset, pp->p_offset);
3558 
3559 	/*
3560 	 *
3561 	 */
3562 	VM_STAT_ADD(hashout_count);
3563 	index = PAGE_HASH_FUNC(vp, pp->p_offset);
3564 	if (phm == NULL) {
3565 		VM_STAT_ADD(hashout_not_held);
3566 		nphm = PAGE_HASH_MUTEX(index);
3567 		mutex_enter(nphm);
3568 	}
3569 	ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1);
3570 
3571 
3572 	/*
3573 	 * grab page vnode mutex and remove it...
3574 	 */
3575 	vphm = page_vnode_mutex(vp);
3576 	mutex_enter(vphm);
3577 
3578 	page_do_hashout(pp);
3579 
3580 	mutex_exit(vphm);
3581 	if (phm == NULL)
3582 		mutex_exit(nphm);
3583 
3584 	/*
3585 	 * Wake up processes waiting for this page.  The page's
3586 	 * identity has been changed, and is probably not the
3587 	 * desired page any longer.
3588 	 */
3589 	sep = page_se_mutex(pp);
3590 	mutex_enter(sep);
3591 	pp->p_selock &= ~SE_EWANTED;
3592 	if (CV_HAS_WAITERS(&pp->p_cv))
3593 		cv_broadcast(&pp->p_cv);
3594 	mutex_exit(sep);
3595 }
3596 
3597 /*
3598  * Add the page to the front of a linked list of pages
3599  * using the p_next & p_prev pointers for the list.
3600  * The caller is responsible for protecting the list pointers.
3601  */
3602 void
3603 page_add(page_t **ppp, page_t *pp)
3604 {
3605 	ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
3606 
3607 	page_add_common(ppp, pp);
3608 }
3609 
3610 
3611 
3612 /*
3613  *  Common code for page_add() and mach_page_add()
3614  */
3615 void
3616 page_add_common(page_t **ppp, page_t *pp)
3617 {
3618 	if (*ppp == NULL) {
3619 		pp->p_next = pp->p_prev = pp;
3620 	} else {
3621 		pp->p_next = *ppp;
3622 		pp->p_prev = (*ppp)->p_prev;
3623 		(*ppp)->p_prev = pp;
3624 		pp->p_prev->p_next = pp;
3625 	}
3626 	*ppp = pp;
3627 }
3628 
3629 
3630 /*
3631  * Remove this page from a linked list of pages
3632  * using the p_next & p_prev pointers for the list.
3633  *
3634  * The caller is responsible for protecting the list pointers.
3635  */
3636 void
3637 page_sub(page_t **ppp, page_t *pp)
3638 {
3639 	ASSERT((PP_ISFREE(pp)) ? 1 :
3640 	    (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
3641 
3642 	if (*ppp == NULL || pp == NULL) {
3643 		panic("page_sub: bad arg(s): pp %p, *ppp %p",
3644 		    (void *)pp, (void *)(*ppp));
3645 		/*NOTREACHED*/
3646 	}
3647 
3648 	page_sub_common(ppp, pp);
3649 }
3650 
3651 
3652 /*
3653  *  Common code for page_sub() and mach_page_sub()
3654  */
3655 void
3656 page_sub_common(page_t **ppp, page_t *pp)
3657 {
3658 	if (*ppp == pp)
3659 		*ppp = pp->p_next;		/* go to next page */
3660 
3661 	if (*ppp == pp)
3662 		*ppp = NULL;			/* page list is gone */
3663 	else {
3664 		pp->p_prev->p_next = pp->p_next;
3665 		pp->p_next->p_prev = pp->p_prev;
3666 	}
3667 	pp->p_prev = pp->p_next = pp;		/* make pp a list of one */
3668 }
3669 
3670 
3671 /*
3672  * Break page list cppp into two lists with npages in the first list.
3673  * The tail is returned in nppp.
3674  */
3675 void
3676 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages)
3677 {
3678 	page_t *s1pp = *oppp;
3679 	page_t *s2pp;
3680 	page_t *e1pp, *e2pp;
3681 	long n = 0;
3682 
3683 	if (s1pp == NULL) {
3684 		*nppp = NULL;
3685 		return;
3686 	}
3687 	if (npages == 0) {
3688 		*nppp = s1pp;
3689 		*oppp = NULL;
3690 		return;
3691 	}
3692 	for (n = 0, s2pp = *oppp; n < npages; n++) {
3693 		s2pp = s2pp->p_next;
3694 	}
3695 	/* Fix head and tail of new lists */
3696 	e1pp = s2pp->p_prev;
3697 	e2pp = s1pp->p_prev;
3698 	s1pp->p_prev = e1pp;
3699 	e1pp->p_next = s1pp;
3700 	s2pp->p_prev = e2pp;
3701 	e2pp->p_next = s2pp;
3702 
3703 	/* second list empty */
3704 	if (s2pp == s1pp) {
3705 		*oppp = s1pp;
3706 		*nppp = NULL;
3707 	} else {
3708 		*oppp = s1pp;
3709 		*nppp = s2pp;
3710 	}
3711 }
3712 
3713 /*
3714  * Concatenate page list nppp onto the end of list ppp.
3715  */
3716 void
3717 page_list_concat(page_t **ppp, page_t **nppp)
3718 {
3719 	page_t *s1pp, *s2pp, *e1pp, *e2pp;
3720 
3721 	if (*nppp == NULL) {
3722 		return;
3723 	}
3724 	if (*ppp == NULL) {
3725 		*ppp = *nppp;
3726 		return;
3727 	}
3728 	s1pp = *ppp;
3729 	e1pp =  s1pp->p_prev;
3730 	s2pp = *nppp;
3731 	e2pp = s2pp->p_prev;
3732 	s1pp->p_prev = e2pp;
3733 	e2pp->p_next = s1pp;
3734 	e1pp->p_next = s2pp;
3735 	s2pp->p_prev = e1pp;
3736 }
3737 
3738 /*
3739  * return the next page in the page list
3740  */
3741 page_t *
3742 page_list_next(page_t *pp)
3743 {
3744 	return (pp->p_next);
3745 }
3746 
3747 
3748 /*
3749  * Add the page to the front of the linked list of pages
3750  * using p_vpnext/p_vpprev pointers for the list.
3751  *
3752  * The caller is responsible for protecting the lists.
3753  */
3754 void
3755 page_vpadd(page_t **ppp, page_t *pp)
3756 {
3757 	if (*ppp == NULL) {
3758 		pp->p_vpnext = pp->p_vpprev = pp;
3759 	} else {
3760 		pp->p_vpnext = *ppp;
3761 		pp->p_vpprev = (*ppp)->p_vpprev;
3762 		(*ppp)->p_vpprev = pp;
3763 		pp->p_vpprev->p_vpnext = pp;
3764 	}
3765 	*ppp = pp;
3766 }
3767 
3768 /*
3769  * Remove this page from the linked list of pages
3770  * using p_vpnext/p_vpprev pointers for the list.
3771  *
3772  * The caller is responsible for protecting the lists.
3773  */
3774 void
3775 page_vpsub(page_t **ppp, page_t *pp)
3776 {
3777 	if (*ppp == NULL || pp == NULL) {
3778 		panic("page_vpsub: bad arg(s): pp %p, *ppp %p",
3779 		    (void *)pp, (void *)(*ppp));
3780 		/*NOTREACHED*/
3781 	}
3782 
3783 	if (*ppp == pp)
3784 		*ppp = pp->p_vpnext;		/* go to next page */
3785 
3786 	if (*ppp == pp)
3787 		*ppp = NULL;			/* page list is gone */
3788 	else {
3789 		pp->p_vpprev->p_vpnext = pp->p_vpnext;
3790 		pp->p_vpnext->p_vpprev = pp->p_vpprev;
3791 	}
3792 	pp->p_vpprev = pp->p_vpnext = pp;	/* make pp a list of one */
3793 }
3794 
3795 /*
3796  * Lock a physical page into memory "long term".  Used to support "lock
3797  * in memory" functions.  Accepts the page to be locked, and a cow variable
3798  * to indicate whether a the lock will travel to the new page during
3799  * a potential copy-on-write.
3800  */
3801 int
3802 page_pp_lock(
3803 	page_t *pp,			/* page to be locked */
3804 	int cow,			/* cow lock */
3805 	int kernel)			/* must succeed -- ignore checking */
3806 {
3807 	int r = 0;			/* result -- assume failure */
3808 
3809 	ASSERT(PAGE_LOCKED(pp));
3810 
3811 	page_struct_lock(pp);
3812 	/*
3813 	 * Acquire the "freemem_lock" for availrmem.
3814 	 */
3815 	if (cow) {
3816 		mutex_enter(&freemem_lock);
3817 		if ((availrmem > pages_pp_maximum) &&
3818 		    (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
3819 			availrmem--;
3820 			pages_locked++;
3821 			mutex_exit(&freemem_lock);
3822 			r = 1;
3823 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
3824 				cmn_err(CE_WARN,
3825 				    "COW lock limit reached on pfn 0x%lx",
3826 				    page_pptonum(pp));
3827 			}
3828 		} else
3829 			mutex_exit(&freemem_lock);
3830 	} else {
3831 		if (pp->p_lckcnt) {
3832 			if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
3833 				r = 1;
3834 				if (++pp->p_lckcnt ==
3835 				    (ushort_t)PAGE_LOCK_MAXIMUM) {
3836 					cmn_err(CE_WARN, "Page lock limit "
3837 					    "reached on pfn 0x%lx",
3838 					    page_pptonum(pp));
3839 				}
3840 			}
3841 		} else {
3842 			if (kernel) {
3843 				/* availrmem accounting done by caller */
3844 				++pp->p_lckcnt;
3845 				r = 1;
3846 			} else {
3847 				mutex_enter(&freemem_lock);
3848 				if (availrmem > pages_pp_maximum) {
3849 					availrmem--;
3850 					pages_locked++;
3851 					++pp->p_lckcnt;
3852 					r = 1;
3853 				}
3854 				mutex_exit(&freemem_lock);
3855 			}
3856 		}
3857 	}
3858 	page_struct_unlock(pp);
3859 	return (r);
3860 }
3861 
3862 /*
3863  * Decommit a lock on a physical page frame.  Account for cow locks if
3864  * appropriate.
3865  */
3866 void
3867 page_pp_unlock(
3868 	page_t *pp,			/* page to be unlocked */
3869 	int cow,			/* expect cow lock */
3870 	int kernel)			/* this was a kernel lock */
3871 {
3872 	ASSERT(PAGE_LOCKED(pp));
3873 
3874 	page_struct_lock(pp);
3875 	/*
3876 	 * Acquire the "freemem_lock" for availrmem.
3877 	 * If cowcnt or lcknt is already 0 do nothing; i.e., we
3878 	 * could be called to unlock even if nothing is locked. This could
3879 	 * happen if locked file pages were truncated (removing the lock)
3880 	 * and the file was grown again and new pages faulted in; the new
3881 	 * pages are unlocked but the segment still thinks they're locked.
3882 	 */
3883 	if (cow) {
3884 		if (pp->p_cowcnt) {
3885 			mutex_enter(&freemem_lock);
3886 			pp->p_cowcnt--;
3887 			availrmem++;
3888 			pages_locked--;
3889 			mutex_exit(&freemem_lock);
3890 		}
3891 	} else {
3892 		if (pp->p_lckcnt && --pp->p_lckcnt == 0) {
3893 			if (!kernel) {
3894 				mutex_enter(&freemem_lock);
3895 				availrmem++;
3896 				pages_locked--;
3897 				mutex_exit(&freemem_lock);
3898 			}
3899 		}
3900 	}
3901 	page_struct_unlock(pp);
3902 }
3903 
3904 /*
3905  * This routine reserves availrmem for npages;
3906  * 	flags: KM_NOSLEEP or KM_SLEEP
3907  * 	returns 1 on success or 0 on failure
3908  */
3909 int
3910 page_resv(pgcnt_t npages, uint_t flags)
3911 {
3912 	mutex_enter(&freemem_lock);
3913 	while (availrmem < tune.t_minarmem + npages) {
3914 		if (flags & KM_NOSLEEP) {
3915 			mutex_exit(&freemem_lock);
3916 			return (0);
3917 		}
3918 		mutex_exit(&freemem_lock);
3919 		page_needfree(npages);
3920 		kmem_reap();
3921 		delay(hz >> 2);
3922 		page_needfree(-(spgcnt_t)npages);
3923 		mutex_enter(&freemem_lock);
3924 	}
3925 	availrmem -= npages;
3926 	mutex_exit(&freemem_lock);
3927 	return (1);
3928 }
3929 
3930 /*
3931  * This routine unreserves availrmem for npages;
3932  */
3933 void
3934 page_unresv(pgcnt_t npages)
3935 {
3936 	mutex_enter(&freemem_lock);
3937 	availrmem += npages;
3938 	mutex_exit(&freemem_lock);
3939 }
3940 
3941 /*
3942  * See Statement at the beginning of segvn_lockop() regarding
3943  * the way we handle cowcnts and lckcnts.
3944  *
3945  * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage
3946  * that breaks COW has PROT_WRITE.
3947  *
3948  * Note that, we may also break COW in case we are softlocking
3949  * on read access during physio;
3950  * in this softlock case, the vpage may not have PROT_WRITE.
3951  * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp'
3952  * if the vpage doesn't have PROT_WRITE.
3953  *
3954  * This routine is never called if we are stealing a page
3955  * in anon_private.
3956  *
3957  * The caller subtracted from availrmem for read only mapping.
3958  * if lckcnt is 1 increment availrmem.
3959  */
3960 void
3961 page_pp_useclaim(
3962 	page_t *opp,		/* original page frame losing lock */
3963 	page_t *npp,		/* new page frame gaining lock */
3964 	uint_t	write_perm) 	/* set if vpage has PROT_WRITE */
3965 {
3966 	int payback = 0;
3967 	int nidx, oidx;
3968 
3969 	ASSERT(PAGE_LOCKED(opp));
3970 	ASSERT(PAGE_LOCKED(npp));
3971 
3972 	/*
3973 	 * Since we have two pages we probably have two locks.  We need to take
3974 	 * them in a defined order to avoid deadlocks.  It's also possible they
3975 	 * both hash to the same lock in which case this is a non-issue.
3976 	 */
3977 	nidx = PAGE_LLOCK_HASH(PP_PAGEROOT(npp));
3978 	oidx = PAGE_LLOCK_HASH(PP_PAGEROOT(opp));
3979 	if (nidx < oidx) {
3980 		page_struct_lock(npp);
3981 		page_struct_lock(opp);
3982 	} else if (oidx < nidx) {
3983 		page_struct_lock(opp);
3984 		page_struct_lock(npp);
3985 	} else {	/* The pages hash to the same lock */
3986 		page_struct_lock(npp);
3987 	}
3988 
3989 	ASSERT(npp->p_cowcnt == 0);
3990 	ASSERT(npp->p_lckcnt == 0);
3991 
3992 	/* Don't use claim if nothing is locked (see page_pp_unlock above) */
3993 	if ((write_perm && opp->p_cowcnt != 0) ||
3994 	    (!write_perm && opp->p_lckcnt != 0)) {
3995 
3996 		if (write_perm) {
3997 			npp->p_cowcnt++;
3998 			ASSERT(opp->p_cowcnt != 0);
3999 			opp->p_cowcnt--;
4000 		} else {
4001 
4002 			ASSERT(opp->p_lckcnt != 0);
4003 
4004 			/*
4005 			 * We didn't need availrmem decremented if p_lckcnt on
4006 			 * original page is 1. Here, we are unlocking
4007 			 * read-only copy belonging to original page and
4008 			 * are locking a copy belonging to new page.
4009 			 */
4010 			if (opp->p_lckcnt == 1)
4011 				payback = 1;
4012 
4013 			npp->p_lckcnt++;
4014 			opp->p_lckcnt--;
4015 		}
4016 	}
4017 	if (payback) {
4018 		mutex_enter(&freemem_lock);
4019 		availrmem++;
4020 		pages_useclaim--;
4021 		mutex_exit(&freemem_lock);
4022 	}
4023 
4024 	if (nidx < oidx) {
4025 		page_struct_unlock(opp);
4026 		page_struct_unlock(npp);
4027 	} else if (oidx < nidx) {
4028 		page_struct_unlock(npp);
4029 		page_struct_unlock(opp);
4030 	} else {	/* The pages hash to the same lock */
4031 		page_struct_unlock(npp);
4032 	}
4033 }
4034 
4035 /*
4036  * Simple claim adjust functions -- used to support changes in
4037  * claims due to changes in access permissions.  Used by segvn_setprot().
4038  */
4039 int
4040 page_addclaim(page_t *pp)
4041 {
4042 	int r = 0;			/* result */
4043 
4044 	ASSERT(PAGE_LOCKED(pp));
4045 
4046 	page_struct_lock(pp);
4047 	ASSERT(pp->p_lckcnt != 0);
4048 
4049 	if (pp->p_lckcnt == 1) {
4050 		if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
4051 			--pp->p_lckcnt;
4052 			r = 1;
4053 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4054 				cmn_err(CE_WARN,
4055 				    "COW lock limit reached on pfn 0x%lx",
4056 				    page_pptonum(pp));
4057 			}
4058 		}
4059 	} else {
4060 		mutex_enter(&freemem_lock);
4061 		if ((availrmem > pages_pp_maximum) &&
4062 		    (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
4063 			--availrmem;
4064 			++pages_claimed;
4065 			mutex_exit(&freemem_lock);
4066 			--pp->p_lckcnt;
4067 			r = 1;
4068 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4069 				cmn_err(CE_WARN,
4070 				    "COW lock limit reached on pfn 0x%lx",
4071 				    page_pptonum(pp));
4072 			}
4073 		} else
4074 			mutex_exit(&freemem_lock);
4075 	}
4076 	page_struct_unlock(pp);
4077 	return (r);
4078 }
4079 
4080 int
4081 page_subclaim(page_t *pp)
4082 {
4083 	int r = 0;
4084 
4085 	ASSERT(PAGE_LOCKED(pp));
4086 
4087 	page_struct_lock(pp);
4088 	ASSERT(pp->p_cowcnt != 0);
4089 
4090 	if (pp->p_lckcnt) {
4091 		if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
4092 			r = 1;
4093 			/*
4094 			 * for availrmem
4095 			 */
4096 			mutex_enter(&freemem_lock);
4097 			availrmem++;
4098 			pages_claimed--;
4099 			mutex_exit(&freemem_lock);
4100 
4101 			pp->p_cowcnt--;
4102 
4103 			if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4104 				cmn_err(CE_WARN,
4105 				    "Page lock limit reached on pfn 0x%lx",
4106 				    page_pptonum(pp));
4107 			}
4108 		}
4109 	} else {
4110 		r = 1;
4111 		pp->p_cowcnt--;
4112 		pp->p_lckcnt++;
4113 	}
4114 	page_struct_unlock(pp);
4115 	return (r);
4116 }
4117 
4118 /*
4119  * Variant of page_addclaim(), where ppa[] contains the pages of a single large
4120  * page.
4121  */
4122 int
4123 page_addclaim_pages(page_t  **ppa)
4124 {
4125 	pgcnt_t	lckpgs = 0, pg_idx;
4126 
4127 	VM_STAT_ADD(pagecnt.pc_addclaim_pages);
4128 
4129 	/*
4130 	 * Only need to take the page struct lock on the large page root.
4131 	 */
4132 	page_struct_lock(ppa[0]);
4133 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4134 
4135 		ASSERT(PAGE_LOCKED(ppa[pg_idx]));
4136 		ASSERT(ppa[pg_idx]->p_lckcnt != 0);
4137 		if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4138 			page_struct_unlock(ppa[0]);
4139 			return (0);
4140 		}
4141 		if (ppa[pg_idx]->p_lckcnt > 1)
4142 			lckpgs++;
4143 	}
4144 
4145 	if (lckpgs != 0) {
4146 		mutex_enter(&freemem_lock);
4147 		if (availrmem >= pages_pp_maximum + lckpgs) {
4148 			availrmem -= lckpgs;
4149 			pages_claimed += lckpgs;
4150 		} else {
4151 			mutex_exit(&freemem_lock);
4152 			page_struct_unlock(ppa[0]);
4153 			return (0);
4154 		}
4155 		mutex_exit(&freemem_lock);
4156 	}
4157 
4158 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4159 		ppa[pg_idx]->p_lckcnt--;
4160 		ppa[pg_idx]->p_cowcnt++;
4161 	}
4162 	page_struct_unlock(ppa[0]);
4163 	return (1);
4164 }
4165 
4166 /*
4167  * Variant of page_subclaim(), where ppa[] contains the pages of a single large
4168  * page.
4169  */
4170 int
4171 page_subclaim_pages(page_t  **ppa)
4172 {
4173 	pgcnt_t	ulckpgs = 0, pg_idx;
4174 
4175 	VM_STAT_ADD(pagecnt.pc_subclaim_pages);
4176 
4177 	/*
4178 	 * Only need to take the page struct lock on the large page root.
4179 	 */
4180 	page_struct_lock(ppa[0]);
4181 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4182 
4183 		ASSERT(PAGE_LOCKED(ppa[pg_idx]));
4184 		ASSERT(ppa[pg_idx]->p_cowcnt != 0);
4185 		if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4186 			page_struct_unlock(ppa[0]);
4187 			return (0);
4188 		}
4189 		if (ppa[pg_idx]->p_lckcnt != 0)
4190 			ulckpgs++;
4191 	}
4192 
4193 	if (ulckpgs != 0) {
4194 		mutex_enter(&freemem_lock);
4195 		availrmem += ulckpgs;
4196 		pages_claimed -= ulckpgs;
4197 		mutex_exit(&freemem_lock);
4198 	}
4199 
4200 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4201 		ppa[pg_idx]->p_cowcnt--;
4202 		ppa[pg_idx]->p_lckcnt++;
4203 
4204 	}
4205 	page_struct_unlock(ppa[0]);
4206 	return (1);
4207 }
4208 
4209 page_t *
4210 page_numtopp(pfn_t pfnum, se_t se)
4211 {
4212 	page_t *pp;
4213 
4214 retry:
4215 	pp = page_numtopp_nolock(pfnum);
4216 	if (pp == NULL) {
4217 		return ((page_t *)NULL);
4218 	}
4219 
4220 	/*
4221 	 * Acquire the appropriate lock on the page.
4222 	 */
4223 	while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) {
4224 		if (page_pptonum(pp) != pfnum)
4225 			goto retry;
4226 		continue;
4227 	}
4228 
4229 	if (page_pptonum(pp) != pfnum) {
4230 		page_unlock(pp);
4231 		goto retry;
4232 	}
4233 
4234 	return (pp);
4235 }
4236 
4237 page_t *
4238 page_numtopp_noreclaim(pfn_t pfnum, se_t se)
4239 {
4240 	page_t *pp;
4241 
4242 retry:
4243 	pp = page_numtopp_nolock(pfnum);
4244 	if (pp == NULL) {
4245 		return ((page_t *)NULL);
4246 	}
4247 
4248 	/*
4249 	 * Acquire the appropriate lock on the page.
4250 	 */
4251 	while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) {
4252 		if (page_pptonum(pp) != pfnum)
4253 			goto retry;
4254 		continue;
4255 	}
4256 
4257 	if (page_pptonum(pp) != pfnum) {
4258 		page_unlock(pp);
4259 		goto retry;
4260 	}
4261 
4262 	return (pp);
4263 }
4264 
4265 /*
4266  * This routine is like page_numtopp, but will only return page structs
4267  * for pages which are ok for loading into hardware using the page struct.
4268  */
4269 page_t *
4270 page_numtopp_nowait(pfn_t pfnum, se_t se)
4271 {
4272 	page_t *pp;
4273 
4274 retry:
4275 	pp = page_numtopp_nolock(pfnum);
4276 	if (pp == NULL) {
4277 		return ((page_t *)NULL);
4278 	}
4279 
4280 	/*
4281 	 * Try to acquire the appropriate lock on the page.
4282 	 */
4283 	if (PP_ISFREE(pp))
4284 		pp = NULL;
4285 	else {
4286 		if (!page_trylock(pp, se))
4287 			pp = NULL;
4288 		else {
4289 			if (page_pptonum(pp) != pfnum) {
4290 				page_unlock(pp);
4291 				goto retry;
4292 			}
4293 			if (PP_ISFREE(pp)) {
4294 				page_unlock(pp);
4295 				pp = NULL;
4296 			}
4297 		}
4298 	}
4299 	return (pp);
4300 }
4301 
4302 /*
4303  * Returns a count of dirty pages that are in the process
4304  * of being written out.  If 'cleanit' is set, try to push the page.
4305  */
4306 pgcnt_t
4307 page_busy(int cleanit)
4308 {
4309 	page_t *page0 = page_first();
4310 	page_t *pp = page0;
4311 	pgcnt_t nppbusy = 0;
4312 	u_offset_t off;
4313 
4314 	do {
4315 		vnode_t *vp = pp->p_vnode;
4316 		/*
4317 		 * A page is a candidate for syncing if it is:
4318 		 *
4319 		 * (a)	On neither the freelist nor the cachelist
4320 		 * (b)	Hashed onto a vnode
4321 		 * (c)	Not a kernel page
4322 		 * (d)	Dirty
4323 		 * (e)	Not part of a swapfile
4324 		 * (f)	a page which belongs to a real vnode; eg has a non-null
4325 		 *	v_vfsp pointer.
4326 		 * (g)	Backed by a filesystem which doesn't have a
4327 		 *	stubbed-out sync operation
4328 		 */
4329 		if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) &&
4330 		    hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL &&
4331 		    vfs_can_sync(vp->v_vfsp)) {
4332 			nppbusy++;
4333 
4334 			if (!cleanit)
4335 				continue;
4336 			if (!page_trylock(pp, SE_EXCL))
4337 				continue;
4338 
4339 			if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) ||
4340 			    pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
4341 			    !(hat_pagesync(pp,
4342 			    HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) {
4343 				page_unlock(pp);
4344 				continue;
4345 			}
4346 			off = pp->p_offset;
4347 			VN_HOLD(vp);
4348 			page_unlock(pp);
4349 			(void) VOP_PUTPAGE(vp, off, PAGESIZE,
4350 			    B_ASYNC | B_FREE, kcred, NULL);
4351 			VN_RELE(vp);
4352 		}
4353 	} while ((pp = page_next(pp)) != page0);
4354 
4355 	return (nppbusy);
4356 }
4357 
4358 void page_invalidate_pages(void);
4359 
4360 /*
4361  * callback handler to vm sub-system
4362  *
4363  * callers make sure no recursive entries to this func.
4364  */
4365 /*ARGSUSED*/
4366 boolean_t
4367 callb_vm_cpr(void *arg, int code)
4368 {
4369 	if (code == CB_CODE_CPR_CHKPT)
4370 		page_invalidate_pages();
4371 	return (B_TRUE);
4372 }
4373 
4374 /*
4375  * Invalidate all pages of the system.
4376  * It shouldn't be called until all user page activities are all stopped.
4377  */
4378 void
4379 page_invalidate_pages()
4380 {
4381 	page_t *pp;
4382 	page_t *page0;
4383 	pgcnt_t nbusypages;
4384 	int retry = 0;
4385 	const int MAXRETRIES = 4;
4386 top:
4387 	/*
4388 	 * Flush dirty pages and destroy the clean ones.
4389 	 */
4390 	nbusypages = 0;
4391 
4392 	pp = page0 = page_first();
4393 	do {
4394 		struct vnode	*vp;
4395 		u_offset_t	offset;
4396 		int		mod;
4397 
4398 		/*
4399 		 * skip the page if it has no vnode or the page associated
4400 		 * with the kernel vnode or prom allocated kernel mem.
4401 		 */
4402 		if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp))
4403 			continue;
4404 
4405 		/*
4406 		 * skip the page which is already free invalidated.
4407 		 */
4408 		if (PP_ISFREE(pp) && PP_ISAGED(pp))
4409 			continue;
4410 
4411 		/*
4412 		 * skip pages that are already locked or can't be "exclusively"
4413 		 * locked or are already free.  After we lock the page, check
4414 		 * the free and age bits again to be sure it's not destroyed
4415 		 * yet.
4416 		 * To achieve max. parallelization, we use page_trylock instead
4417 		 * of page_lock so that we don't get block on individual pages
4418 		 * while we have thousands of other pages to process.
4419 		 */
4420 		if (!page_trylock(pp, SE_EXCL)) {
4421 			nbusypages++;
4422 			continue;
4423 		} else if (PP_ISFREE(pp)) {
4424 			if (!PP_ISAGED(pp)) {
4425 				page_destroy_free(pp);
4426 			} else {
4427 				page_unlock(pp);
4428 			}
4429 			continue;
4430 		}
4431 		/*
4432 		 * Is this page involved in some I/O? shared?
4433 		 *
4434 		 * The page_struct_lock need not be acquired to
4435 		 * examine these fields since the page has an
4436 		 * "exclusive" lock.
4437 		 */
4438 		if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
4439 			page_unlock(pp);
4440 			continue;
4441 		}
4442 
4443 		if (vp->v_type == VCHR) {
4444 			panic("vp->v_type == VCHR");
4445 			/*NOTREACHED*/
4446 		}
4447 
4448 		if (!page_try_demote_pages(pp)) {
4449 			page_unlock(pp);
4450 			continue;
4451 		}
4452 
4453 		/*
4454 		 * Check the modified bit. Leave the bits alone in hardware
4455 		 * (they will be modified if we do the putpage).
4456 		 */
4457 		mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD)
4458 		    & P_MOD);
4459 		if (mod) {
4460 			offset = pp->p_offset;
4461 			/*
4462 			 * Hold the vnode before releasing the page lock
4463 			 * to prevent it from being freed and re-used by
4464 			 * some other thread.
4465 			 */
4466 			VN_HOLD(vp);
4467 			page_unlock(pp);
4468 			/*
4469 			 * No error return is checked here. Callers such as
4470 			 * cpr deals with the dirty pages at the dump time
4471 			 * if this putpage fails.
4472 			 */
4473 			(void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL,
4474 			    kcred, NULL);
4475 			VN_RELE(vp);
4476 		} else {
4477 			/*LINTED: constant in conditional context*/
4478 			VN_DISPOSE(pp, B_INVAL, 0, kcred);
4479 		}
4480 	} while ((pp = page_next(pp)) != page0);
4481 	if (nbusypages && retry++ < MAXRETRIES) {
4482 		delay(1);
4483 		goto top;
4484 	}
4485 }
4486 
4487 /*
4488  * Replace the page "old" with the page "new" on the page hash and vnode lists
4489  *
4490  * the replacement must be done in place, ie the equivalent sequence:
4491  *
4492  *	vp = old->p_vnode;
4493  *	off = old->p_offset;
4494  *	page_do_hashout(old)
4495  *	page_do_hashin(new, vp, off)
4496  *
4497  * doesn't work, since
4498  *  1) if old is the only page on the vnode, the v_pages list has a window
4499  *     where it looks empty. This will break file system assumptions.
4500  * and
4501  *  2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list.
4502  */
4503 static void
4504 page_do_relocate_hash(page_t *new, page_t *old)
4505 {
4506 	page_t	**hash_list;
4507 	vnode_t	*vp = old->p_vnode;
4508 	kmutex_t *sep;
4509 
4510 	ASSERT(PAGE_EXCL(old));
4511 	ASSERT(PAGE_EXCL(new));
4512 	ASSERT(vp != NULL);
4513 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
4514 	ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset))));
4515 
4516 	/*
4517 	 * First find old page on the page hash list
4518 	 */
4519 	hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)];
4520 
4521 	for (;;) {
4522 		if (*hash_list == old)
4523 			break;
4524 		if (*hash_list == NULL) {
4525 			panic("page_do_hashout");
4526 			/*NOTREACHED*/
4527 		}
4528 		hash_list = &(*hash_list)->p_hash;
4529 	}
4530 
4531 	/*
4532 	 * update new and replace old with new on the page hash list
4533 	 */
4534 	new->p_vnode = old->p_vnode;
4535 	new->p_offset = old->p_offset;
4536 	new->p_hash = old->p_hash;
4537 	*hash_list = new;
4538 
4539 	if ((new->p_vnode->v_flag & VISSWAP) != 0)
4540 		PP_SETSWAP(new);
4541 
4542 	/*
4543 	 * replace old with new on the vnode's page list
4544 	 */
4545 	if (old->p_vpnext == old) {
4546 		new->p_vpnext = new;
4547 		new->p_vpprev = new;
4548 	} else {
4549 		new->p_vpnext = old->p_vpnext;
4550 		new->p_vpprev = old->p_vpprev;
4551 		new->p_vpnext->p_vpprev = new;
4552 		new->p_vpprev->p_vpnext = new;
4553 	}
4554 	if (vp->v_pages == old)
4555 		vp->v_pages = new;
4556 
4557 	/*
4558 	 * clear out the old page
4559 	 */
4560 	old->p_hash = NULL;
4561 	old->p_vpnext = NULL;
4562 	old->p_vpprev = NULL;
4563 	old->p_vnode = NULL;
4564 	PP_CLRSWAP(old);
4565 	old->p_offset = (u_offset_t)-1;
4566 	page_clr_all_props(old);
4567 
4568 	/*
4569 	 * Wake up processes waiting for this page.  The page's
4570 	 * identity has been changed, and is probably not the
4571 	 * desired page any longer.
4572 	 */
4573 	sep = page_se_mutex(old);
4574 	mutex_enter(sep);
4575 	old->p_selock &= ~SE_EWANTED;
4576 	if (CV_HAS_WAITERS(&old->p_cv))
4577 		cv_broadcast(&old->p_cv);
4578 	mutex_exit(sep);
4579 }
4580 
4581 /*
4582  * This function moves the identity of page "pp_old" to page "pp_new".
4583  * Both pages must be locked on entry.  "pp_new" is free, has no identity,
4584  * and need not be hashed out from anywhere.
4585  */
4586 void
4587 page_relocate_hash(page_t *pp_new, page_t *pp_old)
4588 {
4589 	vnode_t *vp = pp_old->p_vnode;
4590 	u_offset_t off = pp_old->p_offset;
4591 	kmutex_t *phm, *vphm;
4592 
4593 	/*
4594 	 * Rehash two pages
4595 	 */
4596 	ASSERT(PAGE_EXCL(pp_old));
4597 	ASSERT(PAGE_EXCL(pp_new));
4598 	ASSERT(vp != NULL);
4599 	ASSERT(pp_new->p_vnode == NULL);
4600 
4601 	/*
4602 	 * hashout then hashin while holding the mutexes
4603 	 */
4604 	phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off));
4605 	mutex_enter(phm);
4606 	vphm = page_vnode_mutex(vp);
4607 	mutex_enter(vphm);
4608 
4609 	page_do_relocate_hash(pp_new, pp_old);
4610 
4611 	/* The following comment preserved from page_flip(). */
4612 	pp_new->p_fsdata = pp_old->p_fsdata;
4613 	pp_old->p_fsdata = 0;
4614 	mutex_exit(vphm);
4615 	mutex_exit(phm);
4616 
4617 	/*
4618 	 * The page_struct_lock need not be acquired for lckcnt and
4619 	 * cowcnt since the page has an "exclusive" lock.
4620 	 */
4621 	ASSERT(pp_new->p_lckcnt == 0);
4622 	ASSERT(pp_new->p_cowcnt == 0);
4623 	pp_new->p_lckcnt = pp_old->p_lckcnt;
4624 	pp_new->p_cowcnt = pp_old->p_cowcnt;
4625 	pp_old->p_lckcnt = pp_old->p_cowcnt = 0;
4626 
4627 }
4628 
4629 /*
4630  * Helper routine used to lock all remaining members of a
4631  * large page. The caller is responsible for passing in a locked
4632  * pp. If pp is a large page, then it succeeds in locking all the
4633  * remaining constituent pages or it returns with only the
4634  * original page locked.
4635  *
4636  * Returns 1 on success, 0 on failure.
4637  *
4638  * If success is returned this routine guarantees p_szc for all constituent
4639  * pages of a large page pp belongs to can't change. To achieve this we
4640  * recheck szc of pp after locking all constituent pages and retry if szc
4641  * changed (it could only decrease). Since hat_page_demote() needs an EXCL
4642  * lock on one of constituent pages it can't be running after all constituent
4643  * pages are locked.  hat_page_demote() with a lock on a constituent page
4644  * outside of this large page (i.e. pp belonged to a larger large page) is
4645  * already done with all constituent pages of pp since the root's p_szc is
4646  * changed last. Therefore no need to synchronize with hat_page_demote() that
4647  * locked a constituent page outside of pp's current large page.
4648  */
4649 #ifdef DEBUG
4650 uint32_t gpg_trylock_mtbf = 0;
4651 #endif
4652 
4653 int
4654 group_page_trylock(page_t *pp, se_t se)
4655 {
4656 	page_t  *tpp;
4657 	pgcnt_t	npgs, i, j;
4658 	uint_t pszc = pp->p_szc;
4659 
4660 #ifdef DEBUG
4661 	if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) {
4662 		return (0);
4663 	}
4664 #endif
4665 
4666 	if (pp != PP_GROUPLEADER(pp, pszc)) {
4667 		return (0);
4668 	}
4669 
4670 retry:
4671 	ASSERT(PAGE_LOCKED_SE(pp, se));
4672 	ASSERT(!PP_ISFREE(pp));
4673 	if (pszc == 0) {
4674 		return (1);
4675 	}
4676 	npgs = page_get_pagecnt(pszc);
4677 	tpp = pp + 1;
4678 	for (i = 1; i < npgs; i++, tpp++) {
4679 		if (!page_trylock(tpp, se)) {
4680 			tpp = pp + 1;
4681 			for (j = 1; j < i; j++, tpp++) {
4682 				page_unlock(tpp);
4683 			}
4684 			return (0);
4685 		}
4686 	}
4687 	if (pp->p_szc != pszc) {
4688 		ASSERT(pp->p_szc < pszc);
4689 		ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) &&
4690 		    !IS_SWAPFSVP(pp->p_vnode));
4691 		tpp = pp + 1;
4692 		for (i = 1; i < npgs; i++, tpp++) {
4693 			page_unlock(tpp);
4694 		}
4695 		pszc = pp->p_szc;
4696 		goto retry;
4697 	}
4698 	return (1);
4699 }
4700 
4701 void
4702 group_page_unlock(page_t *pp)
4703 {
4704 	page_t *tpp;
4705 	pgcnt_t	npgs, i;
4706 
4707 	ASSERT(PAGE_LOCKED(pp));
4708 	ASSERT(!PP_ISFREE(pp));
4709 	ASSERT(pp == PP_PAGEROOT(pp));
4710 	npgs = page_get_pagecnt(pp->p_szc);
4711 	for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) {
4712 		page_unlock(tpp);
4713 	}
4714 }
4715 
4716 /*
4717  * returns
4718  * 0 		: on success and *nrelocp is number of relocated PAGESIZE pages
4719  * ERANGE	: this is not a base page
4720  * EBUSY	: failure to get locks on the page/pages
4721  * ENOMEM	: failure to obtain replacement pages
4722  * EAGAIN	: OBP has not yet completed its boot-time handoff to the kernel
4723  * EIO		: An error occurred while trying to copy the page data
4724  *
4725  * Return with all constituent members of target and replacement
4726  * SE_EXCL locked. It is the callers responsibility to drop the
4727  * locks.
4728  */
4729 int
4730 do_page_relocate(
4731 	page_t **target,
4732 	page_t **replacement,
4733 	int grouplock,
4734 	spgcnt_t *nrelocp,
4735 	lgrp_t *lgrp)
4736 {
4737 	page_t *first_repl;
4738 	page_t *repl;
4739 	page_t *targ;
4740 	page_t *pl = NULL;
4741 	uint_t ppattr;
4742 	pfn_t   pfn, repl_pfn;
4743 	uint_t	szc;
4744 	spgcnt_t npgs, i;
4745 	int repl_contig = 0;
4746 	uint_t flags = 0;
4747 	spgcnt_t dofree = 0;
4748 
4749 	*nrelocp = 0;
4750 
4751 #if defined(__sparc)
4752 	/*
4753 	 * We need to wait till OBP has completed
4754 	 * its boot-time handoff of its resources to the kernel
4755 	 * before we allow page relocation
4756 	 */
4757 	if (page_relocate_ready == 0) {
4758 		return (EAGAIN);
4759 	}
4760 #endif
4761 
4762 	/*
4763 	 * If this is not a base page,
4764 	 * just return with 0x0 pages relocated.
4765 	 */
4766 	targ = *target;
4767 	ASSERT(PAGE_EXCL(targ));
4768 	ASSERT(!PP_ISFREE(targ));
4769 	szc = targ->p_szc;
4770 	ASSERT(szc < mmu_page_sizes);
4771 	VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
4772 	pfn = targ->p_pagenum;
4773 	if (pfn != PFN_BASE(pfn, szc)) {
4774 		VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]);
4775 		return (ERANGE);
4776 	}
4777 
4778 	if ((repl = *replacement) != NULL && repl->p_szc >= szc) {
4779 		repl_pfn = repl->p_pagenum;
4780 		if (repl_pfn != PFN_BASE(repl_pfn, szc)) {
4781 			VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]);
4782 			return (ERANGE);
4783 		}
4784 		repl_contig = 1;
4785 	}
4786 
4787 	/*
4788 	 * We must lock all members of this large page or we cannot
4789 	 * relocate any part of it.
4790 	 */
4791 	if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) {
4792 		VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]);
4793 		return (EBUSY);
4794 	}
4795 
4796 	/*
4797 	 * reread szc it could have been decreased before
4798 	 * group_page_trylock() was done.
4799 	 */
4800 	szc = targ->p_szc;
4801 	ASSERT(szc < mmu_page_sizes);
4802 	VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
4803 	ASSERT(pfn == PFN_BASE(pfn, szc));
4804 
4805 	npgs = page_get_pagecnt(targ->p_szc);
4806 
4807 	if (repl == NULL) {
4808 		dofree = npgs;		/* Size of target page in MMU pages */
4809 		if (!page_create_wait(dofree, 0)) {
4810 			if (grouplock != 0) {
4811 				group_page_unlock(targ);
4812 			}
4813 			VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
4814 			return (ENOMEM);
4815 		}
4816 
4817 		/*
4818 		 * seg kmem pages require that the target and replacement
4819 		 * page be the same pagesize.
4820 		 */
4821 		flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0;
4822 		repl = page_get_replacement_page(targ, lgrp, flags);
4823 		if (repl == NULL) {
4824 			if (grouplock != 0) {
4825 				group_page_unlock(targ);
4826 			}
4827 			page_create_putback(dofree);
4828 			VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
4829 			return (ENOMEM);
4830 		}
4831 	}
4832 #ifdef DEBUG
4833 	else {
4834 		ASSERT(PAGE_LOCKED(repl));
4835 	}
4836 #endif /* DEBUG */
4837 
4838 #if defined(__sparc)
4839 	/*
4840 	 * Let hat_page_relocate() complete the relocation if it's kernel page
4841 	 */
4842 	if (VN_ISKAS(targ->p_vnode)) {
4843 		*replacement = repl;
4844 		if (hat_page_relocate(target, replacement, nrelocp) != 0) {
4845 			if (grouplock != 0) {
4846 				group_page_unlock(targ);
4847 			}
4848 			if (dofree) {
4849 				*replacement = NULL;
4850 				page_free_replacement_page(repl);
4851 				page_create_putback(dofree);
4852 			}
4853 			VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]);
4854 			return (EAGAIN);
4855 		}
4856 		VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
4857 		return (0);
4858 	}
4859 #else
4860 #if defined(lint)
4861 	dofree = dofree;
4862 #endif
4863 #endif
4864 
4865 	first_repl = repl;
4866 
4867 	for (i = 0; i < npgs; i++) {
4868 		ASSERT(PAGE_EXCL(targ));
4869 		ASSERT(targ->p_slckcnt == 0);
4870 		ASSERT(repl->p_slckcnt == 0);
4871 
4872 		(void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD);
4873 
4874 		ASSERT(hat_page_getshare(targ) == 0);
4875 		ASSERT(!PP_ISFREE(targ));
4876 		ASSERT(targ->p_pagenum == (pfn + i));
4877 		ASSERT(repl_contig == 0 ||
4878 		    repl->p_pagenum == (repl_pfn + i));
4879 
4880 		/*
4881 		 * Copy the page contents and attributes then
4882 		 * relocate the page in the page hash.
4883 		 */
4884 		if (ppcopy(targ, repl) == 0) {
4885 			targ = *target;
4886 			repl = first_repl;
4887 			VM_STAT_ADD(vmm_vmstats.ppr_copyfail);
4888 			if (grouplock != 0) {
4889 				group_page_unlock(targ);
4890 			}
4891 			if (dofree) {
4892 				*replacement = NULL;
4893 				page_free_replacement_page(repl);
4894 				page_create_putback(dofree);
4895 			}
4896 			return (EIO);
4897 		}
4898 
4899 		targ++;
4900 		if (repl_contig != 0) {
4901 			repl++;
4902 		} else {
4903 			repl = repl->p_next;
4904 		}
4905 	}
4906 
4907 	repl = first_repl;
4908 	targ = *target;
4909 
4910 	for (i = 0; i < npgs; i++) {
4911 		ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO));
4912 		page_clr_all_props(repl);
4913 		page_set_props(repl, ppattr);
4914 		page_relocate_hash(repl, targ);
4915 
4916 		ASSERT(hat_page_getshare(targ) == 0);
4917 		ASSERT(hat_page_getshare(repl) == 0);
4918 		/*
4919 		 * Now clear the props on targ, after the
4920 		 * page_relocate_hash(), they no longer
4921 		 * have any meaning.
4922 		 */
4923 		page_clr_all_props(targ);
4924 		ASSERT(targ->p_next == targ);
4925 		ASSERT(targ->p_prev == targ);
4926 		page_list_concat(&pl, &targ);
4927 
4928 		targ++;
4929 		if (repl_contig != 0) {
4930 			repl++;
4931 		} else {
4932 			repl = repl->p_next;
4933 		}
4934 	}
4935 	/* assert that we have come full circle with repl */
4936 	ASSERT(repl_contig == 1 || first_repl == repl);
4937 
4938 	*target = pl;
4939 	if (*replacement == NULL) {
4940 		ASSERT(first_repl == repl);
4941 		*replacement = repl;
4942 	}
4943 	VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
4944 	*nrelocp = npgs;
4945 	return (0);
4946 }
4947 /*
4948  * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated.
4949  */
4950 int
4951 page_relocate(
4952 	page_t **target,
4953 	page_t **replacement,
4954 	int grouplock,
4955 	int freetarget,
4956 	spgcnt_t *nrelocp,
4957 	lgrp_t *lgrp)
4958 {
4959 	spgcnt_t ret;
4960 
4961 	/* do_page_relocate returns 0 on success or errno value */
4962 	ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp);
4963 
4964 	if (ret != 0 || freetarget == 0) {
4965 		return (ret);
4966 	}
4967 	if (*nrelocp == 1) {
4968 		ASSERT(*target != NULL);
4969 		page_free(*target, 1);
4970 	} else {
4971 		page_t *tpp = *target;
4972 		uint_t szc = tpp->p_szc;
4973 		pgcnt_t npgs = page_get_pagecnt(szc);
4974 		ASSERT(npgs > 1);
4975 		ASSERT(szc != 0);
4976 		do {
4977 			ASSERT(PAGE_EXCL(tpp));
4978 			ASSERT(!hat_page_is_mapped(tpp));
4979 			ASSERT(tpp->p_szc == szc);
4980 			PP_SETFREE(tpp);
4981 			PP_SETAGED(tpp);
4982 			npgs--;
4983 		} while ((tpp = tpp->p_next) != *target);
4984 		ASSERT(npgs == 0);
4985 		page_list_add_pages(*target, 0);
4986 		npgs = page_get_pagecnt(szc);
4987 		page_create_putback(npgs);
4988 	}
4989 	return (ret);
4990 }
4991 
4992 /*
4993  * it is up to the caller to deal with pcf accounting.
4994  */
4995 void
4996 page_free_replacement_page(page_t *pplist)
4997 {
4998 	page_t *pp;
4999 
5000 	while (pplist != NULL) {
5001 		/*
5002 		 * pp_targ is a linked list.
5003 		 */
5004 		pp = pplist;
5005 		if (pp->p_szc == 0) {
5006 			page_sub(&pplist, pp);
5007 			page_clr_all_props(pp);
5008 			PP_SETFREE(pp);
5009 			PP_SETAGED(pp);
5010 			page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
5011 			page_unlock(pp);
5012 			VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]);
5013 		} else {
5014 			spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc);
5015 			page_t *tpp;
5016 			page_list_break(&pp, &pplist, curnpgs);
5017 			tpp = pp;
5018 			do {
5019 				ASSERT(PAGE_EXCL(tpp));
5020 				ASSERT(!hat_page_is_mapped(tpp));
5021 				page_clr_all_props(tpp);
5022 				PP_SETFREE(tpp);
5023 				PP_SETAGED(tpp);
5024 			} while ((tpp = tpp->p_next) != pp);
5025 			page_list_add_pages(pp, 0);
5026 			VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]);
5027 		}
5028 	}
5029 }
5030 
5031 /*
5032  * Relocate target to non-relocatable replacement page.
5033  */
5034 int
5035 page_relocate_cage(page_t **target, page_t **replacement)
5036 {
5037 	page_t *tpp, *rpp;
5038 	spgcnt_t pgcnt, npgs;
5039 	int result;
5040 
5041 	tpp = *target;
5042 
5043 	ASSERT(PAGE_EXCL(tpp));
5044 	ASSERT(tpp->p_szc == 0);
5045 
5046 	pgcnt = btop(page_get_pagesize(tpp->p_szc));
5047 
5048 	do {
5049 		(void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC);
5050 		rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC);
5051 		if (rpp == NULL) {
5052 			page_create_putback(pgcnt);
5053 			kcage_cageout_wakeup();
5054 		}
5055 	} while (rpp == NULL);
5056 
5057 	ASSERT(PP_ISNORELOC(rpp));
5058 
5059 	result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL);
5060 
5061 	if (result == 0) {
5062 		*replacement = rpp;
5063 		if (pgcnt != npgs)
5064 			panic("page_relocate_cage: partial relocation");
5065 	}
5066 
5067 	return (result);
5068 }
5069 
5070 /*
5071  * Release the page lock on a page, place on cachelist
5072  * tail if no longer mapped. Caller can let us know if
5073  * the page is known to be clean.
5074  */
5075 int
5076 page_release(page_t *pp, int checkmod)
5077 {
5078 	int status;
5079 
5080 	ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) &&
5081 	    (pp->p_vnode != NULL));
5082 
5083 	if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) &&
5084 	    ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) &&
5085 	    pp->p_lckcnt == 0 && pp->p_cowcnt == 0 &&
5086 	    !hat_page_is_mapped(pp)) {
5087 
5088 		/*
5089 		 * If page is modified, unlock it
5090 		 *
5091 		 * (p_nrm & P_MOD) bit has the latest stuff because:
5092 		 * (1) We found that this page doesn't have any mappings
5093 		 *	_after_ holding SE_EXCL and
5094 		 * (2) We didn't drop SE_EXCL lock after the check in (1)
5095 		 */
5096 		if (checkmod && hat_ismod(pp)) {
5097 			page_unlock(pp);
5098 			status = PGREL_MOD;
5099 		} else {
5100 			/*LINTED: constant in conditional context*/
5101 			VN_DISPOSE(pp, B_FREE, 0, kcred);
5102 			status = PGREL_CLEAN;
5103 		}
5104 	} else {
5105 		page_unlock(pp);
5106 		status = PGREL_NOTREL;
5107 	}
5108 	return (status);
5109 }
5110 
5111 /*
5112  * Given a constituent page, try to demote the large page on the freelist.
5113  *
5114  * Returns nonzero if the page could be demoted successfully. Returns with
5115  * the constituent page still locked.
5116  */
5117 int
5118 page_try_demote_free_pages(page_t *pp)
5119 {
5120 	page_t *rootpp = pp;
5121 	pfn_t	pfn = page_pptonum(pp);
5122 	spgcnt_t npgs;
5123 	uint_t	szc = pp->p_szc;
5124 
5125 	ASSERT(PP_ISFREE(pp));
5126 	ASSERT(PAGE_EXCL(pp));
5127 
5128 	/*
5129 	 * Adjust rootpp and lock it, if `pp' is not the base
5130 	 * constituent page.
5131 	 */
5132 	npgs = page_get_pagecnt(pp->p_szc);
5133 	if (npgs == 1) {
5134 		return (0);
5135 	}
5136 
5137 	if (!IS_P2ALIGNED(pfn, npgs)) {
5138 		pfn = P2ALIGN(pfn, npgs);
5139 		rootpp = page_numtopp_nolock(pfn);
5140 	}
5141 
5142 	if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) {
5143 		return (0);
5144 	}
5145 
5146 	if (rootpp->p_szc != szc) {
5147 		if (pp != rootpp)
5148 			page_unlock(rootpp);
5149 		return (0);
5150 	}
5151 
5152 	page_demote_free_pages(rootpp);
5153 
5154 	if (pp != rootpp)
5155 		page_unlock(rootpp);
5156 
5157 	ASSERT(PP_ISFREE(pp));
5158 	ASSERT(PAGE_EXCL(pp));
5159 	return (1);
5160 }
5161 
5162 /*
5163  * Given a constituent page, try to demote the large page.
5164  *
5165  * Returns nonzero if the page could be demoted successfully. Returns with
5166  * the constituent page still locked.
5167  */
5168 int
5169 page_try_demote_pages(page_t *pp)
5170 {
5171 	page_t *tpp, *rootpp = pp;
5172 	pfn_t	pfn = page_pptonum(pp);
5173 	spgcnt_t i, npgs;
5174 	uint_t	szc = pp->p_szc;
5175 	vnode_t *vp = pp->p_vnode;
5176 
5177 	ASSERT(PAGE_EXCL(pp));
5178 
5179 	VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]);
5180 
5181 	if (pp->p_szc == 0) {
5182 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]);
5183 		return (1);
5184 	}
5185 
5186 	if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) {
5187 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]);
5188 		page_demote_vp_pages(pp);
5189 		ASSERT(pp->p_szc == 0);
5190 		return (1);
5191 	}
5192 
5193 	/*
5194 	 * Adjust rootpp if passed in is not the base
5195 	 * constituent page.
5196 	 */
5197 	npgs = page_get_pagecnt(pp->p_szc);
5198 	ASSERT(npgs > 1);
5199 	if (!IS_P2ALIGNED(pfn, npgs)) {
5200 		pfn = P2ALIGN(pfn, npgs);
5201 		rootpp = page_numtopp_nolock(pfn);
5202 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]);
5203 		ASSERT(rootpp->p_vnode != NULL);
5204 		ASSERT(rootpp->p_szc == szc);
5205 	}
5206 
5207 	/*
5208 	 * We can't demote kernel pages since we can't hat_unload()
5209 	 * the mappings.
5210 	 */
5211 	if (VN_ISKAS(rootpp->p_vnode))
5212 		return (0);
5213 
5214 	/*
5215 	 * Attempt to lock all constituent pages except the page passed
5216 	 * in since it's already locked.
5217 	 */
5218 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5219 		ASSERT(!PP_ISFREE(tpp));
5220 		ASSERT(tpp->p_vnode != NULL);
5221 
5222 		if (tpp != pp && !page_trylock(tpp, SE_EXCL))
5223 			break;
5224 		ASSERT(tpp->p_szc == rootpp->p_szc);
5225 		ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i);
5226 	}
5227 
5228 	/*
5229 	 * If we failed to lock them all then unlock what we have
5230 	 * locked so far and bail.
5231 	 */
5232 	if (i < npgs) {
5233 		tpp = rootpp;
5234 		while (i-- > 0) {
5235 			if (tpp != pp)
5236 				page_unlock(tpp);
5237 			tpp++;
5238 		}
5239 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]);
5240 		return (0);
5241 	}
5242 
5243 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5244 		ASSERT(PAGE_EXCL(tpp));
5245 		ASSERT(tpp->p_slckcnt == 0);
5246 		(void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
5247 		tpp->p_szc = 0;
5248 	}
5249 
5250 	/*
5251 	 * Unlock all pages except the page passed in.
5252 	 */
5253 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5254 		ASSERT(!hat_page_is_mapped(tpp));
5255 		if (tpp != pp)
5256 			page_unlock(tpp);
5257 	}
5258 
5259 	VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]);
5260 	return (1);
5261 }
5262 
5263 /*
5264  * Called by page_free() and page_destroy() to demote the page size code
5265  * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero
5266  * p_szc on free list, neither can we just clear p_szc of a single page_t
5267  * within a large page since it will break other code that relies on p_szc
5268  * being the same for all page_t's of a large page). Anonymous pages should
5269  * never end up here because anon_map_getpages() cannot deal with p_szc
5270  * changes after a single constituent page is locked.  While anonymous or
5271  * kernel large pages are demoted or freed the entire large page at a time
5272  * with all constituent pages locked EXCL for the file system pages we
5273  * have to be able to demote a large page (i.e. decrease all constituent pages
5274  * p_szc) with only just an EXCL lock on one of constituent pages. The reason
5275  * we can easily deal with anonymous page demotion the entire large page at a
5276  * time is that those operation originate at address space level and concern
5277  * the entire large page region with actual demotion only done when pages are
5278  * not shared with any other processes (therefore we can always get EXCL lock
5279  * on all anonymous constituent pages after clearing segment page
5280  * cache). However file system pages can be truncated or invalidated at a
5281  * PAGESIZE level from the file system side and end up in page_free() or
5282  * page_destroy() (we also allow only part of the large page to be SOFTLOCKed
5283  * and therefore pageout should be able to demote a large page by EXCL locking
5284  * any constituent page that is not under SOFTLOCK). In those cases we cannot
5285  * rely on being able to lock EXCL all constituent pages.
5286  *
5287  * To prevent szc changes on file system pages one has to lock all constituent
5288  * pages at least SHARED (or call page_szc_lock()). The only subsystem that
5289  * doesn't rely on locking all constituent pages (or using page_szc_lock()) to
5290  * prevent szc changes is hat layer that uses its own page level mlist
5291  * locks. hat assumes that szc doesn't change after mlist lock for a page is
5292  * taken. Therefore we need to change szc under hat level locks if we only
5293  * have an EXCL lock on a single constituent page and hat still references any
5294  * of constituent pages.  (Note we can't "ignore" hat layer by simply
5295  * hat_pageunload() all constituent pages without having EXCL locks on all of
5296  * constituent pages). We use hat_page_demote() call to safely demote szc of
5297  * all constituent pages under hat locks when we only have an EXCL lock on one
5298  * of constituent pages.
5299  *
5300  * This routine calls page_szc_lock() before calling hat_page_demote() to
5301  * allow segvn in one special case not to lock all constituent pages SHARED
5302  * before calling hat_memload_array() that relies on p_szc not changing even
5303  * before hat level mlist lock is taken.  In that case segvn uses
5304  * page_szc_lock() to prevent hat_page_demote() changing p_szc values.
5305  *
5306  * Anonymous or kernel page demotion still has to lock all pages exclusively
5307  * and do hat_pageunload() on all constituent pages before demoting the page
5308  * therefore there's no need for anonymous or kernel page demotion to use
5309  * hat_page_demote() mechanism.
5310  *
5311  * hat_page_demote() removes all large mappings that map pp and then decreases
5312  * p_szc starting from the last constituent page of the large page. By working
5313  * from the tail of a large page in pfn decreasing order allows one looking at
5314  * the root page to know that hat_page_demote() is done for root's szc area.
5315  * e.g. if a root page has szc 1 one knows it only has to lock all constituent
5316  * pages within szc 1 area to prevent szc changes because hat_page_demote()
5317  * that started on this page when it had szc > 1 is done for this szc 1 area.
5318  *
5319  * We are guaranteed that all constituent pages of pp's large page belong to
5320  * the same vnode with the consecutive offsets increasing in the direction of
5321  * the pfn i.e. the identity of constituent pages can't change until their
5322  * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove
5323  * large mappings to pp even though we don't lock any constituent page except
5324  * pp (i.e. we won't unload e.g. kernel locked page).
5325  */
5326 static void
5327 page_demote_vp_pages(page_t *pp)
5328 {
5329 	kmutex_t *mtx;
5330 
5331 	ASSERT(PAGE_EXCL(pp));
5332 	ASSERT(!PP_ISFREE(pp));
5333 	ASSERT(pp->p_vnode != NULL);
5334 	ASSERT(!IS_SWAPFSVP(pp->p_vnode));
5335 	ASSERT(!PP_ISKAS(pp));
5336 
5337 	VM_STAT_ADD(pagecnt.pc_demote_pages[0]);
5338 
5339 	mtx = page_szc_lock(pp);
5340 	if (mtx != NULL) {
5341 		hat_page_demote(pp);
5342 		mutex_exit(mtx);
5343 	}
5344 	ASSERT(pp->p_szc == 0);
5345 }
5346 
5347 /*
5348  * Mark any existing pages for migration in the given range
5349  */
5350 void
5351 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len,
5352     struct anon_map *amp, ulong_t anon_index, vnode_t *vp,
5353     u_offset_t vnoff, int rflag)
5354 {
5355 	struct anon	*ap;
5356 	vnode_t		*curvp;
5357 	lgrp_t		*from;
5358 	pgcnt_t		nlocked;
5359 	u_offset_t	off;
5360 	pfn_t		pfn;
5361 	size_t		pgsz;
5362 	size_t		segpgsz;
5363 	pgcnt_t		pages;
5364 	uint_t		pszc;
5365 	page_t		*pp0, *pp;
5366 	caddr_t		va;
5367 	ulong_t		an_idx;
5368 	anon_sync_obj_t	cookie;
5369 
5370 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
5371 
5372 	/*
5373 	 * Don't do anything if don't need to do lgroup optimizations
5374 	 * on this system
5375 	 */
5376 	if (!lgrp_optimizations())
5377 		return;
5378 
5379 	/*
5380 	 * Align address and length to (potentially large) page boundary
5381 	 */
5382 	segpgsz = page_get_pagesize(seg->s_szc);
5383 	addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz);
5384 	if (rflag)
5385 		len = P2ROUNDUP(len, segpgsz);
5386 
5387 	/*
5388 	 * Do one (large) page at a time
5389 	 */
5390 	va = addr;
5391 	while (va < addr + len) {
5392 		/*
5393 		 * Lookup (root) page for vnode and offset corresponding to
5394 		 * this virtual address
5395 		 * Try anonmap first since there may be copy-on-write
5396 		 * pages, but initialize vnode pointer and offset using
5397 		 * vnode arguments just in case there isn't an amp.
5398 		 */
5399 		curvp = vp;
5400 		off = vnoff + va - seg->s_base;
5401 		if (amp) {
5402 			ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
5403 			an_idx = anon_index + seg_page(seg, va);
5404 			anon_array_enter(amp, an_idx, &cookie);
5405 			ap = anon_get_ptr(amp->ahp, an_idx);
5406 			if (ap)
5407 				swap_xlate(ap, &curvp, &off);
5408 			anon_array_exit(&cookie);
5409 			ANON_LOCK_EXIT(&amp->a_rwlock);
5410 		}
5411 
5412 		pp = NULL;
5413 		if (curvp)
5414 			pp = page_lookup(curvp, off, SE_SHARED);
5415 
5416 		/*
5417 		 * If there isn't a page at this virtual address,
5418 		 * skip to next page
5419 		 */
5420 		if (pp == NULL) {
5421 			va += PAGESIZE;
5422 			continue;
5423 		}
5424 
5425 		/*
5426 		 * Figure out which lgroup this page is in for kstats
5427 		 */
5428 		pfn = page_pptonum(pp);
5429 		from = lgrp_pfn_to_lgrp(pfn);
5430 
5431 		/*
5432 		 * Get page size, and round up and skip to next page boundary
5433 		 * if unaligned address
5434 		 */
5435 		pszc = pp->p_szc;
5436 		pgsz = page_get_pagesize(pszc);
5437 		pages = btop(pgsz);
5438 		if (!IS_P2ALIGNED(va, pgsz) ||
5439 		    !IS_P2ALIGNED(pfn, pages) ||
5440 		    pgsz > segpgsz) {
5441 			pgsz = MIN(pgsz, segpgsz);
5442 			page_unlock(pp);
5443 			pages = btop(P2END((uintptr_t)va, pgsz) -
5444 			    (uintptr_t)va);
5445 			va = (caddr_t)P2END((uintptr_t)va, pgsz);
5446 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, pages);
5447 			continue;
5448 		}
5449 
5450 		/*
5451 		 * Upgrade to exclusive lock on page
5452 		 */
5453 		if (!page_tryupgrade(pp)) {
5454 			page_unlock(pp);
5455 			va += pgsz;
5456 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
5457 			    btop(pgsz));
5458 			continue;
5459 		}
5460 
5461 		pp0 = pp++;
5462 		nlocked = 1;
5463 
5464 		/*
5465 		 * Lock constituent pages if this is large page
5466 		 */
5467 		if (pages > 1) {
5468 			/*
5469 			 * Lock all constituents except root page, since it
5470 			 * should be locked already.
5471 			 */
5472 			for (; nlocked < pages; nlocked++) {
5473 				if (!page_trylock(pp, SE_EXCL)) {
5474 					break;
5475 				}
5476 				if (PP_ISFREE(pp) ||
5477 				    pp->p_szc != pszc) {
5478 					/*
5479 					 * hat_page_demote() raced in with us.
5480 					 */
5481 					ASSERT(!IS_SWAPFSVP(curvp));
5482 					page_unlock(pp);
5483 					break;
5484 				}
5485 				pp++;
5486 			}
5487 		}
5488 
5489 		/*
5490 		 * If all constituent pages couldn't be locked,
5491 		 * unlock pages locked so far and skip to next page.
5492 		 */
5493 		if (nlocked < pages) {
5494 			while (pp0 < pp) {
5495 				page_unlock(pp0++);
5496 			}
5497 			va += pgsz;
5498 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
5499 			    btop(pgsz));
5500 			continue;
5501 		}
5502 
5503 		/*
5504 		 * hat_page_demote() can no longer happen
5505 		 * since last cons page had the right p_szc after
5506 		 * all cons pages were locked. all cons pages
5507 		 * should now have the same p_szc.
5508 		 */
5509 
5510 		/*
5511 		 * All constituent pages locked successfully, so mark
5512 		 * large page for migration and unload the mappings of
5513 		 * constituent pages, so a fault will occur on any part of the
5514 		 * large page
5515 		 */
5516 		PP_SETMIGRATE(pp0);
5517 		while (pp0 < pp) {
5518 			(void) hat_pageunload(pp0, HAT_FORCE_PGUNLOAD);
5519 			ASSERT(hat_page_getshare(pp0) == 0);
5520 			page_unlock(pp0++);
5521 		}
5522 		lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked);
5523 
5524 		va += pgsz;
5525 	}
5526 }
5527 
5528 /*
5529  * Migrate any pages that have been marked for migration in the given range
5530  */
5531 void
5532 page_migrate(
5533 	struct seg	*seg,
5534 	caddr_t		addr,
5535 	page_t		**ppa,
5536 	pgcnt_t		npages)
5537 {
5538 	lgrp_t		*from;
5539 	lgrp_t		*to;
5540 	page_t		*newpp;
5541 	page_t		*pp;
5542 	pfn_t		pfn;
5543 	size_t		pgsz;
5544 	spgcnt_t	page_cnt;
5545 	spgcnt_t	i;
5546 	uint_t		pszc;
5547 
5548 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
5549 
5550 	while (npages > 0) {
5551 		pp = *ppa;
5552 		pszc = pp->p_szc;
5553 		pgsz = page_get_pagesize(pszc);
5554 		page_cnt = btop(pgsz);
5555 
5556 		/*
5557 		 * Check to see whether this page is marked for migration
5558 		 *
5559 		 * Assume that root page of large page is marked for
5560 		 * migration and none of the other constituent pages
5561 		 * are marked.  This really simplifies clearing the
5562 		 * migrate bit by not having to clear it from each
5563 		 * constituent page.
5564 		 *
5565 		 * note we don't want to relocate an entire large page if
5566 		 * someone is only using one subpage.
5567 		 */
5568 		if (npages < page_cnt)
5569 			break;
5570 
5571 		/*
5572 		 * Is it marked for migration?
5573 		 */
5574 		if (!PP_ISMIGRATE(pp))
5575 			goto next;
5576 
5577 		/*
5578 		 * Determine lgroups that page is being migrated between
5579 		 */
5580 		pfn = page_pptonum(pp);
5581 		if (!IS_P2ALIGNED(pfn, page_cnt)) {
5582 			break;
5583 		}
5584 		from = lgrp_pfn_to_lgrp(pfn);
5585 		to = lgrp_mem_choose(seg, addr, pgsz);
5586 
5587 		/*
5588 		 * Need to get exclusive lock's to migrate
5589 		 */
5590 		for (i = 0; i < page_cnt; i++) {
5591 			ASSERT(PAGE_LOCKED(ppa[i]));
5592 			if (page_pptonum(ppa[i]) != pfn + i ||
5593 			    ppa[i]->p_szc != pszc) {
5594 				break;
5595 			}
5596 			if (!page_tryupgrade(ppa[i])) {
5597 				lgrp_stat_add(from->lgrp_id,
5598 				    LGRP_PM_FAIL_LOCK_PGS,
5599 				    page_cnt);
5600 				break;
5601 			}
5602 
5603 			/*
5604 			 * Check to see whether we are trying to migrate
5605 			 * page to lgroup where it is allocated already.
5606 			 * If so, clear the migrate bit and skip to next
5607 			 * page.
5608 			 */
5609 			if (i == 0 && to == from) {
5610 				PP_CLRMIGRATE(ppa[0]);
5611 				page_downgrade(ppa[0]);
5612 				goto next;
5613 			}
5614 		}
5615 
5616 		/*
5617 		 * If all constituent pages couldn't be locked,
5618 		 * unlock pages locked so far and skip to next page.
5619 		 */
5620 		if (i != page_cnt) {
5621 			while (--i != -1) {
5622 				page_downgrade(ppa[i]);
5623 			}
5624 			goto next;
5625 		}
5626 
5627 		(void) page_create_wait(page_cnt, PG_WAIT);
5628 		newpp = page_get_replacement_page(pp, to, PGR_SAMESZC);
5629 		if (newpp == NULL) {
5630 			page_create_putback(page_cnt);
5631 			for (i = 0; i < page_cnt; i++) {
5632 				page_downgrade(ppa[i]);
5633 			}
5634 			lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS,
5635 			    page_cnt);
5636 			goto next;
5637 		}
5638 		ASSERT(newpp->p_szc == pszc);
5639 		/*
5640 		 * Clear migrate bit and relocate page
5641 		 */
5642 		PP_CLRMIGRATE(pp);
5643 		if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) {
5644 			panic("page_migrate: page_relocate failed");
5645 		}
5646 		ASSERT(page_cnt * PAGESIZE == pgsz);
5647 
5648 		/*
5649 		 * Keep stats for number of pages migrated from and to
5650 		 * each lgroup
5651 		 */
5652 		lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt);
5653 		lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt);
5654 		/*
5655 		 * update the page_t array we were passed in and
5656 		 * unlink constituent pages of a large page.
5657 		 */
5658 		for (i = 0; i < page_cnt; ++i, ++pp) {
5659 			ASSERT(PAGE_EXCL(newpp));
5660 			ASSERT(newpp->p_szc == pszc);
5661 			ppa[i] = newpp;
5662 			pp = newpp;
5663 			page_sub(&newpp, pp);
5664 			page_downgrade(pp);
5665 		}
5666 		ASSERT(newpp == NULL);
5667 next:
5668 		addr += pgsz;
5669 		ppa += page_cnt;
5670 		npages -= page_cnt;
5671 	}
5672 }
5673 
5674 uint_t page_reclaim_maxcnt = 60; /* max total iterations */
5675 uint_t page_reclaim_nofree_maxcnt = 3; /* max iterations without progress */
5676 /*
5677  * Reclaim/reserve availrmem for npages.
5678  * If there is not enough memory start reaping seg, kmem caches.
5679  * Start pageout scanner (via page_needfree()).
5680  * Exit after ~ MAX_CNT s regardless of how much memory has been released.
5681  * Note: There is no guarantee that any availrmem will be freed as
5682  * this memory typically is locked (kernel heap) or reserved for swap.
5683  * Also due to memory fragmentation kmem allocator may not be able
5684  * to free any memory (single user allocated buffer will prevent
5685  * freeing slab or a page).
5686  */
5687 int
5688 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust)
5689 {
5690 	int	i = 0;
5691 	int	i_nofree = 0;
5692 	int	ret = 0;
5693 	pgcnt_t	deficit;
5694 	pgcnt_t old_availrmem = 0;
5695 
5696 	mutex_enter(&freemem_lock);
5697 	while (availrmem < tune.t_minarmem + npages + epages &&
5698 	    i++ < page_reclaim_maxcnt) {
5699 		/* ensure we made some progress in the last few iterations */
5700 		if (old_availrmem < availrmem) {
5701 			old_availrmem = availrmem;
5702 			i_nofree = 0;
5703 		} else if (i_nofree++ >= page_reclaim_nofree_maxcnt) {
5704 			break;
5705 		}
5706 
5707 		deficit = tune.t_minarmem + npages + epages - availrmem;
5708 		mutex_exit(&freemem_lock);
5709 		page_needfree(deficit);
5710 		kmem_reap();
5711 		delay(hz);
5712 		page_needfree(-(spgcnt_t)deficit);
5713 		mutex_enter(&freemem_lock);
5714 	}
5715 
5716 	if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) {
5717 		availrmem -= npages;
5718 		ret = 1;
5719 	}
5720 
5721 	mutex_exit(&freemem_lock);
5722 
5723 	return (ret);
5724 }
5725 
5726 /*
5727  * Search the memory segments to locate the desired page.  Within a
5728  * segment, pages increase linearly with one page structure per
5729  * physical page frame (size PAGESIZE).  The search begins
5730  * with the segment that was accessed last, to take advantage of locality.
5731  * If the hint misses, we start from the beginning of the sorted memseg list
5732  */
5733 
5734 
5735 /*
5736  * Some data structures for pfn to pp lookup.
5737  */
5738 ulong_t mhash_per_slot;
5739 struct memseg *memseg_hash[N_MEM_SLOTS];
5740 
5741 page_t *
5742 page_numtopp_nolock(pfn_t pfnum)
5743 {
5744 	struct memseg *seg;
5745 	page_t *pp;
5746 	vm_cpu_data_t *vc;
5747 
5748 	/*
5749 	 * We need to disable kernel preemption while referencing the
5750 	 * cpu_vm_data field in order to prevent us from being switched to
5751 	 * another cpu and trying to reference it after it has been freed.
5752 	 * This will keep us on cpu and prevent it from being removed while
5753 	 * we are still on it.
5754 	 *
5755 	 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
5756 	 * which is being resued by DR who will flush those references
5757 	 * before modifying the reused memseg.  See memseg_cpu_vm_flush().
5758 	 */
5759 	kpreempt_disable();
5760 	vc = CPU->cpu_vm_data;
5761 	ASSERT(vc != NULL);
5762 
5763 	MEMSEG_STAT_INCR(nsearch);
5764 
5765 	/* Try last winner first */
5766 	if (((seg = vc->vc_pnum_memseg) != NULL) &&
5767 	    (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5768 		MEMSEG_STAT_INCR(nlastwon);
5769 		pp = seg->pages + (pfnum - seg->pages_base);
5770 		if (pp->p_pagenum == pfnum) {
5771 			kpreempt_enable();
5772 			return ((page_t *)pp);
5773 		}
5774 	}
5775 
5776 	/* Else Try hash */
5777 	if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
5778 	    (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5779 		MEMSEG_STAT_INCR(nhashwon);
5780 		vc->vc_pnum_memseg = seg;
5781 		pp = seg->pages + (pfnum - seg->pages_base);
5782 		if (pp->p_pagenum == pfnum) {
5783 			kpreempt_enable();
5784 			return ((page_t *)pp);
5785 		}
5786 	}
5787 
5788 	/* Else Brute force */
5789 	for (seg = memsegs; seg != NULL; seg = seg->next) {
5790 		if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5791 			vc->vc_pnum_memseg = seg;
5792 			pp = seg->pages + (pfnum - seg->pages_base);
5793 			if (pp->p_pagenum == pfnum) {
5794 				kpreempt_enable();
5795 				return ((page_t *)pp);
5796 			}
5797 		}
5798 	}
5799 	vc->vc_pnum_memseg = NULL;
5800 	kpreempt_enable();
5801 	MEMSEG_STAT_INCR(nnotfound);
5802 	return ((page_t *)NULL);
5803 
5804 }
5805 
5806 struct memseg *
5807 page_numtomemseg_nolock(pfn_t pfnum)
5808 {
5809 	struct memseg *seg;
5810 	page_t *pp;
5811 
5812 	/*
5813 	 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
5814 	 * which is being resued by DR who will flush those references
5815 	 * before modifying the reused memseg.  See memseg_cpu_vm_flush().
5816 	 */
5817 	kpreempt_disable();
5818 	/* Try hash */
5819 	if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
5820 	    (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5821 		pp = seg->pages + (pfnum - seg->pages_base);
5822 		if (pp->p_pagenum == pfnum) {
5823 			kpreempt_enable();
5824 			return (seg);
5825 		}
5826 	}
5827 
5828 	/* Else Brute force */
5829 	for (seg = memsegs; seg != NULL; seg = seg->next) {
5830 		if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5831 			pp = seg->pages + (pfnum - seg->pages_base);
5832 			if (pp->p_pagenum == pfnum) {
5833 				kpreempt_enable();
5834 				return (seg);
5835 			}
5836 		}
5837 	}
5838 	kpreempt_enable();
5839 	return ((struct memseg *)NULL);
5840 }
5841 
5842 /*
5843  * Given a page and a count return the page struct that is
5844  * n structs away from the current one in the global page
5845  * list.
5846  *
5847  * This function wraps to the first page upon
5848  * reaching the end of the memseg list.
5849  */
5850 page_t *
5851 page_nextn(page_t *pp, ulong_t n)
5852 {
5853 	struct memseg *seg;
5854 	page_t *ppn;
5855 	vm_cpu_data_t *vc;
5856 
5857 	/*
5858 	 * We need to disable kernel preemption while referencing the
5859 	 * cpu_vm_data field in order to prevent us from being switched to
5860 	 * another cpu and trying to reference it after it has been freed.
5861 	 * This will keep us on cpu and prevent it from being removed while
5862 	 * we are still on it.
5863 	 *
5864 	 * We may be caching a memseg in vc_pnum_memseg/vc_pnext_memseg
5865 	 * which is being resued by DR who will flush those references
5866 	 * before modifying the reused memseg.  See memseg_cpu_vm_flush().
5867 	 */
5868 	kpreempt_disable();
5869 	vc = (vm_cpu_data_t *)CPU->cpu_vm_data;
5870 
5871 	ASSERT(vc != NULL);
5872 
5873 	if (((seg = vc->vc_pnext_memseg) == NULL) ||
5874 	    (seg->pages_base == seg->pages_end) ||
5875 	    !(pp >= seg->pages && pp < seg->epages)) {
5876 
5877 		for (seg = memsegs; seg; seg = seg->next) {
5878 			if (pp >= seg->pages && pp < seg->epages)
5879 				break;
5880 		}
5881 
5882 		if (seg == NULL) {
5883 			/* Memory delete got in, return something valid. */
5884 			/* TODO: fix me. */
5885 			seg = memsegs;
5886 			pp = seg->pages;
5887 		}
5888 	}
5889 
5890 	/* check for wraparound - possible if n is large */
5891 	while ((ppn = (pp + n)) >= seg->epages || ppn < pp) {
5892 		n -= seg->epages - pp;
5893 		seg = seg->next;
5894 		if (seg == NULL)
5895 			seg = memsegs;
5896 		pp = seg->pages;
5897 	}
5898 	vc->vc_pnext_memseg = seg;
5899 	kpreempt_enable();
5900 	return (ppn);
5901 }
5902 
5903 /*
5904  * Initialize for a loop using page_next_scan_large().
5905  */
5906 page_t *
5907 page_next_scan_init(void **cookie)
5908 {
5909 	ASSERT(cookie != NULL);
5910 	*cookie = (void *)memsegs;
5911 	return ((page_t *)memsegs->pages);
5912 }
5913 
5914 /*
5915  * Return the next page in a scan of page_t's, assuming we want
5916  * to skip over sub-pages within larger page sizes.
5917  *
5918  * The cookie is used to keep track of the current memseg.
5919  */
5920 page_t *
5921 page_next_scan_large(
5922 	page_t		*pp,
5923 	ulong_t		*n,
5924 	void		**cookie)
5925 {
5926 	struct memseg	*seg = (struct memseg *)*cookie;
5927 	page_t		*new_pp;
5928 	ulong_t		cnt;
5929 	pfn_t		pfn;
5930 
5931 
5932 	/*
5933 	 * get the count of page_t's to skip based on the page size
5934 	 */
5935 	ASSERT(pp != NULL);
5936 	if (pp->p_szc == 0) {
5937 		cnt = 1;
5938 	} else {
5939 		pfn = page_pptonum(pp);
5940 		cnt = page_get_pagecnt(pp->p_szc);
5941 		cnt -= pfn & (cnt - 1);
5942 	}
5943 	*n += cnt;
5944 	new_pp = pp + cnt;
5945 
5946 	/*
5947 	 * Catch if we went past the end of the current memory segment. If so,
5948 	 * just move to the next segment with pages.
5949 	 */
5950 	if (new_pp >= seg->epages || seg->pages_base == seg->pages_end) {
5951 		do {
5952 			seg = seg->next;
5953 			if (seg == NULL)
5954 				seg = memsegs;
5955 		} while (seg->pages_base == seg->pages_end);
5956 		new_pp = seg->pages;
5957 		*cookie = (void *)seg;
5958 	}
5959 
5960 	return (new_pp);
5961 }
5962 
5963 
5964 /*
5965  * Returns next page in list. Note: this function wraps
5966  * to the first page in the list upon reaching the end
5967  * of the list. Callers should be aware of this fact.
5968  */
5969 
5970 /* We should change this be a #define */
5971 
5972 page_t *
5973 page_next(page_t *pp)
5974 {
5975 	return (page_nextn(pp, 1));
5976 }
5977 
5978 page_t *
5979 page_first()
5980 {
5981 	return ((page_t *)memsegs->pages);
5982 }
5983 
5984 
5985 /*
5986  * This routine is called at boot with the initial memory configuration
5987  * and when memory is added or removed.
5988  */
5989 void
5990 build_pfn_hash()
5991 {
5992 	pfn_t cur;
5993 	pgcnt_t index;
5994 	struct memseg *pseg;
5995 	int	i;
5996 
5997 	/*
5998 	 * Clear memseg_hash array.
5999 	 * Since memory add/delete is designed to operate concurrently
6000 	 * with normal operation, the hash rebuild must be able to run
6001 	 * concurrently with page_numtopp_nolock(). To support this
6002 	 * functionality, assignments to memseg_hash array members must
6003 	 * be done atomically.
6004 	 *
6005 	 * NOTE: bzero() does not currently guarantee this for kernel
6006 	 * threads, and cannot be used here.
6007 	 */
6008 	for (i = 0; i < N_MEM_SLOTS; i++)
6009 		memseg_hash[i] = NULL;
6010 
6011 	hat_kpm_mseghash_clear(N_MEM_SLOTS);
6012 
6013 	/*
6014 	 * Physmax is the last valid pfn.
6015 	 */
6016 	mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT;
6017 	for (pseg = memsegs; pseg != NULL; pseg = pseg->next) {
6018 		index = MEMSEG_PFN_HASH(pseg->pages_base);
6019 		cur = pseg->pages_base;
6020 		do {
6021 			if (index >= N_MEM_SLOTS)
6022 				index = MEMSEG_PFN_HASH(cur);
6023 
6024 			if (memseg_hash[index] == NULL ||
6025 			    memseg_hash[index]->pages_base > pseg->pages_base) {
6026 				memseg_hash[index] = pseg;
6027 				hat_kpm_mseghash_update(index, pseg);
6028 			}
6029 			cur += mhash_per_slot;
6030 			index++;
6031 		} while (cur < pseg->pages_end);
6032 	}
6033 }
6034 
6035 /*
6036  * Return the pagenum for the pp
6037  */
6038 pfn_t
6039 page_pptonum(page_t *pp)
6040 {
6041 	return (pp->p_pagenum);
6042 }
6043 
6044 /*
6045  * interface to the referenced and modified etc bits
6046  * in the PSM part of the page struct
6047  * when no locking is desired.
6048  */
6049 void
6050 page_set_props(page_t *pp, uint_t flags)
6051 {
6052 	ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0);
6053 	pp->p_nrm |= (uchar_t)flags;
6054 }
6055 
6056 void
6057 page_clr_all_props(page_t *pp)
6058 {
6059 	pp->p_nrm = 0;
6060 }
6061 
6062 /*
6063  * Clear p_lckcnt and p_cowcnt, adjusting freemem if required.
6064  */
6065 int
6066 page_clear_lck_cow(page_t *pp, int adjust)
6067 {
6068 	int	f_amount;
6069 
6070 	ASSERT(PAGE_EXCL(pp));
6071 
6072 	/*
6073 	 * The page_struct_lock need not be acquired here since
6074 	 * we require the caller hold the page exclusively locked.
6075 	 */
6076 	f_amount = 0;
6077 	if (pp->p_lckcnt) {
6078 		f_amount = 1;
6079 		pp->p_lckcnt = 0;
6080 	}
6081 	if (pp->p_cowcnt) {
6082 		f_amount += pp->p_cowcnt;
6083 		pp->p_cowcnt = 0;
6084 	}
6085 
6086 	if (adjust && f_amount) {
6087 		mutex_enter(&freemem_lock);
6088 		availrmem += f_amount;
6089 		mutex_exit(&freemem_lock);
6090 	}
6091 
6092 	return (f_amount);
6093 }
6094 
6095 /*
6096  * The following functions is called from free_vp_pages()
6097  * for an inexact estimate of a newly free'd page...
6098  */
6099 ulong_t
6100 page_share_cnt(page_t *pp)
6101 {
6102 	return (hat_page_getshare(pp));
6103 }
6104 
6105 int
6106 page_isshared(page_t *pp)
6107 {
6108 	return (hat_page_checkshare(pp, 1));
6109 }
6110 
6111 int
6112 page_isfree(page_t *pp)
6113 {
6114 	return (PP_ISFREE(pp));
6115 }
6116 
6117 int
6118 page_isref(page_t *pp)
6119 {
6120 	return (hat_page_getattr(pp, P_REF));
6121 }
6122 
6123 int
6124 page_ismod(page_t *pp)
6125 {
6126 	return (hat_page_getattr(pp, P_MOD));
6127 }
6128 
6129 /*
6130  * The following code all currently relates to the page capture logic:
6131  *
6132  * This logic is used for cases where there is a desire to claim a certain
6133  * physical page in the system for the caller.  As it may not be possible
6134  * to capture the page immediately, the p_toxic bits are used in the page
6135  * structure to indicate that someone wants to capture this page.  When the
6136  * page gets unlocked, the toxic flag will be noted and an attempt to capture
6137  * the page will be made.  If it is successful, the original callers callback
6138  * will be called with the page to do with it what they please.
6139  *
6140  * There is also an async thread which wakes up to attempt to capture
6141  * pages occasionally which have the capture bit set.  All of the pages which
6142  * need to be captured asynchronously have been inserted into the
6143  * page_capture_hash and thus this thread walks that hash list.  Items in the
6144  * hash have an expiration time so this thread handles that as well by removing
6145  * the item from the hash if it has expired.
6146  *
6147  * Some important things to note are:
6148  * - if the PR_CAPTURE bit is set on a page, then the page is in the
6149  *   page_capture_hash.  The page_capture_hash_head.pchh_mutex is needed
6150  *   to set and clear this bit, and while the lock is held is the only time
6151  *   you can add or remove an entry from the hash.
6152  * - the PR_CAPTURE bit can only be set and cleared while holding the
6153  *   page_capture_hash_head.pchh_mutex
6154  * - the t_flag field of the thread struct is used with the T_CAPTURING
6155  *   flag to prevent recursion while dealing with large pages.
6156  * - pages which need to be retired never expire on the page_capture_hash.
6157  */
6158 
6159 static void page_capture_thread(void);
6160 static kthread_t *pc_thread_id;
6161 kcondvar_t pc_cv;
6162 static kmutex_t pc_thread_mutex;
6163 static clock_t pc_thread_shortwait;
6164 static clock_t pc_thread_longwait;
6165 static int pc_thread_retry;
6166 
6167 struct page_capture_callback pc_cb[PC_NUM_CALLBACKS];
6168 
6169 /* Note that this is a circular linked list */
6170 typedef struct page_capture_hash_bucket {
6171 	page_t *pp;
6172 	uchar_t szc;
6173 	uchar_t pri;
6174 	uint_t flags;
6175 	clock_t expires;	/* lbolt at which this request expires. */
6176 	void *datap;		/* Cached data passed in for callback */
6177 	struct page_capture_hash_bucket *next;
6178 	struct page_capture_hash_bucket *prev;
6179 } page_capture_hash_bucket_t;
6180 
6181 #define	PC_PRI_HI	0	/* capture now */
6182 #define	PC_PRI_LO	1	/* capture later */
6183 #define	PC_NUM_PRI	2
6184 
6185 #define	PAGE_CAPTURE_PRIO(pp) (PP_ISRAF(pp) ? PC_PRI_LO : PC_PRI_HI)
6186 
6187 
6188 /*
6189  * Each hash bucket will have it's own mutex and two lists which are:
6190  * active (0):	represents requests which have not been processed by
6191  *		the page_capture async thread yet.
6192  * walked (1):	represents requests which have been processed by the
6193  *		page_capture async thread within it's given walk of this bucket.
6194  *
6195  * These are all needed so that we can synchronize all async page_capture
6196  * events.  When the async thread moves to a new bucket, it will append the
6197  * walked list to the active list and walk each item one at a time, moving it
6198  * from the active list to the walked list.  Thus if there is an async request
6199  * outstanding for a given page, it will always be in one of the two lists.
6200  * New requests will always be added to the active list.
6201  * If we were not able to capture a page before the request expired, we'd free
6202  * up the request structure which would indicate to page_capture that there is
6203  * no longer a need for the given page, and clear the PR_CAPTURE flag if
6204  * possible.
6205  */
6206 typedef struct page_capture_hash_head {
6207 	kmutex_t pchh_mutex;
6208 	uint_t num_pages[PC_NUM_PRI];
6209 	page_capture_hash_bucket_t lists[2]; /* sentinel nodes */
6210 } page_capture_hash_head_t;
6211 
6212 #ifdef DEBUG
6213 #define	NUM_PAGE_CAPTURE_BUCKETS 4
6214 #else
6215 #define	NUM_PAGE_CAPTURE_BUCKETS 64
6216 #endif
6217 
6218 page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS];
6219 
6220 /* for now use a very simple hash based upon the size of a page struct */
6221 #define	PAGE_CAPTURE_HASH(pp)	\
6222 	((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1)))
6223 
6224 extern pgcnt_t swapfs_minfree;
6225 
6226 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap);
6227 
6228 /*
6229  * a callback function is required for page capture requests.
6230  */
6231 void
6232 page_capture_register_callback(uint_t index, clock_t duration,
6233     int (*cb_func)(page_t *, void *, uint_t))
6234 {
6235 	ASSERT(pc_cb[index].cb_active == 0);
6236 	ASSERT(cb_func != NULL);
6237 	rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER);
6238 	pc_cb[index].duration = duration;
6239 	pc_cb[index].cb_func = cb_func;
6240 	pc_cb[index].cb_active = 1;
6241 	rw_exit(&pc_cb[index].cb_rwlock);
6242 }
6243 
6244 void
6245 page_capture_unregister_callback(uint_t index)
6246 {
6247 	int i, j;
6248 	struct page_capture_hash_bucket *bp1;
6249 	struct page_capture_hash_bucket *bp2;
6250 	struct page_capture_hash_bucket *head = NULL;
6251 	uint_t flags = (1 << index);
6252 
6253 	rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER);
6254 	ASSERT(pc_cb[index].cb_active == 1);
6255 	pc_cb[index].duration = 0;	/* Paranoia */
6256 	pc_cb[index].cb_func = NULL;	/* Paranoia */
6257 	pc_cb[index].cb_active = 0;
6258 	rw_exit(&pc_cb[index].cb_rwlock);
6259 
6260 	/*
6261 	 * Just move all the entries to a private list which we can walk
6262 	 * through without the need to hold any locks.
6263 	 * No more requests can get added to the hash lists for this consumer
6264 	 * as the cb_active field for the callback has been cleared.
6265 	 */
6266 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
6267 		mutex_enter(&page_capture_hash[i].pchh_mutex);
6268 		for (j = 0; j < 2; j++) {
6269 			bp1 = page_capture_hash[i].lists[j].next;
6270 			/* walk through all but first (sentinel) element */
6271 			while (bp1 != &page_capture_hash[i].lists[j]) {
6272 				bp2 = bp1;
6273 				if (bp2->flags & flags) {
6274 					bp1 = bp2->next;
6275 					bp1->prev = bp2->prev;
6276 					bp2->prev->next = bp1;
6277 					bp2->next = head;
6278 					head = bp2;
6279 					/*
6280 					 * Clear the PR_CAPTURE bit as we
6281 					 * hold appropriate locks here.
6282 					 */
6283 					page_clrtoxic(head->pp, PR_CAPTURE);
6284 					page_capture_hash[i].
6285 					    num_pages[bp2->pri]--;
6286 					continue;
6287 				}
6288 				bp1 = bp1->next;
6289 			}
6290 		}
6291 		mutex_exit(&page_capture_hash[i].pchh_mutex);
6292 	}
6293 
6294 	while (head != NULL) {
6295 		bp1 = head;
6296 		head = head->next;
6297 		kmem_free(bp1, sizeof (*bp1));
6298 	}
6299 }
6300 
6301 
6302 /*
6303  * Find pp in the active list and move it to the walked list if it
6304  * exists.
6305  * Note that most often pp should be at the front of the active list
6306  * as it is currently used and thus there is no other sort of optimization
6307  * being done here as this is a linked list data structure.
6308  * Returns 1 on successful move or 0 if page could not be found.
6309  */
6310 static int
6311 page_capture_move_to_walked(page_t *pp)
6312 {
6313 	page_capture_hash_bucket_t *bp;
6314 	int index;
6315 
6316 	index = PAGE_CAPTURE_HASH(pp);
6317 
6318 	mutex_enter(&page_capture_hash[index].pchh_mutex);
6319 	bp = page_capture_hash[index].lists[0].next;
6320 	while (bp != &page_capture_hash[index].lists[0]) {
6321 		if (bp->pp == pp) {
6322 			/* Remove from old list */
6323 			bp->next->prev = bp->prev;
6324 			bp->prev->next = bp->next;
6325 
6326 			/* Add to new list */
6327 			bp->next = page_capture_hash[index].lists[1].next;
6328 			bp->prev = &page_capture_hash[index].lists[1];
6329 			page_capture_hash[index].lists[1].next = bp;
6330 			bp->next->prev = bp;
6331 
6332 			/*
6333 			 * There is a small probability of page on a free
6334 			 * list being retired while being allocated
6335 			 * and before P_RAF is set on it. The page may
6336 			 * end up marked as high priority request instead
6337 			 * of low priority request.
6338 			 * If P_RAF page is not marked as low priority request
6339 			 * change it to low priority request.
6340 			 */
6341 			page_capture_hash[index].num_pages[bp->pri]--;
6342 			bp->pri = PAGE_CAPTURE_PRIO(pp);
6343 			page_capture_hash[index].num_pages[bp->pri]++;
6344 			mutex_exit(&page_capture_hash[index].pchh_mutex);
6345 			return (1);
6346 		}
6347 		bp = bp->next;
6348 	}
6349 	mutex_exit(&page_capture_hash[index].pchh_mutex);
6350 	return (0);
6351 }
6352 
6353 /*
6354  * Add a new entry to the page capture hash.  The only case where a new
6355  * entry is not added is when the page capture consumer is no longer registered.
6356  * In this case, we'll silently not add the page to the hash.  We know that
6357  * page retire will always be registered for the case where we are currently
6358  * unretiring a page and thus there are no conflicts.
6359  */
6360 static void
6361 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap)
6362 {
6363 	page_capture_hash_bucket_t *bp1;
6364 	page_capture_hash_bucket_t *bp2;
6365 	int index;
6366 	int cb_index;
6367 	int i;
6368 	uchar_t pri;
6369 #ifdef DEBUG
6370 	page_capture_hash_bucket_t *tp1;
6371 	int l;
6372 #endif
6373 
6374 	ASSERT(!(flags & CAPTURE_ASYNC));
6375 
6376 	bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP);
6377 
6378 	bp1->pp = pp;
6379 	bp1->szc = szc;
6380 	bp1->flags = flags;
6381 	bp1->datap = datap;
6382 
6383 	for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6384 		if ((flags >> cb_index) & 1) {
6385 			break;
6386 		}
6387 	}
6388 
6389 	ASSERT(cb_index != PC_NUM_CALLBACKS);
6390 
6391 	rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER);
6392 	if (pc_cb[cb_index].cb_active) {
6393 		if (pc_cb[cb_index].duration == -1) {
6394 			bp1->expires = (clock_t)-1;
6395 		} else {
6396 			bp1->expires = ddi_get_lbolt() +
6397 			    pc_cb[cb_index].duration;
6398 		}
6399 	} else {
6400 		/* There's no callback registered so don't add to the hash */
6401 		rw_exit(&pc_cb[cb_index].cb_rwlock);
6402 		kmem_free(bp1, sizeof (*bp1));
6403 		return;
6404 	}
6405 
6406 	index = PAGE_CAPTURE_HASH(pp);
6407 
6408 	/*
6409 	 * Only allow capture flag to be modified under this mutex.
6410 	 * Prevents multiple entries for same page getting added.
6411 	 */
6412 	mutex_enter(&page_capture_hash[index].pchh_mutex);
6413 
6414 	/*
6415 	 * if not already on the hash, set capture bit and add to the hash
6416 	 */
6417 	if (!(pp->p_toxic & PR_CAPTURE)) {
6418 #ifdef DEBUG
6419 		/* Check for duplicate entries */
6420 		for (l = 0; l < 2; l++) {
6421 			tp1 = page_capture_hash[index].lists[l].next;
6422 			while (tp1 != &page_capture_hash[index].lists[l]) {
6423 				if (tp1->pp == pp) {
6424 					panic("page pp 0x%p already on hash "
6425 					    "at 0x%p\n",
6426 					    (void *)pp, (void *)tp1);
6427 				}
6428 				tp1 = tp1->next;
6429 			}
6430 		}
6431 
6432 #endif
6433 		page_settoxic(pp, PR_CAPTURE);
6434 		pri = PAGE_CAPTURE_PRIO(pp);
6435 		bp1->pri = pri;
6436 		bp1->next = page_capture_hash[index].lists[0].next;
6437 		bp1->prev = &page_capture_hash[index].lists[0];
6438 		bp1->next->prev = bp1;
6439 		page_capture_hash[index].lists[0].next = bp1;
6440 		page_capture_hash[index].num_pages[pri]++;
6441 		if (flags & CAPTURE_RETIRE) {
6442 			page_retire_incr_pend_count(datap);
6443 		}
6444 		mutex_exit(&page_capture_hash[index].pchh_mutex);
6445 		rw_exit(&pc_cb[cb_index].cb_rwlock);
6446 		cv_signal(&pc_cv);
6447 		return;
6448 	}
6449 
6450 	/*
6451 	 * A page retire request will replace any other request.
6452 	 * A second physmem request which is for a different process than
6453 	 * the currently registered one will be dropped as there is
6454 	 * no way to hold the private data for both calls.
6455 	 * In the future, once there are more callers, this will have to
6456 	 * be worked out better as there needs to be private storage for
6457 	 * at least each type of caller (maybe have datap be an array of
6458 	 * *void's so that we can index based upon callers index).
6459 	 */
6460 
6461 	/* walk hash list to update expire time */
6462 	for (i = 0; i < 2; i++) {
6463 		bp2 = page_capture_hash[index].lists[i].next;
6464 		while (bp2 != &page_capture_hash[index].lists[i]) {
6465 			if (bp2->pp == pp) {
6466 				if (flags & CAPTURE_RETIRE) {
6467 					if (!(bp2->flags & CAPTURE_RETIRE)) {
6468 						page_retire_incr_pend_count(
6469 						    datap);
6470 						bp2->flags = flags;
6471 						bp2->expires = bp1->expires;
6472 						bp2->datap = datap;
6473 					}
6474 				} else {
6475 					ASSERT(flags & CAPTURE_PHYSMEM);
6476 					if (!(bp2->flags & CAPTURE_RETIRE) &&
6477 					    (datap == bp2->datap)) {
6478 						bp2->expires = bp1->expires;
6479 					}
6480 				}
6481 				mutex_exit(&page_capture_hash[index].
6482 				    pchh_mutex);
6483 				rw_exit(&pc_cb[cb_index].cb_rwlock);
6484 				kmem_free(bp1, sizeof (*bp1));
6485 				return;
6486 			}
6487 			bp2 = bp2->next;
6488 		}
6489 	}
6490 
6491 	/*
6492 	 * the PR_CAPTURE flag is protected by the page_capture_hash mutexes
6493 	 * and thus it either has to be set or not set and can't change
6494 	 * while holding the mutex above.
6495 	 */
6496 	panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n",
6497 	    (void *)pp);
6498 }
6499 
6500 /*
6501  * We have a page in our hands, lets try and make it ours by turning
6502  * it into a clean page like it had just come off the freelists.
6503  *
6504  * Returns 0 on success, with the page still EXCL locked.
6505  * On failure, the page will be unlocked, and returns EAGAIN
6506  */
6507 static int
6508 page_capture_clean_page(page_t *pp)
6509 {
6510 	page_t *newpp;
6511 	int skip_unlock = 0;
6512 	spgcnt_t count;
6513 	page_t *tpp;
6514 	int ret = 0;
6515 	int extra;
6516 
6517 	ASSERT(PAGE_EXCL(pp));
6518 	ASSERT(!PP_RETIRED(pp));
6519 	ASSERT(curthread->t_flag & T_CAPTURING);
6520 
6521 	if (PP_ISFREE(pp)) {
6522 		if (!page_reclaim(pp, NULL)) {
6523 			skip_unlock = 1;
6524 			ret = EAGAIN;
6525 			goto cleanup;
6526 		}
6527 		ASSERT(pp->p_szc == 0);
6528 		if (pp->p_vnode != NULL) {
6529 			/*
6530 			 * Since this page came from the
6531 			 * cachelist, we must destroy the
6532 			 * old vnode association.
6533 			 */
6534 			page_hashout(pp, NULL);
6535 		}
6536 		goto cleanup;
6537 	}
6538 
6539 	/*
6540 	 * If we know page_relocate will fail, skip it
6541 	 * It could still fail due to a UE on another page but we
6542 	 * can't do anything about that.
6543 	 */
6544 	if (pp->p_toxic & PR_UE) {
6545 		goto skip_relocate;
6546 	}
6547 
6548 	/*
6549 	 * It's possible that pages can not have a vnode as fsflush comes
6550 	 * through and cleans up these pages.  It's ugly but that's how it is.
6551 	 */
6552 	if (pp->p_vnode == NULL) {
6553 		goto skip_relocate;
6554 	}
6555 
6556 	/*
6557 	 * Page was not free, so lets try to relocate it.
6558 	 * page_relocate only works with root pages, so if this is not a root
6559 	 * page, we need to demote it to try and relocate it.
6560 	 * Unfortunately this is the best we can do right now.
6561 	 */
6562 	newpp = NULL;
6563 	if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) {
6564 		if (page_try_demote_pages(pp) == 0) {
6565 			ret = EAGAIN;
6566 			goto cleanup;
6567 		}
6568 	}
6569 	ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL);
6570 	if (ret == 0) {
6571 		page_t *npp;
6572 		/* unlock the new page(s) */
6573 		while (count-- > 0) {
6574 			ASSERT(newpp != NULL);
6575 			npp = newpp;
6576 			page_sub(&newpp, npp);
6577 			page_unlock(npp);
6578 		}
6579 		ASSERT(newpp == NULL);
6580 		/*
6581 		 * Check to see if the page we have is too large.
6582 		 * If so, demote it freeing up the extra pages.
6583 		 */
6584 		if (pp->p_szc > 0) {
6585 			/* For now demote extra pages to szc == 0 */
6586 			extra = page_get_pagecnt(pp->p_szc) - 1;
6587 			while (extra > 0) {
6588 				tpp = pp->p_next;
6589 				page_sub(&pp, tpp);
6590 				tpp->p_szc = 0;
6591 				page_free(tpp, 1);
6592 				extra--;
6593 			}
6594 			/* Make sure to set our page to szc 0 as well */
6595 			ASSERT(pp->p_next == pp && pp->p_prev == pp);
6596 			pp->p_szc = 0;
6597 		}
6598 		goto cleanup;
6599 	} else if (ret == EIO) {
6600 		ret = EAGAIN;
6601 		goto cleanup;
6602 	} else {
6603 		/*
6604 		 * Need to reset return type as we failed to relocate the page
6605 		 * but that does not mean that some of the next steps will not
6606 		 * work.
6607 		 */
6608 		ret = 0;
6609 	}
6610 
6611 skip_relocate:
6612 
6613 	if (pp->p_szc > 0) {
6614 		if (page_try_demote_pages(pp) == 0) {
6615 			ret = EAGAIN;
6616 			goto cleanup;
6617 		}
6618 	}
6619 
6620 	ASSERT(pp->p_szc == 0);
6621 
6622 	if (hat_ismod(pp)) {
6623 		ret = EAGAIN;
6624 		goto cleanup;
6625 	}
6626 	if (PP_ISKAS(pp)) {
6627 		ret = EAGAIN;
6628 		goto cleanup;
6629 	}
6630 	if (pp->p_lckcnt || pp->p_cowcnt) {
6631 		ret = EAGAIN;
6632 		goto cleanup;
6633 	}
6634 
6635 	(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
6636 	ASSERT(!hat_page_is_mapped(pp));
6637 
6638 	if (hat_ismod(pp)) {
6639 		/*
6640 		 * This is a semi-odd case as the page is now modified but not
6641 		 * mapped as we just unloaded the mappings above.
6642 		 */
6643 		ret = EAGAIN;
6644 		goto cleanup;
6645 	}
6646 	if (pp->p_vnode != NULL) {
6647 		page_hashout(pp, NULL);
6648 	}
6649 
6650 	/*
6651 	 * At this point, the page should be in a clean state and
6652 	 * we can do whatever we want with it.
6653 	 */
6654 
6655 cleanup:
6656 	if (ret != 0) {
6657 		if (!skip_unlock) {
6658 			page_unlock(pp);
6659 		}
6660 	} else {
6661 		ASSERT(pp->p_szc == 0);
6662 		ASSERT(PAGE_EXCL(pp));
6663 
6664 		pp->p_next = pp;
6665 		pp->p_prev = pp;
6666 	}
6667 	return (ret);
6668 }
6669 
6670 /*
6671  * Various callers of page_trycapture() can have different restrictions upon
6672  * what memory they have access to.
6673  * Returns 0 on success, with the following error codes on failure:
6674  *      EPERM - The requested page is long term locked, and thus repeated
6675  *              requests to capture this page will likely fail.
6676  *      ENOMEM - There was not enough free memory in the system to safely
6677  *              map the requested page.
6678  *      ENOENT - The requested page was inside the kernel cage, and the
6679  *              PHYSMEM_CAGE flag was not set.
6680  */
6681 int
6682 page_capture_pre_checks(page_t *pp, uint_t flags)
6683 {
6684 	ASSERT(pp != NULL);
6685 
6686 #if defined(__sparc)
6687 	if (pp->p_vnode == &promvp) {
6688 		return (EPERM);
6689 	}
6690 
6691 	if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE) &&
6692 	    (flags & CAPTURE_PHYSMEM)) {
6693 		return (ENOENT);
6694 	}
6695 
6696 	if (PP_ISNORELOCKERNEL(pp)) {
6697 		return (EPERM);
6698 	}
6699 #else
6700 	if (PP_ISKAS(pp)) {
6701 		return (EPERM);
6702 	}
6703 #endif /* __sparc */
6704 
6705 	/* only physmem currently has the restrictions checked below */
6706 	if (!(flags & CAPTURE_PHYSMEM)) {
6707 		return (0);
6708 	}
6709 
6710 	if (availrmem < swapfs_minfree) {
6711 		/*
6712 		 * We won't try to capture this page as we are
6713 		 * running low on memory.
6714 		 */
6715 		return (ENOMEM);
6716 	}
6717 	return (0);
6718 }
6719 
6720 /*
6721  * Once we have a page in our mits, go ahead and complete the capture
6722  * operation.
6723  * Returns 1 on failure where page is no longer needed
6724  * Returns 0 on success
6725  * Returns -1 if there was a transient failure.
6726  * Failure cases must release the SE_EXCL lock on pp (usually via page_free).
6727  */
6728 int
6729 page_capture_take_action(page_t *pp, uint_t flags, void *datap)
6730 {
6731 	int cb_index;
6732 	int ret = 0;
6733 	page_capture_hash_bucket_t *bp1;
6734 	page_capture_hash_bucket_t *bp2;
6735 	int index;
6736 	int found = 0;
6737 	int i;
6738 
6739 	ASSERT(PAGE_EXCL(pp));
6740 	ASSERT(curthread->t_flag & T_CAPTURING);
6741 
6742 	for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6743 		if ((flags >> cb_index) & 1) {
6744 			break;
6745 		}
6746 	}
6747 	ASSERT(cb_index < PC_NUM_CALLBACKS);
6748 
6749 	/*
6750 	 * Remove the entry from the page_capture hash, but don't free it yet
6751 	 * as we may need to put it back.
6752 	 * Since we own the page at this point in time, we should find it
6753 	 * in the hash if this is an ASYNC call.  If we don't it's likely
6754 	 * that the page_capture_async() thread decided that this request
6755 	 * had expired, in which case we just continue on.
6756 	 */
6757 	if (flags & CAPTURE_ASYNC) {
6758 
6759 		index = PAGE_CAPTURE_HASH(pp);
6760 
6761 		mutex_enter(&page_capture_hash[index].pchh_mutex);
6762 		for (i = 0; i < 2 && !found; i++) {
6763 			bp1 = page_capture_hash[index].lists[i].next;
6764 			while (bp1 != &page_capture_hash[index].lists[i]) {
6765 				if (bp1->pp == pp) {
6766 					bp1->next->prev = bp1->prev;
6767 					bp1->prev->next = bp1->next;
6768 					page_capture_hash[index].
6769 					    num_pages[bp1->pri]--;
6770 					page_clrtoxic(pp, PR_CAPTURE);
6771 					found = 1;
6772 					break;
6773 				}
6774 				bp1 = bp1->next;
6775 			}
6776 		}
6777 		mutex_exit(&page_capture_hash[index].pchh_mutex);
6778 	}
6779 
6780 	/* Synchronize with the unregister func. */
6781 	rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER);
6782 	if (!pc_cb[cb_index].cb_active) {
6783 		page_free(pp, 1);
6784 		rw_exit(&pc_cb[cb_index].cb_rwlock);
6785 		if (found) {
6786 			kmem_free(bp1, sizeof (*bp1));
6787 		}
6788 		return (1);
6789 	}
6790 
6791 	/*
6792 	 * We need to remove the entry from the page capture hash and turn off
6793 	 * the PR_CAPTURE bit before calling the callback.  We'll need to cache
6794 	 * the entry here, and then based upon the return value, cleanup
6795 	 * appropriately or re-add it to the hash, making sure that someone else
6796 	 * hasn't already done so.
6797 	 * It should be rare for the callback to fail and thus it's ok for
6798 	 * the failure path to be a bit complicated as the success path is
6799 	 * cleaner and the locking rules are easier to follow.
6800 	 */
6801 
6802 	ret = pc_cb[cb_index].cb_func(pp, datap, flags);
6803 
6804 	rw_exit(&pc_cb[cb_index].cb_rwlock);
6805 
6806 	/*
6807 	 * If this was an ASYNC request, we need to cleanup the hash if the
6808 	 * callback was successful or if the request was no longer valid.
6809 	 * For non-ASYNC requests, we return failure to map and the caller
6810 	 * will take care of adding the request to the hash.
6811 	 * Note also that the callback itself is responsible for the page
6812 	 * at this point in time in terms of locking ...  The most common
6813 	 * case for the failure path should just be a page_free.
6814 	 */
6815 	if (ret >= 0) {
6816 		if (found) {
6817 			if (bp1->flags & CAPTURE_RETIRE) {
6818 				page_retire_decr_pend_count(datap);
6819 			}
6820 			kmem_free(bp1, sizeof (*bp1));
6821 		}
6822 		return (ret);
6823 	}
6824 	if (!found) {
6825 		return (ret);
6826 	}
6827 
6828 	ASSERT(flags & CAPTURE_ASYNC);
6829 
6830 	/*
6831 	 * Check for expiration time first as we can just free it up if it's
6832 	 * expired.
6833 	 */
6834 	if (ddi_get_lbolt() > bp1->expires && bp1->expires != -1) {
6835 		kmem_free(bp1, sizeof (*bp1));
6836 		return (ret);
6837 	}
6838 
6839 	/*
6840 	 * The callback failed and there used to be an entry in the hash for
6841 	 * this page, so we need to add it back to the hash.
6842 	 */
6843 	mutex_enter(&page_capture_hash[index].pchh_mutex);
6844 	if (!(pp->p_toxic & PR_CAPTURE)) {
6845 		/* just add bp1 back to head of walked list */
6846 		page_settoxic(pp, PR_CAPTURE);
6847 		bp1->next = page_capture_hash[index].lists[1].next;
6848 		bp1->prev = &page_capture_hash[index].lists[1];
6849 		bp1->next->prev = bp1;
6850 		bp1->pri = PAGE_CAPTURE_PRIO(pp);
6851 		page_capture_hash[index].lists[1].next = bp1;
6852 		page_capture_hash[index].num_pages[bp1->pri]++;
6853 		mutex_exit(&page_capture_hash[index].pchh_mutex);
6854 		return (ret);
6855 	}
6856 
6857 	/*
6858 	 * Otherwise there was a new capture request added to list
6859 	 * Need to make sure that our original data is represented if
6860 	 * appropriate.
6861 	 */
6862 	for (i = 0; i < 2; i++) {
6863 		bp2 = page_capture_hash[index].lists[i].next;
6864 		while (bp2 != &page_capture_hash[index].lists[i]) {
6865 			if (bp2->pp == pp) {
6866 				if (bp1->flags & CAPTURE_RETIRE) {
6867 					if (!(bp2->flags & CAPTURE_RETIRE)) {
6868 						bp2->szc = bp1->szc;
6869 						bp2->flags = bp1->flags;
6870 						bp2->expires = bp1->expires;
6871 						bp2->datap = bp1->datap;
6872 					}
6873 				} else {
6874 					ASSERT(bp1->flags & CAPTURE_PHYSMEM);
6875 					if (!(bp2->flags & CAPTURE_RETIRE)) {
6876 						bp2->szc = bp1->szc;
6877 						bp2->flags = bp1->flags;
6878 						bp2->expires = bp1->expires;
6879 						bp2->datap = bp1->datap;
6880 					}
6881 				}
6882 				page_capture_hash[index].num_pages[bp2->pri]--;
6883 				bp2->pri = PAGE_CAPTURE_PRIO(pp);
6884 				page_capture_hash[index].num_pages[bp2->pri]++;
6885 				mutex_exit(&page_capture_hash[index].
6886 				    pchh_mutex);
6887 				kmem_free(bp1, sizeof (*bp1));
6888 				return (ret);
6889 			}
6890 			bp2 = bp2->next;
6891 		}
6892 	}
6893 	panic("PR_CAPTURE set but not on hash for pp 0x%p\n", (void *)pp);
6894 	/*NOTREACHED*/
6895 }
6896 
6897 /*
6898  * Try to capture the given page for the caller specified in the flags
6899  * parameter.  The page will either be captured and handed over to the
6900  * appropriate callback, or will be queued up in the page capture hash
6901  * to be captured asynchronously.
6902  * If the current request is due to an async capture, the page must be
6903  * exclusively locked before calling this function.
6904  * Currently szc must be 0 but in the future this should be expandable to
6905  * other page sizes.
6906  * Returns 0 on success, with the following error codes on failure:
6907  *      EPERM - The requested page is long term locked, and thus repeated
6908  *              requests to capture this page will likely fail.
6909  *      ENOMEM - There was not enough free memory in the system to safely
6910  *              map the requested page.
6911  *      ENOENT - The requested page was inside the kernel cage, and the
6912  *              CAPTURE_GET_CAGE flag was not set.
6913  *	EAGAIN - The requested page could not be capturead at this point in
6914  *		time but future requests will likely work.
6915  *	EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag
6916  *		was not set.
6917  */
6918 int
6919 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap)
6920 {
6921 	int ret;
6922 	int cb_index;
6923 
6924 	if (flags & CAPTURE_ASYNC) {
6925 		ASSERT(PAGE_EXCL(pp));
6926 		goto async;
6927 	}
6928 
6929 	/* Make sure there's enough availrmem ... */
6930 	ret = page_capture_pre_checks(pp, flags);
6931 	if (ret != 0) {
6932 		return (ret);
6933 	}
6934 
6935 	if (!page_trylock(pp, SE_EXCL)) {
6936 		for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
6937 			if ((flags >> cb_index) & 1) {
6938 				break;
6939 			}
6940 		}
6941 		ASSERT(cb_index < PC_NUM_CALLBACKS);
6942 		ret = EAGAIN;
6943 		/* Special case for retired pages */
6944 		if (PP_RETIRED(pp)) {
6945 			if (flags & CAPTURE_GET_RETIRED) {
6946 				if (!page_unretire_pp(pp, PR_UNR_TEMP)) {
6947 					/*
6948 					 * Need to set capture bit and add to
6949 					 * hash so that the page will be
6950 					 * retired when freed.
6951 					 */
6952 					page_capture_add_hash(pp, szc,
6953 					    CAPTURE_RETIRE, NULL);
6954 					ret = 0;
6955 					goto own_page;
6956 				}
6957 			} else {
6958 				return (EBUSY);
6959 			}
6960 		}
6961 		page_capture_add_hash(pp, szc, flags, datap);
6962 		return (ret);
6963 	}
6964 
6965 async:
6966 	ASSERT(PAGE_EXCL(pp));
6967 
6968 	/* Need to check for physmem async requests that availrmem is sane */
6969 	if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) ==
6970 	    (CAPTURE_ASYNC | CAPTURE_PHYSMEM) &&
6971 	    (availrmem < swapfs_minfree)) {
6972 		page_unlock(pp);
6973 		return (ENOMEM);
6974 	}
6975 
6976 	ret = page_capture_clean_page(pp);
6977 
6978 	if (ret != 0) {
6979 		/* We failed to get the page, so lets add it to the hash */
6980 		if (!(flags & CAPTURE_ASYNC)) {
6981 			page_capture_add_hash(pp, szc, flags, datap);
6982 		}
6983 		return (ret);
6984 	}
6985 
6986 own_page:
6987 	ASSERT(PAGE_EXCL(pp));
6988 	ASSERT(pp->p_szc == 0);
6989 
6990 	/* Call the callback */
6991 	ret = page_capture_take_action(pp, flags, datap);
6992 
6993 	if (ret == 0) {
6994 		return (0);
6995 	}
6996 
6997 	/*
6998 	 * Note that in the failure cases from page_capture_take_action, the
6999 	 * EXCL lock will have already been dropped.
7000 	 */
7001 	if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) {
7002 		page_capture_add_hash(pp, szc, flags, datap);
7003 	}
7004 	return (EAGAIN);
7005 }
7006 
7007 int
7008 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap)
7009 {
7010 	int ret;
7011 
7012 	curthread->t_flag |= T_CAPTURING;
7013 	ret = page_itrycapture(pp, szc, flags, datap);
7014 	curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */
7015 	return (ret);
7016 }
7017 
7018 /*
7019  * When unlocking a page which has the PR_CAPTURE bit set, this routine
7020  * gets called to try and capture the page.
7021  */
7022 void
7023 page_unlock_capture(page_t *pp)
7024 {
7025 	page_capture_hash_bucket_t *bp;
7026 	int index;
7027 	int i;
7028 	uint_t szc;
7029 	uint_t flags = 0;
7030 	void *datap;
7031 	kmutex_t *mp;
7032 	extern vnode_t retired_pages;
7033 
7034 	/*
7035 	 * We need to protect against a possible deadlock here where we own
7036 	 * the vnode page hash mutex and want to acquire it again as there
7037 	 * are locations in the code, where we unlock a page while holding
7038 	 * the mutex which can lead to the page being captured and eventually
7039 	 * end up here.  As we may be hashing out the old page and hashing into
7040 	 * the retire vnode, we need to make sure we don't own them.
7041 	 * Other callbacks who do hash operations also need to make sure that
7042 	 * before they hashin to a vnode that they do not currently own the
7043 	 * vphm mutex otherwise there will be a panic.
7044 	 */
7045 	if (mutex_owned(page_vnode_mutex(&retired_pages))) {
7046 		page_unlock_nocapture(pp);
7047 		return;
7048 	}
7049 	if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) {
7050 		page_unlock_nocapture(pp);
7051 		return;
7052 	}
7053 
7054 	index = PAGE_CAPTURE_HASH(pp);
7055 
7056 	mp = &page_capture_hash[index].pchh_mutex;
7057 	mutex_enter(mp);
7058 	for (i = 0; i < 2; i++) {
7059 		bp = page_capture_hash[index].lists[i].next;
7060 		while (bp != &page_capture_hash[index].lists[i]) {
7061 			if (bp->pp == pp) {
7062 				szc = bp->szc;
7063 				flags = bp->flags | CAPTURE_ASYNC;
7064 				datap = bp->datap;
7065 				mutex_exit(mp);
7066 				(void) page_trycapture(pp, szc, flags, datap);
7067 				return;
7068 			}
7069 			bp = bp->next;
7070 		}
7071 	}
7072 
7073 	/* Failed to find page in hash so clear flags and unlock it. */
7074 	page_clrtoxic(pp, PR_CAPTURE);
7075 	page_unlock(pp);
7076 
7077 	mutex_exit(mp);
7078 }
7079 
7080 void
7081 page_capture_init()
7082 {
7083 	int i;
7084 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7085 		page_capture_hash[i].lists[0].next =
7086 		    &page_capture_hash[i].lists[0];
7087 		page_capture_hash[i].lists[0].prev =
7088 		    &page_capture_hash[i].lists[0];
7089 		page_capture_hash[i].lists[1].next =
7090 		    &page_capture_hash[i].lists[1];
7091 		page_capture_hash[i].lists[1].prev =
7092 		    &page_capture_hash[i].lists[1];
7093 	}
7094 
7095 	pc_thread_shortwait = 23 * hz;
7096 	pc_thread_longwait = 1201 * hz;
7097 	pc_thread_retry = 3;
7098 	mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL);
7099 	cv_init(&pc_cv, NULL, CV_DEFAULT, NULL);
7100 	pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0,
7101 	    TS_RUN, minclsyspri);
7102 }
7103 
7104 /*
7105  * It is necessary to scrub any failing pages prior to reboot in order to
7106  * prevent a latent error trap from occurring on the next boot.
7107  */
7108 void
7109 page_retire_mdboot()
7110 {
7111 	page_t *pp;
7112 	int i, j;
7113 	page_capture_hash_bucket_t *bp;
7114 	uchar_t pri;
7115 
7116 	/* walk lists looking for pages to scrub */
7117 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7118 		for (pri = 0; pri < PC_NUM_PRI; pri++) {
7119 			if (page_capture_hash[i].num_pages[pri] != 0) {
7120 				break;
7121 			}
7122 		}
7123 		if (pri == PC_NUM_PRI)
7124 			continue;
7125 
7126 		mutex_enter(&page_capture_hash[i].pchh_mutex);
7127 
7128 		for (j = 0; j < 2; j++) {
7129 			bp = page_capture_hash[i].lists[j].next;
7130 			while (bp != &page_capture_hash[i].lists[j]) {
7131 				pp = bp->pp;
7132 				if (PP_TOXIC(pp)) {
7133 					if (page_trylock(pp, SE_EXCL)) {
7134 						PP_CLRFREE(pp);
7135 						pagescrub(pp, 0, PAGESIZE);
7136 						page_unlock(pp);
7137 					}
7138 				}
7139 				bp = bp->next;
7140 			}
7141 		}
7142 		mutex_exit(&page_capture_hash[i].pchh_mutex);
7143 	}
7144 }
7145 
7146 /*
7147  * Walk the page_capture_hash trying to capture pages and also cleanup old
7148  * entries which have expired.
7149  */
7150 void
7151 page_capture_async()
7152 {
7153 	page_t *pp;
7154 	int i;
7155 	int ret;
7156 	page_capture_hash_bucket_t *bp1, *bp2;
7157 	uint_t szc;
7158 	uint_t flags;
7159 	void *datap;
7160 	uchar_t pri;
7161 
7162 	/* If there are outstanding pages to be captured, get to work */
7163 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7164 		for (pri = 0; pri < PC_NUM_PRI; pri++) {
7165 			if (page_capture_hash[i].num_pages[pri] != 0)
7166 				break;
7167 		}
7168 		if (pri == PC_NUM_PRI)
7169 			continue;
7170 
7171 		/* Append list 1 to list 0 and then walk through list 0 */
7172 		mutex_enter(&page_capture_hash[i].pchh_mutex);
7173 		bp1 = &page_capture_hash[i].lists[1];
7174 		bp2 = bp1->next;
7175 		if (bp1 != bp2) {
7176 			bp1->prev->next = page_capture_hash[i].lists[0].next;
7177 			bp2->prev = &page_capture_hash[i].lists[0];
7178 			page_capture_hash[i].lists[0].next->prev = bp1->prev;
7179 			page_capture_hash[i].lists[0].next = bp2;
7180 			bp1->next = bp1;
7181 			bp1->prev = bp1;
7182 		}
7183 
7184 		/* list[1] will be empty now */
7185 
7186 		bp1 = page_capture_hash[i].lists[0].next;
7187 		while (bp1 != &page_capture_hash[i].lists[0]) {
7188 			/* Check expiration time */
7189 			if ((ddi_get_lbolt() > bp1->expires &&
7190 			    bp1->expires != -1) ||
7191 			    page_deleted(bp1->pp)) {
7192 				page_capture_hash[i].lists[0].next = bp1->next;
7193 				bp1->next->prev =
7194 				    &page_capture_hash[i].lists[0];
7195 				page_capture_hash[i].num_pages[bp1->pri]--;
7196 
7197 				/*
7198 				 * We can safely remove the PR_CAPTURE bit
7199 				 * without holding the EXCL lock on the page
7200 				 * as the PR_CAPTURE bit requres that the
7201 				 * page_capture_hash[].pchh_mutex be held
7202 				 * to modify it.
7203 				 */
7204 				page_clrtoxic(bp1->pp, PR_CAPTURE);
7205 				mutex_exit(&page_capture_hash[i].pchh_mutex);
7206 				kmem_free(bp1, sizeof (*bp1));
7207 				mutex_enter(&page_capture_hash[i].pchh_mutex);
7208 				bp1 = page_capture_hash[i].lists[0].next;
7209 				continue;
7210 			}
7211 			pp = bp1->pp;
7212 			szc = bp1->szc;
7213 			flags = bp1->flags;
7214 			datap = bp1->datap;
7215 			mutex_exit(&page_capture_hash[i].pchh_mutex);
7216 			if (page_trylock(pp, SE_EXCL)) {
7217 				ret = page_trycapture(pp, szc,
7218 				    flags | CAPTURE_ASYNC, datap);
7219 			} else {
7220 				ret = 1;	/* move to walked hash */
7221 			}
7222 
7223 			if (ret != 0) {
7224 				/* Move to walked hash */
7225 				(void) page_capture_move_to_walked(pp);
7226 			}
7227 			mutex_enter(&page_capture_hash[i].pchh_mutex);
7228 			bp1 = page_capture_hash[i].lists[0].next;
7229 		}
7230 
7231 		mutex_exit(&page_capture_hash[i].pchh_mutex);
7232 	}
7233 }
7234 
7235 /*
7236  * This function is called by the page_capture_thread, and is needed in
7237  * in order to initiate aio cleanup, so that pages used in aio
7238  * will be unlocked and subsequently retired by page_capture_thread.
7239  */
7240 static int
7241 do_aio_cleanup(void)
7242 {
7243 	proc_t *procp;
7244 	int (*aio_cleanup_dr_delete_memory)(proc_t *);
7245 	int cleaned = 0;
7246 
7247 	if (modload("sys", "kaio") == -1) {
7248 		cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio");
7249 		return (0);
7250 	}
7251 	/*
7252 	 * We use the aio_cleanup_dr_delete_memory function to
7253 	 * initiate the actual clean up; this function will wake
7254 	 * up the per-process aio_cleanup_thread.
7255 	 */
7256 	aio_cleanup_dr_delete_memory = (int (*)(proc_t *))
7257 	    modgetsymvalue("aio_cleanup_dr_delete_memory", 0);
7258 	if (aio_cleanup_dr_delete_memory == NULL) {
7259 		cmn_err(CE_WARN,
7260 	    "aio_cleanup_dr_delete_memory not found in kaio");
7261 		return (0);
7262 	}
7263 	mutex_enter(&pidlock);
7264 	for (procp = practive; (procp != NULL); procp = procp->p_next) {
7265 		mutex_enter(&procp->p_lock);
7266 		if (procp->p_aio != NULL) {
7267 			/* cleanup proc's outstanding kaio */
7268 			cleaned += (*aio_cleanup_dr_delete_memory)(procp);
7269 		}
7270 		mutex_exit(&procp->p_lock);
7271 	}
7272 	mutex_exit(&pidlock);
7273 	return (cleaned);
7274 }
7275 
7276 /*
7277  * helper function for page_capture_thread
7278  */
7279 static void
7280 page_capture_handle_outstanding(void)
7281 {
7282 	int ntry;
7283 
7284 	/* Reap pages before attempting capture pages */
7285 	kmem_reap();
7286 
7287 	if ((page_retire_pend_count() > page_retire_pend_kas_count()) &&
7288 	    hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
7289 		/*
7290 		 * Note: Purging only for platforms that support
7291 		 * ISM hat_pageunload() - mainly SPARC. On x86/x64
7292 		 * platforms ISM pages SE_SHARED locked until destroyed.
7293 		 */
7294 
7295 		/* disable and purge seg_pcache */
7296 		(void) seg_p_disable();
7297 		for (ntry = 0; ntry < pc_thread_retry; ntry++) {
7298 			if (!page_retire_pend_count())
7299 				break;
7300 			if (do_aio_cleanup()) {
7301 				/*
7302 				 * allow the apps cleanup threads
7303 				 * to run
7304 				 */
7305 				delay(pc_thread_shortwait);
7306 			}
7307 			page_capture_async();
7308 		}
7309 		/* reenable seg_pcache */
7310 		seg_p_enable();
7311 
7312 		/* completed what can be done.  break out */
7313 		return;
7314 	}
7315 
7316 	/*
7317 	 * For kernel pages and/or unsupported HAT_DYNAMIC_ISM_UNMAP, reap
7318 	 * and then attempt to capture.
7319 	 */
7320 	seg_preap();
7321 	page_capture_async();
7322 }
7323 
7324 /*
7325  * The page_capture_thread loops forever, looking to see if there are
7326  * pages still waiting to be captured.
7327  */
7328 static void
7329 page_capture_thread(void)
7330 {
7331 	callb_cpr_t c;
7332 	int i;
7333 	int high_pri_pages;
7334 	int low_pri_pages;
7335 	clock_t timeout;
7336 
7337 	CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture");
7338 
7339 	mutex_enter(&pc_thread_mutex);
7340 	for (;;) {
7341 		high_pri_pages = 0;
7342 		low_pri_pages = 0;
7343 		for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
7344 			high_pri_pages +=
7345 			    page_capture_hash[i].num_pages[PC_PRI_HI];
7346 			low_pri_pages +=
7347 			    page_capture_hash[i].num_pages[PC_PRI_LO];
7348 		}
7349 
7350 		timeout = pc_thread_longwait;
7351 		if (high_pri_pages != 0) {
7352 			timeout = pc_thread_shortwait;
7353 			page_capture_handle_outstanding();
7354 		} else if (low_pri_pages != 0) {
7355 			page_capture_async();
7356 		}
7357 		CALLB_CPR_SAFE_BEGIN(&c);
7358 		(void) cv_reltimedwait(&pc_cv, &pc_thread_mutex,
7359 		    timeout, TR_CLOCK_TICK);
7360 		CALLB_CPR_SAFE_END(&c, &pc_thread_mutex);
7361 	}
7362 	/*NOTREACHED*/
7363 }
7364 /*
7365  * Attempt to locate a bucket that has enough pages to satisfy the request.
7366  * The initial check is done without the lock to avoid unneeded contention.
7367  * The function returns 1 if enough pages were found, else 0 if it could not
7368  * find enough pages in a bucket.
7369  */
7370 static int
7371 pcf_decrement_bucket(pgcnt_t npages)
7372 {
7373 	struct pcf	*p;
7374 	struct pcf	*q;
7375 	int i;
7376 
7377 	p = &pcf[PCF_INDEX()];
7378 	q = &pcf[pcf_fanout];
7379 	for (i = 0; i < pcf_fanout; i++) {
7380 		if (p->pcf_count > npages) {
7381 			/*
7382 			 * a good one to try.
7383 			 */
7384 			mutex_enter(&p->pcf_lock);
7385 			if (p->pcf_count > npages) {
7386 				p->pcf_count -= (uint_t)npages;
7387 				/*
7388 				 * freemem is not protected by any lock.
7389 				 * Thus, we cannot have any assertion
7390 				 * containing freemem here.
7391 				 */
7392 				freemem -= npages;
7393 				mutex_exit(&p->pcf_lock);
7394 				return (1);
7395 			}
7396 			mutex_exit(&p->pcf_lock);
7397 		}
7398 		p++;
7399 		if (p >= q) {
7400 			p = pcf;
7401 		}
7402 	}
7403 	return (0);
7404 }
7405 
7406 /*
7407  * Arguments:
7408  *	pcftotal_ret:	If the value is not NULL and we have walked all the
7409  *			buckets but did not find enough pages then it will
7410  *			be set to the total number of pages in all the pcf
7411  *			buckets.
7412  *	npages:		Is the number of pages we have been requested to
7413  *			find.
7414  *	unlock:		If set to 0 we will leave the buckets locked if the
7415  *			requested number of pages are not found.
7416  *
7417  * Go and try to satisfy the page request  from any number of buckets.
7418  * This can be a very expensive operation as we have to lock the buckets
7419  * we are checking (and keep them locked), starting at bucket 0.
7420  *
7421  * The function returns 1 if enough pages were found, else 0 if it could not
7422  * find enough pages in the buckets.
7423  *
7424  */
7425 static int
7426 pcf_decrement_multiple(pgcnt_t *pcftotal_ret, pgcnt_t npages, int unlock)
7427 {
7428 	struct pcf	*p;
7429 	pgcnt_t pcftotal;
7430 	int i;
7431 
7432 	p = pcf;
7433 	/* try to collect pages from several pcf bins */
7434 	for (pcftotal = 0, i = 0; i < pcf_fanout; i++) {
7435 		mutex_enter(&p->pcf_lock);
7436 		pcftotal += p->pcf_count;
7437 		if (pcftotal >= npages) {
7438 			/*
7439 			 * Wow!  There are enough pages laying around
7440 			 * to satisfy the request.  Do the accounting,
7441 			 * drop the locks we acquired, and go back.
7442 			 *
7443 			 * freemem is not protected by any lock. So,
7444 			 * we cannot have any assertion containing
7445 			 * freemem.
7446 			 */
7447 			freemem -= npages;
7448 			while (p >= pcf) {
7449 				if (p->pcf_count <= npages) {
7450 					npages -= p->pcf_count;
7451 					p->pcf_count = 0;
7452 				} else {
7453 					p->pcf_count -= (uint_t)npages;
7454 					npages = 0;
7455 				}
7456 				mutex_exit(&p->pcf_lock);
7457 				p--;
7458 			}
7459 			ASSERT(npages == 0);
7460 			return (1);
7461 		}
7462 		p++;
7463 	}
7464 	if (unlock) {
7465 		/* failed to collect pages - release the locks */
7466 		while (--p >= pcf) {
7467 			mutex_exit(&p->pcf_lock);
7468 		}
7469 	}
7470 	if (pcftotal_ret != NULL)
7471 		*pcftotal_ret = pcftotal;
7472 	return (0);
7473 }
7474