xref: /titanic_51/usr/src/uts/common/vm/vm_page.c (revision 06e1a7147edd272b7296f208141627a5b1191731)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989  AT&T	*/
27 /*	  All Rights Reserved  	*/
28 
29 /*
30  * University Copyright- Copyright (c) 1982, 1986, 1988
31  * The Regents of the University of California
32  * All Rights Reserved
33  *
34  * University Acknowledgment- Portions of this document are derived from
35  * software developed by the University of California, Berkeley, and its
36  * contributors.
37  */
38 
39 #pragma ident	"%Z%%M%	%I%	%E% SMI"
40 
41 /*
42  * VM - physical page management.
43  */
44 
45 #include <sys/types.h>
46 #include <sys/t_lock.h>
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/errno.h>
50 #include <sys/time.h>
51 #include <sys/vnode.h>
52 #include <sys/vm.h>
53 #include <sys/vtrace.h>
54 #include <sys/swap.h>
55 #include <sys/cmn_err.h>
56 #include <sys/tuneable.h>
57 #include <sys/sysmacros.h>
58 #include <sys/cpuvar.h>
59 #include <sys/callb.h>
60 #include <sys/debug.h>
61 #include <sys/tnf_probe.h>
62 #include <sys/condvar_impl.h>
63 #include <sys/mem_config.h>
64 #include <sys/mem_cage.h>
65 #include <sys/kmem.h>
66 #include <sys/atomic.h>
67 #include <sys/strlog.h>
68 #include <sys/mman.h>
69 #include <sys/ontrap.h>
70 #include <sys/lgrp.h>
71 #include <sys/vfs.h>
72 
73 #include <vm/hat.h>
74 #include <vm/anon.h>
75 #include <vm/page.h>
76 #include <vm/seg.h>
77 #include <vm/pvn.h>
78 #include <vm/seg_kmem.h>
79 #include <vm/vm_dep.h>
80 
81 #include <fs/fs_subr.h>
82 
83 static int nopageage = 0;
84 
85 static pgcnt_t max_page_get;	/* max page_get request size in pages */
86 pgcnt_t total_pages = 0;	/* total number of pages (used by /proc) */
87 
88 /*
89  * freemem_lock protects all freemem variables:
90  * availrmem. Also this lock protects the globals which track the
91  * availrmem changes for accurate kernel footprint calculation.
92  * See below for an explanation of these
93  * globals.
94  */
95 kmutex_t freemem_lock;
96 pgcnt_t availrmem;
97 pgcnt_t availrmem_initial;
98 
99 /*
100  * These globals track availrmem changes to get a more accurate
101  * estimate of tke kernel size. Historically pp_kernel is used for
102  * kernel size and is based on availrmem. But availrmem is adjusted for
103  * locked pages in the system not just for kernel locked pages.
104  * These new counters will track the pages locked through segvn and
105  * by explicit user locking.
106  *
107  * segvn_pages_locked : This keeps track on a global basis how many pages
108  * are currently locked because of I/O.
109  *
110  * pages_locked : How many pages are locked becuase of user specified
111  * locking through mlock or plock.
112  *
113  * pages_useclaim,pages_claimed : These two variables track the
114  * cliam adjustments because of the protection changes on a segvn segment.
115  *
116  * All these globals are protected by the same lock which protects availrmem.
117  */
118 pgcnt_t segvn_pages_locked;
119 pgcnt_t pages_locked;
120 pgcnt_t pages_useclaim;
121 pgcnt_t pages_claimed;
122 
123 
124 /*
125  * new_freemem_lock protects freemem, freemem_wait & freemem_cv.
126  */
127 static kmutex_t	new_freemem_lock;
128 static uint_t	freemem_wait;	/* someone waiting for freemem */
129 static kcondvar_t freemem_cv;
130 
131 /*
132  * The logical page free list is maintained as two lists, the 'free'
133  * and the 'cache' lists.
134  * The free list contains those pages that should be reused first.
135  *
136  * The implementation of the lists is machine dependent.
137  * page_get_freelist(), page_get_cachelist(),
138  * page_list_sub(), and page_list_add()
139  * form the interface to the machine dependent implementation.
140  *
141  * Pages with p_free set are on the cache list.
142  * Pages with p_free and p_age set are on the free list,
143  *
144  * A page may be locked while on either list.
145  */
146 
147 /*
148  * free list accounting stuff.
149  *
150  *
151  * Spread out the value for the number of pages on the
152  * page free and page cache lists.  If there is just one
153  * value, then it must be under just one lock.
154  * The lock contention and cache traffic are a real bother.
155  *
156  * When we acquire and then drop a single pcf lock
157  * we can start in the middle of the array of pcf structures.
158  * If we acquire more than one pcf lock at a time, we need to
159  * start at the front to avoid deadlocking.
160  *
161  * pcf_count holds the number of pages in each pool.
162  *
163  * pcf_block is set when page_create_get_something() has asked the
164  * PSM page freelist and page cachelist routines without specifying
165  * a color and nothing came back.  This is used to block anything
166  * else from moving pages from one list to the other while the
167  * lists are searched again.  If a page is freeed while pcf_block is
168  * set, then pcf_reserve is incremented.  pcgs_unblock() takes care
169  * of clearning pcf_block, doing the wakeups, etc.
170  */
171 
172 #if NCPU <= 4
173 #define	PAD	2
174 #define	PCF_FANOUT	4
175 static	uint_t	pcf_mask = PCF_FANOUT - 1;
176 #else
177 #define	PAD	10
178 #ifdef sun4v
179 #define	PCF_FANOUT	32
180 #else
181 #define	PCF_FANOUT	128
182 #endif
183 static	uint_t	pcf_mask = PCF_FANOUT - 1;
184 #endif
185 
186 struct pcf {
187 	kmutex_t	pcf_lock;	/* protects the structure */
188 	uint_t		pcf_count;	/* page count */
189 	uint_t		pcf_wait;	/* number of waiters */
190 	uint_t		pcf_block; 	/* pcgs flag to page_free() */
191 	uint_t		pcf_reserve; 	/* pages freed after pcf_block set */
192 	uint_t		pcf_fill[PAD];	/* to line up on the caches */
193 };
194 
195 static struct	pcf	pcf[PCF_FANOUT];
196 #define	PCF_INDEX()	((CPU->cpu_id) & (pcf_mask))
197 
198 kmutex_t	pcgs_lock;		/* serializes page_create_get_ */
199 kmutex_t	pcgs_cagelock;		/* serializes NOSLEEP cage allocs */
200 kmutex_t	pcgs_wait_lock;		/* used for delay in pcgs */
201 static kcondvar_t	pcgs_cv;	/* cv for delay in pcgs */
202 
203 #ifdef VM_STATS
204 
205 /*
206  * No locks, but so what, they are only statistics.
207  */
208 
209 static struct page_tcnt {
210 	int	pc_free_cache;		/* free's into cache list */
211 	int	pc_free_dontneed;	/* free's with dontneed */
212 	int	pc_free_pageout;	/* free's from pageout */
213 	int	pc_free_free;		/* free's into free list */
214 	int	pc_free_pages;		/* free's into large page free list */
215 	int	pc_destroy_pages;	/* large page destroy's */
216 	int	pc_get_cache;		/* get's from cache list */
217 	int	pc_get_free;		/* get's from free list */
218 	int	pc_reclaim;		/* reclaim's */
219 	int	pc_abortfree;		/* abort's of free pages */
220 	int	pc_find_hit;		/* find's that find page */
221 	int	pc_find_miss;		/* find's that don't find page */
222 	int	pc_destroy_free;	/* # of free pages destroyed */
223 #define	PC_HASH_CNT	(4*PAGE_HASHAVELEN)
224 	int	pc_find_hashlen[PC_HASH_CNT+1];
225 	int	pc_addclaim_pages;
226 	int	pc_subclaim_pages;
227 	int	pc_free_replacement_page[2];
228 	int	pc_try_demote_pages[6];
229 	int	pc_demote_pages[2];
230 } pagecnt;
231 
232 uint_t	hashin_count;
233 uint_t	hashin_not_held;
234 uint_t	hashin_already;
235 
236 uint_t	hashout_count;
237 uint_t	hashout_not_held;
238 
239 uint_t	page_create_count;
240 uint_t	page_create_not_enough;
241 uint_t	page_create_not_enough_again;
242 uint_t	page_create_zero;
243 uint_t	page_create_hashout;
244 uint_t	page_create_page_lock_failed;
245 uint_t	page_create_trylock_failed;
246 uint_t	page_create_found_one;
247 uint_t	page_create_hashin_failed;
248 uint_t	page_create_dropped_phm;
249 
250 uint_t	page_create_new;
251 uint_t	page_create_exists;
252 uint_t	page_create_putbacks;
253 uint_t	page_create_overshoot;
254 
255 uint_t	page_reclaim_zero;
256 uint_t	page_reclaim_zero_locked;
257 
258 uint_t	page_rename_exists;
259 uint_t	page_rename_count;
260 
261 uint_t	page_lookup_cnt[20];
262 uint_t	page_lookup_nowait_cnt[10];
263 uint_t	page_find_cnt;
264 uint_t	page_exists_cnt;
265 uint_t	page_exists_forreal_cnt;
266 uint_t	page_lookup_dev_cnt;
267 uint_t	get_cachelist_cnt;
268 uint_t	page_create_cnt[10];
269 uint_t	alloc_pages[8];
270 uint_t	page_exphcontg[19];
271 uint_t  page_create_large_cnt[10];
272 
273 /*
274  * Collects statistics.
275  */
276 #define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
277 	uint_t	mylen = 0; \
278 			\
279 	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \
280 		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
281 			break; \
282 	} \
283 	if ((pp) != NULL) \
284 		pagecnt.pc_find_hit++; \
285 	else \
286 		pagecnt.pc_find_miss++; \
287 	if (mylen > PC_HASH_CNT) \
288 		mylen = PC_HASH_CNT; \
289 	pagecnt.pc_find_hashlen[mylen]++; \
290 }
291 
292 #else	/* VM_STATS */
293 
294 /*
295  * Don't collect statistics
296  */
297 #define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
298 	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
299 		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
300 			break; \
301 	} \
302 }
303 
304 #endif	/* VM_STATS */
305 
306 
307 
308 #ifdef DEBUG
309 #define	MEMSEG_SEARCH_STATS
310 #endif
311 
312 #ifdef MEMSEG_SEARCH_STATS
313 struct memseg_stats {
314     uint_t nsearch;
315     uint_t nlastwon;
316     uint_t nhashwon;
317     uint_t nnotfound;
318 } memseg_stats;
319 
320 #define	MEMSEG_STAT_INCR(v) \
321 	atomic_add_32(&memseg_stats.v, 1)
322 #else
323 #define	MEMSEG_STAT_INCR(x)
324 #endif
325 
326 struct memseg *memsegs;		/* list of memory segments */
327 
328 
329 static void page_init_mem_config(void);
330 static int page_do_hashin(page_t *, vnode_t *, u_offset_t);
331 static void page_do_hashout(page_t *);
332 
333 static void page_demote_vp_pages(page_t *);
334 
335 /*
336  * vm subsystem related initialization
337  */
338 void
339 vm_init(void)
340 {
341 	boolean_t callb_vm_cpr(void *, int);
342 
343 	(void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm");
344 	page_init_mem_config();
345 	page_retire_init();
346 }
347 
348 /*
349  * This function is called at startup and when memory is added or deleted.
350  */
351 void
352 init_pages_pp_maximum()
353 {
354 	static pgcnt_t p_min;
355 	static pgcnt_t pages_pp_maximum_startup;
356 	static pgcnt_t avrmem_delta;
357 	static int init_done;
358 	static int user_set;	/* true if set in /etc/system */
359 
360 	if (init_done == 0) {
361 
362 		/* If the user specified a value, save it */
363 		if (pages_pp_maximum != 0) {
364 			user_set = 1;
365 			pages_pp_maximum_startup = pages_pp_maximum;
366 		}
367 
368 		/*
369 		 * Setting of pages_pp_maximum is based first time
370 		 * on the value of availrmem just after the start-up
371 		 * allocations. To preserve this relationship at run
372 		 * time, use a delta from availrmem_initial.
373 		 */
374 		ASSERT(availrmem_initial >= availrmem);
375 		avrmem_delta = availrmem_initial - availrmem;
376 
377 		/* The allowable floor of pages_pp_maximum */
378 		p_min = tune.t_minarmem + 100;
379 
380 		/* Make sure we don't come through here again. */
381 		init_done = 1;
382 	}
383 	/*
384 	 * Determine pages_pp_maximum, the number of currently available
385 	 * pages (availrmem) that can't be `locked'. If not set by
386 	 * the user, we set it to 4% of the currently available memory
387 	 * plus 4MB.
388 	 * But we also insist that it be greater than tune.t_minarmem;
389 	 * otherwise a process could lock down a lot of memory, get swapped
390 	 * out, and never have enough to get swapped back in.
391 	 */
392 	if (user_set)
393 		pages_pp_maximum = pages_pp_maximum_startup;
394 	else
395 		pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25)
396 		    + btop(4 * 1024 * 1024);
397 
398 	if (pages_pp_maximum <= p_min) {
399 		pages_pp_maximum = p_min;
400 	}
401 }
402 
403 void
404 set_max_page_get(pgcnt_t target_total_pages)
405 {
406 	max_page_get = target_total_pages / 2;
407 }
408 
409 static pgcnt_t pending_delete;
410 
411 /*ARGSUSED*/
412 static void
413 page_mem_config_post_add(
414 	void *arg,
415 	pgcnt_t delta_pages)
416 {
417 	set_max_page_get(total_pages - pending_delete);
418 	init_pages_pp_maximum();
419 }
420 
421 /*ARGSUSED*/
422 static int
423 page_mem_config_pre_del(
424 	void *arg,
425 	pgcnt_t delta_pages)
426 {
427 	pgcnt_t nv;
428 
429 	nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages);
430 	set_max_page_get(total_pages - nv);
431 	return (0);
432 }
433 
434 /*ARGSUSED*/
435 static void
436 page_mem_config_post_del(
437 	void *arg,
438 	pgcnt_t delta_pages,
439 	int cancelled)
440 {
441 	pgcnt_t nv;
442 
443 	nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages);
444 	set_max_page_get(total_pages - nv);
445 	if (!cancelled)
446 		init_pages_pp_maximum();
447 }
448 
449 static kphysm_setup_vector_t page_mem_config_vec = {
450 	KPHYSM_SETUP_VECTOR_VERSION,
451 	page_mem_config_post_add,
452 	page_mem_config_pre_del,
453 	page_mem_config_post_del,
454 };
455 
456 static void
457 page_init_mem_config(void)
458 {
459 	int ret;
460 
461 	ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL);
462 	ASSERT(ret == 0);
463 }
464 
465 /*
466  * Evenly spread out the PCF counters for large free pages
467  */
468 static void
469 page_free_large_ctr(pgcnt_t npages)
470 {
471 	static struct pcf	*p = pcf;
472 	pgcnt_t			lump;
473 
474 	freemem += npages;
475 
476 	lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT;
477 
478 	while (npages > 0) {
479 
480 		ASSERT(!p->pcf_block);
481 
482 		if (lump < npages) {
483 			p->pcf_count += (uint_t)lump;
484 			npages -= lump;
485 		} else {
486 			p->pcf_count += (uint_t)npages;
487 			npages = 0;
488 		}
489 
490 		ASSERT(!p->pcf_wait);
491 
492 		if (++p > &pcf[PCF_FANOUT - 1])
493 			p = pcf;
494 	}
495 
496 	ASSERT(npages == 0);
497 }
498 
499 /*
500  * Add a physical chunk of memory to the system freee lists during startup.
501  * Platform specific startup() allocates the memory for the page structs.
502  *
503  * num	- number of page structures
504  * base - page number (pfn) to be associated with the first page.
505  *
506  * Since we are doing this during startup (ie. single threaded), we will
507  * use shortcut routines to avoid any locking overhead while putting all
508  * these pages on the freelists.
509  *
510  * NOTE: Any changes performed to page_free(), must also be performed to
511  *	 add_physmem() since this is how we initialize all page_t's at
512  *	 boot time.
513  */
514 void
515 add_physmem(
516 	page_t	*pp,
517 	pgcnt_t	num,
518 	pfn_t	pnum)
519 {
520 	page_t	*root = NULL;
521 	uint_t	szc = page_num_pagesizes() - 1;
522 	pgcnt_t	large = page_get_pagecnt(szc);
523 	pgcnt_t	cnt = 0;
524 
525 	TRACE_2(TR_FAC_VM, TR_PAGE_INIT,
526 		"add_physmem:pp %p num %lu", pp, num);
527 
528 	/*
529 	 * Arbitrarily limit the max page_get request
530 	 * to 1/2 of the page structs we have.
531 	 */
532 	total_pages += num;
533 	set_max_page_get(total_pages);
534 
535 	PLCNT_MODIFY_MAX(pnum, (long)num);
536 
537 	/*
538 	 * The physical space for the pages array
539 	 * representing ram pages has already been
540 	 * allocated.  Here we initialize each lock
541 	 * in the page structure, and put each on
542 	 * the free list
543 	 */
544 	for (; num; pp++, pnum++, num--) {
545 
546 		/*
547 		 * this needs to fill in the page number
548 		 * and do any other arch specific initialization
549 		 */
550 		add_physmem_cb(pp, pnum);
551 
552 		pp->p_lckcnt = 0;
553 		pp->p_cowcnt = 0;
554 		pp->p_slckcnt = 0;
555 
556 		/*
557 		 * Initialize the page lock as unlocked, since nobody
558 		 * can see or access this page yet.
559 		 */
560 		pp->p_selock = 0;
561 
562 		/*
563 		 * Initialize IO lock
564 		 */
565 		page_iolock_init(pp);
566 
567 		/*
568 		 * initialize other fields in the page_t
569 		 */
570 		PP_SETFREE(pp);
571 		page_clr_all_props(pp);
572 		PP_SETAGED(pp);
573 		pp->p_offset = (u_offset_t)-1;
574 		pp->p_next = pp;
575 		pp->p_prev = pp;
576 
577 		/*
578 		 * Simple case: System doesn't support large pages.
579 		 */
580 		if (szc == 0) {
581 			pp->p_szc = 0;
582 			page_free_at_startup(pp);
583 			continue;
584 		}
585 
586 		/*
587 		 * Handle unaligned pages, we collect them up onto
588 		 * the root page until we have a full large page.
589 		 */
590 		if (!IS_P2ALIGNED(pnum, large)) {
591 
592 			/*
593 			 * If not in a large page,
594 			 * just free as small page.
595 			 */
596 			if (root == NULL) {
597 				pp->p_szc = 0;
598 				page_free_at_startup(pp);
599 				continue;
600 			}
601 
602 			/*
603 			 * Link a constituent page into the large page.
604 			 */
605 			pp->p_szc = szc;
606 			page_list_concat(&root, &pp);
607 
608 			/*
609 			 * When large page is fully formed, free it.
610 			 */
611 			if (++cnt == large) {
612 				page_free_large_ctr(cnt);
613 				page_list_add_pages(root, PG_LIST_ISINIT);
614 				root = NULL;
615 				cnt = 0;
616 			}
617 			continue;
618 		}
619 
620 		/*
621 		 * At this point we have a page number which
622 		 * is aligned. We assert that we aren't already
623 		 * in a different large page.
624 		 */
625 		ASSERT(IS_P2ALIGNED(pnum, large));
626 		ASSERT(root == NULL && cnt == 0);
627 
628 		/*
629 		 * If insufficient number of pages left to form
630 		 * a large page, just free the small page.
631 		 */
632 		if (num < large) {
633 			pp->p_szc = 0;
634 			page_free_at_startup(pp);
635 			continue;
636 		}
637 
638 		/*
639 		 * Otherwise start a new large page.
640 		 */
641 		pp->p_szc = szc;
642 		cnt++;
643 		root = pp;
644 	}
645 	ASSERT(root == NULL && cnt == 0);
646 }
647 
648 /*
649  * Find a page representing the specified [vp, offset].
650  * If we find the page but it is intransit coming in,
651  * it will have an "exclusive" lock and we wait for
652  * the i/o to complete.  A page found on the free list
653  * is always reclaimed and then locked.  On success, the page
654  * is locked, its data is valid and it isn't on the free
655  * list, while a NULL is returned if the page doesn't exist.
656  */
657 page_t *
658 page_lookup(vnode_t *vp, u_offset_t off, se_t se)
659 {
660 	return (page_lookup_create(vp, off, se, NULL, NULL, 0));
661 }
662 
663 /*
664  * Find a page representing the specified [vp, offset].
665  * We either return the one we found or, if passed in,
666  * create one with identity of [vp, offset] of the
667  * pre-allocated page. If we find exsisting page but it is
668  * intransit coming in, it will have an "exclusive" lock
669  * and we wait for the i/o to complete.  A page found on
670  * the free list is always reclaimed and then locked.
671  * On success, the page is locked, its data is valid and
672  * it isn't on the free list, while a NULL is returned
673  * if the page doesn't exist and newpp is NULL;
674  */
675 page_t *
676 page_lookup_create(
677 	vnode_t *vp,
678 	u_offset_t off,
679 	se_t se,
680 	page_t *newpp,
681 	spgcnt_t *nrelocp,
682 	int flags)
683 {
684 	page_t		*pp;
685 	kmutex_t	*phm;
686 	ulong_t		index;
687 	uint_t		hash_locked;
688 	uint_t		es;
689 
690 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
691 	VM_STAT_ADD(page_lookup_cnt[0]);
692 	ASSERT(newpp ? PAGE_EXCL(newpp) : 1);
693 
694 	/*
695 	 * Acquire the appropriate page hash lock since
696 	 * we have to search the hash list.  Pages that
697 	 * hash to this list can't change identity while
698 	 * this lock is held.
699 	 */
700 	hash_locked = 0;
701 	index = PAGE_HASH_FUNC(vp, off);
702 	phm = NULL;
703 top:
704 	PAGE_HASH_SEARCH(index, pp, vp, off);
705 	if (pp != NULL) {
706 		VM_STAT_ADD(page_lookup_cnt[1]);
707 		es = (newpp != NULL) ? 1 : 0;
708 		es |= flags;
709 		if (!hash_locked) {
710 			VM_STAT_ADD(page_lookup_cnt[2]);
711 			if (!page_try_reclaim_lock(pp, se, es)) {
712 				/*
713 				 * On a miss, acquire the phm.  Then
714 				 * next time, page_lock() will be called,
715 				 * causing a wait if the page is busy.
716 				 * just looping with page_trylock() would
717 				 * get pretty boring.
718 				 */
719 				VM_STAT_ADD(page_lookup_cnt[3]);
720 				phm = PAGE_HASH_MUTEX(index);
721 				mutex_enter(phm);
722 				hash_locked = 1;
723 				goto top;
724 			}
725 		} else {
726 			VM_STAT_ADD(page_lookup_cnt[4]);
727 			if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) {
728 				VM_STAT_ADD(page_lookup_cnt[5]);
729 				goto top;
730 			}
731 		}
732 
733 		/*
734 		 * Since `pp' is locked it can not change identity now.
735 		 * Reconfirm we locked the correct page.
736 		 *
737 		 * Both the p_vnode and p_offset *must* be cast volatile
738 		 * to force a reload of their values: The PAGE_HASH_SEARCH
739 		 * macro will have stuffed p_vnode and p_offset into
740 		 * registers before calling page_trylock(); another thread,
741 		 * actually holding the hash lock, could have changed the
742 		 * page's identity in memory, but our registers would not
743 		 * be changed, fooling the reconfirmation.  If the hash
744 		 * lock was held during the search, the casting would
745 		 * not be needed.
746 		 */
747 		VM_STAT_ADD(page_lookup_cnt[6]);
748 		if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
749 		    ((volatile u_offset_t)(pp->p_offset) != off)) {
750 			VM_STAT_ADD(page_lookup_cnt[7]);
751 			if (hash_locked) {
752 				panic("page_lookup_create: lost page %p",
753 				    (void *)pp);
754 				/*NOTREACHED*/
755 			}
756 			page_unlock(pp);
757 			phm = PAGE_HASH_MUTEX(index);
758 			mutex_enter(phm);
759 			hash_locked = 1;
760 			goto top;
761 		}
762 
763 		/*
764 		 * If page_trylock() was called, then pp may still be on
765 		 * the cachelist (can't be on the free list, it would not
766 		 * have been found in the search).  If it is on the
767 		 * cachelist it must be pulled now. To pull the page from
768 		 * the cachelist, it must be exclusively locked.
769 		 *
770 		 * The other big difference between page_trylock() and
771 		 * page_lock(), is that page_lock() will pull the
772 		 * page from whatever free list (the cache list in this
773 		 * case) the page is on.  If page_trylock() was used
774 		 * above, then we have to do the reclaim ourselves.
775 		 */
776 		if ((!hash_locked) && (PP_ISFREE(pp))) {
777 			ASSERT(PP_ISAGED(pp) == 0);
778 			VM_STAT_ADD(page_lookup_cnt[8]);
779 
780 			/*
781 			 * page_relcaim will insure that we
782 			 * have this page exclusively
783 			 */
784 
785 			if (!page_reclaim(pp, NULL)) {
786 				/*
787 				 * Page_reclaim dropped whatever lock
788 				 * we held.
789 				 */
790 				VM_STAT_ADD(page_lookup_cnt[9]);
791 				phm = PAGE_HASH_MUTEX(index);
792 				mutex_enter(phm);
793 				hash_locked = 1;
794 				goto top;
795 			} else if (se == SE_SHARED && newpp == NULL) {
796 				VM_STAT_ADD(page_lookup_cnt[10]);
797 				page_downgrade(pp);
798 			}
799 		}
800 
801 		if (hash_locked) {
802 			mutex_exit(phm);
803 		}
804 
805 		if (newpp != NULL && pp->p_szc < newpp->p_szc &&
806 		    PAGE_EXCL(pp) && nrelocp != NULL) {
807 			ASSERT(nrelocp != NULL);
808 			(void) page_relocate(&pp, &newpp, 1, 1, nrelocp,
809 			    NULL);
810 			if (*nrelocp > 0) {
811 				VM_STAT_COND_ADD(*nrelocp == 1,
812 				    page_lookup_cnt[11]);
813 				VM_STAT_COND_ADD(*nrelocp > 1,
814 				    page_lookup_cnt[12]);
815 				pp = newpp;
816 				se = SE_EXCL;
817 			} else {
818 				if (se == SE_SHARED) {
819 					page_downgrade(pp);
820 				}
821 				VM_STAT_ADD(page_lookup_cnt[13]);
822 			}
823 		} else if (newpp != NULL && nrelocp != NULL) {
824 			if (PAGE_EXCL(pp) && se == SE_SHARED) {
825 				page_downgrade(pp);
826 			}
827 			VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc,
828 			    page_lookup_cnt[14]);
829 			VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc,
830 			    page_lookup_cnt[15]);
831 			VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc,
832 			    page_lookup_cnt[16]);
833 		} else if (newpp != NULL && PAGE_EXCL(pp)) {
834 			se = SE_EXCL;
835 		}
836 	} else if (!hash_locked) {
837 		VM_STAT_ADD(page_lookup_cnt[17]);
838 		phm = PAGE_HASH_MUTEX(index);
839 		mutex_enter(phm);
840 		hash_locked = 1;
841 		goto top;
842 	} else if (newpp != NULL) {
843 		/*
844 		 * If we have a preallocated page then
845 		 * insert it now and basically behave like
846 		 * page_create.
847 		 */
848 		VM_STAT_ADD(page_lookup_cnt[18]);
849 		/*
850 		 * Since we hold the page hash mutex and
851 		 * just searched for this page, page_hashin
852 		 * had better not fail.  If it does, that
853 		 * means some thread did not follow the
854 		 * page hash mutex rules.  Panic now and
855 		 * get it over with.  As usual, go down
856 		 * holding all the locks.
857 		 */
858 		ASSERT(MUTEX_HELD(phm));
859 		if (!page_hashin(newpp, vp, off, phm)) {
860 			ASSERT(MUTEX_HELD(phm));
861 			panic("page_lookup_create: hashin failed %p %p %llx %p",
862 			    (void *)newpp, (void *)vp, off, (void *)phm);
863 			/*NOTREACHED*/
864 		}
865 		ASSERT(MUTEX_HELD(phm));
866 		mutex_exit(phm);
867 		phm = NULL;
868 		page_set_props(newpp, P_REF);
869 		page_io_lock(newpp);
870 		pp = newpp;
871 		se = SE_EXCL;
872 	} else {
873 		VM_STAT_ADD(page_lookup_cnt[19]);
874 		mutex_exit(phm);
875 	}
876 
877 	ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
878 
879 	ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1);
880 
881 	return (pp);
882 }
883 
884 /*
885  * Search the hash list for the page representing the
886  * specified [vp, offset] and return it locked.  Skip
887  * free pages and pages that cannot be locked as requested.
888  * Used while attempting to kluster pages.
889  */
890 page_t *
891 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se)
892 {
893 	page_t		*pp;
894 	kmutex_t	*phm;
895 	ulong_t		index;
896 	uint_t		locked;
897 
898 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
899 	VM_STAT_ADD(page_lookup_nowait_cnt[0]);
900 
901 	index = PAGE_HASH_FUNC(vp, off);
902 	PAGE_HASH_SEARCH(index, pp, vp, off);
903 	locked = 0;
904 	if (pp == NULL) {
905 top:
906 		VM_STAT_ADD(page_lookup_nowait_cnt[1]);
907 		locked = 1;
908 		phm = PAGE_HASH_MUTEX(index);
909 		mutex_enter(phm);
910 		PAGE_HASH_SEARCH(index, pp, vp, off);
911 	}
912 
913 	if (pp == NULL || PP_ISFREE(pp)) {
914 		VM_STAT_ADD(page_lookup_nowait_cnt[2]);
915 		pp = NULL;
916 	} else {
917 		if (!page_trylock(pp, se)) {
918 			VM_STAT_ADD(page_lookup_nowait_cnt[3]);
919 			pp = NULL;
920 		} else {
921 			VM_STAT_ADD(page_lookup_nowait_cnt[4]);
922 			/*
923 			 * See the comment in page_lookup()
924 			 */
925 			if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
926 			    ((u_offset_t)(pp->p_offset) != off)) {
927 				VM_STAT_ADD(page_lookup_nowait_cnt[5]);
928 				if (locked) {
929 					panic("page_lookup_nowait %p",
930 					    (void *)pp);
931 					/*NOTREACHED*/
932 				}
933 				page_unlock(pp);
934 				goto top;
935 			}
936 			if (PP_ISFREE(pp)) {
937 				VM_STAT_ADD(page_lookup_nowait_cnt[6]);
938 				page_unlock(pp);
939 				pp = NULL;
940 			}
941 		}
942 	}
943 	if (locked) {
944 		VM_STAT_ADD(page_lookup_nowait_cnt[7]);
945 		mutex_exit(phm);
946 	}
947 
948 	ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
949 
950 	return (pp);
951 }
952 
953 /*
954  * Search the hash list for a page with the specified [vp, off]
955  * that is known to exist and is already locked.  This routine
956  * is typically used by segment SOFTUNLOCK routines.
957  */
958 page_t *
959 page_find(vnode_t *vp, u_offset_t off)
960 {
961 	page_t		*pp;
962 	kmutex_t	*phm;
963 	ulong_t		index;
964 
965 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
966 	VM_STAT_ADD(page_find_cnt);
967 
968 	index = PAGE_HASH_FUNC(vp, off);
969 	phm = PAGE_HASH_MUTEX(index);
970 
971 	mutex_enter(phm);
972 	PAGE_HASH_SEARCH(index, pp, vp, off);
973 	mutex_exit(phm);
974 
975 	ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr);
976 	return (pp);
977 }
978 
979 /*
980  * Determine whether a page with the specified [vp, off]
981  * currently exists in the system.  Obviously this should
982  * only be considered as a hint since nothing prevents the
983  * page from disappearing or appearing immediately after
984  * the return from this routine. Subsequently, we don't
985  * even bother to lock the list.
986  */
987 page_t *
988 page_exists(vnode_t *vp, u_offset_t off)
989 {
990 	page_t	*pp;
991 	ulong_t		index;
992 
993 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
994 	VM_STAT_ADD(page_exists_cnt);
995 
996 	index = PAGE_HASH_FUNC(vp, off);
997 	PAGE_HASH_SEARCH(index, pp, vp, off);
998 
999 	return (pp);
1000 }
1001 
1002 /*
1003  * Determine if physically contiguous pages exist for [vp, off] - [vp, off +
1004  * page_size(szc)) range.  if they exist and ppa is not NULL fill ppa array
1005  * with these pages locked SHARED. If necessary reclaim pages from
1006  * freelist. Return 1 if contiguous pages exist and 0 otherwise.
1007  *
1008  * If we fail to lock pages still return 1 if pages exist and contiguous.
1009  * But in this case return value is just a hint. ppa array won't be filled.
1010  * Caller should initialize ppa[0] as NULL to distinguish return value.
1011  *
1012  * Returns 0 if pages don't exist or not physically contiguous.
1013  *
1014  * This routine doesn't work for anonymous(swapfs) pages.
1015  */
1016 int
1017 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[])
1018 {
1019 	pgcnt_t pages;
1020 	pfn_t pfn;
1021 	page_t *rootpp;
1022 	pgcnt_t i;
1023 	pgcnt_t j;
1024 	u_offset_t save_off = off;
1025 	ulong_t index;
1026 	kmutex_t *phm;
1027 	page_t *pp;
1028 	uint_t pszc;
1029 	int loopcnt = 0;
1030 
1031 	ASSERT(szc != 0);
1032 	ASSERT(vp != NULL);
1033 	ASSERT(!IS_SWAPFSVP(vp));
1034 	ASSERT(vp != &kvp);
1035 
1036 again:
1037 	if (++loopcnt > 3) {
1038 		VM_STAT_ADD(page_exphcontg[0]);
1039 		return (0);
1040 	}
1041 
1042 	index = PAGE_HASH_FUNC(vp, off);
1043 	phm = PAGE_HASH_MUTEX(index);
1044 
1045 	mutex_enter(phm);
1046 	PAGE_HASH_SEARCH(index, pp, vp, off);
1047 	mutex_exit(phm);
1048 
1049 	VM_STAT_ADD(page_exphcontg[1]);
1050 
1051 	if (pp == NULL) {
1052 		VM_STAT_ADD(page_exphcontg[2]);
1053 		return (0);
1054 	}
1055 
1056 	pages = page_get_pagecnt(szc);
1057 	rootpp = pp;
1058 	pfn = rootpp->p_pagenum;
1059 
1060 	if ((pszc = pp->p_szc) >= szc && ppa != NULL) {
1061 		VM_STAT_ADD(page_exphcontg[3]);
1062 		if (!page_trylock(pp, SE_SHARED)) {
1063 			VM_STAT_ADD(page_exphcontg[4]);
1064 			return (1);
1065 		}
1066 		if (pp->p_szc != pszc || pp->p_vnode != vp ||
1067 		    pp->p_offset != off) {
1068 			VM_STAT_ADD(page_exphcontg[5]);
1069 			page_unlock(pp);
1070 			off = save_off;
1071 			goto again;
1072 		}
1073 		/*
1074 		 * szc was non zero and vnode and offset matched after we
1075 		 * locked the page it means it can't become free on us.
1076 		 */
1077 		ASSERT(!PP_ISFREE(pp));
1078 		if (!IS_P2ALIGNED(pfn, pages)) {
1079 			page_unlock(pp);
1080 			return (0);
1081 		}
1082 		ppa[0] = pp;
1083 		pp++;
1084 		off += PAGESIZE;
1085 		pfn++;
1086 		for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
1087 			if (!page_trylock(pp, SE_SHARED)) {
1088 				VM_STAT_ADD(page_exphcontg[6]);
1089 				pp--;
1090 				while (i-- > 0) {
1091 					page_unlock(pp);
1092 					pp--;
1093 				}
1094 				ppa[0] = NULL;
1095 				return (1);
1096 			}
1097 			if (pp->p_szc != pszc) {
1098 				VM_STAT_ADD(page_exphcontg[7]);
1099 				page_unlock(pp);
1100 				pp--;
1101 				while (i-- > 0) {
1102 					page_unlock(pp);
1103 					pp--;
1104 				}
1105 				ppa[0] = NULL;
1106 				off = save_off;
1107 				goto again;
1108 			}
1109 			/*
1110 			 * szc the same as for previous already locked pages
1111 			 * with right identity. Since this page had correct
1112 			 * szc after we locked it can't get freed or destroyed
1113 			 * and therefore must have the expected identity.
1114 			 */
1115 			ASSERT(!PP_ISFREE(pp));
1116 			if (pp->p_vnode != vp ||
1117 			    pp->p_offset != off) {
1118 				panic("page_exists_physcontig: "
1119 				    "large page identity doesn't match");
1120 			}
1121 			ppa[i] = pp;
1122 			ASSERT(pp->p_pagenum == pfn);
1123 		}
1124 		VM_STAT_ADD(page_exphcontg[8]);
1125 		ppa[pages] = NULL;
1126 		return (1);
1127 	} else if (pszc >= szc) {
1128 		VM_STAT_ADD(page_exphcontg[9]);
1129 		if (!IS_P2ALIGNED(pfn, pages)) {
1130 			return (0);
1131 		}
1132 		return (1);
1133 	}
1134 
1135 	if (!IS_P2ALIGNED(pfn, pages)) {
1136 		VM_STAT_ADD(page_exphcontg[10]);
1137 		return (0);
1138 	}
1139 
1140 	if (page_numtomemseg_nolock(pfn) !=
1141 	    page_numtomemseg_nolock(pfn + pages - 1)) {
1142 		VM_STAT_ADD(page_exphcontg[11]);
1143 		return (0);
1144 	}
1145 
1146 	/*
1147 	 * We loop up 4 times across pages to promote page size.
1148 	 * We're extra cautious to promote page size atomically with respect
1149 	 * to everybody else.  But we can probably optimize into 1 loop if
1150 	 * this becomes an issue.
1151 	 */
1152 
1153 	for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
1154 		ASSERT(pp->p_pagenum == pfn);
1155 		if (!page_trylock(pp, SE_EXCL)) {
1156 			VM_STAT_ADD(page_exphcontg[12]);
1157 			break;
1158 		}
1159 		if (pp->p_vnode != vp ||
1160 		    pp->p_offset != off) {
1161 			VM_STAT_ADD(page_exphcontg[13]);
1162 			page_unlock(pp);
1163 			break;
1164 		}
1165 		if (pp->p_szc >= szc) {
1166 			ASSERT(i == 0);
1167 			page_unlock(pp);
1168 			off = save_off;
1169 			goto again;
1170 		}
1171 	}
1172 
1173 	if (i != pages) {
1174 		VM_STAT_ADD(page_exphcontg[14]);
1175 		--pp;
1176 		while (i-- > 0) {
1177 			page_unlock(pp);
1178 			--pp;
1179 		}
1180 		return (0);
1181 	}
1182 
1183 	pp = rootpp;
1184 	for (i = 0; i < pages; i++, pp++) {
1185 		if (PP_ISFREE(pp)) {
1186 			VM_STAT_ADD(page_exphcontg[15]);
1187 			ASSERT(!PP_ISAGED(pp));
1188 			ASSERT(pp->p_szc == 0);
1189 			if (!page_reclaim(pp, NULL)) {
1190 				break;
1191 			}
1192 		} else {
1193 			ASSERT(pp->p_szc < szc);
1194 			VM_STAT_ADD(page_exphcontg[16]);
1195 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
1196 		}
1197 	}
1198 	if (i < pages) {
1199 		VM_STAT_ADD(page_exphcontg[17]);
1200 		/*
1201 		 * page_reclaim failed because we were out of memory.
1202 		 * drop the rest of the locks and return because this page
1203 		 * must be already reallocated anyway.
1204 		 */
1205 		pp = rootpp;
1206 		for (j = 0; j < pages; j++, pp++) {
1207 			if (j != i) {
1208 				page_unlock(pp);
1209 			}
1210 		}
1211 		return (0);
1212 	}
1213 
1214 	off = save_off;
1215 	pp = rootpp;
1216 	for (i = 0; i < pages; i++, pp++, off += PAGESIZE) {
1217 		ASSERT(PAGE_EXCL(pp));
1218 		ASSERT(!PP_ISFREE(pp));
1219 		ASSERT(!hat_page_is_mapped(pp));
1220 		ASSERT(pp->p_vnode == vp);
1221 		ASSERT(pp->p_offset == off);
1222 		pp->p_szc = szc;
1223 	}
1224 	pp = rootpp;
1225 	for (i = 0; i < pages; i++, pp++) {
1226 		if (ppa == NULL) {
1227 			page_unlock(pp);
1228 		} else {
1229 			ppa[i] = pp;
1230 			page_downgrade(ppa[i]);
1231 		}
1232 	}
1233 	if (ppa != NULL) {
1234 		ppa[pages] = NULL;
1235 	}
1236 	VM_STAT_ADD(page_exphcontg[18]);
1237 	ASSERT(vp->v_pages != NULL);
1238 	return (1);
1239 }
1240 
1241 /*
1242  * Determine whether a page with the specified [vp, off]
1243  * currently exists in the system and if so return its
1244  * size code. Obviously this should only be considered as
1245  * a hint since nothing prevents the page from disappearing
1246  * or appearing immediately after the return from this routine.
1247  */
1248 int
1249 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc)
1250 {
1251 	page_t		*pp;
1252 	kmutex_t	*phm;
1253 	ulong_t		index;
1254 	int		rc = 0;
1255 
1256 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
1257 	ASSERT(szc != NULL);
1258 	VM_STAT_ADD(page_exists_forreal_cnt);
1259 
1260 	index = PAGE_HASH_FUNC(vp, off);
1261 	phm = PAGE_HASH_MUTEX(index);
1262 
1263 	mutex_enter(phm);
1264 	PAGE_HASH_SEARCH(index, pp, vp, off);
1265 	if (pp != NULL) {
1266 		*szc = pp->p_szc;
1267 		rc = 1;
1268 	}
1269 	mutex_exit(phm);
1270 	return (rc);
1271 }
1272 
1273 /* wakeup threads waiting for pages in page_create_get_something() */
1274 void
1275 wakeup_pcgs(void)
1276 {
1277 	if (!CV_HAS_WAITERS(&pcgs_cv))
1278 		return;
1279 	cv_broadcast(&pcgs_cv);
1280 }
1281 
1282 /*
1283  * 'freemem' is used all over the kernel as an indication of how many
1284  * pages are free (either on the cache list or on the free page list)
1285  * in the system.  In very few places is a really accurate 'freemem'
1286  * needed.  To avoid contention of the lock protecting a the
1287  * single freemem, it was spread out into NCPU buckets.  Set_freemem
1288  * sets freemem to the total of all NCPU buckets.  It is called from
1289  * clock() on each TICK.
1290  */
1291 void
1292 set_freemem()
1293 {
1294 	struct pcf	*p;
1295 	ulong_t		t;
1296 	uint_t		i;
1297 
1298 	t = 0;
1299 	p = pcf;
1300 	for (i = 0;  i < PCF_FANOUT; i++) {
1301 		t += p->pcf_count;
1302 		p++;
1303 	}
1304 	freemem = t;
1305 
1306 	/*
1307 	 * Don't worry about grabbing mutex.  It's not that
1308 	 * critical if we miss a tick or two.  This is
1309 	 * where we wakeup possible delayers in
1310 	 * page_create_get_something().
1311 	 */
1312 	wakeup_pcgs();
1313 }
1314 
1315 ulong_t
1316 get_freemem()
1317 {
1318 	struct pcf	*p;
1319 	ulong_t		t;
1320 	uint_t		i;
1321 
1322 	t = 0;
1323 	p = pcf;
1324 	for (i = 0; i < PCF_FANOUT; i++) {
1325 		t += p->pcf_count;
1326 		p++;
1327 	}
1328 	/*
1329 	 * We just calculated it, might as well set it.
1330 	 */
1331 	freemem = t;
1332 	return (t);
1333 }
1334 
1335 /*
1336  * Acquire all of the page cache & free (pcf) locks.
1337  */
1338 void
1339 pcf_acquire_all()
1340 {
1341 	struct pcf	*p;
1342 	uint_t		i;
1343 
1344 	p = pcf;
1345 	for (i = 0; i < PCF_FANOUT; i++) {
1346 		mutex_enter(&p->pcf_lock);
1347 		p++;
1348 	}
1349 }
1350 
1351 /*
1352  * Release all the pcf_locks.
1353  */
1354 void
1355 pcf_release_all()
1356 {
1357 	struct pcf	*p;
1358 	uint_t		i;
1359 
1360 	p = pcf;
1361 	for (i = 0; i < PCF_FANOUT; i++) {
1362 		mutex_exit(&p->pcf_lock);
1363 		p++;
1364 	}
1365 }
1366 
1367 /*
1368  * Inform the VM system that we need some pages freed up.
1369  * Calls must be symmetric, e.g.:
1370  *
1371  *	page_needfree(100);
1372  *	wait a bit;
1373  *	page_needfree(-100);
1374  */
1375 void
1376 page_needfree(spgcnt_t npages)
1377 {
1378 	mutex_enter(&new_freemem_lock);
1379 	needfree += npages;
1380 	mutex_exit(&new_freemem_lock);
1381 }
1382 
1383 /*
1384  * Throttle for page_create(): try to prevent freemem from dropping
1385  * below throttlefree.  We can't provide a 100% guarantee because
1386  * KM_NOSLEEP allocations, page_reclaim(), and various other things
1387  * nibble away at the freelist.  However, we can block all PG_WAIT
1388  * allocations until memory becomes available.  The motivation is
1389  * that several things can fall apart when there's no free memory:
1390  *
1391  * (1) If pageout() needs memory to push a page, the system deadlocks.
1392  *
1393  * (2) By (broken) specification, timeout(9F) can neither fail nor
1394  *     block, so it has no choice but to panic the system if it
1395  *     cannot allocate a callout structure.
1396  *
1397  * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block;
1398  *     it panics if it cannot allocate a callback structure.
1399  *
1400  * (4) Untold numbers of third-party drivers have not yet been hardened
1401  *     against KM_NOSLEEP and/or allocb() failures; they simply assume
1402  *     success and panic the system with a data fault on failure.
1403  *     (The long-term solution to this particular problem is to ship
1404  *     hostile fault-injecting DEBUG kernels with the DDK.)
1405  *
1406  * It is theoretically impossible to guarantee success of non-blocking
1407  * allocations, but in practice, this throttle is very hard to break.
1408  */
1409 static int
1410 page_create_throttle(pgcnt_t npages, int flags)
1411 {
1412 	ulong_t	fm;
1413 	uint_t	i;
1414 	pgcnt_t tf;	/* effective value of throttlefree */
1415 
1416 	/*
1417 	 * Never deny pages when:
1418 	 * - it's a thread that cannot block [NOMEMWAIT()]
1419 	 * - the allocation cannot block and must not fail
1420 	 * - the allocation cannot block and is pageout dispensated
1421 	 */
1422 	if (NOMEMWAIT() ||
1423 	    ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) ||
1424 	    ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE))
1425 		return (1);
1426 
1427 	/*
1428 	 * If the allocation can't block, we look favorably upon it
1429 	 * unless we're below pageout_reserve.  In that case we fail
1430 	 * the allocation because we want to make sure there are a few
1431 	 * pages available for pageout.
1432 	 */
1433 	if ((flags & PG_WAIT) == 0)
1434 		return (freemem >= npages + pageout_reserve);
1435 
1436 	/* Calculate the effective throttlefree value */
1437 	tf = throttlefree -
1438 	    ((flags & PG_PUSHPAGE) ? pageout_reserve : 0);
1439 
1440 	cv_signal(&proc_pageout->p_cv);
1441 
1442 	while (freemem < npages + tf) {
1443 		pcf_acquire_all();
1444 		mutex_enter(&new_freemem_lock);
1445 		fm = 0;
1446 		for (i = 0; i < PCF_FANOUT; i++) {
1447 			fm += pcf[i].pcf_count;
1448 			pcf[i].pcf_wait++;
1449 			mutex_exit(&pcf[i].pcf_lock);
1450 		}
1451 		freemem = fm;
1452 		needfree += npages;
1453 		freemem_wait++;
1454 		cv_wait(&freemem_cv, &new_freemem_lock);
1455 		freemem_wait--;
1456 		needfree -= npages;
1457 		mutex_exit(&new_freemem_lock);
1458 	}
1459 	return (1);
1460 }
1461 
1462 /*
1463  * page_create_wait() is called to either coalecse pages from the
1464  * different pcf buckets or to wait because there simply are not
1465  * enough pages to satisfy the caller's request.
1466  *
1467  * Sadly, this is called from platform/vm/vm_machdep.c
1468  */
1469 int
1470 page_create_wait(size_t npages, uint_t flags)
1471 {
1472 	pgcnt_t		total;
1473 	uint_t		i;
1474 	struct pcf	*p;
1475 
1476 	/*
1477 	 * Wait until there are enough free pages to satisfy our
1478 	 * entire request.
1479 	 * We set needfree += npages before prodding pageout, to make sure
1480 	 * it does real work when npages > lotsfree > freemem.
1481 	 */
1482 	VM_STAT_ADD(page_create_not_enough);
1483 
1484 	ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1);
1485 checkagain:
1486 	if ((flags & PG_NORELOC) &&
1487 	    kcage_freemem < kcage_throttlefree + npages)
1488 		(void) kcage_create_throttle(npages, flags);
1489 
1490 	if (freemem < npages + throttlefree)
1491 		if (!page_create_throttle(npages, flags))
1492 			return (0);
1493 
1494 	/*
1495 	 * Since page_create_va() looked at every
1496 	 * bucket, assume we are going to have to wait.
1497 	 * Get all of the pcf locks.
1498 	 */
1499 	total = 0;
1500 	p = pcf;
1501 	for (i = 0; i < PCF_FANOUT; i++) {
1502 		mutex_enter(&p->pcf_lock);
1503 		total += p->pcf_count;
1504 		if (total >= npages) {
1505 			/*
1506 			 * Wow!  There are enough pages laying around
1507 			 * to satisfy the request.  Do the accounting,
1508 			 * drop the locks we acquired, and go back.
1509 			 *
1510 			 * freemem is not protected by any lock. So,
1511 			 * we cannot have any assertion containing
1512 			 * freemem.
1513 			 */
1514 			freemem -= npages;
1515 
1516 			while (p >= pcf) {
1517 				if (p->pcf_count <= npages) {
1518 					npages -= p->pcf_count;
1519 					p->pcf_count = 0;
1520 				} else {
1521 					p->pcf_count -= (uint_t)npages;
1522 					npages = 0;
1523 				}
1524 				mutex_exit(&p->pcf_lock);
1525 				p--;
1526 			}
1527 			ASSERT(npages == 0);
1528 			return (1);
1529 		}
1530 		p++;
1531 	}
1532 
1533 	/*
1534 	 * All of the pcf locks are held, there are not enough pages
1535 	 * to satisfy the request (npages < total).
1536 	 * Be sure to acquire the new_freemem_lock before dropping
1537 	 * the pcf locks.  This prevents dropping wakeups in page_free().
1538 	 * The order is always pcf_lock then new_freemem_lock.
1539 	 *
1540 	 * Since we hold all the pcf locks, it is a good time to set freemem.
1541 	 *
1542 	 * If the caller does not want to wait, return now.
1543 	 * Else turn the pageout daemon loose to find something
1544 	 * and wait till it does.
1545 	 *
1546 	 */
1547 	freemem = total;
1548 
1549 	if ((flags & PG_WAIT) == 0) {
1550 		pcf_release_all();
1551 
1552 		TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM,
1553 		"page_create_nomem:npages %ld freemem %ld", npages, freemem);
1554 		return (0);
1555 	}
1556 
1557 	ASSERT(proc_pageout != NULL);
1558 	cv_signal(&proc_pageout->p_cv);
1559 
1560 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START,
1561 	    "page_create_sleep_start: freemem %ld needfree %ld",
1562 	    freemem, needfree);
1563 
1564 	/*
1565 	 * We are going to wait.
1566 	 * We currently hold all of the pcf_locks,
1567 	 * get the new_freemem_lock (it protects freemem_wait),
1568 	 * before dropping the pcf_locks.
1569 	 */
1570 	mutex_enter(&new_freemem_lock);
1571 
1572 	p = pcf;
1573 	for (i = 0; i < PCF_FANOUT; i++) {
1574 		p->pcf_wait++;
1575 		mutex_exit(&p->pcf_lock);
1576 		p++;
1577 	}
1578 
1579 	needfree += npages;
1580 	freemem_wait++;
1581 
1582 	cv_wait(&freemem_cv, &new_freemem_lock);
1583 
1584 	freemem_wait--;
1585 	needfree -= npages;
1586 
1587 	mutex_exit(&new_freemem_lock);
1588 
1589 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END,
1590 	    "page_create_sleep_end: freemem %ld needfree %ld",
1591 	    freemem, needfree);
1592 
1593 	VM_STAT_ADD(page_create_not_enough_again);
1594 	goto checkagain;
1595 }
1596 
1597 /*
1598  * A routine to do the opposite of page_create_wait().
1599  */
1600 void
1601 page_create_putback(spgcnt_t npages)
1602 {
1603 	struct pcf	*p;
1604 	pgcnt_t		lump;
1605 	uint_t		*which;
1606 
1607 	/*
1608 	 * When a contiguous lump is broken up, we have to
1609 	 * deal with lots of pages (min 64) so lets spread
1610 	 * the wealth around.
1611 	 */
1612 	lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT;
1613 	freemem += npages;
1614 
1615 	for (p = pcf; (npages > 0) && (p < &pcf[PCF_FANOUT]); p++) {
1616 		which = &p->pcf_count;
1617 
1618 		mutex_enter(&p->pcf_lock);
1619 
1620 		if (p->pcf_block) {
1621 			which = &p->pcf_reserve;
1622 		}
1623 
1624 		if (lump < npages) {
1625 			*which += (uint_t)lump;
1626 			npages -= lump;
1627 		} else {
1628 			*which += (uint_t)npages;
1629 			npages = 0;
1630 		}
1631 
1632 		if (p->pcf_wait) {
1633 			mutex_enter(&new_freemem_lock);
1634 			/*
1635 			 * Check to see if some other thread
1636 			 * is actually waiting.  Another bucket
1637 			 * may have woken it up by now.  If there
1638 			 * are no waiters, then set our pcf_wait
1639 			 * count to zero to avoid coming in here
1640 			 * next time.
1641 			 */
1642 			if (freemem_wait) {
1643 				if (npages > 1) {
1644 					cv_broadcast(&freemem_cv);
1645 				} else {
1646 					cv_signal(&freemem_cv);
1647 				}
1648 				p->pcf_wait--;
1649 			} else {
1650 				p->pcf_wait = 0;
1651 			}
1652 			mutex_exit(&new_freemem_lock);
1653 		}
1654 		mutex_exit(&p->pcf_lock);
1655 	}
1656 	ASSERT(npages == 0);
1657 }
1658 
1659 /*
1660  * A helper routine for page_create_get_something.
1661  * The indenting got to deep down there.
1662  * Unblock the pcf counters.  Any pages freed after
1663  * pcf_block got set are moved to pcf_count and
1664  * wakeups (cv_broadcast() or cv_signal()) are done as needed.
1665  */
1666 static void
1667 pcgs_unblock(void)
1668 {
1669 	int		i;
1670 	struct pcf	*p;
1671 
1672 	/* Update freemem while we're here. */
1673 	freemem = 0;
1674 	p = pcf;
1675 	for (i = 0; i < PCF_FANOUT; i++) {
1676 		mutex_enter(&p->pcf_lock);
1677 		ASSERT(p->pcf_count == 0);
1678 		p->pcf_count = p->pcf_reserve;
1679 		p->pcf_block = 0;
1680 		freemem += p->pcf_count;
1681 		if (p->pcf_wait) {
1682 			mutex_enter(&new_freemem_lock);
1683 			if (freemem_wait) {
1684 				if (p->pcf_reserve > 1) {
1685 					cv_broadcast(&freemem_cv);
1686 					p->pcf_wait = 0;
1687 				} else {
1688 					cv_signal(&freemem_cv);
1689 					p->pcf_wait--;
1690 				}
1691 			} else {
1692 				p->pcf_wait = 0;
1693 			}
1694 			mutex_exit(&new_freemem_lock);
1695 		}
1696 		p->pcf_reserve = 0;
1697 		mutex_exit(&p->pcf_lock);
1698 		p++;
1699 	}
1700 }
1701 
1702 /*
1703  * Called from page_create_va() when both the cache and free lists
1704  * have been checked once.
1705  *
1706  * Either returns a page or panics since the accounting was done
1707  * way before we got here.
1708  *
1709  * We don't come here often, so leave the accounting on permanently.
1710  */
1711 
1712 #define	MAX_PCGS	100
1713 
1714 #ifdef	DEBUG
1715 #define	PCGS_TRIES	100
1716 #else	/* DEBUG */
1717 #define	PCGS_TRIES	10
1718 #endif	/* DEBUG */
1719 
1720 #ifdef	VM_STATS
1721 uint_t	pcgs_counts[PCGS_TRIES];
1722 uint_t	pcgs_too_many;
1723 uint_t	pcgs_entered;
1724 uint_t	pcgs_entered_noreloc;
1725 uint_t	pcgs_locked;
1726 uint_t	pcgs_cagelocked;
1727 #endif	/* VM_STATS */
1728 
1729 static page_t *
1730 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg,
1731     caddr_t vaddr, uint_t flags)
1732 {
1733 	uint_t		count;
1734 	page_t		*pp;
1735 	uint_t		locked, i;
1736 	struct	pcf	*p;
1737 	lgrp_t		*lgrp;
1738 	int		cagelocked = 0;
1739 
1740 	VM_STAT_ADD(pcgs_entered);
1741 
1742 	/*
1743 	 * Tap any reserve freelists: if we fail now, we'll die
1744 	 * since the page(s) we're looking for have already been
1745 	 * accounted for.
1746 	 */
1747 	flags |= PG_PANIC;
1748 
1749 	if ((flags & PG_NORELOC) != 0) {
1750 		VM_STAT_ADD(pcgs_entered_noreloc);
1751 		/*
1752 		 * Requests for free pages from critical threads
1753 		 * such as pageout still won't throttle here, but
1754 		 * we must try again, to give the cageout thread
1755 		 * another chance to catch up. Since we already
1756 		 * accounted for the pages, we had better get them
1757 		 * this time.
1758 		 *
1759 		 * N.B. All non-critical threads acquire the pcgs_cagelock
1760 		 * to serialize access to the freelists. This implements a
1761 		 * turnstile-type synchornization to avoid starvation of
1762 		 * critical requests for PG_NORELOC memory by non-critical
1763 		 * threads: all non-critical threads must acquire a 'ticket'
1764 		 * before passing through, which entails making sure
1765 		 * kcage_freemem won't fall below minfree prior to grabbing
1766 		 * pages from the freelists.
1767 		 */
1768 		if (kcage_create_throttle(1, flags) == KCT_NONCRIT) {
1769 			mutex_enter(&pcgs_cagelock);
1770 			cagelocked = 1;
1771 			VM_STAT_ADD(pcgs_cagelocked);
1772 		}
1773 	}
1774 
1775 	/*
1776 	 * Time to get serious.
1777 	 * We failed to get a `correctly colored' page from both the
1778 	 * free and cache lists.
1779 	 * We escalate in stage.
1780 	 *
1781 	 * First try both lists without worring about color.
1782 	 *
1783 	 * Then, grab all page accounting locks (ie. pcf[]) and
1784 	 * steal any pages that they have and set the pcf_block flag to
1785 	 * stop deletions from the lists.  This will help because
1786 	 * a page can get added to the free list while we are looking
1787 	 * at the cache list, then another page could be added to the cache
1788 	 * list allowing the page on the free list to be removed as we
1789 	 * move from looking at the cache list to the free list. This
1790 	 * could happen over and over. We would never find the page
1791 	 * we have accounted for.
1792 	 *
1793 	 * Noreloc pages are a subset of the global (relocatable) page pool.
1794 	 * They are not tracked separately in the pcf bins, so it is
1795 	 * impossible to know when doing pcf accounting if the available
1796 	 * page(s) are noreloc pages or not. When looking for a noreloc page
1797 	 * it is quite easy to end up here even if the global (relocatable)
1798 	 * page pool has plenty of free pages but the noreloc pool is empty.
1799 	 *
1800 	 * When the noreloc pool is empty (or low), additional noreloc pages
1801 	 * are created by converting pages from the global page pool. This
1802 	 * process will stall during pcf accounting if the pcf bins are
1803 	 * already locked. Such is the case when a noreloc allocation is
1804 	 * looping here in page_create_get_something waiting for more noreloc
1805 	 * pages to appear.
1806 	 *
1807 	 * Short of adding a new field to the pcf bins to accurately track
1808 	 * the number of free noreloc pages, we instead do not grab the
1809 	 * pcgs_lock, do not set the pcf blocks and do not timeout when
1810 	 * allocating a noreloc page. This allows noreloc allocations to
1811 	 * loop without blocking global page pool allocations.
1812 	 *
1813 	 * NOTE: the behaviour of page_create_get_something has not changed
1814 	 * for the case of global page pool allocations.
1815 	 */
1816 
1817 	flags &= ~PG_MATCH_COLOR;
1818 	locked = 0;
1819 #if defined(__i386) || defined(__amd64)
1820 	/*
1821 	 * page_create_get_something may be called because 4g memory may be
1822 	 * depleted. Set flags to allow for relocation of base page below
1823 	 * 4g if necessary.
1824 	 */
1825 	if (physmax4g)
1826 		flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
1827 #endif
1828 
1829 	lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
1830 
1831 	for (count = 0; kcage_on || count < MAX_PCGS; count++) {
1832 		pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
1833 		    flags, lgrp);
1834 		if (pp == NULL) {
1835 			pp = page_get_cachelist(vp, off, seg, vaddr,
1836 				flags, lgrp);
1837 		}
1838 		if (pp == NULL) {
1839 			/*
1840 			 * Serialize.  Don't fight with other pcgs().
1841 			 */
1842 			if (!locked && (!kcage_on || !(flags & PG_NORELOC))) {
1843 				mutex_enter(&pcgs_lock);
1844 				VM_STAT_ADD(pcgs_locked);
1845 				locked = 1;
1846 				p = pcf;
1847 				for (i = 0; i < PCF_FANOUT; i++) {
1848 					mutex_enter(&p->pcf_lock);
1849 					ASSERT(p->pcf_block == 0);
1850 					p->pcf_block = 1;
1851 					p->pcf_reserve = p->pcf_count;
1852 					p->pcf_count = 0;
1853 					mutex_exit(&p->pcf_lock);
1854 					p++;
1855 				}
1856 				freemem = 0;
1857 			}
1858 
1859 			if (count) {
1860 				/*
1861 				 * Since page_free() puts pages on
1862 				 * a list then accounts for it, we
1863 				 * just have to wait for page_free()
1864 				 * to unlock any page it was working
1865 				 * with. The page_lock()-page_reclaim()
1866 				 * path falls in the same boat.
1867 				 *
1868 				 * We don't need to check on the
1869 				 * PG_WAIT flag, we have already
1870 				 * accounted for the page we are
1871 				 * looking for in page_create_va().
1872 				 *
1873 				 * We just wait a moment to let any
1874 				 * locked pages on the lists free up,
1875 				 * then continue around and try again.
1876 				 *
1877 				 * Will be awakened by set_freemem().
1878 				 */
1879 				mutex_enter(&pcgs_wait_lock);
1880 				cv_wait(&pcgs_cv, &pcgs_wait_lock);
1881 				mutex_exit(&pcgs_wait_lock);
1882 			}
1883 		} else {
1884 #ifdef VM_STATS
1885 			if (count >= PCGS_TRIES) {
1886 				VM_STAT_ADD(pcgs_too_many);
1887 			} else {
1888 				VM_STAT_ADD(pcgs_counts[count]);
1889 			}
1890 #endif
1891 			if (locked) {
1892 				pcgs_unblock();
1893 				mutex_exit(&pcgs_lock);
1894 			}
1895 			if (cagelocked)
1896 				mutex_exit(&pcgs_cagelock);
1897 			return (pp);
1898 		}
1899 	}
1900 	/*
1901 	 * we go down holding the pcf locks.
1902 	 */
1903 	panic("no %spage found %d",
1904 	    ((flags & PG_NORELOC) ? "non-reloc " : ""), count);
1905 	/*NOTREACHED*/
1906 }
1907 
1908 /*
1909  * Create enough pages for "bytes" worth of data starting at
1910  * "off" in "vp".
1911  *
1912  *	Where flag must be one of:
1913  *
1914  *		PG_EXCL:	Exclusive create (fail if any page already
1915  *				exists in the page cache) which does not
1916  *				wait for memory to become available.
1917  *
1918  *		PG_WAIT:	Non-exclusive create which can wait for
1919  *				memory to become available.
1920  *
1921  *		PG_PHYSCONTIG:	Allocate physically contiguous pages.
1922  *				(Not Supported)
1923  *
1924  * A doubly linked list of pages is returned to the caller.  Each page
1925  * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock)
1926  * lock.
1927  *
1928  * Unable to change the parameters to page_create() in a minor release,
1929  * we renamed page_create() to page_create_va(), changed all known calls
1930  * from page_create() to page_create_va(), and created this wrapper.
1931  *
1932  * Upon a major release, we should break compatibility by deleting this
1933  * wrapper, and replacing all the strings "page_create_va", with "page_create".
1934  *
1935  * NOTE: There is a copy of this interface as page_create_io() in
1936  *	 i86/vm/vm_machdep.c. Any bugs fixed here should be applied
1937  *	 there.
1938  */
1939 page_t *
1940 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags)
1941 {
1942 	caddr_t random_vaddr;
1943 	struct seg kseg;
1944 
1945 #ifdef DEBUG
1946 	cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p",
1947 	    (void *)caller());
1948 #endif
1949 
1950 	random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^
1951 	    (uintptr_t)(off >> PAGESHIFT));
1952 	kseg.s_as = &kas;
1953 
1954 	return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr));
1955 }
1956 
1957 #ifdef DEBUG
1958 uint32_t pg_alloc_pgs_mtbf = 0;
1959 #endif
1960 
1961 /*
1962  * Used for large page support. It will attempt to allocate
1963  * a large page(s) off the freelist.
1964  *
1965  * Returns non zero on failure.
1966  */
1967 int
1968 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr,
1969     page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz)
1970 {
1971 	pgcnt_t		npgs, curnpgs, totpgs;
1972 	size_t		pgsz;
1973 	page_t		*pplist = NULL, *pp;
1974 	int		err = 0;
1975 	lgrp_t		*lgrp;
1976 
1977 	ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1));
1978 
1979 	VM_STAT_ADD(alloc_pages[0]);
1980 
1981 #ifdef DEBUG
1982 	if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) {
1983 		return (ENOMEM);
1984 	}
1985 #endif
1986 
1987 	pgsz = page_get_pagesize(szc);
1988 	totpgs = curnpgs = npgs = pgsz >> PAGESHIFT;
1989 
1990 	ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0);
1991 	/*
1992 	 * One must be NULL but not both.
1993 	 * And one must be non NULL but not both.
1994 	 */
1995 	ASSERT(basepp != NULL || ppa != NULL);
1996 	ASSERT(basepp == NULL || ppa == NULL);
1997 
1998 	(void) page_create_wait(npgs, PG_WAIT);
1999 
2000 	while (npgs && szc) {
2001 		lgrp = lgrp_mem_choose(seg, addr, pgsz);
2002 		pp = page_get_freelist(vp, 0, seg, addr, pgsz, 0, lgrp);
2003 		if (pp != NULL) {
2004 			VM_STAT_ADD(alloc_pages[1]);
2005 			page_list_concat(&pplist, &pp);
2006 			ASSERT(npgs >= curnpgs);
2007 			npgs -= curnpgs;
2008 		} else if (anypgsz) {
2009 			VM_STAT_ADD(alloc_pages[2]);
2010 			szc--;
2011 			pgsz = page_get_pagesize(szc);
2012 			curnpgs = pgsz >> PAGESHIFT;
2013 		} else {
2014 			VM_STAT_ADD(alloc_pages[3]);
2015 			ASSERT(npgs == totpgs);
2016 			page_create_putback(npgs);
2017 			return (ENOMEM);
2018 		}
2019 	}
2020 	if (szc == 0) {
2021 		VM_STAT_ADD(alloc_pages[4]);
2022 		ASSERT(npgs != 0);
2023 		page_create_putback(npgs);
2024 		err = ENOMEM;
2025 	} else if (basepp != NULL) {
2026 		ASSERT(npgs == 0);
2027 		ASSERT(ppa == NULL);
2028 		*basepp = pplist;
2029 	}
2030 
2031 	npgs = totpgs - npgs;
2032 	pp = pplist;
2033 
2034 	/*
2035 	 * Clear the free and age bits. Also if we were passed in a ppa then
2036 	 * fill it in with all the constituent pages from the large page. But
2037 	 * if we failed to allocate all the pages just free what we got.
2038 	 */
2039 	while (npgs != 0) {
2040 		ASSERT(PP_ISFREE(pp));
2041 		ASSERT(PP_ISAGED(pp));
2042 		if (ppa != NULL || err != 0) {
2043 			if (err == 0) {
2044 				VM_STAT_ADD(alloc_pages[5]);
2045 				PP_CLRFREE(pp);
2046 				PP_CLRAGED(pp);
2047 				page_sub(&pplist, pp);
2048 				*ppa++ = pp;
2049 				npgs--;
2050 			} else {
2051 				VM_STAT_ADD(alloc_pages[6]);
2052 				ASSERT(pp->p_szc != 0);
2053 				curnpgs = page_get_pagecnt(pp->p_szc);
2054 				page_list_break(&pp, &pplist, curnpgs);
2055 				page_list_add_pages(pp, 0);
2056 				page_create_putback(curnpgs);
2057 				ASSERT(npgs >= curnpgs);
2058 				npgs -= curnpgs;
2059 			}
2060 			pp = pplist;
2061 		} else {
2062 			VM_STAT_ADD(alloc_pages[7]);
2063 			PP_CLRFREE(pp);
2064 			PP_CLRAGED(pp);
2065 			pp = pp->p_next;
2066 			npgs--;
2067 		}
2068 	}
2069 	return (err);
2070 }
2071 
2072 /*
2073  * Get a single large page off of the freelists, and set it up for use.
2074  * Number of bytes requested must be a supported page size.
2075  *
2076  * Note that this call may fail even if there is sufficient
2077  * memory available or PG_WAIT is set, so the caller must
2078  * be willing to fallback on page_create_va(), block and retry,
2079  * or fail the requester.
2080  */
2081 page_t *
2082 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
2083     struct seg *seg, caddr_t vaddr, void *arg)
2084 {
2085 	pgcnt_t		npages, pcftotal;
2086 	page_t		*pp;
2087 	page_t		*rootpp;
2088 	lgrp_t		*lgrp;
2089 	uint_t		enough;
2090 	uint_t		pcf_index;
2091 	uint_t		i;
2092 	struct pcf	*p;
2093 	struct pcf	*q;
2094 	lgrp_id_t	*lgrpid = (lgrp_id_t *)arg;
2095 
2096 	ASSERT(vp != NULL);
2097 
2098 	ASSERT((flags & ~(PG_EXCL | PG_WAIT |
2099 		    PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0);
2100 	/* but no others */
2101 
2102 	ASSERT((flags & PG_EXCL) == PG_EXCL);
2103 
2104 	npages = btop(bytes);
2105 
2106 	if (!kcage_on || panicstr) {
2107 		/*
2108 		 * Cage is OFF, or we are single threaded in
2109 		 * panic, so make everything a RELOC request.
2110 		 */
2111 		flags &= ~PG_NORELOC;
2112 	}
2113 
2114 	/*
2115 	 * Make sure there's adequate physical memory available.
2116 	 * Note: PG_WAIT is ignored here.
2117 	 */
2118 	if (freemem <= throttlefree + npages) {
2119 		VM_STAT_ADD(page_create_large_cnt[1]);
2120 		return (NULL);
2121 	}
2122 
2123 	/*
2124 	 * If cage is on, dampen draw from cage when available
2125 	 * cage space is low.
2126 	 */
2127 	if ((flags & (PG_NORELOC | PG_WAIT)) ==  (PG_NORELOC | PG_WAIT) &&
2128 	    kcage_freemem < kcage_throttlefree + npages) {
2129 
2130 		/*
2131 		 * The cage is on, the caller wants PG_NORELOC
2132 		 * pages and available cage memory is very low.
2133 		 * Call kcage_create_throttle() to attempt to
2134 		 * control demand on the cage.
2135 		 */
2136 		if (kcage_create_throttle(npages, flags) == KCT_FAILURE) {
2137 			VM_STAT_ADD(page_create_large_cnt[2]);
2138 			return (NULL);
2139 		}
2140 	}
2141 
2142 	enough = 0;
2143 	pcf_index = PCF_INDEX();
2144 	p = &pcf[pcf_index];
2145 	q = &pcf[PCF_FANOUT];
2146 	for (pcftotal = 0, i = 0; i < PCF_FANOUT; i++) {
2147 		if (p->pcf_count > npages) {
2148 			/*
2149 			 * a good one to try.
2150 			 */
2151 			mutex_enter(&p->pcf_lock);
2152 			if (p->pcf_count > npages) {
2153 				p->pcf_count -= (uint_t)npages;
2154 				/*
2155 				 * freemem is not protected by any lock.
2156 				 * Thus, we cannot have any assertion
2157 				 * containing freemem here.
2158 				 */
2159 				freemem -= npages;
2160 				enough = 1;
2161 				mutex_exit(&p->pcf_lock);
2162 				break;
2163 			}
2164 			mutex_exit(&p->pcf_lock);
2165 		}
2166 		pcftotal += p->pcf_count;
2167 		p++;
2168 		if (p >= q) {
2169 			p = pcf;
2170 		}
2171 	}
2172 
2173 	if (!enough) {
2174 		/* If there isn't enough memory available, give up. */
2175 		if (pcftotal < npages) {
2176 			VM_STAT_ADD(page_create_large_cnt[3]);
2177 			return (NULL);
2178 		}
2179 
2180 		/* try to collect pages from several pcf bins */
2181 		for (p = pcf, pcftotal = 0, i = 0; i < PCF_FANOUT; i++) {
2182 			mutex_enter(&p->pcf_lock);
2183 			pcftotal += p->pcf_count;
2184 			if (pcftotal >= npages) {
2185 				/*
2186 				 * Wow!  There are enough pages laying around
2187 				 * to satisfy the request.  Do the accounting,
2188 				 * drop the locks we acquired, and go back.
2189 				 *
2190 				 * freemem is not protected by any lock. So,
2191 				 * we cannot have any assertion containing
2192 				 * freemem.
2193 				 */
2194 				pgcnt_t	tpages = npages;
2195 				freemem -= npages;
2196 				while (p >= pcf) {
2197 					if (p->pcf_count <= tpages) {
2198 						tpages -= p->pcf_count;
2199 						p->pcf_count = 0;
2200 					} else {
2201 						p->pcf_count -= (uint_t)tpages;
2202 						tpages = 0;
2203 					}
2204 					mutex_exit(&p->pcf_lock);
2205 					p--;
2206 				}
2207 				ASSERT(tpages == 0);
2208 				break;
2209 			}
2210 			p++;
2211 		}
2212 		if (i == PCF_FANOUT) {
2213 			/* failed to collect pages - release the locks */
2214 			while (--p >= pcf) {
2215 				mutex_exit(&p->pcf_lock);
2216 			}
2217 			VM_STAT_ADD(page_create_large_cnt[4]);
2218 			return (NULL);
2219 		}
2220 	}
2221 
2222 	/*
2223 	 * This is where this function behaves fundamentally differently
2224 	 * than page_create_va(); since we're intending to map the page
2225 	 * with a single TTE, we have to get it as a physically contiguous
2226 	 * hardware pagesize chunk.  If we can't, we fail.
2227 	 */
2228 	if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max &&
2229 		LGRP_EXISTS(lgrp_table[*lgrpid]))
2230 		lgrp = lgrp_table[*lgrpid];
2231 	else
2232 		lgrp = lgrp_mem_choose(seg, vaddr, bytes);
2233 
2234 	if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr,
2235 	    bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) {
2236 		page_create_putback(npages);
2237 		VM_STAT_ADD(page_create_large_cnt[5]);
2238 		return (NULL);
2239 	}
2240 
2241 	/*
2242 	 * if we got the page with the wrong mtype give it back this is a
2243 	 * workaround for CR 6249718. When CR 6249718 is fixed we never get
2244 	 * inside "if" and the workaround becomes just a nop
2245 	 */
2246 	if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) {
2247 		page_list_add_pages(rootpp, 0);
2248 		page_create_putback(npages);
2249 		VM_STAT_ADD(page_create_large_cnt[6]);
2250 		return (NULL);
2251 	}
2252 
2253 	/*
2254 	 * If satisfying this request has left us with too little
2255 	 * memory, start the wheels turning to get some back.  The
2256 	 * first clause of the test prevents waking up the pageout
2257 	 * daemon in situations where it would decide that there's
2258 	 * nothing to do.
2259 	 */
2260 	if (nscan < desscan && freemem < minfree) {
2261 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
2262 		    "pageout_cv_signal:freemem %ld", freemem);
2263 		cv_signal(&proc_pageout->p_cv);
2264 	}
2265 
2266 	pp = rootpp;
2267 	while (npages--) {
2268 		ASSERT(PAGE_EXCL(pp));
2269 		ASSERT(pp->p_vnode == NULL);
2270 		ASSERT(!hat_page_is_mapped(pp));
2271 		PP_CLRFREE(pp);
2272 		PP_CLRAGED(pp);
2273 		if (!page_hashin(pp, vp, off, NULL))
2274 			panic("page_create_large: hashin failed: page %p",
2275 			    (void *)pp);
2276 		page_io_lock(pp);
2277 		off += PAGESIZE;
2278 		pp = pp->p_next;
2279 	}
2280 
2281 	VM_STAT_ADD(page_create_large_cnt[0]);
2282 	return (rootpp);
2283 }
2284 
2285 page_t *
2286 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
2287     struct seg *seg, caddr_t vaddr)
2288 {
2289 	page_t		*plist = NULL;
2290 	pgcnt_t		npages;
2291 	pgcnt_t		found_on_free = 0;
2292 	pgcnt_t		pages_req;
2293 	page_t		*npp = NULL;
2294 	uint_t		enough;
2295 	uint_t		i;
2296 	uint_t		pcf_index;
2297 	struct pcf	*p;
2298 	struct pcf	*q;
2299 	lgrp_t		*lgrp;
2300 
2301 	TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
2302 		"page_create_start:vp %p off %llx bytes %lu flags %x",
2303 		vp, off, bytes, flags);
2304 
2305 	ASSERT(bytes != 0 && vp != NULL);
2306 
2307 	if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) {
2308 		panic("page_create: invalid flags");
2309 		/*NOTREACHED*/
2310 	}
2311 	ASSERT((flags & ~(PG_EXCL | PG_WAIT |
2312 	    PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0);
2313 	    /* but no others */
2314 
2315 	pages_req = npages = btopr(bytes);
2316 	/*
2317 	 * Try to see whether request is too large to *ever* be
2318 	 * satisfied, in order to prevent deadlock.  We arbitrarily
2319 	 * decide to limit maximum size requests to max_page_get.
2320 	 */
2321 	if (npages >= max_page_get) {
2322 		if ((flags & PG_WAIT) == 0) {
2323 			TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG,
2324 			    "page_create_toobig:vp %p off %llx npages "
2325 			    "%lu max_page_get %lu",
2326 			    vp, off, npages, max_page_get);
2327 			return (NULL);
2328 		} else {
2329 			cmn_err(CE_WARN,
2330 			    "Request for too much kernel memory "
2331 			    "(%lu bytes), will hang forever", bytes);
2332 			for (;;)
2333 				delay(1000000000);
2334 		}
2335 	}
2336 
2337 	if (!kcage_on || panicstr) {
2338 		/*
2339 		 * Cage is OFF, or we are single threaded in
2340 		 * panic, so make everything a RELOC request.
2341 		 */
2342 		flags &= ~PG_NORELOC;
2343 	}
2344 
2345 	if (freemem <= throttlefree + npages)
2346 		if (!page_create_throttle(npages, flags))
2347 			return (NULL);
2348 
2349 	/*
2350 	 * If cage is on, dampen draw from cage when available
2351 	 * cage space is low.
2352 	 */
2353 	if ((flags & PG_NORELOC) &&
2354 		kcage_freemem < kcage_throttlefree + npages) {
2355 
2356 		/*
2357 		 * The cage is on, the caller wants PG_NORELOC
2358 		 * pages and available cage memory is very low.
2359 		 * Call kcage_create_throttle() to attempt to
2360 		 * control demand on the cage.
2361 		 */
2362 		if (kcage_create_throttle(npages, flags) == KCT_FAILURE)
2363 			return (NULL);
2364 	}
2365 
2366 	VM_STAT_ADD(page_create_cnt[0]);
2367 
2368 	enough = 0;
2369 	pcf_index = PCF_INDEX();
2370 
2371 	p = &pcf[pcf_index];
2372 	q = &pcf[PCF_FANOUT];
2373 	for (i = 0; i < PCF_FANOUT; i++) {
2374 		if (p->pcf_count > npages) {
2375 			/*
2376 			 * a good one to try.
2377 			 */
2378 			mutex_enter(&p->pcf_lock);
2379 			if (p->pcf_count > npages) {
2380 				p->pcf_count -= (uint_t)npages;
2381 				/*
2382 				 * freemem is not protected by any lock.
2383 				 * Thus, we cannot have any assertion
2384 				 * containing freemem here.
2385 				 */
2386 				freemem -= npages;
2387 				enough = 1;
2388 				mutex_exit(&p->pcf_lock);
2389 				break;
2390 			}
2391 			mutex_exit(&p->pcf_lock);
2392 		}
2393 		p++;
2394 		if (p >= q) {
2395 			p = pcf;
2396 		}
2397 	}
2398 
2399 	if (!enough) {
2400 		/*
2401 		 * Have to look harder.  If npages is greater than
2402 		 * one, then we might have to coalecse the counters.
2403 		 *
2404 		 * Go wait.  We come back having accounted
2405 		 * for the memory.
2406 		 */
2407 		VM_STAT_ADD(page_create_cnt[1]);
2408 		if (!page_create_wait(npages, flags)) {
2409 			VM_STAT_ADD(page_create_cnt[2]);
2410 			return (NULL);
2411 		}
2412 	}
2413 
2414 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
2415 		"page_create_success:vp %p off %llx", vp, off);
2416 
2417 	/*
2418 	 * If satisfying this request has left us with too little
2419 	 * memory, start the wheels turning to get some back.  The
2420 	 * first clause of the test prevents waking up the pageout
2421 	 * daemon in situations where it would decide that there's
2422 	 * nothing to do.
2423 	 */
2424 	if (nscan < desscan && freemem < minfree) {
2425 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
2426 			"pageout_cv_signal:freemem %ld", freemem);
2427 		cv_signal(&proc_pageout->p_cv);
2428 	}
2429 
2430 	/*
2431 	 * Loop around collecting the requested number of pages.
2432 	 * Most of the time, we have to `create' a new page. With
2433 	 * this in mind, pull the page off the free list before
2434 	 * getting the hash lock.  This will minimize the hash
2435 	 * lock hold time, nesting, and the like.  If it turns
2436 	 * out we don't need the page, we put it back at the end.
2437 	 */
2438 	while (npages--) {
2439 		page_t		*pp;
2440 		kmutex_t	*phm = NULL;
2441 		ulong_t		index;
2442 
2443 		index = PAGE_HASH_FUNC(vp, off);
2444 top:
2445 		ASSERT(phm == NULL);
2446 		ASSERT(index == PAGE_HASH_FUNC(vp, off));
2447 		ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
2448 
2449 		if (npp == NULL) {
2450 			/*
2451 			 * Try to get a page from the freelist (ie,
2452 			 * a page with no [vp, off] tag).  If that
2453 			 * fails, use the cachelist.
2454 			 *
2455 			 * During the first attempt at both the free
2456 			 * and cache lists we try for the correct color.
2457 			 */
2458 			/*
2459 			 * XXXX-how do we deal with virtual indexed
2460 			 * caches and and colors?
2461 			 */
2462 			VM_STAT_ADD(page_create_cnt[4]);
2463 			/*
2464 			 * Get lgroup to allocate next page of shared memory
2465 			 * from and use it to specify where to allocate
2466 			 * the physical memory
2467 			 */
2468 			lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
2469 			npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
2470 			    flags | PG_MATCH_COLOR, lgrp);
2471 			if (npp == NULL) {
2472 				npp = page_get_cachelist(vp, off, seg,
2473 				    vaddr, flags | PG_MATCH_COLOR, lgrp);
2474 				if (npp == NULL) {
2475 					npp = page_create_get_something(vp,
2476 					    off, seg, vaddr,
2477 					    flags & ~PG_MATCH_COLOR);
2478 				}
2479 
2480 				if (PP_ISAGED(npp) == 0) {
2481 					/*
2482 					 * Since this page came from the
2483 					 * cachelist, we must destroy the
2484 					 * old vnode association.
2485 					 */
2486 					page_hashout(npp, NULL);
2487 				}
2488 			}
2489 		}
2490 
2491 		/*
2492 		 * We own this page!
2493 		 */
2494 		ASSERT(PAGE_EXCL(npp));
2495 		ASSERT(npp->p_vnode == NULL);
2496 		ASSERT(!hat_page_is_mapped(npp));
2497 		PP_CLRFREE(npp);
2498 		PP_CLRAGED(npp);
2499 
2500 		/*
2501 		 * Here we have a page in our hot little mits and are
2502 		 * just waiting to stuff it on the appropriate lists.
2503 		 * Get the mutex and check to see if it really does
2504 		 * not exist.
2505 		 */
2506 		phm = PAGE_HASH_MUTEX(index);
2507 		mutex_enter(phm);
2508 		PAGE_HASH_SEARCH(index, pp, vp, off);
2509 		if (pp == NULL) {
2510 			VM_STAT_ADD(page_create_new);
2511 			pp = npp;
2512 			npp = NULL;
2513 			if (!page_hashin(pp, vp, off, phm)) {
2514 				/*
2515 				 * Since we hold the page hash mutex and
2516 				 * just searched for this page, page_hashin
2517 				 * had better not fail.  If it does, that
2518 				 * means somethread did not follow the
2519 				 * page hash mutex rules.  Panic now and
2520 				 * get it over with.  As usual, go down
2521 				 * holding all the locks.
2522 				 */
2523 				ASSERT(MUTEX_HELD(phm));
2524 				panic("page_create: "
2525 				    "hashin failed %p %p %llx %p",
2526 				    (void *)pp, (void *)vp, off, (void *)phm);
2527 				/*NOTREACHED*/
2528 			}
2529 			ASSERT(MUTEX_HELD(phm));
2530 			mutex_exit(phm);
2531 			phm = NULL;
2532 
2533 			/*
2534 			 * Hat layer locking need not be done to set
2535 			 * the following bits since the page is not hashed
2536 			 * and was on the free list (i.e., had no mappings).
2537 			 *
2538 			 * Set the reference bit to protect
2539 			 * against immediate pageout
2540 			 *
2541 			 * XXXmh modify freelist code to set reference
2542 			 * bit so we don't have to do it here.
2543 			 */
2544 			page_set_props(pp, P_REF);
2545 			found_on_free++;
2546 		} else {
2547 			VM_STAT_ADD(page_create_exists);
2548 			if (flags & PG_EXCL) {
2549 				/*
2550 				 * Found an existing page, and the caller
2551 				 * wanted all new pages.  Undo all of the work
2552 				 * we have done.
2553 				 */
2554 				mutex_exit(phm);
2555 				phm = NULL;
2556 				while (plist != NULL) {
2557 					pp = plist;
2558 					page_sub(&plist, pp);
2559 					page_io_unlock(pp);
2560 					/* large pages should not end up here */
2561 					ASSERT(pp->p_szc == 0);
2562 					/*LINTED: constant in conditional ctx*/
2563 					VN_DISPOSE(pp, B_INVAL, 0, kcred);
2564 				}
2565 				VM_STAT_ADD(page_create_found_one);
2566 				goto fail;
2567 			}
2568 			ASSERT(flags & PG_WAIT);
2569 			if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) {
2570 				/*
2571 				 * Start all over again if we blocked trying
2572 				 * to lock the page.
2573 				 */
2574 				mutex_exit(phm);
2575 				VM_STAT_ADD(page_create_page_lock_failed);
2576 				phm = NULL;
2577 				goto top;
2578 			}
2579 			mutex_exit(phm);
2580 			phm = NULL;
2581 
2582 			if (PP_ISFREE(pp)) {
2583 				ASSERT(PP_ISAGED(pp) == 0);
2584 				VM_STAT_ADD(pagecnt.pc_get_cache);
2585 				page_list_sub(pp, PG_CACHE_LIST);
2586 				PP_CLRFREE(pp);
2587 				found_on_free++;
2588 			}
2589 		}
2590 
2591 		/*
2592 		 * Got a page!  It is locked.  Acquire the i/o
2593 		 * lock since we are going to use the p_next and
2594 		 * p_prev fields to link the requested pages together.
2595 		 */
2596 		page_io_lock(pp);
2597 		page_add(&plist, pp);
2598 		plist = plist->p_next;
2599 		off += PAGESIZE;
2600 		vaddr += PAGESIZE;
2601 	}
2602 
2603 	ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1);
2604 fail:
2605 	if (npp != NULL) {
2606 		/*
2607 		 * Did not need this page after all.
2608 		 * Put it back on the free list.
2609 		 */
2610 		VM_STAT_ADD(page_create_putbacks);
2611 		PP_SETFREE(npp);
2612 		PP_SETAGED(npp);
2613 		npp->p_offset = (u_offset_t)-1;
2614 		page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
2615 		page_unlock(npp);
2616 
2617 	}
2618 
2619 	ASSERT(pages_req >= found_on_free);
2620 
2621 	{
2622 		uint_t overshoot = (uint_t)(pages_req - found_on_free);
2623 
2624 		if (overshoot) {
2625 			VM_STAT_ADD(page_create_overshoot);
2626 			p = &pcf[pcf_index];
2627 			mutex_enter(&p->pcf_lock);
2628 			if (p->pcf_block) {
2629 				p->pcf_reserve += overshoot;
2630 			} else {
2631 				p->pcf_count += overshoot;
2632 				if (p->pcf_wait) {
2633 					mutex_enter(&new_freemem_lock);
2634 					if (freemem_wait) {
2635 						cv_signal(&freemem_cv);
2636 						p->pcf_wait--;
2637 					} else {
2638 						p->pcf_wait = 0;
2639 					}
2640 					mutex_exit(&new_freemem_lock);
2641 				}
2642 			}
2643 			mutex_exit(&p->pcf_lock);
2644 			/* freemem is approximate, so this test OK */
2645 			if (!p->pcf_block)
2646 				freemem += overshoot;
2647 		}
2648 	}
2649 
2650 	return (plist);
2651 }
2652 
2653 /*
2654  * One or more constituent pages of this large page has been marked
2655  * toxic. Simply demote the large page to PAGESIZE pages and let
2656  * page_free() handle it. This routine should only be called by
2657  * large page free routines (page_free_pages() and page_destroy_pages().
2658  * All pages are locked SE_EXCL and have already been marked free.
2659  */
2660 static void
2661 page_free_toxic_pages(page_t *rootpp)
2662 {
2663 	page_t	*tpp;
2664 	pgcnt_t	i, pgcnt = page_get_pagecnt(rootpp->p_szc);
2665 	uint_t	szc = rootpp->p_szc;
2666 
2667 	for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) {
2668 		ASSERT(tpp->p_szc == szc);
2669 		ASSERT((PAGE_EXCL(tpp) &&
2670 		    !page_iolock_assert(tpp)) || panicstr);
2671 		tpp->p_szc = 0;
2672 	}
2673 
2674 	while (rootpp != NULL) {
2675 		tpp = rootpp;
2676 		page_sub(&rootpp, tpp);
2677 		ASSERT(PP_ISFREE(tpp));
2678 		PP_CLRFREE(tpp);
2679 		page_free(tpp, 1);
2680 	}
2681 }
2682 
2683 /*
2684  * Put page on the "free" list.
2685  * The free list is really two lists maintained by
2686  * the PSM of whatever machine we happen to be on.
2687  */
2688 void
2689 page_free(page_t *pp, int dontneed)
2690 {
2691 	struct pcf	*p;
2692 	uint_t		pcf_index;
2693 
2694 	ASSERT((PAGE_EXCL(pp) &&
2695 	    !page_iolock_assert(pp)) || panicstr);
2696 
2697 	if (PP_ISFREE(pp)) {
2698 		panic("page_free: page %p is free", (void *)pp);
2699 	}
2700 
2701 	if (pp->p_szc != 0) {
2702 		if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
2703 		    pp->p_vnode == &kvp) {
2704 			panic("page_free: anon or kernel "
2705 			    "or no vnode large page %p", (void *)pp);
2706 		}
2707 		page_demote_vp_pages(pp);
2708 		ASSERT(pp->p_szc == 0);
2709 	}
2710 
2711 	/*
2712 	 * The page_struct_lock need not be acquired to examine these
2713 	 * fields since the page has an "exclusive" lock.
2714 	 */
2715 	if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
2716 	    pp->p_slckcnt != 0) {
2717 		panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d "
2718 		    "slckcnt = %d", pp, page_pptonum(pp), pp->p_lckcnt,
2719 		    pp->p_cowcnt, pp->p_slckcnt);
2720 		/*NOTREACHED*/
2721 	}
2722 
2723 	ASSERT(!hat_page_getshare(pp));
2724 
2725 	PP_SETFREE(pp);
2726 	ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) ||
2727 	    !hat_ismod(pp));
2728 	page_clr_all_props(pp);
2729 	ASSERT(!hat_page_getshare(pp));
2730 
2731 	/*
2732 	 * Now we add the page to the head of the free list.
2733 	 * But if this page is associated with a paged vnode
2734 	 * then we adjust the head forward so that the page is
2735 	 * effectively at the end of the list.
2736 	 */
2737 	if (pp->p_vnode == NULL) {
2738 		/*
2739 		 * Page has no identity, put it on the free list.
2740 		 */
2741 		PP_SETAGED(pp);
2742 		pp->p_offset = (u_offset_t)-1;
2743 		page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
2744 		VM_STAT_ADD(pagecnt.pc_free_free);
2745 		TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
2746 		    "page_free_free:pp %p", pp);
2747 	} else {
2748 		PP_CLRAGED(pp);
2749 
2750 		if (!dontneed || nopageage) {
2751 			/* move it to the tail of the list */
2752 			page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL);
2753 
2754 			VM_STAT_ADD(pagecnt.pc_free_cache);
2755 			TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL,
2756 			    "page_free_cache_tail:pp %p", pp);
2757 		} else {
2758 			page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD);
2759 
2760 			VM_STAT_ADD(pagecnt.pc_free_dontneed);
2761 			TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD,
2762 			    "page_free_cache_head:pp %p", pp);
2763 		}
2764 	}
2765 	page_unlock(pp);
2766 
2767 	/*
2768 	 * Now do the `freemem' accounting.
2769 	 */
2770 	pcf_index = PCF_INDEX();
2771 	p = &pcf[pcf_index];
2772 
2773 	mutex_enter(&p->pcf_lock);
2774 	if (p->pcf_block) {
2775 		p->pcf_reserve += 1;
2776 	} else {
2777 		p->pcf_count += 1;
2778 		if (p->pcf_wait) {
2779 			mutex_enter(&new_freemem_lock);
2780 			/*
2781 			 * Check to see if some other thread
2782 			 * is actually waiting.  Another bucket
2783 			 * may have woken it up by now.  If there
2784 			 * are no waiters, then set our pcf_wait
2785 			 * count to zero to avoid coming in here
2786 			 * next time.  Also, since only one page
2787 			 * was put on the free list, just wake
2788 			 * up one waiter.
2789 			 */
2790 			if (freemem_wait) {
2791 				cv_signal(&freemem_cv);
2792 				p->pcf_wait--;
2793 			} else {
2794 				p->pcf_wait = 0;
2795 			}
2796 			mutex_exit(&new_freemem_lock);
2797 		}
2798 	}
2799 	mutex_exit(&p->pcf_lock);
2800 
2801 	/* freemem is approximate, so this test OK */
2802 	if (!p->pcf_block)
2803 		freemem += 1;
2804 }
2805 
2806 /*
2807  * Put page on the "free" list during intial startup.
2808  * This happens during initial single threaded execution.
2809  */
2810 void
2811 page_free_at_startup(page_t *pp)
2812 {
2813 	struct pcf	*p;
2814 	uint_t		pcf_index;
2815 
2816 	page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT);
2817 	VM_STAT_ADD(pagecnt.pc_free_free);
2818 
2819 	/*
2820 	 * Now do the `freemem' accounting.
2821 	 */
2822 	pcf_index = PCF_INDEX();
2823 	p = &pcf[pcf_index];
2824 
2825 	ASSERT(p->pcf_block == 0);
2826 	ASSERT(p->pcf_wait == 0);
2827 	p->pcf_count += 1;
2828 
2829 	/* freemem is approximate, so this is OK */
2830 	freemem += 1;
2831 }
2832 
2833 void
2834 page_free_pages(page_t *pp)
2835 {
2836 	page_t	*tpp, *rootpp = NULL;
2837 	pgcnt_t	pgcnt = page_get_pagecnt(pp->p_szc);
2838 	pgcnt_t	i;
2839 	uint_t	szc = pp->p_szc;
2840 
2841 	VM_STAT_ADD(pagecnt.pc_free_pages);
2842 	TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
2843 	    "page_free_free:pp %p", pp);
2844 
2845 	ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
2846 	if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
2847 		panic("page_free_pages: not root page %p", (void *)pp);
2848 		/*NOTREACHED*/
2849 	}
2850 
2851 	for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
2852 		ASSERT((PAGE_EXCL(tpp) &&
2853 		    !page_iolock_assert(tpp)) || panicstr);
2854 		if (PP_ISFREE(tpp)) {
2855 			panic("page_free_pages: page %p is free", (void *)tpp);
2856 			/*NOTREACHED*/
2857 		}
2858 		if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 ||
2859 		    tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) {
2860 			panic("page_free_pages %p", (void *)tpp);
2861 			/*NOTREACHED*/
2862 		}
2863 
2864 		ASSERT(!hat_page_getshare(tpp));
2865 		ASSERT(tpp->p_vnode == NULL);
2866 		ASSERT(tpp->p_szc == szc);
2867 
2868 		PP_SETFREE(tpp);
2869 		page_clr_all_props(tpp);
2870 		PP_SETAGED(tpp);
2871 		tpp->p_offset = (u_offset_t)-1;
2872 		ASSERT(tpp->p_next == tpp);
2873 		ASSERT(tpp->p_prev == tpp);
2874 		page_list_concat(&rootpp, &tpp);
2875 	}
2876 	ASSERT(rootpp == pp);
2877 
2878 	page_list_add_pages(rootpp, 0);
2879 	page_create_putback(pgcnt);
2880 }
2881 
2882 int free_pages = 1;
2883 
2884 /*
2885  * This routine attempts to return pages to the cachelist via page_release().
2886  * It does not *have* to be successful in all cases, since the pageout scanner
2887  * will catch any pages it misses.  It does need to be fast and not introduce
2888  * too much overhead.
2889  *
2890  * If a page isn't found on the unlocked sweep of the page_hash bucket, we
2891  * don't lock and retry.  This is ok, since the page scanner will eventually
2892  * find any page we miss in free_vp_pages().
2893  */
2894 void
2895 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len)
2896 {
2897 	page_t *pp;
2898 	u_offset_t eoff;
2899 	extern int swap_in_range(vnode_t *, u_offset_t, size_t);
2900 
2901 	eoff = off + len;
2902 
2903 	if (free_pages == 0)
2904 		return;
2905 	if (swap_in_range(vp, off, len))
2906 		return;
2907 
2908 	for (; off < eoff; off += PAGESIZE) {
2909 
2910 		/*
2911 		 * find the page using a fast, but inexact search. It'll be OK
2912 		 * if a few pages slip through the cracks here.
2913 		 */
2914 		pp = page_exists(vp, off);
2915 
2916 		/*
2917 		 * If we didn't find the page (it may not exist), the page
2918 		 * is free, looks still in use (shared), or we can't lock it,
2919 		 * just give up.
2920 		 */
2921 		if (pp == NULL ||
2922 		    PP_ISFREE(pp) ||
2923 		    page_share_cnt(pp) > 0 ||
2924 		    !page_trylock(pp, SE_EXCL))
2925 			continue;
2926 
2927 		/*
2928 		 * Once we have locked pp, verify that it's still the
2929 		 * correct page and not already free
2930 		 */
2931 		ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL));
2932 		if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) {
2933 			page_unlock(pp);
2934 			continue;
2935 		}
2936 
2937 		/*
2938 		 * try to release the page...
2939 		 */
2940 		(void) page_release(pp, 1);
2941 	}
2942 }
2943 
2944 /*
2945  * Reclaim the given page from the free list.
2946  * Returns 1 on success or 0 on failure.
2947  *
2948  * The page is unlocked if it can't be reclaimed (when freemem == 0).
2949  * If `lock' is non-null, it will be dropped and re-acquired if
2950  * the routine must wait while freemem is 0.
2951  *
2952  * As it turns out, boot_getpages() does this.  It picks a page,
2953  * based on where OBP mapped in some address, gets its pfn, searches
2954  * the memsegs, locks the page, then pulls it off the free list!
2955  */
2956 int
2957 page_reclaim(page_t *pp, kmutex_t *lock)
2958 {
2959 	struct pcf	*p;
2960 	uint_t		pcf_index;
2961 	struct cpu	*cpup;
2962 	uint_t		i;
2963 	pgcnt_t		npgs, need;
2964 	pgcnt_t		collected = 0;
2965 
2966 	ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1);
2967 	ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp));
2968 
2969 	npgs = page_get_pagecnt(pp->p_szc);
2970 
2971 	/*
2972 	 * If `freemem' is 0, we cannot reclaim this page from the
2973 	 * freelist, so release every lock we might hold: the page,
2974 	 * and the `lock' before blocking.
2975 	 *
2976 	 * The only way `freemem' can become 0 while there are pages
2977 	 * marked free (have their p->p_free bit set) is when the
2978 	 * system is low on memory and doing a page_create().  In
2979 	 * order to guarantee that once page_create() starts acquiring
2980 	 * pages it will be able to get all that it needs since `freemem'
2981 	 * was decreased by the requested amount.  So, we need to release
2982 	 * this page, and let page_create() have it.
2983 	 *
2984 	 * Since `freemem' being zero is not supposed to happen, just
2985 	 * use the usual hash stuff as a starting point.  If that bucket
2986 	 * is empty, then assume the worst, and start at the beginning
2987 	 * of the pcf array.  If we always start at the beginning
2988 	 * when acquiring more than one pcf lock, there won't be any
2989 	 * deadlock problems.
2990 	 */
2991 
2992 	/* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */
2993 
2994 	if (freemem <= throttlefree && !page_create_throttle(npgs, 0)) {
2995 		pcf_acquire_all();
2996 		goto page_reclaim_nomem;
2997 	}
2998 
2999 	pcf_index = PCF_INDEX();
3000 	p = &pcf[pcf_index];
3001 	mutex_enter(&p->pcf_lock);
3002 	if (p->pcf_count >= npgs) {
3003 		collected = npgs;
3004 		p->pcf_count -= npgs;
3005 	}
3006 	mutex_exit(&p->pcf_lock);
3007 	need = npgs - collected;
3008 
3009 	if (need > 0) {
3010 		VM_STAT_ADD(page_reclaim_zero);
3011 		/*
3012 		 * Check again. Its possible that some other thread
3013 		 * could have been right behind us, and added one
3014 		 * to a list somewhere.  Acquire each of the pcf locks
3015 		 * until we find a page.
3016 		 */
3017 		p = pcf;
3018 		for (i = 0; i < PCF_FANOUT; i++) {
3019 			mutex_enter(&p->pcf_lock);
3020 			if (p->pcf_count) {
3021 				if (p->pcf_count >= need) {
3022 					p->pcf_count -= need;
3023 					collected += need;
3024 					need = 0;
3025 					break;
3026 				} else if (p->pcf_count) {
3027 					collected += p->pcf_count;
3028 					need -= p->pcf_count;
3029 					p->pcf_count = 0;
3030 				}
3031 			}
3032 			p++;
3033 		}
3034 
3035 		if (need > 0) {
3036 page_reclaim_nomem:
3037 			/*
3038 			 * We really can't have page `pp'.
3039 			 * Time for the no-memory dance with
3040 			 * page_free().  This is just like
3041 			 * page_create_wait().  Plus the added
3042 			 * attraction of releasing whatever mutex
3043 			 * we held when we were called with in `lock'.
3044 			 * Page_unlock() will wakeup any thread
3045 			 * waiting around for this page.
3046 			 */
3047 			if (lock) {
3048 				VM_STAT_ADD(page_reclaim_zero_locked);
3049 				mutex_exit(lock);
3050 			}
3051 			page_unlock(pp);
3052 
3053 			/*
3054 			 * get this before we drop all the pcf locks.
3055 			 */
3056 			mutex_enter(&new_freemem_lock);
3057 
3058 			p = pcf;
3059 			p->pcf_count += collected;
3060 			for (i = 0; i < PCF_FANOUT; i++) {
3061 				p->pcf_wait++;
3062 				mutex_exit(&p->pcf_lock);
3063 				p++;
3064 			}
3065 
3066 			freemem_wait++;
3067 			cv_wait(&freemem_cv, &new_freemem_lock);
3068 			freemem_wait--;
3069 
3070 			mutex_exit(&new_freemem_lock);
3071 
3072 			if (lock) {
3073 				mutex_enter(lock);
3074 			}
3075 			return (0);
3076 		}
3077 
3078 		/*
3079 		 * We beat the PCF bins over the head until
3080 		 * we got the memory that we wanted.
3081 		 * The pcf accounting has been done,
3082 		 * though none of the pcf_wait flags have been set,
3083 		 * drop the locks and continue on.
3084 		 */
3085 		ASSERT(collected == npgs);
3086 		while (p >= pcf) {
3087 			mutex_exit(&p->pcf_lock);
3088 			p--;
3089 		}
3090 	}
3091 
3092 	/*
3093 	 * freemem is not protected by any lock. Thus, we cannot
3094 	 * have any assertion containing freemem here.
3095 	 */
3096 	freemem -= npgs;
3097 
3098 	VM_STAT_ADD(pagecnt.pc_reclaim);
3099 	if (PP_ISAGED(pp)) {
3100 		if (npgs > 1) {
3101 			page_list_sub_pages(pp, pp->p_szc);
3102 		} else {
3103 			page_list_sub(pp, PG_FREE_LIST);
3104 		}
3105 		TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE,
3106 		    "page_reclaim_free:pp %p", pp);
3107 	} else {
3108 		ASSERT(npgs == 1);
3109 		page_list_sub(pp, PG_CACHE_LIST);
3110 		TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE,
3111 		    "page_reclaim_cache:pp %p", pp);
3112 	}
3113 
3114 	/*
3115 	 * clear the p_free & p_age bits since this page is no longer
3116 	 * on the free list.  Notice that there was a brief time where
3117 	 * a page is marked as free, but is not on the list.
3118 	 *
3119 	 * Set the reference bit to protect against immediate pageout.
3120 	 */
3121 	for (i = 0; i < npgs; i++, pp++) {
3122 		PP_CLRFREE(pp);
3123 		PP_CLRAGED(pp);
3124 		page_set_props(pp, P_REF);
3125 	}
3126 
3127 	CPU_STATS_ENTER_K();
3128 	cpup = CPU;	/* get cpup now that CPU cannot change */
3129 	CPU_STATS_ADDQ(cpup, vm, pgrec, 1);
3130 	CPU_STATS_ADDQ(cpup, vm, pgfrec, 1);
3131 	CPU_STATS_EXIT_K();
3132 
3133 	return (1);
3134 }
3135 
3136 
3137 
3138 /*
3139  * Destroy identity of the page and put it back on
3140  * the page free list.  Assumes that the caller has
3141  * acquired the "exclusive" lock on the page.
3142  */
3143 void
3144 page_destroy(page_t *pp, int dontfree)
3145 {
3146 	ASSERT((PAGE_EXCL(pp) &&
3147 	    !page_iolock_assert(pp)) || panicstr);
3148 	ASSERT(pp->p_slckcnt == 0 || panicstr);
3149 
3150 	if (pp->p_szc != 0) {
3151 		if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
3152 		    pp->p_vnode == &kvp) {
3153 			panic("page_destroy: anon or kernel or no vnode "
3154 			    "large page %p", (void *)pp);
3155 		}
3156 		page_demote_vp_pages(pp);
3157 		ASSERT(pp->p_szc == 0);
3158 	}
3159 
3160 	TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp);
3161 
3162 	/*
3163 	 * Unload translations, if any, then hash out the
3164 	 * page to erase its identity.
3165 	 */
3166 	(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
3167 	page_hashout(pp, NULL);
3168 
3169 	if (!dontfree) {
3170 		/*
3171 		 * Acquire the "freemem_lock" for availrmem.
3172 		 * The page_struct_lock need not be acquired for lckcnt
3173 		 * and cowcnt since the page has an "exclusive" lock.
3174 		 */
3175 		if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) {
3176 			mutex_enter(&freemem_lock);
3177 			if (pp->p_lckcnt != 0) {
3178 				availrmem++;
3179 				pp->p_lckcnt = 0;
3180 			}
3181 			if (pp->p_cowcnt != 0) {
3182 				availrmem += pp->p_cowcnt;
3183 				pp->p_cowcnt = 0;
3184 			}
3185 			mutex_exit(&freemem_lock);
3186 		}
3187 		/*
3188 		 * Put the page on the "free" list.
3189 		 */
3190 		page_free(pp, 0);
3191 	}
3192 }
3193 
3194 void
3195 page_destroy_pages(page_t *pp)
3196 {
3197 
3198 	page_t	*tpp, *rootpp = NULL;
3199 	pgcnt_t	pgcnt = page_get_pagecnt(pp->p_szc);
3200 	pgcnt_t	i, pglcks = 0;
3201 	uint_t	szc = pp->p_szc;
3202 
3203 	ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
3204 
3205 	VM_STAT_ADD(pagecnt.pc_destroy_pages);
3206 
3207 	TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp);
3208 
3209 	if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
3210 		panic("page_destroy_pages: not root page %p", (void *)pp);
3211 		/*NOTREACHED*/
3212 	}
3213 
3214 	for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
3215 		ASSERT((PAGE_EXCL(tpp) &&
3216 		    !page_iolock_assert(tpp)) || panicstr);
3217 		ASSERT(tpp->p_slckcnt == 0 || panicstr);
3218 		(void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
3219 		page_hashout(tpp, NULL);
3220 		ASSERT(tpp->p_offset == (u_offset_t)-1);
3221 		if (tpp->p_lckcnt != 0) {
3222 			pglcks++;
3223 			tpp->p_lckcnt = 0;
3224 		} else if (tpp->p_cowcnt != 0) {
3225 			pglcks += tpp->p_cowcnt;
3226 			tpp->p_cowcnt = 0;
3227 		}
3228 		ASSERT(!hat_page_getshare(tpp));
3229 		ASSERT(tpp->p_vnode == NULL);
3230 		ASSERT(tpp->p_szc == szc);
3231 
3232 		PP_SETFREE(tpp);
3233 		page_clr_all_props(tpp);
3234 		PP_SETAGED(tpp);
3235 		ASSERT(tpp->p_next == tpp);
3236 		ASSERT(tpp->p_prev == tpp);
3237 		page_list_concat(&rootpp, &tpp);
3238 	}
3239 
3240 	ASSERT(rootpp == pp);
3241 	if (pglcks != 0) {
3242 		mutex_enter(&freemem_lock);
3243 		availrmem += pglcks;
3244 		mutex_exit(&freemem_lock);
3245 	}
3246 
3247 	page_list_add_pages(rootpp, 0);
3248 	page_create_putback(pgcnt);
3249 }
3250 
3251 /*
3252  * Similar to page_destroy(), but destroys pages which are
3253  * locked and known to be on the page free list.  Since
3254  * the page is known to be free and locked, no one can access
3255  * it.
3256  *
3257  * Also, the number of free pages does not change.
3258  */
3259 void
3260 page_destroy_free(page_t *pp)
3261 {
3262 	ASSERT(PAGE_EXCL(pp));
3263 	ASSERT(PP_ISFREE(pp));
3264 	ASSERT(pp->p_vnode);
3265 	ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0);
3266 	ASSERT(!hat_page_is_mapped(pp));
3267 	ASSERT(PP_ISAGED(pp) == 0);
3268 	ASSERT(pp->p_szc == 0);
3269 
3270 	VM_STAT_ADD(pagecnt.pc_destroy_free);
3271 	page_list_sub(pp, PG_CACHE_LIST);
3272 
3273 	page_hashout(pp, NULL);
3274 	ASSERT(pp->p_vnode == NULL);
3275 	ASSERT(pp->p_offset == (u_offset_t)-1);
3276 	ASSERT(pp->p_hash == NULL);
3277 
3278 	PP_SETAGED(pp);
3279 	page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
3280 	page_unlock(pp);
3281 
3282 	mutex_enter(&new_freemem_lock);
3283 	if (freemem_wait) {
3284 		cv_signal(&freemem_cv);
3285 	}
3286 	mutex_exit(&new_freemem_lock);
3287 }
3288 
3289 /*
3290  * Rename the page "opp" to have an identity specified
3291  * by [vp, off].  If a page already exists with this name
3292  * it is locked and destroyed.  Note that the page's
3293  * translations are not unloaded during the rename.
3294  *
3295  * This routine is used by the anon layer to "steal" the
3296  * original page and is not unlike destroying a page and
3297  * creating a new page using the same page frame.
3298  *
3299  * XXX -- Could deadlock if caller 1 tries to rename A to B while
3300  * caller 2 tries to rename B to A.
3301  */
3302 void
3303 page_rename(page_t *opp, vnode_t *vp, u_offset_t off)
3304 {
3305 	page_t		*pp;
3306 	int		olckcnt = 0;
3307 	int		ocowcnt = 0;
3308 	kmutex_t	*phm;
3309 	ulong_t		index;
3310 
3311 	ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp));
3312 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3313 	ASSERT(PP_ISFREE(opp) == 0);
3314 
3315 	VM_STAT_ADD(page_rename_count);
3316 
3317 	TRACE_3(TR_FAC_VM, TR_PAGE_RENAME,
3318 		"page rename:pp %p vp %p off %llx", opp, vp, off);
3319 
3320 	/*
3321 	 * CacheFS may call page_rename for a large NFS page
3322 	 * when both CacheFS and NFS mount points are used
3323 	 * by applications. Demote this large page before
3324 	 * renaming it, to ensure that there are no "partial"
3325 	 * large pages left lying around.
3326 	 */
3327 	if (opp->p_szc != 0) {
3328 		vnode_t *ovp = opp->p_vnode;
3329 		ASSERT(ovp != NULL);
3330 		ASSERT(!IS_SWAPFSVP(ovp));
3331 		ASSERT(ovp != &kvp);
3332 		page_demote_vp_pages(opp);
3333 		ASSERT(opp->p_szc == 0);
3334 	}
3335 
3336 	page_hashout(opp, NULL);
3337 	PP_CLRAGED(opp);
3338 
3339 	/*
3340 	 * Acquire the appropriate page hash lock, since
3341 	 * we're going to rename the page.
3342 	 */
3343 	index = PAGE_HASH_FUNC(vp, off);
3344 	phm = PAGE_HASH_MUTEX(index);
3345 	mutex_enter(phm);
3346 top:
3347 	/*
3348 	 * Look for an existing page with this name and destroy it if found.
3349 	 * By holding the page hash lock all the way to the page_hashin()
3350 	 * call, we are assured that no page can be created with this
3351 	 * identity.  In the case when the phm lock is dropped to undo any
3352 	 * hat layer mappings, the existing page is held with an "exclusive"
3353 	 * lock, again preventing another page from being created with
3354 	 * this identity.
3355 	 */
3356 	PAGE_HASH_SEARCH(index, pp, vp, off);
3357 	if (pp != NULL) {
3358 		VM_STAT_ADD(page_rename_exists);
3359 
3360 		/*
3361 		 * As it turns out, this is one of only two places where
3362 		 * page_lock() needs to hold the passed in lock in the
3363 		 * successful case.  In all of the others, the lock could
3364 		 * be dropped as soon as the attempt is made to lock
3365 		 * the page.  It is tempting to add yet another arguement,
3366 		 * PL_KEEP or PL_DROP, to let page_lock know what to do.
3367 		 */
3368 		if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) {
3369 			/*
3370 			 * Went to sleep because the page could not
3371 			 * be locked.  We were woken up when the page
3372 			 * was unlocked, or when the page was destroyed.
3373 			 * In either case, `phm' was dropped while we
3374 			 * slept.  Hence we should not just roar through
3375 			 * this loop.
3376 			 */
3377 			goto top;
3378 		}
3379 
3380 		/*
3381 		 * If an existing page is a large page, then demote
3382 		 * it to ensure that no "partial" large pages are
3383 		 * "created" after page_rename. An existing page
3384 		 * can be a CacheFS page, and can't belong to swapfs.
3385 		 */
3386 		if (hat_page_is_mapped(pp)) {
3387 			/*
3388 			 * Unload translations.  Since we hold the
3389 			 * exclusive lock on this page, the page
3390 			 * can not be changed while we drop phm.
3391 			 * This is also not a lock protocol violation,
3392 			 * but rather the proper way to do things.
3393 			 */
3394 			mutex_exit(phm);
3395 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
3396 			if (pp->p_szc != 0) {
3397 				ASSERT(!IS_SWAPFSVP(vp));
3398 				ASSERT(vp != &kvp);
3399 				page_demote_vp_pages(pp);
3400 				ASSERT(pp->p_szc == 0);
3401 			}
3402 			mutex_enter(phm);
3403 		} else if (pp->p_szc != 0) {
3404 			ASSERT(!IS_SWAPFSVP(vp));
3405 			ASSERT(vp != &kvp);
3406 			mutex_exit(phm);
3407 			page_demote_vp_pages(pp);
3408 			ASSERT(pp->p_szc == 0);
3409 			mutex_enter(phm);
3410 		}
3411 		page_hashout(pp, phm);
3412 	}
3413 	/*
3414 	 * Hash in the page with the new identity.
3415 	 */
3416 	if (!page_hashin(opp, vp, off, phm)) {
3417 		/*
3418 		 * We were holding phm while we searched for [vp, off]
3419 		 * and only dropped phm if we found and locked a page.
3420 		 * If we can't create this page now, then some thing
3421 		 * is really broken.
3422 		 */
3423 		panic("page_rename: Can't hash in page: %p", (void *)pp);
3424 		/*NOTREACHED*/
3425 	}
3426 
3427 	ASSERT(MUTEX_HELD(phm));
3428 	mutex_exit(phm);
3429 
3430 	/*
3431 	 * Now that we have dropped phm, lets get around to finishing up
3432 	 * with pp.
3433 	 */
3434 	if (pp != NULL) {
3435 		ASSERT(!hat_page_is_mapped(pp));
3436 		/* for now large pages should not end up here */
3437 		ASSERT(pp->p_szc == 0);
3438 		/*
3439 		 * Save the locks for transfer to the new page and then
3440 		 * clear them so page_free doesn't think they're important.
3441 		 * The page_struct_lock need not be acquired for lckcnt and
3442 		 * cowcnt since the page has an "exclusive" lock.
3443 		 */
3444 		olckcnt = pp->p_lckcnt;
3445 		ocowcnt = pp->p_cowcnt;
3446 		pp->p_lckcnt = pp->p_cowcnt = 0;
3447 
3448 		/*
3449 		 * Put the page on the "free" list after we drop
3450 		 * the lock.  The less work under the lock the better.
3451 		 */
3452 		/*LINTED: constant in conditional context*/
3453 		VN_DISPOSE(pp, B_FREE, 0, kcred);
3454 	}
3455 
3456 	/*
3457 	 * Transfer the lock count from the old page (if any).
3458 	 * The page_struct_lock need not be acquired for lckcnt and
3459 	 * cowcnt since the page has an "exclusive" lock.
3460 	 */
3461 	opp->p_lckcnt += olckcnt;
3462 	opp->p_cowcnt += ocowcnt;
3463 }
3464 
3465 /*
3466  * low level routine to add page `pp' to the hash and vp chains for [vp, offset]
3467  *
3468  * Pages are normally inserted at the start of a vnode's v_pages list.
3469  * If the vnode is VMODSORT and the page is modified, it goes at the end.
3470  * This can happen when a modified page is relocated for DR.
3471  *
3472  * Returns 1 on success and 0 on failure.
3473  */
3474 static int
3475 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset)
3476 {
3477 	page_t		**listp;
3478 	page_t		*tp;
3479 	ulong_t		index;
3480 
3481 	ASSERT(PAGE_EXCL(pp));
3482 	ASSERT(vp != NULL);
3483 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
3484 
3485 	/*
3486 	 * Be sure to set these up before the page is inserted on the hash
3487 	 * list.  As soon as the page is placed on the list some other
3488 	 * thread might get confused and wonder how this page could
3489 	 * possibly hash to this list.
3490 	 */
3491 	pp->p_vnode = vp;
3492 	pp->p_offset = offset;
3493 
3494 	/*
3495 	 * record if this page is on a swap vnode
3496 	 */
3497 	if ((vp->v_flag & VISSWAP) != 0)
3498 		PP_SETSWAP(pp);
3499 
3500 	index = PAGE_HASH_FUNC(vp, offset);
3501 	ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index)));
3502 	listp = &page_hash[index];
3503 
3504 	/*
3505 	 * If this page is already hashed in, fail this attempt to add it.
3506 	 */
3507 	for (tp = *listp; tp != NULL; tp = tp->p_hash) {
3508 		if (tp->p_vnode == vp && tp->p_offset == offset) {
3509 			pp->p_vnode = NULL;
3510 			pp->p_offset = (u_offset_t)(-1);
3511 			return (0);
3512 		}
3513 	}
3514 	pp->p_hash = *listp;
3515 	*listp = pp;
3516 
3517 	/*
3518 	 * Add the page to the vnode's list of pages
3519 	 */
3520 	if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp))
3521 		listp = &vp->v_pages->p_vpprev->p_vpnext;
3522 	else
3523 		listp = &vp->v_pages;
3524 
3525 	page_vpadd(listp, pp);
3526 
3527 	return (1);
3528 }
3529 
3530 /*
3531  * Add page `pp' to both the hash and vp chains for [vp, offset].
3532  *
3533  * Returns 1 on success and 0 on failure.
3534  * If hold is passed in, it is not dropped.
3535  */
3536 int
3537 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold)
3538 {
3539 	kmutex_t	*phm = NULL;
3540 	kmutex_t	*vphm;
3541 	int		rc;
3542 
3543 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
3544 
3545 	TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN,
3546 		"page_hashin:pp %p vp %p offset %llx",
3547 		pp, vp, offset);
3548 
3549 	VM_STAT_ADD(hashin_count);
3550 
3551 	if (hold != NULL)
3552 		phm = hold;
3553 	else {
3554 		VM_STAT_ADD(hashin_not_held);
3555 		phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset));
3556 		mutex_enter(phm);
3557 	}
3558 
3559 	vphm = page_vnode_mutex(vp);
3560 	mutex_enter(vphm);
3561 	rc = page_do_hashin(pp, vp, offset);
3562 	mutex_exit(vphm);
3563 	if (hold == NULL)
3564 		mutex_exit(phm);
3565 	if (rc == 0)
3566 		VM_STAT_ADD(hashin_already);
3567 	return (rc);
3568 }
3569 
3570 /*
3571  * Remove page ``pp'' from the hash and vp chains and remove vp association.
3572  * All mutexes must be held
3573  */
3574 static void
3575 page_do_hashout(page_t *pp)
3576 {
3577 	page_t	**hpp;
3578 	page_t	*hp;
3579 	vnode_t	*vp = pp->p_vnode;
3580 
3581 	ASSERT(vp != NULL);
3582 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
3583 
3584 	/*
3585 	 * First, take pp off of its hash chain.
3586 	 */
3587 	hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)];
3588 
3589 	for (;;) {
3590 		hp = *hpp;
3591 		if (hp == pp)
3592 			break;
3593 		if (hp == NULL) {
3594 			panic("page_do_hashout");
3595 			/*NOTREACHED*/
3596 		}
3597 		hpp = &hp->p_hash;
3598 	}
3599 	*hpp = pp->p_hash;
3600 
3601 	/*
3602 	 * Now remove it from its associated vnode.
3603 	 */
3604 	if (vp->v_pages)
3605 		page_vpsub(&vp->v_pages, pp);
3606 
3607 	pp->p_hash = NULL;
3608 	page_clr_all_props(pp);
3609 	PP_CLRSWAP(pp);
3610 	pp->p_vnode = NULL;
3611 	pp->p_offset = (u_offset_t)-1;
3612 }
3613 
3614 /*
3615  * Remove page ``pp'' from the hash and vp chains and remove vp association.
3616  *
3617  * When `phm' is non-NULL it contains the address of the mutex protecting the
3618  * hash list pp is on.  It is not dropped.
3619  */
3620 void
3621 page_hashout(page_t *pp, kmutex_t *phm)
3622 {
3623 	vnode_t		*vp;
3624 	ulong_t		index;
3625 	kmutex_t	*nphm;
3626 	kmutex_t	*vphm;
3627 	kmutex_t	*sep;
3628 
3629 	ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1);
3630 	ASSERT(pp->p_vnode != NULL);
3631 	ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr);
3632 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode)));
3633 
3634 	vp = pp->p_vnode;
3635 
3636 	TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT,
3637 		"page_hashout:pp %p vp %p", pp, vp);
3638 
3639 	/* Kernel probe */
3640 	TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */,
3641 	    tnf_opaque, vnode, vp,
3642 	    tnf_offset, offset, pp->p_offset);
3643 
3644 	/*
3645 	 *
3646 	 */
3647 	VM_STAT_ADD(hashout_count);
3648 	index = PAGE_HASH_FUNC(vp, pp->p_offset);
3649 	if (phm == NULL) {
3650 		VM_STAT_ADD(hashout_not_held);
3651 		nphm = PAGE_HASH_MUTEX(index);
3652 		mutex_enter(nphm);
3653 	}
3654 	ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1);
3655 
3656 
3657 	/*
3658 	 * grab page vnode mutex and remove it...
3659 	 */
3660 	vphm = page_vnode_mutex(vp);
3661 	mutex_enter(vphm);
3662 
3663 	page_do_hashout(pp);
3664 
3665 	mutex_exit(vphm);
3666 	if (phm == NULL)
3667 		mutex_exit(nphm);
3668 
3669 	/*
3670 	 * Wake up processes waiting for this page.  The page's
3671 	 * identity has been changed, and is probably not the
3672 	 * desired page any longer.
3673 	 */
3674 	sep = page_se_mutex(pp);
3675 	mutex_enter(sep);
3676 	pp->p_selock &= ~SE_EWANTED;
3677 	if (CV_HAS_WAITERS(&pp->p_cv))
3678 		cv_broadcast(&pp->p_cv);
3679 	mutex_exit(sep);
3680 }
3681 
3682 /*
3683  * Add the page to the front of a linked list of pages
3684  * using the p_next & p_prev pointers for the list.
3685  * The caller is responsible for protecting the list pointers.
3686  */
3687 void
3688 page_add(page_t **ppp, page_t *pp)
3689 {
3690 	ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
3691 
3692 	page_add_common(ppp, pp);
3693 }
3694 
3695 
3696 
3697 /*
3698  *  Common code for page_add() and mach_page_add()
3699  */
3700 void
3701 page_add_common(page_t **ppp, page_t *pp)
3702 {
3703 	if (*ppp == NULL) {
3704 		pp->p_next = pp->p_prev = pp;
3705 	} else {
3706 		pp->p_next = *ppp;
3707 		pp->p_prev = (*ppp)->p_prev;
3708 		(*ppp)->p_prev = pp;
3709 		pp->p_prev->p_next = pp;
3710 	}
3711 	*ppp = pp;
3712 }
3713 
3714 
3715 /*
3716  * Remove this page from a linked list of pages
3717  * using the p_next & p_prev pointers for the list.
3718  *
3719  * The caller is responsible for protecting the list pointers.
3720  */
3721 void
3722 page_sub(page_t **ppp, page_t *pp)
3723 {
3724 	ASSERT((PP_ISFREE(pp)) ? 1 :
3725 	    (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
3726 
3727 	if (*ppp == NULL || pp == NULL) {
3728 		panic("page_sub: bad arg(s): pp %p, *ppp %p",
3729 		    (void *)pp, (void *)(*ppp));
3730 		/*NOTREACHED*/
3731 	}
3732 
3733 	page_sub_common(ppp, pp);
3734 }
3735 
3736 
3737 /*
3738  *  Common code for page_sub() and mach_page_sub()
3739  */
3740 void
3741 page_sub_common(page_t **ppp, page_t *pp)
3742 {
3743 	if (*ppp == pp)
3744 		*ppp = pp->p_next;		/* go to next page */
3745 
3746 	if (*ppp == pp)
3747 		*ppp = NULL;			/* page list is gone */
3748 	else {
3749 		pp->p_prev->p_next = pp->p_next;
3750 		pp->p_next->p_prev = pp->p_prev;
3751 	}
3752 	pp->p_prev = pp->p_next = pp;		/* make pp a list of one */
3753 }
3754 
3755 
3756 /*
3757  * Break page list cppp into two lists with npages in the first list.
3758  * The tail is returned in nppp.
3759  */
3760 void
3761 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages)
3762 {
3763 	page_t *s1pp = *oppp;
3764 	page_t *s2pp;
3765 	page_t *e1pp, *e2pp;
3766 	long n = 0;
3767 
3768 	if (s1pp == NULL) {
3769 		*nppp = NULL;
3770 		return;
3771 	}
3772 	if (npages == 0) {
3773 		*nppp = s1pp;
3774 		*oppp = NULL;
3775 		return;
3776 	}
3777 	for (n = 0, s2pp = *oppp; n < npages; n++) {
3778 		s2pp = s2pp->p_next;
3779 	}
3780 	/* Fix head and tail of new lists */
3781 	e1pp = s2pp->p_prev;
3782 	e2pp = s1pp->p_prev;
3783 	s1pp->p_prev = e1pp;
3784 	e1pp->p_next = s1pp;
3785 	s2pp->p_prev = e2pp;
3786 	e2pp->p_next = s2pp;
3787 
3788 	/* second list empty */
3789 	if (s2pp == s1pp) {
3790 		*oppp = s1pp;
3791 		*nppp = NULL;
3792 	} else {
3793 		*oppp = s1pp;
3794 		*nppp = s2pp;
3795 	}
3796 }
3797 
3798 /*
3799  * Concatenate page list nppp onto the end of list ppp.
3800  */
3801 void
3802 page_list_concat(page_t **ppp, page_t **nppp)
3803 {
3804 	page_t *s1pp, *s2pp, *e1pp, *e2pp;
3805 
3806 	if (*nppp == NULL) {
3807 		return;
3808 	}
3809 	if (*ppp == NULL) {
3810 		*ppp = *nppp;
3811 		return;
3812 	}
3813 	s1pp = *ppp;
3814 	e1pp =  s1pp->p_prev;
3815 	s2pp = *nppp;
3816 	e2pp = s2pp->p_prev;
3817 	s1pp->p_prev = e2pp;
3818 	e2pp->p_next = s1pp;
3819 	e1pp->p_next = s2pp;
3820 	s2pp->p_prev = e1pp;
3821 }
3822 
3823 /*
3824  * return the next page in the page list
3825  */
3826 page_t *
3827 page_list_next(page_t *pp)
3828 {
3829 	return (pp->p_next);
3830 }
3831 
3832 
3833 /*
3834  * Add the page to the front of the linked list of pages
3835  * using p_vpnext/p_vpprev pointers for the list.
3836  *
3837  * The caller is responsible for protecting the lists.
3838  */
3839 void
3840 page_vpadd(page_t **ppp, page_t *pp)
3841 {
3842 	if (*ppp == NULL) {
3843 		pp->p_vpnext = pp->p_vpprev = pp;
3844 	} else {
3845 		pp->p_vpnext = *ppp;
3846 		pp->p_vpprev = (*ppp)->p_vpprev;
3847 		(*ppp)->p_vpprev = pp;
3848 		pp->p_vpprev->p_vpnext = pp;
3849 	}
3850 	*ppp = pp;
3851 }
3852 
3853 /*
3854  * Remove this page from the linked list of pages
3855  * using p_vpnext/p_vpprev pointers for the list.
3856  *
3857  * The caller is responsible for protecting the lists.
3858  */
3859 void
3860 page_vpsub(page_t **ppp, page_t *pp)
3861 {
3862 	if (*ppp == NULL || pp == NULL) {
3863 		panic("page_vpsub: bad arg(s): pp %p, *ppp %p",
3864 		    (void *)pp, (void *)(*ppp));
3865 		/*NOTREACHED*/
3866 	}
3867 
3868 	if (*ppp == pp)
3869 		*ppp = pp->p_vpnext;		/* go to next page */
3870 
3871 	if (*ppp == pp)
3872 		*ppp = NULL;			/* page list is gone */
3873 	else {
3874 		pp->p_vpprev->p_vpnext = pp->p_vpnext;
3875 		pp->p_vpnext->p_vpprev = pp->p_vpprev;
3876 	}
3877 	pp->p_vpprev = pp->p_vpnext = pp;	/* make pp a list of one */
3878 }
3879 
3880 /*
3881  * Lock a physical page into memory "long term".  Used to support "lock
3882  * in memory" functions.  Accepts the page to be locked, and a cow variable
3883  * to indicate whether a the lock will travel to the new page during
3884  * a potential copy-on-write.
3885  */
3886 int
3887 page_pp_lock(
3888 	page_t *pp,			/* page to be locked */
3889 	int cow,			/* cow lock */
3890 	int kernel)			/* must succeed -- ignore checking */
3891 {
3892 	int r = 0;			/* result -- assume failure */
3893 
3894 	ASSERT(PAGE_LOCKED(pp));
3895 
3896 	page_struct_lock(pp);
3897 	/*
3898 	 * Acquire the "freemem_lock" for availrmem.
3899 	 */
3900 	if (cow) {
3901 		mutex_enter(&freemem_lock);
3902 		if ((availrmem > pages_pp_maximum) &&
3903 		    (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
3904 			availrmem--;
3905 			pages_locked++;
3906 			mutex_exit(&freemem_lock);
3907 			r = 1;
3908 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
3909 				cmn_err(CE_WARN,
3910 				    "COW lock limit reached on pfn 0x%lx",
3911 				    page_pptonum(pp));
3912 			}
3913 		} else
3914 			mutex_exit(&freemem_lock);
3915 	} else {
3916 		if (pp->p_lckcnt) {
3917 			if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
3918 				r = 1;
3919 				if (++pp->p_lckcnt ==
3920 				    (ushort_t)PAGE_LOCK_MAXIMUM) {
3921 					cmn_err(CE_WARN, "Page lock limit "
3922 					    "reached on pfn 0x%lx",
3923 					    page_pptonum(pp));
3924 				}
3925 			}
3926 		} else {
3927 			if (kernel) {
3928 				/* availrmem accounting done by caller */
3929 				++pp->p_lckcnt;
3930 				r = 1;
3931 			} else {
3932 				mutex_enter(&freemem_lock);
3933 				if (availrmem > pages_pp_maximum) {
3934 					availrmem--;
3935 					pages_locked++;
3936 					++pp->p_lckcnt;
3937 					r = 1;
3938 				}
3939 				mutex_exit(&freemem_lock);
3940 			}
3941 		}
3942 	}
3943 	page_struct_unlock(pp);
3944 	return (r);
3945 }
3946 
3947 /*
3948  * Decommit a lock on a physical page frame.  Account for cow locks if
3949  * appropriate.
3950  */
3951 void
3952 page_pp_unlock(
3953 	page_t *pp,			/* page to be unlocked */
3954 	int cow,			/* expect cow lock */
3955 	int kernel)			/* this was a kernel lock */
3956 {
3957 	ASSERT(PAGE_LOCKED(pp));
3958 
3959 	page_struct_lock(pp);
3960 	/*
3961 	 * Acquire the "freemem_lock" for availrmem.
3962 	 * If cowcnt or lcknt is already 0 do nothing; i.e., we
3963 	 * could be called to unlock even if nothing is locked. This could
3964 	 * happen if locked file pages were truncated (removing the lock)
3965 	 * and the file was grown again and new pages faulted in; the new
3966 	 * pages are unlocked but the segment still thinks they're locked.
3967 	 */
3968 	if (cow) {
3969 		if (pp->p_cowcnt) {
3970 			mutex_enter(&freemem_lock);
3971 			pp->p_cowcnt--;
3972 			availrmem++;
3973 			pages_locked--;
3974 			mutex_exit(&freemem_lock);
3975 		}
3976 	} else {
3977 		if (pp->p_lckcnt && --pp->p_lckcnt == 0) {
3978 			if (!kernel) {
3979 				mutex_enter(&freemem_lock);
3980 				availrmem++;
3981 				pages_locked--;
3982 				mutex_exit(&freemem_lock);
3983 			}
3984 		}
3985 	}
3986 	page_struct_unlock(pp);
3987 }
3988 
3989 /*
3990  * This routine reserves availrmem for npages;
3991  * 	flags: KM_NOSLEEP or KM_SLEEP
3992  * 	returns 1 on success or 0 on failure
3993  */
3994 int
3995 page_resv(pgcnt_t npages, uint_t flags)
3996 {
3997 	mutex_enter(&freemem_lock);
3998 	while (availrmem < tune.t_minarmem + npages) {
3999 		if (flags & KM_NOSLEEP) {
4000 			mutex_exit(&freemem_lock);
4001 			return (0);
4002 		}
4003 		mutex_exit(&freemem_lock);
4004 		page_needfree(npages);
4005 		kmem_reap();
4006 		delay(hz >> 2);
4007 		page_needfree(-(spgcnt_t)npages);
4008 		mutex_enter(&freemem_lock);
4009 	}
4010 	availrmem -= npages;
4011 	mutex_exit(&freemem_lock);
4012 	return (1);
4013 }
4014 
4015 /*
4016  * This routine unreserves availrmem for npages;
4017  */
4018 void
4019 page_unresv(pgcnt_t npages)
4020 {
4021 	mutex_enter(&freemem_lock);
4022 	availrmem += npages;
4023 	mutex_exit(&freemem_lock);
4024 }
4025 
4026 /*
4027  * See Statement at the beginning of segvn_lockop() regarding
4028  * the way we handle cowcnts and lckcnts.
4029  *
4030  * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage
4031  * that breaks COW has PROT_WRITE.
4032  *
4033  * Note that, we may also break COW in case we are softlocking
4034  * on read access during physio;
4035  * in this softlock case, the vpage may not have PROT_WRITE.
4036  * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp'
4037  * if the vpage doesn't have PROT_WRITE.
4038  *
4039  * This routine is never called if we are stealing a page
4040  * in anon_private.
4041  *
4042  * The caller subtracted from availrmem for read only mapping.
4043  * if lckcnt is 1 increment availrmem.
4044  */
4045 void
4046 page_pp_useclaim(
4047 	page_t *opp,		/* original page frame losing lock */
4048 	page_t *npp,		/* new page frame gaining lock */
4049 	uint_t	write_perm) 	/* set if vpage has PROT_WRITE */
4050 {
4051 	int payback = 0;
4052 
4053 	ASSERT(PAGE_LOCKED(opp));
4054 	ASSERT(PAGE_LOCKED(npp));
4055 
4056 	page_struct_lock(opp);
4057 
4058 	ASSERT(npp->p_cowcnt == 0);
4059 	ASSERT(npp->p_lckcnt == 0);
4060 
4061 	/* Don't use claim if nothing is locked (see page_pp_unlock above) */
4062 	if ((write_perm && opp->p_cowcnt != 0) ||
4063 	    (!write_perm && opp->p_lckcnt != 0)) {
4064 
4065 		if (write_perm) {
4066 			npp->p_cowcnt++;
4067 			ASSERT(opp->p_cowcnt != 0);
4068 			opp->p_cowcnt--;
4069 		} else {
4070 
4071 			ASSERT(opp->p_lckcnt != 0);
4072 
4073 			/*
4074 			 * We didn't need availrmem decremented if p_lckcnt on
4075 			 * original page is 1. Here, we are unlocking
4076 			 * read-only copy belonging to original page and
4077 			 * are locking a copy belonging to new page.
4078 			 */
4079 			if (opp->p_lckcnt == 1)
4080 				payback = 1;
4081 
4082 			npp->p_lckcnt++;
4083 			opp->p_lckcnt--;
4084 		}
4085 	}
4086 	if (payback) {
4087 		mutex_enter(&freemem_lock);
4088 		availrmem++;
4089 		pages_useclaim--;
4090 		mutex_exit(&freemem_lock);
4091 	}
4092 	page_struct_unlock(opp);
4093 }
4094 
4095 /*
4096  * Simple claim adjust functions -- used to support changes in
4097  * claims due to changes in access permissions.  Used by segvn_setprot().
4098  */
4099 int
4100 page_addclaim(page_t *pp)
4101 {
4102 	int r = 0;			/* result */
4103 
4104 	ASSERT(PAGE_LOCKED(pp));
4105 
4106 	page_struct_lock(pp);
4107 	ASSERT(pp->p_lckcnt != 0);
4108 
4109 	if (pp->p_lckcnt == 1) {
4110 		if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
4111 			--pp->p_lckcnt;
4112 			r = 1;
4113 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4114 				cmn_err(CE_WARN,
4115 				    "COW lock limit reached on pfn 0x%lx",
4116 				    page_pptonum(pp));
4117 			}
4118 		}
4119 	} else {
4120 		mutex_enter(&freemem_lock);
4121 		if ((availrmem > pages_pp_maximum) &&
4122 		    (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
4123 			--availrmem;
4124 			++pages_claimed;
4125 			mutex_exit(&freemem_lock);
4126 			--pp->p_lckcnt;
4127 			r = 1;
4128 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4129 				cmn_err(CE_WARN,
4130 				    "COW lock limit reached on pfn 0x%lx",
4131 				    page_pptonum(pp));
4132 			}
4133 		} else
4134 			mutex_exit(&freemem_lock);
4135 	}
4136 	page_struct_unlock(pp);
4137 	return (r);
4138 }
4139 
4140 int
4141 page_subclaim(page_t *pp)
4142 {
4143 	int r = 0;
4144 
4145 	ASSERT(PAGE_LOCKED(pp));
4146 
4147 	page_struct_lock(pp);
4148 	ASSERT(pp->p_cowcnt != 0);
4149 
4150 	if (pp->p_lckcnt) {
4151 		if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
4152 			r = 1;
4153 			/*
4154 			 * for availrmem
4155 			 */
4156 			mutex_enter(&freemem_lock);
4157 			availrmem++;
4158 			pages_claimed--;
4159 			mutex_exit(&freemem_lock);
4160 
4161 			pp->p_cowcnt--;
4162 
4163 			if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4164 				cmn_err(CE_WARN,
4165 				    "Page lock limit reached on pfn 0x%lx",
4166 				    page_pptonum(pp));
4167 			}
4168 		}
4169 	} else {
4170 		r = 1;
4171 		pp->p_cowcnt--;
4172 		pp->p_lckcnt++;
4173 	}
4174 	page_struct_unlock(pp);
4175 	return (r);
4176 }
4177 
4178 int
4179 page_addclaim_pages(page_t  **ppa)
4180 {
4181 
4182 	pgcnt_t	lckpgs = 0, pg_idx;
4183 
4184 	VM_STAT_ADD(pagecnt.pc_addclaim_pages);
4185 
4186 	mutex_enter(&page_llock);
4187 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4188 
4189 		ASSERT(PAGE_LOCKED(ppa[pg_idx]));
4190 		ASSERT(ppa[pg_idx]->p_lckcnt != 0);
4191 		if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4192 			mutex_exit(&page_llock);
4193 			return (0);
4194 		}
4195 		if (ppa[pg_idx]->p_lckcnt > 1)
4196 			lckpgs++;
4197 	}
4198 
4199 	if (lckpgs != 0) {
4200 		mutex_enter(&freemem_lock);
4201 		if (availrmem >= pages_pp_maximum + lckpgs) {
4202 			availrmem -= lckpgs;
4203 			pages_claimed += lckpgs;
4204 		} else {
4205 			mutex_exit(&freemem_lock);
4206 			mutex_exit(&page_llock);
4207 			return (0);
4208 		}
4209 		mutex_exit(&freemem_lock);
4210 	}
4211 
4212 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4213 		ppa[pg_idx]->p_lckcnt--;
4214 		ppa[pg_idx]->p_cowcnt++;
4215 	}
4216 	mutex_exit(&page_llock);
4217 	return (1);
4218 }
4219 
4220 int
4221 page_subclaim_pages(page_t  **ppa)
4222 {
4223 	pgcnt_t	ulckpgs = 0, pg_idx;
4224 
4225 	VM_STAT_ADD(pagecnt.pc_subclaim_pages);
4226 
4227 	mutex_enter(&page_llock);
4228 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4229 
4230 		ASSERT(PAGE_LOCKED(ppa[pg_idx]));
4231 		ASSERT(ppa[pg_idx]->p_cowcnt != 0);
4232 		if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
4233 			mutex_exit(&page_llock);
4234 			return (0);
4235 		}
4236 		if (ppa[pg_idx]->p_lckcnt != 0)
4237 			ulckpgs++;
4238 	}
4239 
4240 	if (ulckpgs != 0) {
4241 		mutex_enter(&freemem_lock);
4242 		availrmem += ulckpgs;
4243 		pages_claimed -= ulckpgs;
4244 		mutex_exit(&freemem_lock);
4245 	}
4246 
4247 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
4248 		ppa[pg_idx]->p_cowcnt--;
4249 		ppa[pg_idx]->p_lckcnt++;
4250 
4251 	}
4252 	mutex_exit(&page_llock);
4253 	return (1);
4254 }
4255 
4256 page_t *
4257 page_numtopp(pfn_t pfnum, se_t se)
4258 {
4259 	page_t *pp;
4260 
4261 retry:
4262 	pp = page_numtopp_nolock(pfnum);
4263 	if (pp == NULL) {
4264 		return ((page_t *)NULL);
4265 	}
4266 
4267 	/*
4268 	 * Acquire the appropriate lock on the page.
4269 	 */
4270 	while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) {
4271 		if (page_pptonum(pp) != pfnum)
4272 			goto retry;
4273 		continue;
4274 	}
4275 
4276 	if (page_pptonum(pp) != pfnum) {
4277 		page_unlock(pp);
4278 		goto retry;
4279 	}
4280 
4281 	return (pp);
4282 }
4283 
4284 page_t *
4285 page_numtopp_noreclaim(pfn_t pfnum, se_t se)
4286 {
4287 	page_t *pp;
4288 
4289 retry:
4290 	pp = page_numtopp_nolock(pfnum);
4291 	if (pp == NULL) {
4292 		return ((page_t *)NULL);
4293 	}
4294 
4295 	/*
4296 	 * Acquire the appropriate lock on the page.
4297 	 */
4298 	while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) {
4299 		if (page_pptonum(pp) != pfnum)
4300 			goto retry;
4301 		continue;
4302 	}
4303 
4304 	if (page_pptonum(pp) != pfnum) {
4305 		page_unlock(pp);
4306 		goto retry;
4307 	}
4308 
4309 	return (pp);
4310 }
4311 
4312 /*
4313  * This routine is like page_numtopp, but will only return page structs
4314  * for pages which are ok for loading into hardware using the page struct.
4315  */
4316 page_t *
4317 page_numtopp_nowait(pfn_t pfnum, se_t se)
4318 {
4319 	page_t *pp;
4320 
4321 retry:
4322 	pp = page_numtopp_nolock(pfnum);
4323 	if (pp == NULL) {
4324 		return ((page_t *)NULL);
4325 	}
4326 
4327 	/*
4328 	 * Try to acquire the appropriate lock on the page.
4329 	 */
4330 	if (PP_ISFREE(pp))
4331 		pp = NULL;
4332 	else {
4333 		if (!page_trylock(pp, se))
4334 			pp = NULL;
4335 		else {
4336 			if (page_pptonum(pp) != pfnum) {
4337 				page_unlock(pp);
4338 				goto retry;
4339 			}
4340 			if (PP_ISFREE(pp)) {
4341 				page_unlock(pp);
4342 				pp = NULL;
4343 			}
4344 		}
4345 	}
4346 	return (pp);
4347 }
4348 
4349 /*
4350  * Returns a count of dirty pages that are in the process
4351  * of being written out.  If 'cleanit' is set, try to push the page.
4352  */
4353 pgcnt_t
4354 page_busy(int cleanit)
4355 {
4356 	page_t *page0 = page_first();
4357 	page_t *pp = page0;
4358 	pgcnt_t nppbusy = 0;
4359 	u_offset_t off;
4360 
4361 	do {
4362 		vnode_t *vp = pp->p_vnode;
4363 
4364 		/*
4365 		 * A page is a candidate for syncing if it is:
4366 		 *
4367 		 * (a)	On neither the freelist nor the cachelist
4368 		 * (b)	Hashed onto a vnode
4369 		 * (c)	Not a kernel page
4370 		 * (d)	Dirty
4371 		 * (e)	Not part of a swapfile
4372 		 * (f)	a page which belongs to a real vnode; eg has a non-null
4373 		 *	v_vfsp pointer.
4374 		 * (g)	Backed by a filesystem which doesn't have a
4375 		 *	stubbed-out sync operation
4376 		 */
4377 		if (!PP_ISFREE(pp) && vp != NULL && vp != &kvp &&
4378 		    hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL &&
4379 		    vfs_can_sync(vp->v_vfsp)) {
4380 			nppbusy++;
4381 			vfs_syncprogress();
4382 
4383 			if (!cleanit)
4384 				continue;
4385 			if (!page_trylock(pp, SE_EXCL))
4386 				continue;
4387 
4388 			if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) ||
4389 			    pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
4390 			    !(hat_pagesync(pp,
4391 			    HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) {
4392 				page_unlock(pp);
4393 				continue;
4394 			}
4395 			off = pp->p_offset;
4396 			VN_HOLD(vp);
4397 			page_unlock(pp);
4398 			(void) VOP_PUTPAGE(vp, off, PAGESIZE,
4399 			    B_ASYNC | B_FREE, kcred);
4400 			VN_RELE(vp);
4401 		}
4402 	} while ((pp = page_next(pp)) != page0);
4403 
4404 	return (nppbusy);
4405 }
4406 
4407 void page_invalidate_pages(void);
4408 
4409 /*
4410  * callback handler to vm sub-system
4411  *
4412  * callers make sure no recursive entries to this func.
4413  */
4414 /*ARGSUSED*/
4415 boolean_t
4416 callb_vm_cpr(void *arg, int code)
4417 {
4418 	if (code == CB_CODE_CPR_CHKPT)
4419 		page_invalidate_pages();
4420 	return (B_TRUE);
4421 }
4422 
4423 /*
4424  * Invalidate all pages of the system.
4425  * It shouldn't be called until all user page activities are all stopped.
4426  */
4427 void
4428 page_invalidate_pages()
4429 {
4430 	page_t *pp;
4431 	page_t *page0;
4432 	pgcnt_t nbusypages;
4433 	int retry = 0;
4434 	const int MAXRETRIES = 4;
4435 #if defined(__sparc)
4436 	extern struct vnode prom_ppages;
4437 #endif /* __sparc */
4438 
4439 top:
4440 	/*
4441 	 * Flush dirty pages and destory the clean ones.
4442 	 */
4443 	nbusypages = 0;
4444 
4445 	pp = page0 = page_first();
4446 	do {
4447 		struct vnode	*vp;
4448 		u_offset_t	offset;
4449 		int		mod;
4450 
4451 		/*
4452 		 * skip the page if it has no vnode or the page associated
4453 		 * with the kernel vnode or prom allocated kernel mem.
4454 		 */
4455 #if defined(__sparc)
4456 		if ((vp = pp->p_vnode) == NULL || vp == &kvp ||
4457 		    vp == &prom_ppages)
4458 #else /* x86 doesn't have prom or prom_ppage */
4459 		if ((vp = pp->p_vnode) == NULL || vp == &kvp)
4460 #endif /* __sparc */
4461 			continue;
4462 
4463 		/*
4464 		 * skip the page which is already free invalidated.
4465 		 */
4466 		if (PP_ISFREE(pp) && PP_ISAGED(pp))
4467 			continue;
4468 
4469 		/*
4470 		 * skip pages that are already locked or can't be "exclusively"
4471 		 * locked or are already free.  After we lock the page, check
4472 		 * the free and age bits again to be sure it's not destroied
4473 		 * yet.
4474 		 * To achieve max. parallelization, we use page_trylock instead
4475 		 * of page_lock so that we don't get block on individual pages
4476 		 * while we have thousands of other pages to process.
4477 		 */
4478 		if (!page_trylock(pp, SE_EXCL)) {
4479 			nbusypages++;
4480 			continue;
4481 		} else if (PP_ISFREE(pp)) {
4482 			if (!PP_ISAGED(pp)) {
4483 				page_destroy_free(pp);
4484 			} else {
4485 				page_unlock(pp);
4486 			}
4487 			continue;
4488 		}
4489 		/*
4490 		 * Is this page involved in some I/O? shared?
4491 		 *
4492 		 * The page_struct_lock need not be acquired to
4493 		 * examine these fields since the page has an
4494 		 * "exclusive" lock.
4495 		 */
4496 		if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
4497 			page_unlock(pp);
4498 			continue;
4499 		}
4500 
4501 		if (vp->v_type == VCHR) {
4502 			panic("vp->v_type == VCHR");
4503 			/*NOTREACHED*/
4504 		}
4505 
4506 		if (!page_try_demote_pages(pp)) {
4507 			page_unlock(pp);
4508 			continue;
4509 		}
4510 
4511 		/*
4512 		 * Check the modified bit. Leave the bits alone in hardware
4513 		 * (they will be modified if we do the putpage).
4514 		 */
4515 		mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD)
4516 			& P_MOD);
4517 		if (mod) {
4518 			offset = pp->p_offset;
4519 			/*
4520 			 * Hold the vnode before releasing the page lock
4521 			 * to prevent it from being freed and re-used by
4522 			 * some other thread.
4523 			 */
4524 			VN_HOLD(vp);
4525 			page_unlock(pp);
4526 			/*
4527 			 * No error return is checked here. Callers such as
4528 			 * cpr deals with the dirty pages at the dump time
4529 			 * if this putpage fails.
4530 			 */
4531 			(void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL,
4532 			    kcred);
4533 			VN_RELE(vp);
4534 		} else {
4535 			page_destroy(pp, 0);
4536 		}
4537 	} while ((pp = page_next(pp)) != page0);
4538 	if (nbusypages && retry++ < MAXRETRIES) {
4539 		delay(1);
4540 		goto top;
4541 	}
4542 }
4543 
4544 /*
4545  * Replace the page "old" with the page "new" on the page hash and vnode lists
4546  *
4547  * the replacemnt must be done in place, ie the equivalent sequence:
4548  *
4549  *	vp = old->p_vnode;
4550  *	off = old->p_offset;
4551  *	page_do_hashout(old)
4552  *	page_do_hashin(new, vp, off)
4553  *
4554  * doesn't work, since
4555  *  1) if old is the only page on the vnode, the v_pages list has a window
4556  *     where it looks empty. This will break file system assumptions.
4557  * and
4558  *  2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list.
4559  */
4560 static void
4561 page_do_relocate_hash(page_t *new, page_t *old)
4562 {
4563 	page_t	**hash_list;
4564 	vnode_t	*vp = old->p_vnode;
4565 	kmutex_t *sep;
4566 
4567 	ASSERT(PAGE_EXCL(old));
4568 	ASSERT(PAGE_EXCL(new));
4569 	ASSERT(vp != NULL);
4570 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
4571 	ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset))));
4572 
4573 	/*
4574 	 * First find old page on the page hash list
4575 	 */
4576 	hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)];
4577 
4578 	for (;;) {
4579 		if (*hash_list == old)
4580 			break;
4581 		if (*hash_list == NULL) {
4582 			panic("page_do_hashout");
4583 			/*NOTREACHED*/
4584 		}
4585 		hash_list = &(*hash_list)->p_hash;
4586 	}
4587 
4588 	/*
4589 	 * update new and replace old with new on the page hash list
4590 	 */
4591 	new->p_vnode = old->p_vnode;
4592 	new->p_offset = old->p_offset;
4593 	new->p_hash = old->p_hash;
4594 	*hash_list = new;
4595 
4596 	if ((new->p_vnode->v_flag & VISSWAP) != 0)
4597 		PP_SETSWAP(new);
4598 
4599 	/*
4600 	 * replace old with new on the vnode's page list
4601 	 */
4602 	if (old->p_vpnext == old) {
4603 		new->p_vpnext = new;
4604 		new->p_vpprev = new;
4605 	} else {
4606 		new->p_vpnext = old->p_vpnext;
4607 		new->p_vpprev = old->p_vpprev;
4608 		new->p_vpnext->p_vpprev = new;
4609 		new->p_vpprev->p_vpnext = new;
4610 	}
4611 	if (vp->v_pages == old)
4612 		vp->v_pages = new;
4613 
4614 	/*
4615 	 * clear out the old page
4616 	 */
4617 	old->p_hash = NULL;
4618 	old->p_vpnext = NULL;
4619 	old->p_vpprev = NULL;
4620 	old->p_vnode = NULL;
4621 	PP_CLRSWAP(old);
4622 	old->p_offset = (u_offset_t)-1;
4623 	page_clr_all_props(old);
4624 
4625 	/*
4626 	 * Wake up processes waiting for this page.  The page's
4627 	 * identity has been changed, and is probably not the
4628 	 * desired page any longer.
4629 	 */
4630 	sep = page_se_mutex(old);
4631 	mutex_enter(sep);
4632 	old->p_selock &= ~SE_EWANTED;
4633 	if (CV_HAS_WAITERS(&old->p_cv))
4634 		cv_broadcast(&old->p_cv);
4635 	mutex_exit(sep);
4636 }
4637 
4638 /*
4639  * This function moves the identity of page "pp_old" to page "pp_new".
4640  * Both pages must be locked on entry.  "pp_new" is free, has no identity,
4641  * and need not be hashed out from anywhere.
4642  */
4643 void
4644 page_relocate_hash(page_t *pp_new, page_t *pp_old)
4645 {
4646 	vnode_t *vp = pp_old->p_vnode;
4647 	u_offset_t off = pp_old->p_offset;
4648 	kmutex_t *phm, *vphm;
4649 
4650 	/*
4651 	 * Rehash two pages
4652 	 */
4653 	ASSERT(PAGE_EXCL(pp_old));
4654 	ASSERT(PAGE_EXCL(pp_new));
4655 	ASSERT(vp != NULL);
4656 	ASSERT(pp_new->p_vnode == NULL);
4657 
4658 	/*
4659 	 * hashout then hashin while holding the mutexes
4660 	 */
4661 	phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off));
4662 	mutex_enter(phm);
4663 	vphm = page_vnode_mutex(vp);
4664 	mutex_enter(vphm);
4665 
4666 	page_do_relocate_hash(pp_new, pp_old);
4667 
4668 	mutex_exit(vphm);
4669 	mutex_exit(phm);
4670 
4671 	/*
4672 	 * The page_struct_lock need not be acquired for lckcnt and
4673 	 * cowcnt since the page has an "exclusive" lock.
4674 	 */
4675 	ASSERT(pp_new->p_lckcnt == 0);
4676 	ASSERT(pp_new->p_cowcnt == 0);
4677 	pp_new->p_lckcnt = pp_old->p_lckcnt;
4678 	pp_new->p_cowcnt = pp_old->p_cowcnt;
4679 	pp_old->p_lckcnt = pp_old->p_cowcnt = 0;
4680 
4681 	/* The following comment preserved from page_flip(). */
4682 	/* XXX - Do we need to protect fsdata? */
4683 	pp_new->p_fsdata = pp_old->p_fsdata;
4684 }
4685 
4686 /*
4687  * Helper routine used to lock all remaining members of a
4688  * large page. The caller is responsible for passing in a locked
4689  * pp. If pp is a large page, then it succeeds in locking all the
4690  * remaining constituent pages or it returns with only the
4691  * original page locked.
4692  *
4693  * Returns 1 on success, 0 on failure.
4694  *
4695  * If success is returned this routine gurantees p_szc for all constituent
4696  * pages of a large page pp belongs to can't change. To achieve this we
4697  * recheck szc of pp after locking all constituent pages and retry if szc
4698  * changed (it could only decrease). Since hat_page_demote() needs an EXCL
4699  * lock on one of constituent pages it can't be running after all constituent
4700  * pages are locked.  hat_page_demote() with a lock on a constituent page
4701  * outside of this large page (i.e. pp belonged to a larger large page) is
4702  * already done with all constituent pages of pp since the root's p_szc is
4703  * changed last. Thefore no need to synchronize with hat_page_demote() that
4704  * locked a constituent page outside of pp's current large page.
4705  */
4706 #ifdef DEBUG
4707 uint32_t gpg_trylock_mtbf = 0;
4708 #endif
4709 
4710 int
4711 group_page_trylock(page_t *pp, se_t se)
4712 {
4713 	page_t  *tpp;
4714 	pgcnt_t	npgs, i, j;
4715 	uint_t pszc = pp->p_szc;
4716 
4717 #ifdef DEBUG
4718 	if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) {
4719 		return (0);
4720 	}
4721 #endif
4722 
4723 	if (pp != PP_GROUPLEADER(pp, pszc)) {
4724 		return (0);
4725 	}
4726 
4727 retry:
4728 	ASSERT(PAGE_LOCKED_SE(pp, se));
4729 	ASSERT(!PP_ISFREE(pp));
4730 	if (pszc == 0) {
4731 		return (1);
4732 	}
4733 	npgs = page_get_pagecnt(pszc);
4734 	tpp = pp + 1;
4735 	for (i = 1; i < npgs; i++, tpp++) {
4736 		if (!page_trylock(tpp, se)) {
4737 			tpp = pp + 1;
4738 			for (j = 1; j < i; j++, tpp++) {
4739 				page_unlock(tpp);
4740 			}
4741 			return (0);
4742 		}
4743 	}
4744 	if (pp->p_szc != pszc) {
4745 		ASSERT(pp->p_szc < pszc);
4746 		ASSERT(pp->p_vnode != NULL && pp->p_vnode != &kvp &&
4747 		    !IS_SWAPFSVP(pp->p_vnode));
4748 		tpp = pp + 1;
4749 		for (i = 1; i < npgs; i++, tpp++) {
4750 			page_unlock(tpp);
4751 		}
4752 		pszc = pp->p_szc;
4753 		goto retry;
4754 	}
4755 	return (1);
4756 }
4757 
4758 void
4759 group_page_unlock(page_t *pp)
4760 {
4761 	page_t *tpp;
4762 	pgcnt_t	npgs, i;
4763 
4764 	ASSERT(PAGE_LOCKED(pp));
4765 	ASSERT(!PP_ISFREE(pp));
4766 	ASSERT(pp == PP_PAGEROOT(pp));
4767 	npgs = page_get_pagecnt(pp->p_szc);
4768 	for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) {
4769 		page_unlock(tpp);
4770 	}
4771 }
4772 
4773 /*
4774  * returns
4775  * 0 		: on success and *nrelocp is number of relocated PAGESIZE pages
4776  * ERANGE	: this is not a base page
4777  * EBUSY	: failure to get locks on the page/pages
4778  * ENOMEM	: failure to obtain replacement pages
4779  * EAGAIN	: OBP has not yet completed its boot-time handoff to the kernel
4780  *
4781  * Return with all constituent members of target and replacement
4782  * SE_EXCL locked. It is the callers responsibility to drop the
4783  * locks.
4784  */
4785 int
4786 do_page_relocate(
4787 	page_t **target,
4788 	page_t **replacement,
4789 	int grouplock,
4790 	spgcnt_t *nrelocp,
4791 	lgrp_t *lgrp)
4792 {
4793 #ifdef DEBUG
4794 	page_t *first_repl;
4795 #endif /* DEBUG */
4796 	page_t *repl;
4797 	page_t *targ;
4798 	page_t *pl = NULL;
4799 	uint_t ppattr;
4800 	pfn_t   pfn, repl_pfn;
4801 	uint_t	szc;
4802 	spgcnt_t npgs, i;
4803 	int repl_contig = 0;
4804 	uint_t flags = 0;
4805 	spgcnt_t dofree = 0;
4806 
4807 	*nrelocp = 0;
4808 
4809 #if defined(__sparc)
4810 	/*
4811 	 * We need to wait till OBP has completed
4812 	 * its boot-time handoff of its resources to the kernel
4813 	 * before we allow page relocation
4814 	 */
4815 	if (page_relocate_ready == 0) {
4816 		return (EAGAIN);
4817 	}
4818 #endif
4819 
4820 	/*
4821 	 * If this is not a base page,
4822 	 * just return with 0x0 pages relocated.
4823 	 */
4824 	targ = *target;
4825 	ASSERT(PAGE_EXCL(targ));
4826 	ASSERT(!PP_ISFREE(targ));
4827 	szc = targ->p_szc;
4828 	ASSERT(szc < mmu_page_sizes);
4829 	VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
4830 	pfn = targ->p_pagenum;
4831 	if (pfn != PFN_BASE(pfn, szc)) {
4832 		VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]);
4833 		return (ERANGE);
4834 	}
4835 
4836 	if ((repl = *replacement) != NULL && repl->p_szc >= szc) {
4837 		repl_pfn = repl->p_pagenum;
4838 		if (repl_pfn != PFN_BASE(repl_pfn, szc)) {
4839 			VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]);
4840 			return (ERANGE);
4841 		}
4842 		repl_contig = 1;
4843 	}
4844 
4845 	/*
4846 	 * We must lock all members of this large page or we cannot
4847 	 * relocate any part of it.
4848 	 */
4849 	if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) {
4850 		VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]);
4851 		return (EBUSY);
4852 	}
4853 
4854 	/*
4855 	 * reread szc it could have been decreased before
4856 	 * group_page_trylock() was done.
4857 	 */
4858 	szc = targ->p_szc;
4859 	ASSERT(szc < mmu_page_sizes);
4860 	VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
4861 	ASSERT(pfn == PFN_BASE(pfn, szc));
4862 
4863 	npgs = page_get_pagecnt(targ->p_szc);
4864 
4865 	if (repl == NULL) {
4866 		dofree = npgs;		/* Size of target page in MMU pages */
4867 		if (!page_create_wait(dofree, 0)) {
4868 			if (grouplock != 0) {
4869 				group_page_unlock(targ);
4870 			}
4871 			VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
4872 			return (ENOMEM);
4873 		}
4874 
4875 		/*
4876 		 * seg kmem pages require that the target and replacement
4877 		 * page be the same pagesize.
4878 		 */
4879 		flags = (targ->p_vnode == &kvp) ? PGR_SAMESZC : 0;
4880 		repl = page_get_replacement_page(targ, lgrp, flags);
4881 		if (repl == NULL) {
4882 			if (grouplock != 0) {
4883 				group_page_unlock(targ);
4884 			}
4885 			page_create_putback(dofree);
4886 			VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
4887 			return (ENOMEM);
4888 		}
4889 	}
4890 #ifdef DEBUG
4891 	else {
4892 		ASSERT(PAGE_LOCKED(repl));
4893 	}
4894 #endif /* DEBUG */
4895 
4896 #if defined(__sparc)
4897 	/*
4898 	 * Let hat_page_relocate() complete the relocation if it's kernel page
4899 	 */
4900 	if (targ->p_vnode == &kvp) {
4901 		*replacement = repl;
4902 		if (hat_page_relocate(target, replacement, nrelocp) != 0) {
4903 			if (grouplock != 0) {
4904 				group_page_unlock(targ);
4905 			}
4906 			if (dofree) {
4907 				*replacement = NULL;
4908 				page_free_replacement_page(repl);
4909 				page_create_putback(dofree);
4910 			}
4911 			VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]);
4912 			return (EAGAIN);
4913 		}
4914 		VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
4915 		return (0);
4916 	}
4917 #else
4918 #if defined(lint)
4919 	dofree = dofree;
4920 #endif
4921 #endif
4922 
4923 #ifdef DEBUG
4924 	first_repl = repl;
4925 #endif /* DEBUG */
4926 
4927 	for (i = 0; i < npgs; i++) {
4928 		ASSERT(PAGE_EXCL(targ));
4929 		ASSERT(targ->p_slckcnt == 0);
4930 		ASSERT(repl->p_slckcnt == 0);
4931 
4932 		(void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD);
4933 
4934 		ASSERT(hat_page_getshare(targ) == 0);
4935 		ASSERT(!PP_ISFREE(targ));
4936 		ASSERT(targ->p_pagenum == (pfn + i));
4937 		ASSERT(repl_contig == 0 ||
4938 		    repl->p_pagenum == (repl_pfn + i));
4939 
4940 		/*
4941 		 * Copy the page contents and attributes then
4942 		 * relocate the page in the page hash.
4943 		 */
4944 		ppcopy(targ, repl);
4945 		ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO));
4946 		page_clr_all_props(repl);
4947 		page_set_props(repl, ppattr);
4948 		page_relocate_hash(repl, targ);
4949 
4950 		ASSERT(hat_page_getshare(targ) == 0);
4951 		ASSERT(hat_page_getshare(repl) == 0);
4952 		/*
4953 		 * Now clear the props on targ, after the
4954 		 * page_relocate_hash(), they no longer
4955 		 * have any meaning.
4956 		 */
4957 		page_clr_all_props(targ);
4958 		ASSERT(targ->p_next == targ);
4959 		ASSERT(targ->p_prev == targ);
4960 		page_list_concat(&pl, &targ);
4961 
4962 		targ++;
4963 		if (repl_contig != 0) {
4964 			repl++;
4965 		} else {
4966 			repl = repl->p_next;
4967 		}
4968 	}
4969 	/* assert that we have come full circle with repl */
4970 	ASSERT(repl_contig == 1 || first_repl == repl);
4971 
4972 	*target = pl;
4973 	if (*replacement == NULL) {
4974 		ASSERT(first_repl == repl);
4975 		*replacement = repl;
4976 	}
4977 	VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
4978 	*nrelocp = npgs;
4979 	return (0);
4980 }
4981 /*
4982  * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated.
4983  */
4984 int
4985 page_relocate(
4986 	page_t **target,
4987 	page_t **replacement,
4988 	int grouplock,
4989 	int freetarget,
4990 	spgcnt_t *nrelocp,
4991 	lgrp_t *lgrp)
4992 {
4993 	spgcnt_t ret;
4994 
4995 	/* do_page_relocate returns 0 on success or errno value */
4996 	ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp);
4997 
4998 	if (ret != 0 || freetarget == 0) {
4999 		return (ret);
5000 	}
5001 	if (*nrelocp == 1) {
5002 		ASSERT(*target != NULL);
5003 		page_free(*target, 1);
5004 	} else {
5005 		page_t *tpp = *target;
5006 		uint_t szc = tpp->p_szc;
5007 		pgcnt_t npgs = page_get_pagecnt(szc);
5008 		ASSERT(npgs > 1);
5009 		ASSERT(szc != 0);
5010 		do {
5011 			ASSERT(PAGE_EXCL(tpp));
5012 			ASSERT(!hat_page_is_mapped(tpp));
5013 			ASSERT(tpp->p_szc == szc);
5014 			PP_SETFREE(tpp);
5015 			PP_SETAGED(tpp);
5016 			npgs--;
5017 		} while ((tpp = tpp->p_next) != *target);
5018 		ASSERT(npgs == 0);
5019 		page_list_add_pages(*target, 0);
5020 		npgs = page_get_pagecnt(szc);
5021 		page_create_putback(npgs);
5022 	}
5023 	return (ret);
5024 }
5025 
5026 /*
5027  * it is up to the caller to deal with pcf accounting.
5028  */
5029 void
5030 page_free_replacement_page(page_t *pplist)
5031 {
5032 	page_t *pp;
5033 
5034 	while (pplist != NULL) {
5035 		/*
5036 		 * pp_targ is a linked list.
5037 		 */
5038 		pp = pplist;
5039 		if (pp->p_szc == 0) {
5040 			page_sub(&pplist, pp);
5041 			page_clr_all_props(pp);
5042 			PP_SETFREE(pp);
5043 			PP_SETAGED(pp);
5044 			page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
5045 			page_unlock(pp);
5046 			VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]);
5047 		} else {
5048 			spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc);
5049 			page_t *tpp;
5050 			page_list_break(&pp, &pplist, curnpgs);
5051 			tpp = pp;
5052 			do {
5053 				ASSERT(PAGE_EXCL(tpp));
5054 				ASSERT(!hat_page_is_mapped(tpp));
5055 				page_clr_all_props(pp);
5056 				PP_SETFREE(tpp);
5057 				PP_SETAGED(tpp);
5058 			} while ((tpp = tpp->p_next) != pp);
5059 			page_list_add_pages(pp, 0);
5060 			VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]);
5061 		}
5062 	}
5063 }
5064 
5065 /*
5066  * Relocate target to non-relocatable replacement page.
5067  */
5068 int
5069 page_relocate_cage(page_t **target, page_t **replacement)
5070 {
5071 	page_t *tpp, *rpp;
5072 	spgcnt_t pgcnt, npgs;
5073 	int result;
5074 
5075 	tpp = *target;
5076 
5077 	ASSERT(PAGE_EXCL(tpp));
5078 	ASSERT(tpp->p_szc == 0);
5079 
5080 	pgcnt = btop(page_get_pagesize(tpp->p_szc));
5081 
5082 	do {
5083 		(void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC);
5084 		rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC);
5085 		if (rpp == NULL) {
5086 			page_create_putback(pgcnt);
5087 			kcage_cageout_wakeup();
5088 		}
5089 	} while (rpp == NULL);
5090 
5091 	ASSERT(PP_ISNORELOC(rpp));
5092 
5093 	result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL);
5094 
5095 	if (result == 0) {
5096 		*replacement = rpp;
5097 		if (pgcnt != npgs)
5098 			panic("page_relocate_cage: partial relocation");
5099 	}
5100 
5101 	return (result);
5102 }
5103 
5104 /*
5105  * Release the page lock on a page, place on cachelist
5106  * tail if no longer mapped. Caller can let us know if
5107  * the page is known to be clean.
5108  */
5109 int
5110 page_release(page_t *pp, int checkmod)
5111 {
5112 	int status;
5113 
5114 	ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) &&
5115 		(pp->p_vnode != NULL));
5116 
5117 	if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) &&
5118 	    ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) &&
5119 	    pp->p_lckcnt == 0 && pp->p_cowcnt == 0 &&
5120 	    !hat_page_is_mapped(pp)) {
5121 
5122 		/*
5123 		 * If page is modified, unlock it
5124 		 *
5125 		 * (p_nrm & P_MOD) bit has the latest stuff because:
5126 		 * (1) We found that this page doesn't have any mappings
5127 		 *	_after_ holding SE_EXCL and
5128 		 * (2) We didn't drop SE_EXCL lock after the check in (1)
5129 		 */
5130 		if (checkmod && hat_ismod(pp)) {
5131 			page_unlock(pp);
5132 			status = PGREL_MOD;
5133 		} else {
5134 			/*LINTED: constant in conditional context*/
5135 			VN_DISPOSE(pp, B_FREE, 0, kcred);
5136 			status = PGREL_CLEAN;
5137 		}
5138 	} else {
5139 		page_unlock(pp);
5140 		status = PGREL_NOTREL;
5141 	}
5142 	return (status);
5143 }
5144 
5145 /*
5146  * Given a constituent page, try to demote the large page on the freelist.
5147  *
5148  * Returns nonzero if the page could be demoted successfully. Returns with
5149  * the constituent page still locked.
5150  */
5151 int
5152 page_try_demote_free_pages(page_t *pp)
5153 {
5154 	page_t *rootpp = pp;
5155 	pfn_t	pfn = page_pptonum(pp);
5156 	spgcnt_t npgs;
5157 	uint_t	szc = pp->p_szc;
5158 
5159 	ASSERT(PP_ISFREE(pp));
5160 	ASSERT(PAGE_EXCL(pp));
5161 
5162 	/*
5163 	 * Adjust rootpp and lock it, if `pp' is not the base
5164 	 * constituent page.
5165 	 */
5166 	npgs = page_get_pagecnt(pp->p_szc);
5167 	if (npgs == 1) {
5168 		return (0);
5169 	}
5170 
5171 	if (!IS_P2ALIGNED(pfn, npgs)) {
5172 		pfn = P2ALIGN(pfn, npgs);
5173 		rootpp = page_numtopp_nolock(pfn);
5174 	}
5175 
5176 	if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) {
5177 		return (0);
5178 	}
5179 
5180 	if (rootpp->p_szc != szc) {
5181 		if (pp != rootpp)
5182 			page_unlock(rootpp);
5183 		return (0);
5184 	}
5185 
5186 	page_demote_free_pages(rootpp);
5187 
5188 	if (pp != rootpp)
5189 		page_unlock(rootpp);
5190 
5191 	ASSERT(PP_ISFREE(pp));
5192 	ASSERT(PAGE_EXCL(pp));
5193 	return (1);
5194 }
5195 
5196 /*
5197  * Given a constituent page, try to demote the large page.
5198  *
5199  * Returns nonzero if the page could be demoted successfully. Returns with
5200  * the constituent page still locked.
5201  */
5202 int
5203 page_try_demote_pages(page_t *pp)
5204 {
5205 	page_t *tpp, *rootpp = pp;
5206 	pfn_t	pfn = page_pptonum(pp);
5207 	spgcnt_t i, npgs;
5208 	uint_t	szc = pp->p_szc;
5209 	vnode_t *vp = pp->p_vnode;
5210 
5211 	ASSERT(PAGE_EXCL(pp));
5212 
5213 	VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]);
5214 
5215 	if (pp->p_szc == 0) {
5216 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]);
5217 		return (1);
5218 	}
5219 
5220 	if (vp != NULL && !IS_SWAPFSVP(vp) && vp != &kvp) {
5221 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]);
5222 		page_demote_vp_pages(pp);
5223 		ASSERT(pp->p_szc == 0);
5224 		return (1);
5225 	}
5226 
5227 	/*
5228 	 * Adjust rootpp if passed in is not the base
5229 	 * constituent page.
5230 	 */
5231 	npgs = page_get_pagecnt(pp->p_szc);
5232 	ASSERT(npgs > 1);
5233 	if (!IS_P2ALIGNED(pfn, npgs)) {
5234 		pfn = P2ALIGN(pfn, npgs);
5235 		rootpp = page_numtopp_nolock(pfn);
5236 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]);
5237 		ASSERT(rootpp->p_vnode != NULL);
5238 		ASSERT(rootpp->p_szc == szc);
5239 	}
5240 
5241 	/*
5242 	 * We can't demote kernel pages since we can't hat_unload()
5243 	 * the mappings.
5244 	 */
5245 	if (rootpp->p_vnode == &kvp)
5246 		return (0);
5247 
5248 	/*
5249 	 * Attempt to lock all constituent pages except the page passed
5250 	 * in since it's already locked.
5251 	 */
5252 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5253 		ASSERT(!PP_ISFREE(tpp));
5254 		ASSERT(tpp->p_vnode != NULL);
5255 
5256 		if (tpp != pp && !page_trylock(tpp, SE_EXCL))
5257 			break;
5258 		ASSERT(tpp->p_szc == rootpp->p_szc);
5259 		ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i);
5260 	}
5261 
5262 	/*
5263 	 * If we failed to lock them all then unlock what we have
5264 	 * locked so far and bail.
5265 	 */
5266 	if (i < npgs) {
5267 		tpp = rootpp;
5268 		while (i-- > 0) {
5269 			if (tpp != pp)
5270 				page_unlock(tpp);
5271 			tpp++;
5272 		}
5273 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]);
5274 		return (0);
5275 	}
5276 
5277 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5278 		ASSERT(PAGE_EXCL(tpp));
5279 		ASSERT(tpp->p_slckcnt == 0);
5280 		(void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
5281 		tpp->p_szc = 0;
5282 	}
5283 
5284 	/*
5285 	 * Unlock all pages except the page passed in.
5286 	 */
5287 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
5288 		ASSERT(!hat_page_is_mapped(tpp));
5289 		if (tpp != pp)
5290 			page_unlock(tpp);
5291 	}
5292 
5293 	VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]);
5294 	return (1);
5295 }
5296 
5297 /*
5298  * Called by page_free() and page_destroy() to demote the page size code
5299  * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero
5300  * p_szc on free list, neither can we just clear p_szc of a single page_t
5301  * within a large page since it will break other code that relies on p_szc
5302  * being the same for all page_t's of a large page). Anonymous pages should
5303  * never end up here because anon_map_getpages() cannot deal with p_szc
5304  * changes after a single constituent page is locked.  While anonymous or
5305  * kernel large pages are demoted or freed the entire large page at a time
5306  * with all constituent pages locked EXCL for the file system pages we
5307  * have to be able to demote a large page (i.e. decrease all constituent pages
5308  * p_szc) with only just an EXCL lock on one of constituent pages. The reason
5309  * we can easily deal with anonymous page demotion the entire large page at a
5310  * time is that those operation originate at address space level and concern
5311  * the entire large page region with actual demotion only done when pages are
5312  * not shared with any other processes (therefore we can always get EXCL lock
5313  * on all anonymous constituent pages after clearing segment page
5314  * cache). However file system pages can be truncated or invalidated at a
5315  * PAGESIZE level from the file system side and end up in page_free() or
5316  * page_destroy() (we also allow only part of the large page to be SOFTLOCKed
5317  * and therfore pageout should be able to demote a large page by EXCL locking
5318  * any constituent page that is not under SOFTLOCK). In those cases we cannot
5319  * rely on being able to lock EXCL all constituent pages.
5320  *
5321  * To prevent szc changes on file system pages one has to lock all constituent
5322  * pages at least SHARED (or call page_szc_lock()). The only subsystem that
5323  * doesn't rely on locking all constituent pages (or using page_szc_lock()) to
5324  * prevent szc changes is hat layer that uses its own page level mlist
5325  * locks. hat assumes that szc doesn't change after mlist lock for a page is
5326  * taken. Therefore we need to change szc under hat level locks if we only
5327  * have an EXCL lock on a single constituent page and hat still references any
5328  * of constituent pages.  (Note we can't "ignore" hat layer by simply
5329  * hat_pageunload() all constituent pages without having EXCL locks on all of
5330  * constituent pages). We use hat_page_demote() call to safely demote szc of
5331  * all constituent pages under hat locks when we only have an EXCL lock on one
5332  * of constituent pages.
5333  *
5334  * This routine calls page_szc_lock() before calling hat_page_demote() to
5335  * allow segvn in one special case not to lock all constituent pages SHARED
5336  * before calling hat_memload_array() that relies on p_szc not changeing even
5337  * before hat level mlist lock is taken.  In that case segvn uses
5338  * page_szc_lock() to prevent hat_page_demote() changeing p_szc values.
5339  *
5340  * Anonymous or kernel page demotion still has to lock all pages exclusively
5341  * and do hat_pageunload() on all constituent pages before demoting the page
5342  * therefore there's no need for anonymous or kernel page demotion to use
5343  * hat_page_demote() mechanism.
5344  *
5345  * hat_page_demote() removes all large mappings that map pp and then decreases
5346  * p_szc starting from the last constituent page of the large page. By working
5347  * from the tail of a large page in pfn decreasing order allows one looking at
5348  * the root page to know that hat_page_demote() is done for root's szc area.
5349  * e.g. if a root page has szc 1 one knows it only has to lock all constituent
5350  * pages within szc 1 area to prevent szc changes because hat_page_demote()
5351  * that started on this page when it had szc > 1 is done for this szc 1 area.
5352  *
5353  * We are guranteed that all constituent pages of pp's large page belong to
5354  * the same vnode with the consecutive offsets increasing in the direction of
5355  * the pfn i.e. the identity of constituent pages can't change until their
5356  * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove
5357  * large mappings to pp even though we don't lock any constituent page except
5358  * pp (i.e. we won't unload e.g. kernel locked page).
5359  */
5360 static void
5361 page_demote_vp_pages(page_t *pp)
5362 {
5363 	kmutex_t *mtx;
5364 
5365 	ASSERT(PAGE_EXCL(pp));
5366 	ASSERT(!PP_ISFREE(pp));
5367 	ASSERT(pp->p_vnode != NULL);
5368 	ASSERT(!IS_SWAPFSVP(pp->p_vnode));
5369 	ASSERT(pp->p_vnode != &kvp);
5370 
5371 	VM_STAT_ADD(pagecnt.pc_demote_pages[0]);
5372 
5373 	mtx = page_szc_lock(pp);
5374 	if (mtx != NULL) {
5375 		hat_page_demote(pp);
5376 		mutex_exit(mtx);
5377 	}
5378 	ASSERT(pp->p_szc == 0);
5379 }
5380 
5381 /*
5382  * Mark any existing pages for migration in the given range
5383  */
5384 void
5385 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len,
5386     struct anon_map *amp, ulong_t anon_index, vnode_t *vp,
5387     u_offset_t vnoff, int rflag)
5388 {
5389 	struct anon	*ap;
5390 	vnode_t		*curvp;
5391 	lgrp_t		*from;
5392 	pgcnt_t		i;
5393 	pgcnt_t		nlocked;
5394 	u_offset_t	off;
5395 	pfn_t		pfn;
5396 	size_t		pgsz;
5397 	size_t		segpgsz;
5398 	pgcnt_t		pages;
5399 	uint_t		pszc;
5400 	page_t		**ppa;
5401 	pgcnt_t		ppa_nentries;
5402 	page_t		*pp;
5403 	caddr_t		va;
5404 	ulong_t		an_idx;
5405 	anon_sync_obj_t	cookie;
5406 
5407 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
5408 
5409 	/*
5410 	 * Don't do anything if don't need to do lgroup optimizations
5411 	 * on this system
5412 	 */
5413 	if (!lgrp_optimizations())
5414 		return;
5415 
5416 	/*
5417 	 * Align address and length to (potentially large) page boundary
5418 	 */
5419 	segpgsz = page_get_pagesize(seg->s_szc);
5420 	addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz);
5421 	if (rflag)
5422 		len = P2ROUNDUP(len, segpgsz);
5423 
5424 	/*
5425 	 * Allocate page array to accomodate largest page size
5426 	 */
5427 	pgsz = page_get_pagesize(page_num_pagesizes() - 1);
5428 	ppa_nentries = btop(pgsz);
5429 	ppa = kmem_zalloc(ppa_nentries * sizeof (page_t *), KM_SLEEP);
5430 
5431 	/*
5432 	 * Do one (large) page at a time
5433 	 */
5434 	va = addr;
5435 	while (va < addr + len) {
5436 		/*
5437 		 * Lookup (root) page for vnode and offset corresponding to
5438 		 * this virtual address
5439 		 * Try anonmap first since there may be copy-on-write
5440 		 * pages, but initialize vnode pointer and offset using
5441 		 * vnode arguments just in case there isn't an amp.
5442 		 */
5443 		curvp = vp;
5444 		off = vnoff + va - seg->s_base;
5445 		if (amp) {
5446 			ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
5447 			an_idx = anon_index + seg_page(seg, va);
5448 			anon_array_enter(amp, an_idx, &cookie);
5449 			ap = anon_get_ptr(amp->ahp, an_idx);
5450 			if (ap)
5451 				swap_xlate(ap, &curvp, &off);
5452 			anon_array_exit(&cookie);
5453 			ANON_LOCK_EXIT(&amp->a_rwlock);
5454 		}
5455 
5456 		pp = NULL;
5457 		if (curvp)
5458 			pp = page_lookup(curvp, off, SE_SHARED);
5459 
5460 		/*
5461 		 * If there isn't a page at this virtual address,
5462 		 * skip to next page
5463 		 */
5464 		if (pp == NULL) {
5465 			va += PAGESIZE;
5466 			continue;
5467 		}
5468 
5469 		/*
5470 		 * Figure out which lgroup this page is in for kstats
5471 		 */
5472 		pfn = page_pptonum(pp);
5473 		from = lgrp_pfn_to_lgrp(pfn);
5474 
5475 		/*
5476 		 * Get page size, and round up and skip to next page boundary
5477 		 * if unaligned address
5478 		 */
5479 		pszc = pp->p_szc;
5480 		pgsz = page_get_pagesize(pszc);
5481 		pages = btop(pgsz);
5482 		if (!IS_P2ALIGNED(va, pgsz) ||
5483 		    !IS_P2ALIGNED(pfn, pages) ||
5484 		    pgsz > segpgsz) {
5485 			pgsz = MIN(pgsz, segpgsz);
5486 			page_unlock(pp);
5487 			i = btop(P2END((uintptr_t)va, pgsz) -
5488 			    (uintptr_t)va);
5489 			va = (caddr_t)P2END((uintptr_t)va, pgsz);
5490 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, i);
5491 			continue;
5492 		}
5493 
5494 		/*
5495 		 * Upgrade to exclusive lock on page
5496 		 */
5497 		if (!page_tryupgrade(pp)) {
5498 			page_unlock(pp);
5499 			va += pgsz;
5500 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
5501 			    btop(pgsz));
5502 			continue;
5503 		}
5504 
5505 		/*
5506 		 * Remember pages locked exclusively and how many
5507 		 */
5508 		ppa[0] = pp;
5509 		nlocked = 1;
5510 
5511 		/*
5512 		 * Lock constituent pages if this is large page
5513 		 */
5514 		if (pages > 1) {
5515 			/*
5516 			 * Lock all constituents except root page, since it
5517 			 * should be locked already.
5518 			 */
5519 			for (i = 1; i < pages; i++) {
5520 				pp++;
5521 				if (!page_trylock(pp, SE_EXCL)) {
5522 					break;
5523 				}
5524 				if (PP_ISFREE(pp) ||
5525 				    pp->p_szc != pszc) {
5526 					/*
5527 					 * hat_page_demote() raced in with us.
5528 					 */
5529 					ASSERT(!IS_SWAPFSVP(curvp));
5530 					page_unlock(pp);
5531 					break;
5532 				}
5533 				ppa[nlocked] = pp;
5534 				nlocked++;
5535 			}
5536 		}
5537 
5538 		/*
5539 		 * If all constituent pages couldn't be locked,
5540 		 * unlock pages locked so far and skip to next page.
5541 		 */
5542 		if (nlocked != pages) {
5543 			for (i = 0; i < nlocked; i++)
5544 				page_unlock(ppa[i]);
5545 			va += pgsz;
5546 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
5547 			    btop(pgsz));
5548 			continue;
5549 		}
5550 
5551 		/*
5552 		 * hat_page_demote() can no longer happen
5553 		 * since last cons page had the right p_szc after
5554 		 * all cons pages were locked. all cons pages
5555 		 * should now have the same p_szc.
5556 		 */
5557 
5558 		/*
5559 		 * All constituent pages locked successfully, so mark
5560 		 * large page for migration and unload the mappings of
5561 		 * constituent pages, so a fault will occur on any part of the
5562 		 * large page
5563 		 */
5564 		PP_SETMIGRATE(ppa[0]);
5565 		for (i = 0; i < nlocked; i++) {
5566 			pp = ppa[i];
5567 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
5568 			ASSERT(hat_page_getshare(pp) == 0);
5569 			page_unlock(pp);
5570 		}
5571 		lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked);
5572 
5573 		va += pgsz;
5574 	}
5575 	kmem_free(ppa, ppa_nentries * sizeof (page_t *));
5576 }
5577 
5578 /*
5579  * Migrate any pages that have been marked for migration in the given range
5580  */
5581 void
5582 page_migrate(
5583 	struct seg	*seg,
5584 	caddr_t		addr,
5585 	page_t		**ppa,
5586 	pgcnt_t		npages)
5587 {
5588 	lgrp_t		*from;
5589 	lgrp_t		*to;
5590 	page_t		*newpp;
5591 	page_t		*pp;
5592 	pfn_t		pfn;
5593 	size_t		pgsz;
5594 	spgcnt_t	page_cnt;
5595 	spgcnt_t	i;
5596 	uint_t		pszc;
5597 
5598 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
5599 
5600 	while (npages > 0) {
5601 		pp = *ppa;
5602 		pszc = pp->p_szc;
5603 		pgsz = page_get_pagesize(pszc);
5604 		page_cnt = btop(pgsz);
5605 
5606 		/*
5607 		 * Check to see whether this page is marked for migration
5608 		 *
5609 		 * Assume that root page of large page is marked for
5610 		 * migration and none of the other constituent pages
5611 		 * are marked.  This really simplifies clearing the
5612 		 * migrate bit by not having to clear it from each
5613 		 * constituent page.
5614 		 *
5615 		 * note we don't want to relocate an entire large page if
5616 		 * someone is only using one subpage.
5617 		 */
5618 		if (npages < page_cnt)
5619 			break;
5620 
5621 		/*
5622 		 * Is it marked for migration?
5623 		 */
5624 		if (!PP_ISMIGRATE(pp))
5625 			goto next;
5626 
5627 		/*
5628 		 * Determine lgroups that page is being migrated between
5629 		 */
5630 		pfn = page_pptonum(pp);
5631 		if (!IS_P2ALIGNED(pfn, page_cnt)) {
5632 			break;
5633 		}
5634 		from = lgrp_pfn_to_lgrp(pfn);
5635 		to = lgrp_mem_choose(seg, addr, pgsz);
5636 
5637 		/*
5638 		 * Check to see whether we are trying to migrate page to lgroup
5639 		 * where it is allocated already
5640 		 */
5641 		if (to == from) {
5642 			PP_CLRMIGRATE(pp);
5643 			goto next;
5644 		}
5645 
5646 		/*
5647 		 * Need to get exclusive lock's to migrate
5648 		 */
5649 		for (i = 0; i < page_cnt; i++) {
5650 			ASSERT(PAGE_LOCKED(ppa[i]));
5651 			if (page_pptonum(ppa[i]) != pfn + i ||
5652 			    ppa[i]->p_szc != pszc) {
5653 				break;
5654 			}
5655 			if (!page_tryupgrade(ppa[i])) {
5656 				lgrp_stat_add(from->lgrp_id,
5657 				    LGRP_PM_FAIL_LOCK_PGS,
5658 				    page_cnt);
5659 				break;
5660 			}
5661 		}
5662 		if (i != page_cnt) {
5663 			while (--i != -1) {
5664 				page_downgrade(ppa[i]);
5665 			}
5666 			goto next;
5667 		}
5668 
5669 		(void) page_create_wait(page_cnt, PG_WAIT);
5670 		newpp = page_get_replacement_page(pp, to, PGR_SAMESZC);
5671 		if (newpp == NULL) {
5672 			page_create_putback(page_cnt);
5673 			for (i = 0; i < page_cnt; i++) {
5674 				page_downgrade(ppa[i]);
5675 			}
5676 			lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS,
5677 			    page_cnt);
5678 			goto next;
5679 		}
5680 		ASSERT(newpp->p_szc == pszc);
5681 		/*
5682 		 * Clear migrate bit and relocate page
5683 		 */
5684 		PP_CLRMIGRATE(pp);
5685 		if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) {
5686 			panic("page_migrate: page_relocate failed");
5687 		}
5688 		ASSERT(page_cnt * PAGESIZE == pgsz);
5689 
5690 		/*
5691 		 * Keep stats for number of pages migrated from and to
5692 		 * each lgroup
5693 		 */
5694 		lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt);
5695 		lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt);
5696 		/*
5697 		 * update the page_t array we were passed in and
5698 		 * unlink constituent pages of a large page.
5699 		 */
5700 		for (i = 0; i < page_cnt; ++i, ++pp) {
5701 			ASSERT(PAGE_EXCL(newpp));
5702 			ASSERT(newpp->p_szc == pszc);
5703 			ppa[i] = newpp;
5704 			pp = newpp;
5705 			page_sub(&newpp, pp);
5706 			page_downgrade(pp);
5707 		}
5708 		ASSERT(newpp == NULL);
5709 next:
5710 		addr += pgsz;
5711 		ppa += page_cnt;
5712 		npages -= page_cnt;
5713 	}
5714 }
5715 
5716 ulong_t mem_waiters 	= 0;
5717 ulong_t	max_count 	= 20;
5718 #define	MAX_DELAY	0x1ff
5719 
5720 /*
5721  * Check if enough memory is available to proceed.
5722  * Depending on system configuration and how much memory is
5723  * reserved for swap we need to check against two variables.
5724  * e.g. on systems with little physical swap availrmem can be
5725  * more reliable indicator of how much memory is available.
5726  * On systems with large phys swap freemem can be better indicator.
5727  * If freemem drops below threshold level don't return an error
5728  * immediately but wake up pageout to free memory and block.
5729  * This is done number of times. If pageout is not able to free
5730  * memory within certain time return an error.
5731  * The same applies for availrmem but kmem_reap is used to
5732  * free memory.
5733  */
5734 int
5735 page_mem_avail(pgcnt_t npages)
5736 {
5737 	ulong_t count;
5738 
5739 #if defined(__i386)
5740 	if (freemem > desfree + npages &&
5741 	    availrmem > swapfs_reserve + npages &&
5742 	    btop(vmem_size(heap_arena, VMEM_FREE)) > tune.t_minarmem +
5743 	    npages)
5744 		return (1);
5745 #else
5746 	if (freemem > desfree + npages &&
5747 	    availrmem > swapfs_reserve + npages)
5748 		return (1);
5749 #endif
5750 
5751 	count = max_count;
5752 	atomic_add_long(&mem_waiters, 1);
5753 
5754 	while (freemem < desfree + npages && --count) {
5755 		cv_signal(&proc_pageout->p_cv);
5756 		if (delay_sig(hz + (mem_waiters & MAX_DELAY))) {
5757 			atomic_add_long(&mem_waiters, -1);
5758 			return (0);
5759 		}
5760 	}
5761 	if (count == 0) {
5762 		atomic_add_long(&mem_waiters, -1);
5763 		return (0);
5764 	}
5765 
5766 	count = max_count;
5767 	while (availrmem < swapfs_reserve + npages && --count) {
5768 		kmem_reap();
5769 		if (delay_sig(hz + (mem_waiters & MAX_DELAY))) {
5770 			atomic_add_long(&mem_waiters, -1);
5771 			return (0);
5772 		}
5773 	}
5774 	atomic_add_long(&mem_waiters, -1);
5775 	if (count == 0)
5776 		return (0);
5777 
5778 #if defined(__i386)
5779 	if (btop(vmem_size(heap_arena, VMEM_FREE)) <
5780 	    tune.t_minarmem + npages)
5781 		return (0);
5782 #endif
5783 	return (1);
5784 }
5785 
5786 #define	MAX_CNT	60	/* max num of iterations */
5787 /*
5788  * Reclaim/reserve availrmem for npages.
5789  * If there is not enough memory start reaping seg, kmem caches.
5790  * Start pageout scanner (via page_needfree()).
5791  * Exit after ~ MAX_CNT s regardless of how much memory has been released.
5792  * Note: There is no guarantee that any availrmem will be freed as
5793  * this memory typically is locked (kernel heap) or reserved for swap.
5794  * Also due to memory fragmentation kmem allocator may not be able
5795  * to free any memory (single user allocated buffer will prevent
5796  * freeing slab or a page).
5797  */
5798 int
5799 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust)
5800 {
5801 	int	i = 0;
5802 	int	ret = 0;
5803 	pgcnt_t	deficit;
5804 	pgcnt_t old_availrmem;
5805 
5806 	mutex_enter(&freemem_lock);
5807 	old_availrmem = availrmem - 1;
5808 	while ((availrmem < tune.t_minarmem + npages + epages) &&
5809 	    (old_availrmem < availrmem) && (i++ < MAX_CNT)) {
5810 		old_availrmem = availrmem;
5811 		deficit = tune.t_minarmem + npages + epages - availrmem;
5812 		mutex_exit(&freemem_lock);
5813 		page_needfree(deficit);
5814 		seg_preap();
5815 		kmem_reap();
5816 		delay(hz);
5817 		page_needfree(-(spgcnt_t)deficit);
5818 		mutex_enter(&freemem_lock);
5819 	}
5820 
5821 	if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) {
5822 		availrmem -= npages;
5823 		ret = 1;
5824 	}
5825 
5826 	mutex_exit(&freemem_lock);
5827 
5828 	return (ret);
5829 }
5830 
5831 /*
5832  * Search the memory segments to locate the desired page.  Within a
5833  * segment, pages increase linearly with one page structure per
5834  * physical page frame (size PAGESIZE).  The search begins
5835  * with the segment that was accessed last, to take advantage of locality.
5836  * If the hint misses, we start from the beginning of the sorted memseg list
5837  */
5838 
5839 
5840 /*
5841  * Some data structures for pfn to pp lookup.
5842  */
5843 ulong_t mhash_per_slot;
5844 struct memseg *memseg_hash[N_MEM_SLOTS];
5845 
5846 page_t *
5847 page_numtopp_nolock(pfn_t pfnum)
5848 {
5849 	struct memseg *seg;
5850 	page_t *pp;
5851 	vm_cpu_data_t *vc = CPU->cpu_vm_data;
5852 
5853 	ASSERT(vc != NULL);
5854 
5855 	MEMSEG_STAT_INCR(nsearch);
5856 
5857 	/* Try last winner first */
5858 	if (((seg = vc->vc_pnum_memseg) != NULL) &&
5859 		(pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5860 		MEMSEG_STAT_INCR(nlastwon);
5861 		pp = seg->pages + (pfnum - seg->pages_base);
5862 		if (pp->p_pagenum == pfnum)
5863 			return ((page_t *)pp);
5864 	}
5865 
5866 	/* Else Try hash */
5867 	if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
5868 		(pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5869 		MEMSEG_STAT_INCR(nhashwon);
5870 		vc->vc_pnum_memseg = seg;
5871 		pp = seg->pages + (pfnum - seg->pages_base);
5872 		if (pp->p_pagenum == pfnum)
5873 			return ((page_t *)pp);
5874 	}
5875 
5876 	/* Else Brute force */
5877 	for (seg = memsegs; seg != NULL; seg = seg->next) {
5878 		if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5879 			vc->vc_pnum_memseg = seg;
5880 			pp = seg->pages + (pfnum - seg->pages_base);
5881 			return ((page_t *)pp);
5882 		}
5883 	}
5884 	vc->vc_pnum_memseg = NULL;
5885 	MEMSEG_STAT_INCR(nnotfound);
5886 	return ((page_t *)NULL);
5887 
5888 }
5889 
5890 struct memseg *
5891 page_numtomemseg_nolock(pfn_t pfnum)
5892 {
5893 	struct memseg *seg;
5894 	page_t *pp;
5895 
5896 	/* Try hash */
5897 	if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
5898 		(pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
5899 		pp = seg->pages + (pfnum - seg->pages_base);
5900 		if (pp->p_pagenum == pfnum)
5901 			return (seg);
5902 	}
5903 
5904 	/* Else Brute force */
5905 	for (seg = memsegs; seg != NULL; seg = seg->next) {
5906 		if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5907 			return (seg);
5908 		}
5909 	}
5910 	return ((struct memseg *)NULL);
5911 }
5912 
5913 /*
5914  * Given a page and a count return the page struct that is
5915  * n structs away from the current one in the global page
5916  * list.
5917  *
5918  * This function wraps to the first page upon
5919  * reaching the end of the memseg list.
5920  */
5921 page_t *
5922 page_nextn(page_t *pp, ulong_t n)
5923 {
5924 	struct memseg *seg;
5925 	page_t *ppn;
5926 	vm_cpu_data_t *vc = (vm_cpu_data_t *)CPU->cpu_vm_data;
5927 
5928 	ASSERT(vc != NULL);
5929 
5930 	if (((seg = vc->vc_pnext_memseg) == NULL) ||
5931 	    (seg->pages_base == seg->pages_end) ||
5932 	    !(pp >= seg->pages && pp < seg->epages)) {
5933 
5934 		for (seg = memsegs; seg; seg = seg->next) {
5935 			if (pp >= seg->pages && pp < seg->epages)
5936 				break;
5937 		}
5938 
5939 		if (seg == NULL) {
5940 			/* Memory delete got in, return something valid. */
5941 			/* TODO: fix me. */
5942 			seg = memsegs;
5943 			pp = seg->pages;
5944 		}
5945 	}
5946 
5947 	/* check for wraparound - possible if n is large */
5948 	while ((ppn = (pp + n)) >= seg->epages || ppn < pp) {
5949 		n -= seg->epages - pp;
5950 		seg = seg->next;
5951 		if (seg == NULL)
5952 			seg = memsegs;
5953 		pp = seg->pages;
5954 	}
5955 	vc->vc_pnext_memseg = seg;
5956 	return (ppn);
5957 }
5958 
5959 /*
5960  * Initialize for a loop using page_next_scan_large().
5961  */
5962 page_t *
5963 page_next_scan_init(void **cookie)
5964 {
5965 	ASSERT(cookie != NULL);
5966 	*cookie = (void *)memsegs;
5967 	return ((page_t *)memsegs->pages);
5968 }
5969 
5970 /*
5971  * Return the next page in a scan of page_t's, assuming we want
5972  * to skip over sub-pages within larger page sizes.
5973  *
5974  * The cookie is used to keep track of the current memseg.
5975  */
5976 page_t *
5977 page_next_scan_large(
5978 	page_t		*pp,
5979 	ulong_t		*n,
5980 	void		**cookie)
5981 {
5982 	struct memseg	*seg = (struct memseg *)*cookie;
5983 	page_t		*new_pp;
5984 	ulong_t		cnt;
5985 	pfn_t		pfn;
5986 
5987 
5988 	/*
5989 	 * get the count of page_t's to skip based on the page size
5990 	 */
5991 	ASSERT(pp != NULL);
5992 	if (pp->p_szc == 0) {
5993 		cnt = 1;
5994 	} else {
5995 		pfn = page_pptonum(pp);
5996 		cnt = page_get_pagecnt(pp->p_szc);
5997 		cnt -= pfn & (cnt - 1);
5998 	}
5999 	*n += cnt;
6000 	new_pp = pp + cnt;
6001 
6002 	/*
6003 	 * Catch if we went past the end of the current memory segment. If so,
6004 	 * just move to the next segment with pages.
6005 	 */
6006 	if (new_pp >= seg->epages) {
6007 		do {
6008 			seg = seg->next;
6009 			if (seg == NULL)
6010 				seg = memsegs;
6011 		} while (seg->pages == seg->epages);
6012 		new_pp = seg->pages;
6013 		*cookie = (void *)seg;
6014 	}
6015 
6016 	return (new_pp);
6017 }
6018 
6019 
6020 /*
6021  * Returns next page in list. Note: this function wraps
6022  * to the first page in the list upon reaching the end
6023  * of the list. Callers should be aware of this fact.
6024  */
6025 
6026 /* We should change this be a #define */
6027 
6028 page_t *
6029 page_next(page_t *pp)
6030 {
6031 	return (page_nextn(pp, 1));
6032 }
6033 
6034 page_t *
6035 page_first()
6036 {
6037 	return ((page_t *)memsegs->pages);
6038 }
6039 
6040 
6041 /*
6042  * This routine is called at boot with the initial memory configuration
6043  * and when memory is added or removed.
6044  */
6045 void
6046 build_pfn_hash()
6047 {
6048 	pfn_t cur;
6049 	pgcnt_t index;
6050 	struct memseg *pseg;
6051 	int	i;
6052 
6053 	/*
6054 	 * Clear memseg_hash array.
6055 	 * Since memory add/delete is designed to operate concurrently
6056 	 * with normal operation, the hash rebuild must be able to run
6057 	 * concurrently with page_numtopp_nolock(). To support this
6058 	 * functionality, assignments to memseg_hash array members must
6059 	 * be done atomically.
6060 	 *
6061 	 * NOTE: bzero() does not currently guarantee this for kernel
6062 	 * threads, and cannot be used here.
6063 	 */
6064 	for (i = 0; i < N_MEM_SLOTS; i++)
6065 		memseg_hash[i] = NULL;
6066 
6067 	hat_kpm_mseghash_clear(N_MEM_SLOTS);
6068 
6069 	/*
6070 	 * Physmax is the last valid pfn.
6071 	 */
6072 	mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT;
6073 	for (pseg = memsegs; pseg != NULL; pseg = pseg->next) {
6074 		index = MEMSEG_PFN_HASH(pseg->pages_base);
6075 		cur = pseg->pages_base;
6076 		do {
6077 			if (index >= N_MEM_SLOTS)
6078 				index = MEMSEG_PFN_HASH(cur);
6079 
6080 			if (memseg_hash[index] == NULL ||
6081 			    memseg_hash[index]->pages_base > pseg->pages_base) {
6082 				memseg_hash[index] = pseg;
6083 				hat_kpm_mseghash_update(index, pseg);
6084 			}
6085 			cur += mhash_per_slot;
6086 			index++;
6087 		} while (cur < pseg->pages_end);
6088 	}
6089 }
6090 
6091 /*
6092  * Return the pagenum for the pp
6093  */
6094 pfn_t
6095 page_pptonum(page_t *pp)
6096 {
6097 	return (pp->p_pagenum);
6098 }
6099 
6100 /*
6101  * interface to the referenced and modified etc bits
6102  * in the PSM part of the page struct
6103  * when no locking is desired.
6104  */
6105 void
6106 page_set_props(page_t *pp, uint_t flags)
6107 {
6108 	ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0);
6109 	pp->p_nrm |= (uchar_t)flags;
6110 }
6111 
6112 void
6113 page_clr_all_props(page_t *pp)
6114 {
6115 	pp->p_nrm = 0;
6116 }
6117 
6118 /*
6119  * Clear p_lckcnt and p_cowcnt, adjusting freemem if required.
6120  */
6121 int
6122 page_clear_lck_cow(page_t *pp, int adjust)
6123 {
6124 	int	f_amount;
6125 
6126 	ASSERT(PAGE_EXCL(pp));
6127 
6128 	/*
6129 	 * The page_struct_lock need not be acquired here since
6130 	 * we require the caller hold the page exclusively locked.
6131 	 */
6132 	f_amount = 0;
6133 	if (pp->p_lckcnt) {
6134 		f_amount = 1;
6135 		pp->p_lckcnt = 0;
6136 	}
6137 	if (pp->p_cowcnt) {
6138 		f_amount += pp->p_cowcnt;
6139 		pp->p_cowcnt = 0;
6140 	}
6141 
6142 	if (adjust && f_amount) {
6143 		mutex_enter(&freemem_lock);
6144 		availrmem += f_amount;
6145 		mutex_exit(&freemem_lock);
6146 	}
6147 
6148 	return (f_amount);
6149 }
6150 
6151 /*
6152  * The following functions is called from free_vp_pages()
6153  * for an inexact estimate of a newly free'd page...
6154  */
6155 ulong_t
6156 page_share_cnt(page_t *pp)
6157 {
6158 	return (hat_page_getshare(pp));
6159 }
6160 
6161 int
6162 page_isshared(page_t *pp)
6163 {
6164 	return (hat_page_getshare(pp) > 1);
6165 }
6166 
6167 int
6168 page_isfree(page_t *pp)
6169 {
6170 	return (PP_ISFREE(pp));
6171 }
6172 
6173 int
6174 page_isref(page_t *pp)
6175 {
6176 	return (hat_page_getattr(pp, P_REF));
6177 }
6178 
6179 int
6180 page_ismod(page_t *pp)
6181 {
6182 	return (hat_page_getattr(pp, P_MOD));
6183 }
6184