xref: /freebsd/sys/vm/vm_page.h (revision 934bfc128efae7dec678dd52f75a28ac3538384a)
160727d8bSWarner Losh /*-
2796df753SPedro F. Giffuni  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
351369649SPedro F. Giffuni  *
4df8bae1dSRodney W. Grimes  * Copyright (c) 1991, 1993
5df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
6df8bae1dSRodney W. Grimes  *
7df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
8df8bae1dSRodney W. Grimes  * The Mach Operating System project at Carnegie-Mellon University.
9df8bae1dSRodney W. Grimes  *
10df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
11df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
12df8bae1dSRodney W. Grimes  * are met:
13df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
14df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
15df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
16df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
17df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
18fbbd9655SWarner Losh  * 3. Neither the name of the University nor the names of its contributors
19df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
20df8bae1dSRodney W. Grimes  *    without specific prior written permission.
21df8bae1dSRodney W. Grimes  *
22df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
33df8bae1dSRodney W. Grimes  *
343c4dd356SDavid Greenman  *	from: @(#)vm_page.h	8.2 (Berkeley) 12/13/93
35df8bae1dSRodney W. Grimes  *
36df8bae1dSRodney W. Grimes  *
37df8bae1dSRodney W. Grimes  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38df8bae1dSRodney W. Grimes  * All rights reserved.
39df8bae1dSRodney W. Grimes  *
40df8bae1dSRodney W. Grimes  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41df8bae1dSRodney W. Grimes  *
42df8bae1dSRodney W. Grimes  * Permission to use, copy, modify and distribute this software and
43df8bae1dSRodney W. Grimes  * its documentation is hereby granted, provided that both the copyright
44df8bae1dSRodney W. Grimes  * notice and this permission notice appear in all copies of the
45df8bae1dSRodney W. Grimes  * software, derivative works or modified versions, and any portions
46df8bae1dSRodney W. Grimes  * thereof, and that both notices appear in supporting documentation.
47df8bae1dSRodney W. Grimes  *
48df8bae1dSRodney W. Grimes  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49df8bae1dSRodney W. Grimes  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50df8bae1dSRodney W. Grimes  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51df8bae1dSRodney W. Grimes  *
52df8bae1dSRodney W. Grimes  * Carnegie Mellon requests users of this software to return to
53df8bae1dSRodney W. Grimes  *
54df8bae1dSRodney W. Grimes  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
55df8bae1dSRodney W. Grimes  *  School of Computer Science
56df8bae1dSRodney W. Grimes  *  Carnegie Mellon University
57df8bae1dSRodney W. Grimes  *  Pittsburgh PA 15213-3890
58df8bae1dSRodney W. Grimes  *
59df8bae1dSRodney W. Grimes  * any improvements or extensions that they make and grant Carnegie the
60df8bae1dSRodney W. Grimes  * rights to redistribute these changes.
613c4dd356SDavid Greenman  *
62c3aac50fSPeter Wemm  * $FreeBSD$
63df8bae1dSRodney W. Grimes  */
64df8bae1dSRodney W. Grimes 
65df8bae1dSRodney W. Grimes /*
66df8bae1dSRodney W. Grimes  *	Resident memory system definitions.
67df8bae1dSRodney W. Grimes  */
68df8bae1dSRodney W. Grimes 
69df8bae1dSRodney W. Grimes #ifndef	_VM_PAGE_
70df8bae1dSRodney W. Grimes #define	_VM_PAGE_
71df8bae1dSRodney W. Grimes 
72f919ebdeSDavid Greenman #include <vm/pmap.h>
73431fb8abSMark Johnston #include <vm/_vm_phys.h>
74069e9bc1SDoug Rabson 
75df8bae1dSRodney W. Grimes /*
76df8bae1dSRodney W. Grimes  *	Management of resident (logical) pages.
77df8bae1dSRodney W. Grimes  *
78df8bae1dSRodney W. Grimes  *	A small structure is kept for each resident
79df8bae1dSRodney W. Grimes  *	page, indexed by page number.  Each structure
80da384208SAlan Cox  *	is an element of several collections:
81df8bae1dSRodney W. Grimes  *
82da384208SAlan Cox  *		A radix tree used to quickly
83df8bae1dSRodney W. Grimes  *		perform object/offset lookups
84df8bae1dSRodney W. Grimes  *
85df8bae1dSRodney W. Grimes  *		A list of all pages for a given object,
86df8bae1dSRodney W. Grimes  *		so they can be quickly deactivated at
87df8bae1dSRodney W. Grimes  *		time of deallocation.
88df8bae1dSRodney W. Grimes  *
89df8bae1dSRodney W. Grimes  *		An ordered list of pages due for pageout.
90df8bae1dSRodney W. Grimes  *
91df8bae1dSRodney W. Grimes  *	In addition, the structure contains the object
92df8bae1dSRodney W. Grimes  *	and offset to which this page belongs (for pageout),
93df8bae1dSRodney W. Grimes  *	and sundry status bits.
94df8bae1dSRodney W. Grimes  *
953c76db4cSAlan Cox  *	In general, operations on this structure's mutable fields are
960012f373SJeff Roberson  *	synchronized using either one of or a combination of locks.  If a
970012f373SJeff Roberson  *	field is annotated with two of these locks then holding either is
980012f373SJeff Roberson  *	sufficient for read access but both are required for write access.
99efec381dSMark Johnston  *	The queue lock for a page depends on the value of its queue field and is
100efec381dSMark Johnston  *	described in detail below.
1010012f373SJeff Roberson  *
1020012f373SJeff Roberson  *	The following annotations are possible:
103958d8f52SMark Johnston  *	(A) the field must be accessed using atomic(9) and may require
104958d8f52SMark Johnston  *	    additional synchronization.
1050012f373SJeff Roberson  *	(B) the page busy lock.
1060012f373SJeff Roberson  *	(C) the field is immutable.
107efec381dSMark Johnston  *	(F) the per-domain lock for the free queues.
1080012f373SJeff Roberson  *	(M) Machine dependent, defined by pmap layer.
1090012f373SJeff Roberson  *	(O) the object that the page belongs to.
1100012f373SJeff Roberson  *	(Q) the page's queue lock.
1110012f373SJeff Roberson  *
1120012f373SJeff Roberson  *	The busy lock is an embedded reader-writer lock that protects the
1130012f373SJeff Roberson  *	page's contents and identity (i.e., its <object, pindex> tuple) as
1140012f373SJeff Roberson  *	well as certain valid/dirty modifications.  To avoid bloating the
1150012f373SJeff Roberson  *	the page structure, the busy lock lacks some of the features available
1160012f373SJeff Roberson  *	the kernel's general-purpose synchronization primitives.  As a result,
1170012f373SJeff Roberson  *	busy lock ordering rules are not verified, lock recursion is not
1180012f373SJeff Roberson  *	detected, and an attempt to xbusy a busy page or sbusy an xbusy page
1190012f373SJeff Roberson  *	results will trigger a panic rather than causing the thread to block.
1200012f373SJeff Roberson  *	vm_page_sleep_if_busy() can be used to sleep until the page's busy
1210012f373SJeff Roberson  *	state changes, after which the caller must re-lookup the page and
1220012f373SJeff Roberson  *	re-evaluate its state.  vm_page_busy_acquire() will block until
1230012f373SJeff Roberson  *	the lock is acquired.
1240012f373SJeff Roberson  *
1250012f373SJeff Roberson  *	The valid field is protected by the page busy lock (B) and object
1260012f373SJeff Roberson  *	lock (O).  Transitions from invalid to valid are generally done
1270012f373SJeff Roberson  *	via I/O or zero filling and do not require the object lock.
1280012f373SJeff Roberson  *	These must be protected with the busy lock to prevent page-in or
1290012f373SJeff Roberson  *	creation races.  Page invalidation generally happens as a result
1300012f373SJeff Roberson  *	of truncate or msync.  When invalidated, pages must not be present
1310012f373SJeff Roberson  *	in pmap and must hold the object lock to prevent concurrent
1320012f373SJeff Roberson  *	speculative read-only mappings that do not require busy.  I/O
1330012f373SJeff Roberson  *	routines may check for validity without a lock if they are prepared
1340012f373SJeff Roberson  *	to handle invalidation races with higher level locks (vnode) or are
1350012f373SJeff Roberson  *	unconcerned with races so long as they hold a reference to prevent
1360012f373SJeff Roberson  *	recycling.  When a valid bit is set while holding a shared busy
1370012f373SJeff Roberson  *	lock (A) atomic operations are used to protect against concurrent
1380012f373SJeff Roberson  *	modification.
1390ce3ba8cSKip Macy  *
140abb9b935SKonstantin Belousov  *	In contrast, the synchronization of accesses to the page's
1410012f373SJeff Roberson  *	dirty field is a mix of machine dependent (M) and busy (B).  In
1420012f373SJeff Roberson  *	the machine-independent layer, the page busy must be held to
1430012f373SJeff Roberson  *	operate on the field.  However, the pmap layer is permitted to
1440012f373SJeff Roberson  *	set all bits within the field without holding that lock.  If the
1450012f373SJeff Roberson  *	underlying architecture does not support atomic read-modify-write
146abb9b935SKonstantin Belousov  *	operations on the field's type, then the machine-independent
1472042bb37SKonstantin Belousov  *	layer uses a 32-bit atomic on the aligned 32-bit word that
148abb9b935SKonstantin Belousov  *	contains the dirty field.  In the machine-independent layer,
149abb9b935SKonstantin Belousov  *	the implementation of read-modify-write operations on the
1500012f373SJeff Roberson  *	field is encapsulated in vm_page_clear_dirty_mask().  An
1510012f373SJeff Roberson  *	exclusive busy lock combined with pmap_remove_{write/all}() is the
1520012f373SJeff Roberson  *	only way to ensure a page can not become dirty.  I/O generally
1530012f373SJeff Roberson  *	removes the page from pmap to ensure exclusive access and atomic
1540012f373SJeff Roberson  *	writes.
1551d3a1bcfSMark Johnston  *
156fee2a2faSMark Johnston  *	The ref_count field tracks references to the page.  References that
157fee2a2faSMark Johnston  *	prevent the page from being reclaimable are called wirings and are
158fee2a2faSMark Johnston  *	counted in the low bits of ref_count.  The containing object's
159fee2a2faSMark Johnston  *	reference, if one exists, is counted using the VPRC_OBJREF bit in the
160fee2a2faSMark Johnston  *	ref_count field.  Additionally, the VPRC_BLOCKED bit is used to
161fee2a2faSMark Johnston  *	atomically check for wirings and prevent new wirings via
162fee2a2faSMark Johnston  *	pmap_extract_and_hold().  When a page belongs to an object, it may be
163fee2a2faSMark Johnston  *	wired only when the object is locked, or the page is busy, or by
164fee2a2faSMark Johnston  *	pmap_extract_and_hold().  As a result, if the object is locked and the
165fee2a2faSMark Johnston  *	page is not busy (or is exclusively busied by the current thread), and
166fee2a2faSMark Johnston  *	the page is unmapped, its wire count will not increase.  The ref_count
167fee2a2faSMark Johnston  *	field is updated using atomic operations in most cases, except when it
168fee2a2faSMark Johnston  *	is known that no other references to the page exist, such as in the page
169fee2a2faSMark Johnston  *	allocator.  A page may be present in the page queues, or even actively
170fee2a2faSMark Johnston  *	scanned by the page daemon, without an explicitly counted referenced.
171fee2a2faSMark Johnston  *	The page daemon must therefore handle the possibility of a concurrent
172fee2a2faSMark Johnston  *	free of the page.
1731d3a1bcfSMark Johnston  *
174dc71caa0SMark Johnston  *	The queue state of a page consists of the queue and act_count fields of
175dc71caa0SMark Johnston  *	its atomically updated state, and the subset of atomic flags specified
176dc71caa0SMark Johnston  *	by PGA_QUEUE_STATE_MASK.  The queue field contains the page's page queue
177dc71caa0SMark Johnston  *	index, or PQ_NONE if it does not belong to a page queue.  To modify the
178dc71caa0SMark Johnston  *	queue field, the page queue lock corresponding to the old value must be
179dc71caa0SMark Johnston  *	held, unless that value is PQ_NONE, in which case the queue index must
180dc71caa0SMark Johnston  *	be updated using an atomic RMW operation.  There is one exception to
181dc71caa0SMark Johnston  *	this rule: the page daemon may transition the queue field from
182dc71caa0SMark Johnston  *	PQ_INACTIVE to PQ_NONE immediately prior to freeing the page during an
183dc71caa0SMark Johnston  *	inactive queue scan.  At that point the page is already dequeued and no
184dc71caa0SMark Johnston  *	other references to that vm_page structure can exist.  The PGA_ENQUEUED
185dc71caa0SMark Johnston  *	flag, when set, indicates that the page structure is physically inserted
186dc71caa0SMark Johnston  *	into the queue corresponding to the page's queue index, and may only be
187dc71caa0SMark Johnston  *	set or cleared with the corresponding page queue lock held.
1885cd29d0fSMark Johnston  *
189dc71caa0SMark Johnston  *	To avoid contention on page queue locks, page queue operations (enqueue,
190dc71caa0SMark Johnston  *	dequeue, requeue) are batched using fixed-size per-CPU queues.  A
191dc71caa0SMark Johnston  *	deferred operation is requested by setting one of the flags in
192dc71caa0SMark Johnston  *	PGA_QUEUE_OP_MASK and inserting an entry into a batch queue.  When a
193dc71caa0SMark Johnston  *	queue is full, an attempt to insert a new entry will lock the page
194dc71caa0SMark Johnston  *	queues and trigger processing of the pending entries.  The
195dc71caa0SMark Johnston  *	type-stability of vm_page structures is crucial to this scheme since the
196dc71caa0SMark Johnston  *	processing of entries in a given batch queue may be deferred
197dc71caa0SMark Johnston  *	indefinitely.  In particular, a page may be freed with pending batch
198dc71caa0SMark Johnston  *	queue entries.  The page queue operation flags must be set using atomic
199dc71caa0SMark Johnston  *	RWM operations.
200df8bae1dSRodney W. Grimes  */
201df8bae1dSRodney W. Grimes 
202561cc9fcSKonstantin Belousov #if PAGE_SIZE == 4096
203561cc9fcSKonstantin Belousov #define VM_PAGE_BITS_ALL 0xffu
204561cc9fcSKonstantin Belousov typedef uint8_t vm_page_bits_t;
205561cc9fcSKonstantin Belousov #elif PAGE_SIZE == 8192
206561cc9fcSKonstantin Belousov #define VM_PAGE_BITS_ALL 0xffffu
207561cc9fcSKonstantin Belousov typedef uint16_t vm_page_bits_t;
208561cc9fcSKonstantin Belousov #elif PAGE_SIZE == 16384
209561cc9fcSKonstantin Belousov #define VM_PAGE_BITS_ALL 0xffffffffu
210561cc9fcSKonstantin Belousov typedef uint32_t vm_page_bits_t;
211561cc9fcSKonstantin Belousov #elif PAGE_SIZE == 32768
212561cc9fcSKonstantin Belousov #define VM_PAGE_BITS_ALL 0xfffffffffffffffflu
213561cc9fcSKonstantin Belousov typedef uint64_t vm_page_bits_t;
214561cc9fcSKonstantin Belousov #endif
215561cc9fcSKonstantin Belousov 
2165cff1f4dSMark Johnston typedef union vm_page_astate {
2175cff1f4dSMark Johnston 	struct {
2185cff1f4dSMark Johnston 		uint16_t flags;
2195cff1f4dSMark Johnston 		uint8_t	queue;
2205cff1f4dSMark Johnston 		uint8_t act_count;
2215cff1f4dSMark Johnston 	};
2225cff1f4dSMark Johnston 	uint32_t _bits;
2235cff1f4dSMark Johnston } vm_page_astate_t;
2245cff1f4dSMark Johnston 
225df8bae1dSRodney W. Grimes struct vm_page {
226c325e866SKonstantin Belousov 	union {
227c325e866SKonstantin Belousov 		TAILQ_ENTRY(vm_page) q; /* page queue or free list (Q) */
228c325e866SKonstantin Belousov 		struct {
229c325e866SKonstantin Belousov 			SLIST_ENTRY(vm_page) ss; /* private slists */
230c325e866SKonstantin Belousov 		} s;
231c325e866SKonstantin Belousov 		struct {
232c325e866SKonstantin Belousov 			u_long p;
233c325e866SKonstantin Belousov 			u_long v;
234c325e866SKonstantin Belousov 		} memguard;
235584061b4SJeff Roberson 		struct {
236584061b4SJeff Roberson 			void *slab;
237584061b4SJeff Roberson 			void *zone;
238584061b4SJeff Roberson 		} uma;
239c325e866SKonstantin Belousov 	} plinks;
240e3975643SJake Burkholder 	TAILQ_ENTRY(vm_page) listq;	/* pages in same object (O) */
241fee2a2faSMark Johnston 	vm_object_t object;		/* which object am I in (O) */
24243319c11SAlan Cox 	vm_pindex_t pindex;		/* offset into object (O,P) */
2435cd29d0fSMark Johnston 	vm_paddr_t phys_addr;		/* physical address of page (C) */
244763df3ecSPedro F. Giffuni 	struct md_page md;		/* machine dependent stuff */
245b119329dSMark Johnston 	u_int ref_count;		/* page references (A) */
246958d8f52SMark Johnston 	u_int busy_lock;		/* busy owners lock (A) */
247958d8f52SMark Johnston 	union vm_page_astate a;		/* state accessed atomically (A) */
248eeacb3b0SMark Johnston 	uint8_t order;			/* index of the buddy queue (F) */
249eeacb3b0SMark Johnston 	uint8_t pool;			/* vm_phys freepool index (F) */
2503a2ba997SMark Johnston 	uint8_t flags;			/* page PG_* flags (P) */
251e8bcf696SMark Johnston 	uint8_t oflags;			/* page VPO_* flags (O) */
252dd05fa19SAlan Cox 	int8_t psind;			/* pagesizes[] index (O) */
2531d3a1bcfSMark Johnston 	int8_t segind;			/* vm_phys segment index (C) */
254c325e866SKonstantin Belousov 	/* NOTE that these must support one bit per DEV_BSIZE in a page */
255bd7e5f99SJohn Dyson 	/* so, on normal X86 kernels, they must be at least 8 bits wide */
2560012f373SJeff Roberson 	vm_page_bits_t valid;		/* valid DEV_BSIZE chunk map (O,B) */
2570012f373SJeff Roberson 	vm_page_bits_t dirty;		/* dirty DEV_BSIZE chunk map (M,B) */
258df8bae1dSRodney W. Grimes };
259df8bae1dSRodney W. Grimes 
2605786be7cSAlan Cox /*
261fee2a2faSMark Johnston  * Special bits used in the ref_count field.
262fee2a2faSMark Johnston  *
263fee2a2faSMark Johnston  * ref_count is normally used to count wirings that prevent the page from being
264fee2a2faSMark Johnston  * reclaimed, but also supports several special types of references that do not
265fee2a2faSMark Johnston  * prevent reclamation.  Accesses to the ref_count field must be atomic unless
266fee2a2faSMark Johnston  * the page is unallocated.
267fee2a2faSMark Johnston  *
268fee2a2faSMark Johnston  * VPRC_OBJREF is the reference held by the containing object.  It can set or
269fee2a2faSMark Johnston  * cleared only when the corresponding object's write lock is held.
270fee2a2faSMark Johnston  *
271fee2a2faSMark Johnston  * VPRC_BLOCKED is used to atomically block wirings via pmap lookups while
272efec381dSMark Johnston  * attempting to tear down all mappings of a given page.  The page busy lock and
273fee2a2faSMark Johnston  * object write lock must both be held in order to set or clear this bit.
274fee2a2faSMark Johnston  */
275fee2a2faSMark Johnston #define	VPRC_BLOCKED	0x40000000u	/* mappings are being removed */
276fee2a2faSMark Johnston #define	VPRC_OBJREF	0x80000000u	/* object reference, cleared with (O) */
277fee2a2faSMark Johnston #define	VPRC_WIRE_COUNT(c)	((c) & ~(VPRC_BLOCKED | VPRC_OBJREF))
278fee2a2faSMark Johnston #define	VPRC_WIRE_COUNT_MAX	(~(VPRC_BLOCKED | VPRC_OBJREF))
279fee2a2faSMark Johnston 
280fee2a2faSMark Johnston /*
2815786be7cSAlan Cox  * Page flags stored in oflags:
2825786be7cSAlan Cox  *
2835786be7cSAlan Cox  * Access to these page flags is synchronized by the lock on the object
2845786be7cSAlan Cox  * containing the page (O).
285d98d0ce2SKonstantin Belousov  *
286d98d0ce2SKonstantin Belousov  * Note: VPO_UNMANAGED (used by OBJT_DEVICE, OBJT_PHYS and OBJT_SG)
287d98d0ce2SKonstantin Belousov  * 	 indicates that the page is not under PV management but
288d98d0ce2SKonstantin Belousov  * 	 otherwise should be treated as a normal page.  Pages not
289d98d0ce2SKonstantin Belousov  * 	 under PV management cannot be paged out via the
290d98d0ce2SKonstantin Belousov  * 	 object/vm_page_t because there is no knowledge of their pte
291d98d0ce2SKonstantin Belousov  * 	 mappings, and such pages are also not on any PQ queue.
292d98d0ce2SKonstantin Belousov  *
2935786be7cSAlan Cox  */
29449bfa624SAlan Cox #define	VPO_KMEM_EXEC	0x01		/* kmem mapping allows execution */
295c7aebda8SAttilio Rao #define	VPO_SWAPSLEEP	0x02		/* waiting for swap to finish */
296081a4881SAlan Cox #define	VPO_UNMANAGED	0x04		/* no PV management for page */
297081a4881SAlan Cox #define	VPO_SWAPINPROG	0x08		/* swap I/O in progress on page */
2985786be7cSAlan Cox 
299c7aebda8SAttilio Rao /*
300c7aebda8SAttilio Rao  * Busy page implementation details.
301c7aebda8SAttilio Rao  * The algorithm is taken mostly by rwlock(9) and sx(9) locks implementation,
302c7aebda8SAttilio Rao  * even if the support for owner identity is removed because of size
303c7aebda8SAttilio Rao  * constraints.  Checks on lock recursion are then not possible, while the
304c7aebda8SAttilio Rao  * lock assertions effectiveness is someway reduced.
305c7aebda8SAttilio Rao  */
306c7aebda8SAttilio Rao #define	VPB_BIT_SHARED		0x01
307c7aebda8SAttilio Rao #define	VPB_BIT_EXCLUSIVE	0x02
308c7aebda8SAttilio Rao #define	VPB_BIT_WAITERS		0x04
309c7aebda8SAttilio Rao #define	VPB_BIT_FLAGMASK						\
310c7aebda8SAttilio Rao 	(VPB_BIT_SHARED | VPB_BIT_EXCLUSIVE | VPB_BIT_WAITERS)
311c7aebda8SAttilio Rao 
312c7aebda8SAttilio Rao #define	VPB_SHARERS_SHIFT	3
313c7aebda8SAttilio Rao #define	VPB_SHARERS(x)							\
314c7aebda8SAttilio Rao 	(((x) & ~VPB_BIT_FLAGMASK) >> VPB_SHARERS_SHIFT)
315c7aebda8SAttilio Rao #define	VPB_SHARERS_WORD(x)	((x) << VPB_SHARERS_SHIFT | VPB_BIT_SHARED)
316c7aebda8SAttilio Rao #define	VPB_ONE_SHARER		(1 << VPB_SHARERS_SHIFT)
317c7aebda8SAttilio Rao 
318b631c36fSKonstantin Belousov #define	VPB_SINGLE_EXCLUSIVE	VPB_BIT_EXCLUSIVE
319b631c36fSKonstantin Belousov #ifdef INVARIANTS
320b631c36fSKonstantin Belousov #define	VPB_CURTHREAD_EXCLUSIVE						\
321b631c36fSKonstantin Belousov 	(VPB_BIT_EXCLUSIVE | ((u_int)(uintptr_t)curthread & ~VPB_BIT_FLAGMASK))
322b631c36fSKonstantin Belousov #else
323b631c36fSKonstantin Belousov #define	VPB_CURTHREAD_EXCLUSIVE	VPB_SINGLE_EXCLUSIVE
324b631c36fSKonstantin Belousov #endif
325c7aebda8SAttilio Rao 
326c7aebda8SAttilio Rao #define	VPB_UNBUSIED		VPB_SHARERS_WORD(0)
327c7aebda8SAttilio Rao 
328ee9e43f8SJeff Roberson /* Freed lock blocks both shared and exclusive. */
329ee9e43f8SJeff Roberson #define	VPB_FREED		(0xffffffff - VPB_BIT_SHARED)
330ee9e43f8SJeff Roberson 
33144e46b9eSAlan Cox #define	PQ_NONE		255
33244e46b9eSAlan Cox #define	PQ_INACTIVE	0
33344e46b9eSAlan Cox #define	PQ_ACTIVE	1
334ebcddc72SAlan Cox #define	PQ_LAUNDRY	2
335b1fd102eSMark Johnston #define	PQ_UNSWAPPABLE	3
336b1fd102eSMark Johnston #define	PQ_COUNT	4
337ef39c05bSAlexander Leidinger 
338b9e8fb64SKonstantin Belousov #ifndef VM_PAGE_HAVE_PGLIST
3398d220203SAlan Cox TAILQ_HEAD(pglist, vm_page);
340b9e8fb64SKonstantin Belousov #define VM_PAGE_HAVE_PGLIST
341b9e8fb64SKonstantin Belousov #endif
342c325e866SKonstantin Belousov SLIST_HEAD(spglist, vm_page);
34370c17636SAlan Cox 
344449c2e92SKonstantin Belousov #ifdef _KERNEL
345bfc8c24cSGleb Smirnoff extern vm_page_t bogus_page;
346449c2e92SKonstantin Belousov #endif	/* _KERNEL */
347449c2e92SKonstantin Belousov 
3484ceaf45dSAttilio Rao extern struct mtx_padalign pa_lock[];
3492965a453SKip Macy 
3502965a453SKip Macy #if defined(__arm__)
3512965a453SKip Macy #define	PDRSHIFT	PDR_SHIFT
3522965a453SKip Macy #elif !defined(PDRSHIFT)
3532965a453SKip Macy #define PDRSHIFT	21
3542965a453SKip Macy #endif
3552965a453SKip Macy 
3562965a453SKip Macy #define	pa_index(pa)	((pa) >> PDRSHIFT)
3574ceaf45dSAttilio Rao #define	PA_LOCKPTR(pa)	((struct mtx *)(&pa_lock[pa_index(pa) % PA_LOCK_COUNT]))
3582965a453SKip Macy #define	PA_LOCKOBJPTR(pa)	((struct lock_object *)PA_LOCKPTR((pa)))
3592965a453SKip Macy #define	PA_LOCK(pa)	mtx_lock(PA_LOCKPTR(pa))
3602965a453SKip Macy #define	PA_TRYLOCK(pa)	mtx_trylock(PA_LOCKPTR(pa))
3612965a453SKip Macy #define	PA_UNLOCK(pa)	mtx_unlock(PA_LOCKPTR(pa))
3622965a453SKip Macy #define	PA_UNLOCK_COND(pa) 			\
3632965a453SKip Macy 	do {		   			\
364567e51e1SAlan Cox 		if ((pa) != 0) {		\
365567e51e1SAlan Cox 			PA_UNLOCK((pa));	\
366567e51e1SAlan Cox 			(pa) = 0;		\
367567e51e1SAlan Cox 		}				\
3682965a453SKip Macy 	} while (0)
3692965a453SKip Macy 
3702965a453SKip Macy #define	PA_LOCK_ASSERT(pa, a)	mtx_assert(PA_LOCKPTR(pa), (a))
3712965a453SKip Macy 
372f4b36404SMatt Macy #if defined(KLD_MODULE) && !defined(KLD_TIED)
373cf1911a9SKonstantin Belousov #define	vm_page_lock(m)		vm_page_lock_KBI((m), LOCK_FILE, LOCK_LINE)
374cf1911a9SKonstantin Belousov #define	vm_page_unlock(m)	vm_page_unlock_KBI((m), LOCK_FILE, LOCK_LINE)
375cf1911a9SKonstantin Belousov #define	vm_page_trylock(m)	vm_page_trylock_KBI((m), LOCK_FILE, LOCK_LINE)
376cf1911a9SKonstantin Belousov #else	/* !KLD_MODULE */
3772965a453SKip Macy #define	vm_page_lockptr(m)	(PA_LOCKPTR(VM_PAGE_TO_PHYS((m))))
3782965a453SKip Macy #define	vm_page_lock(m)		mtx_lock(vm_page_lockptr((m)))
3792965a453SKip Macy #define	vm_page_unlock(m)	mtx_unlock(vm_page_lockptr((m)))
3802965a453SKip Macy #define	vm_page_trylock(m)	mtx_trylock(vm_page_lockptr((m)))
381ef5ba5a3SAlan Cox #endif
382ef5ba5a3SAlan Cox #if defined(INVARIANTS)
383b4171812SAlan Cox #define	vm_page_assert_locked(m)		\
384b4171812SAlan Cox     vm_page_assert_locked_KBI((m), __FILE__, __LINE__)
385ef5ba5a3SAlan Cox #define	vm_page_lock_assert(m, a)		\
386ef5ba5a3SAlan Cox     vm_page_lock_assert_KBI((m), (a), __FILE__, __LINE__)
387ef5ba5a3SAlan Cox #else
388b4171812SAlan Cox #define	vm_page_assert_locked(m)
389ef5ba5a3SAlan Cox #define	vm_page_lock_assert(m, a)
390cf1911a9SKonstantin Belousov #endif
391e67e0775SAlan Cox 
392df8bae1dSRodney W. Grimes /*
393369763e3SAlan Cox  * The vm_page's aflags are updated using atomic operations.  To set or clear
394369763e3SAlan Cox  * these flags, the functions vm_page_aflag_set() and vm_page_aflag_clear()
395369763e3SAlan Cox  * must be used.  Neither these flags nor these functions are part of the KBI.
3963407fefeSKonstantin Belousov  *
397b4171812SAlan Cox  * PGA_REFERENCED may be cleared only if the page is locked.  It is set by
398b4171812SAlan Cox  * both the MI and MD VM layers.  However, kernel loadable modules should not
399b4171812SAlan Cox  * directly set this flag.  They should call vm_page_reference() instead.
400ce186587SAlan Cox  *
401afb69e6bSKonstantin Belousov  * PGA_WRITEABLE is set exclusively on managed pages by pmap_enter().
402afb69e6bSKonstantin Belousov  * When it does so, the object must be locked, or the page must be
403afb69e6bSKonstantin Belousov  * exclusive busied.  The MI VM layer must never access this flag
404afb69e6bSKonstantin Belousov  * directly.  Instead, it should call pmap_page_is_write_mapped().
40557bd5cceSNathan Whitehorn  *
40657bd5cceSNathan Whitehorn  * PGA_EXECUTABLE may be set by pmap routines, and indicates that a page has
4076031c68dSAlan Cox  * at least one executable mapping.  It is not consumed by the MI VM layer.
4085cd29d0fSMark Johnston  *
409fff5403fSJeff Roberson  * PGA_NOSYNC must be set and cleared with the page busy lock held.
410fff5403fSJeff Roberson  *
4115cd29d0fSMark Johnston  * PGA_ENQUEUED is set and cleared when a page is inserted into or removed
4125cd29d0fSMark Johnston  * from a page queue, respectively.  It determines whether the plinks.q field
413efec381dSMark Johnston  * of the page is valid.  To set or clear this flag, page's "queue" field must
414efec381dSMark Johnston  * be a valid queue index, and the corresponding page queue lock must be held.
4155cd29d0fSMark Johnston  *
4165cd29d0fSMark Johnston  * PGA_DEQUEUE is set when the page is scheduled to be dequeued from a page
4175cd29d0fSMark Johnston  * queue, and cleared when the dequeue request is processed.  A page may
4185cd29d0fSMark Johnston  * have PGA_DEQUEUE set and PGA_ENQUEUED cleared, for instance if a dequeue
4195cd29d0fSMark Johnston  * is requested after the page is scheduled to be enqueued but before it is
420efec381dSMark Johnston  * actually inserted into the page queue.
4215cd29d0fSMark Johnston  *
4225cd29d0fSMark Johnston  * PGA_REQUEUE is set when the page is scheduled to be enqueued or requeued
423efec381dSMark Johnston  * in its page queue.
4245cd29d0fSMark Johnston  *
4255cd29d0fSMark Johnston  * PGA_REQUEUE_HEAD is a special flag for enqueuing pages near the head of
426efec381dSMark Johnston  * the inactive queue, thus bypassing LRU.
427efec381dSMark Johnston  *
428efec381dSMark Johnston  * The PGA_DEQUEUE, PGA_REQUEUE and PGA_REQUEUE_HEAD flags must be set using an
429efec381dSMark Johnston  * atomic RMW operation to ensure that the "queue" field is a valid queue index,
430efec381dSMark Johnston  * and the corresponding page queue lock must be held when clearing any of the
431efec381dSMark Johnston  * flags.
432a8081778SJeff Roberson  *
433a8081778SJeff Roberson  * PGA_SWAP_FREE is used to defer freeing swap space to the pageout daemon
434a8081778SJeff Roberson  * when the context that dirties the page does not have the object write lock
435a8081778SJeff Roberson  * held.
436df8bae1dSRodney W. Grimes  */
4373a2ba997SMark Johnston #define	PGA_WRITEABLE	0x0001		/* page may be mapped writeable */
4383a2ba997SMark Johnston #define	PGA_REFERENCED	0x0002		/* page has been referenced */
4393a2ba997SMark Johnston #define	PGA_EXECUTABLE	0x0004		/* page may be mapped executable */
4403a2ba997SMark Johnston #define	PGA_ENQUEUED	0x0008		/* page is enqueued in a page queue */
4413a2ba997SMark Johnston #define	PGA_DEQUEUE	0x0010		/* page is due to be dequeued */
4423a2ba997SMark Johnston #define	PGA_REQUEUE	0x0020		/* page is due to be requeued */
4433a2ba997SMark Johnston #define	PGA_REQUEUE_HEAD 0x0040		/* page requeue should bypass LRU */
4443a2ba997SMark Johnston #define	PGA_NOSYNC	0x0080		/* do not collect for syncer */
445a8081778SJeff Roberson #define	PGA_SWAP_FREE	0x0100		/* page with swap space was dirtied */
446a8081778SJeff Roberson #define	PGA_SWAP_SPACE	0x0200		/* page has allocated swap space */
4475cd29d0fSMark Johnston 
4486fbaf685SMark Johnston #define	PGA_QUEUE_OP_MASK	(PGA_DEQUEUE | PGA_REQUEUE | PGA_REQUEUE_HEAD)
4496fbaf685SMark Johnston #define	PGA_QUEUE_STATE_MASK	(PGA_ENQUEUED | PGA_QUEUE_OP_MASK)
4503407fefeSKonstantin Belousov 
4513407fefeSKonstantin Belousov /*
452efec381dSMark Johnston  * Page flags.  Updates to these flags are not synchronized, and thus they must
453efec381dSMark Johnston  * be set during page allocation or free to avoid races.
454d9a73522SMark Johnston  *
455d9a73522SMark Johnston  * The PG_PCPU_CACHE flag is set at allocation time if the page was
456d9a73522SMark Johnston  * allocated from a per-CPU cache.  It is cleared the next time that the
457d9a73522SMark Johnston  * page is allocated from the physical memory allocator.
4583407fefeSKonstantin Belousov  */
4593a2ba997SMark Johnston #define	PG_PCPU_CACHE	0x01		/* was allocated from per-CPU caches */
4603a2ba997SMark Johnston #define	PG_FICTITIOUS	0x02		/* physical page doesn't exist */
4613a2ba997SMark Johnston #define	PG_ZERO		0x04		/* page is zeroed */
4623a2ba997SMark Johnston #define	PG_MARKER	0x08		/* special queue marker page */
4633a2ba997SMark Johnston #define	PG_NODUMP	0x10		/* don't include this page in a dump */
464df8bae1dSRodney W. Grimes 
46524a1cce3SDavid Greenman /*
46624a1cce3SDavid Greenman  * Misc constants.
46724a1cce3SDavid Greenman  */
46824a1cce3SDavid Greenman #define ACT_DECLINE		1
46924a1cce3SDavid Greenman #define ACT_ADVANCE		3
47038efa82bSJohn Dyson #define ACT_INIT		5
4715070c7f8SJohn Dyson #define ACT_MAX			64
472df8bae1dSRodney W. Grimes 
473c4473420SPeter Wemm #ifdef _KERNEL
47404a18977SAlan Cox 
475d950c589SKonstantin Belousov #include <sys/kassert.h>
476369763e3SAlan Cox #include <machine/atomic.h>
477369763e3SAlan Cox 
478df8bae1dSRodney W. Grimes /*
479b1fd102eSMark Johnston  * Each pageable resident page falls into one of five lists:
480df8bae1dSRodney W. Grimes  *
481df8bae1dSRodney W. Grimes  *	free
482df8bae1dSRodney W. Grimes  *		Available for allocation now.
48324a1cce3SDavid Greenman  *
484df8bae1dSRodney W. Grimes  *	inactive
4856c5e9bbdSMike Pritchard  *		Low activity, candidates for reclamation.
4863453bca8SAlan Cox  *		This list is approximately LRU ordered.
4873453bca8SAlan Cox  *
4883453bca8SAlan Cox  *	laundry
489df8bae1dSRodney W. Grimes  *		This is the list of pages that should be
490df8bae1dSRodney W. Grimes  *		paged out next.
49124a1cce3SDavid Greenman  *
492b1fd102eSMark Johnston  *	unswappable
493b1fd102eSMark Johnston  *		Dirty anonymous pages that cannot be paged
494b1fd102eSMark Johnston  *		out because no swap device is configured.
495b1fd102eSMark Johnston  *
496df8bae1dSRodney W. Grimes  *	active
4973453bca8SAlan Cox  *		Pages that are "active", i.e., they have been
49824a1cce3SDavid Greenman  *		recently referenced.
49910ad4d48SJohn Dyson  *
500df8bae1dSRodney W. Grimes  */
501df8bae1dSRodney W. Grimes 
5020d94caffSDavid Greenman extern vm_page_t vm_page_array;		/* First resident page in table */
50313a0b7bcSKonstantin Belousov extern long vm_page_array_size;		/* number of vm_page_t's */
5040d94caffSDavid Greenman extern long first_page;			/* first physical page number */
5050d94caffSDavid Greenman 
506df8bae1dSRodney W. Grimes #define VM_PAGE_TO_PHYS(entry)	((entry)->phys_addr)
507df8bae1dSRodney W. Grimes 
508a5073058SJason A. Harmening /*
5097c989c15SJason A. Harmening  * PHYS_TO_VM_PAGE() returns the vm_page_t object that represents a memory
510a5073058SJason A. Harmening  * page to which the given physical address belongs. The correct vm_page_t
511a5073058SJason A. Harmening  * object is returned for addresses that are not page-aligned.
512a5073058SJason A. Harmening  */
513b6de32bdSKonstantin Belousov vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa);
514df8bae1dSRodney W. Grimes 
51589fc8bdbSGleb Smirnoff /*
51689fc8bdbSGleb Smirnoff  * Page allocation parameters for vm_page for the functions
51789fc8bdbSGleb Smirnoff  * vm_page_alloc(), vm_page_grab(), vm_page_alloc_contig() and
51889fc8bdbSGleb Smirnoff  * vm_page_alloc_freelist().  Some functions support only a subset
51989fc8bdbSGleb Smirnoff  * of the flags, and ignore others, see the flags legend.
52089fc8bdbSGleb Smirnoff  *
5215471caf6SAlan Cox  * The meaning of VM_ALLOC_ZERO differs slightly between the vm_page_alloc*()
5225471caf6SAlan Cox  * and the vm_page_grab*() functions.  See these functions for details.
5235471caf6SAlan Cox  *
52489fc8bdbSGleb Smirnoff  * Bits 0 - 1 define class.
52589fc8bdbSGleb Smirnoff  * Bits 2 - 15 dedicated for flags.
52689fc8bdbSGleb Smirnoff  * Legend:
52789fc8bdbSGleb Smirnoff  * (a) - vm_page_alloc() supports the flag.
52889fc8bdbSGleb Smirnoff  * (c) - vm_page_alloc_contig() supports the flag.
52989fc8bdbSGleb Smirnoff  * (g) - vm_page_grab() supports the flag.
530b498f71bSMark Johnston  * (n) - vm_page_alloc_noobj() and vm_page_alloc_freelist() support the flag.
5315471caf6SAlan Cox  * (p) - vm_page_grab_pages() supports the flag.
53289fc8bdbSGleb Smirnoff  * Bits above 15 define the count of additional pages that the caller
53389fc8bdbSGleb Smirnoff  * intends to allocate.
53489fc8bdbSGleb Smirnoff  */
5356d40c3d3SDavid Greenman #define VM_ALLOC_NORMAL		0
5366d40c3d3SDavid Greenman #define VM_ALLOC_INTERRUPT	1
5376d40c3d3SDavid Greenman #define VM_ALLOC_SYSTEM		2
538827b2fa0SAlan Cox #define	VM_ALLOC_CLASS_MASK	3
539b498f71bSMark Johnston #define	VM_ALLOC_WAITOK		0x0008	/* (acn) Sleep and retry */
540b498f71bSMark Johnston #define	VM_ALLOC_WAITFAIL	0x0010	/* (acn) Sleep and return error */
541b498f71bSMark Johnston #define	VM_ALLOC_WIRED		0x0020	/* (acgnp) Allocate a wired page */
542b498f71bSMark Johnston #define	VM_ALLOC_ZERO		0x0040	/* (acgnp) Allocate a zeroed page */
543660344caSRyan Stone #define	VM_ALLOC_NORECLAIM	0x0080	/* (c) Do not reclaim after failure */
544a9d6f1feSMark Johnston #define	VM_ALLOC_AVAIL0		0x0100
5455471caf6SAlan Cox #define	VM_ALLOC_NOBUSY		0x0200	/* (acgp) Do not excl busy the page */
546c7575748SJeff Roberson #define	VM_ALLOC_NOCREAT	0x0400	/* (gp) Don't create a page */
547a9d6f1feSMark Johnston #define	VM_ALLOC_AVAIL1		0x0800
5485471caf6SAlan Cox #define	VM_ALLOC_IGN_SBUSY	0x1000	/* (gp) Ignore shared busy flag */
54989fc8bdbSGleb Smirnoff #define	VM_ALLOC_NODUMP		0x2000	/* (ag) don't include in dump */
5505471caf6SAlan Cox #define	VM_ALLOC_SBUSY		0x4000	/* (acgp) Shared busy the page */
551b498f71bSMark Johnston #define	VM_ALLOC_NOWAIT		0x8000	/* (acgnp) Do not sleep */
552c40cf9bcSMark Johnston #define	VM_ALLOC_COUNT_MAX	0xffff
5535f195aa3SKonstantin Belousov #define	VM_ALLOC_COUNT_SHIFT	16
554c40cf9bcSMark Johnston #define	VM_ALLOC_COUNT_MASK	(VM_ALLOC_COUNT(VM_ALLOC_COUNT_MAX))
555c40cf9bcSMark Johnston #define	VM_ALLOC_COUNT(count)	({				\
556c40cf9bcSMark Johnston 	KASSERT((count) <= VM_ALLOC_COUNT_MAX,			\
557c40cf9bcSMark Johnston 	    ("%s: invalid VM_ALLOC_COUNT value", __func__));	\
558c40cf9bcSMark Johnston 	(count) << VM_ALLOC_COUNT_SHIFT;			\
559c40cf9bcSMark Johnston })
5600d94caffSDavid Greenman 
561b32ecf44SKonstantin Belousov #ifdef M_NOWAIT
562b32ecf44SKonstantin Belousov static inline int
563b32ecf44SKonstantin Belousov malloc2vm_flags(int malloc_flags)
564b32ecf44SKonstantin Belousov {
565b32ecf44SKonstantin Belousov 	int pflags;
566b32ecf44SKonstantin Belousov 
567962b064aSKonstantin Belousov 	KASSERT((malloc_flags & M_USE_RESERVE) == 0 ||
568962b064aSKonstantin Belousov 	    (malloc_flags & M_NOWAIT) != 0,
569962b064aSKonstantin Belousov 	    ("M_USE_RESERVE requires M_NOWAIT"));
570b32ecf44SKonstantin Belousov 	pflags = (malloc_flags & M_USE_RESERVE) != 0 ? VM_ALLOC_INTERRUPT :
571b32ecf44SKonstantin Belousov 	    VM_ALLOC_SYSTEM;
572b32ecf44SKonstantin Belousov 	if ((malloc_flags & M_ZERO) != 0)
573b32ecf44SKonstantin Belousov 		pflags |= VM_ALLOC_ZERO;
574b32ecf44SKonstantin Belousov 	if ((malloc_flags & M_NODUMP) != 0)
575b32ecf44SKonstantin Belousov 		pflags |= VM_ALLOC_NODUMP;
5768d6fbbb8SJeff Roberson 	if ((malloc_flags & M_NOWAIT))
5778d6fbbb8SJeff Roberson 		pflags |= VM_ALLOC_NOWAIT;
5788d6fbbb8SJeff Roberson 	if ((malloc_flags & M_WAITOK))
5798d6fbbb8SJeff Roberson 		pflags |= VM_ALLOC_WAITOK;
580660344caSRyan Stone 	if ((malloc_flags & M_NORECLAIM))
581660344caSRyan Stone 		pflags |= VM_ALLOC_NORECLAIM;
582b32ecf44SKonstantin Belousov 	return (pflags);
583b32ecf44SKonstantin Belousov }
584b32ecf44SKonstantin Belousov #endif
585b32ecf44SKonstantin Belousov 
58688302601SAlan Cox /*
58788302601SAlan Cox  * Predicates supported by vm_page_ps_test():
58888302601SAlan Cox  *
58988302601SAlan Cox  *	PS_ALL_DIRTY is true only if the entire (super)page is dirty.
59088302601SAlan Cox  *	However, it can be spuriously false when the (super)page has become
59188302601SAlan Cox  *	dirty in the pmap but that information has not been propagated to the
59288302601SAlan Cox  *	machine-independent layer.
59388302601SAlan Cox  */
59488302601SAlan Cox #define	PS_ALL_DIRTY	0x1
59588302601SAlan Cox #define	PS_ALL_VALID	0x2
59688302601SAlan Cox #define	PS_NONE_BUSY	0x4
59788302601SAlan Cox 
598fb1d575cSJeff Roberson bool vm_page_busy_acquire(vm_page_t m, int allocflags);
599c7aebda8SAttilio Rao void vm_page_busy_downgrade(vm_page_t m);
60063e97555SJeff Roberson int vm_page_busy_tryupgrade(vm_page_t m);
60187b64663SMark Johnston bool vm_page_busy_sleep(vm_page_t m, const char *msg, int allocflags);
602f212367bSJeff Roberson void vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m,
60387b64663SMark Johnston     vm_pindex_t pindex, const char *wmesg, int allocflags);
6041b40f8c0SMatthew Dillon void vm_page_free(vm_page_t m);
6051b40f8c0SMatthew Dillon void vm_page_free_zero(vm_page_t m);
6061b40f8c0SMatthew Dillon 
6071b40f8c0SMatthew Dillon void vm_page_activate (vm_page_t);
6082051980fSAlan Cox void vm_page_advise(vm_page_t m, int advice);
6091b40f8c0SMatthew Dillon vm_page_t vm_page_alloc(vm_object_t, vm_pindex_t, int);
610ef435ae7SJeff Roberson vm_page_t vm_page_alloc_domain(vm_object_t, vm_pindex_t, int, int);
61133fff5d5SMark Johnston vm_page_t vm_page_alloc_after(vm_object_t, vm_pindex_t, int, vm_page_t);
612ef435ae7SJeff Roberson vm_page_t vm_page_alloc_domain_after(vm_object_t, vm_pindex_t, int, int,
613ef435ae7SJeff Roberson     vm_page_t);
614fbd80bd0SAlan Cox vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
615fbd80bd0SAlan Cox     u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
616fbd80bd0SAlan Cox     vm_paddr_t boundary, vm_memattr_t memattr);
617ef435ae7SJeff Roberson vm_page_t vm_page_alloc_contig_domain(vm_object_t object,
618ef435ae7SJeff Roberson     vm_pindex_t pindex, int domain, int req, u_long npages, vm_paddr_t low,
619ef435ae7SJeff Roberson     vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
620ef435ae7SJeff Roberson     vm_memattr_t memattr);
621aa546366SJayachandran C. vm_page_t vm_page_alloc_freelist(int, int);
622ef435ae7SJeff Roberson vm_page_t vm_page_alloc_freelist_domain(int, int, int);
623b498f71bSMark Johnston vm_page_t vm_page_alloc_noobj(int);
624b498f71bSMark Johnston vm_page_t vm_page_alloc_noobj_domain(int, int);
62592db9f3bSMark Johnston vm_page_t vm_page_alloc_noobj_contig(int req, u_long npages, vm_paddr_t low,
62692db9f3bSMark Johnston     vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
62792db9f3bSMark Johnston     vm_memattr_t memattr);
62892db9f3bSMark Johnston vm_page_t vm_page_alloc_noobj_contig_domain(int domain, int req, u_long npages,
62992db9f3bSMark Johnston     vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
63092db9f3bSMark Johnston     vm_memattr_t memattr);
6317f935055SJeff Roberson void vm_page_bits_set(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t set);
632ffc568baSScott Long bool vm_page_blacklist_add(vm_paddr_t pa, bool verbose);
6331b40f8c0SMatthew Dillon vm_page_t vm_page_grab(vm_object_t, vm_pindex_t, int);
634c49be4f1SJeff Roberson vm_page_t vm_page_grab_unlocked(vm_object_t, vm_pindex_t, int);
6359df950b3SMark Johnston int vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
6365471caf6SAlan Cox     vm_page_t *ma, int count);
637c49be4f1SJeff Roberson int vm_page_grab_pages_unlocked(vm_object_t object, vm_pindex_t pindex,
638c49be4f1SJeff Roberson     int allocflags, vm_page_t *ma, int count);
639c7575748SJeff Roberson int vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex,
640c7575748SJeff Roberson     int allocflags);
641c49be4f1SJeff Roberson int vm_page_grab_valid_unlocked(vm_page_t *mp, vm_object_t object,
642c49be4f1SJeff Roberson     vm_pindex_t pindex, int allocflags);
6431b40f8c0SMatthew Dillon void vm_page_deactivate(vm_page_t);
6443138cd36SMark Johnston void vm_page_deactivate_noreuse(vm_page_t);
6458d220203SAlan Cox void vm_page_dequeue(vm_page_t m);
646e8bcf696SMark Johnston void vm_page_dequeue_deferred(vm_page_t m);
647b382c10aSKonstantin Belousov vm_page_t vm_page_find_least(vm_object_t, vm_pindex_t);
6484dfa06e1SChuck Silvers void vm_page_free_invalid(vm_page_t);
64910cf2560SAlan Cox vm_page_t vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr);
650e461aae7SKonstantin Belousov void vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr);
6510292c54bSConrad Meyer void vm_page_init_marker(vm_page_t marker, int queue, uint16_t aflags);
6525b10e79eSKonstantin Belousov void vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segind);
653e946b949SAttilio Rao int vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t);
6540012f373SJeff Roberson void vm_page_invalid(vm_page_t m);
655ebcddc72SAlan Cox void vm_page_launder(vm_page_t m);
6561b40f8c0SMatthew Dillon vm_page_t vm_page_lookup(vm_object_t, vm_pindex_t);
657c2c6fb90SBryan Drewery vm_page_t vm_page_lookup_unlocked(vm_object_t, vm_pindex_t);
65891b4f427SAlan Cox vm_page_t vm_page_next(vm_page_t m);
659386eba08SMark Johnston void vm_page_pqbatch_drain(void);
660386eba08SMark Johnston void vm_page_pqbatch_submit(vm_page_t m, uint8_t queue);
661f3f38e25SMark Johnston bool vm_page_pqstate_commit(vm_page_t m, vm_page_astate_t *old,
662f3f38e25SMark Johnston     vm_page_astate_t new);
66391b4f427SAlan Cox vm_page_t vm_page_prev(vm_page_t m);
66488302601SAlan Cox bool vm_page_ps_test(vm_page_t m, int flags, vm_page_t skip_m);
66510cf2560SAlan Cox void vm_page_putfake(vm_page_t m);
666b6c00483SKonstantin Belousov void vm_page_readahead_finish(vm_page_t m);
667c869e672SAlan Cox bool vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low,
668c869e672SAlan Cox     vm_paddr_t high, u_long alignment, vm_paddr_t boundary);
6693f289c3fSJeff Roberson bool vm_page_reclaim_contig_domain(int domain, int req, u_long npages,
670ef435ae7SJeff Roberson     vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary);
6713407fefeSKonstantin Belousov void vm_page_reference(vm_page_t m);
67298549e2dSMark Johnston #define	VPR_TRYFREE	0x01
67398549e2dSMark Johnston #define	VPR_NOREUSE	0x02
67498549e2dSMark Johnston void vm_page_release(vm_page_t m, int flags);
67598549e2dSMark Johnston void vm_page_release_locked(vm_page_t m, int flags);
6766be21eb7SJeff Roberson vm_page_t vm_page_relookup(vm_object_t, vm_pindex_t);
6770fd977b3SMark Johnston bool vm_page_remove(vm_page_t);
6783cf3b4e6SJeff Roberson bool vm_page_remove_xbusy(vm_page_t);
679e946b949SAttilio Rao int vm_page_rename(vm_page_t, vm_object_t, vm_pindex_t);
6803cf3b4e6SJeff Roberson void vm_page_replace(vm_page_t mnew, vm_object_t object,
6813cf3b4e6SJeff Roberson     vm_pindex_t pindex, vm_page_t mold);
682c7aebda8SAttilio Rao int vm_page_sbusied(vm_page_t m);
683c869e672SAlan Cox vm_page_t vm_page_scan_contig(u_long npages, vm_page_t m_start,
684c869e672SAlan Cox     vm_page_t m_end, u_long alignment, vm_paddr_t boundary, int options);
685a8081778SJeff Roberson vm_page_bits_t vm_page_set_dirty(vm_page_t m);
686dc874f98SKonstantin Belousov void vm_page_set_valid_range(vm_page_t m, int base, int size);
687889eb0fcSAlan Cox vm_offset_t vm_page_startup(vm_offset_t vaddr);
688c7aebda8SAttilio Rao void vm_page_sunbusy(vm_page_t m);
689fee2a2faSMark Johnston bool vm_page_try_remove_all(vm_page_t m);
690fee2a2faSMark Johnston bool vm_page_try_remove_write(vm_page_t m);
691c7aebda8SAttilio Rao int vm_page_trysbusy(vm_page_t m);
692205be21dSJeff Roberson int vm_page_tryxbusy(vm_page_t m);
6938c22654dSAlan Cox void vm_page_unhold_pages(vm_page_t *ma, int count);
694b1fd102eSMark Johnston void vm_page_unswappable(vm_page_t m);
695fee2a2faSMark Johnston void vm_page_unwire(vm_page_t m, uint8_t queue);
6961d3a1bcfSMark Johnston bool vm_page_unwire_noq(vm_page_t m);
69710cf2560SAlan Cox void vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr);
6981b40f8c0SMatthew Dillon void vm_page_wire(vm_page_t);
699fee2a2faSMark Johnston bool vm_page_wire_mapped(vm_page_t m);
700c7aebda8SAttilio Rao void vm_page_xunbusy_hard(vm_page_t m);
701b631c36fSKonstantin Belousov void vm_page_xunbusy_hard_unchecked(vm_page_t m);
7021b40f8c0SMatthew Dillon void vm_page_set_validclean (vm_page_t, int, int);
7031b40f8c0SMatthew Dillon void vm_page_clear_dirty(vm_page_t, int, int);
7041b40f8c0SMatthew Dillon void vm_page_set_invalid(vm_page_t, int, int);
7050012f373SJeff Roberson void vm_page_valid(vm_page_t m);
7061b40f8c0SMatthew Dillon int vm_page_is_valid(vm_page_t, int, int);
7071b40f8c0SMatthew Dillon void vm_page_test_dirty(vm_page_t);
708561cc9fcSKonstantin Belousov vm_page_bits_t vm_page_bits(int base, int size);
7098d17e694SJulian Elischer void vm_page_zero_invalid(vm_page_t m, boolean_t setvalid);
7108c8ee2eeSKonstantin Belousov void vm_page_free_pages_toq(struct spglist *free, bool update_wire_count);
71198cb733cSKenneth D. Merry 
712eddc9291SAlan Cox void vm_page_dirty_KBI(vm_page_t m);
713cf1911a9SKonstantin Belousov void vm_page_lock_KBI(vm_page_t m, const char *file, int line);
714cf1911a9SKonstantin Belousov void vm_page_unlock_KBI(vm_page_t m, const char *file, int line);
715cf1911a9SKonstantin Belousov int vm_page_trylock_KBI(vm_page_t m, const char *file, int line);
716cf1911a9SKonstantin Belousov #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
717b4171812SAlan Cox void vm_page_assert_locked_KBI(vm_page_t m, const char *file, int line);
718cf1911a9SKonstantin Belousov void vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line);
719cf1911a9SKonstantin Belousov #endif
720cf1911a9SKonstantin Belousov 
721958d8f52SMark Johnston #define	vm_page_busy_fetch(m)	atomic_load_int(&(m)->busy_lock)
722958d8f52SMark Johnston 
7230012f373SJeff Roberson #define	vm_page_assert_busied(m)					\
7240012f373SJeff Roberson 	KASSERT(vm_page_busied(m),					\
7250012f373SJeff Roberson 	    ("vm_page_assert_busied: page %p not busy @ %s:%d", \
7260012f373SJeff Roberson 	    (m), __FILE__, __LINE__))
7270012f373SJeff Roberson 
728c7aebda8SAttilio Rao #define	vm_page_assert_sbusied(m)					\
729c7aebda8SAttilio Rao 	KASSERT(vm_page_sbusied(m),					\
730c7aebda8SAttilio Rao 	    ("vm_page_assert_sbusied: page %p not shared busy @ %s:%d", \
731dc62d559SConrad Meyer 	    (m), __FILE__, __LINE__))
732c7aebda8SAttilio Rao 
733c7aebda8SAttilio Rao #define	vm_page_assert_unbusied(m)					\
734958d8f52SMark Johnston 	KASSERT((vm_page_busy_fetch(m) & ~VPB_BIT_WAITERS) !=		\
735ee9e43f8SJeff Roberson 	    VPB_CURTHREAD_EXCLUSIVE,					\
736ee9e43f8SJeff Roberson 	    ("vm_page_assert_xbusied: page %p busy_lock %#x owned"	\
737ee9e43f8SJeff Roberson             " by me @ %s:%d",						\
738ee9e43f8SJeff Roberson 	    (m), (m)->busy_lock, __FILE__, __LINE__));			\
739c7aebda8SAttilio Rao 
740b631c36fSKonstantin Belousov #define	vm_page_assert_xbusied_unchecked(m) do {			\
741c7aebda8SAttilio Rao 	KASSERT(vm_page_xbusied(m),					\
742c7aebda8SAttilio Rao 	    ("vm_page_assert_xbusied: page %p not exclusive busy @ %s:%d", \
743b631c36fSKonstantin Belousov 	    (m), __FILE__, __LINE__));					\
744b631c36fSKonstantin Belousov } while (0)
745b631c36fSKonstantin Belousov #define	vm_page_assert_xbusied(m) do {					\
746b631c36fSKonstantin Belousov 	vm_page_assert_xbusied_unchecked(m);				\
747958d8f52SMark Johnston 	KASSERT((vm_page_busy_fetch(m) & ~VPB_BIT_WAITERS) ==		\
748b631c36fSKonstantin Belousov 	    VPB_CURTHREAD_EXCLUSIVE,					\
749b631c36fSKonstantin Belousov 	    ("vm_page_assert_xbusied: page %p busy_lock %#x not owned"	\
750b631c36fSKonstantin Belousov             " by me @ %s:%d",						\
751b631c36fSKonstantin Belousov 	    (m), (m)->busy_lock, __FILE__, __LINE__));			\
752b631c36fSKonstantin Belousov } while (0)
753c7aebda8SAttilio Rao 
754c7aebda8SAttilio Rao #define	vm_page_busied(m)						\
755958d8f52SMark Johnston 	(vm_page_busy_fetch(m) != VPB_UNBUSIED)
756c7aebda8SAttilio Rao 
757c7aebda8SAttilio Rao #define	vm_page_xbusied(m)						\
758958d8f52SMark Johnston 	((vm_page_busy_fetch(m) & VPB_SINGLE_EXCLUSIVE) != 0)
759c7aebda8SAttilio Rao 
760ee9e43f8SJeff Roberson #define	vm_page_busy_freed(m)						\
761958d8f52SMark Johnston 	(vm_page_busy_fetch(m) == VPB_FREED)
762ee9e43f8SJeff Roberson 
763505cd5d1SKonstantin Belousov /* Note: page m's lock must not be owned by the caller. */
764c7aebda8SAttilio Rao #define	vm_page_xunbusy(m) do {						\
765c7aebda8SAttilio Rao 	if (!atomic_cmpset_rel_int(&(m)->busy_lock,			\
766b631c36fSKonstantin Belousov 	    VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED))			\
767c7aebda8SAttilio Rao 		vm_page_xunbusy_hard(m);				\
768c7aebda8SAttilio Rao } while (0)
769b631c36fSKonstantin Belousov #define	vm_page_xunbusy_unchecked(m) do {				\
770b631c36fSKonstantin Belousov 	if (!atomic_cmpset_rel_int(&(m)->busy_lock,			\
771b631c36fSKonstantin Belousov 	    VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED))			\
772b631c36fSKonstantin Belousov 		vm_page_xunbusy_hard_unchecked(m);			\
773b631c36fSKonstantin Belousov } while (0)
774c7aebda8SAttilio Rao 
7753b1025d2SKonstantin Belousov #ifdef INVARIANTS
776205be21dSJeff Roberson void vm_page_object_busy_assert(vm_page_t m);
777205be21dSJeff Roberson #define	VM_PAGE_OBJECT_BUSY_ASSERT(m)	vm_page_object_busy_assert(m)
7783a2ba997SMark Johnston void vm_page_assert_pga_writeable(vm_page_t m, uint16_t bits);
779afb69e6bSKonstantin Belousov #define	VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits)				\
780afb69e6bSKonstantin Belousov 	vm_page_assert_pga_writeable(m, bits)
781958d8f52SMark Johnston /*
782958d8f52SMark Johnston  * Claim ownership of a page's xbusy state.  In non-INVARIANTS kernels this
783958d8f52SMark Johnston  * operation is a no-op since ownership is not tracked.  In particular
784958d8f52SMark Johnston  * this macro does not provide any synchronization with the previous owner.
785958d8f52SMark Johnston  */
786e9ceb9ddSJeff Roberson #define	vm_page_xbusy_claim(m) do {					\
787f72e5be5SMark Johnston 	u_int _busy_lock;						\
788f72e5be5SMark Johnston 									\
789e9ceb9ddSJeff Roberson 	vm_page_assert_xbusied_unchecked((m));				\
790f72e5be5SMark Johnston 	do {								\
791958d8f52SMark Johnston 		_busy_lock = vm_page_busy_fetch(m);			\
792f72e5be5SMark Johnston 	} while (!atomic_cmpset_int(&(m)->busy_lock, _busy_lock,	\
793f72e5be5SMark Johnston 	    (_busy_lock & VPB_BIT_FLAGMASK) | VPB_CURTHREAD_EXCLUSIVE)); \
794e9ceb9ddSJeff Roberson } while (0)
7953b1025d2SKonstantin Belousov #else
796205be21dSJeff Roberson #define	VM_PAGE_OBJECT_BUSY_ASSERT(m)	(void)0
797afb69e6bSKonstantin Belousov #define	VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits)	(void)0
798e9ceb9ddSJeff Roberson #define	vm_page_xbusy_claim(m)
7993b1025d2SKonstantin Belousov #endif
8003b1025d2SKonstantin Belousov 
801c2f22e97SMark Johnston #if BYTE_ORDER == BIG_ENDIAN
802c2f22e97SMark Johnston #define	VM_PAGE_AFLAG_SHIFT	16
803c2f22e97SMark Johnston #else
804c2f22e97SMark Johnston #define	VM_PAGE_AFLAG_SHIFT	0
805c2f22e97SMark Johnston #endif
8067cdeaf33SMark Johnston 
80741fd4b94SMark Johnston /*
8086fbaf685SMark Johnston  *	Load a snapshot of a page's 32-bit atomic state.
80941fd4b94SMark Johnston  */
8106fbaf685SMark Johnston static inline vm_page_astate_t
8116fbaf685SMark Johnston vm_page_astate_load(vm_page_t m)
8126fbaf685SMark Johnston {
8136fbaf685SMark Johnston 	vm_page_astate_t a;
81441fd4b94SMark Johnston 
815cbc080b4SMark Johnston 	a._bits = atomic_load_32(&m->a._bits);
8166fbaf685SMark Johnston 	return (a);
8176fbaf685SMark Johnston }
8186fbaf685SMark Johnston 
8196fbaf685SMark Johnston /*
8206fbaf685SMark Johnston  *	Atomically compare and set a page's atomic state.
8216fbaf685SMark Johnston  */
8226fbaf685SMark Johnston static inline bool
8236fbaf685SMark Johnston vm_page_astate_fcmpset(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new)
8246fbaf685SMark Johnston {
8256fbaf685SMark Johnston 
8266fbaf685SMark Johnston 	KASSERT(new.queue == PQ_INACTIVE || (new.flags & PGA_REQUEUE_HEAD) == 0,
8276fbaf685SMark Johnston 	    ("%s: invalid head requeue request for page %p", __func__, m));
8286fbaf685SMark Johnston 	KASSERT((new.flags & PGA_ENQUEUED) == 0 || new.queue != PQ_NONE,
8296fbaf685SMark Johnston 	    ("%s: setting PGA_ENQUEUED with PQ_NONE in page %p", __func__, m));
8306fbaf685SMark Johnston 	KASSERT(new._bits != old->_bits,
8316fbaf685SMark Johnston 	    ("%s: bits are unchanged", __func__));
8326fbaf685SMark Johnston 
8336fbaf685SMark Johnston 	return (atomic_fcmpset_32(&m->a._bits, &old->_bits, new._bits) != 0);
8346fbaf685SMark Johnston }
835369763e3SAlan Cox 
836369763e3SAlan Cox /*
837369763e3SAlan Cox  *	Clear the given bits in the specified page.
838369763e3SAlan Cox  */
839369763e3SAlan Cox static inline void
8403a2ba997SMark Johnston vm_page_aflag_clear(vm_page_t m, uint16_t bits)
841369763e3SAlan Cox {
842369763e3SAlan Cox 	uint32_t *addr, val;
843369763e3SAlan Cox 
844369763e3SAlan Cox 	/*
845369763e3SAlan Cox 	 * Access the whole 32-bit word containing the aflags field with an
846369763e3SAlan Cox 	 * atomic update.  Parallel non-atomic updates to the other fields
847369763e3SAlan Cox 	 * within this word are handled properly by the atomic update.
848369763e3SAlan Cox 	 */
8496fbaf685SMark Johnston 	addr = (void *)&m->a;
8507cdeaf33SMark Johnston 	val = bits << VM_PAGE_AFLAG_SHIFT;
851369763e3SAlan Cox 	atomic_clear_32(addr, val);
852369763e3SAlan Cox }
853369763e3SAlan Cox 
854369763e3SAlan Cox /*
855369763e3SAlan Cox  *	Set the given bits in the specified page.
856369763e3SAlan Cox  */
857369763e3SAlan Cox static inline void
8583a2ba997SMark Johnston vm_page_aflag_set(vm_page_t m, uint16_t bits)
859369763e3SAlan Cox {
860369763e3SAlan Cox 	uint32_t *addr, val;
861369763e3SAlan Cox 
862afb69e6bSKonstantin Belousov 	VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits);
863369763e3SAlan Cox 
864369763e3SAlan Cox 	/*
865369763e3SAlan Cox 	 * Access the whole 32-bit word containing the aflags field with an
866369763e3SAlan Cox 	 * atomic update.  Parallel non-atomic updates to the other fields
867369763e3SAlan Cox 	 * within this word are handled properly by the atomic update.
868369763e3SAlan Cox 	 */
8696fbaf685SMark Johnston 	addr = (void *)&m->a;
8707cdeaf33SMark Johnston 	val = bits << VM_PAGE_AFLAG_SHIFT;
871369763e3SAlan Cox 	atomic_set_32(addr, val);
872369763e3SAlan Cox }
873369763e3SAlan Cox 
874e8bcf696SMark Johnston /*
875eddc9291SAlan Cox  *	vm_page_dirty:
876eddc9291SAlan Cox  *
877eddc9291SAlan Cox  *	Set all bits in the page's dirty field.
878eddc9291SAlan Cox  *
879eddc9291SAlan Cox  *	The object containing the specified page must be locked if the
880eddc9291SAlan Cox  *	call is made from the machine-independent layer.
881eddc9291SAlan Cox  *
882eddc9291SAlan Cox  *	See vm_page_clear_dirty_mask().
883eddc9291SAlan Cox  */
884eddc9291SAlan Cox static __inline void
885eddc9291SAlan Cox vm_page_dirty(vm_page_t m)
886eddc9291SAlan Cox {
887eddc9291SAlan Cox 
888eddc9291SAlan Cox 	/* Use vm_page_dirty_KBI() under INVARIANTS to save memory. */
889f4b36404SMatt Macy #if (defined(KLD_MODULE) && !defined(KLD_TIED)) || defined(INVARIANTS)
890eddc9291SAlan Cox 	vm_page_dirty_KBI(m);
891eddc9291SAlan Cox #else
892eddc9291SAlan Cox 	m->dirty = VM_PAGE_BITS_ALL;
893eddc9291SAlan Cox #endif
894eddc9291SAlan Cox }
895eddc9291SAlan Cox 
896eddc9291SAlan Cox /*
897f3b676f0SAlan Cox  *	vm_page_undirty:
898f3b676f0SAlan Cox  *
899f3b676f0SAlan Cox  *	Set page to not be dirty.  Note: does not clear pmap modify bits
900f3b676f0SAlan Cox  */
901f3b676f0SAlan Cox static __inline void
902f3b676f0SAlan Cox vm_page_undirty(vm_page_t m)
903f3b676f0SAlan Cox {
9043b1025d2SKonstantin Belousov 
905205be21dSJeff Roberson 	VM_PAGE_OBJECT_BUSY_ASSERT(m);
906f3b676f0SAlan Cox 	m->dirty = 0;
907f3b676f0SAlan Cox }
908f3b676f0SAlan Cox 
909f3f38e25SMark Johnston static inline uint8_t
910f3f38e25SMark Johnston _vm_page_queue(vm_page_astate_t as)
911f3f38e25SMark Johnston {
912f3f38e25SMark Johnston 
913f3f38e25SMark Johnston 	if ((as.flags & PGA_DEQUEUE) != 0)
914f3f38e25SMark Johnston 		return (PQ_NONE);
915f3f38e25SMark Johnston 	return (as.queue);
916f3f38e25SMark Johnston }
917f3f38e25SMark Johnston 
9181b5c869dSMark Johnston /*
9191b5c869dSMark Johnston  *	vm_page_queue:
9201b5c869dSMark Johnston  *
921f3f38e25SMark Johnston  *	Return the index of the queue containing m.
9221b5c869dSMark Johnston  */
9231b5c869dSMark Johnston static inline uint8_t
9241b5c869dSMark Johnston vm_page_queue(vm_page_t m)
9251b5c869dSMark Johnston {
9261b5c869dSMark Johnston 
927f3f38e25SMark Johnston 	return (_vm_page_queue(vm_page_astate_load(m)));
9281b5c869dSMark Johnston }
9291b5c869dSMark Johnston 
930ebcddc72SAlan Cox static inline bool
931ebcddc72SAlan Cox vm_page_active(vm_page_t m)
932ebcddc72SAlan Cox {
933ebcddc72SAlan Cox 
9341b5c869dSMark Johnston 	return (vm_page_queue(m) == PQ_ACTIVE);
935ebcddc72SAlan Cox }
936ebcddc72SAlan Cox 
937ebcddc72SAlan Cox static inline bool
938ebcddc72SAlan Cox vm_page_inactive(vm_page_t m)
939ebcddc72SAlan Cox {
940ebcddc72SAlan Cox 
9411b5c869dSMark Johnston 	return (vm_page_queue(m) == PQ_INACTIVE);
942ebcddc72SAlan Cox }
943ebcddc72SAlan Cox 
944ebcddc72SAlan Cox static inline bool
945ebcddc72SAlan Cox vm_page_in_laundry(vm_page_t m)
946ebcddc72SAlan Cox {
9471b5c869dSMark Johnston 	uint8_t queue;
948ebcddc72SAlan Cox 
9491b5c869dSMark Johnston 	queue = vm_page_queue(m);
9501b5c869dSMark Johnston 	return (queue == PQ_LAUNDRY || queue == PQ_UNSWAPPABLE);
9515cd29d0fSMark Johnston }
9525cd29d0fSMark Johnston 
9535cd29d0fSMark Johnston /*
954fee2a2faSMark Johnston  *	vm_page_drop:
955fee2a2faSMark Johnston  *
956fee2a2faSMark Johnston  *	Release a reference to a page and return the old reference count.
957fee2a2faSMark Johnston  */
958fee2a2faSMark Johnston static inline u_int
959fee2a2faSMark Johnston vm_page_drop(vm_page_t m, u_int val)
960fee2a2faSMark Johnston {
96138547d59SMark Johnston 	u_int old;
962fee2a2faSMark Johnston 
963fee2a2faSMark Johnston 	/*
964fee2a2faSMark Johnston 	 * Synchronize with vm_page_free_prep(): ensure that all updates to the
965fee2a2faSMark Johnston 	 * page structure are visible before it is freed.
966fee2a2faSMark Johnston 	 */
967fee2a2faSMark Johnston 	atomic_thread_fence_rel();
96838547d59SMark Johnston 	old = atomic_fetchadd_int(&m->ref_count, -val);
96938547d59SMark Johnston 	KASSERT(old != VPRC_BLOCKED,
97038547d59SMark Johnston 	    ("vm_page_drop: page %p has an invalid refcount value", m));
97138547d59SMark Johnston 	return (old);
972fee2a2faSMark Johnston }
973fee2a2faSMark Johnston 
974fee2a2faSMark Johnston /*
975eeacb3b0SMark Johnston  *	vm_page_wired:
9761d3a1bcfSMark Johnston  *
977fee2a2faSMark Johnston  *	Perform a racy check to determine whether a reference prevents the page
978fee2a2faSMark Johnston  *	from being reclaimable.  If the page's object is locked, and the page is
979a9ea09e5SMark Johnston  *	unmapped and exclusively busied by the current thread, no new wirings
980a9ea09e5SMark Johnston  *	may be created.
9811d3a1bcfSMark Johnston  */
9821d3a1bcfSMark Johnston static inline bool
983d842aa51SMark Johnston vm_page_wired(vm_page_t m)
984d842aa51SMark Johnston {
985d842aa51SMark Johnston 
986fee2a2faSMark Johnston 	return (VPRC_WIRE_COUNT(m->ref_count) > 0);
987d842aa51SMark Johnston }
988d842aa51SMark Johnston 
9890012f373SJeff Roberson static inline bool
9900012f373SJeff Roberson vm_page_all_valid(vm_page_t m)
9910012f373SJeff Roberson {
9920012f373SJeff Roberson 
9930012f373SJeff Roberson 	return (m->valid == VM_PAGE_BITS_ALL);
9940012f373SJeff Roberson }
9950012f373SJeff Roberson 
9960012f373SJeff Roberson static inline bool
997*934bfc12SKonstantin Belousov vm_page_any_valid(vm_page_t m)
998*934bfc12SKonstantin Belousov {
999*934bfc12SKonstantin Belousov 
1000*934bfc12SKonstantin Belousov 	return (m->valid != 0);
1001*934bfc12SKonstantin Belousov }
1002*934bfc12SKonstantin Belousov 
1003*934bfc12SKonstantin Belousov static inline bool
10040012f373SJeff Roberson vm_page_none_valid(vm_page_t m)
10050012f373SJeff Roberson {
10060012f373SJeff Roberson 
10070012f373SJeff Roberson 	return (m->valid == 0);
10080012f373SJeff Roberson }
10090012f373SJeff Roberson 
1010431fb8abSMark Johnston static inline int
1011431fb8abSMark Johnston vm_page_domain(vm_page_t m)
1012431fb8abSMark Johnston {
1013431fb8abSMark Johnston #ifdef NUMA
1014431fb8abSMark Johnston 	int domn, segind;
1015431fb8abSMark Johnston 
1016431fb8abSMark Johnston 	segind = m->segind;
1017431fb8abSMark Johnston 	KASSERT(segind < vm_phys_nsegs, ("segind %d m %p", segind, m));
1018431fb8abSMark Johnston 	domn = vm_phys_segs[segind].domain;
1019431fb8abSMark Johnston 	KASSERT(domn >= 0 && domn < vm_ndomains, ("domain %d m %p", domn, m));
1020431fb8abSMark Johnston 	return (domn);
1021431fb8abSMark Johnston #else
1022431fb8abSMark Johnston 	return (0);
1023431fb8abSMark Johnston #endif
1024431fb8abSMark Johnston }
1025431fb8abSMark Johnston 
1026c4473420SPeter Wemm #endif				/* _KERNEL */
1027df8bae1dSRodney W. Grimes #endif				/* !_VM_PAGE_ */
1028