xref: /freebsd/sys/vm/vm_glue.c (revision 89f6b8632cc94bca2738b4fcc26e1189ef4f5dde)
160727d8bSWarner Losh /*-
2df8bae1dSRodney W. Grimes  * Copyright (c) 1991, 1993
3df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
4df8bae1dSRodney W. Grimes  *
5df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
6df8bae1dSRodney W. Grimes  * The Mach Operating System project at Carnegie-Mellon University.
7df8bae1dSRodney W. Grimes  *
8df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
9df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
10df8bae1dSRodney W. Grimes  * are met:
11df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
12df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
13df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
14df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
15df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
16df8bae1dSRodney W. Grimes  * 4. Neither the name of the University nor the names of its contributors
17df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
18df8bae1dSRodney W. Grimes  *    without specific prior written permission.
19df8bae1dSRodney W. Grimes  *
20df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
31df8bae1dSRodney W. Grimes  *
323c4dd356SDavid Greenman  *	from: @(#)vm_glue.c	8.6 (Berkeley) 1/5/94
33df8bae1dSRodney W. Grimes  *
34df8bae1dSRodney W. Grimes  *
35df8bae1dSRodney W. Grimes  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
36df8bae1dSRodney W. Grimes  * All rights reserved.
37df8bae1dSRodney W. Grimes  *
38df8bae1dSRodney W. Grimes  * Permission to use, copy, modify and distribute this software and
39df8bae1dSRodney W. Grimes  * its documentation is hereby granted, provided that both the copyright
40df8bae1dSRodney W. Grimes  * notice and this permission notice appear in all copies of the
41df8bae1dSRodney W. Grimes  * software, derivative works or modified versions, and any portions
42df8bae1dSRodney W. Grimes  * thereof, and that both notices appear in supporting documentation.
43df8bae1dSRodney W. Grimes  *
44df8bae1dSRodney W. Grimes  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
45df8bae1dSRodney W. Grimes  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
46df8bae1dSRodney W. Grimes  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
47df8bae1dSRodney W. Grimes  *
48df8bae1dSRodney W. Grimes  * Carnegie Mellon requests users of this software to return to
49df8bae1dSRodney W. Grimes  *
50df8bae1dSRodney W. Grimes  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
51df8bae1dSRodney W. Grimes  *  School of Computer Science
52df8bae1dSRodney W. Grimes  *  Carnegie Mellon University
53df8bae1dSRodney W. Grimes  *  Pittsburgh PA 15213-3890
54df8bae1dSRodney W. Grimes  *
55df8bae1dSRodney W. Grimes  * any improvements or extensions that they make and grant Carnegie the
56df8bae1dSRodney W. Grimes  * rights to redistribute these changes.
57df8bae1dSRodney W. Grimes  */
58df8bae1dSRodney W. Grimes 
59874651b1SDavid E. O'Brien #include <sys/cdefs.h>
60874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$");
61874651b1SDavid E. O'Brien 
62faa5f8d8SAndrzej Bialecki #include "opt_vm.h"
6315a7ad60SPeter Wemm #include "opt_kstack_pages.h"
6415a7ad60SPeter Wemm #include "opt_kstack_max_pages.h"
65e9822d92SJoerg Wunsch 
66df8bae1dSRodney W. Grimes #include <sys/param.h>
67df8bae1dSRodney W. Grimes #include <sys/systm.h>
68104a9b7eSAlexander Kabaev #include <sys/limits.h>
69fb919e4dSMark Murray #include <sys/lock.h>
70fb919e4dSMark Murray #include <sys/mutex.h>
71df8bae1dSRodney W. Grimes #include <sys/proc.h>
721ba5ad42SEdward Tomasz Napierala #include <sys/racct.h>
73df8bae1dSRodney W. Grimes #include <sys/resourcevar.h>
74*89f6b863SAttilio Rao #include <sys/rwlock.h>
75da61b9a6SAlan Cox #include <sys/sched.h>
76da61b9a6SAlan Cox #include <sys/sf_buf.h>
773aa12267SBruce Evans #include <sys/shm.h>
78efeaf95aSDavid Greenman #include <sys/vmmeter.h>
791005a129SJohn Baldwin #include <sys/sx.h>
80ceb0cf87SJohn Dyson #include <sys/sysctl.h>
81e878d997SKonstantin Belousov #include <sys/_kstack_cache.h>
828a945d10SKonstantin Belousov #include <sys/eventhandler.h>
8326f9a767SRodney W. Grimes #include <sys/kernel.h>
840384fff8SJason Evans #include <sys/ktr.h>
85a2a1c95cSPeter Wemm #include <sys/unistd.h>
8626f9a767SRodney W. Grimes 
87df8bae1dSRodney W. Grimes #include <vm/vm.h>
88efeaf95aSDavid Greenman #include <vm/vm_param.h>
89efeaf95aSDavid Greenman #include <vm/pmap.h>
90efeaf95aSDavid Greenman #include <vm/vm_map.h>
91df8bae1dSRodney W. Grimes #include <vm/vm_page.h>
9226f9a767SRodney W. Grimes #include <vm/vm_pageout.h>
93a136efe9SPeter Wemm #include <vm/vm_object.h>
94df8bae1dSRodney W. Grimes #include <vm/vm_kern.h>
95efeaf95aSDavid Greenman #include <vm/vm_extern.h>
96a136efe9SPeter Wemm #include <vm/vm_pager.h>
9792da00bbSMatthew Dillon #include <vm/swap_pager.h>
98efeaf95aSDavid Greenman 
992b14f991SJulian Elischer /*
1002b14f991SJulian Elischer  * System initialization
1012b14f991SJulian Elischer  *
1022b14f991SJulian Elischer  * THIS MUST BE THE LAST INITIALIZATION ITEM!!!
1032b14f991SJulian Elischer  *
1042b14f991SJulian Elischer  * Note: run scheduling should be divorced from the vm system.
1052b14f991SJulian Elischer  */
10611caded3SAlfred Perlstein static void scheduler(void *);
107237fdd78SRobert Watson SYSINIT(scheduler, SI_SUB_RUN_SCHEDULER, SI_ORDER_ANY, scheduler, NULL);
1082b14f991SJulian Elischer 
109e50f5c2eSBruce Evans #ifndef NO_SWAPPING
110b61ce5b0SJeff Roberson static int swapout(struct proc *);
111b61ce5b0SJeff Roberson static void swapclear(struct proc *);
112ac45ee97SAlan Cox static void vm_thread_swapin(struct thread *td);
113ac45ee97SAlan Cox static void vm_thread_swapout(struct thread *td);
114e50f5c2eSBruce Evans #endif
115f708ef1bSPoul-Henning Kamp 
11643a90f3aSAlan Cox /*
11743a90f3aSAlan Cox  * MPSAFE
1182d5c7e45SMatthew Dillon  *
1192d5c7e45SMatthew Dillon  * WARNING!  This code calls vm_map_check_protection() which only checks
1202d5c7e45SMatthew Dillon  * the associated vm_map_entry range.  It does not determine whether the
1212d5c7e45SMatthew Dillon  * contents of the memory is actually readable or writable.  In most cases
1222d5c7e45SMatthew Dillon  * just checking the vm_map_entry is sufficient within the kernel's address
1232d5c7e45SMatthew Dillon  * space.
12443a90f3aSAlan Cox  */
125df8bae1dSRodney W. Grimes int
126df8bae1dSRodney W. Grimes kernacc(addr, len, rw)
127c3dfdfd1SAlfred Perlstein 	void *addr;
128df8bae1dSRodney W. Grimes 	int len, rw;
129df8bae1dSRodney W. Grimes {
130df8bae1dSRodney W. Grimes 	boolean_t rv;
131df8bae1dSRodney W. Grimes 	vm_offset_t saddr, eaddr;
13202c58685SPoul-Henning Kamp 	vm_prot_t prot;
133df8bae1dSRodney W. Grimes 
134e50f5c2eSBruce Evans 	KASSERT((rw & ~VM_PROT_ALL) == 0,
13502c58685SPoul-Henning Kamp 	    ("illegal ``rw'' argument to kernacc (%x)\n", rw));
13675337a56SAlan Cox 
13775337a56SAlan Cox 	if ((vm_offset_t)addr + len > kernel_map->max_offset ||
13875337a56SAlan Cox 	    (vm_offset_t)addr + len < (vm_offset_t)addr)
13975337a56SAlan Cox 		return (FALSE);
14075337a56SAlan Cox 
14102c58685SPoul-Henning Kamp 	prot = rw;
1426cde7a16SDavid Greenman 	saddr = trunc_page((vm_offset_t)addr);
1436cde7a16SDavid Greenman 	eaddr = round_page((vm_offset_t)addr + len);
144d8834602SAlan Cox 	vm_map_lock_read(kernel_map);
145df8bae1dSRodney W. Grimes 	rv = vm_map_check_protection(kernel_map, saddr, eaddr, prot);
146d8834602SAlan Cox 	vm_map_unlock_read(kernel_map);
147df8bae1dSRodney W. Grimes 	return (rv == TRUE);
148df8bae1dSRodney W. Grimes }
149df8bae1dSRodney W. Grimes 
15043a90f3aSAlan Cox /*
15143a90f3aSAlan Cox  * MPSAFE
1522d5c7e45SMatthew Dillon  *
1532d5c7e45SMatthew Dillon  * WARNING!  This code calls vm_map_check_protection() which only checks
1542d5c7e45SMatthew Dillon  * the associated vm_map_entry range.  It does not determine whether the
1552d5c7e45SMatthew Dillon  * contents of the memory is actually readable or writable.  vmapbuf(),
1562d5c7e45SMatthew Dillon  * vm_fault_quick(), or copyin()/copout()/su*()/fu*() functions should be
1572d5c7e45SMatthew Dillon  * used in conjuction with this call.
15843a90f3aSAlan Cox  */
159df8bae1dSRodney W. Grimes int
160df8bae1dSRodney W. Grimes useracc(addr, len, rw)
161c3dfdfd1SAlfred Perlstein 	void *addr;
162df8bae1dSRodney W. Grimes 	int len, rw;
163df8bae1dSRodney W. Grimes {
164df8bae1dSRodney W. Grimes 	boolean_t rv;
16502c58685SPoul-Henning Kamp 	vm_prot_t prot;
16605ba50f5SJake Burkholder 	vm_map_t map;
167df8bae1dSRodney W. Grimes 
168e50f5c2eSBruce Evans 	KASSERT((rw & ~VM_PROT_ALL) == 0,
16902c58685SPoul-Henning Kamp 	    ("illegal ``rw'' argument to useracc (%x)\n", rw));
17002c58685SPoul-Henning Kamp 	prot = rw;
17105ba50f5SJake Burkholder 	map = &curproc->p_vmspace->vm_map;
17205ba50f5SJake Burkholder 	if ((vm_offset_t)addr + len > vm_map_max(map) ||
17305ba50f5SJake Burkholder 	    (vm_offset_t)addr + len < (vm_offset_t)addr) {
17426f9a767SRodney W. Grimes 		return (FALSE);
17526f9a767SRodney W. Grimes 	}
176d8834602SAlan Cox 	vm_map_lock_read(map);
17705ba50f5SJake Burkholder 	rv = vm_map_check_protection(map, trunc_page((vm_offset_t)addr),
17805ba50f5SJake Burkholder 	    round_page((vm_offset_t)addr + len), prot);
179d8834602SAlan Cox 	vm_map_unlock_read(map);
180df8bae1dSRodney W. Grimes 	return (rv == TRUE);
181df8bae1dSRodney W. Grimes }
182df8bae1dSRodney W. Grimes 
18316929939SDon Lewis int
184f0ea4612SDon Lewis vslock(void *addr, size_t len)
18516929939SDon Lewis {
186bb734798SDon Lewis 	vm_offset_t end, last, start;
187bb734798SDon Lewis 	vm_size_t npages;
188bb734798SDon Lewis 	int error;
18916929939SDon Lewis 
190bb734798SDon Lewis 	last = (vm_offset_t)addr + len;
191ce8660e3SDon Lewis 	start = trunc_page((vm_offset_t)addr);
192bb734798SDon Lewis 	end = round_page(last);
193bb734798SDon Lewis 	if (last < (vm_offset_t)addr || end < (vm_offset_t)addr)
19416929939SDon Lewis 		return (EINVAL);
19516929939SDon Lewis 	npages = atop(end - start);
19616929939SDon Lewis 	if (npages > vm_page_max_wired)
19716929939SDon Lewis 		return (ENOMEM);
19816929939SDon Lewis #if 0
19916929939SDon Lewis 	/*
20016929939SDon Lewis 	 * XXX - not yet
20116929939SDon Lewis 	 *
20216929939SDon Lewis 	 * The limit for transient usage of wired pages should be
20316929939SDon Lewis 	 * larger than for "permanent" wired pages (mlock()).
20416929939SDon Lewis 	 *
20516929939SDon Lewis 	 * Also, the sysctl code, which is the only present user
20616929939SDon Lewis 	 * of vslock(), does a hard loop on EAGAIN.
20716929939SDon Lewis 	 */
2082feb50bfSAttilio Rao 	if (npages + cnt.v_wire_count > vm_page_max_wired)
20916929939SDon Lewis 		return (EAGAIN);
21016929939SDon Lewis #endif
211ce8660e3SDon Lewis 	error = vm_map_wire(&curproc->p_vmspace->vm_map, start, end,
212d9b2500eSBrian Feldman 	    VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES);
213ce8660e3SDon Lewis 	/*
214ce8660e3SDon Lewis 	 * Return EFAULT on error to match copy{in,out}() behaviour
215ce8660e3SDon Lewis 	 * rather than returning ENOMEM like mlock() would.
216ce8660e3SDon Lewis 	 */
217ce8660e3SDon Lewis 	return (error == KERN_SUCCESS ? 0 : EFAULT);
21816929939SDon Lewis }
21916929939SDon Lewis 
220ce8660e3SDon Lewis void
221f0ea4612SDon Lewis vsunlock(void *addr, size_t len)
22216929939SDon Lewis {
22316929939SDon Lewis 
224ce8660e3SDon Lewis 	/* Rely on the parameter sanity checks performed by vslock(). */
225ce8660e3SDon Lewis 	(void)vm_map_unwire(&curproc->p_vmspace->vm_map,
226ce8660e3SDon Lewis 	    trunc_page((vm_offset_t)addr), round_page((vm_offset_t)addr + len),
22716929939SDon Lewis 	    VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES);
22816929939SDon Lewis }
22916929939SDon Lewis 
230da61b9a6SAlan Cox /*
231da61b9a6SAlan Cox  * Pin the page contained within the given object at the given offset.  If the
232da61b9a6SAlan Cox  * page is not resident, allocate and load it using the given object's pager.
233da61b9a6SAlan Cox  * Return the pinned page if successful; otherwise, return NULL.
234da61b9a6SAlan Cox  */
235da61b9a6SAlan Cox static vm_page_t
236da61b9a6SAlan Cox vm_imgact_hold_page(vm_object_t object, vm_ooffset_t offset)
237da61b9a6SAlan Cox {
238da61b9a6SAlan Cox 	vm_page_t m, ma[1];
239da61b9a6SAlan Cox 	vm_pindex_t pindex;
240da61b9a6SAlan Cox 	int rv;
241da61b9a6SAlan Cox 
242*89f6b863SAttilio Rao 	VM_OBJECT_WLOCK(object);
243da61b9a6SAlan Cox 	pindex = OFF_TO_IDX(offset);
244da61b9a6SAlan Cox 	m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
2450a2e596aSAlan Cox 	if (m->valid != VM_PAGE_BITS_ALL) {
246da61b9a6SAlan Cox 		ma[0] = m;
247da61b9a6SAlan Cox 		rv = vm_pager_get_pages(object, ma, 1, 0);
248da61b9a6SAlan Cox 		m = vm_page_lookup(object, pindex);
249da61b9a6SAlan Cox 		if (m == NULL)
250da61b9a6SAlan Cox 			goto out;
251d1a6e42dSAlan Cox 		if (rv != VM_PAGER_OK) {
2522965a453SKip Macy 			vm_page_lock(m);
253da61b9a6SAlan Cox 			vm_page_free(m);
2542965a453SKip Macy 			vm_page_unlock(m);
255da61b9a6SAlan Cox 			m = NULL;
256da61b9a6SAlan Cox 			goto out;
257da61b9a6SAlan Cox 		}
258da61b9a6SAlan Cox 	}
2592965a453SKip Macy 	vm_page_lock(m);
260da61b9a6SAlan Cox 	vm_page_hold(m);
2612965a453SKip Macy 	vm_page_unlock(m);
26266bdd5d6SAlan Cox 	vm_page_wakeup(m);
263da61b9a6SAlan Cox out:
264*89f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(object);
265da61b9a6SAlan Cox 	return (m);
266da61b9a6SAlan Cox }
267da61b9a6SAlan Cox 
268da61b9a6SAlan Cox /*
269da61b9a6SAlan Cox  * Return a CPU private mapping to the page at the given offset within the
270da61b9a6SAlan Cox  * given object.  The page is pinned before it is mapped.
271da61b9a6SAlan Cox  */
272da61b9a6SAlan Cox struct sf_buf *
273da61b9a6SAlan Cox vm_imgact_map_page(vm_object_t object, vm_ooffset_t offset)
274da61b9a6SAlan Cox {
275da61b9a6SAlan Cox 	vm_page_t m;
276da61b9a6SAlan Cox 
277da61b9a6SAlan Cox 	m = vm_imgact_hold_page(object, offset);
278da61b9a6SAlan Cox 	if (m == NULL)
279da61b9a6SAlan Cox 		return (NULL);
280da61b9a6SAlan Cox 	sched_pin();
281da61b9a6SAlan Cox 	return (sf_buf_alloc(m, SFB_CPUPRIVATE));
282da61b9a6SAlan Cox }
283da61b9a6SAlan Cox 
284da61b9a6SAlan Cox /*
285da61b9a6SAlan Cox  * Destroy the given CPU private mapping and unpin the page that it mapped.
286da61b9a6SAlan Cox  */
287da61b9a6SAlan Cox void
288da61b9a6SAlan Cox vm_imgact_unmap_page(struct sf_buf *sf)
289da61b9a6SAlan Cox {
290da61b9a6SAlan Cox 	vm_page_t m;
291da61b9a6SAlan Cox 
292da61b9a6SAlan Cox 	m = sf_buf_page(sf);
293da61b9a6SAlan Cox 	sf_buf_free(sf);
294da61b9a6SAlan Cox 	sched_unpin();
2952965a453SKip Macy 	vm_page_lock(m);
296da61b9a6SAlan Cox 	vm_page_unhold(m);
2972965a453SKip Macy 	vm_page_unlock(m);
298da61b9a6SAlan Cox }
299da61b9a6SAlan Cox 
3001a4fcaebSMarcel Moolenaar void
3011a4fcaebSMarcel Moolenaar vm_sync_icache(vm_map_t map, vm_offset_t va, vm_offset_t sz)
3021a4fcaebSMarcel Moolenaar {
3031a4fcaebSMarcel Moolenaar 
3041a4fcaebSMarcel Moolenaar 	pmap_sync_icache(map->pmap, va, sz);
3051a4fcaebSMarcel Moolenaar }
3061a4fcaebSMarcel Moolenaar 
307e878d997SKonstantin Belousov struct kstack_cache_entry *kstack_cache;
3088a945d10SKonstantin Belousov static int kstack_cache_size = 128;
3098a945d10SKonstantin Belousov static int kstacks;
3108a945d10SKonstantin Belousov static struct mtx kstack_cache_mtx;
31125c1e164SAndre Oppermann MTX_SYSINIT(kstack_cache, &kstack_cache_mtx, "kstkch", MTX_DEF);
31225c1e164SAndre Oppermann 
3138a945d10SKonstantin Belousov SYSCTL_INT(_vm, OID_AUTO, kstack_cache_size, CTLFLAG_RW, &kstack_cache_size, 0,
3148a945d10SKonstantin Belousov     "");
3158a945d10SKonstantin Belousov SYSCTL_INT(_vm, OID_AUTO, kstacks, CTLFLAG_RD, &kstacks, 0,
3168a945d10SKonstantin Belousov     "");
3178a945d10SKonstantin Belousov 
31849a2507bSAlan Cox #ifndef KSTACK_MAX_PAGES
31949a2507bSAlan Cox #define KSTACK_MAX_PAGES 32
32049a2507bSAlan Cox #endif
32149a2507bSAlan Cox 
32249a2507bSAlan Cox /*
32349a2507bSAlan Cox  * Create the kernel stack (including pcb for i386) for a new thread.
32449a2507bSAlan Cox  * This routine directly affects the fork perf for a process and
32549a2507bSAlan Cox  * create performance for a thread.
32649a2507bSAlan Cox  */
32789b57fcfSKonstantin Belousov int
32849a2507bSAlan Cox vm_thread_new(struct thread *td, int pages)
32949a2507bSAlan Cox {
33049a2507bSAlan Cox 	vm_object_t ksobj;
33149a2507bSAlan Cox 	vm_offset_t ks;
33249a2507bSAlan Cox 	vm_page_t m, ma[KSTACK_MAX_PAGES];
3338a945d10SKonstantin Belousov 	struct kstack_cache_entry *ks_ce;
33449a2507bSAlan Cox 	int i;
33549a2507bSAlan Cox 
33649a2507bSAlan Cox 	/* Bounds check */
33749a2507bSAlan Cox 	if (pages <= 1)
33849a2507bSAlan Cox 		pages = KSTACK_PAGES;
33949a2507bSAlan Cox 	else if (pages > KSTACK_MAX_PAGES)
34049a2507bSAlan Cox 		pages = KSTACK_MAX_PAGES;
3418a945d10SKonstantin Belousov 
3428a945d10SKonstantin Belousov 	if (pages == KSTACK_PAGES) {
3438a945d10SKonstantin Belousov 		mtx_lock(&kstack_cache_mtx);
3448a945d10SKonstantin Belousov 		if (kstack_cache != NULL) {
3458a945d10SKonstantin Belousov 			ks_ce = kstack_cache;
3468a945d10SKonstantin Belousov 			kstack_cache = ks_ce->next_ks_entry;
3478a945d10SKonstantin Belousov 			mtx_unlock(&kstack_cache_mtx);
3488a945d10SKonstantin Belousov 
3498a945d10SKonstantin Belousov 			td->td_kstack_obj = ks_ce->ksobj;
3508a945d10SKonstantin Belousov 			td->td_kstack = (vm_offset_t)ks_ce;
3518a945d10SKonstantin Belousov 			td->td_kstack_pages = KSTACK_PAGES;
3528a945d10SKonstantin Belousov 			return (1);
3538a945d10SKonstantin Belousov 		}
3548a945d10SKonstantin Belousov 		mtx_unlock(&kstack_cache_mtx);
3558a945d10SKonstantin Belousov 	}
3568a945d10SKonstantin Belousov 
35749a2507bSAlan Cox 	/*
35849a2507bSAlan Cox 	 * Allocate an object for the kstack.
35949a2507bSAlan Cox 	 */
36049a2507bSAlan Cox 	ksobj = vm_object_allocate(OBJT_DEFAULT, pages);
361374ae2a3SJeff Roberson 
36249a2507bSAlan Cox 	/*
36349a2507bSAlan Cox 	 * Get a kernel virtual address for this thread's kstack.
36449a2507bSAlan Cox 	 */
365ca596a25SJuli Mallett #if defined(__mips__)
366ca596a25SJuli Mallett 	/*
367ca596a25SJuli Mallett 	 * We need to align the kstack's mapped address to fit within
368ca596a25SJuli Mallett 	 * a single TLB entry.
369ca596a25SJuli Mallett 	 */
370ca596a25SJuli Mallett 	ks = kmem_alloc_nofault_space(kernel_map,
371ca596a25SJuli Mallett 	    (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE, VMFS_TLB_ALIGNED_SPACE);
372ca596a25SJuli Mallett #else
37349a2507bSAlan Cox 	ks = kmem_alloc_nofault(kernel_map,
37449a2507bSAlan Cox 	   (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE);
375ca596a25SJuli Mallett #endif
37689b57fcfSKonstantin Belousov 	if (ks == 0) {
37789b57fcfSKonstantin Belousov 		printf("vm_thread_new: kstack allocation failed\n");
37889b57fcfSKonstantin Belousov 		vm_object_deallocate(ksobj);
37989b57fcfSKonstantin Belousov 		return (0);
38089b57fcfSKonstantin Belousov 	}
38189b57fcfSKonstantin Belousov 
3828a945d10SKonstantin Belousov 	atomic_add_int(&kstacks, 1);
38349a2507bSAlan Cox 	if (KSTACK_GUARD_PAGES != 0) {
38449a2507bSAlan Cox 		pmap_qremove(ks, KSTACK_GUARD_PAGES);
38549a2507bSAlan Cox 		ks += KSTACK_GUARD_PAGES * PAGE_SIZE;
38649a2507bSAlan Cox 	}
38789b57fcfSKonstantin Belousov 	td->td_kstack_obj = ksobj;
38849a2507bSAlan Cox 	td->td_kstack = ks;
38949a2507bSAlan Cox 	/*
39049a2507bSAlan Cox 	 * Knowing the number of pages allocated is useful when you
39149a2507bSAlan Cox 	 * want to deallocate them.
39249a2507bSAlan Cox 	 */
39349a2507bSAlan Cox 	td->td_kstack_pages = pages;
39449a2507bSAlan Cox 	/*
39549a2507bSAlan Cox 	 * For the length of the stack, link in a real page of ram for each
39649a2507bSAlan Cox 	 * page of stack.
39749a2507bSAlan Cox 	 */
398*89f6b863SAttilio Rao 	VM_OBJECT_WLOCK(ksobj);
39949a2507bSAlan Cox 	for (i = 0; i < pages; i++) {
40049a2507bSAlan Cox 		/*
40149a2507bSAlan Cox 		 * Get a kernel stack page.
40249a2507bSAlan Cox 		 */
403ddf4bb37SAlan Cox 		m = vm_page_grab(ksobj, i, VM_ALLOC_NOBUSY |
40449a2507bSAlan Cox 		    VM_ALLOC_NORMAL | VM_ALLOC_RETRY | VM_ALLOC_WIRED);
40549a2507bSAlan Cox 		ma[i] = m;
40649a2507bSAlan Cox 		m->valid = VM_PAGE_BITS_ALL;
40749a2507bSAlan Cox 	}
408*89f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(ksobj);
40949a2507bSAlan Cox 	pmap_qenter(ks, ma, pages);
41089b57fcfSKonstantin Belousov 	return (1);
41149a2507bSAlan Cox }
41249a2507bSAlan Cox 
4138a945d10SKonstantin Belousov static void
4148a945d10SKonstantin Belousov vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages)
41549a2507bSAlan Cox {
41649a2507bSAlan Cox 	vm_page_t m;
4178a945d10SKonstantin Belousov 	int i;
41849a2507bSAlan Cox 
4198a945d10SKonstantin Belousov 	atomic_add_int(&kstacks, -1);
42049a2507bSAlan Cox 	pmap_qremove(ks, pages);
421*89f6b863SAttilio Rao 	VM_OBJECT_WLOCK(ksobj);
42249a2507bSAlan Cox 	for (i = 0; i < pages; i++) {
42349a2507bSAlan Cox 		m = vm_page_lookup(ksobj, i);
42449a2507bSAlan Cox 		if (m == NULL)
42549a2507bSAlan Cox 			panic("vm_thread_dispose: kstack already missing?");
4262965a453SKip Macy 		vm_page_lock(m);
42749a2507bSAlan Cox 		vm_page_unwire(m, 0);
42849a2507bSAlan Cox 		vm_page_free(m);
4292965a453SKip Macy 		vm_page_unlock(m);
43049a2507bSAlan Cox 	}
431*89f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(ksobj);
43249a2507bSAlan Cox 	vm_object_deallocate(ksobj);
43349a2507bSAlan Cox 	kmem_free(kernel_map, ks - (KSTACK_GUARD_PAGES * PAGE_SIZE),
43449a2507bSAlan Cox 	    (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE);
435c3cf0b47SKonstantin Belousov }
436c3cf0b47SKonstantin Belousov 
437c3cf0b47SKonstantin Belousov /*
4388a945d10SKonstantin Belousov  * Dispose of a thread's kernel stack.
4398a945d10SKonstantin Belousov  */
4408a945d10SKonstantin Belousov void
4418a945d10SKonstantin Belousov vm_thread_dispose(struct thread *td)
4428a945d10SKonstantin Belousov {
4438a945d10SKonstantin Belousov 	vm_object_t ksobj;
4448a945d10SKonstantin Belousov 	vm_offset_t ks;
4458a945d10SKonstantin Belousov 	struct kstack_cache_entry *ks_ce;
4468a945d10SKonstantin Belousov 	int pages;
4478a945d10SKonstantin Belousov 
4488a945d10SKonstantin Belousov 	pages = td->td_kstack_pages;
4498a945d10SKonstantin Belousov 	ksobj = td->td_kstack_obj;
4508a945d10SKonstantin Belousov 	ks = td->td_kstack;
4518a945d10SKonstantin Belousov 	td->td_kstack = 0;
4528a945d10SKonstantin Belousov 	td->td_kstack_pages = 0;
4538a945d10SKonstantin Belousov 	if (pages == KSTACK_PAGES && kstacks <= kstack_cache_size) {
4548a945d10SKonstantin Belousov 		ks_ce = (struct kstack_cache_entry *)ks;
4558a945d10SKonstantin Belousov 		ks_ce->ksobj = ksobj;
4568a945d10SKonstantin Belousov 		mtx_lock(&kstack_cache_mtx);
4578a945d10SKonstantin Belousov 		ks_ce->next_ks_entry = kstack_cache;
4588a945d10SKonstantin Belousov 		kstack_cache = ks_ce;
4598a945d10SKonstantin Belousov 		mtx_unlock(&kstack_cache_mtx);
4608a945d10SKonstantin Belousov 		return;
4618a945d10SKonstantin Belousov 	}
4628a945d10SKonstantin Belousov 	vm_thread_stack_dispose(ksobj, ks, pages);
4638a945d10SKonstantin Belousov }
4648a945d10SKonstantin Belousov 
4658a945d10SKonstantin Belousov static void
4668a945d10SKonstantin Belousov vm_thread_stack_lowmem(void *nulll)
4678a945d10SKonstantin Belousov {
4688a945d10SKonstantin Belousov 	struct kstack_cache_entry *ks_ce, *ks_ce1;
4698a945d10SKonstantin Belousov 
4708a945d10SKonstantin Belousov 	mtx_lock(&kstack_cache_mtx);
4718a945d10SKonstantin Belousov 	ks_ce = kstack_cache;
4728a945d10SKonstantin Belousov 	kstack_cache = NULL;
4738a945d10SKonstantin Belousov 	mtx_unlock(&kstack_cache_mtx);
4748a945d10SKonstantin Belousov 
4758a945d10SKonstantin Belousov 	while (ks_ce != NULL) {
4768a945d10SKonstantin Belousov 		ks_ce1 = ks_ce;
4778a945d10SKonstantin Belousov 		ks_ce = ks_ce->next_ks_entry;
4788a945d10SKonstantin Belousov 
4798a945d10SKonstantin Belousov 		vm_thread_stack_dispose(ks_ce1->ksobj, (vm_offset_t)ks_ce1,
4808a945d10SKonstantin Belousov 		    KSTACK_PAGES);
4818a945d10SKonstantin Belousov 	}
4828a945d10SKonstantin Belousov }
4838a945d10SKonstantin Belousov 
4848a945d10SKonstantin Belousov static void
4858a945d10SKonstantin Belousov kstack_cache_init(void *nulll)
4868a945d10SKonstantin Belousov {
4878a945d10SKonstantin Belousov 
4888a945d10SKonstantin Belousov 	EVENTHANDLER_REGISTER(vm_lowmem, vm_thread_stack_lowmem, NULL,
4898a945d10SKonstantin Belousov 	    EVENTHANDLER_PRI_ANY);
4908a945d10SKonstantin Belousov }
4918a945d10SKonstantin Belousov 
4928a945d10SKonstantin Belousov SYSINIT(vm_kstacks, SI_SUB_KTHREAD_INIT, SI_ORDER_ANY, kstack_cache_init, NULL);
4938a945d10SKonstantin Belousov 
494ac45ee97SAlan Cox #ifndef NO_SWAPPING
4958a945d10SKonstantin Belousov /*
49649a2507bSAlan Cox  * Allow a thread's kernel stack to be paged out.
49749a2507bSAlan Cox  */
498ac45ee97SAlan Cox static void
49949a2507bSAlan Cox vm_thread_swapout(struct thread *td)
50049a2507bSAlan Cox {
50149a2507bSAlan Cox 	vm_object_t ksobj;
50249a2507bSAlan Cox 	vm_page_t m;
50349a2507bSAlan Cox 	int i, pages;
50449a2507bSAlan Cox 
505710338e9SMarcel Moolenaar 	cpu_thread_swapout(td);
50649a2507bSAlan Cox 	pages = td->td_kstack_pages;
50749a2507bSAlan Cox 	ksobj = td->td_kstack_obj;
50849a2507bSAlan Cox 	pmap_qremove(td->td_kstack, pages);
509*89f6b863SAttilio Rao 	VM_OBJECT_WLOCK(ksobj);
51049a2507bSAlan Cox 	for (i = 0; i < pages; i++) {
51149a2507bSAlan Cox 		m = vm_page_lookup(ksobj, i);
51249a2507bSAlan Cox 		if (m == NULL)
51349a2507bSAlan Cox 			panic("vm_thread_swapout: kstack already missing?");
51449a2507bSAlan Cox 		vm_page_dirty(m);
5152965a453SKip Macy 		vm_page_lock(m);
51649a2507bSAlan Cox 		vm_page_unwire(m, 0);
5172965a453SKip Macy 		vm_page_unlock(m);
51849a2507bSAlan Cox 	}
519*89f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(ksobj);
52049a2507bSAlan Cox }
52149a2507bSAlan Cox 
52249a2507bSAlan Cox /*
52349a2507bSAlan Cox  * Bring the kernel stack for a specified thread back in.
52449a2507bSAlan Cox  */
525ac45ee97SAlan Cox static void
52649a2507bSAlan Cox vm_thread_swapin(struct thread *td)
52749a2507bSAlan Cox {
52849a2507bSAlan Cox 	vm_object_t ksobj;
5296fb8c0c1SKonstantin Belousov 	vm_page_t ma[KSTACK_MAX_PAGES];
5306fb8c0c1SKonstantin Belousov 	int i, j, k, pages, rv;
53149a2507bSAlan Cox 
53249a2507bSAlan Cox 	pages = td->td_kstack_pages;
53349a2507bSAlan Cox 	ksobj = td->td_kstack_obj;
534*89f6b863SAttilio Rao 	VM_OBJECT_WLOCK(ksobj);
5356fb8c0c1SKonstantin Belousov 	for (i = 0; i < pages; i++)
5366fb8c0c1SKonstantin Belousov 		ma[i] = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY |
5376c68c971SAlan Cox 		    VM_ALLOC_WIRED);
5386fb8c0c1SKonstantin Belousov 	for (i = 0; i < pages; i++) {
5396fb8c0c1SKonstantin Belousov 		if (ma[i]->valid != VM_PAGE_BITS_ALL) {
5406fb8c0c1SKonstantin Belousov 			KASSERT(ma[i]->oflags & VPO_BUSY,
5416fb8c0c1SKonstantin Belousov 			    ("lost busy 1"));
5426fb8c0c1SKonstantin Belousov 			vm_object_pip_add(ksobj, 1);
5436fb8c0c1SKonstantin Belousov 			for (j = i + 1; j < pages; j++) {
5446fb8c0c1SKonstantin Belousov 				KASSERT(ma[j]->valid == VM_PAGE_BITS_ALL ||
5456fb8c0c1SKonstantin Belousov 				    (ma[j]->oflags & VPO_BUSY),
5466fb8c0c1SKonstantin Belousov 				    ("lost busy 2"));
5476fb8c0c1SKonstantin Belousov 				if (ma[j]->valid == VM_PAGE_BITS_ALL)
5486fb8c0c1SKonstantin Belousov 					break;
54949a2507bSAlan Cox 			}
5506fb8c0c1SKonstantin Belousov 			rv = vm_pager_get_pages(ksobj, ma + i, j - i, 0);
5516fb8c0c1SKonstantin Belousov 			if (rv != VM_PAGER_OK)
5526fb8c0c1SKonstantin Belousov 	panic("vm_thread_swapin: cannot get kstack for proc: %d",
5536fb8c0c1SKonstantin Belousov 				    td->td_proc->p_pid);
5546fb8c0c1SKonstantin Belousov 			vm_object_pip_wakeup(ksobj);
5556fb8c0c1SKonstantin Belousov 			for (k = i; k < j; k++)
5566fb8c0c1SKonstantin Belousov 				ma[k] = vm_page_lookup(ksobj, k);
5576fb8c0c1SKonstantin Belousov 			vm_page_wakeup(ma[i]);
5586fb8c0c1SKonstantin Belousov 		} else if (ma[i]->oflags & VPO_BUSY)
5596fb8c0c1SKonstantin Belousov 			vm_page_wakeup(ma[i]);
56049a2507bSAlan Cox 	}
561*89f6b863SAttilio Rao 	VM_OBJECT_WUNLOCK(ksobj);
56249a2507bSAlan Cox 	pmap_qenter(td->td_kstack, ma, pages);
563710338e9SMarcel Moolenaar 	cpu_thread_swapin(td);
56449a2507bSAlan Cox }
565ac45ee97SAlan Cox #endif /* !NO_SWAPPING */
56649a2507bSAlan Cox 
567a136efe9SPeter Wemm /*
568df8bae1dSRodney W. Grimes  * Implement fork's actions on an address space.
569df8bae1dSRodney W. Grimes  * Here we arrange for the address space to be copied or referenced,
570df8bae1dSRodney W. Grimes  * allocate a user struct (pcb and kernel stack), then call the
571df8bae1dSRodney W. Grimes  * machine-dependent layer to fill those in and make the new process
572a2a1c95cSPeter Wemm  * ready to run.  The new process is set up so that it returns directly
573a2a1c95cSPeter Wemm  * to user mode to avoid stack copying and relocation problems.
574df8bae1dSRodney W. Grimes  */
57589b57fcfSKonstantin Belousov int
57689b57fcfSKonstantin Belousov vm_forkproc(td, p2, td2, vm2, flags)
577b40ce416SJulian Elischer 	struct thread *td;
578b40ce416SJulian Elischer 	struct proc *p2;
579079b7badSJulian Elischer 	struct thread *td2;
58089b57fcfSKonstantin Belousov 	struct vmspace *vm2;
581a2a1c95cSPeter Wemm 	int flags;
582df8bae1dSRodney W. Grimes {
583b40ce416SJulian Elischer 	struct proc *p1 = td->td_proc;
58489b57fcfSKonstantin Belousov 	int error;
585df8bae1dSRodney W. Grimes 
58691c28bfdSLuoqi Chen 	if ((flags & RFPROC) == 0) {
58791c28bfdSLuoqi Chen 		/*
58891c28bfdSLuoqi Chen 		 * Divorce the memory, if it is shared, essentially
58991c28bfdSLuoqi Chen 		 * this changes shared memory amongst threads, into
59091c28bfdSLuoqi Chen 		 * COW locally.
59191c28bfdSLuoqi Chen 		 */
59291c28bfdSLuoqi Chen 		if ((flags & RFMEM) == 0) {
59391c28bfdSLuoqi Chen 			if (p1->p_vmspace->vm_refcnt > 1) {
59489b57fcfSKonstantin Belousov 				error = vmspace_unshare(p1);
59589b57fcfSKonstantin Belousov 				if (error)
59689b57fcfSKonstantin Belousov 					return (error);
59791c28bfdSLuoqi Chen 			}
59891c28bfdSLuoqi Chen 		}
599079b7badSJulian Elischer 		cpu_fork(td, p2, td2, flags);
60089b57fcfSKonstantin Belousov 		return (0);
60191c28bfdSLuoqi Chen 	}
60291c28bfdSLuoqi Chen 
6035856e12eSJohn Dyson 	if (flags & RFMEM) {
6045856e12eSJohn Dyson 		p2->p_vmspace = p1->p_vmspace;
6051a276a3fSAlan Cox 		atomic_add_int(&p1->p_vmspace->vm_refcnt, 1);
6065856e12eSJohn Dyson 	}
6075856e12eSJohn Dyson 
60890ecac61SMatthew Dillon 	while (vm_page_count_severe()) {
60926f9a767SRodney W. Grimes 		VM_WAIT;
6100d94caffSDavid Greenman 	}
61126f9a767SRodney W. Grimes 
6125856e12eSJohn Dyson 	if ((flags & RFMEM) == 0) {
61389b57fcfSKonstantin Belousov 		p2->p_vmspace = vm2;
614df8bae1dSRodney W. Grimes 		if (p1->p_vmspace->vm_shm)
615dabee6feSPeter Wemm 			shmfork(p1, p2);
616a2a1c95cSPeter Wemm 	}
617df8bae1dSRodney W. Grimes 
61839fb8e6bSJulian Elischer 	/*
619a2a1c95cSPeter Wemm 	 * cpu_fork will copy and update the pcb, set up the kernel stack,
620a2a1c95cSPeter Wemm 	 * and make the child ready to run.
621df8bae1dSRodney W. Grimes 	 */
622079b7badSJulian Elischer 	cpu_fork(td, p2, td2, flags);
62389b57fcfSKonstantin Belousov 	return (0);
624df8bae1dSRodney W. Grimes }
625df8bae1dSRodney W. Grimes 
626df8bae1dSRodney W. Grimes /*
627eb30c1c0SPeter Wemm  * Called after process has been wait(2)'ed apon and is being reaped.
628eb30c1c0SPeter Wemm  * The idea is to reclaim resources that we could not reclaim while
629eb30c1c0SPeter Wemm  * the process was still executing.
630eb30c1c0SPeter Wemm  */
631eb30c1c0SPeter Wemm void
632eb30c1c0SPeter Wemm vm_waitproc(p)
633eb30c1c0SPeter Wemm 	struct proc *p;
634eb30c1c0SPeter Wemm {
635eb30c1c0SPeter Wemm 
636582ec34cSAlfred Perlstein 	vmspace_exitfree(p);		/* and clean-out the vmspace */
637eb30c1c0SPeter Wemm }
638eb30c1c0SPeter Wemm 
63926f9a767SRodney W. Grimes void
64026f9a767SRodney W. Grimes faultin(p)
64126f9a767SRodney W. Grimes 	struct proc *p;
64226f9a767SRodney W. Grimes {
64311edc1e0SJohn Baldwin #ifdef NO_SWAPPING
64411edc1e0SJohn Baldwin 
64511edc1e0SJohn Baldwin 	PROC_LOCK_ASSERT(p, MA_OWNED);
646b61ce5b0SJeff Roberson 	if ((p->p_flag & P_INMEM) == 0)
64711edc1e0SJohn Baldwin 		panic("faultin: proc swapped out with NO_SWAPPING!");
64811edc1e0SJohn Baldwin #else /* !NO_SWAPPING */
649664f718bSJohn Baldwin 	struct thread *td;
65026f9a767SRodney W. Grimes 
651c96d52a9SJohn Baldwin 	PROC_LOCK_ASSERT(p, MA_OWNED);
6521d7b9ed2SJulian Elischer 	/*
6531d7b9ed2SJulian Elischer 	 * If another process is swapping in this process,
6541d7b9ed2SJulian Elischer 	 * just wait until it finishes.
6551d7b9ed2SJulian Elischer 	 */
656b61ce5b0SJeff Roberson 	if (p->p_flag & P_SWAPPINGIN) {
657b61ce5b0SJeff Roberson 		while (p->p_flag & P_SWAPPINGIN)
658b61ce5b0SJeff Roberson 			msleep(&p->p_flag, &p->p_mtx, PVM, "faultin", 0);
659b61ce5b0SJeff Roberson 		return;
660b61ce5b0SJeff Roberson 	}
661b61ce5b0SJeff Roberson 	if ((p->p_flag & P_INMEM) == 0) {
662664f718bSJohn Baldwin 		/*
663664f718bSJohn Baldwin 		 * Don't let another thread swap process p out while we are
664664f718bSJohn Baldwin 		 * busy swapping it in.
665664f718bSJohn Baldwin 		 */
666664f718bSJohn Baldwin 		++p->p_lock;
667b61ce5b0SJeff Roberson 		p->p_flag |= P_SWAPPINGIN;
66845ece682SJohn Baldwin 		PROC_UNLOCK(p);
66926f9a767SRodney W. Grimes 
670b61ce5b0SJeff Roberson 		/*
671b61ce5b0SJeff Roberson 		 * We hold no lock here because the list of threads
672b61ce5b0SJeff Roberson 		 * can not change while all threads in the process are
673b61ce5b0SJeff Roberson 		 * swapped out.
674b61ce5b0SJeff Roberson 		 */
675664f718bSJohn Baldwin 		FOREACH_THREAD_IN_PROC(p, td)
67649a2507bSAlan Cox 			vm_thread_swapin(td);
67745ece682SJohn Baldwin 		PROC_LOCK(p);
678b61ce5b0SJeff Roberson 		swapclear(p);
679258853abSJeff Roberson 		p->p_swtick = ticks;
68026f9a767SRodney W. Grimes 
681b61ce5b0SJeff Roberson 		wakeup(&p->p_flag);
68226f9a767SRodney W. Grimes 
683664f718bSJohn Baldwin 		/* Allow other threads to swap p out now. */
68426f9a767SRodney W. Grimes 		--p->p_lock;
68526f9a767SRodney W. Grimes 	}
68611edc1e0SJohn Baldwin #endif /* NO_SWAPPING */
68726f9a767SRodney W. Grimes }
68826f9a767SRodney W. Grimes 
689df8bae1dSRodney W. Grimes /*
69026f9a767SRodney W. Grimes  * This swapin algorithm attempts to swap-in processes only if there
69126f9a767SRodney W. Grimes  * is enough space for them.  Of course, if a process waits for a long
69226f9a767SRodney W. Grimes  * time, it will be swapped in anyway.
6930384fff8SJason Evans  *
69410c447faSAlan Cox  * Giant is held on entry.
695df8bae1dSRodney W. Grimes  */
6962b14f991SJulian Elischer /* ARGSUSED*/
6972b14f991SJulian Elischer static void
698d841aaa7SBruce Evans scheduler(dummy)
699d841aaa7SBruce Evans 	void *dummy;
700df8bae1dSRodney W. Grimes {
70154d92145SMatthew Dillon 	struct proc *p;
702e602ba25SJulian Elischer 	struct thread *td;
703df8bae1dSRodney W. Grimes 	struct proc *pp;
704258853abSJeff Roberson 	int slptime;
705258853abSJeff Roberson 	int swtime;
706df8bae1dSRodney W. Grimes 	int ppri;
707258853abSJeff Roberson 	int pri;
708df8bae1dSRodney W. Grimes 
709c96d52a9SJohn Baldwin 	mtx_assert(&Giant, MA_OWNED | MA_NOTRECURSED);
71010c447faSAlan Cox 	mtx_unlock(&Giant);
7110384fff8SJason Evans 
712df8bae1dSRodney W. Grimes loop:
71390ecac61SMatthew Dillon 	if (vm_page_count_min()) {
7140d94caffSDavid Greenman 		VM_WAIT;
71590ecac61SMatthew Dillon 		goto loop;
7160d94caffSDavid Greenman 	}
71726f9a767SRodney W. Grimes 
718df8bae1dSRodney W. Grimes 	pp = NULL;
719df8bae1dSRodney W. Grimes 	ppri = INT_MIN;
7201005a129SJohn Baldwin 	sx_slock(&allproc_lock);
721b40ce416SJulian Elischer 	FOREACH_PROC_IN_SYSTEM(p) {
722b61ce5b0SJeff Roberson 		PROC_LOCK(p);
723e806d352SJohn Baldwin 		if (p->p_state == PRS_NEW ||
724e806d352SJohn Baldwin 		    p->p_flag & (P_SWAPPINGOUT | P_SWAPPINGIN | P_INMEM)) {
725b61ce5b0SJeff Roberson 			PROC_UNLOCK(p);
726e602ba25SJulian Elischer 			continue;
727e602ba25SJulian Elischer 		}
728258853abSJeff Roberson 		swtime = (ticks - p->p_swtick) / hz;
729e602ba25SJulian Elischer 		FOREACH_THREAD_IN_PROC(p, td) {
7301d7b9ed2SJulian Elischer 			/*
73171fad9fdSJulian Elischer 			 * An otherwise runnable thread of a process
73271fad9fdSJulian Elischer 			 * swapped out has only the TDI_SWAPPED bit set.
73371fad9fdSJulian Elischer 			 *
7341d7b9ed2SJulian Elischer 			 */
735982d11f8SJeff Roberson 			thread_lock(td);
73671fad9fdSJulian Elischer 			if (td->td_inhibitors == TDI_SWAPPED) {
737258853abSJeff Roberson 				slptime = (ticks - td->td_slptick) / hz;
738258853abSJeff Roberson 				pri = swtime + slptime;
739b61ce5b0SJeff Roberson 				if ((td->td_flags & TDF_SWAPINREQ) == 0)
740fa885116SJulian Elischer 					pri -= p->p_nice * 8;
74126f9a767SRodney W. Grimes 				/*
742ad1e7d28SJulian Elischer 				 * if this thread is higher priority
743b40ce416SJulian Elischer 				 * and there is enough space, then select
744b40ce416SJulian Elischer 				 * this process instead of the previous
745b40ce416SJulian Elischer 				 * selection.
74626f9a767SRodney W. Grimes 				 */
7470d94caffSDavid Greenman 				if (pri > ppri) {
748df8bae1dSRodney W. Grimes 					pp = p;
749df8bae1dSRodney W. Grimes 					ppri = pri;
750df8bae1dSRodney W. Grimes 				}
751df8bae1dSRodney W. Grimes 			}
752982d11f8SJeff Roberson 			thread_unlock(td);
753b40ce416SJulian Elischer 		}
754b61ce5b0SJeff Roberson 		PROC_UNLOCK(p);
755df8bae1dSRodney W. Grimes 	}
7561005a129SJohn Baldwin 	sx_sunlock(&allproc_lock);
75726f9a767SRodney W. Grimes 
758df8bae1dSRodney W. Grimes 	/*
759a669a6e9SJohn Dyson 	 * Nothing to do, back to sleep.
760df8bae1dSRodney W. Grimes 	 */
761df8bae1dSRodney W. Grimes 	if ((p = pp) == NULL) {
76250a57dfbSKonstantin Belousov 		tsleep(&proc0, PVM, "sched", MAXSLP * hz / 2);
763df8bae1dSRodney W. Grimes 		goto loop;
764df8bae1dSRodney W. Grimes 	}
7651d7b9ed2SJulian Elischer 	PROC_LOCK(p);
7661d7b9ed2SJulian Elischer 
7671d7b9ed2SJulian Elischer 	/*
7681d7b9ed2SJulian Elischer 	 * Another process may be bringing or may have already
7691d7b9ed2SJulian Elischer 	 * brought this process in while we traverse all threads.
7701d7b9ed2SJulian Elischer 	 * Or, this process may even be being swapped out again.
7711d7b9ed2SJulian Elischer 	 */
772b61ce5b0SJeff Roberson 	if (p->p_flag & (P_INMEM | P_SWAPPINGOUT | P_SWAPPINGIN)) {
7731d7b9ed2SJulian Elischer 		PROC_UNLOCK(p);
7741d7b9ed2SJulian Elischer 		goto loop;
7751d7b9ed2SJulian Elischer 	}
7761d7b9ed2SJulian Elischer 
777df8bae1dSRodney W. Grimes 	/*
77826f9a767SRodney W. Grimes 	 * We would like to bring someone in. (only if there is space).
779e602ba25SJulian Elischer 	 * [What checks the space? ]
780df8bae1dSRodney W. Grimes 	 */
78126f9a767SRodney W. Grimes 	faultin(p);
78245ece682SJohn Baldwin 	PROC_UNLOCK(p);
783df8bae1dSRodney W. Grimes 	goto loop;
784df8bae1dSRodney W. Grimes }
785df8bae1dSRodney W. Grimes 
786da7bbd2cSJohn Baldwin void
787da7bbd2cSJohn Baldwin kick_proc0(void)
788d13ec713SStephan Uphoff {
789d13ec713SStephan Uphoff 
790da7bbd2cSJohn Baldwin 	wakeup(&proc0);
791d13ec713SStephan Uphoff }
792d13ec713SStephan Uphoff 
7935afce282SDavid Greenman #ifndef NO_SWAPPING
7945afce282SDavid Greenman 
795ceb0cf87SJohn Dyson /*
796ceb0cf87SJohn Dyson  * Swap_idle_threshold1 is the guaranteed swapped in time for a process
797ceb0cf87SJohn Dyson  */
798303b270bSEivind Eklund static int swap_idle_threshold1 = 2;
7992a3eeaa2STom Rhodes SYSCTL_INT(_vm, OID_AUTO, swap_idle_threshold1, CTLFLAG_RW,
8009faaf3b3STom Rhodes     &swap_idle_threshold1, 0, "Guaranteed swapped in time for a process");
801ceb0cf87SJohn Dyson 
802ceb0cf87SJohn Dyson /*
803ceb0cf87SJohn Dyson  * Swap_idle_threshold2 is the time that a process can be idle before
804ceb0cf87SJohn Dyson  * it will be swapped out, if idle swapping is enabled.
805ceb0cf87SJohn Dyson  */
806303b270bSEivind Eklund static int swap_idle_threshold2 = 10;
8072a3eeaa2STom Rhodes SYSCTL_INT(_vm, OID_AUTO, swap_idle_threshold2, CTLFLAG_RW,
8089faaf3b3STom Rhodes     &swap_idle_threshold2, 0, "Time before a process will be swapped out");
809ceb0cf87SJohn Dyson 
810df8bae1dSRodney W. Grimes /*
81150a57dfbSKonstantin Belousov  * First, if any processes have been sleeping or stopped for at least
81250a57dfbSKonstantin Belousov  * "swap_idle_threshold1" seconds, they are swapped out.  If, however,
81350a57dfbSKonstantin Belousov  * no such processes exist, then the longest-sleeping or stopped
81450a57dfbSKonstantin Belousov  * process is swapped out.  Finally, and only as a last resort, if
81550a57dfbSKonstantin Belousov  * there are no sleeping or stopped processes, the longest-resident
81650a57dfbSKonstantin Belousov  * process is swapped out.
817df8bae1dSRodney W. Grimes  */
818df8bae1dSRodney W. Grimes void
8193a2dc656SJohn Dyson swapout_procs(action)
8203a2dc656SJohn Dyson int action;
821df8bae1dSRodney W. Grimes {
82254d92145SMatthew Dillon 	struct proc *p;
823e602ba25SJulian Elischer 	struct thread *td;
824df8bae1dSRodney W. Grimes 	int didswap = 0;
825df8bae1dSRodney W. Grimes 
8260d94caffSDavid Greenman retry:
8273a2189d4SJohn Baldwin 	sx_slock(&allproc_lock);
828e602ba25SJulian Elischer 	FOREACH_PROC_IN_SYSTEM(p) {
829b18bfc3dSJohn Dyson 		struct vmspace *vm;
830b40ce416SJulian Elischer 		int minslptime = 100000;
831258853abSJeff Roberson 		int slptime;
832b18bfc3dSJohn Dyson 
8339eb881f8SSeigo Tanimura 		/*
834b1f99ebeSSeigo Tanimura 		 * Watch out for a process in
835b1f99ebeSSeigo Tanimura 		 * creation.  It may have no
8361c865ac7SJohn Baldwin 		 * address space or lock yet.
8371c865ac7SJohn Baldwin 		 */
838b61ce5b0SJeff Roberson 		if (p->p_state == PRS_NEW)
8391c865ac7SJohn Baldwin 			continue;
8401c865ac7SJohn Baldwin 		/*
841b1f99ebeSSeigo Tanimura 		 * An aio daemon switches its
842b1f99ebeSSeigo Tanimura 		 * address space while running.
843b1f99ebeSSeigo Tanimura 		 * Perform a quick check whether
844b1f99ebeSSeigo Tanimura 		 * a process has P_SYSTEM.
8459eb881f8SSeigo Tanimura 		 */
8468f887403SJohn Baldwin 		if ((p->p_flag & P_SYSTEM) != 0)
847b1f99ebeSSeigo Tanimura 			continue;
8481c865ac7SJohn Baldwin 		/*
8491c865ac7SJohn Baldwin 		 * Do not swapout a process that
8501c865ac7SJohn Baldwin 		 * is waiting for VM data
8511c865ac7SJohn Baldwin 		 * structures as there is a possible
8521c865ac7SJohn Baldwin 		 * deadlock.  Test this first as
8531c865ac7SJohn Baldwin 		 * this may block.
8541c865ac7SJohn Baldwin 		 *
8551c865ac7SJohn Baldwin 		 * Lock the map until swapout
8561c865ac7SJohn Baldwin 		 * finishes, or a thread of this
8571c865ac7SJohn Baldwin 		 * process may attempt to alter
8581c865ac7SJohn Baldwin 		 * the map.
8591c865ac7SJohn Baldwin 		 */
86057051fdcSTor Egge 		vm = vmspace_acquire_ref(p);
86157051fdcSTor Egge 		if (vm == NULL)
86257051fdcSTor Egge 			continue;
8639eb881f8SSeigo Tanimura 		if (!vm_map_trylock(&vm->vm_map))
8649eb881f8SSeigo Tanimura 			goto nextproc1;
8659eb881f8SSeigo Tanimura 
8665074aecdSJohn Baldwin 		PROC_LOCK(p);
86769b40456SJohn Baldwin 		if (p->p_lock != 0 ||
8681279572aSDavid Xu 		    (p->p_flag & (P_STOPPED_SINGLE|P_TRACED|P_SYSTEM|P_WEXIT)
8691279572aSDavid Xu 		    ) != 0) {
870374ae2a3SJeff Roberson 			goto nextproc;
8715074aecdSJohn Baldwin 		}
87223955314SAlfred Perlstein 		/*
87323955314SAlfred Perlstein 		 * only aiod changes vmspace, however it will be
87423955314SAlfred Perlstein 		 * skipped because of the if statement above checking
87523955314SAlfred Perlstein 		 * for P_SYSTEM
87623955314SAlfred Perlstein 		 */
877b61ce5b0SJeff Roberson 		if ((p->p_flag & (P_INMEM|P_SWAPPINGOUT|P_SWAPPINGIN)) != P_INMEM)
878374ae2a3SJeff Roberson 			goto nextproc;
87969b40456SJohn Baldwin 
880e602ba25SJulian Elischer 		switch (p->p_state) {
8810d94caffSDavid Greenman 		default:
882e602ba25SJulian Elischer 			/* Don't swap out processes in any sort
883e602ba25SJulian Elischer 			 * of 'special' state. */
8848f887403SJohn Baldwin 			break;
885df8bae1dSRodney W. Grimes 
886e602ba25SJulian Elischer 		case PRS_NORMAL:
88726f9a767SRodney W. Grimes 			/*
888bfbfac11SDavid Greenman 			 * do not swapout a realtime process
889b40ce416SJulian Elischer 			 * Check all the thread groups..
890bfbfac11SDavid Greenman 			 */
8918460a577SJohn Birrell 			FOREACH_THREAD_IN_PROC(p, td) {
892b61ce5b0SJeff Roberson 				thread_lock(td);
893b61ce5b0SJeff Roberson 				if (PRI_IS_REALTIME(td->td_pri_class)) {
894b61ce5b0SJeff Roberson 					thread_unlock(td);
895b40ce416SJulian Elischer 					goto nextproc;
896b61ce5b0SJeff Roberson 				}
897258853abSJeff Roberson 				slptime = (ticks - td->td_slptick) / hz;
898bfbfac11SDavid Greenman 				/*
8999eb881f8SSeigo Tanimura 				 * Guarantee swap_idle_threshold1
900ceb0cf87SJohn Dyson 				 * time in memory.
9010d94caffSDavid Greenman 				 */
902258853abSJeff Roberson 				if (slptime < swap_idle_threshold1) {
903b61ce5b0SJeff Roberson 					thread_unlock(td);
904b40ce416SJulian Elischer 					goto nextproc;
905b61ce5b0SJeff Roberson 				}
9069eb881f8SSeigo Tanimura 
9071d7b9ed2SJulian Elischer 				/*
9089eb881f8SSeigo Tanimura 				 * Do not swapout a process if it is
9099eb881f8SSeigo Tanimura 				 * waiting on a critical event of some
9109eb881f8SSeigo Tanimura 				 * kind or there is a thread whose
9119eb881f8SSeigo Tanimura 				 * pageable memory may be accessed.
9121d7b9ed2SJulian Elischer 				 *
9131d7b9ed2SJulian Elischer 				 * This could be refined to support
9141d7b9ed2SJulian Elischer 				 * swapping out a thread.
9151d7b9ed2SJulian Elischer 				 */
916c5aa6b58SJeff Roberson 				if (!thread_safetoswapout(td)) {
917b61ce5b0SJeff Roberson 					thread_unlock(td);
918e602ba25SJulian Elischer 					goto nextproc;
919b61ce5b0SJeff Roberson 				}
920ceb0cf87SJohn Dyson 				/*
921b40ce416SJulian Elischer 				 * If the system is under memory stress,
922b40ce416SJulian Elischer 				 * or if we are swapping
923b40ce416SJulian Elischer 				 * idle processes >= swap_idle_threshold2,
924b40ce416SJulian Elischer 				 * then swap the process out.
925ceb0cf87SJohn Dyson 				 */
926ceb0cf87SJohn Dyson 				if (((action & VM_SWAP_NORMAL) == 0) &&
927ceb0cf87SJohn Dyson 				    (((action & VM_SWAP_IDLE) == 0) ||
928258853abSJeff Roberson 				    (slptime < swap_idle_threshold2))) {
929b61ce5b0SJeff Roberson 					thread_unlock(td);
930b40ce416SJulian Elischer 					goto nextproc;
931b61ce5b0SJeff Roberson 				}
9329eb881f8SSeigo Tanimura 
933258853abSJeff Roberson 				if (minslptime > slptime)
934258853abSJeff Roberson 					minslptime = slptime;
935b61ce5b0SJeff Roberson 				thread_unlock(td);
936b40ce416SJulian Elischer 			}
9370d94caffSDavid Greenman 
93811b224dcSDavid Greenman 			/*
93917d9d0d0SDavid Schultz 			 * If the pageout daemon didn't free enough pages,
94017d9d0d0SDavid Schultz 			 * or if this process is idle and the system is
94117d9d0d0SDavid Schultz 			 * configured to swap proactively, swap it out.
94211b224dcSDavid Greenman 			 */
943ceb0cf87SJohn Dyson 			if ((action & VM_SWAP_NORMAL) ||
944ceb0cf87SJohn Dyson 				((action & VM_SWAP_IDLE) &&
945b40ce416SJulian Elischer 				 (minslptime > swap_idle_threshold2))) {
946b61ce5b0SJeff Roberson 				if (swapout(p) == 0)
947df8bae1dSRodney W. Grimes 					didswap++;
948664f718bSJohn Baldwin 				PROC_UNLOCK(p);
9499eb881f8SSeigo Tanimura 				vm_map_unlock(&vm->vm_map);
9509eb881f8SSeigo Tanimura 				vmspace_free(vm);
9519eb881f8SSeigo Tanimura 				sx_sunlock(&allproc_lock);
9520d94caffSDavid Greenman 				goto retry;
953c96d52a9SJohn Baldwin 			}
9548f887403SJohn Baldwin 		}
955374ae2a3SJeff Roberson nextproc:
9569eb881f8SSeigo Tanimura 		PROC_UNLOCK(p);
9579eb881f8SSeigo Tanimura 		vm_map_unlock(&vm->vm_map);
9589eb881f8SSeigo Tanimura nextproc1:
9599eb881f8SSeigo Tanimura 		vmspace_free(vm);
96030171114SPeter Wemm 		continue;
961ceb0cf87SJohn Dyson 	}
9621005a129SJohn Baldwin 	sx_sunlock(&allproc_lock);
96326f9a767SRodney W. Grimes 	/*
96426f9a767SRodney W. Grimes 	 * If we swapped something out, and another process needed memory,
96526f9a767SRodney W. Grimes 	 * then wakeup the sched process.
96626f9a767SRodney W. Grimes 	 */
9670d94caffSDavid Greenman 	if (didswap)
96824a1cce3SDavid Greenman 		wakeup(&proc0);
969df8bae1dSRodney W. Grimes }
970df8bae1dSRodney W. Grimes 
971f708ef1bSPoul-Henning Kamp static void
972b61ce5b0SJeff Roberson swapclear(p)
973b61ce5b0SJeff Roberson 	struct proc *p;
974b61ce5b0SJeff Roberson {
975b61ce5b0SJeff Roberson 	struct thread *td;
976b61ce5b0SJeff Roberson 
977b61ce5b0SJeff Roberson 	PROC_LOCK_ASSERT(p, MA_OWNED);
978b61ce5b0SJeff Roberson 
979b61ce5b0SJeff Roberson 	FOREACH_THREAD_IN_PROC(p, td) {
980b61ce5b0SJeff Roberson 		thread_lock(td);
981b61ce5b0SJeff Roberson 		td->td_flags |= TDF_INMEM;
982b61ce5b0SJeff Roberson 		td->td_flags &= ~TDF_SWAPINREQ;
983b61ce5b0SJeff Roberson 		TD_CLR_SWAPPED(td);
984b61ce5b0SJeff Roberson 		if (TD_CAN_RUN(td))
985da7bbd2cSJohn Baldwin 			if (setrunnable(td)) {
986da7bbd2cSJohn Baldwin #ifdef INVARIANTS
987da7bbd2cSJohn Baldwin 				/*
988da7bbd2cSJohn Baldwin 				 * XXX: We just cleared TDI_SWAPPED
989da7bbd2cSJohn Baldwin 				 * above and set TDF_INMEM, so this
990da7bbd2cSJohn Baldwin 				 * should never happen.
991da7bbd2cSJohn Baldwin 				 */
992da7bbd2cSJohn Baldwin 				panic("not waking up swapper");
993da7bbd2cSJohn Baldwin #endif
994da7bbd2cSJohn Baldwin 			}
995b61ce5b0SJeff Roberson 		thread_unlock(td);
996b61ce5b0SJeff Roberson 	}
997b61ce5b0SJeff Roberson 	p->p_flag &= ~(P_SWAPPINGIN|P_SWAPPINGOUT);
998b61ce5b0SJeff Roberson 	p->p_flag |= P_INMEM;
999b61ce5b0SJeff Roberson }
1000b61ce5b0SJeff Roberson 
1001b61ce5b0SJeff Roberson static int
1002df8bae1dSRodney W. Grimes swapout(p)
100354d92145SMatthew Dillon 	struct proc *p;
1004df8bae1dSRodney W. Grimes {
1005b40ce416SJulian Elischer 	struct thread *td;
1006df8bae1dSRodney W. Grimes 
1007ea754954SJohn Baldwin 	PROC_LOCK_ASSERT(p, MA_OWNED);
1008d3a34985SJohn Dyson #if defined(SWAP_DEBUG)
1009d3a34985SJohn Dyson 	printf("swapping out %d\n", p->p_pid);
1010d3a34985SJohn Dyson #endif
10111d7b9ed2SJulian Elischer 
10121d7b9ed2SJulian Elischer 	/*
10139eb881f8SSeigo Tanimura 	 * The states of this process and its threads may have changed
10149eb881f8SSeigo Tanimura 	 * by now.  Assuming that there is only one pageout daemon thread,
10159eb881f8SSeigo Tanimura 	 * this process should still be in memory.
10169eb881f8SSeigo Tanimura 	 */
1017b61ce5b0SJeff Roberson 	KASSERT((p->p_flag & (P_INMEM|P_SWAPPINGOUT|P_SWAPPINGIN)) == P_INMEM,
10189eb881f8SSeigo Tanimura 		("swapout: lost a swapout race?"));
10199eb881f8SSeigo Tanimura 
1020df8bae1dSRodney W. Grimes 	/*
102126f9a767SRodney W. Grimes 	 * remember the process resident count
1022df8bae1dSRodney W. Grimes 	 */
1023b1028ad1SLuoqi Chen 	p->p_vmspace->vm_swrss = vmspace_resident_count(p->p_vmspace);
1024b61ce5b0SJeff Roberson 	/*
1025b61ce5b0SJeff Roberson 	 * Check and mark all threads before we proceed.
1026b61ce5b0SJeff Roberson 	 */
1027b61ce5b0SJeff Roberson 	p->p_flag &= ~P_INMEM;
1028b61ce5b0SJeff Roberson 	p->p_flag |= P_SWAPPINGOUT;
1029982d11f8SJeff Roberson 	FOREACH_THREAD_IN_PROC(p, td) {
1030982d11f8SJeff Roberson 		thread_lock(td);
1031b61ce5b0SJeff Roberson 		if (!thread_safetoswapout(td)) {
1032b61ce5b0SJeff Roberson 			thread_unlock(td);
1033b61ce5b0SJeff Roberson 			swapclear(p);
1034b61ce5b0SJeff Roberson 			return (EBUSY);
1035b61ce5b0SJeff Roberson 		}
1036b61ce5b0SJeff Roberson 		td->td_flags &= ~TDF_INMEM;
1037664f718bSJohn Baldwin 		TD_SET_SWAPPED(td);
1038982d11f8SJeff Roberson 		thread_unlock(td);
1039982d11f8SJeff Roberson 	}
1040b61ce5b0SJeff Roberson 	td = FIRST_THREAD_IN_PROC(p);
1041b61ce5b0SJeff Roberson 	++td->td_ru.ru_nswap;
1042b61ce5b0SJeff Roberson 	PROC_UNLOCK(p);
104326f9a767SRodney W. Grimes 
1044b61ce5b0SJeff Roberson 	/*
1045b61ce5b0SJeff Roberson 	 * This list is stable because all threads are now prevented from
1046b61ce5b0SJeff Roberson 	 * running.  The list is only modified in the context of a running
1047b61ce5b0SJeff Roberson 	 * thread in this process.
1048b61ce5b0SJeff Roberson 	 */
1049664f718bSJohn Baldwin 	FOREACH_THREAD_IN_PROC(p, td)
105049a2507bSAlan Cox 		vm_thread_swapout(td);
1051664f718bSJohn Baldwin 
1052664f718bSJohn Baldwin 	PROC_LOCK(p);
1053b61ce5b0SJeff Roberson 	p->p_flag &= ~P_SWAPPINGOUT;
1054258853abSJeff Roberson 	p->p_swtick = ticks;
1055b61ce5b0SJeff Roberson 	return (0);
1056df8bae1dSRodney W. Grimes }
10575afce282SDavid Greenman #endif /* !NO_SWAPPING */
1058