160727d8bSWarner Losh /*- 2df8bae1dSRodney W. Grimes * Copyright (c) 1991, 1993 3df8bae1dSRodney W. Grimes * The Regents of the University of California. All rights reserved. 4df8bae1dSRodney W. Grimes * 5df8bae1dSRodney W. Grimes * This code is derived from software contributed to Berkeley by 6df8bae1dSRodney W. Grimes * The Mach Operating System project at Carnegie-Mellon University. 7df8bae1dSRodney W. Grimes * 8df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 9df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 10df8bae1dSRodney W. Grimes * are met: 11df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 12df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 13df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 14df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 15df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 16df8bae1dSRodney W. Grimes * 4. Neither the name of the University nor the names of its contributors 17df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 18df8bae1dSRodney W. Grimes * without specific prior written permission. 19df8bae1dSRodney W. Grimes * 20df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30df8bae1dSRodney W. Grimes * SUCH DAMAGE. 31df8bae1dSRodney W. Grimes * 323c4dd356SDavid Greenman * from: @(#)vm_glue.c 8.6 (Berkeley) 1/5/94 33df8bae1dSRodney W. Grimes * 34df8bae1dSRodney W. Grimes * 35df8bae1dSRodney W. Grimes * Copyright (c) 1987, 1990 Carnegie-Mellon University. 36df8bae1dSRodney W. Grimes * All rights reserved. 37df8bae1dSRodney W. Grimes * 38df8bae1dSRodney W. Grimes * Permission to use, copy, modify and distribute this software and 39df8bae1dSRodney W. Grimes * its documentation is hereby granted, provided that both the copyright 40df8bae1dSRodney W. Grimes * notice and this permission notice appear in all copies of the 41df8bae1dSRodney W. Grimes * software, derivative works or modified versions, and any portions 42df8bae1dSRodney W. Grimes * thereof, and that both notices appear in supporting documentation. 43df8bae1dSRodney W. Grimes * 44df8bae1dSRodney W. Grimes * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 45df8bae1dSRodney W. Grimes * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 46df8bae1dSRodney W. Grimes * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 47df8bae1dSRodney W. Grimes * 48df8bae1dSRodney W. Grimes * Carnegie Mellon requests users of this software to return to 49df8bae1dSRodney W. Grimes * 50df8bae1dSRodney W. Grimes * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 51df8bae1dSRodney W. Grimes * School of Computer Science 52df8bae1dSRodney W. Grimes * Carnegie Mellon University 53df8bae1dSRodney W. Grimes * Pittsburgh PA 15213-3890 54df8bae1dSRodney W. Grimes * 55df8bae1dSRodney W. Grimes * any improvements or extensions that they make and grant Carnegie the 56df8bae1dSRodney W. Grimes * rights to redistribute these changes. 57df8bae1dSRodney W. Grimes */ 58df8bae1dSRodney W. Grimes 59874651b1SDavid E. O'Brien #include <sys/cdefs.h> 60874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$"); 61874651b1SDavid E. O'Brien 62faa5f8d8SAndrzej Bialecki #include "opt_vm.h" 6315a7ad60SPeter Wemm #include "opt_kstack_pages.h" 6415a7ad60SPeter Wemm #include "opt_kstack_max_pages.h" 65e9822d92SJoerg Wunsch 66df8bae1dSRodney W. Grimes #include <sys/param.h> 67df8bae1dSRodney W. Grimes #include <sys/systm.h> 68104a9b7eSAlexander Kabaev #include <sys/limits.h> 69fb919e4dSMark Murray #include <sys/lock.h> 70fb919e4dSMark Murray #include <sys/mutex.h> 71df8bae1dSRodney W. Grimes #include <sys/proc.h> 721ba5ad42SEdward Tomasz Napierala #include <sys/racct.h> 73df8bae1dSRodney W. Grimes #include <sys/resourcevar.h> 74*89f6b863SAttilio Rao #include <sys/rwlock.h> 75da61b9a6SAlan Cox #include <sys/sched.h> 76da61b9a6SAlan Cox #include <sys/sf_buf.h> 773aa12267SBruce Evans #include <sys/shm.h> 78efeaf95aSDavid Greenman #include <sys/vmmeter.h> 791005a129SJohn Baldwin #include <sys/sx.h> 80ceb0cf87SJohn Dyson #include <sys/sysctl.h> 81e878d997SKonstantin Belousov #include <sys/_kstack_cache.h> 828a945d10SKonstantin Belousov #include <sys/eventhandler.h> 8326f9a767SRodney W. Grimes #include <sys/kernel.h> 840384fff8SJason Evans #include <sys/ktr.h> 85a2a1c95cSPeter Wemm #include <sys/unistd.h> 8626f9a767SRodney W. Grimes 87df8bae1dSRodney W. Grimes #include <vm/vm.h> 88efeaf95aSDavid Greenman #include <vm/vm_param.h> 89efeaf95aSDavid Greenman #include <vm/pmap.h> 90efeaf95aSDavid Greenman #include <vm/vm_map.h> 91df8bae1dSRodney W. Grimes #include <vm/vm_page.h> 9226f9a767SRodney W. Grimes #include <vm/vm_pageout.h> 93a136efe9SPeter Wemm #include <vm/vm_object.h> 94df8bae1dSRodney W. Grimes #include <vm/vm_kern.h> 95efeaf95aSDavid Greenman #include <vm/vm_extern.h> 96a136efe9SPeter Wemm #include <vm/vm_pager.h> 9792da00bbSMatthew Dillon #include <vm/swap_pager.h> 98efeaf95aSDavid Greenman 992b14f991SJulian Elischer /* 1002b14f991SJulian Elischer * System initialization 1012b14f991SJulian Elischer * 1022b14f991SJulian Elischer * THIS MUST BE THE LAST INITIALIZATION ITEM!!! 1032b14f991SJulian Elischer * 1042b14f991SJulian Elischer * Note: run scheduling should be divorced from the vm system. 1052b14f991SJulian Elischer */ 10611caded3SAlfred Perlstein static void scheduler(void *); 107237fdd78SRobert Watson SYSINIT(scheduler, SI_SUB_RUN_SCHEDULER, SI_ORDER_ANY, scheduler, NULL); 1082b14f991SJulian Elischer 109e50f5c2eSBruce Evans #ifndef NO_SWAPPING 110b61ce5b0SJeff Roberson static int swapout(struct proc *); 111b61ce5b0SJeff Roberson static void swapclear(struct proc *); 112ac45ee97SAlan Cox static void vm_thread_swapin(struct thread *td); 113ac45ee97SAlan Cox static void vm_thread_swapout(struct thread *td); 114e50f5c2eSBruce Evans #endif 115f708ef1bSPoul-Henning Kamp 11643a90f3aSAlan Cox /* 11743a90f3aSAlan Cox * MPSAFE 1182d5c7e45SMatthew Dillon * 1192d5c7e45SMatthew Dillon * WARNING! This code calls vm_map_check_protection() which only checks 1202d5c7e45SMatthew Dillon * the associated vm_map_entry range. It does not determine whether the 1212d5c7e45SMatthew Dillon * contents of the memory is actually readable or writable. In most cases 1222d5c7e45SMatthew Dillon * just checking the vm_map_entry is sufficient within the kernel's address 1232d5c7e45SMatthew Dillon * space. 12443a90f3aSAlan Cox */ 125df8bae1dSRodney W. Grimes int 126df8bae1dSRodney W. Grimes kernacc(addr, len, rw) 127c3dfdfd1SAlfred Perlstein void *addr; 128df8bae1dSRodney W. Grimes int len, rw; 129df8bae1dSRodney W. Grimes { 130df8bae1dSRodney W. Grimes boolean_t rv; 131df8bae1dSRodney W. Grimes vm_offset_t saddr, eaddr; 13202c58685SPoul-Henning Kamp vm_prot_t prot; 133df8bae1dSRodney W. Grimes 134e50f5c2eSBruce Evans KASSERT((rw & ~VM_PROT_ALL) == 0, 13502c58685SPoul-Henning Kamp ("illegal ``rw'' argument to kernacc (%x)\n", rw)); 13675337a56SAlan Cox 13775337a56SAlan Cox if ((vm_offset_t)addr + len > kernel_map->max_offset || 13875337a56SAlan Cox (vm_offset_t)addr + len < (vm_offset_t)addr) 13975337a56SAlan Cox return (FALSE); 14075337a56SAlan Cox 14102c58685SPoul-Henning Kamp prot = rw; 1426cde7a16SDavid Greenman saddr = trunc_page((vm_offset_t)addr); 1436cde7a16SDavid Greenman eaddr = round_page((vm_offset_t)addr + len); 144d8834602SAlan Cox vm_map_lock_read(kernel_map); 145df8bae1dSRodney W. Grimes rv = vm_map_check_protection(kernel_map, saddr, eaddr, prot); 146d8834602SAlan Cox vm_map_unlock_read(kernel_map); 147df8bae1dSRodney W. Grimes return (rv == TRUE); 148df8bae1dSRodney W. Grimes } 149df8bae1dSRodney W. Grimes 15043a90f3aSAlan Cox /* 15143a90f3aSAlan Cox * MPSAFE 1522d5c7e45SMatthew Dillon * 1532d5c7e45SMatthew Dillon * WARNING! This code calls vm_map_check_protection() which only checks 1542d5c7e45SMatthew Dillon * the associated vm_map_entry range. It does not determine whether the 1552d5c7e45SMatthew Dillon * contents of the memory is actually readable or writable. vmapbuf(), 1562d5c7e45SMatthew Dillon * vm_fault_quick(), or copyin()/copout()/su*()/fu*() functions should be 1572d5c7e45SMatthew Dillon * used in conjuction with this call. 15843a90f3aSAlan Cox */ 159df8bae1dSRodney W. Grimes int 160df8bae1dSRodney W. Grimes useracc(addr, len, rw) 161c3dfdfd1SAlfred Perlstein void *addr; 162df8bae1dSRodney W. Grimes int len, rw; 163df8bae1dSRodney W. Grimes { 164df8bae1dSRodney W. Grimes boolean_t rv; 16502c58685SPoul-Henning Kamp vm_prot_t prot; 16605ba50f5SJake Burkholder vm_map_t map; 167df8bae1dSRodney W. Grimes 168e50f5c2eSBruce Evans KASSERT((rw & ~VM_PROT_ALL) == 0, 16902c58685SPoul-Henning Kamp ("illegal ``rw'' argument to useracc (%x)\n", rw)); 17002c58685SPoul-Henning Kamp prot = rw; 17105ba50f5SJake Burkholder map = &curproc->p_vmspace->vm_map; 17205ba50f5SJake Burkholder if ((vm_offset_t)addr + len > vm_map_max(map) || 17305ba50f5SJake Burkholder (vm_offset_t)addr + len < (vm_offset_t)addr) { 17426f9a767SRodney W. Grimes return (FALSE); 17526f9a767SRodney W. Grimes } 176d8834602SAlan Cox vm_map_lock_read(map); 17705ba50f5SJake Burkholder rv = vm_map_check_protection(map, trunc_page((vm_offset_t)addr), 17805ba50f5SJake Burkholder round_page((vm_offset_t)addr + len), prot); 179d8834602SAlan Cox vm_map_unlock_read(map); 180df8bae1dSRodney W. Grimes return (rv == TRUE); 181df8bae1dSRodney W. Grimes } 182df8bae1dSRodney W. Grimes 18316929939SDon Lewis int 184f0ea4612SDon Lewis vslock(void *addr, size_t len) 18516929939SDon Lewis { 186bb734798SDon Lewis vm_offset_t end, last, start; 187bb734798SDon Lewis vm_size_t npages; 188bb734798SDon Lewis int error; 18916929939SDon Lewis 190bb734798SDon Lewis last = (vm_offset_t)addr + len; 191ce8660e3SDon Lewis start = trunc_page((vm_offset_t)addr); 192bb734798SDon Lewis end = round_page(last); 193bb734798SDon Lewis if (last < (vm_offset_t)addr || end < (vm_offset_t)addr) 19416929939SDon Lewis return (EINVAL); 19516929939SDon Lewis npages = atop(end - start); 19616929939SDon Lewis if (npages > vm_page_max_wired) 19716929939SDon Lewis return (ENOMEM); 19816929939SDon Lewis #if 0 19916929939SDon Lewis /* 20016929939SDon Lewis * XXX - not yet 20116929939SDon Lewis * 20216929939SDon Lewis * The limit for transient usage of wired pages should be 20316929939SDon Lewis * larger than for "permanent" wired pages (mlock()). 20416929939SDon Lewis * 20516929939SDon Lewis * Also, the sysctl code, which is the only present user 20616929939SDon Lewis * of vslock(), does a hard loop on EAGAIN. 20716929939SDon Lewis */ 2082feb50bfSAttilio Rao if (npages + cnt.v_wire_count > vm_page_max_wired) 20916929939SDon Lewis return (EAGAIN); 21016929939SDon Lewis #endif 211ce8660e3SDon Lewis error = vm_map_wire(&curproc->p_vmspace->vm_map, start, end, 212d9b2500eSBrian Feldman VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); 213ce8660e3SDon Lewis /* 214ce8660e3SDon Lewis * Return EFAULT on error to match copy{in,out}() behaviour 215ce8660e3SDon Lewis * rather than returning ENOMEM like mlock() would. 216ce8660e3SDon Lewis */ 217ce8660e3SDon Lewis return (error == KERN_SUCCESS ? 0 : EFAULT); 21816929939SDon Lewis } 21916929939SDon Lewis 220ce8660e3SDon Lewis void 221f0ea4612SDon Lewis vsunlock(void *addr, size_t len) 22216929939SDon Lewis { 22316929939SDon Lewis 224ce8660e3SDon Lewis /* Rely on the parameter sanity checks performed by vslock(). */ 225ce8660e3SDon Lewis (void)vm_map_unwire(&curproc->p_vmspace->vm_map, 226ce8660e3SDon Lewis trunc_page((vm_offset_t)addr), round_page((vm_offset_t)addr + len), 22716929939SDon Lewis VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); 22816929939SDon Lewis } 22916929939SDon Lewis 230da61b9a6SAlan Cox /* 231da61b9a6SAlan Cox * Pin the page contained within the given object at the given offset. If the 232da61b9a6SAlan Cox * page is not resident, allocate and load it using the given object's pager. 233da61b9a6SAlan Cox * Return the pinned page if successful; otherwise, return NULL. 234da61b9a6SAlan Cox */ 235da61b9a6SAlan Cox static vm_page_t 236da61b9a6SAlan Cox vm_imgact_hold_page(vm_object_t object, vm_ooffset_t offset) 237da61b9a6SAlan Cox { 238da61b9a6SAlan Cox vm_page_t m, ma[1]; 239da61b9a6SAlan Cox vm_pindex_t pindex; 240da61b9a6SAlan Cox int rv; 241da61b9a6SAlan Cox 242*89f6b863SAttilio Rao VM_OBJECT_WLOCK(object); 243da61b9a6SAlan Cox pindex = OFF_TO_IDX(offset); 244da61b9a6SAlan Cox m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 2450a2e596aSAlan Cox if (m->valid != VM_PAGE_BITS_ALL) { 246da61b9a6SAlan Cox ma[0] = m; 247da61b9a6SAlan Cox rv = vm_pager_get_pages(object, ma, 1, 0); 248da61b9a6SAlan Cox m = vm_page_lookup(object, pindex); 249da61b9a6SAlan Cox if (m == NULL) 250da61b9a6SAlan Cox goto out; 251d1a6e42dSAlan Cox if (rv != VM_PAGER_OK) { 2522965a453SKip Macy vm_page_lock(m); 253da61b9a6SAlan Cox vm_page_free(m); 2542965a453SKip Macy vm_page_unlock(m); 255da61b9a6SAlan Cox m = NULL; 256da61b9a6SAlan Cox goto out; 257da61b9a6SAlan Cox } 258da61b9a6SAlan Cox } 2592965a453SKip Macy vm_page_lock(m); 260da61b9a6SAlan Cox vm_page_hold(m); 2612965a453SKip Macy vm_page_unlock(m); 26266bdd5d6SAlan Cox vm_page_wakeup(m); 263da61b9a6SAlan Cox out: 264*89f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 265da61b9a6SAlan Cox return (m); 266da61b9a6SAlan Cox } 267da61b9a6SAlan Cox 268da61b9a6SAlan Cox /* 269da61b9a6SAlan Cox * Return a CPU private mapping to the page at the given offset within the 270da61b9a6SAlan Cox * given object. The page is pinned before it is mapped. 271da61b9a6SAlan Cox */ 272da61b9a6SAlan Cox struct sf_buf * 273da61b9a6SAlan Cox vm_imgact_map_page(vm_object_t object, vm_ooffset_t offset) 274da61b9a6SAlan Cox { 275da61b9a6SAlan Cox vm_page_t m; 276da61b9a6SAlan Cox 277da61b9a6SAlan Cox m = vm_imgact_hold_page(object, offset); 278da61b9a6SAlan Cox if (m == NULL) 279da61b9a6SAlan Cox return (NULL); 280da61b9a6SAlan Cox sched_pin(); 281da61b9a6SAlan Cox return (sf_buf_alloc(m, SFB_CPUPRIVATE)); 282da61b9a6SAlan Cox } 283da61b9a6SAlan Cox 284da61b9a6SAlan Cox /* 285da61b9a6SAlan Cox * Destroy the given CPU private mapping and unpin the page that it mapped. 286da61b9a6SAlan Cox */ 287da61b9a6SAlan Cox void 288da61b9a6SAlan Cox vm_imgact_unmap_page(struct sf_buf *sf) 289da61b9a6SAlan Cox { 290da61b9a6SAlan Cox vm_page_t m; 291da61b9a6SAlan Cox 292da61b9a6SAlan Cox m = sf_buf_page(sf); 293da61b9a6SAlan Cox sf_buf_free(sf); 294da61b9a6SAlan Cox sched_unpin(); 2952965a453SKip Macy vm_page_lock(m); 296da61b9a6SAlan Cox vm_page_unhold(m); 2972965a453SKip Macy vm_page_unlock(m); 298da61b9a6SAlan Cox } 299da61b9a6SAlan Cox 3001a4fcaebSMarcel Moolenaar void 3011a4fcaebSMarcel Moolenaar vm_sync_icache(vm_map_t map, vm_offset_t va, vm_offset_t sz) 3021a4fcaebSMarcel Moolenaar { 3031a4fcaebSMarcel Moolenaar 3041a4fcaebSMarcel Moolenaar pmap_sync_icache(map->pmap, va, sz); 3051a4fcaebSMarcel Moolenaar } 3061a4fcaebSMarcel Moolenaar 307e878d997SKonstantin Belousov struct kstack_cache_entry *kstack_cache; 3088a945d10SKonstantin Belousov static int kstack_cache_size = 128; 3098a945d10SKonstantin Belousov static int kstacks; 3108a945d10SKonstantin Belousov static struct mtx kstack_cache_mtx; 31125c1e164SAndre Oppermann MTX_SYSINIT(kstack_cache, &kstack_cache_mtx, "kstkch", MTX_DEF); 31225c1e164SAndre Oppermann 3138a945d10SKonstantin Belousov SYSCTL_INT(_vm, OID_AUTO, kstack_cache_size, CTLFLAG_RW, &kstack_cache_size, 0, 3148a945d10SKonstantin Belousov ""); 3158a945d10SKonstantin Belousov SYSCTL_INT(_vm, OID_AUTO, kstacks, CTLFLAG_RD, &kstacks, 0, 3168a945d10SKonstantin Belousov ""); 3178a945d10SKonstantin Belousov 31849a2507bSAlan Cox #ifndef KSTACK_MAX_PAGES 31949a2507bSAlan Cox #define KSTACK_MAX_PAGES 32 32049a2507bSAlan Cox #endif 32149a2507bSAlan Cox 32249a2507bSAlan Cox /* 32349a2507bSAlan Cox * Create the kernel stack (including pcb for i386) for a new thread. 32449a2507bSAlan Cox * This routine directly affects the fork perf for a process and 32549a2507bSAlan Cox * create performance for a thread. 32649a2507bSAlan Cox */ 32789b57fcfSKonstantin Belousov int 32849a2507bSAlan Cox vm_thread_new(struct thread *td, int pages) 32949a2507bSAlan Cox { 33049a2507bSAlan Cox vm_object_t ksobj; 33149a2507bSAlan Cox vm_offset_t ks; 33249a2507bSAlan Cox vm_page_t m, ma[KSTACK_MAX_PAGES]; 3338a945d10SKonstantin Belousov struct kstack_cache_entry *ks_ce; 33449a2507bSAlan Cox int i; 33549a2507bSAlan Cox 33649a2507bSAlan Cox /* Bounds check */ 33749a2507bSAlan Cox if (pages <= 1) 33849a2507bSAlan Cox pages = KSTACK_PAGES; 33949a2507bSAlan Cox else if (pages > KSTACK_MAX_PAGES) 34049a2507bSAlan Cox pages = KSTACK_MAX_PAGES; 3418a945d10SKonstantin Belousov 3428a945d10SKonstantin Belousov if (pages == KSTACK_PAGES) { 3438a945d10SKonstantin Belousov mtx_lock(&kstack_cache_mtx); 3448a945d10SKonstantin Belousov if (kstack_cache != NULL) { 3458a945d10SKonstantin Belousov ks_ce = kstack_cache; 3468a945d10SKonstantin Belousov kstack_cache = ks_ce->next_ks_entry; 3478a945d10SKonstantin Belousov mtx_unlock(&kstack_cache_mtx); 3488a945d10SKonstantin Belousov 3498a945d10SKonstantin Belousov td->td_kstack_obj = ks_ce->ksobj; 3508a945d10SKonstantin Belousov td->td_kstack = (vm_offset_t)ks_ce; 3518a945d10SKonstantin Belousov td->td_kstack_pages = KSTACK_PAGES; 3528a945d10SKonstantin Belousov return (1); 3538a945d10SKonstantin Belousov } 3548a945d10SKonstantin Belousov mtx_unlock(&kstack_cache_mtx); 3558a945d10SKonstantin Belousov } 3568a945d10SKonstantin Belousov 35749a2507bSAlan Cox /* 35849a2507bSAlan Cox * Allocate an object for the kstack. 35949a2507bSAlan Cox */ 36049a2507bSAlan Cox ksobj = vm_object_allocate(OBJT_DEFAULT, pages); 361374ae2a3SJeff Roberson 36249a2507bSAlan Cox /* 36349a2507bSAlan Cox * Get a kernel virtual address for this thread's kstack. 36449a2507bSAlan Cox */ 365ca596a25SJuli Mallett #if defined(__mips__) 366ca596a25SJuli Mallett /* 367ca596a25SJuli Mallett * We need to align the kstack's mapped address to fit within 368ca596a25SJuli Mallett * a single TLB entry. 369ca596a25SJuli Mallett */ 370ca596a25SJuli Mallett ks = kmem_alloc_nofault_space(kernel_map, 371ca596a25SJuli Mallett (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE, VMFS_TLB_ALIGNED_SPACE); 372ca596a25SJuli Mallett #else 37349a2507bSAlan Cox ks = kmem_alloc_nofault(kernel_map, 37449a2507bSAlan Cox (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE); 375ca596a25SJuli Mallett #endif 37689b57fcfSKonstantin Belousov if (ks == 0) { 37789b57fcfSKonstantin Belousov printf("vm_thread_new: kstack allocation failed\n"); 37889b57fcfSKonstantin Belousov vm_object_deallocate(ksobj); 37989b57fcfSKonstantin Belousov return (0); 38089b57fcfSKonstantin Belousov } 38189b57fcfSKonstantin Belousov 3828a945d10SKonstantin Belousov atomic_add_int(&kstacks, 1); 38349a2507bSAlan Cox if (KSTACK_GUARD_PAGES != 0) { 38449a2507bSAlan Cox pmap_qremove(ks, KSTACK_GUARD_PAGES); 38549a2507bSAlan Cox ks += KSTACK_GUARD_PAGES * PAGE_SIZE; 38649a2507bSAlan Cox } 38789b57fcfSKonstantin Belousov td->td_kstack_obj = ksobj; 38849a2507bSAlan Cox td->td_kstack = ks; 38949a2507bSAlan Cox /* 39049a2507bSAlan Cox * Knowing the number of pages allocated is useful when you 39149a2507bSAlan Cox * want to deallocate them. 39249a2507bSAlan Cox */ 39349a2507bSAlan Cox td->td_kstack_pages = pages; 39449a2507bSAlan Cox /* 39549a2507bSAlan Cox * For the length of the stack, link in a real page of ram for each 39649a2507bSAlan Cox * page of stack. 39749a2507bSAlan Cox */ 398*89f6b863SAttilio Rao VM_OBJECT_WLOCK(ksobj); 39949a2507bSAlan Cox for (i = 0; i < pages; i++) { 40049a2507bSAlan Cox /* 40149a2507bSAlan Cox * Get a kernel stack page. 40249a2507bSAlan Cox */ 403ddf4bb37SAlan Cox m = vm_page_grab(ksobj, i, VM_ALLOC_NOBUSY | 40449a2507bSAlan Cox VM_ALLOC_NORMAL | VM_ALLOC_RETRY | VM_ALLOC_WIRED); 40549a2507bSAlan Cox ma[i] = m; 40649a2507bSAlan Cox m->valid = VM_PAGE_BITS_ALL; 40749a2507bSAlan Cox } 408*89f6b863SAttilio Rao VM_OBJECT_WUNLOCK(ksobj); 40949a2507bSAlan Cox pmap_qenter(ks, ma, pages); 41089b57fcfSKonstantin Belousov return (1); 41149a2507bSAlan Cox } 41249a2507bSAlan Cox 4138a945d10SKonstantin Belousov static void 4148a945d10SKonstantin Belousov vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages) 41549a2507bSAlan Cox { 41649a2507bSAlan Cox vm_page_t m; 4178a945d10SKonstantin Belousov int i; 41849a2507bSAlan Cox 4198a945d10SKonstantin Belousov atomic_add_int(&kstacks, -1); 42049a2507bSAlan Cox pmap_qremove(ks, pages); 421*89f6b863SAttilio Rao VM_OBJECT_WLOCK(ksobj); 42249a2507bSAlan Cox for (i = 0; i < pages; i++) { 42349a2507bSAlan Cox m = vm_page_lookup(ksobj, i); 42449a2507bSAlan Cox if (m == NULL) 42549a2507bSAlan Cox panic("vm_thread_dispose: kstack already missing?"); 4262965a453SKip Macy vm_page_lock(m); 42749a2507bSAlan Cox vm_page_unwire(m, 0); 42849a2507bSAlan Cox vm_page_free(m); 4292965a453SKip Macy vm_page_unlock(m); 43049a2507bSAlan Cox } 431*89f6b863SAttilio Rao VM_OBJECT_WUNLOCK(ksobj); 43249a2507bSAlan Cox vm_object_deallocate(ksobj); 43349a2507bSAlan Cox kmem_free(kernel_map, ks - (KSTACK_GUARD_PAGES * PAGE_SIZE), 43449a2507bSAlan Cox (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE); 435c3cf0b47SKonstantin Belousov } 436c3cf0b47SKonstantin Belousov 437c3cf0b47SKonstantin Belousov /* 4388a945d10SKonstantin Belousov * Dispose of a thread's kernel stack. 4398a945d10SKonstantin Belousov */ 4408a945d10SKonstantin Belousov void 4418a945d10SKonstantin Belousov vm_thread_dispose(struct thread *td) 4428a945d10SKonstantin Belousov { 4438a945d10SKonstantin Belousov vm_object_t ksobj; 4448a945d10SKonstantin Belousov vm_offset_t ks; 4458a945d10SKonstantin Belousov struct kstack_cache_entry *ks_ce; 4468a945d10SKonstantin Belousov int pages; 4478a945d10SKonstantin Belousov 4488a945d10SKonstantin Belousov pages = td->td_kstack_pages; 4498a945d10SKonstantin Belousov ksobj = td->td_kstack_obj; 4508a945d10SKonstantin Belousov ks = td->td_kstack; 4518a945d10SKonstantin Belousov td->td_kstack = 0; 4528a945d10SKonstantin Belousov td->td_kstack_pages = 0; 4538a945d10SKonstantin Belousov if (pages == KSTACK_PAGES && kstacks <= kstack_cache_size) { 4548a945d10SKonstantin Belousov ks_ce = (struct kstack_cache_entry *)ks; 4558a945d10SKonstantin Belousov ks_ce->ksobj = ksobj; 4568a945d10SKonstantin Belousov mtx_lock(&kstack_cache_mtx); 4578a945d10SKonstantin Belousov ks_ce->next_ks_entry = kstack_cache; 4588a945d10SKonstantin Belousov kstack_cache = ks_ce; 4598a945d10SKonstantin Belousov mtx_unlock(&kstack_cache_mtx); 4608a945d10SKonstantin Belousov return; 4618a945d10SKonstantin Belousov } 4628a945d10SKonstantin Belousov vm_thread_stack_dispose(ksobj, ks, pages); 4638a945d10SKonstantin Belousov } 4648a945d10SKonstantin Belousov 4658a945d10SKonstantin Belousov static void 4668a945d10SKonstantin Belousov vm_thread_stack_lowmem(void *nulll) 4678a945d10SKonstantin Belousov { 4688a945d10SKonstantin Belousov struct kstack_cache_entry *ks_ce, *ks_ce1; 4698a945d10SKonstantin Belousov 4708a945d10SKonstantin Belousov mtx_lock(&kstack_cache_mtx); 4718a945d10SKonstantin Belousov ks_ce = kstack_cache; 4728a945d10SKonstantin Belousov kstack_cache = NULL; 4738a945d10SKonstantin Belousov mtx_unlock(&kstack_cache_mtx); 4748a945d10SKonstantin Belousov 4758a945d10SKonstantin Belousov while (ks_ce != NULL) { 4768a945d10SKonstantin Belousov ks_ce1 = ks_ce; 4778a945d10SKonstantin Belousov ks_ce = ks_ce->next_ks_entry; 4788a945d10SKonstantin Belousov 4798a945d10SKonstantin Belousov vm_thread_stack_dispose(ks_ce1->ksobj, (vm_offset_t)ks_ce1, 4808a945d10SKonstantin Belousov KSTACK_PAGES); 4818a945d10SKonstantin Belousov } 4828a945d10SKonstantin Belousov } 4838a945d10SKonstantin Belousov 4848a945d10SKonstantin Belousov static void 4858a945d10SKonstantin Belousov kstack_cache_init(void *nulll) 4868a945d10SKonstantin Belousov { 4878a945d10SKonstantin Belousov 4888a945d10SKonstantin Belousov EVENTHANDLER_REGISTER(vm_lowmem, vm_thread_stack_lowmem, NULL, 4898a945d10SKonstantin Belousov EVENTHANDLER_PRI_ANY); 4908a945d10SKonstantin Belousov } 4918a945d10SKonstantin Belousov 4928a945d10SKonstantin Belousov SYSINIT(vm_kstacks, SI_SUB_KTHREAD_INIT, SI_ORDER_ANY, kstack_cache_init, NULL); 4938a945d10SKonstantin Belousov 494ac45ee97SAlan Cox #ifndef NO_SWAPPING 4958a945d10SKonstantin Belousov /* 49649a2507bSAlan Cox * Allow a thread's kernel stack to be paged out. 49749a2507bSAlan Cox */ 498ac45ee97SAlan Cox static void 49949a2507bSAlan Cox vm_thread_swapout(struct thread *td) 50049a2507bSAlan Cox { 50149a2507bSAlan Cox vm_object_t ksobj; 50249a2507bSAlan Cox vm_page_t m; 50349a2507bSAlan Cox int i, pages; 50449a2507bSAlan Cox 505710338e9SMarcel Moolenaar cpu_thread_swapout(td); 50649a2507bSAlan Cox pages = td->td_kstack_pages; 50749a2507bSAlan Cox ksobj = td->td_kstack_obj; 50849a2507bSAlan Cox pmap_qremove(td->td_kstack, pages); 509*89f6b863SAttilio Rao VM_OBJECT_WLOCK(ksobj); 51049a2507bSAlan Cox for (i = 0; i < pages; i++) { 51149a2507bSAlan Cox m = vm_page_lookup(ksobj, i); 51249a2507bSAlan Cox if (m == NULL) 51349a2507bSAlan Cox panic("vm_thread_swapout: kstack already missing?"); 51449a2507bSAlan Cox vm_page_dirty(m); 5152965a453SKip Macy vm_page_lock(m); 51649a2507bSAlan Cox vm_page_unwire(m, 0); 5172965a453SKip Macy vm_page_unlock(m); 51849a2507bSAlan Cox } 519*89f6b863SAttilio Rao VM_OBJECT_WUNLOCK(ksobj); 52049a2507bSAlan Cox } 52149a2507bSAlan Cox 52249a2507bSAlan Cox /* 52349a2507bSAlan Cox * Bring the kernel stack for a specified thread back in. 52449a2507bSAlan Cox */ 525ac45ee97SAlan Cox static void 52649a2507bSAlan Cox vm_thread_swapin(struct thread *td) 52749a2507bSAlan Cox { 52849a2507bSAlan Cox vm_object_t ksobj; 5296fb8c0c1SKonstantin Belousov vm_page_t ma[KSTACK_MAX_PAGES]; 5306fb8c0c1SKonstantin Belousov int i, j, k, pages, rv; 53149a2507bSAlan Cox 53249a2507bSAlan Cox pages = td->td_kstack_pages; 53349a2507bSAlan Cox ksobj = td->td_kstack_obj; 534*89f6b863SAttilio Rao VM_OBJECT_WLOCK(ksobj); 5356fb8c0c1SKonstantin Belousov for (i = 0; i < pages; i++) 5366fb8c0c1SKonstantin Belousov ma[i] = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY | 5376c68c971SAlan Cox VM_ALLOC_WIRED); 5386fb8c0c1SKonstantin Belousov for (i = 0; i < pages; i++) { 5396fb8c0c1SKonstantin Belousov if (ma[i]->valid != VM_PAGE_BITS_ALL) { 5406fb8c0c1SKonstantin Belousov KASSERT(ma[i]->oflags & VPO_BUSY, 5416fb8c0c1SKonstantin Belousov ("lost busy 1")); 5426fb8c0c1SKonstantin Belousov vm_object_pip_add(ksobj, 1); 5436fb8c0c1SKonstantin Belousov for (j = i + 1; j < pages; j++) { 5446fb8c0c1SKonstantin Belousov KASSERT(ma[j]->valid == VM_PAGE_BITS_ALL || 5456fb8c0c1SKonstantin Belousov (ma[j]->oflags & VPO_BUSY), 5466fb8c0c1SKonstantin Belousov ("lost busy 2")); 5476fb8c0c1SKonstantin Belousov if (ma[j]->valid == VM_PAGE_BITS_ALL) 5486fb8c0c1SKonstantin Belousov break; 54949a2507bSAlan Cox } 5506fb8c0c1SKonstantin Belousov rv = vm_pager_get_pages(ksobj, ma + i, j - i, 0); 5516fb8c0c1SKonstantin Belousov if (rv != VM_PAGER_OK) 5526fb8c0c1SKonstantin Belousov panic("vm_thread_swapin: cannot get kstack for proc: %d", 5536fb8c0c1SKonstantin Belousov td->td_proc->p_pid); 5546fb8c0c1SKonstantin Belousov vm_object_pip_wakeup(ksobj); 5556fb8c0c1SKonstantin Belousov for (k = i; k < j; k++) 5566fb8c0c1SKonstantin Belousov ma[k] = vm_page_lookup(ksobj, k); 5576fb8c0c1SKonstantin Belousov vm_page_wakeup(ma[i]); 5586fb8c0c1SKonstantin Belousov } else if (ma[i]->oflags & VPO_BUSY) 5596fb8c0c1SKonstantin Belousov vm_page_wakeup(ma[i]); 56049a2507bSAlan Cox } 561*89f6b863SAttilio Rao VM_OBJECT_WUNLOCK(ksobj); 56249a2507bSAlan Cox pmap_qenter(td->td_kstack, ma, pages); 563710338e9SMarcel Moolenaar cpu_thread_swapin(td); 56449a2507bSAlan Cox } 565ac45ee97SAlan Cox #endif /* !NO_SWAPPING */ 56649a2507bSAlan Cox 567a136efe9SPeter Wemm /* 568df8bae1dSRodney W. Grimes * Implement fork's actions on an address space. 569df8bae1dSRodney W. Grimes * Here we arrange for the address space to be copied or referenced, 570df8bae1dSRodney W. Grimes * allocate a user struct (pcb and kernel stack), then call the 571df8bae1dSRodney W. Grimes * machine-dependent layer to fill those in and make the new process 572a2a1c95cSPeter Wemm * ready to run. The new process is set up so that it returns directly 573a2a1c95cSPeter Wemm * to user mode to avoid stack copying and relocation problems. 574df8bae1dSRodney W. Grimes */ 57589b57fcfSKonstantin Belousov int 57689b57fcfSKonstantin Belousov vm_forkproc(td, p2, td2, vm2, flags) 577b40ce416SJulian Elischer struct thread *td; 578b40ce416SJulian Elischer struct proc *p2; 579079b7badSJulian Elischer struct thread *td2; 58089b57fcfSKonstantin Belousov struct vmspace *vm2; 581a2a1c95cSPeter Wemm int flags; 582df8bae1dSRodney W. Grimes { 583b40ce416SJulian Elischer struct proc *p1 = td->td_proc; 58489b57fcfSKonstantin Belousov int error; 585df8bae1dSRodney W. Grimes 58691c28bfdSLuoqi Chen if ((flags & RFPROC) == 0) { 58791c28bfdSLuoqi Chen /* 58891c28bfdSLuoqi Chen * Divorce the memory, if it is shared, essentially 58991c28bfdSLuoqi Chen * this changes shared memory amongst threads, into 59091c28bfdSLuoqi Chen * COW locally. 59191c28bfdSLuoqi Chen */ 59291c28bfdSLuoqi Chen if ((flags & RFMEM) == 0) { 59391c28bfdSLuoqi Chen if (p1->p_vmspace->vm_refcnt > 1) { 59489b57fcfSKonstantin Belousov error = vmspace_unshare(p1); 59589b57fcfSKonstantin Belousov if (error) 59689b57fcfSKonstantin Belousov return (error); 59791c28bfdSLuoqi Chen } 59891c28bfdSLuoqi Chen } 599079b7badSJulian Elischer cpu_fork(td, p2, td2, flags); 60089b57fcfSKonstantin Belousov return (0); 60191c28bfdSLuoqi Chen } 60291c28bfdSLuoqi Chen 6035856e12eSJohn Dyson if (flags & RFMEM) { 6045856e12eSJohn Dyson p2->p_vmspace = p1->p_vmspace; 6051a276a3fSAlan Cox atomic_add_int(&p1->p_vmspace->vm_refcnt, 1); 6065856e12eSJohn Dyson } 6075856e12eSJohn Dyson 60890ecac61SMatthew Dillon while (vm_page_count_severe()) { 60926f9a767SRodney W. Grimes VM_WAIT; 6100d94caffSDavid Greenman } 61126f9a767SRodney W. Grimes 6125856e12eSJohn Dyson if ((flags & RFMEM) == 0) { 61389b57fcfSKonstantin Belousov p2->p_vmspace = vm2; 614df8bae1dSRodney W. Grimes if (p1->p_vmspace->vm_shm) 615dabee6feSPeter Wemm shmfork(p1, p2); 616a2a1c95cSPeter Wemm } 617df8bae1dSRodney W. Grimes 61839fb8e6bSJulian Elischer /* 619a2a1c95cSPeter Wemm * cpu_fork will copy and update the pcb, set up the kernel stack, 620a2a1c95cSPeter Wemm * and make the child ready to run. 621df8bae1dSRodney W. Grimes */ 622079b7badSJulian Elischer cpu_fork(td, p2, td2, flags); 62389b57fcfSKonstantin Belousov return (0); 624df8bae1dSRodney W. Grimes } 625df8bae1dSRodney W. Grimes 626df8bae1dSRodney W. Grimes /* 627eb30c1c0SPeter Wemm * Called after process has been wait(2)'ed apon and is being reaped. 628eb30c1c0SPeter Wemm * The idea is to reclaim resources that we could not reclaim while 629eb30c1c0SPeter Wemm * the process was still executing. 630eb30c1c0SPeter Wemm */ 631eb30c1c0SPeter Wemm void 632eb30c1c0SPeter Wemm vm_waitproc(p) 633eb30c1c0SPeter Wemm struct proc *p; 634eb30c1c0SPeter Wemm { 635eb30c1c0SPeter Wemm 636582ec34cSAlfred Perlstein vmspace_exitfree(p); /* and clean-out the vmspace */ 637eb30c1c0SPeter Wemm } 638eb30c1c0SPeter Wemm 63926f9a767SRodney W. Grimes void 64026f9a767SRodney W. Grimes faultin(p) 64126f9a767SRodney W. Grimes struct proc *p; 64226f9a767SRodney W. Grimes { 64311edc1e0SJohn Baldwin #ifdef NO_SWAPPING 64411edc1e0SJohn Baldwin 64511edc1e0SJohn Baldwin PROC_LOCK_ASSERT(p, MA_OWNED); 646b61ce5b0SJeff Roberson if ((p->p_flag & P_INMEM) == 0) 64711edc1e0SJohn Baldwin panic("faultin: proc swapped out with NO_SWAPPING!"); 64811edc1e0SJohn Baldwin #else /* !NO_SWAPPING */ 649664f718bSJohn Baldwin struct thread *td; 65026f9a767SRodney W. Grimes 651c96d52a9SJohn Baldwin PROC_LOCK_ASSERT(p, MA_OWNED); 6521d7b9ed2SJulian Elischer /* 6531d7b9ed2SJulian Elischer * If another process is swapping in this process, 6541d7b9ed2SJulian Elischer * just wait until it finishes. 6551d7b9ed2SJulian Elischer */ 656b61ce5b0SJeff Roberson if (p->p_flag & P_SWAPPINGIN) { 657b61ce5b0SJeff Roberson while (p->p_flag & P_SWAPPINGIN) 658b61ce5b0SJeff Roberson msleep(&p->p_flag, &p->p_mtx, PVM, "faultin", 0); 659b61ce5b0SJeff Roberson return; 660b61ce5b0SJeff Roberson } 661b61ce5b0SJeff Roberson if ((p->p_flag & P_INMEM) == 0) { 662664f718bSJohn Baldwin /* 663664f718bSJohn Baldwin * Don't let another thread swap process p out while we are 664664f718bSJohn Baldwin * busy swapping it in. 665664f718bSJohn Baldwin */ 666664f718bSJohn Baldwin ++p->p_lock; 667b61ce5b0SJeff Roberson p->p_flag |= P_SWAPPINGIN; 66845ece682SJohn Baldwin PROC_UNLOCK(p); 66926f9a767SRodney W. Grimes 670b61ce5b0SJeff Roberson /* 671b61ce5b0SJeff Roberson * We hold no lock here because the list of threads 672b61ce5b0SJeff Roberson * can not change while all threads in the process are 673b61ce5b0SJeff Roberson * swapped out. 674b61ce5b0SJeff Roberson */ 675664f718bSJohn Baldwin FOREACH_THREAD_IN_PROC(p, td) 67649a2507bSAlan Cox vm_thread_swapin(td); 67745ece682SJohn Baldwin PROC_LOCK(p); 678b61ce5b0SJeff Roberson swapclear(p); 679258853abSJeff Roberson p->p_swtick = ticks; 68026f9a767SRodney W. Grimes 681b61ce5b0SJeff Roberson wakeup(&p->p_flag); 68226f9a767SRodney W. Grimes 683664f718bSJohn Baldwin /* Allow other threads to swap p out now. */ 68426f9a767SRodney W. Grimes --p->p_lock; 68526f9a767SRodney W. Grimes } 68611edc1e0SJohn Baldwin #endif /* NO_SWAPPING */ 68726f9a767SRodney W. Grimes } 68826f9a767SRodney W. Grimes 689df8bae1dSRodney W. Grimes /* 69026f9a767SRodney W. Grimes * This swapin algorithm attempts to swap-in processes only if there 69126f9a767SRodney W. Grimes * is enough space for them. Of course, if a process waits for a long 69226f9a767SRodney W. Grimes * time, it will be swapped in anyway. 6930384fff8SJason Evans * 69410c447faSAlan Cox * Giant is held on entry. 695df8bae1dSRodney W. Grimes */ 6962b14f991SJulian Elischer /* ARGSUSED*/ 6972b14f991SJulian Elischer static void 698d841aaa7SBruce Evans scheduler(dummy) 699d841aaa7SBruce Evans void *dummy; 700df8bae1dSRodney W. Grimes { 70154d92145SMatthew Dillon struct proc *p; 702e602ba25SJulian Elischer struct thread *td; 703df8bae1dSRodney W. Grimes struct proc *pp; 704258853abSJeff Roberson int slptime; 705258853abSJeff Roberson int swtime; 706df8bae1dSRodney W. Grimes int ppri; 707258853abSJeff Roberson int pri; 708df8bae1dSRodney W. Grimes 709c96d52a9SJohn Baldwin mtx_assert(&Giant, MA_OWNED | MA_NOTRECURSED); 71010c447faSAlan Cox mtx_unlock(&Giant); 7110384fff8SJason Evans 712df8bae1dSRodney W. Grimes loop: 71390ecac61SMatthew Dillon if (vm_page_count_min()) { 7140d94caffSDavid Greenman VM_WAIT; 71590ecac61SMatthew Dillon goto loop; 7160d94caffSDavid Greenman } 71726f9a767SRodney W. Grimes 718df8bae1dSRodney W. Grimes pp = NULL; 719df8bae1dSRodney W. Grimes ppri = INT_MIN; 7201005a129SJohn Baldwin sx_slock(&allproc_lock); 721b40ce416SJulian Elischer FOREACH_PROC_IN_SYSTEM(p) { 722b61ce5b0SJeff Roberson PROC_LOCK(p); 723e806d352SJohn Baldwin if (p->p_state == PRS_NEW || 724e806d352SJohn Baldwin p->p_flag & (P_SWAPPINGOUT | P_SWAPPINGIN | P_INMEM)) { 725b61ce5b0SJeff Roberson PROC_UNLOCK(p); 726e602ba25SJulian Elischer continue; 727e602ba25SJulian Elischer } 728258853abSJeff Roberson swtime = (ticks - p->p_swtick) / hz; 729e602ba25SJulian Elischer FOREACH_THREAD_IN_PROC(p, td) { 7301d7b9ed2SJulian Elischer /* 73171fad9fdSJulian Elischer * An otherwise runnable thread of a process 73271fad9fdSJulian Elischer * swapped out has only the TDI_SWAPPED bit set. 73371fad9fdSJulian Elischer * 7341d7b9ed2SJulian Elischer */ 735982d11f8SJeff Roberson thread_lock(td); 73671fad9fdSJulian Elischer if (td->td_inhibitors == TDI_SWAPPED) { 737258853abSJeff Roberson slptime = (ticks - td->td_slptick) / hz; 738258853abSJeff Roberson pri = swtime + slptime; 739b61ce5b0SJeff Roberson if ((td->td_flags & TDF_SWAPINREQ) == 0) 740fa885116SJulian Elischer pri -= p->p_nice * 8; 74126f9a767SRodney W. Grimes /* 742ad1e7d28SJulian Elischer * if this thread is higher priority 743b40ce416SJulian Elischer * and there is enough space, then select 744b40ce416SJulian Elischer * this process instead of the previous 745b40ce416SJulian Elischer * selection. 74626f9a767SRodney W. Grimes */ 7470d94caffSDavid Greenman if (pri > ppri) { 748df8bae1dSRodney W. Grimes pp = p; 749df8bae1dSRodney W. Grimes ppri = pri; 750df8bae1dSRodney W. Grimes } 751df8bae1dSRodney W. Grimes } 752982d11f8SJeff Roberson thread_unlock(td); 753b40ce416SJulian Elischer } 754b61ce5b0SJeff Roberson PROC_UNLOCK(p); 755df8bae1dSRodney W. Grimes } 7561005a129SJohn Baldwin sx_sunlock(&allproc_lock); 75726f9a767SRodney W. Grimes 758df8bae1dSRodney W. Grimes /* 759a669a6e9SJohn Dyson * Nothing to do, back to sleep. 760df8bae1dSRodney W. Grimes */ 761df8bae1dSRodney W. Grimes if ((p = pp) == NULL) { 76250a57dfbSKonstantin Belousov tsleep(&proc0, PVM, "sched", MAXSLP * hz / 2); 763df8bae1dSRodney W. Grimes goto loop; 764df8bae1dSRodney W. Grimes } 7651d7b9ed2SJulian Elischer PROC_LOCK(p); 7661d7b9ed2SJulian Elischer 7671d7b9ed2SJulian Elischer /* 7681d7b9ed2SJulian Elischer * Another process may be bringing or may have already 7691d7b9ed2SJulian Elischer * brought this process in while we traverse all threads. 7701d7b9ed2SJulian Elischer * Or, this process may even be being swapped out again. 7711d7b9ed2SJulian Elischer */ 772b61ce5b0SJeff Roberson if (p->p_flag & (P_INMEM | P_SWAPPINGOUT | P_SWAPPINGIN)) { 7731d7b9ed2SJulian Elischer PROC_UNLOCK(p); 7741d7b9ed2SJulian Elischer goto loop; 7751d7b9ed2SJulian Elischer } 7761d7b9ed2SJulian Elischer 777df8bae1dSRodney W. Grimes /* 77826f9a767SRodney W. Grimes * We would like to bring someone in. (only if there is space). 779e602ba25SJulian Elischer * [What checks the space? ] 780df8bae1dSRodney W. Grimes */ 78126f9a767SRodney W. Grimes faultin(p); 78245ece682SJohn Baldwin PROC_UNLOCK(p); 783df8bae1dSRodney W. Grimes goto loop; 784df8bae1dSRodney W. Grimes } 785df8bae1dSRodney W. Grimes 786da7bbd2cSJohn Baldwin void 787da7bbd2cSJohn Baldwin kick_proc0(void) 788d13ec713SStephan Uphoff { 789d13ec713SStephan Uphoff 790da7bbd2cSJohn Baldwin wakeup(&proc0); 791d13ec713SStephan Uphoff } 792d13ec713SStephan Uphoff 7935afce282SDavid Greenman #ifndef NO_SWAPPING 7945afce282SDavid Greenman 795ceb0cf87SJohn Dyson /* 796ceb0cf87SJohn Dyson * Swap_idle_threshold1 is the guaranteed swapped in time for a process 797ceb0cf87SJohn Dyson */ 798303b270bSEivind Eklund static int swap_idle_threshold1 = 2; 7992a3eeaa2STom Rhodes SYSCTL_INT(_vm, OID_AUTO, swap_idle_threshold1, CTLFLAG_RW, 8009faaf3b3STom Rhodes &swap_idle_threshold1, 0, "Guaranteed swapped in time for a process"); 801ceb0cf87SJohn Dyson 802ceb0cf87SJohn Dyson /* 803ceb0cf87SJohn Dyson * Swap_idle_threshold2 is the time that a process can be idle before 804ceb0cf87SJohn Dyson * it will be swapped out, if idle swapping is enabled. 805ceb0cf87SJohn Dyson */ 806303b270bSEivind Eklund static int swap_idle_threshold2 = 10; 8072a3eeaa2STom Rhodes SYSCTL_INT(_vm, OID_AUTO, swap_idle_threshold2, CTLFLAG_RW, 8089faaf3b3STom Rhodes &swap_idle_threshold2, 0, "Time before a process will be swapped out"); 809ceb0cf87SJohn Dyson 810df8bae1dSRodney W. Grimes /* 81150a57dfbSKonstantin Belousov * First, if any processes have been sleeping or stopped for at least 81250a57dfbSKonstantin Belousov * "swap_idle_threshold1" seconds, they are swapped out. If, however, 81350a57dfbSKonstantin Belousov * no such processes exist, then the longest-sleeping or stopped 81450a57dfbSKonstantin Belousov * process is swapped out. Finally, and only as a last resort, if 81550a57dfbSKonstantin Belousov * there are no sleeping or stopped processes, the longest-resident 81650a57dfbSKonstantin Belousov * process is swapped out. 817df8bae1dSRodney W. Grimes */ 818df8bae1dSRodney W. Grimes void 8193a2dc656SJohn Dyson swapout_procs(action) 8203a2dc656SJohn Dyson int action; 821df8bae1dSRodney W. Grimes { 82254d92145SMatthew Dillon struct proc *p; 823e602ba25SJulian Elischer struct thread *td; 824df8bae1dSRodney W. Grimes int didswap = 0; 825df8bae1dSRodney W. Grimes 8260d94caffSDavid Greenman retry: 8273a2189d4SJohn Baldwin sx_slock(&allproc_lock); 828e602ba25SJulian Elischer FOREACH_PROC_IN_SYSTEM(p) { 829b18bfc3dSJohn Dyson struct vmspace *vm; 830b40ce416SJulian Elischer int minslptime = 100000; 831258853abSJeff Roberson int slptime; 832b18bfc3dSJohn Dyson 8339eb881f8SSeigo Tanimura /* 834b1f99ebeSSeigo Tanimura * Watch out for a process in 835b1f99ebeSSeigo Tanimura * creation. It may have no 8361c865ac7SJohn Baldwin * address space or lock yet. 8371c865ac7SJohn Baldwin */ 838b61ce5b0SJeff Roberson if (p->p_state == PRS_NEW) 8391c865ac7SJohn Baldwin continue; 8401c865ac7SJohn Baldwin /* 841b1f99ebeSSeigo Tanimura * An aio daemon switches its 842b1f99ebeSSeigo Tanimura * address space while running. 843b1f99ebeSSeigo Tanimura * Perform a quick check whether 844b1f99ebeSSeigo Tanimura * a process has P_SYSTEM. 8459eb881f8SSeigo Tanimura */ 8468f887403SJohn Baldwin if ((p->p_flag & P_SYSTEM) != 0) 847b1f99ebeSSeigo Tanimura continue; 8481c865ac7SJohn Baldwin /* 8491c865ac7SJohn Baldwin * Do not swapout a process that 8501c865ac7SJohn Baldwin * is waiting for VM data 8511c865ac7SJohn Baldwin * structures as there is a possible 8521c865ac7SJohn Baldwin * deadlock. Test this first as 8531c865ac7SJohn Baldwin * this may block. 8541c865ac7SJohn Baldwin * 8551c865ac7SJohn Baldwin * Lock the map until swapout 8561c865ac7SJohn Baldwin * finishes, or a thread of this 8571c865ac7SJohn Baldwin * process may attempt to alter 8581c865ac7SJohn Baldwin * the map. 8591c865ac7SJohn Baldwin */ 86057051fdcSTor Egge vm = vmspace_acquire_ref(p); 86157051fdcSTor Egge if (vm == NULL) 86257051fdcSTor Egge continue; 8639eb881f8SSeigo Tanimura if (!vm_map_trylock(&vm->vm_map)) 8649eb881f8SSeigo Tanimura goto nextproc1; 8659eb881f8SSeigo Tanimura 8665074aecdSJohn Baldwin PROC_LOCK(p); 86769b40456SJohn Baldwin if (p->p_lock != 0 || 8681279572aSDavid Xu (p->p_flag & (P_STOPPED_SINGLE|P_TRACED|P_SYSTEM|P_WEXIT) 8691279572aSDavid Xu ) != 0) { 870374ae2a3SJeff Roberson goto nextproc; 8715074aecdSJohn Baldwin } 87223955314SAlfred Perlstein /* 87323955314SAlfred Perlstein * only aiod changes vmspace, however it will be 87423955314SAlfred Perlstein * skipped because of the if statement above checking 87523955314SAlfred Perlstein * for P_SYSTEM 87623955314SAlfred Perlstein */ 877b61ce5b0SJeff Roberson if ((p->p_flag & (P_INMEM|P_SWAPPINGOUT|P_SWAPPINGIN)) != P_INMEM) 878374ae2a3SJeff Roberson goto nextproc; 87969b40456SJohn Baldwin 880e602ba25SJulian Elischer switch (p->p_state) { 8810d94caffSDavid Greenman default: 882e602ba25SJulian Elischer /* Don't swap out processes in any sort 883e602ba25SJulian Elischer * of 'special' state. */ 8848f887403SJohn Baldwin break; 885df8bae1dSRodney W. Grimes 886e602ba25SJulian Elischer case PRS_NORMAL: 88726f9a767SRodney W. Grimes /* 888bfbfac11SDavid Greenman * do not swapout a realtime process 889b40ce416SJulian Elischer * Check all the thread groups.. 890bfbfac11SDavid Greenman */ 8918460a577SJohn Birrell FOREACH_THREAD_IN_PROC(p, td) { 892b61ce5b0SJeff Roberson thread_lock(td); 893b61ce5b0SJeff Roberson if (PRI_IS_REALTIME(td->td_pri_class)) { 894b61ce5b0SJeff Roberson thread_unlock(td); 895b40ce416SJulian Elischer goto nextproc; 896b61ce5b0SJeff Roberson } 897258853abSJeff Roberson slptime = (ticks - td->td_slptick) / hz; 898bfbfac11SDavid Greenman /* 8999eb881f8SSeigo Tanimura * Guarantee swap_idle_threshold1 900ceb0cf87SJohn Dyson * time in memory. 9010d94caffSDavid Greenman */ 902258853abSJeff Roberson if (slptime < swap_idle_threshold1) { 903b61ce5b0SJeff Roberson thread_unlock(td); 904b40ce416SJulian Elischer goto nextproc; 905b61ce5b0SJeff Roberson } 9069eb881f8SSeigo Tanimura 9071d7b9ed2SJulian Elischer /* 9089eb881f8SSeigo Tanimura * Do not swapout a process if it is 9099eb881f8SSeigo Tanimura * waiting on a critical event of some 9109eb881f8SSeigo Tanimura * kind or there is a thread whose 9119eb881f8SSeigo Tanimura * pageable memory may be accessed. 9121d7b9ed2SJulian Elischer * 9131d7b9ed2SJulian Elischer * This could be refined to support 9141d7b9ed2SJulian Elischer * swapping out a thread. 9151d7b9ed2SJulian Elischer */ 916c5aa6b58SJeff Roberson if (!thread_safetoswapout(td)) { 917b61ce5b0SJeff Roberson thread_unlock(td); 918e602ba25SJulian Elischer goto nextproc; 919b61ce5b0SJeff Roberson } 920ceb0cf87SJohn Dyson /* 921b40ce416SJulian Elischer * If the system is under memory stress, 922b40ce416SJulian Elischer * or if we are swapping 923b40ce416SJulian Elischer * idle processes >= swap_idle_threshold2, 924b40ce416SJulian Elischer * then swap the process out. 925ceb0cf87SJohn Dyson */ 926ceb0cf87SJohn Dyson if (((action & VM_SWAP_NORMAL) == 0) && 927ceb0cf87SJohn Dyson (((action & VM_SWAP_IDLE) == 0) || 928258853abSJeff Roberson (slptime < swap_idle_threshold2))) { 929b61ce5b0SJeff Roberson thread_unlock(td); 930b40ce416SJulian Elischer goto nextproc; 931b61ce5b0SJeff Roberson } 9329eb881f8SSeigo Tanimura 933258853abSJeff Roberson if (minslptime > slptime) 934258853abSJeff Roberson minslptime = slptime; 935b61ce5b0SJeff Roberson thread_unlock(td); 936b40ce416SJulian Elischer } 9370d94caffSDavid Greenman 93811b224dcSDavid Greenman /* 93917d9d0d0SDavid Schultz * If the pageout daemon didn't free enough pages, 94017d9d0d0SDavid Schultz * or if this process is idle and the system is 94117d9d0d0SDavid Schultz * configured to swap proactively, swap it out. 94211b224dcSDavid Greenman */ 943ceb0cf87SJohn Dyson if ((action & VM_SWAP_NORMAL) || 944ceb0cf87SJohn Dyson ((action & VM_SWAP_IDLE) && 945b40ce416SJulian Elischer (minslptime > swap_idle_threshold2))) { 946b61ce5b0SJeff Roberson if (swapout(p) == 0) 947df8bae1dSRodney W. Grimes didswap++; 948664f718bSJohn Baldwin PROC_UNLOCK(p); 9499eb881f8SSeigo Tanimura vm_map_unlock(&vm->vm_map); 9509eb881f8SSeigo Tanimura vmspace_free(vm); 9519eb881f8SSeigo Tanimura sx_sunlock(&allproc_lock); 9520d94caffSDavid Greenman goto retry; 953c96d52a9SJohn Baldwin } 9548f887403SJohn Baldwin } 955374ae2a3SJeff Roberson nextproc: 9569eb881f8SSeigo Tanimura PROC_UNLOCK(p); 9579eb881f8SSeigo Tanimura vm_map_unlock(&vm->vm_map); 9589eb881f8SSeigo Tanimura nextproc1: 9599eb881f8SSeigo Tanimura vmspace_free(vm); 96030171114SPeter Wemm continue; 961ceb0cf87SJohn Dyson } 9621005a129SJohn Baldwin sx_sunlock(&allproc_lock); 96326f9a767SRodney W. Grimes /* 96426f9a767SRodney W. Grimes * If we swapped something out, and another process needed memory, 96526f9a767SRodney W. Grimes * then wakeup the sched process. 96626f9a767SRodney W. Grimes */ 9670d94caffSDavid Greenman if (didswap) 96824a1cce3SDavid Greenman wakeup(&proc0); 969df8bae1dSRodney W. Grimes } 970df8bae1dSRodney W. Grimes 971f708ef1bSPoul-Henning Kamp static void 972b61ce5b0SJeff Roberson swapclear(p) 973b61ce5b0SJeff Roberson struct proc *p; 974b61ce5b0SJeff Roberson { 975b61ce5b0SJeff Roberson struct thread *td; 976b61ce5b0SJeff Roberson 977b61ce5b0SJeff Roberson PROC_LOCK_ASSERT(p, MA_OWNED); 978b61ce5b0SJeff Roberson 979b61ce5b0SJeff Roberson FOREACH_THREAD_IN_PROC(p, td) { 980b61ce5b0SJeff Roberson thread_lock(td); 981b61ce5b0SJeff Roberson td->td_flags |= TDF_INMEM; 982b61ce5b0SJeff Roberson td->td_flags &= ~TDF_SWAPINREQ; 983b61ce5b0SJeff Roberson TD_CLR_SWAPPED(td); 984b61ce5b0SJeff Roberson if (TD_CAN_RUN(td)) 985da7bbd2cSJohn Baldwin if (setrunnable(td)) { 986da7bbd2cSJohn Baldwin #ifdef INVARIANTS 987da7bbd2cSJohn Baldwin /* 988da7bbd2cSJohn Baldwin * XXX: We just cleared TDI_SWAPPED 989da7bbd2cSJohn Baldwin * above and set TDF_INMEM, so this 990da7bbd2cSJohn Baldwin * should never happen. 991da7bbd2cSJohn Baldwin */ 992da7bbd2cSJohn Baldwin panic("not waking up swapper"); 993da7bbd2cSJohn Baldwin #endif 994da7bbd2cSJohn Baldwin } 995b61ce5b0SJeff Roberson thread_unlock(td); 996b61ce5b0SJeff Roberson } 997b61ce5b0SJeff Roberson p->p_flag &= ~(P_SWAPPINGIN|P_SWAPPINGOUT); 998b61ce5b0SJeff Roberson p->p_flag |= P_INMEM; 999b61ce5b0SJeff Roberson } 1000b61ce5b0SJeff Roberson 1001b61ce5b0SJeff Roberson static int 1002df8bae1dSRodney W. Grimes swapout(p) 100354d92145SMatthew Dillon struct proc *p; 1004df8bae1dSRodney W. Grimes { 1005b40ce416SJulian Elischer struct thread *td; 1006df8bae1dSRodney W. Grimes 1007ea754954SJohn Baldwin PROC_LOCK_ASSERT(p, MA_OWNED); 1008d3a34985SJohn Dyson #if defined(SWAP_DEBUG) 1009d3a34985SJohn Dyson printf("swapping out %d\n", p->p_pid); 1010d3a34985SJohn Dyson #endif 10111d7b9ed2SJulian Elischer 10121d7b9ed2SJulian Elischer /* 10139eb881f8SSeigo Tanimura * The states of this process and its threads may have changed 10149eb881f8SSeigo Tanimura * by now. Assuming that there is only one pageout daemon thread, 10159eb881f8SSeigo Tanimura * this process should still be in memory. 10169eb881f8SSeigo Tanimura */ 1017b61ce5b0SJeff Roberson KASSERT((p->p_flag & (P_INMEM|P_SWAPPINGOUT|P_SWAPPINGIN)) == P_INMEM, 10189eb881f8SSeigo Tanimura ("swapout: lost a swapout race?")); 10199eb881f8SSeigo Tanimura 1020df8bae1dSRodney W. Grimes /* 102126f9a767SRodney W. Grimes * remember the process resident count 1022df8bae1dSRodney W. Grimes */ 1023b1028ad1SLuoqi Chen p->p_vmspace->vm_swrss = vmspace_resident_count(p->p_vmspace); 1024b61ce5b0SJeff Roberson /* 1025b61ce5b0SJeff Roberson * Check and mark all threads before we proceed. 1026b61ce5b0SJeff Roberson */ 1027b61ce5b0SJeff Roberson p->p_flag &= ~P_INMEM; 1028b61ce5b0SJeff Roberson p->p_flag |= P_SWAPPINGOUT; 1029982d11f8SJeff Roberson FOREACH_THREAD_IN_PROC(p, td) { 1030982d11f8SJeff Roberson thread_lock(td); 1031b61ce5b0SJeff Roberson if (!thread_safetoswapout(td)) { 1032b61ce5b0SJeff Roberson thread_unlock(td); 1033b61ce5b0SJeff Roberson swapclear(p); 1034b61ce5b0SJeff Roberson return (EBUSY); 1035b61ce5b0SJeff Roberson } 1036b61ce5b0SJeff Roberson td->td_flags &= ~TDF_INMEM; 1037664f718bSJohn Baldwin TD_SET_SWAPPED(td); 1038982d11f8SJeff Roberson thread_unlock(td); 1039982d11f8SJeff Roberson } 1040b61ce5b0SJeff Roberson td = FIRST_THREAD_IN_PROC(p); 1041b61ce5b0SJeff Roberson ++td->td_ru.ru_nswap; 1042b61ce5b0SJeff Roberson PROC_UNLOCK(p); 104326f9a767SRodney W. Grimes 1044b61ce5b0SJeff Roberson /* 1045b61ce5b0SJeff Roberson * This list is stable because all threads are now prevented from 1046b61ce5b0SJeff Roberson * running. The list is only modified in the context of a running 1047b61ce5b0SJeff Roberson * thread in this process. 1048b61ce5b0SJeff Roberson */ 1049664f718bSJohn Baldwin FOREACH_THREAD_IN_PROC(p, td) 105049a2507bSAlan Cox vm_thread_swapout(td); 1051664f718bSJohn Baldwin 1052664f718bSJohn Baldwin PROC_LOCK(p); 1053b61ce5b0SJeff Roberson p->p_flag &= ~P_SWAPPINGOUT; 1054258853abSJeff Roberson p->p_swtick = ticks; 1055b61ce5b0SJeff Roberson return (0); 1056df8bae1dSRodney W. Grimes } 10575afce282SDavid Greenman #endif /* !NO_SWAPPING */ 1058