xref: /freebsd/sys/vm/vm_kern.c (revision ca596a25f07f94a05515cd150b1d7e6f81c1894d)
160727d8bSWarner Losh /*-
2df8bae1dSRodney W. Grimes  * Copyright (c) 1991, 1993
3df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
4df8bae1dSRodney W. Grimes  *
5df8bae1dSRodney W. Grimes  * This code is derived from software contributed to Berkeley by
6df8bae1dSRodney W. Grimes  * The Mach Operating System project at Carnegie-Mellon University.
7df8bae1dSRodney W. Grimes  *
8df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
9df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
10df8bae1dSRodney W. Grimes  * are met:
11df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
12df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
13df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
14df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
15df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
16df8bae1dSRodney W. Grimes  * 4. Neither the name of the University nor the names of its contributors
17df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
18df8bae1dSRodney W. Grimes  *    without specific prior written permission.
19df8bae1dSRodney W. Grimes  *
20df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
31df8bae1dSRodney W. Grimes  *
323c4dd356SDavid Greenman  *	from: @(#)vm_kern.c	8.3 (Berkeley) 1/12/94
33df8bae1dSRodney W. Grimes  *
34df8bae1dSRodney W. Grimes  *
35df8bae1dSRodney W. Grimes  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
36df8bae1dSRodney W. Grimes  * All rights reserved.
37df8bae1dSRodney W. Grimes  *
38df8bae1dSRodney W. Grimes  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
39df8bae1dSRodney W. Grimes  *
40df8bae1dSRodney W. Grimes  * Permission to use, copy, modify and distribute this software and
41df8bae1dSRodney W. Grimes  * its documentation is hereby granted, provided that both the copyright
42df8bae1dSRodney W. Grimes  * notice and this permission notice appear in all copies of the
43df8bae1dSRodney W. Grimes  * software, derivative works or modified versions, and any portions
44df8bae1dSRodney W. Grimes  * thereof, and that both notices appear in supporting documentation.
45df8bae1dSRodney W. Grimes  *
46df8bae1dSRodney W. Grimes  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
47df8bae1dSRodney W. Grimes  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
48df8bae1dSRodney W. Grimes  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
49df8bae1dSRodney W. Grimes  *
50df8bae1dSRodney W. Grimes  * Carnegie Mellon requests users of this software to return to
51df8bae1dSRodney W. Grimes  *
52df8bae1dSRodney W. Grimes  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
53df8bae1dSRodney W. Grimes  *  School of Computer Science
54df8bae1dSRodney W. Grimes  *  Carnegie Mellon University
55df8bae1dSRodney W. Grimes  *  Pittsburgh PA 15213-3890
56df8bae1dSRodney W. Grimes  *
57df8bae1dSRodney W. Grimes  * any improvements or extensions that they make and grant Carnegie the
58df8bae1dSRodney W. Grimes  * rights to redistribute these changes.
59df8bae1dSRodney W. Grimes  */
60df8bae1dSRodney W. Grimes 
61df8bae1dSRodney W. Grimes /*
62df8bae1dSRodney W. Grimes  *	Kernel memory management.
63df8bae1dSRodney W. Grimes  */
64df8bae1dSRodney W. Grimes 
65874651b1SDavid E. O'Brien #include <sys/cdefs.h>
66874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$");
67874651b1SDavid E. O'Brien 
68df8bae1dSRodney W. Grimes #include <sys/param.h>
69df8bae1dSRodney W. Grimes #include <sys/systm.h>
7060363fb9SLuigi Rizzo #include <sys/kernel.h>		/* for ticks and hz */
710f2c2ce0SPawel Jakub Dawidek #include <sys/eventhandler.h>
72fb919e4dSMark Murray #include <sys/lock.h>
73fb919e4dSMark Murray #include <sys/mutex.h>
74f23b4c91SGarrett Wollman #include <sys/proc.h>
75a1f6d91cSDavid Greenman #include <sys/malloc.h>
7686f08737SRobert Watson #include <sys/sysctl.h>
77df8bae1dSRodney W. Grimes 
78df8bae1dSRodney W. Grimes #include <vm/vm.h>
79efeaf95aSDavid Greenman #include <vm/vm_param.h>
80efeaf95aSDavid Greenman #include <vm/pmap.h>
81efeaf95aSDavid Greenman #include <vm/vm_map.h>
82efeaf95aSDavid Greenman #include <vm/vm_object.h>
83df8bae1dSRodney W. Grimes #include <vm/vm_page.h>
84df8bae1dSRodney W. Grimes #include <vm/vm_pageout.h>
859b4288a3SBruce Evans #include <vm/vm_extern.h>
860f2c2ce0SPawel Jakub Dawidek #include <vm/uma.h>
87df8bae1dSRodney W. Grimes 
885b0a7408SJohn Dyson vm_map_t kernel_map=0;
895b0a7408SJohn Dyson vm_map_t kmem_map=0;
905b0a7408SJohn Dyson vm_map_t exec_map=0;
91cebde069SMike Silbersack vm_map_t pipe_map;
925b0a7408SJohn Dyson vm_map_t buffer_map=0;
93f23b4c91SGarrett Wollman 
94df8bae1dSRodney W. Grimes /*
95a839bdc8SDmitrij Tejblum  *	kmem_alloc_nofault:
96a839bdc8SDmitrij Tejblum  *
97b77c2bcdSAlan Cox  *	Allocate a virtual address range with no underlying object and
98b77c2bcdSAlan Cox  *	no initial mapping to physical memory.  Any mapping from this
99b77c2bcdSAlan Cox  *	range to physical memory must be explicitly created prior to
100b77c2bcdSAlan Cox  *	its use, typically with pmap_qenter().  Any attempt to create
101b77c2bcdSAlan Cox  *	a mapping on demand through vm_fault() will result in a panic.
102a839bdc8SDmitrij Tejblum  */
103a839bdc8SDmitrij Tejblum vm_offset_t
104a839bdc8SDmitrij Tejblum kmem_alloc_nofault(map, size)
105a839bdc8SDmitrij Tejblum 	vm_map_t map;
106030f2369SAlfred Perlstein 	vm_size_t size;
107a839bdc8SDmitrij Tejblum {
108a839bdc8SDmitrij Tejblum 	vm_offset_t addr;
109030f2369SAlfred Perlstein 	int result;
110a839bdc8SDmitrij Tejblum 
111a839bdc8SDmitrij Tejblum 	size = round_page(size);
112a839bdc8SDmitrij Tejblum 	addr = vm_map_min(map);
1133202ed75SAlan Cox 	result = vm_map_find(map, NULL, 0, &addr, size, VMFS_ANY_SPACE,
1143202ed75SAlan Cox 	    VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
115a839bdc8SDmitrij Tejblum 	if (result != KERN_SUCCESS) {
116a839bdc8SDmitrij Tejblum 		return (0);
117a839bdc8SDmitrij Tejblum 	}
118a839bdc8SDmitrij Tejblum 	return (addr);
119a839bdc8SDmitrij Tejblum }
120a839bdc8SDmitrij Tejblum 
121a839bdc8SDmitrij Tejblum /*
122*ca596a25SJuli Mallett  *	kmem_alloc_nofault_space:
123*ca596a25SJuli Mallett  *
124*ca596a25SJuli Mallett  *	Allocate a virtual address range with no underlying object and
125*ca596a25SJuli Mallett  *	no initial mapping to physical memory within the specified
126*ca596a25SJuli Mallett  *	address space.  Any mapping from this range to physical memory
127*ca596a25SJuli Mallett  *	must be explicitly created prior to its use, typically with
128*ca596a25SJuli Mallett  *	pmap_qenter().  Any attempt to create a mapping on demand
129*ca596a25SJuli Mallett  *	through vm_fault() will result in a panic.
130*ca596a25SJuli Mallett  */
131*ca596a25SJuli Mallett vm_offset_t
132*ca596a25SJuli Mallett kmem_alloc_nofault_space(map, size, find_space)
133*ca596a25SJuli Mallett 	vm_map_t map;
134*ca596a25SJuli Mallett 	vm_size_t size;
135*ca596a25SJuli Mallett 	int find_space;
136*ca596a25SJuli Mallett {
137*ca596a25SJuli Mallett 	vm_offset_t addr;
138*ca596a25SJuli Mallett 	int result;
139*ca596a25SJuli Mallett 
140*ca596a25SJuli Mallett 	size = round_page(size);
141*ca596a25SJuli Mallett 	addr = vm_map_min(map);
142*ca596a25SJuli Mallett 	result = vm_map_find(map, NULL, 0, &addr, size, find_space,
143*ca596a25SJuli Mallett 	    VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
144*ca596a25SJuli Mallett 	if (result != KERN_SUCCESS) {
145*ca596a25SJuli Mallett 		return (0);
146*ca596a25SJuli Mallett 	}
147*ca596a25SJuli Mallett 	return (addr);
148*ca596a25SJuli Mallett }
149*ca596a25SJuli Mallett 
150*ca596a25SJuli Mallett /*
151df8bae1dSRodney W. Grimes  *	Allocate wired-down memory in the kernel's address map
152df8bae1dSRodney W. Grimes  *	or a submap.
153df8bae1dSRodney W. Grimes  */
1540d94caffSDavid Greenman vm_offset_t
1550d94caffSDavid Greenman kmem_alloc(map, size)
156030f2369SAlfred Perlstein 	vm_map_t map;
157030f2369SAlfred Perlstein 	vm_size_t size;
158df8bae1dSRodney W. Grimes {
159df8bae1dSRodney W. Grimes 	vm_offset_t addr;
160030f2369SAlfred Perlstein 	vm_offset_t offset;
161df8bae1dSRodney W. Grimes 	vm_offset_t i;
162df8bae1dSRodney W. Grimes 
163df8bae1dSRodney W. Grimes 	size = round_page(size);
164df8bae1dSRodney W. Grimes 
165df8bae1dSRodney W. Grimes 	/*
1660d94caffSDavid Greenman 	 * Use the kernel object for wired-down kernel pages. Assume that no
1670d94caffSDavid Greenman 	 * region of the kernel object is referenced more than once.
168df8bae1dSRodney W. Grimes 	 */
169df8bae1dSRodney W. Grimes 
170df8bae1dSRodney W. Grimes 	/*
1710d94caffSDavid Greenman 	 * Locate sufficient space in the map.  This will give us the final
1720d94caffSDavid Greenman 	 * virtual address for the new memory, and thus will tell us the
1730d94caffSDavid Greenman 	 * offset within the kernel map.
174df8bae1dSRodney W. Grimes 	 */
175df8bae1dSRodney W. Grimes 	vm_map_lock(map);
176e47ed70bSJohn Dyson 	if (vm_map_findspace(map, vm_map_min(map), size, &addr)) {
177df8bae1dSRodney W. Grimes 		vm_map_unlock(map);
178df8bae1dSRodney W. Grimes 		return (0);
179df8bae1dSRodney W. Grimes 	}
180df8bae1dSRodney W. Grimes 	offset = addr - VM_MIN_KERNEL_ADDRESS;
181df8bae1dSRodney W. Grimes 	vm_object_reference(kernel_object);
182bd7e5f99SJohn Dyson 	vm_map_insert(map, kernel_object, offset, addr, addr + size,
183bd7e5f99SJohn Dyson 		VM_PROT_ALL, VM_PROT_ALL, 0);
184df8bae1dSRodney W. Grimes 	vm_map_unlock(map);
185df8bae1dSRodney W. Grimes 
186df8bae1dSRodney W. Grimes 	/*
1870d94caffSDavid Greenman 	 * Guarantee that there are pages already in this object before
1888f101a2fSJonathan Mini 	 * calling vm_map_wire.  This is to prevent the following
1890d94caffSDavid Greenman 	 * scenario:
190df8bae1dSRodney W. Grimes 	 *
1910d94caffSDavid Greenman 	 * 1) Threads have swapped out, so that there is a pager for the
1920d94caffSDavid Greenman 	 * kernel_object. 2) The kmsg zone is empty, and so we are
1938f101a2fSJonathan Mini 	 * kmem_allocing a new page for it. 3) vm_map_wire calls vm_fault;
1940d94caffSDavid Greenman 	 * there is no page, but there is a pager, so we call
1950d94caffSDavid Greenman 	 * pager_data_request.  But the kmsg zone is empty, so we must
1960d94caffSDavid Greenman 	 * kmem_alloc. 4) goto 1 5) Even if the kmsg zone is not empty: when
1970d94caffSDavid Greenman 	 * we get the data back from the pager, it will be (very stale)
1980d94caffSDavid Greenman 	 * non-zero data.  kmem_alloc is defined to return zero-filled memory.
199df8bae1dSRodney W. Grimes 	 *
2000d94caffSDavid Greenman 	 * We're intentionally not activating the pages we allocate to prevent a
2018f101a2fSJonathan Mini 	 * race with page-out.  vm_map_wire will wire the pages.
202df8bae1dSRodney W. Grimes 	 */
20349c06616SAlan Cox 	VM_OBJECT_LOCK(kernel_object);
204df8bae1dSRodney W. Grimes 	for (i = 0; i < size; i += PAGE_SIZE) {
205df8bae1dSRodney W. Grimes 		vm_page_t mem;
206df8bae1dSRodney W. Grimes 
20795461b45SJohn Dyson 		mem = vm_page_grab(kernel_object, OFF_TO_IDX(offset + i),
208ddf4bb37SAlan Cox 		    VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_RETRY);
2097fb0c17eSDavid Greenman 		mem->valid = VM_PAGE_BITS_ALL;
2109f5c801bSAlan Cox 		KASSERT((mem->flags & PG_UNMANAGED) != 0,
2119f5c801bSAlan Cox 		    ("kmem_alloc: page %p is managed", mem));
212df8bae1dSRodney W. Grimes 	}
21349c06616SAlan Cox 	VM_OBJECT_UNLOCK(kernel_object);
214df8bae1dSRodney W. Grimes 
215df8bae1dSRodney W. Grimes 	/*
216df8bae1dSRodney W. Grimes 	 * And finally, mark the data as non-pageable.
217df8bae1dSRodney W. Grimes 	 */
218abd498aaSBruce M Simpson 	(void) vm_map_wire(map, addr, addr + size,
219abd498aaSBruce M Simpson 	    VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES);
220df8bae1dSRodney W. Grimes 
221df8bae1dSRodney W. Grimes 	return (addr);
222df8bae1dSRodney W. Grimes }
223df8bae1dSRodney W. Grimes 
224df8bae1dSRodney W. Grimes /*
225df8bae1dSRodney W. Grimes  *	kmem_free:
226df8bae1dSRodney W. Grimes  *
227df8bae1dSRodney W. Grimes  *	Release a region of kernel virtual memory allocated
228df8bae1dSRodney W. Grimes  *	with kmem_alloc, and return the physical pages
229df8bae1dSRodney W. Grimes  *	associated with that region.
2301c7c3c6aSMatthew Dillon  *
2311c7c3c6aSMatthew Dillon  *	This routine may not block on kernel maps.
232df8bae1dSRodney W. Grimes  */
2330d94caffSDavid Greenman void
2340d94caffSDavid Greenman kmem_free(map, addr, size)
235df8bae1dSRodney W. Grimes 	vm_map_t map;
236030f2369SAlfred Perlstein 	vm_offset_t addr;
237df8bae1dSRodney W. Grimes 	vm_size_t size;
238df8bae1dSRodney W. Grimes {
23923955314SAlfred Perlstein 
240df8bae1dSRodney W. Grimes 	(void) vm_map_remove(map, trunc_page(addr), round_page(addr + size));
241df8bae1dSRodney W. Grimes }
242df8bae1dSRodney W. Grimes 
243df8bae1dSRodney W. Grimes /*
244df8bae1dSRodney W. Grimes  *	kmem_suballoc:
245df8bae1dSRodney W. Grimes  *
246df8bae1dSRodney W. Grimes  *	Allocates a map to manage a subrange
247df8bae1dSRodney W. Grimes  *	of the kernel virtual address space.
248df8bae1dSRodney W. Grimes  *
249df8bae1dSRodney W. Grimes  *	Arguments are as follows:
250df8bae1dSRodney W. Grimes  *
251df8bae1dSRodney W. Grimes  *	parent		Map to take range from
252df8bae1dSRodney W. Grimes  *	min, max	Returned endpoints of map
253030f2369SAlfred Perlstein  *	size		Size of range to find
2543202ed75SAlan Cox  *	superpage_align	Request that min is superpage aligned
255df8bae1dSRodney W. Grimes  */
2560d94caffSDavid Greenman vm_map_t
2573202ed75SAlan Cox kmem_suballoc(vm_map_t parent, vm_offset_t *min, vm_offset_t *max,
2583202ed75SAlan Cox     vm_size_t size, boolean_t superpage_align)
259df8bae1dSRodney W. Grimes {
2606e4f51d1SAlfred Perlstein 	int ret;
261df8bae1dSRodney W. Grimes 	vm_map_t result;
26223955314SAlfred Perlstein 
263df8bae1dSRodney W. Grimes 	size = round_page(size);
264df8bae1dSRodney W. Grimes 
2652bc24aa9SAlan Cox 	*min = vm_map_min(parent);
2663202ed75SAlan Cox 	ret = vm_map_find(parent, NULL, 0, min, size, superpage_align ?
2673364c323SKonstantin Belousov 	    VMFS_ALIGNED_SPACE : VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL,
2683364c323SKonstantin Belousov 	    MAP_ACC_NO_CHARGE);
26924dedba9SAlan Cox 	if (ret != KERN_SUCCESS)
27024dedba9SAlan Cox 		panic("kmem_suballoc: bad status return of %d", ret);
271df8bae1dSRodney W. Grimes 	*max = *min + size;
2722d8acc0fSJohn Dyson 	result = vm_map_create(vm_map_pmap(parent), *min, *max);
273df8bae1dSRodney W. Grimes 	if (result == NULL)
274df8bae1dSRodney W. Grimes 		panic("kmem_suballoc: cannot create submap");
2756e4f51d1SAlfred Perlstein 	if (vm_map_submap(parent, *min, *max, result) != KERN_SUCCESS)
276df8bae1dSRodney W. Grimes 		panic("kmem_suballoc: unable to change range to submap");
277df8bae1dSRodney W. Grimes 	return (result);
278df8bae1dSRodney W. Grimes }
279df8bae1dSRodney W. Grimes 
280df8bae1dSRodney W. Grimes /*
2811c7c3c6aSMatthew Dillon  *	kmem_malloc:
2821c7c3c6aSMatthew Dillon  *
283df8bae1dSRodney W. Grimes  * 	Allocate wired-down memory in the kernel's address map for the higher
284df8bae1dSRodney W. Grimes  * 	level kernel memory allocator (kern/kern_malloc.c).  We cannot use
285df8bae1dSRodney W. Grimes  * 	kmem_alloc() because we may need to allocate memory at interrupt
286df8bae1dSRodney W. Grimes  * 	level where we cannot block (canwait == FALSE).
287df8bae1dSRodney W. Grimes  *
288df8bae1dSRodney W. Grimes  * 	This routine has its own private kernel submap (kmem_map) and object
289df8bae1dSRodney W. Grimes  * 	(kmem_object).  This, combined with the fact that only malloc uses
290df8bae1dSRodney W. Grimes  * 	this routine, ensures that we will never block in map or object waits.
291df8bae1dSRodney W. Grimes  *
292df8bae1dSRodney W. Grimes  * 	We don't worry about expanding the map (adding entries) since entries
293df8bae1dSRodney W. Grimes  * 	for wired maps are statically allocated.
2941c7c3c6aSMatthew Dillon  *
29508442f8aSBosko Milekic  *	`map' is ONLY allowed to be kmem_map or one of the mbuf submaps to
29608442f8aSBosko Milekic  *	which we never free.
297df8bae1dSRodney W. Grimes  */
298df8bae1dSRodney W. Grimes vm_offset_t
2991c7c3c6aSMatthew Dillon kmem_malloc(map, size, flags)
300030f2369SAlfred Perlstein 	vm_map_t map;
301030f2369SAlfred Perlstein 	vm_size_t size;
3021c7c3c6aSMatthew Dillon 	int flags;
303df8bae1dSRodney W. Grimes {
304030f2369SAlfred Perlstein 	vm_offset_t offset, i;
305655c3490SKonstantin Belousov 	vm_map_entry_t entry;
306df8bae1dSRodney W. Grimes 	vm_offset_t addr;
307df8bae1dSRodney W. Grimes 	vm_page_t m;
3081e081f88SJeff Roberson 	int pflags;
309df8bae1dSRodney W. Grimes 
310df8bae1dSRodney W. Grimes 	size = round_page(size);
311df8bae1dSRodney W. Grimes 	addr = vm_map_min(map);
312df8bae1dSRodney W. Grimes 
313df8bae1dSRodney W. Grimes 	/*
3140d94caffSDavid Greenman 	 * Locate sufficient space in the map.  This will give us the final
3150d94caffSDavid Greenman 	 * virtual address for the new memory, and thus will tell us the
3160d94caffSDavid Greenman 	 * offset within the kernel map.
317df8bae1dSRodney W. Grimes 	 */
318df8bae1dSRodney W. Grimes 	vm_map_lock(map);
319e47ed70bSJohn Dyson 	if (vm_map_findspace(map, vm_map_min(map), size, &addr)) {
320df8bae1dSRodney W. Grimes 		vm_map_unlock(map);
3210f2c2ce0SPawel Jakub Dawidek                 if ((flags & M_NOWAIT) == 0) {
32279c2840dSPawel Jakub Dawidek 			for (i = 0; i < 8; i++) {
3230f2c2ce0SPawel Jakub Dawidek 				EVENTHANDLER_INVOKE(vm_lowmem, 0);
3240f2c2ce0SPawel Jakub Dawidek 				uma_reclaim();
3250f2c2ce0SPawel Jakub Dawidek 				vm_map_lock(map);
32679c2840dSPawel Jakub Dawidek 				if (vm_map_findspace(map, vm_map_min(map),
32779c2840dSPawel Jakub Dawidek 				    size, &addr) == 0) {
32879c2840dSPawel Jakub Dawidek 					break;
32979c2840dSPawel Jakub Dawidek 				}
3300f2c2ce0SPawel Jakub Dawidek 				vm_map_unlock(map);
33179c2840dSPawel Jakub Dawidek 				tsleep(&i, 0, "nokva", (hz / 4) * (i + 1));
33279c2840dSPawel Jakub Dawidek 			}
33379c2840dSPawel Jakub Dawidek 			if (i == 8) {
3343efc015bSPeter Wemm 				panic("kmem_malloc(%ld): kmem_map too small: %ld total allocated",
3353efc015bSPeter Wemm 				    (long)size, (long)map->size);
3360f2c2ce0SPawel Jakub Dawidek 			}
3370f2c2ce0SPawel Jakub Dawidek 		} else {
338f31c239dSAlan Cox 			return (0);
339df8bae1dSRodney W. Grimes 		}
3400f2c2ce0SPawel Jakub Dawidek 	}
3410891ef4cSJohn Dyson 	offset = addr - VM_MIN_KERNEL_ADDRESS;
342df8bae1dSRodney W. Grimes 	vm_object_reference(kmem_object);
343bd7e5f99SJohn Dyson 	vm_map_insert(map, kmem_object, offset, addr, addr + size,
344bd7e5f99SJohn Dyson 		VM_PROT_ALL, VM_PROT_ALL, 0);
345df8bae1dSRodney W. Grimes 
34695f24639SJeff Roberson 	if ((flags & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
347a623fedeSAlan Cox 		pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED;
34895f24639SJeff Roberson 	else
349a623fedeSAlan Cox 		pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED;
35095f24639SJeff Roberson 
35195f24639SJeff Roberson 	if (flags & M_ZERO)
35295f24639SJeff Roberson 		pflags |= VM_ALLOC_ZERO;
35395f24639SJeff Roberson 
354acbff226SAlan Cox 	VM_OBJECT_LOCK(kmem_object);
3551e081f88SJeff Roberson 	for (i = 0; i < size; i += PAGE_SIZE) {
3561e081f88SJeff Roberson retry:
35795f24639SJeff Roberson 		m = vm_page_alloc(kmem_object, OFF_TO_IDX(offset + i), pflags);
358df8bae1dSRodney W. Grimes 
359df8bae1dSRodney W. Grimes 		/*
3600d94caffSDavid Greenman 		 * Ran out of space, free everything up and return. Don't need
3610d94caffSDavid Greenman 		 * to lock page queues here as we know that the pages we got
3620d94caffSDavid Greenman 		 * aren't on any queues.
363df8bae1dSRodney W. Grimes 		 */
364df8bae1dSRodney W. Grimes 		if (m == NULL) {
3651c7c3c6aSMatthew Dillon 			if ((flags & M_NOWAIT) == 0) {
366acbff226SAlan Cox 				VM_OBJECT_UNLOCK(kmem_object);
367c7003c69SAlan Cox 				vm_map_unlock(map);
368b18bfc3dSJohn Dyson 				VM_WAIT;
369c7003c69SAlan Cox 				vm_map_lock(map);
370acbff226SAlan Cox 				VM_OBJECT_LOCK(kmem_object);
371b18bfc3dSJohn Dyson 				goto retry;
372b18bfc3dSJohn Dyson 			}
373ff91d780STor Egge 			/*
374ff91d780STor Egge 			 * Free the pages before removing the map entry.
375ff91d780STor Egge 			 * They are already marked busy.  Calling
376ff91d780STor Egge 			 * vm_map_delete before the pages has been freed or
377ff91d780STor Egge 			 * unbusied will cause a deadlock.
378ff91d780STor Egge 			 */
379ff91d780STor Egge 			while (i != 0) {
380ff91d780STor Egge 				i -= PAGE_SIZE;
381ff91d780STor Egge 				m = vm_page_lookup(kmem_object,
382ff91d780STor Egge 						   OFF_TO_IDX(offset + i));
38357123de6SAlan Cox 				vm_page_lock_queues();
384a623fedeSAlan Cox 				vm_page_unwire(m, 0);
385ff91d780STor Egge 				vm_page_free(m);
38657123de6SAlan Cox 				vm_page_unlock_queues();
387ff91d780STor Egge 			}
388acbff226SAlan Cox 			VM_OBJECT_UNLOCK(kmem_object);
389655c3490SKonstantin Belousov 			vm_map_delete(map, addr, addr + size);
390df8bae1dSRodney W. Grimes 			vm_map_unlock(map);
391f31c239dSAlan Cox 			return (0);
392df8bae1dSRodney W. Grimes 		}
3931e081f88SJeff Roberson 		if (flags & M_ZERO && (m->flags & PG_ZERO) == 0)
394fff6062aSAlan Cox 			pmap_zero_page(m);
39549c06616SAlan Cox 		m->valid = VM_PAGE_BITS_ALL;
3969f5c801bSAlan Cox 		KASSERT((m->flags & PG_UNMANAGED) != 0,
3979f5c801bSAlan Cox 		    ("kmem_malloc: page %p is managed", m));
398df8bae1dSRodney W. Grimes 	}
399acbff226SAlan Cox 	VM_OBJECT_UNLOCK(kmem_object);
400df8bae1dSRodney W. Grimes 
401df8bae1dSRodney W. Grimes 	/*
4020d94caffSDavid Greenman 	 * Mark map entry as non-pageable. Assert: vm_map_insert() will never
4030d94caffSDavid Greenman 	 * be able to extend the previous entry so there will be a new entry
4040d94caffSDavid Greenman 	 * exactly corresponding to this address range and it will have
4050d94caffSDavid Greenman 	 * wired_count == 0.
406df8bae1dSRodney W. Grimes 	 */
407df8bae1dSRodney W. Grimes 	if (!vm_map_lookup_entry(map, addr, &entry) ||
408df8bae1dSRodney W. Grimes 	    entry->start != addr || entry->end != addr + size ||
409c7003c69SAlan Cox 	    entry->wired_count != 0)
410df8bae1dSRodney W. Grimes 		panic("kmem_malloc: entry not found or misaligned");
411c7003c69SAlan Cox 	entry->wired_count = 1;
412df8bae1dSRodney W. Grimes 
413ff5dcf25SAlan Cox 	/*
414ff5dcf25SAlan Cox 	 * At this point, the kmem_object must be unlocked because
415ff5dcf25SAlan Cox 	 * vm_map_simplify_entry() calls vm_object_deallocate(), which
416ff5dcf25SAlan Cox 	 * locks the kmem_object.
417ff5dcf25SAlan Cox 	 */
418b7b2aac2SJohn Dyson 	vm_map_simplify_entry(map, entry);
419b7b2aac2SJohn Dyson 
420df8bae1dSRodney W. Grimes 	/*
4210f3b612aSAlan Cox 	 * Loop thru pages, entering them in the pmap.
422df8bae1dSRodney W. Grimes 	 */
423acbff226SAlan Cox 	VM_OBJECT_LOCK(kmem_object);
424ff5dcf25SAlan Cox 	for (i = 0; i < size; i += PAGE_SIZE) {
425a316d390SJohn Dyson 		m = vm_page_lookup(kmem_object, OFF_TO_IDX(offset + i));
4261c7c3c6aSMatthew Dillon 		/*
4271c7c3c6aSMatthew Dillon 		 * Because this is kernel_pmap, this call will not block.
4281c7c3c6aSMatthew Dillon 		 */
429eb2a0517SAlan Cox 		pmap_enter(kernel_pmap, addr + i, VM_PROT_ALL, m, VM_PROT_ALL,
430eb2a0517SAlan Cox 		    TRUE);
43166bdd5d6SAlan Cox 		vm_page_wakeup(m);
432df8bae1dSRodney W. Grimes 	}
433ff5dcf25SAlan Cox 	VM_OBJECT_UNLOCK(kmem_object);
434df8bae1dSRodney W. Grimes 	vm_map_unlock(map);
435df8bae1dSRodney W. Grimes 
436df8bae1dSRodney W. Grimes 	return (addr);
437df8bae1dSRodney W. Grimes }
438df8bae1dSRodney W. Grimes 
439df8bae1dSRodney W. Grimes /*
4401c7c3c6aSMatthew Dillon  *	kmem_alloc_wait:
441df8bae1dSRodney W. Grimes  *
442df8bae1dSRodney W. Grimes  *	Allocates pageable memory from a sub-map of the kernel.  If the submap
443df8bae1dSRodney W. Grimes  *	has no room, the caller sleeps waiting for more memory in the submap.
444df8bae1dSRodney W. Grimes  *
4451c7c3c6aSMatthew Dillon  *	This routine may block.
446df8bae1dSRodney W. Grimes  */
4470d94caffSDavid Greenman vm_offset_t
4480d94caffSDavid Greenman kmem_alloc_wait(map, size)
449df8bae1dSRodney W. Grimes 	vm_map_t map;
450df8bae1dSRodney W. Grimes 	vm_size_t size;
451df8bae1dSRodney W. Grimes {
452df8bae1dSRodney W. Grimes 	vm_offset_t addr;
45323955314SAlfred Perlstein 
454df8bae1dSRodney W. Grimes 	size = round_page(size);
4553364c323SKonstantin Belousov 	if (!swap_reserve(size))
4563364c323SKonstantin Belousov 		return (0);
457df8bae1dSRodney W. Grimes 
458df8bae1dSRodney W. Grimes 	for (;;) {
459df8bae1dSRodney W. Grimes 		/*
4600d94caffSDavid Greenman 		 * To make this work for more than one map, use the map's lock
4610d94caffSDavid Greenman 		 * to lock out sleepers/wakers.
462df8bae1dSRodney W. Grimes 		 */
463df8bae1dSRodney W. Grimes 		vm_map_lock(map);
464e47ed70bSJohn Dyson 		if (vm_map_findspace(map, vm_map_min(map), size, &addr) == 0)
465df8bae1dSRodney W. Grimes 			break;
466df8bae1dSRodney W. Grimes 		/* no space now; see if we can ever get space */
467df8bae1dSRodney W. Grimes 		if (vm_map_max(map) - vm_map_min(map) < size) {
468df8bae1dSRodney W. Grimes 			vm_map_unlock(map);
4693364c323SKonstantin Belousov 			swap_release(size);
470df8bae1dSRodney W. Grimes 			return (0);
471df8bae1dSRodney W. Grimes 		}
4729688f931SAlan Cox 		map->needs_wakeup = TRUE;
4738ce2d00aSPawel Jakub Dawidek 		vm_map_unlock_and_wait(map, 0);
474df8bae1dSRodney W. Grimes 	}
4753364c323SKonstantin Belousov 	vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_ALL,
4763364c323SKonstantin Belousov 	    VM_PROT_ALL, MAP_ACC_CHARGED);
477df8bae1dSRodney W. Grimes 	vm_map_unlock(map);
478df8bae1dSRodney W. Grimes 	return (addr);
479df8bae1dSRodney W. Grimes }
480df8bae1dSRodney W. Grimes 
481df8bae1dSRodney W. Grimes /*
4821c7c3c6aSMatthew Dillon  *	kmem_free_wakeup:
483df8bae1dSRodney W. Grimes  *
48424a1cce3SDavid Greenman  *	Returns memory to a submap of the kernel, and wakes up any processes
485df8bae1dSRodney W. Grimes  *	waiting for memory in that map.
486df8bae1dSRodney W. Grimes  */
4870d94caffSDavid Greenman void
4880d94caffSDavid Greenman kmem_free_wakeup(map, addr, size)
489df8bae1dSRodney W. Grimes 	vm_map_t map;
490df8bae1dSRodney W. Grimes 	vm_offset_t addr;
491df8bae1dSRodney W. Grimes 	vm_size_t size;
492df8bae1dSRodney W. Grimes {
49323955314SAlfred Perlstein 
494df8bae1dSRodney W. Grimes 	vm_map_lock(map);
495655c3490SKonstantin Belousov 	(void) vm_map_delete(map, trunc_page(addr), round_page(addr + size));
4969688f931SAlan Cox 	if (map->needs_wakeup) {
4979688f931SAlan Cox 		map->needs_wakeup = FALSE;
4989688f931SAlan Cox 		vm_map_wakeup(map);
4999688f931SAlan Cox 	}
500df8bae1dSRodney W. Grimes 	vm_map_unlock(map);
501df8bae1dSRodney W. Grimes }
502df8bae1dSRodney W. Grimes 
503df8bae1dSRodney W. Grimes /*
5041c7c3c6aSMatthew Dillon  * 	kmem_init:
5051c7c3c6aSMatthew Dillon  *
5061c7c3c6aSMatthew Dillon  *	Create the kernel map; insert a mapping covering kernel text,
5071c7c3c6aSMatthew Dillon  *	data, bss, and all space allocated thus far (`boostrap' data).  The
5081c7c3c6aSMatthew Dillon  *	new map will thus map the range between VM_MIN_KERNEL_ADDRESS and
5091c7c3c6aSMatthew Dillon  *	`start' as allocated, and the range between `start' and `end' as free.
510df8bae1dSRodney W. Grimes  */
5110d94caffSDavid Greenman void
5120d94caffSDavid Greenman kmem_init(start, end)
513df8bae1dSRodney W. Grimes 	vm_offset_t start, end;
514df8bae1dSRodney W. Grimes {
515030f2369SAlfred Perlstein 	vm_map_t m;
516df8bae1dSRodney W. Grimes 
5172d8acc0fSJohn Dyson 	m = vm_map_create(kernel_pmap, VM_MIN_KERNEL_ADDRESS, end);
518c9267356SAlan Cox 	m->system_map = 1;
519df8bae1dSRodney W. Grimes 	vm_map_lock(m);
520df8bae1dSRodney W. Grimes 	/* N.B.: cannot use kgdb to debug, starting with this assignment ... */
521df8bae1dSRodney W. Grimes 	kernel_map = m;
522c9267356SAlan Cox 	(void) vm_map_insert(m, NULL, (vm_ooffset_t) 0,
5235cfa90e9SAlan Cox #ifdef __amd64__
5245cfa90e9SAlan Cox 	    KERNBASE,
5255cfa90e9SAlan Cox #else
5265cfa90e9SAlan Cox 	    VM_MIN_KERNEL_ADDRESS,
5275cfa90e9SAlan Cox #endif
5285cfa90e9SAlan Cox 	    start, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
529df8bae1dSRodney W. Grimes 	/* ... and ending with the completion of the above `insert' */
530df8bae1dSRodney W. Grimes 	vm_map_unlock(m);
531df8bae1dSRodney W. Grimes }
53286f08737SRobert Watson 
5339309e63cSRobert Watson #ifdef DIAGNOSTIC
53486f08737SRobert Watson /*
53586f08737SRobert Watson  * Allow userspace to directly trigger the VM drain routine for testing
53686f08737SRobert Watson  * purposes.
53786f08737SRobert Watson  */
53886f08737SRobert Watson static int
53986f08737SRobert Watson debug_vm_lowmem(SYSCTL_HANDLER_ARGS)
54086f08737SRobert Watson {
54186f08737SRobert Watson 	int error, i;
54286f08737SRobert Watson 
54386f08737SRobert Watson 	i = 0;
54486f08737SRobert Watson 	error = sysctl_handle_int(oidp, &i, 0, req);
54586f08737SRobert Watson 	if (error)
54686f08737SRobert Watson 		return (error);
54786f08737SRobert Watson 	if (i)
54886f08737SRobert Watson 		EVENTHANDLER_INVOKE(vm_lowmem, 0);
54986f08737SRobert Watson 	return (0);
55086f08737SRobert Watson }
55186f08737SRobert Watson 
55286f08737SRobert Watson SYSCTL_PROC(_debug, OID_AUTO, vm_lowmem, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
55386f08737SRobert Watson     debug_vm_lowmem, "I", "set to trigger vm_lowmem event");
5549309e63cSRobert Watson #endif
555