xref: /freebsd/sys/vm/vm_kern.c (revision 955c8cbb4960e6cf3602de144b1b9154a5092968)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * The Mach Operating System project at Carnegie-Mellon University.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 4. Neither the name of the University nor the names of its contributors
17  *    may be used to endorse or promote products derived from this software
18  *    without specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  *
32  *	from: @(#)vm_kern.c	8.3 (Berkeley) 1/12/94
33  *
34  *
35  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
36  * All rights reserved.
37  *
38  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
39  *
40  * Permission to use, copy, modify and distribute this software and
41  * its documentation is hereby granted, provided that both the copyright
42  * notice and this permission notice appear in all copies of the
43  * software, derivative works or modified versions, and any portions
44  * thereof, and that both notices appear in supporting documentation.
45  *
46  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
47  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
48  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
49  *
50  * Carnegie Mellon requests users of this software to return to
51  *
52  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
53  *  School of Computer Science
54  *  Carnegie Mellon University
55  *  Pittsburgh PA 15213-3890
56  *
57  * any improvements or extensions that they make and grant Carnegie the
58  * rights to redistribute these changes.
59  */
60 
61 /*
62  *	Kernel memory management.
63  */
64 
65 #include <sys/cdefs.h>
66 __FBSDID("$FreeBSD$");
67 
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/kernel.h>		/* for ticks and hz */
71 #include <sys/eventhandler.h>
72 #include <sys/lock.h>
73 #include <sys/mutex.h>
74 #include <sys/proc.h>
75 #include <sys/malloc.h>
76 #include <sys/sysctl.h>
77 
78 #include <vm/vm.h>
79 #include <vm/vm_param.h>
80 #include <vm/pmap.h>
81 #include <vm/vm_map.h>
82 #include <vm/vm_object.h>
83 #include <vm/vm_page.h>
84 #include <vm/vm_pageout.h>
85 #include <vm/vm_extern.h>
86 #include <vm/uma.h>
87 
88 vm_map_t kernel_map=0;
89 vm_map_t kmem_map=0;
90 vm_map_t exec_map=0;
91 vm_map_t pipe_map;
92 vm_map_t buffer_map=0;
93 
94 const void *zero_region;
95 CTASSERT((ZERO_REGION_SIZE & PAGE_MASK) == 0);
96 
97 /*
98  *	kmem_alloc_nofault:
99  *
100  *	Allocate a virtual address range with no underlying object and
101  *	no initial mapping to physical memory.  Any mapping from this
102  *	range to physical memory must be explicitly created prior to
103  *	its use, typically with pmap_qenter().  Any attempt to create
104  *	a mapping on demand through vm_fault() will result in a panic.
105  */
106 vm_offset_t
107 kmem_alloc_nofault(map, size)
108 	vm_map_t map;
109 	vm_size_t size;
110 {
111 	vm_offset_t addr;
112 	int result;
113 
114 	size = round_page(size);
115 	addr = vm_map_min(map);
116 	result = vm_map_find(map, NULL, 0, &addr, size, VMFS_ANY_SPACE,
117 	    VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
118 	if (result != KERN_SUCCESS) {
119 		return (0);
120 	}
121 	return (addr);
122 }
123 
124 /*
125  *	kmem_alloc_nofault_space:
126  *
127  *	Allocate a virtual address range with no underlying object and
128  *	no initial mapping to physical memory within the specified
129  *	address space.  Any mapping from this range to physical memory
130  *	must be explicitly created prior to its use, typically with
131  *	pmap_qenter().  Any attempt to create a mapping on demand
132  *	through vm_fault() will result in a panic.
133  */
134 vm_offset_t
135 kmem_alloc_nofault_space(map, size, find_space)
136 	vm_map_t map;
137 	vm_size_t size;
138 	int find_space;
139 {
140 	vm_offset_t addr;
141 	int result;
142 
143 	size = round_page(size);
144 	addr = vm_map_min(map);
145 	result = vm_map_find(map, NULL, 0, &addr, size, find_space,
146 	    VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
147 	if (result != KERN_SUCCESS) {
148 		return (0);
149 	}
150 	return (addr);
151 }
152 
153 /*
154  *	Allocate wired-down memory in the kernel's address map
155  *	or a submap.
156  */
157 vm_offset_t
158 kmem_alloc(map, size)
159 	vm_map_t map;
160 	vm_size_t size;
161 {
162 	vm_offset_t addr;
163 	vm_offset_t offset;
164 
165 	size = round_page(size);
166 
167 	/*
168 	 * Use the kernel object for wired-down kernel pages. Assume that no
169 	 * region of the kernel object is referenced more than once.
170 	 */
171 
172 	/*
173 	 * Locate sufficient space in the map.  This will give us the final
174 	 * virtual address for the new memory, and thus will tell us the
175 	 * offset within the kernel map.
176 	 */
177 	vm_map_lock(map);
178 	if (vm_map_findspace(map, vm_map_min(map), size, &addr)) {
179 		vm_map_unlock(map);
180 		return (0);
181 	}
182 	offset = addr - VM_MIN_KERNEL_ADDRESS;
183 	vm_object_reference(kernel_object);
184 	vm_map_insert(map, kernel_object, offset, addr, addr + size,
185 		VM_PROT_ALL, VM_PROT_ALL, 0);
186 	vm_map_unlock(map);
187 
188 	/*
189 	 * And finally, mark the data as non-pageable.
190 	 */
191 	(void) vm_map_wire(map, addr, addr + size,
192 	    VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES);
193 
194 	return (addr);
195 }
196 
197 /*
198  *	Allocates a region from the kernel address map and physical pages
199  *	within the specified address range to the kernel object.  Creates a
200  *	wired mapping from this region to these pages, and returns the
201  *	region's starting virtual address.  The allocated pages are not
202  *	necessarily physically contiguous.  If M_ZERO is specified through the
203  *	given flags, then the pages are zeroed before they are mapped.
204  */
205 vm_offset_t
206 kmem_alloc_attr(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
207     vm_paddr_t high, vm_memattr_t memattr)
208 {
209 	vm_object_t object = kernel_object;
210 	vm_offset_t addr;
211 	vm_ooffset_t end_offset, offset;
212 	vm_page_t m;
213 	int pflags, tries;
214 
215 	size = round_page(size);
216 	vm_map_lock(map);
217 	if (vm_map_findspace(map, vm_map_min(map), size, &addr)) {
218 		vm_map_unlock(map);
219 		return (0);
220 	}
221 	offset = addr - VM_MIN_KERNEL_ADDRESS;
222 	vm_object_reference(object);
223 	vm_map_insert(map, object, offset, addr, addr + size, VM_PROT_ALL,
224 	    VM_PROT_ALL, 0);
225 	pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY;
226 	VM_OBJECT_LOCK(object);
227 	end_offset = offset + size;
228 	for (; offset < end_offset; offset += PAGE_SIZE) {
229 		tries = 0;
230 retry:
231 		m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags, 1,
232 		    low, high, PAGE_SIZE, 0, memattr);
233 		if (m == NULL) {
234 			VM_OBJECT_UNLOCK(object);
235 			if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
236 				vm_map_unlock(map);
237 				vm_pageout_grow_cache(tries, low, high);
238 				vm_map_lock(map);
239 				VM_OBJECT_LOCK(object);
240 				tries++;
241 				goto retry;
242 			}
243 
244 			/*
245 			 * Since the pages that were allocated by any previous
246 			 * iterations of this loop are not busy, they can be
247 			 * freed by vm_object_page_remove(), which is called
248 			 * by vm_map_delete().
249 			 */
250 			vm_map_delete(map, addr, addr + size);
251 			vm_map_unlock(map);
252 			return (0);
253 		}
254 		if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
255 			pmap_zero_page(m);
256 		m->valid = VM_PAGE_BITS_ALL;
257 	}
258 	VM_OBJECT_UNLOCK(object);
259 	vm_map_unlock(map);
260 	vm_map_wire(map, addr, addr + size, VM_MAP_WIRE_SYSTEM |
261 	    VM_MAP_WIRE_NOHOLES);
262 	return (addr);
263 }
264 
265 /*
266  *	Allocates a region from the kernel address map and physically
267  *	contiguous pages within the specified address range to the kernel
268  *	object.  Creates a wired mapping from this region to these pages, and
269  *	returns the region's starting virtual address.  If M_ZERO is specified
270  *	through the given flags, then the pages are zeroed before they are
271  *	mapped.
272  */
273 vm_offset_t
274 kmem_alloc_contig(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low,
275     vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
276     vm_memattr_t memattr)
277 {
278 	vm_object_t object = kernel_object;
279 	vm_offset_t addr;
280 	vm_ooffset_t offset;
281 	vm_page_t end_m, m;
282 	int pflags, tries;
283 
284 	size = round_page(size);
285 	vm_map_lock(map);
286 	if (vm_map_findspace(map, vm_map_min(map), size, &addr)) {
287 		vm_map_unlock(map);
288 		return (0);
289 	}
290 	offset = addr - VM_MIN_KERNEL_ADDRESS;
291 	vm_object_reference(object);
292 	vm_map_insert(map, object, offset, addr, addr + size, VM_PROT_ALL,
293 	    VM_PROT_ALL, 0);
294 	pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY;
295 	VM_OBJECT_LOCK(object);
296 	tries = 0;
297 retry:
298 	m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags,
299 	    atop(size), low, high, alignment, boundary, memattr);
300 	if (m == NULL) {
301 		VM_OBJECT_UNLOCK(object);
302 		if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
303 			vm_map_unlock(map);
304 			vm_pageout_grow_cache(tries, low, high);
305 			vm_map_lock(map);
306 			VM_OBJECT_LOCK(object);
307 			tries++;
308 			goto retry;
309 		}
310 		vm_map_delete(map, addr, addr + size);
311 		vm_map_unlock(map);
312 		return (0);
313 	}
314 	end_m = m + atop(size);
315 	for (; m < end_m; m++) {
316 		if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
317 			pmap_zero_page(m);
318 		m->valid = VM_PAGE_BITS_ALL;
319 	}
320 	VM_OBJECT_UNLOCK(object);
321 	vm_map_unlock(map);
322 	vm_map_wire(map, addr, addr + size, VM_MAP_WIRE_SYSTEM |
323 	    VM_MAP_WIRE_NOHOLES);
324 	return (addr);
325 }
326 
327 /*
328  *	kmem_free:
329  *
330  *	Release a region of kernel virtual memory allocated
331  *	with kmem_alloc, and return the physical pages
332  *	associated with that region.
333  *
334  *	This routine may not block on kernel maps.
335  */
336 void
337 kmem_free(map, addr, size)
338 	vm_map_t map;
339 	vm_offset_t addr;
340 	vm_size_t size;
341 {
342 
343 	(void) vm_map_remove(map, trunc_page(addr), round_page(addr + size));
344 }
345 
346 /*
347  *	kmem_suballoc:
348  *
349  *	Allocates a map to manage a subrange
350  *	of the kernel virtual address space.
351  *
352  *	Arguments are as follows:
353  *
354  *	parent		Map to take range from
355  *	min, max	Returned endpoints of map
356  *	size		Size of range to find
357  *	superpage_align	Request that min is superpage aligned
358  */
359 vm_map_t
360 kmem_suballoc(vm_map_t parent, vm_offset_t *min, vm_offset_t *max,
361     vm_size_t size, boolean_t superpage_align)
362 {
363 	int ret;
364 	vm_map_t result;
365 
366 	size = round_page(size);
367 
368 	*min = vm_map_min(parent);
369 	ret = vm_map_find(parent, NULL, 0, min, size, superpage_align ?
370 	    VMFS_ALIGNED_SPACE : VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL,
371 	    MAP_ACC_NO_CHARGE);
372 	if (ret != KERN_SUCCESS)
373 		panic("kmem_suballoc: bad status return of %d", ret);
374 	*max = *min + size;
375 	result = vm_map_create(vm_map_pmap(parent), *min, *max);
376 	if (result == NULL)
377 		panic("kmem_suballoc: cannot create submap");
378 	if (vm_map_submap(parent, *min, *max, result) != KERN_SUCCESS)
379 		panic("kmem_suballoc: unable to change range to submap");
380 	return (result);
381 }
382 
383 /*
384  *	kmem_malloc:
385  *
386  * 	Allocate wired-down memory in the kernel's address map for the higher
387  * 	level kernel memory allocator (kern/kern_malloc.c).  We cannot use
388  * 	kmem_alloc() because we may need to allocate memory at interrupt
389  * 	level where we cannot block (canwait == FALSE).
390  *
391  * 	This routine has its own private kernel submap (kmem_map) and object
392  * 	(kmem_object).  This, combined with the fact that only malloc uses
393  * 	this routine, ensures that we will never block in map or object waits.
394  *
395  * 	We don't worry about expanding the map (adding entries) since entries
396  * 	for wired maps are statically allocated.
397  *
398  *	`map' is ONLY allowed to be kmem_map or one of the mbuf submaps to
399  *	which we never free.
400  */
401 vm_offset_t
402 kmem_malloc(map, size, flags)
403 	vm_map_t map;
404 	vm_size_t size;
405 	int flags;
406 {
407 	vm_offset_t addr;
408 	int i, rv;
409 
410 	size = round_page(size);
411 	addr = vm_map_min(map);
412 
413 	/*
414 	 * Locate sufficient space in the map.  This will give us the final
415 	 * virtual address for the new memory, and thus will tell us the
416 	 * offset within the kernel map.
417 	 */
418 	vm_map_lock(map);
419 	if (vm_map_findspace(map, vm_map_min(map), size, &addr)) {
420 		vm_map_unlock(map);
421                 if ((flags & M_NOWAIT) == 0) {
422 			for (i = 0; i < 8; i++) {
423 				EVENTHANDLER_INVOKE(vm_lowmem, 0);
424 				uma_reclaim();
425 				vm_map_lock(map);
426 				if (vm_map_findspace(map, vm_map_min(map),
427 				    size, &addr) == 0) {
428 					break;
429 				}
430 				vm_map_unlock(map);
431 				tsleep(&i, 0, "nokva", (hz / 4) * (i + 1));
432 			}
433 			if (i == 8) {
434 				panic("kmem_malloc(%ld): kmem_map too small: %ld total allocated",
435 				    (long)size, (long)map->size);
436 			}
437 		} else {
438 			return (0);
439 		}
440 	}
441 
442 	rv = kmem_back(map, addr, size, flags);
443 	vm_map_unlock(map);
444 	return (rv == KERN_SUCCESS ? addr : 0);
445 }
446 
447 /*
448  *	kmem_back:
449  *
450  *	Allocate physical pages for the specified virtual address range.
451  */
452 int
453 kmem_back(vm_map_t map, vm_offset_t addr, vm_size_t size, int flags)
454 {
455 	vm_offset_t offset, i;
456 	vm_map_entry_t entry;
457 	vm_page_t m;
458 	int pflags;
459 	boolean_t found;
460 
461 	KASSERT(vm_map_locked(map), ("kmem_back: map %p is not locked", map));
462 	offset = addr - VM_MIN_KERNEL_ADDRESS;
463 	vm_object_reference(kmem_object);
464 	vm_map_insert(map, kmem_object, offset, addr, addr + size,
465 	    VM_PROT_ALL, VM_PROT_ALL, 0);
466 
467 	/*
468 	 * Assert: vm_map_insert() will never be able to extend the
469 	 * previous entry so vm_map_lookup_entry() will find a new
470 	 * entry exactly corresponding to this address range and it
471 	 * will have wired_count == 0.
472 	 */
473 	found = vm_map_lookup_entry(map, addr, &entry);
474 	KASSERT(found && entry->start == addr && entry->end == addr + size &&
475 	    entry->wired_count == 0 && (entry->eflags & MAP_ENTRY_IN_TRANSITION)
476 	    == 0, ("kmem_back: entry not found or misaligned"));
477 
478 	pflags = malloc2vm_flags(flags) | VM_ALLOC_WIRED;
479 
480 	VM_OBJECT_LOCK(kmem_object);
481 	for (i = 0; i < size; i += PAGE_SIZE) {
482 retry:
483 		m = vm_page_alloc(kmem_object, OFF_TO_IDX(offset + i), pflags);
484 
485 		/*
486 		 * Ran out of space, free everything up and return. Don't need
487 		 * to lock page queues here as we know that the pages we got
488 		 * aren't on any queues.
489 		 */
490 		if (m == NULL) {
491 			if ((flags & M_NOWAIT) == 0) {
492 				VM_OBJECT_UNLOCK(kmem_object);
493 				entry->eflags |= MAP_ENTRY_IN_TRANSITION;
494 				vm_map_unlock(map);
495 				VM_WAIT;
496 				vm_map_lock(map);
497 				KASSERT(
498 (entry->eflags & (MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_NEEDS_WAKEUP)) ==
499 				    MAP_ENTRY_IN_TRANSITION,
500 				    ("kmem_back: volatile entry"));
501 				entry->eflags &= ~MAP_ENTRY_IN_TRANSITION;
502 				VM_OBJECT_LOCK(kmem_object);
503 				goto retry;
504 			}
505 			/*
506 			 * Free the pages before removing the map entry.
507 			 * They are already marked busy.  Calling
508 			 * vm_map_delete before the pages has been freed or
509 			 * unbusied will cause a deadlock.
510 			 */
511 			while (i != 0) {
512 				i -= PAGE_SIZE;
513 				m = vm_page_lookup(kmem_object,
514 						   OFF_TO_IDX(offset + i));
515 				vm_page_unwire(m, 0);
516 				vm_page_free(m);
517 			}
518 			VM_OBJECT_UNLOCK(kmem_object);
519 			vm_map_delete(map, addr, addr + size);
520 			return (KERN_NO_SPACE);
521 		}
522 		if (flags & M_ZERO && (m->flags & PG_ZERO) == 0)
523 			pmap_zero_page(m);
524 		m->valid = VM_PAGE_BITS_ALL;
525 		KASSERT((m->oflags & VPO_UNMANAGED) != 0,
526 		    ("kmem_malloc: page %p is managed", m));
527 	}
528 	VM_OBJECT_UNLOCK(kmem_object);
529 
530 	/*
531 	 * Mark map entry as non-pageable.  Repeat the assert.
532 	 */
533 	KASSERT(entry->start == addr && entry->end == addr + size &&
534 	    entry->wired_count == 0,
535 	    ("kmem_back: entry not found or misaligned after allocation"));
536 	entry->wired_count = 1;
537 
538 	/*
539 	 * At this point, the kmem_object must be unlocked because
540 	 * vm_map_simplify_entry() calls vm_object_deallocate(), which
541 	 * locks the kmem_object.
542 	 */
543 	vm_map_simplify_entry(map, entry);
544 
545 	/*
546 	 * Loop thru pages, entering them in the pmap.
547 	 */
548 	VM_OBJECT_LOCK(kmem_object);
549 	for (i = 0; i < size; i += PAGE_SIZE) {
550 		m = vm_page_lookup(kmem_object, OFF_TO_IDX(offset + i));
551 		/*
552 		 * Because this is kernel_pmap, this call will not block.
553 		 */
554 		pmap_enter(kernel_pmap, addr + i, VM_PROT_ALL, m, VM_PROT_ALL,
555 		    TRUE);
556 		vm_page_wakeup(m);
557 	}
558 	VM_OBJECT_UNLOCK(kmem_object);
559 
560 	return (KERN_SUCCESS);
561 }
562 
563 /*
564  *	kmem_alloc_wait:
565  *
566  *	Allocates pageable memory from a sub-map of the kernel.  If the submap
567  *	has no room, the caller sleeps waiting for more memory in the submap.
568  *
569  *	This routine may block.
570  */
571 vm_offset_t
572 kmem_alloc_wait(map, size)
573 	vm_map_t map;
574 	vm_size_t size;
575 {
576 	vm_offset_t addr;
577 
578 	size = round_page(size);
579 	if (!swap_reserve(size))
580 		return (0);
581 
582 	for (;;) {
583 		/*
584 		 * To make this work for more than one map, use the map's lock
585 		 * to lock out sleepers/wakers.
586 		 */
587 		vm_map_lock(map);
588 		if (vm_map_findspace(map, vm_map_min(map), size, &addr) == 0)
589 			break;
590 		/* no space now; see if we can ever get space */
591 		if (vm_map_max(map) - vm_map_min(map) < size) {
592 			vm_map_unlock(map);
593 			swap_release(size);
594 			return (0);
595 		}
596 		map->needs_wakeup = TRUE;
597 		vm_map_unlock_and_wait(map, 0);
598 	}
599 	vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_ALL,
600 	    VM_PROT_ALL, MAP_ACC_CHARGED);
601 	vm_map_unlock(map);
602 	return (addr);
603 }
604 
605 /*
606  *	kmem_free_wakeup:
607  *
608  *	Returns memory to a submap of the kernel, and wakes up any processes
609  *	waiting for memory in that map.
610  */
611 void
612 kmem_free_wakeup(map, addr, size)
613 	vm_map_t map;
614 	vm_offset_t addr;
615 	vm_size_t size;
616 {
617 
618 	vm_map_lock(map);
619 	(void) vm_map_delete(map, trunc_page(addr), round_page(addr + size));
620 	if (map->needs_wakeup) {
621 		map->needs_wakeup = FALSE;
622 		vm_map_wakeup(map);
623 	}
624 	vm_map_unlock(map);
625 }
626 
627 static void
628 kmem_init_zero_region(void)
629 {
630 	vm_offset_t addr, i;
631 	vm_page_t m;
632 	int error;
633 
634 	/*
635 	 * Map a single physical page of zeros to a larger virtual range.
636 	 * This requires less looping in places that want large amounts of
637 	 * zeros, while not using much more physical resources.
638 	 */
639 	addr = kmem_alloc_nofault(kernel_map, ZERO_REGION_SIZE);
640 	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
641 	    VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
642 	if ((m->flags & PG_ZERO) == 0)
643 		pmap_zero_page(m);
644 	for (i = 0; i < ZERO_REGION_SIZE; i += PAGE_SIZE)
645 		pmap_qenter(addr + i, &m, 1);
646 	error = vm_map_protect(kernel_map, addr, addr + ZERO_REGION_SIZE,
647 	    VM_PROT_READ, TRUE);
648 	KASSERT(error == 0, ("error=%d", error));
649 
650 	zero_region = (const void *)addr;
651 }
652 
653 /*
654  * 	kmem_init:
655  *
656  *	Create the kernel map; insert a mapping covering kernel text,
657  *	data, bss, and all space allocated thus far (`boostrap' data).  The
658  *	new map will thus map the range between VM_MIN_KERNEL_ADDRESS and
659  *	`start' as allocated, and the range between `start' and `end' as free.
660  */
661 void
662 kmem_init(start, end)
663 	vm_offset_t start, end;
664 {
665 	vm_map_t m;
666 
667 	m = vm_map_create(kernel_pmap, VM_MIN_KERNEL_ADDRESS, end);
668 	m->system_map = 1;
669 	vm_map_lock(m);
670 	/* N.B.: cannot use kgdb to debug, starting with this assignment ... */
671 	kernel_map = m;
672 	(void) vm_map_insert(m, NULL, (vm_ooffset_t) 0,
673 #ifdef __amd64__
674 	    KERNBASE,
675 #else
676 	    VM_MIN_KERNEL_ADDRESS,
677 #endif
678 	    start, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
679 	/* ... and ending with the completion of the above `insert' */
680 	vm_map_unlock(m);
681 
682 	kmem_init_zero_region();
683 }
684 
685 #ifdef DIAGNOSTIC
686 /*
687  * Allow userspace to directly trigger the VM drain routine for testing
688  * purposes.
689  */
690 static int
691 debug_vm_lowmem(SYSCTL_HANDLER_ARGS)
692 {
693 	int error, i;
694 
695 	i = 0;
696 	error = sysctl_handle_int(oidp, &i, 0, req);
697 	if (error)
698 		return (error);
699 	if (i)
700 		EVENTHANDLER_INVOKE(vm_lowmem, 0);
701 	return (0);
702 }
703 
704 SYSCTL_PROC(_debug, OID_AUTO, vm_lowmem, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
705     debug_vm_lowmem, "I", "set to trigger vm_lowmem event");
706 #endif
707