xref: /freebsd/sys/vm/vm_kern.c (revision a273973175e2a4cbbda3fd7b8cddd0bec0857082)
1 /*-
2  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
3  *
4  * Copyright (c) 1991, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  *
7  * This code is derived from software contributed to Berkeley by
8  * The Mach Operating System project at Carnegie-Mellon University.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *	from: @(#)vm_kern.c	8.3 (Berkeley) 1/12/94
35  *
36  *
37  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38  * All rights reserved.
39  *
40  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41  *
42  * Permission to use, copy, modify and distribute this software and
43  * its documentation is hereby granted, provided that both the copyright
44  * notice and this permission notice appear in all copies of the
45  * software, derivative works or modified versions, and any portions
46  * thereof, and that both notices appear in supporting documentation.
47  *
48  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51  *
52  * Carnegie Mellon requests users of this software to return to
53  *
54  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
55  *  School of Computer Science
56  *  Carnegie Mellon University
57  *  Pittsburgh PA 15213-3890
58  *
59  * any improvements or extensions that they make and grant Carnegie the
60  * rights to redistribute these changes.
61  */
62 
63 /*
64  *	Kernel memory management.
65  */
66 
67 #include <sys/cdefs.h>
68 __FBSDID("$FreeBSD$");
69 
70 #include "opt_vm.h"
71 
72 #include <sys/param.h>
73 #include <sys/systm.h>
74 #include <sys/kernel.h>		/* for ticks and hz */
75 #include <sys/domainset.h>
76 #include <sys/eventhandler.h>
77 #include <sys/lock.h>
78 #include <sys/proc.h>
79 #include <sys/malloc.h>
80 #include <sys/rwlock.h>
81 #include <sys/sysctl.h>
82 #include <sys/vmem.h>
83 #include <sys/vmmeter.h>
84 
85 #include <vm/vm.h>
86 #include <vm/vm_param.h>
87 #include <vm/vm_domainset.h>
88 #include <vm/vm_kern.h>
89 #include <vm/pmap.h>
90 #include <vm/vm_map.h>
91 #include <vm/vm_object.h>
92 #include <vm/vm_page.h>
93 #include <vm/vm_pageout.h>
94 #include <vm/vm_phys.h>
95 #include <vm/vm_radix.h>
96 #include <vm/vm_extern.h>
97 #include <vm/uma.h>
98 
99 vm_map_t kernel_map;
100 vm_map_t exec_map;
101 vm_map_t pipe_map;
102 
103 const void *zero_region;
104 CTASSERT((ZERO_REGION_SIZE & PAGE_MASK) == 0);
105 
106 /* NB: Used by kernel debuggers. */
107 const u_long vm_maxuser_address = VM_MAXUSER_ADDRESS;
108 
109 u_int exec_map_entry_size;
110 u_int exec_map_entries;
111 
112 SYSCTL_ULONG(_vm, OID_AUTO, min_kernel_address, CTLFLAG_RD,
113     SYSCTL_NULL_ULONG_PTR, VM_MIN_KERNEL_ADDRESS, "Min kernel address");
114 
115 SYSCTL_ULONG(_vm, OID_AUTO, max_kernel_address, CTLFLAG_RD,
116 #if defined(__arm__) || defined(__sparc64__)
117     &vm_max_kernel_address, 0,
118 #else
119     SYSCTL_NULL_ULONG_PTR, VM_MAX_KERNEL_ADDRESS,
120 #endif
121     "Max kernel address");
122 
123 /*
124  *	kva_alloc:
125  *
126  *	Allocate a virtual address range with no underlying object and
127  *	no initial mapping to physical memory.  Any mapping from this
128  *	range to physical memory must be explicitly created prior to
129  *	its use, typically with pmap_qenter().  Any attempt to create
130  *	a mapping on demand through vm_fault() will result in a panic.
131  */
132 vm_offset_t
133 kva_alloc(vm_size_t size)
134 {
135 	vm_offset_t addr;
136 
137 	size = round_page(size);
138 	if (vmem_alloc(kernel_arena, size, M_BESTFIT | M_NOWAIT, &addr))
139 		return (0);
140 
141 	return (addr);
142 }
143 
144 /*
145  *	kva_free:
146  *
147  *	Release a region of kernel virtual memory allocated
148  *	with kva_alloc, and return the physical pages
149  *	associated with that region.
150  *
151  *	This routine may not block on kernel maps.
152  */
153 void
154 kva_free(vm_offset_t addr, vm_size_t size)
155 {
156 
157 	size = round_page(size);
158 	vmem_free(kernel_arena, addr, size);
159 }
160 
161 /*
162  *	Allocates a region from the kernel address map and physical pages
163  *	within the specified address range to the kernel object.  Creates a
164  *	wired mapping from this region to these pages, and returns the
165  *	region's starting virtual address.  The allocated pages are not
166  *	necessarily physically contiguous.  If M_ZERO is specified through the
167  *	given flags, then the pages are zeroed before they are mapped.
168  */
169 vm_offset_t
170 kmem_alloc_attr_domain(int domain, vm_size_t size, int flags, vm_paddr_t low,
171     vm_paddr_t high, vm_memattr_t memattr)
172 {
173 	vmem_t *vmem;
174 	vm_object_t object = kernel_object;
175 	vm_offset_t addr, i, offset;
176 	vm_page_t m;
177 	int pflags, tries;
178 
179 	size = round_page(size);
180 	vmem = vm_dom[domain].vmd_kernel_arena;
181 	if (vmem_alloc(vmem, size, M_BESTFIT | flags, &addr))
182 		return (0);
183 	offset = addr - VM_MIN_KERNEL_ADDRESS;
184 	pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
185 	pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
186 	pflags |= VM_ALLOC_NOWAIT;
187 	VM_OBJECT_WLOCK(object);
188 	for (i = 0; i < size; i += PAGE_SIZE) {
189 		tries = 0;
190 retry:
191 		m = vm_page_alloc_contig_domain(object, atop(offset + i),
192 		    domain, pflags, 1, low, high, PAGE_SIZE, 0, memattr);
193 		if (m == NULL) {
194 			VM_OBJECT_WUNLOCK(object);
195 			if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
196 				if (!vm_page_reclaim_contig_domain(domain,
197 				    pflags, 1, low, high, PAGE_SIZE, 0) &&
198 				    (flags & M_WAITOK) != 0)
199 					VM_WAIT;
200 				VM_OBJECT_WLOCK(object);
201 				tries++;
202 				goto retry;
203 			}
204 			kmem_unback(object, addr, i);
205 			vmem_free(vmem, addr, size);
206 			return (0);
207 		}
208 		KASSERT(vm_phys_domidx(m) == domain,
209 		    ("kmem_alloc_attr_domain: Domain mismatch %d != %d",
210 		    vm_phys_domidx(m), domain));
211 		if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
212 			pmap_zero_page(m);
213 		m->valid = VM_PAGE_BITS_ALL;
214 		pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL,
215 		    VM_PROT_ALL | PMAP_ENTER_WIRED, 0);
216 	}
217 	VM_OBJECT_WUNLOCK(object);
218 	return (addr);
219 }
220 
221 vm_offset_t
222 kmem_alloc_attr(vmem_t *vmem, vm_size_t size, int flags, vm_paddr_t low,
223     vm_paddr_t high, vm_memattr_t memattr)
224 {
225 	struct vm_domainset_iter di;
226 	vm_offset_t addr;
227 	int domain;
228 
229 	KASSERT(vmem == kernel_arena,
230 	    ("kmem_alloc_attr: Only kernel_arena is supported."));
231 
232 	vm_domainset_iter_malloc_init(&di, kernel_object, &domain, &flags);
233 	do {
234 		addr = kmem_alloc_attr_domain(domain, size, flags, low, high,
235 		    memattr);
236 		if (addr != 0)
237 			break;
238 	} while (vm_domainset_iter_malloc(&di, &domain, &flags) == 0);
239 
240 	return (addr);
241 }
242 
243 /*
244  *	Allocates a region from the kernel address map and physically
245  *	contiguous pages within the specified address range to the kernel
246  *	object.  Creates a wired mapping from this region to these pages, and
247  *	returns the region's starting virtual address.  If M_ZERO is specified
248  *	through the given flags, then the pages are zeroed before they are
249  *	mapped.
250  */
251 vm_offset_t
252 kmem_alloc_contig_domain(int domain, vm_size_t size, int flags, vm_paddr_t low,
253     vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
254     vm_memattr_t memattr)
255 {
256 	vmem_t *vmem;
257 	vm_object_t object = kernel_object;
258 	vm_offset_t addr, offset, tmp;
259 	vm_page_t end_m, m;
260 	u_long npages;
261 	int pflags, tries;
262 
263 	size = round_page(size);
264 	vmem = vm_dom[domain].vmd_kernel_arena;
265 	if (vmem_alloc(vmem, size, flags | M_BESTFIT, &addr))
266 		return (0);
267 	offset = addr - VM_MIN_KERNEL_ADDRESS;
268 	pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
269 	pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
270 	pflags |= VM_ALLOC_NOWAIT;
271 	npages = atop(size);
272 	VM_OBJECT_WLOCK(object);
273 	tries = 0;
274 retry:
275 	m = vm_page_alloc_contig_domain(object, atop(offset), domain, pflags,
276 	    npages, low, high, alignment, boundary, memattr);
277 	if (m == NULL) {
278 		VM_OBJECT_WUNLOCK(object);
279 		if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
280 			if (!vm_page_reclaim_contig_domain(domain, pflags,
281 			    npages, low, high, alignment, boundary) &&
282 			    (flags & M_WAITOK) != 0)
283 				VM_WAIT;
284 			VM_OBJECT_WLOCK(object);
285 			tries++;
286 			goto retry;
287 		}
288 		vmem_free(vmem, addr, size);
289 		return (0);
290 	}
291 	KASSERT(vm_phys_domidx(m) == domain,
292 	    ("kmem_alloc_contig_domain: Domain mismatch %d != %d",
293 	    vm_phys_domidx(m), domain));
294 	end_m = m + npages;
295 	tmp = addr;
296 	for (; m < end_m; m++) {
297 		if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
298 			pmap_zero_page(m);
299 		m->valid = VM_PAGE_BITS_ALL;
300 		pmap_enter(kernel_pmap, tmp, m, VM_PROT_ALL,
301 		    VM_PROT_ALL | PMAP_ENTER_WIRED, 0);
302 		tmp += PAGE_SIZE;
303 	}
304 	VM_OBJECT_WUNLOCK(object);
305 	return (addr);
306 }
307 
308 vm_offset_t
309 kmem_alloc_contig(struct vmem *vmem, vm_size_t size, int flags, vm_paddr_t low,
310     vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
311     vm_memattr_t memattr)
312 {
313 	struct vm_domainset_iter di;
314 	vm_offset_t addr;
315 	int domain;
316 
317 	KASSERT(vmem == kernel_arena,
318 	    ("kmem_alloc_contig: Only kernel_arena is supported."));
319 
320 	vm_domainset_iter_malloc_init(&di, kernel_object, &domain, &flags);
321 	do {
322 		addr = kmem_alloc_contig_domain(domain, size, flags, low, high,
323 		    alignment, boundary, memattr);
324 		if (addr != 0)
325 			break;
326 	} while (vm_domainset_iter_malloc(&di, &domain, &flags) == 0);
327 
328 	return (addr);
329 }
330 
331 /*
332  *	kmem_suballoc:
333  *
334  *	Allocates a map to manage a subrange
335  *	of the kernel virtual address space.
336  *
337  *	Arguments are as follows:
338  *
339  *	parent		Map to take range from
340  *	min, max	Returned endpoints of map
341  *	size		Size of range to find
342  *	superpage_align	Request that min is superpage aligned
343  */
344 vm_map_t
345 kmem_suballoc(vm_map_t parent, vm_offset_t *min, vm_offset_t *max,
346     vm_size_t size, boolean_t superpage_align)
347 {
348 	int ret;
349 	vm_map_t result;
350 
351 	size = round_page(size);
352 
353 	*min = vm_map_min(parent);
354 	ret = vm_map_find(parent, NULL, 0, min, size, 0, superpage_align ?
355 	    VMFS_SUPER_SPACE : VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL,
356 	    MAP_ACC_NO_CHARGE);
357 	if (ret != KERN_SUCCESS)
358 		panic("kmem_suballoc: bad status return of %d", ret);
359 	*max = *min + size;
360 	result = vm_map_create(vm_map_pmap(parent), *min, *max);
361 	if (result == NULL)
362 		panic("kmem_suballoc: cannot create submap");
363 	if (vm_map_submap(parent, *min, *max, result) != KERN_SUCCESS)
364 		panic("kmem_suballoc: unable to change range to submap");
365 	return (result);
366 }
367 
368 /*
369  *	kmem_malloc:
370  *
371  *	Allocate wired-down pages in the kernel's address space.
372  */
373 vm_offset_t
374 kmem_malloc_domain(int domain, vm_size_t size, int flags)
375 {
376 	vmem_t *vmem;
377 	vm_offset_t addr;
378 	int rv;
379 
380 	vmem = vm_dom[domain].vmd_kernel_arena;
381 	size = round_page(size);
382 	if (vmem_alloc(vmem, size, flags | M_BESTFIT, &addr))
383 		return (0);
384 
385 	rv = kmem_back_domain(domain, kernel_object, addr, size, flags);
386 	if (rv != KERN_SUCCESS) {
387 		vmem_free(vmem, addr, size);
388 		return (0);
389 	}
390 	return (addr);
391 }
392 
393 vm_offset_t
394 kmem_malloc(struct vmem *vmem, vm_size_t size, int flags)
395 {
396 	struct vm_domainset_iter di;
397 	vm_offset_t addr;
398 	int domain;
399 
400 	KASSERT(vmem == kernel_arena,
401 	    ("kmem_malloc: Only kernel_arena is supported."));
402 
403 	vm_domainset_iter_malloc_init(&di, kernel_object, &domain, &flags);
404 	do {
405 		addr = kmem_malloc_domain(domain, size, flags);
406 		if (addr != 0)
407 			break;
408 	} while (vm_domainset_iter_malloc(&di, &domain, &flags) == 0);
409 
410 	return (addr);
411 }
412 
413 /*
414  *	kmem_back:
415  *
416  *	Allocate physical pages for the specified virtual address range.
417  */
418 int
419 kmem_back_domain(int domain, vm_object_t object, vm_offset_t addr,
420     vm_size_t size, int flags)
421 {
422 	vm_offset_t offset, i;
423 	vm_page_t m, mpred;
424 	int pflags;
425 
426 	KASSERT(object == kernel_object,
427 	    ("kmem_back_domain: only supports kernel object."));
428 
429 	offset = addr - VM_MIN_KERNEL_ADDRESS;
430 	pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
431 	pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
432 	if (flags & M_WAITOK)
433 		pflags |= VM_ALLOC_WAITFAIL;
434 
435 	i = 0;
436 	VM_OBJECT_WLOCK(object);
437 retry:
438 	mpred = vm_radix_lookup_le(&object->rtree, atop(offset + i));
439 	for (; i < size; i += PAGE_SIZE, mpred = m) {
440 		m = vm_page_alloc_domain_after(object, atop(offset + i),
441 		    domain, pflags, mpred);
442 
443 		/*
444 		 * Ran out of space, free everything up and return. Don't need
445 		 * to lock page queues here as we know that the pages we got
446 		 * aren't on any queues.
447 		 */
448 		if (m == NULL) {
449 			if ((flags & M_NOWAIT) == 0)
450 				goto retry;
451 			VM_OBJECT_WUNLOCK(object);
452 			kmem_unback(object, addr, i);
453 			return (KERN_NO_SPACE);
454 		}
455 		KASSERT(vm_phys_domidx(m) == domain,
456 		    ("kmem_back_domain: Domain mismatch %d != %d",
457 		    vm_phys_domidx(m), domain));
458 		if (flags & M_ZERO && (m->flags & PG_ZERO) == 0)
459 			pmap_zero_page(m);
460 		KASSERT((m->oflags & VPO_UNMANAGED) != 0,
461 		    ("kmem_malloc: page %p is managed", m));
462 		m->valid = VM_PAGE_BITS_ALL;
463 		pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL,
464 		    VM_PROT_ALL | PMAP_ENTER_WIRED, 0);
465 	}
466 	VM_OBJECT_WUNLOCK(object);
467 
468 	return (KERN_SUCCESS);
469 }
470 
471 int
472 kmem_back(vm_object_t object, vm_offset_t addr, vm_size_t size, int flags)
473 {
474 	struct vm_domainset_iter di;
475 	int domain;
476 	int ret;
477 
478 	KASSERT(object == kernel_object,
479 	    ("kmem_back: only supports kernel object."));
480 
481 	vm_domainset_iter_malloc_init(&di, kernel_object, &domain, &flags);
482 	do {
483 		ret = kmem_back_domain(domain, object, addr, size, flags);
484 		if (ret == KERN_SUCCESS)
485 			break;
486 	} while (vm_domainset_iter_malloc(&di, &domain, &flags) == 0);
487 
488 	return (ret);
489 }
490 
491 /*
492  *	kmem_unback:
493  *
494  *	Unmap and free the physical pages underlying the specified virtual
495  *	address range.
496  *
497  *	A physical page must exist within the specified object at each index
498  *	that is being unmapped.
499  */
500 static int
501 _kmem_unback(vm_object_t object, vm_offset_t addr, vm_size_t size)
502 {
503 	vm_page_t m, next;
504 	vm_offset_t end, offset;
505 	int domain;
506 
507 	KASSERT(object == kernel_object,
508 	    ("kmem_unback: only supports kernel object."));
509 
510 	if (size == 0)
511 		return (0);
512 	pmap_remove(kernel_pmap, addr, addr + size);
513 	offset = addr - VM_MIN_KERNEL_ADDRESS;
514 	end = offset + size;
515 	VM_OBJECT_WLOCK(object);
516 	m = vm_page_lookup(object, atop(offset));
517 	domain = vm_phys_domidx(m);
518 	for (; offset < end; offset += PAGE_SIZE, m = next) {
519 		next = vm_page_next(m);
520 		vm_page_unwire(m, PQ_NONE);
521 		vm_page_free(m);
522 	}
523 	VM_OBJECT_WUNLOCK(object);
524 
525 	return (domain);
526 }
527 
528 void
529 kmem_unback(vm_object_t object, vm_offset_t addr, vm_size_t size)
530 {
531 
532 	_kmem_unback(object, addr, size);
533 }
534 
535 /*
536  *	kmem_free:
537  *
538  *	Free memory allocated with kmem_malloc.  The size must match the
539  *	original allocation.
540  */
541 void
542 kmem_free(struct vmem *vmem, vm_offset_t addr, vm_size_t size)
543 {
544 	int domain;
545 
546 	KASSERT(vmem == kernel_arena,
547 	    ("kmem_free: Only kernel_arena is supported."));
548 	size = round_page(size);
549 	domain = _kmem_unback(kernel_object, addr, size);
550 	vmem_free(vm_dom[domain].vmd_kernel_arena, addr, size);
551 }
552 
553 /*
554  *	kmap_alloc_wait:
555  *
556  *	Allocates pageable memory from a sub-map of the kernel.  If the submap
557  *	has no room, the caller sleeps waiting for more memory in the submap.
558  *
559  *	This routine may block.
560  */
561 vm_offset_t
562 kmap_alloc_wait(vm_map_t map, vm_size_t size)
563 {
564 	vm_offset_t addr;
565 
566 	size = round_page(size);
567 	if (!swap_reserve(size))
568 		return (0);
569 
570 	for (;;) {
571 		/*
572 		 * To make this work for more than one map, use the map's lock
573 		 * to lock out sleepers/wakers.
574 		 */
575 		vm_map_lock(map);
576 		if (vm_map_findspace(map, vm_map_min(map), size, &addr) == 0)
577 			break;
578 		/* no space now; see if we can ever get space */
579 		if (vm_map_max(map) - vm_map_min(map) < size) {
580 			vm_map_unlock(map);
581 			swap_release(size);
582 			return (0);
583 		}
584 		map->needs_wakeup = TRUE;
585 		vm_map_unlock_and_wait(map, 0);
586 	}
587 	vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_ALL,
588 	    VM_PROT_ALL, MAP_ACC_CHARGED);
589 	vm_map_unlock(map);
590 	return (addr);
591 }
592 
593 /*
594  *	kmap_free_wakeup:
595  *
596  *	Returns memory to a submap of the kernel, and wakes up any processes
597  *	waiting for memory in that map.
598  */
599 void
600 kmap_free_wakeup(vm_map_t map, vm_offset_t addr, vm_size_t size)
601 {
602 
603 	vm_map_lock(map);
604 	(void) vm_map_delete(map, trunc_page(addr), round_page(addr + size));
605 	if (map->needs_wakeup) {
606 		map->needs_wakeup = FALSE;
607 		vm_map_wakeup(map);
608 	}
609 	vm_map_unlock(map);
610 }
611 
612 void
613 kmem_init_zero_region(void)
614 {
615 	vm_offset_t addr, i;
616 	vm_page_t m;
617 
618 	/*
619 	 * Map a single physical page of zeros to a larger virtual range.
620 	 * This requires less looping in places that want large amounts of
621 	 * zeros, while not using much more physical resources.
622 	 */
623 	addr = kva_alloc(ZERO_REGION_SIZE);
624 	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
625 	    VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
626 	if ((m->flags & PG_ZERO) == 0)
627 		pmap_zero_page(m);
628 	for (i = 0; i < ZERO_REGION_SIZE; i += PAGE_SIZE)
629 		pmap_qenter(addr + i, &m, 1);
630 	pmap_protect(kernel_pmap, addr, addr + ZERO_REGION_SIZE, VM_PROT_READ);
631 
632 	zero_region = (const void *)addr;
633 }
634 
635 /*
636  * 	kmem_init:
637  *
638  *	Create the kernel map; insert a mapping covering kernel text,
639  *	data, bss, and all space allocated thus far (`boostrap' data).  The
640  *	new map will thus map the range between VM_MIN_KERNEL_ADDRESS and
641  *	`start' as allocated, and the range between `start' and `end' as free.
642  */
643 void
644 kmem_init(vm_offset_t start, vm_offset_t end)
645 {
646 	vm_map_t m;
647 
648 	m = vm_map_create(kernel_pmap, VM_MIN_KERNEL_ADDRESS, end);
649 	m->system_map = 1;
650 	vm_map_lock(m);
651 	/* N.B.: cannot use kgdb to debug, starting with this assignment ... */
652 	kernel_map = m;
653 	(void) vm_map_insert(m, NULL, (vm_ooffset_t) 0,
654 #ifdef __amd64__
655 	    KERNBASE,
656 #else
657 	    VM_MIN_KERNEL_ADDRESS,
658 #endif
659 	    start, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT);
660 	/* ... and ending with the completion of the above `insert' */
661 	vm_map_unlock(m);
662 }
663 
664 #ifdef DIAGNOSTIC
665 /*
666  * Allow userspace to directly trigger the VM drain routine for testing
667  * purposes.
668  */
669 static int
670 debug_vm_lowmem(SYSCTL_HANDLER_ARGS)
671 {
672 	int error, i;
673 
674 	i = 0;
675 	error = sysctl_handle_int(oidp, &i, 0, req);
676 	if (error)
677 		return (error);
678 	if ((i & ~(VM_LOW_KMEM | VM_LOW_PAGES)) != 0)
679 		return (EINVAL);
680 	if (i != 0)
681 		EVENTHANDLER_INVOKE(vm_lowmem, i);
682 	return (0);
683 }
684 
685 SYSCTL_PROC(_debug, OID_AUTO, vm_lowmem, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
686     debug_vm_lowmem, "I", "set to trigger vm_lowmem event with given flags");
687 #endif
688