160727d8bSWarner Losh /*- 2796df753SPedro F. Giffuni * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU) 351369649SPedro F. Giffuni * 4df8bae1dSRodney W. Grimes * Copyright (c) 1991, 1993 5df8bae1dSRodney W. Grimes * The Regents of the University of California. All rights reserved. 6df8bae1dSRodney W. Grimes * 7df8bae1dSRodney W. Grimes * This code is derived from software contributed to Berkeley by 8df8bae1dSRodney W. Grimes * The Mach Operating System project at Carnegie-Mellon University. 9df8bae1dSRodney W. Grimes * 10df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 11df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 12df8bae1dSRodney W. Grimes * are met: 13df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 14df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 15df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 16df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 17df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 18fbbd9655SWarner Losh * 3. Neither the name of the University nor the names of its contributors 19df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 20df8bae1dSRodney W. Grimes * without specific prior written permission. 21df8bae1dSRodney W. Grimes * 22df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32df8bae1dSRodney W. Grimes * SUCH DAMAGE. 33df8bae1dSRodney W. Grimes * 343c4dd356SDavid Greenman * from: @(#)vm_kern.c 8.3 (Berkeley) 1/12/94 35df8bae1dSRodney W. Grimes * 36df8bae1dSRodney W. Grimes * 37df8bae1dSRodney W. Grimes * Copyright (c) 1987, 1990 Carnegie-Mellon University. 38df8bae1dSRodney W. Grimes * All rights reserved. 39df8bae1dSRodney W. Grimes * 40df8bae1dSRodney W. Grimes * Authors: Avadis Tevanian, Jr., Michael Wayne Young 41df8bae1dSRodney W. Grimes * 42df8bae1dSRodney W. Grimes * Permission to use, copy, modify and distribute this software and 43df8bae1dSRodney W. Grimes * its documentation is hereby granted, provided that both the copyright 44df8bae1dSRodney W. Grimes * notice and this permission notice appear in all copies of the 45df8bae1dSRodney W. Grimes * software, derivative works or modified versions, and any portions 46df8bae1dSRodney W. Grimes * thereof, and that both notices appear in supporting documentation. 47df8bae1dSRodney W. Grimes * 48df8bae1dSRodney W. Grimes * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 49df8bae1dSRodney W. Grimes * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 50df8bae1dSRodney W. Grimes * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 51df8bae1dSRodney W. Grimes * 52df8bae1dSRodney W. Grimes * Carnegie Mellon requests users of this software to return to 53df8bae1dSRodney W. Grimes * 54df8bae1dSRodney W. Grimes * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 55df8bae1dSRodney W. Grimes * School of Computer Science 56df8bae1dSRodney W. Grimes * Carnegie Mellon University 57df8bae1dSRodney W. Grimes * Pittsburgh PA 15213-3890 58df8bae1dSRodney W. Grimes * 59df8bae1dSRodney W. Grimes * any improvements or extensions that they make and grant Carnegie the 60df8bae1dSRodney W. Grimes * rights to redistribute these changes. 61df8bae1dSRodney W. Grimes */ 62df8bae1dSRodney W. Grimes 63df8bae1dSRodney W. Grimes /* 64df8bae1dSRodney W. Grimes * Kernel memory management. 65df8bae1dSRodney W. Grimes */ 66df8bae1dSRodney W. Grimes 67874651b1SDavid E. O'Brien #include <sys/cdefs.h> 68874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$"); 69874651b1SDavid E. O'Brien 707a469c8eSJeff Roberson #include "opt_vm.h" 717a469c8eSJeff Roberson 72df8bae1dSRodney W. Grimes #include <sys/param.h> 73df8bae1dSRodney W. Grimes #include <sys/systm.h> 742b914b85SMark Johnston #include <sys/asan.h> 757a469c8eSJeff Roberson #include <sys/domainset.h> 760f2c2ce0SPawel Jakub Dawidek #include <sys/eventhandler.h> 772b914b85SMark Johnston #include <sys/kernel.h> 78fb919e4dSMark Murray #include <sys/lock.h> 79a1f6d91cSDavid Greenman #include <sys/malloc.h> 80*540da48dSMark Johnston #include <sys/msan.h> 812b914b85SMark Johnston #include <sys/proc.h> 8289f6b863SAttilio Rao #include <sys/rwlock.h> 8386f08737SRobert Watson #include <sys/sysctl.h> 845df87b21SJeff Roberson #include <sys/vmem.h> 857a469c8eSJeff Roberson #include <sys/vmmeter.h> 86df8bae1dSRodney W. Grimes 87df8bae1dSRodney W. Grimes #include <vm/vm.h> 88efeaf95aSDavid Greenman #include <vm/vm_param.h> 897a469c8eSJeff Roberson #include <vm/vm_domainset.h> 905df87b21SJeff Roberson #include <vm/vm_kern.h> 91efeaf95aSDavid Greenman #include <vm/pmap.h> 92efeaf95aSDavid Greenman #include <vm/vm_map.h> 93efeaf95aSDavid Greenman #include <vm/vm_object.h> 94df8bae1dSRodney W. Grimes #include <vm/vm_page.h> 95df8bae1dSRodney W. Grimes #include <vm/vm_pageout.h> 96e2068d0bSJeff Roberson #include <vm/vm_pagequeue.h> 97431fb8abSMark Johnston #include <vm/vm_phys.h> 9833fff5d5SMark Johnston #include <vm/vm_radix.h> 999b4288a3SBruce Evans #include <vm/vm_extern.h> 1000f2c2ce0SPawel Jakub Dawidek #include <vm/uma.h> 101df8bae1dSRodney W. Grimes 1027dd979dfSMark Johnston struct vm_map kernel_map_store; 1037dd979dfSMark Johnston struct vm_map exec_map_store; 1047dd979dfSMark Johnston struct vm_map pipe_map_store; 105f23b4c91SGarrett Wollman 10689cb2a19SMatthew D Fleming const void *zero_region; 10789cb2a19SMatthew D Fleming CTASSERT((ZERO_REGION_SIZE & PAGE_MASK) == 0); 10889cb2a19SMatthew D Fleming 109645743eaSJohn Baldwin /* NB: Used by kernel debuggers. */ 110645743eaSJohn Baldwin const u_long vm_maxuser_address = VM_MAXUSER_ADDRESS; 111645743eaSJohn Baldwin 112ec492b13SMark Johnston u_int exec_map_entry_size; 113ec492b13SMark Johnston u_int exec_map_entries; 114ec492b13SMark Johnston 11594bfd5b1SMarius Strobl SYSCTL_ULONG(_vm, OID_AUTO, min_kernel_address, CTLFLAG_RD, 116f0188618SHans Petter Selasky SYSCTL_NULL_ULONG_PTR, VM_MIN_KERNEL_ADDRESS, "Min kernel address"); 11794bfd5b1SMarius Strobl 11894bfd5b1SMarius Strobl SYSCTL_ULONG(_vm, OID_AUTO, max_kernel_address, CTLFLAG_RD, 11958aa35d4SWarner Losh #if defined(__arm__) 12094bfd5b1SMarius Strobl &vm_max_kernel_address, 0, 12194bfd5b1SMarius Strobl #else 122f0188618SHans Petter Selasky SYSCTL_NULL_ULONG_PTR, VM_MAX_KERNEL_ADDRESS, 12394bfd5b1SMarius Strobl #endif 12494bfd5b1SMarius Strobl "Max kernel address"); 12594bfd5b1SMarius Strobl 1261aed6d48SMark Johnston #if VM_NRESERVLEVEL > 0 12725ed23cfSMark Johnston #define KVA_QUANTUM_SHIFT (VM_LEVEL_0_ORDER + PAGE_SHIFT) 1281aed6d48SMark Johnston #else 1292b601070SMark Johnston /* On non-superpage architectures we want large import sizes. */ 1302b601070SMark Johnston #define KVA_QUANTUM_SHIFT (8 + PAGE_SHIFT) 1311aed6d48SMark Johnston #endif 13223e875fdSMark Johnston #define KVA_QUANTUM (1ul << KVA_QUANTUM_SHIFT) 133aea9103eSMark Johnston #define KVA_NUMA_IMPORT_QUANTUM (KVA_QUANTUM * 128) 1341aed6d48SMark Johnston 135a81c400eSJeff Roberson extern void uma_startup2(void); 136a81c400eSJeff Roberson 137df8bae1dSRodney W. Grimes /* 1385df87b21SJeff Roberson * kva_alloc: 139a839bdc8SDmitrij Tejblum * 140b77c2bcdSAlan Cox * Allocate a virtual address range with no underlying object and 141b77c2bcdSAlan Cox * no initial mapping to physical memory. Any mapping from this 142b77c2bcdSAlan Cox * range to physical memory must be explicitly created prior to 143b77c2bcdSAlan Cox * its use, typically with pmap_qenter(). Any attempt to create 144b77c2bcdSAlan Cox * a mapping on demand through vm_fault() will result in a panic. 145a839bdc8SDmitrij Tejblum */ 146a839bdc8SDmitrij Tejblum vm_offset_t 1476e309d75SEd Maste kva_alloc(vm_size_t size) 148a839bdc8SDmitrij Tejblum { 149a839bdc8SDmitrij Tejblum vm_offset_t addr; 150a839bdc8SDmitrij Tejblum 151a839bdc8SDmitrij Tejblum size = round_page(size); 1525df87b21SJeff Roberson if (vmem_alloc(kernel_arena, size, M_BESTFIT | M_NOWAIT, &addr)) 153a839bdc8SDmitrij Tejblum return (0); 1545df87b21SJeff Roberson 155a839bdc8SDmitrij Tejblum return (addr); 156a839bdc8SDmitrij Tejblum } 157a839bdc8SDmitrij Tejblum 158a839bdc8SDmitrij Tejblum /* 1595df87b21SJeff Roberson * kva_free: 160ca596a25SJuli Mallett * 1615df87b21SJeff Roberson * Release a region of kernel virtual memory allocated 1625df87b21SJeff Roberson * with kva_alloc, and return the physical pages 1635df87b21SJeff Roberson * associated with that region. 1645df87b21SJeff Roberson * 1655df87b21SJeff Roberson * This routine may not block on kernel maps. 166ca596a25SJuli Mallett */ 1675df87b21SJeff Roberson void 1686e309d75SEd Maste kva_free(vm_offset_t addr, vm_size_t size) 169df8bae1dSRodney W. Grimes { 170df8bae1dSRodney W. Grimes 171df8bae1dSRodney W. Grimes size = round_page(size); 1725df87b21SJeff Roberson vmem_free(kernel_arena, addr, size); 173df8bae1dSRodney W. Grimes } 174df8bae1dSRodney W. Grimes 175*540da48dSMark Johnston /* 176*540da48dSMark Johnston * Update sanitizer shadow state to reflect a new allocation. Force inlining to 177*540da48dSMark Johnston * help make KMSAN origin tracking more precise. 178*540da48dSMark Johnston */ 179*540da48dSMark Johnston static __always_inline void 180*540da48dSMark Johnston kmem_alloc_san(vm_offset_t addr, vm_size_t size, vm_size_t asize, int flags) 181*540da48dSMark Johnston { 182*540da48dSMark Johnston if ((flags & M_ZERO) == 0) { 183*540da48dSMark Johnston kmsan_mark((void *)addr, asize, KMSAN_STATE_UNINIT); 184*540da48dSMark Johnston kmsan_orig((void *)addr, asize, KMSAN_TYPE_KMEM, 185*540da48dSMark Johnston KMSAN_RET_ADDR); 186*540da48dSMark Johnston } else { 187*540da48dSMark Johnston kmsan_mark((void *)addr, asize, KMSAN_STATE_INITED); 188*540da48dSMark Johnston } 189*540da48dSMark Johnston kasan_mark((void *)addr, size, asize, KASAN_KMEM_REDZONE); 190*540da48dSMark Johnston } 191*540da48dSMark Johnston 19233655d95SMark Johnston static vm_page_t 19333655d95SMark Johnston kmem_alloc_contig_pages(vm_object_t object, vm_pindex_t pindex, int domain, 19433655d95SMark Johnston int pflags, u_long npages, vm_paddr_t low, vm_paddr_t high, 19533655d95SMark Johnston u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr) 19633655d95SMark Johnston { 19733655d95SMark Johnston vm_page_t m; 19833655d95SMark Johnston int tries; 199660344caSRyan Stone bool wait, reclaim; 20033655d95SMark Johnston 20133655d95SMark Johnston VM_OBJECT_ASSERT_WLOCKED(object); 20233655d95SMark Johnston 20333655d95SMark Johnston wait = (pflags & VM_ALLOC_WAITOK) != 0; 204660344caSRyan Stone reclaim = (pflags & VM_ALLOC_NORECLAIM) == 0; 20533655d95SMark Johnston pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL); 20633655d95SMark Johnston pflags |= VM_ALLOC_NOWAIT; 20733655d95SMark Johnston for (tries = wait ? 3 : 1;; tries--) { 20833655d95SMark Johnston m = vm_page_alloc_contig_domain(object, pindex, domain, pflags, 20933655d95SMark Johnston npages, low, high, alignment, boundary, memattr); 210660344caSRyan Stone if (m != NULL || tries == 0 || !reclaim) 21133655d95SMark Johnston break; 21233655d95SMark Johnston 21333655d95SMark Johnston VM_OBJECT_WUNLOCK(object); 21433655d95SMark Johnston if (!vm_page_reclaim_contig_domain(domain, pflags, npages, 21533655d95SMark Johnston low, high, alignment, boundary) && wait) 21633655d95SMark Johnston vm_wait_domain(domain); 21733655d95SMark Johnston VM_OBJECT_WLOCK(object); 21833655d95SMark Johnston } 21933655d95SMark Johnston return (m); 22033655d95SMark Johnston } 22133655d95SMark Johnston 222df8bae1dSRodney W. Grimes /* 2230ff0fc84SAlan Cox * Allocates a region from the kernel address map and physical pages 2240ff0fc84SAlan Cox * within the specified address range to the kernel object. Creates a 2250ff0fc84SAlan Cox * wired mapping from this region to these pages, and returns the 2260ff0fc84SAlan Cox * region's starting virtual address. The allocated pages are not 2270ff0fc84SAlan Cox * necessarily physically contiguous. If M_ZERO is specified through the 2280ff0fc84SAlan Cox * given flags, then the pages are zeroed before they are mapped. 2290ff0fc84SAlan Cox */ 2309978bd99SMark Johnston static vm_offset_t 2317a469c8eSJeff Roberson kmem_alloc_attr_domain(int domain, vm_size_t size, int flags, vm_paddr_t low, 2320ff0fc84SAlan Cox vm_paddr_t high, vm_memattr_t memattr) 2330ff0fc84SAlan Cox { 2347a469c8eSJeff Roberson vmem_t *vmem; 23533655d95SMark Johnston vm_object_t object; 236d1780e8dSKonstantin Belousov vm_offset_t addr, i, offset; 2370ff0fc84SAlan Cox vm_page_t m; 2382b914b85SMark Johnston vm_size_t asize; 23933655d95SMark Johnston int pflags; 240d22ff6e6SKonstantin Belousov vm_prot_t prot; 2410ff0fc84SAlan Cox 24233655d95SMark Johnston object = kernel_object; 2432b914b85SMark Johnston asize = round_page(size); 2447a469c8eSJeff Roberson vmem = vm_dom[domain].vmd_kernel_arena; 2452b914b85SMark Johnston if (vmem_alloc(vmem, asize, M_BESTFIT | flags, &addr)) 2460ff0fc84SAlan Cox return (0); 2470ff0fc84SAlan Cox offset = addr - VM_MIN_KERNEL_ADDRESS; 24891e31c3cSJeff Roberson pflags = malloc2vm_flags(flags) | VM_ALLOC_WIRED; 249d22ff6e6SKonstantin Belousov prot = (flags & M_EXEC) != 0 ? VM_PROT_ALL : VM_PROT_RW; 25089f6b863SAttilio Rao VM_OBJECT_WLOCK(object); 2512b914b85SMark Johnston for (i = 0; i < asize; i += PAGE_SIZE) { 25233655d95SMark Johnston m = kmem_alloc_contig_pages(object, atop(offset + i), 2537a469c8eSJeff Roberson domain, pflags, 1, low, high, PAGE_SIZE, 0, memattr); 2540ff0fc84SAlan Cox if (m == NULL) { 25589f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 2569e829b22SAlan Cox kmem_unback(object, addr, i); 2572b914b85SMark Johnston vmem_free(vmem, addr, asize); 2580ff0fc84SAlan Cox return (0); 2590ff0fc84SAlan Cox } 260431fb8abSMark Johnston KASSERT(vm_page_domain(m) == domain, 2617a469c8eSJeff Roberson ("kmem_alloc_attr_domain: Domain mismatch %d != %d", 262431fb8abSMark Johnston vm_page_domain(m), domain)); 2630ff0fc84SAlan Cox if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) 2640ff0fc84SAlan Cox pmap_zero_page(m); 26591e31c3cSJeff Roberson vm_page_valid(m); 266d22ff6e6SKonstantin Belousov pmap_enter(kernel_pmap, addr + i, m, prot, 267d22ff6e6SKonstantin Belousov prot | PMAP_ENTER_WIRED, 0); 2680ff0fc84SAlan Cox } 26989f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 270*540da48dSMark Johnston kmem_alloc_san(addr, size, asize, flags); 2710ff0fc84SAlan Cox return (addr); 2720ff0fc84SAlan Cox } 2730ff0fc84SAlan Cox 2747a469c8eSJeff Roberson vm_offset_t 275db7c2a48SAlan Cox kmem_alloc_attr(vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high, 276db7c2a48SAlan Cox vm_memattr_t memattr) 2777a469c8eSJeff Roberson { 2789978bd99SMark Johnston 2799978bd99SMark Johnston return (kmem_alloc_attr_domainset(DOMAINSET_RR(), size, flags, low, 2809978bd99SMark Johnston high, memattr)); 2819978bd99SMark Johnston } 2829978bd99SMark Johnston 2839978bd99SMark Johnston vm_offset_t 2849978bd99SMark Johnston kmem_alloc_attr_domainset(struct domainset *ds, vm_size_t size, int flags, 2859978bd99SMark Johnston vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr) 2869978bd99SMark Johnston { 2877a469c8eSJeff Roberson struct vm_domainset_iter di; 2887a469c8eSJeff Roberson vm_offset_t addr; 2895fee468eSBryan Drewery int domain; 2907a469c8eSJeff Roberson 2915fee468eSBryan Drewery vm_domainset_iter_policy_init(&di, ds, &domain, &flags); 2927a469c8eSJeff Roberson do { 2937a469c8eSJeff Roberson addr = kmem_alloc_attr_domain(domain, size, flags, low, high, 2947a469c8eSJeff Roberson memattr); 2957a469c8eSJeff Roberson if (addr != 0) 2967a469c8eSJeff Roberson break; 2974c29d2deSMark Johnston } while (vm_domainset_iter_policy(&di, &domain) == 0); 2987a469c8eSJeff Roberson 2997a469c8eSJeff Roberson return (addr); 3007a469c8eSJeff Roberson } 3017a469c8eSJeff Roberson 3020ff0fc84SAlan Cox /* 3030ff0fc84SAlan Cox * Allocates a region from the kernel address map and physically 3040ff0fc84SAlan Cox * contiguous pages within the specified address range to the kernel 3050ff0fc84SAlan Cox * object. Creates a wired mapping from this region to these pages, and 3060ff0fc84SAlan Cox * returns the region's starting virtual address. If M_ZERO is specified 3070ff0fc84SAlan Cox * through the given flags, then the pages are zeroed before they are 3080ff0fc84SAlan Cox * mapped. 3090ff0fc84SAlan Cox */ 3109978bd99SMark Johnston static vm_offset_t 3117a469c8eSJeff Roberson kmem_alloc_contig_domain(int domain, vm_size_t size, int flags, vm_paddr_t low, 3120ff0fc84SAlan Cox vm_paddr_t high, u_long alignment, vm_paddr_t boundary, 3130ff0fc84SAlan Cox vm_memattr_t memattr) 3140ff0fc84SAlan Cox { 3157a469c8eSJeff Roberson vmem_t *vmem; 31633655d95SMark Johnston vm_object_t object; 317d1780e8dSKonstantin Belousov vm_offset_t addr, offset, tmp; 3180ff0fc84SAlan Cox vm_page_t end_m, m; 3192b914b85SMark Johnston vm_size_t asize; 320c869e672SAlan Cox u_long npages; 32133655d95SMark Johnston int pflags; 3220ff0fc84SAlan Cox 32333655d95SMark Johnston object = kernel_object; 3242b914b85SMark Johnston asize = round_page(size); 3257a469c8eSJeff Roberson vmem = vm_dom[domain].vmd_kernel_arena; 3262b914b85SMark Johnston if (vmem_alloc(vmem, asize, flags | M_BESTFIT, &addr)) 3270ff0fc84SAlan Cox return (0); 3280ff0fc84SAlan Cox offset = addr - VM_MIN_KERNEL_ADDRESS; 32991e31c3cSJeff Roberson pflags = malloc2vm_flags(flags) | VM_ALLOC_WIRED; 3302b914b85SMark Johnston npages = atop(asize); 33189f6b863SAttilio Rao VM_OBJECT_WLOCK(object); 33233655d95SMark Johnston m = kmem_alloc_contig_pages(object, atop(offset), domain, 33333655d95SMark Johnston pflags, npages, low, high, alignment, boundary, memattr); 3340ff0fc84SAlan Cox if (m == NULL) { 33589f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 3362b914b85SMark Johnston vmem_free(vmem, addr, asize); 3370ff0fc84SAlan Cox return (0); 3380ff0fc84SAlan Cox } 339431fb8abSMark Johnston KASSERT(vm_page_domain(m) == domain, 3407a469c8eSJeff Roberson ("kmem_alloc_contig_domain: Domain mismatch %d != %d", 341431fb8abSMark Johnston vm_page_domain(m), domain)); 342c869e672SAlan Cox end_m = m + npages; 3435df87b21SJeff Roberson tmp = addr; 3440ff0fc84SAlan Cox for (; m < end_m; m++) { 3450ff0fc84SAlan Cox if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) 3460ff0fc84SAlan Cox pmap_zero_page(m); 34791e31c3cSJeff Roberson vm_page_valid(m); 3480766f278SJonathan T. Looney pmap_enter(kernel_pmap, tmp, m, VM_PROT_RW, 3490766f278SJonathan T. Looney VM_PROT_RW | PMAP_ENTER_WIRED, 0); 3505df87b21SJeff Roberson tmp += PAGE_SIZE; 3510ff0fc84SAlan Cox } 35289f6b863SAttilio Rao VM_OBJECT_WUNLOCK(object); 353*540da48dSMark Johnston kmem_alloc_san(addr, size, asize, flags); 3540ff0fc84SAlan Cox return (addr); 3550ff0fc84SAlan Cox } 3560ff0fc84SAlan Cox 3577a469c8eSJeff Roberson vm_offset_t 35844d0efb2SAlan Cox kmem_alloc_contig(vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high, 35944d0efb2SAlan Cox u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr) 3607a469c8eSJeff Roberson { 3619978bd99SMark Johnston 3629978bd99SMark Johnston return (kmem_alloc_contig_domainset(DOMAINSET_RR(), size, flags, low, 3639978bd99SMark Johnston high, alignment, boundary, memattr)); 3649978bd99SMark Johnston } 3659978bd99SMark Johnston 3669978bd99SMark Johnston vm_offset_t 3679978bd99SMark Johnston kmem_alloc_contig_domainset(struct domainset *ds, vm_size_t size, int flags, 3689978bd99SMark Johnston vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, 3699978bd99SMark Johnston vm_memattr_t memattr) 3709978bd99SMark Johnston { 3717a469c8eSJeff Roberson struct vm_domainset_iter di; 3727a469c8eSJeff Roberson vm_offset_t addr; 3735fee468eSBryan Drewery int domain; 3747a469c8eSJeff Roberson 3755fee468eSBryan Drewery vm_domainset_iter_policy_init(&di, ds, &domain, &flags); 3767a469c8eSJeff Roberson do { 3777a469c8eSJeff Roberson addr = kmem_alloc_contig_domain(domain, size, flags, low, high, 3787a469c8eSJeff Roberson alignment, boundary, memattr); 3797a469c8eSJeff Roberson if (addr != 0) 3807a469c8eSJeff Roberson break; 3814c29d2deSMark Johnston } while (vm_domainset_iter_policy(&di, &domain) == 0); 3827a469c8eSJeff Roberson 3837a469c8eSJeff Roberson return (addr); 3847a469c8eSJeff Roberson } 3857a469c8eSJeff Roberson 3860ff0fc84SAlan Cox /* 3877dd979dfSMark Johnston * kmem_subinit: 388df8bae1dSRodney W. Grimes * 3897dd979dfSMark Johnston * Initializes a map to manage a subrange 390df8bae1dSRodney W. Grimes * of the kernel virtual address space. 391df8bae1dSRodney W. Grimes * 392df8bae1dSRodney W. Grimes * Arguments are as follows: 393df8bae1dSRodney W. Grimes * 394df8bae1dSRodney W. Grimes * parent Map to take range from 395df8bae1dSRodney W. Grimes * min, max Returned endpoints of map 396030f2369SAlfred Perlstein * size Size of range to find 3973202ed75SAlan Cox * superpage_align Request that min is superpage aligned 398df8bae1dSRodney W. Grimes */ 3997dd979dfSMark Johnston void 4007dd979dfSMark Johnston kmem_subinit(vm_map_t map, vm_map_t parent, vm_offset_t *min, vm_offset_t *max, 4017dd979dfSMark Johnston vm_size_t size, bool superpage_align) 402df8bae1dSRodney W. Grimes { 4036e4f51d1SAlfred Perlstein int ret; 40423955314SAlfred Perlstein 405df8bae1dSRodney W. Grimes size = round_page(size); 406df8bae1dSRodney W. Grimes 4072bc24aa9SAlan Cox *min = vm_map_min(parent); 408edb572a3SJohn Baldwin ret = vm_map_find(parent, NULL, 0, min, size, 0, superpage_align ? 4095aa60b6fSJohn Baldwin VMFS_SUPER_SPACE : VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL, 4103364c323SKonstantin Belousov MAP_ACC_NO_CHARGE); 41124dedba9SAlan Cox if (ret != KERN_SUCCESS) 4127dd979dfSMark Johnston panic("kmem_subinit: bad status return of %d", ret); 413df8bae1dSRodney W. Grimes *max = *min + size; 4147dd979dfSMark Johnston vm_map_init(map, vm_map_pmap(parent), *min, *max); 4157dd979dfSMark Johnston if (vm_map_submap(parent, *min, *max, map) != KERN_SUCCESS) 4167dd979dfSMark Johnston panic("kmem_subinit: unable to change range to submap"); 417df8bae1dSRodney W. Grimes } 418df8bae1dSRodney W. Grimes 419df8bae1dSRodney W. Grimes /* 4209978bd99SMark Johnston * kmem_malloc_domain: 4211c7c3c6aSMatthew Dillon * 4225df87b21SJeff Roberson * Allocate wired-down pages in the kernel's address space. 423df8bae1dSRodney W. Grimes */ 4249978bd99SMark Johnston static vm_offset_t 425067fd858SAlan Cox kmem_malloc_domain(int domain, vm_size_t size, int flags) 426df8bae1dSRodney W. Grimes { 4270766f278SJonathan T. Looney vmem_t *arena; 428df8bae1dSRodney W. Grimes vm_offset_t addr; 4292b914b85SMark Johnston vm_size_t asize; 4305df87b21SJeff Roberson int rv; 431df8bae1dSRodney W. Grimes 432067fd858SAlan Cox if (__predict_true((flags & M_EXEC) == 0)) 4330766f278SJonathan T. Looney arena = vm_dom[domain].vmd_kernel_arena; 4340766f278SJonathan T. Looney else 4350766f278SJonathan T. Looney arena = vm_dom[domain].vmd_kernel_rwx_arena; 4362b914b85SMark Johnston asize = round_page(size); 4372b914b85SMark Johnston if (vmem_alloc(arena, asize, flags | M_BESTFIT, &addr)) 4385df87b21SJeff Roberson return (0); 439df8bae1dSRodney W. Grimes 4402b914b85SMark Johnston rv = kmem_back_domain(domain, kernel_object, addr, asize, flags); 4415df87b21SJeff Roberson if (rv != KERN_SUCCESS) { 4422b914b85SMark Johnston vmem_free(arena, addr, asize); 443f31c239dSAlan Cox return (0); 444df8bae1dSRodney W. Grimes } 4452b914b85SMark Johnston kasan_mark((void *)addr, size, asize, KASAN_KMEM_REDZONE); 4465df87b21SJeff Roberson return (addr); 447e3813573SMatthew D Fleming } 448e3813573SMatthew D Fleming 4497a469c8eSJeff Roberson vm_offset_t 45083a90bffSAlan Cox kmem_malloc(vm_size_t size, int flags) 4517a469c8eSJeff Roberson { 4529978bd99SMark Johnston 4539978bd99SMark Johnston return (kmem_malloc_domainset(DOMAINSET_RR(), size, flags)); 4549978bd99SMark Johnston } 4559978bd99SMark Johnston 4569978bd99SMark Johnston vm_offset_t 4579978bd99SMark Johnston kmem_malloc_domainset(struct domainset *ds, vm_size_t size, int flags) 4589978bd99SMark Johnston { 4597a469c8eSJeff Roberson struct vm_domainset_iter di; 4607a469c8eSJeff Roberson vm_offset_t addr; 4617a469c8eSJeff Roberson int domain; 4627a469c8eSJeff Roberson 4639978bd99SMark Johnston vm_domainset_iter_policy_init(&di, ds, &domain, &flags); 4647a469c8eSJeff Roberson do { 465067fd858SAlan Cox addr = kmem_malloc_domain(domain, size, flags); 4667a469c8eSJeff Roberson if (addr != 0) 4677a469c8eSJeff Roberson break; 4684c29d2deSMark Johnston } while (vm_domainset_iter_policy(&di, &domain) == 0); 4697a469c8eSJeff Roberson 4707a469c8eSJeff Roberson return (addr); 4717a469c8eSJeff Roberson } 4727a469c8eSJeff Roberson 473e3813573SMatthew D Fleming /* 47425ed23cfSMark Johnston * kmem_back_domain: 475e3813573SMatthew D Fleming * 47625ed23cfSMark Johnston * Allocate physical pages from the specified domain for the specified 47725ed23cfSMark Johnston * virtual address range. 478e3813573SMatthew D Fleming */ 479e3813573SMatthew D Fleming int 4807a469c8eSJeff Roberson kmem_back_domain(int domain, vm_object_t object, vm_offset_t addr, 4817a469c8eSJeff Roberson vm_size_t size, int flags) 482e3813573SMatthew D Fleming { 483e3813573SMatthew D Fleming vm_offset_t offset, i; 48433fff5d5SMark Johnston vm_page_t m, mpred; 4850766f278SJonathan T. Looney vm_prot_t prot; 486e3813573SMatthew D Fleming int pflags; 487e3813573SMatthew D Fleming 4882e47807cSJeff Roberson KASSERT(object == kernel_object, 4897a469c8eSJeff Roberson ("kmem_back_domain: only supports kernel object.")); 4905df87b21SJeff Roberson 4910891ef4cSJohn Dyson offset = addr - VM_MIN_KERNEL_ADDRESS; 49291e31c3cSJeff Roberson pflags = malloc2vm_flags(flags) | VM_ALLOC_WIRED; 4938d6fbbb8SJeff Roberson pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL); 4948d6fbbb8SJeff Roberson if (flags & M_WAITOK) 4958d6fbbb8SJeff Roberson pflags |= VM_ALLOC_WAITFAIL; 4960766f278SJonathan T. Looney prot = (flags & M_EXEC) != 0 ? VM_PROT_ALL : VM_PROT_RW; 497df8bae1dSRodney W. Grimes 49833fff5d5SMark Johnston i = 0; 49933fff5d5SMark Johnston VM_OBJECT_WLOCK(object); 5008d6fbbb8SJeff Roberson retry: 50133fff5d5SMark Johnston mpred = vm_radix_lookup_le(&object->rtree, atop(offset + i)); 50233fff5d5SMark Johnston for (; i < size; i += PAGE_SIZE, mpred = m) { 5037a469c8eSJeff Roberson m = vm_page_alloc_domain_after(object, atop(offset + i), 5047a469c8eSJeff Roberson domain, pflags, mpred); 505df8bae1dSRodney W. Grimes 506df8bae1dSRodney W. Grimes /* 5070d94caffSDavid Greenman * Ran out of space, free everything up and return. Don't need 5080d94caffSDavid Greenman * to lock page queues here as we know that the pages we got 5090d94caffSDavid Greenman * aren't on any queues. 510df8bae1dSRodney W. Grimes */ 511df8bae1dSRodney W. Grimes if (m == NULL) { 5128d6fbbb8SJeff Roberson if ((flags & M_NOWAIT) == 0) 513b18bfc3dSJohn Dyson goto retry; 5148d6fbbb8SJeff Roberson VM_OBJECT_WUNLOCK(object); 5159e829b22SAlan Cox kmem_unback(object, addr, i); 516e3813573SMatthew D Fleming return (KERN_NO_SPACE); 517df8bae1dSRodney W. Grimes } 518431fb8abSMark Johnston KASSERT(vm_page_domain(m) == domain, 5197a469c8eSJeff Roberson ("kmem_back_domain: Domain mismatch %d != %d", 520431fb8abSMark Johnston vm_page_domain(m), domain)); 5211e081f88SJeff Roberson if (flags & M_ZERO && (m->flags & PG_ZERO) == 0) 522fff6062aSAlan Cox pmap_zero_page(m); 523d98d0ce2SKonstantin Belousov KASSERT((m->oflags & VPO_UNMANAGED) != 0, 5249f5c801bSAlan Cox ("kmem_malloc: page %p is managed", m)); 52591e31c3cSJeff Roberson vm_page_valid(m); 5260766f278SJonathan T. Looney pmap_enter(kernel_pmap, addr + i, m, prot, 5270766f278SJonathan T. Looney prot | PMAP_ENTER_WIRED, 0); 52849bfa624SAlan Cox if (__predict_false((prot & VM_PROT_EXECUTE) != 0)) 52949bfa624SAlan Cox m->oflags |= VPO_KMEM_EXEC; 530df8bae1dSRodney W. Grimes } 5315df87b21SJeff Roberson VM_OBJECT_WUNLOCK(object); 532*540da48dSMark Johnston kmem_alloc_san(addr, size, size, flags); 533e3813573SMatthew D Fleming return (KERN_SUCCESS); 534df8bae1dSRodney W. Grimes } 535df8bae1dSRodney W. Grimes 53625ed23cfSMark Johnston /* 53725ed23cfSMark Johnston * kmem_back: 53825ed23cfSMark Johnston * 53925ed23cfSMark Johnston * Allocate physical pages for the specified virtual address range. 54025ed23cfSMark Johnston */ 5417a469c8eSJeff Roberson int 5427a469c8eSJeff Roberson kmem_back(vm_object_t object, vm_offset_t addr, vm_size_t size, int flags) 5437a469c8eSJeff Roberson { 54425ed23cfSMark Johnston vm_offset_t end, next, start; 54525ed23cfSMark Johnston int domain, rv; 5467a469c8eSJeff Roberson 5477a469c8eSJeff Roberson KASSERT(object == kernel_object, 5487a469c8eSJeff Roberson ("kmem_back: only supports kernel object.")); 5497a469c8eSJeff Roberson 55025ed23cfSMark Johnston for (start = addr, end = addr + size; addr < end; addr = next) { 55125ed23cfSMark Johnston /* 55225ed23cfSMark Johnston * We must ensure that pages backing a given large virtual page 55325ed23cfSMark Johnston * all come from the same physical domain. 55425ed23cfSMark Johnston */ 55525ed23cfSMark Johnston if (vm_ndomains > 1) { 55625ed23cfSMark Johnston domain = (addr >> KVA_QUANTUM_SHIFT) % vm_ndomains; 55730c5525bSAndrew Gallatin while (VM_DOMAIN_EMPTY(domain)) 55830c5525bSAndrew Gallatin domain++; 55925ed23cfSMark Johnston next = roundup2(addr + 1, KVA_QUANTUM); 56025ed23cfSMark Johnston if (next > end || next < start) 56125ed23cfSMark Johnston next = end; 5623d14a7bbSMark Johnston } else { 5633d14a7bbSMark Johnston domain = 0; 56425ed23cfSMark Johnston next = end; 5653d14a7bbSMark Johnston } 56625ed23cfSMark Johnston rv = kmem_back_domain(domain, object, addr, next - addr, flags); 56725ed23cfSMark Johnston if (rv != KERN_SUCCESS) { 56825ed23cfSMark Johnston kmem_unback(object, start, addr - start); 5697a469c8eSJeff Roberson break; 57025ed23cfSMark Johnston } 57125ed23cfSMark Johnston } 57225ed23cfSMark Johnston return (rv); 5737a469c8eSJeff Roberson } 5747a469c8eSJeff Roberson 5759e829b22SAlan Cox /* 5769e829b22SAlan Cox * kmem_unback: 5779e829b22SAlan Cox * 5789e829b22SAlan Cox * Unmap and free the physical pages underlying the specified virtual 5799e829b22SAlan Cox * address range. 5809e829b22SAlan Cox * 5819e829b22SAlan Cox * A physical page must exist within the specified object at each index 5829e829b22SAlan Cox * that is being unmapped. 5839e829b22SAlan Cox */ 58449bfa624SAlan Cox static struct vmem * 5857a469c8eSJeff Roberson _kmem_unback(vm_object_t object, vm_offset_t addr, vm_size_t size) 5865df87b21SJeff Roberson { 58749bfa624SAlan Cox struct vmem *arena; 5887e05ffa6SMark Johnston vm_page_t m, next; 5897e05ffa6SMark Johnston vm_offset_t end, offset; 5907a469c8eSJeff Roberson int domain; 5915df87b21SJeff Roberson 5922e47807cSJeff Roberson KASSERT(object == kernel_object, 5932e47807cSJeff Roberson ("kmem_unback: only supports kernel object.")); 5945df87b21SJeff Roberson 5957a469c8eSJeff Roberson if (size == 0) 59649bfa624SAlan Cox return (NULL); 597fa2f411cSAlan Cox pmap_remove(kernel_pmap, addr, addr + size); 5985df87b21SJeff Roberson offset = addr - VM_MIN_KERNEL_ADDRESS; 5997e05ffa6SMark Johnston end = offset + size; 6005df87b21SJeff Roberson VM_OBJECT_WLOCK(object); 6017a469c8eSJeff Roberson m = vm_page_lookup(object, atop(offset)); 602431fb8abSMark Johnston domain = vm_page_domain(m); 60349bfa624SAlan Cox if (__predict_true((m->oflags & VPO_KMEM_EXEC) == 0)) 60449bfa624SAlan Cox arena = vm_dom[domain].vmd_kernel_arena; 60549bfa624SAlan Cox else 60649bfa624SAlan Cox arena = vm_dom[domain].vmd_kernel_rwx_arena; 6077a469c8eSJeff Roberson for (; offset < end; offset += PAGE_SIZE, m = next) { 6087e05ffa6SMark Johnston next = vm_page_next(m); 609e9ceb9ddSJeff Roberson vm_page_xbusy_claim(m); 61088ea538aSMark Johnston vm_page_unwire_noq(m); 6115df87b21SJeff Roberson vm_page_free(m); 6125df87b21SJeff Roberson } 6135df87b21SJeff Roberson VM_OBJECT_WUNLOCK(object); 6147a469c8eSJeff Roberson 61549bfa624SAlan Cox return (arena); 6167a469c8eSJeff Roberson } 6177a469c8eSJeff Roberson 6187a469c8eSJeff Roberson void 6197a469c8eSJeff Roberson kmem_unback(vm_object_t object, vm_offset_t addr, vm_size_t size) 6207a469c8eSJeff Roberson { 6217a469c8eSJeff Roberson 62249bfa624SAlan Cox (void)_kmem_unback(object, addr, size); 6235df87b21SJeff Roberson } 6245df87b21SJeff Roberson 625df8bae1dSRodney W. Grimes /* 6265df87b21SJeff Roberson * kmem_free: 6275df87b21SJeff Roberson * 6285df87b21SJeff Roberson * Free memory allocated with kmem_malloc. The size must match the 6295df87b21SJeff Roberson * original allocation. 6305df87b21SJeff Roberson */ 6315df87b21SJeff Roberson void 63249bfa624SAlan Cox kmem_free(vm_offset_t addr, vm_size_t size) 6335df87b21SJeff Roberson { 6340766f278SJonathan T. Looney struct vmem *arena; 6350766f278SJonathan T. Looney 6365df87b21SJeff Roberson size = round_page(size); 6372b914b85SMark Johnston kasan_mark((void *)addr, size, size, 0); 63849bfa624SAlan Cox arena = _kmem_unback(kernel_object, addr, size); 63949bfa624SAlan Cox if (arena != NULL) 6400766f278SJonathan T. Looney vmem_free(arena, addr, size); 6415df87b21SJeff Roberson } 6425df87b21SJeff Roberson 6435df87b21SJeff Roberson /* 6445df87b21SJeff Roberson * kmap_alloc_wait: 645df8bae1dSRodney W. Grimes * 646df8bae1dSRodney W. Grimes * Allocates pageable memory from a sub-map of the kernel. If the submap 647df8bae1dSRodney W. Grimes * has no room, the caller sleeps waiting for more memory in the submap. 648df8bae1dSRodney W. Grimes * 6491c7c3c6aSMatthew Dillon * This routine may block. 650df8bae1dSRodney W. Grimes */ 6510d94caffSDavid Greenman vm_offset_t 6526e309d75SEd Maste kmap_alloc_wait(vm_map_t map, vm_size_t size) 653df8bae1dSRodney W. Grimes { 654df8bae1dSRodney W. Grimes vm_offset_t addr; 65523955314SAlfred Perlstein 656df8bae1dSRodney W. Grimes size = round_page(size); 6573364c323SKonstantin Belousov if (!swap_reserve(size)) 6583364c323SKonstantin Belousov return (0); 659df8bae1dSRodney W. Grimes 660df8bae1dSRodney W. Grimes for (;;) { 661df8bae1dSRodney W. Grimes /* 6620d94caffSDavid Greenman * To make this work for more than one map, use the map's lock 6630d94caffSDavid Greenman * to lock out sleepers/wakers. 664df8bae1dSRodney W. Grimes */ 665df8bae1dSRodney W. Grimes vm_map_lock(map); 6669f701172SKonstantin Belousov addr = vm_map_findspace(map, vm_map_min(map), size); 6679f701172SKonstantin Belousov if (addr + size <= vm_map_max(map)) 668df8bae1dSRodney W. Grimes break; 669df8bae1dSRodney W. Grimes /* no space now; see if we can ever get space */ 670df8bae1dSRodney W. Grimes if (vm_map_max(map) - vm_map_min(map) < size) { 671df8bae1dSRodney W. Grimes vm_map_unlock(map); 6723364c323SKonstantin Belousov swap_release(size); 673df8bae1dSRodney W. Grimes return (0); 674df8bae1dSRodney W. Grimes } 6759688f931SAlan Cox map->needs_wakeup = TRUE; 6768ce2d00aSPawel Jakub Dawidek vm_map_unlock_and_wait(map, 0); 677df8bae1dSRodney W. Grimes } 67807702f72SMark Johnston vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_RW, VM_PROT_RW, 67907702f72SMark Johnston MAP_ACC_CHARGED); 680df8bae1dSRodney W. Grimes vm_map_unlock(map); 681df8bae1dSRodney W. Grimes return (addr); 682df8bae1dSRodney W. Grimes } 683df8bae1dSRodney W. Grimes 684df8bae1dSRodney W. Grimes /* 6855df87b21SJeff Roberson * kmap_free_wakeup: 686df8bae1dSRodney W. Grimes * 68724a1cce3SDavid Greenman * Returns memory to a submap of the kernel, and wakes up any processes 688df8bae1dSRodney W. Grimes * waiting for memory in that map. 689df8bae1dSRodney W. Grimes */ 6900d94caffSDavid Greenman void 6916e309d75SEd Maste kmap_free_wakeup(vm_map_t map, vm_offset_t addr, vm_size_t size) 692df8bae1dSRodney W. Grimes { 69323955314SAlfred Perlstein 694df8bae1dSRodney W. Grimes vm_map_lock(map); 695655c3490SKonstantin Belousov (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size)); 6969688f931SAlan Cox if (map->needs_wakeup) { 6979688f931SAlan Cox map->needs_wakeup = FALSE; 6989688f931SAlan Cox vm_map_wakeup(map); 6999688f931SAlan Cox } 700df8bae1dSRodney W. Grimes vm_map_unlock(map); 701df8bae1dSRodney W. Grimes } 702df8bae1dSRodney W. Grimes 7035df87b21SJeff Roberson void 70489cb2a19SMatthew D Fleming kmem_init_zero_region(void) 70589cb2a19SMatthew D Fleming { 706cfb00e5aSMatthew D Fleming vm_offset_t addr, i; 70789cb2a19SMatthew D Fleming vm_page_t m; 70889cb2a19SMatthew D Fleming 709cfb00e5aSMatthew D Fleming /* 710cfb00e5aSMatthew D Fleming * Map a single physical page of zeros to a larger virtual range. 711cfb00e5aSMatthew D Fleming * This requires less looping in places that want large amounts of 712cfb00e5aSMatthew D Fleming * zeros, while not using much more physical resources. 713cfb00e5aSMatthew D Fleming */ 7145df87b21SJeff Roberson addr = kva_alloc(ZERO_REGION_SIZE); 715a4667e09SMark Johnston m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO); 71689cb2a19SMatthew D Fleming for (i = 0; i < ZERO_REGION_SIZE; i += PAGE_SIZE) 71789cb2a19SMatthew D Fleming pmap_qenter(addr + i, &m, 1); 7185df87b21SJeff Roberson pmap_protect(kernel_pmap, addr, addr + ZERO_REGION_SIZE, VM_PROT_READ); 71989cb2a19SMatthew D Fleming 72089cb2a19SMatthew D Fleming zero_region = (const void *)addr; 72189cb2a19SMatthew D Fleming } 72289cb2a19SMatthew D Fleming 723df8bae1dSRodney W. Grimes /* 724969e147aSMark Johnston * Import KVA from the kernel map into the kernel arena. 7251aed6d48SMark Johnston */ 7261aed6d48SMark Johnston static int 7271aed6d48SMark Johnston kva_import(void *unused, vmem_size_t size, int flags, vmem_addr_t *addrp) 7281aed6d48SMark Johnston { 7291aed6d48SMark Johnston vm_offset_t addr; 7301aed6d48SMark Johnston int result; 7311aed6d48SMark Johnston 7321aed6d48SMark Johnston KASSERT((size % KVA_QUANTUM) == 0, 7331aed6d48SMark Johnston ("kva_import: Size %jd is not a multiple of %d", 7341aed6d48SMark Johnston (intmax_t)size, (int)KVA_QUANTUM)); 7351aed6d48SMark Johnston addr = vm_map_min(kernel_map); 7361aed6d48SMark Johnston result = vm_map_find(kernel_map, NULL, 0, &addr, size, 0, 7371aed6d48SMark Johnston VMFS_SUPER_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 7381aed6d48SMark Johnston if (result != KERN_SUCCESS) 7391aed6d48SMark Johnston return (ENOMEM); 7401aed6d48SMark Johnston 7411aed6d48SMark Johnston *addrp = addr; 7421aed6d48SMark Johnston 7431aed6d48SMark Johnston return (0); 7441aed6d48SMark Johnston } 7451aed6d48SMark Johnston 7461aed6d48SMark Johnston /* 747969e147aSMark Johnston * Import KVA from a parent arena into a per-domain arena. Imports must be 748969e147aSMark Johnston * KVA_QUANTUM-aligned and a multiple of KVA_QUANTUM in size. 7491aed6d48SMark Johnston */ 7501aed6d48SMark Johnston static int 751969e147aSMark Johnston kva_import_domain(void *arena, vmem_size_t size, int flags, vmem_addr_t *addrp) 7521aed6d48SMark Johnston { 7531aed6d48SMark Johnston 7541aed6d48SMark Johnston KASSERT((size % KVA_QUANTUM) == 0, 755969e147aSMark Johnston ("kva_import_domain: Size %jd is not a multiple of %d", 7561aed6d48SMark Johnston (intmax_t)size, (int)KVA_QUANTUM)); 7571aed6d48SMark Johnston return (vmem_xalloc(arena, size, KVA_QUANTUM, 0, 0, VMEM_ADDR_MIN, 7581aed6d48SMark Johnston VMEM_ADDR_MAX, flags, addrp)); 7591aed6d48SMark Johnston } 7601aed6d48SMark Johnston 7611aed6d48SMark Johnston /* 7621c7c3c6aSMatthew Dillon * kmem_init: 7631c7c3c6aSMatthew Dillon * 7641c7c3c6aSMatthew Dillon * Create the kernel map; insert a mapping covering kernel text, 7651c7c3c6aSMatthew Dillon * data, bss, and all space allocated thus far (`boostrap' data). The 7661c7c3c6aSMatthew Dillon * new map will thus map the range between VM_MIN_KERNEL_ADDRESS and 7671c7c3c6aSMatthew Dillon * `start' as allocated, and the range between `start' and `end' as free. 7681aed6d48SMark Johnston * Create the kernel vmem arena and its per-domain children. 769df8bae1dSRodney W. Grimes */ 7700d94caffSDavid Greenman void 7716e309d75SEd Maste kmem_init(vm_offset_t start, vm_offset_t end) 772df8bae1dSRodney W. Grimes { 773aea9103eSMark Johnston vm_size_t quantum; 7741aed6d48SMark Johnston int domain; 775df8bae1dSRodney W. Grimes 7767efe14cbSMark Johnston vm_map_init(kernel_map, kernel_pmap, VM_MIN_KERNEL_ADDRESS, end); 7777efe14cbSMark Johnston kernel_map->system_map = 1; 7787efe14cbSMark Johnston vm_map_lock(kernel_map); 779df8bae1dSRodney W. Grimes /* N.B.: cannot use kgdb to debug, starting with this assignment ... */ 7807efe14cbSMark Johnston (void)vm_map_insert(kernel_map, NULL, 0, 7815cfa90e9SAlan Cox #ifdef __amd64__ 7825cfa90e9SAlan Cox KERNBASE, 7835cfa90e9SAlan Cox #else 7845cfa90e9SAlan Cox VM_MIN_KERNEL_ADDRESS, 7855cfa90e9SAlan Cox #endif 7865cfa90e9SAlan Cox start, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 787df8bae1dSRodney W. Grimes /* ... and ending with the completion of the above `insert' */ 7889d75f0dcSMark Johnston 7899d75f0dcSMark Johnston #ifdef __amd64__ 7909d75f0dcSMark Johnston /* 7919d75f0dcSMark Johnston * Mark KVA used for the page array as allocated. Other platforms 7929d75f0dcSMark Johnston * that handle vm_page_array allocation can simply adjust virtual_avail 7939d75f0dcSMark Johnston * instead. 7949d75f0dcSMark Johnston */ 7957dd979dfSMark Johnston (void)vm_map_insert(kernel_map, NULL, 0, (vm_offset_t)vm_page_array, 7969d75f0dcSMark Johnston (vm_offset_t)vm_page_array + round_2mpage(vm_page_array_size * 7979d75f0dcSMark Johnston sizeof(struct vm_page)), 7989d75f0dcSMark Johnston VM_PROT_RW, VM_PROT_RW, MAP_NOFAULT); 7999d75f0dcSMark Johnston #endif 8007dd979dfSMark Johnston vm_map_unlock(kernel_map); 8011aed6d48SMark Johnston 8021aed6d48SMark Johnston /* 803aea9103eSMark Johnston * Use a large import quantum on NUMA systems. This helps minimize 804aea9103eSMark Johnston * interleaving of superpages, reducing internal fragmentation within 805aea9103eSMark Johnston * the per-domain arenas. 806aea9103eSMark Johnston */ 807aea9103eSMark Johnston if (vm_ndomains > 1 && PMAP_HAS_DMAP) 808aea9103eSMark Johnston quantum = KVA_NUMA_IMPORT_QUANTUM; 809aea9103eSMark Johnston else 810aea9103eSMark Johnston quantum = KVA_QUANTUM; 811aea9103eSMark Johnston 812aea9103eSMark Johnston /* 8131aed6d48SMark Johnston * Initialize the kernel_arena. This can grow on demand. 8141aed6d48SMark Johnston */ 8151aed6d48SMark Johnston vmem_init(kernel_arena, "kernel arena", 0, 0, PAGE_SIZE, 0, 0); 816aea9103eSMark Johnston vmem_set_import(kernel_arena, kva_import, NULL, NULL, quantum); 8171aed6d48SMark Johnston 8181aed6d48SMark Johnston for (domain = 0; domain < vm_ndomains; domain++) { 819969e147aSMark Johnston /* 820969e147aSMark Johnston * Initialize the per-domain arenas. These are used to color 821969e147aSMark Johnston * the KVA space in a way that ensures that virtual large pages 822969e147aSMark Johnston * are backed by memory from the same physical domain, 823969e147aSMark Johnston * maximizing the potential for superpage promotion. 824969e147aSMark Johnston */ 8251aed6d48SMark Johnston vm_dom[domain].vmd_kernel_arena = vmem_create( 8261aed6d48SMark Johnston "kernel arena domain", 0, 0, PAGE_SIZE, 0, M_WAITOK); 8271aed6d48SMark Johnston vmem_set_import(vm_dom[domain].vmd_kernel_arena, 828aea9103eSMark Johnston kva_import_domain, NULL, kernel_arena, quantum); 829969e147aSMark Johnston 830969e147aSMark Johnston /* 831969e147aSMark Johnston * In architectures with superpages, maintain separate arenas 832969e147aSMark Johnston * for allocations with permissions that differ from the 833969e147aSMark Johnston * "standard" read/write permissions used for kernel memory, 834969e147aSMark Johnston * so as not to inhibit superpage promotion. 835aea9103eSMark Johnston * 836aea9103eSMark Johnston * Use the base import quantum since this arena is rarely used. 837969e147aSMark Johnston */ 8381aed6d48SMark Johnston #if VM_NRESERVLEVEL > 0 8391aed6d48SMark Johnston vm_dom[domain].vmd_kernel_rwx_arena = vmem_create( 8401aed6d48SMark Johnston "kernel rwx arena domain", 0, 0, PAGE_SIZE, 0, M_WAITOK); 8411aed6d48SMark Johnston vmem_set_import(vm_dom[domain].vmd_kernel_rwx_arena, 842969e147aSMark Johnston kva_import_domain, (vmem_release_t *)vmem_xfree, 843969e147aSMark Johnston kernel_arena, KVA_QUANTUM); 844303b7702SMark Johnston #else 845303b7702SMark Johnston vm_dom[domain].vmd_kernel_rwx_arena = 846303b7702SMark Johnston vm_dom[domain].vmd_kernel_arena; 8471aed6d48SMark Johnston #endif 8481aed6d48SMark Johnston } 849a81c400eSJeff Roberson 850a81c400eSJeff Roberson /* 851a81c400eSJeff Roberson * This must be the very first call so that the virtual address 852a81c400eSJeff Roberson * space used for early allocations is properly marked used in 853a81c400eSJeff Roberson * the map. 854a81c400eSJeff Roberson */ 855a81c400eSJeff Roberson uma_startup2(); 856df8bae1dSRodney W. Grimes } 85786f08737SRobert Watson 858483f692eSMark Johnston /* 859483f692eSMark Johnston * kmem_bootstrap_free: 860483f692eSMark Johnston * 861483f692eSMark Johnston * Free pages backing preloaded data (e.g., kernel modules) to the 862483f692eSMark Johnston * system. Currently only supported on platforms that create a 863483f692eSMark Johnston * vm_phys segment for preloaded data. 864483f692eSMark Johnston */ 865483f692eSMark Johnston void 866483f692eSMark Johnston kmem_bootstrap_free(vm_offset_t start, vm_size_t size) 867483f692eSMark Johnston { 868483f692eSMark Johnston #if defined(__i386__) || defined(__amd64__) 869483f692eSMark Johnston struct vm_domain *vmd; 8706c85795aSMark Johnston vm_offset_t end, va; 871483f692eSMark Johnston vm_paddr_t pa; 872483f692eSMark Johnston vm_page_t m; 873483f692eSMark Johnston 874483f692eSMark Johnston end = trunc_page(start + size); 875483f692eSMark Johnston start = round_page(start); 876483f692eSMark Johnston 877f822c9e2SMark Johnston #ifdef __amd64__ 878f822c9e2SMark Johnston /* 879f822c9e2SMark Johnston * Preloaded files do not have execute permissions by default on amd64. 880f822c9e2SMark Johnston * Restore the default permissions to ensure that the direct map alias 881f822c9e2SMark Johnston * is updated. 882f822c9e2SMark Johnston */ 883f822c9e2SMark Johnston pmap_change_prot(start, end - start, VM_PROT_RW); 884f822c9e2SMark Johnston #endif 8856c85795aSMark Johnston for (va = start; va < end; va += PAGE_SIZE) { 8866c85795aSMark Johnston pa = pmap_kextract(va); 887483f692eSMark Johnston m = PHYS_TO_VM_PAGE(pa); 888483f692eSMark Johnston 889483f692eSMark Johnston vmd = vm_pagequeue_domain(m); 890483f692eSMark Johnston vm_domain_free_lock(vmd); 891483f692eSMark Johnston vm_phys_free_pages(m, 0); 892483f692eSMark Johnston vm_domain_free_unlock(vmd); 893c16bd872SMark Johnston 894c16bd872SMark Johnston vm_domain_freecnt_inc(vmd, 1); 895c16bd872SMark Johnston vm_cnt.v_page_count++; 896483f692eSMark Johnston } 8976c85795aSMark Johnston pmap_remove(kernel_pmap, start, end); 8986c85795aSMark Johnston (void)vmem_add(kernel_arena, start, end - start, M_WAITOK); 899483f692eSMark Johnston #endif 900483f692eSMark Johnston } 901483f692eSMark Johnston 90286f08737SRobert Watson /* 90386f08737SRobert Watson * Allow userspace to directly trigger the VM drain routine for testing 90486f08737SRobert Watson * purposes. 90586f08737SRobert Watson */ 90686f08737SRobert Watson static int 90786f08737SRobert Watson debug_vm_lowmem(SYSCTL_HANDLER_ARGS) 90886f08737SRobert Watson { 90986f08737SRobert Watson int error, i; 91086f08737SRobert Watson 91186f08737SRobert Watson i = 0; 91286f08737SRobert Watson error = sysctl_handle_int(oidp, &i, 0, req); 91351a7be5fSKonstantin Belousov if (error != 0) 91486f08737SRobert Watson return (error); 9159b43bc27SAndriy Gapon if ((i & ~(VM_LOW_KMEM | VM_LOW_PAGES)) != 0) 9169b43bc27SAndriy Gapon return (EINVAL); 9179b43bc27SAndriy Gapon if (i != 0) 9189b43bc27SAndriy Gapon EVENTHANDLER_INVOKE(vm_lowmem, i); 91986f08737SRobert Watson return (0); 92086f08737SRobert Watson } 92151a7be5fSKonstantin Belousov SYSCTL_PROC(_debug, OID_AUTO, vm_lowmem, 92251a7be5fSKonstantin Belousov CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, 0, debug_vm_lowmem, "I", 92351a7be5fSKonstantin Belousov "set to trigger vm_lowmem event with given flags"); 92489619b74SKonstantin Belousov 92589619b74SKonstantin Belousov static int 92689619b74SKonstantin Belousov debug_uma_reclaim(SYSCTL_HANDLER_ARGS) 92789619b74SKonstantin Belousov { 92889619b74SKonstantin Belousov int error, i; 92989619b74SKonstantin Belousov 93089619b74SKonstantin Belousov i = 0; 93189619b74SKonstantin Belousov error = sysctl_handle_int(oidp, &i, 0, req); 932aabe13f1SMark Johnston if (error != 0 || req->newptr == NULL) 93389619b74SKonstantin Belousov return (error); 93489619b74SKonstantin Belousov if (i != UMA_RECLAIM_TRIM && i != UMA_RECLAIM_DRAIN && 93589619b74SKonstantin Belousov i != UMA_RECLAIM_DRAIN_CPU) 93689619b74SKonstantin Belousov return (EINVAL); 93789619b74SKonstantin Belousov uma_reclaim(i); 93889619b74SKonstantin Belousov return (0); 93989619b74SKonstantin Belousov } 94089619b74SKonstantin Belousov SYSCTL_PROC(_debug, OID_AUTO, uma_reclaim, 94189619b74SKonstantin Belousov CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, 0, debug_uma_reclaim, "I", 94289619b74SKonstantin Belousov "set to generate request to reclaim uma caches"); 943aabe13f1SMark Johnston 944aabe13f1SMark Johnston static int 945aabe13f1SMark Johnston debug_uma_reclaim_domain(SYSCTL_HANDLER_ARGS) 946aabe13f1SMark Johnston { 947aabe13f1SMark Johnston int domain, error, request; 948aabe13f1SMark Johnston 949aabe13f1SMark Johnston request = 0; 950aabe13f1SMark Johnston error = sysctl_handle_int(oidp, &request, 0, req); 951aabe13f1SMark Johnston if (error != 0 || req->newptr == NULL) 952aabe13f1SMark Johnston return (error); 953aabe13f1SMark Johnston 954aabe13f1SMark Johnston domain = request >> 4; 955aabe13f1SMark Johnston request &= 0xf; 956aabe13f1SMark Johnston if (request != UMA_RECLAIM_TRIM && request != UMA_RECLAIM_DRAIN && 957aabe13f1SMark Johnston request != UMA_RECLAIM_DRAIN_CPU) 958aabe13f1SMark Johnston return (EINVAL); 959aabe13f1SMark Johnston if (domain < 0 || domain >= vm_ndomains) 960aabe13f1SMark Johnston return (EINVAL); 961aabe13f1SMark Johnston uma_reclaim_domain(request, domain); 962aabe13f1SMark Johnston return (0); 963aabe13f1SMark Johnston } 964aabe13f1SMark Johnston SYSCTL_PROC(_debug, OID_AUTO, uma_reclaim_domain, 965aabe13f1SMark Johnston CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, 0, 966aabe13f1SMark Johnston debug_uma_reclaim_domain, "I", 967aabe13f1SMark Johnston ""); 968