1df8bae1dSRodney W. Grimes /* 2df8bae1dSRodney W. Grimes * Copyright (c) 1987, 1991, 1993 3df8bae1dSRodney W. Grimes * The Regents of the University of California. All rights reserved. 4df8bae1dSRodney W. Grimes * 5df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 6df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 7df8bae1dSRodney W. Grimes * are met: 8df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 9df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 10df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 11df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 12df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 13df8bae1dSRodney W. Grimes * 3. All advertising materials mentioning features or use of this software 14df8bae1dSRodney W. Grimes * must display the following acknowledgement: 15df8bae1dSRodney W. Grimes * This product includes software developed by the University of 16df8bae1dSRodney W. Grimes * California, Berkeley and its contributors. 17df8bae1dSRodney W. Grimes * 4. Neither the name of the University nor the names of its contributors 18df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 19df8bae1dSRodney W. Grimes * without specific prior written permission. 20df8bae1dSRodney W. Grimes * 21df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31df8bae1dSRodney W. Grimes * SUCH DAMAGE. 32df8bae1dSRodney W. Grimes * 33df8bae1dSRodney W. Grimes * @(#)kern_malloc.c 8.3 (Berkeley) 1/4/94 34c3aac50fSPeter Wemm * $FreeBSD$ 35df8bae1dSRodney W. Grimes */ 36df8bae1dSRodney W. Grimes 378a58a9f6SJohn Dyson #include "opt_vm.h" 388a58a9f6SJohn Dyson 39df8bae1dSRodney W. Grimes #include <sys/param.h> 4026f9a767SRodney W. Grimes #include <sys/systm.h> 41df8bae1dSRodney W. Grimes #include <sys/kernel.h> 42fb919e4dSMark Murray #include <sys/lock.h> 43df8bae1dSRodney W. Grimes #include <sys/malloc.h> 4454e7152cSDavid Greenman #include <sys/mbuf.h> 45eec258d2SJohn Baldwin #include <sys/mutex.h> 46efeaf95aSDavid Greenman #include <sys/vmmeter.h> 47a448b62aSJake Burkholder #include <sys/proc.h> 486f267175SJeff Roberson #include <sys/sysctl.h> 499a02e8c6SJason Evans 50df8bae1dSRodney W. Grimes #include <vm/vm.h> 51efeaf95aSDavid Greenman #include <vm/vm_param.h> 52df8bae1dSRodney W. Grimes #include <vm/vm_kern.h> 53efeaf95aSDavid Greenman #include <vm/vm_extern.h> 543075778bSJohn Dyson #include <vm/pmap.h> 553075778bSJohn Dyson #include <vm/vm_map.h> 568355f576SJeff Roberson #include <vm/uma.h> 578355f576SJeff Roberson #include <vm/uma_int.h> 588efc4effSJeff Roberson #include <vm/uma_dbg.h> 59df8bae1dSRodney W. Grimes 60984982d6SPoul-Henning Kamp #if defined(INVARIANTS) && defined(__i386__) 61984982d6SPoul-Henning Kamp #include <machine/cpu.h> 62984982d6SPoul-Henning Kamp #endif 63984982d6SPoul-Henning Kamp 6444a8ff31SArchie Cobbs /* 6544a8ff31SArchie Cobbs * When realloc() is called, if the new size is sufficiently smaller than 6644a8ff31SArchie Cobbs * the old size, realloc() will allocate a new, smaller block to avoid 6744a8ff31SArchie Cobbs * wasting memory. 'Sufficiently smaller' is defined as: newsize <= 6844a8ff31SArchie Cobbs * oldsize / 2^n, where REALLOC_FRACTION defines the value of 'n'. 6944a8ff31SArchie Cobbs */ 7044a8ff31SArchie Cobbs #ifndef REALLOC_FRACTION 7144a8ff31SArchie Cobbs #define REALLOC_FRACTION 1 /* new block if <= half the size */ 7244a8ff31SArchie Cobbs #endif 7344a8ff31SArchie Cobbs 743b6fb885SPoul-Henning Kamp MALLOC_DEFINE(M_CACHE, "cache", "Various Dynamically allocated caches"); 759ef246c6SBruce Evans MALLOC_DEFINE(M_DEVBUF, "devbuf", "device driver memory"); 769ef246c6SBruce Evans MALLOC_DEFINE(M_TEMP, "temp", "misc temporary data buffers"); 779ef246c6SBruce Evans 7882cd038dSYoshinobu Inoue MALLOC_DEFINE(M_IP6OPT, "ip6opt", "IPv6 options"); 7982cd038dSYoshinobu Inoue MALLOC_DEFINE(M_IP6NDP, "ip6ndp", "IPv6 Neighbor Discovery"); 8082cd038dSYoshinobu Inoue 814d77a549SAlfred Perlstein static void kmeminit(void *); 822b14f991SJulian Elischer SYSINIT(kmem, SI_SUB_KMEM, SI_ORDER_FIRST, kmeminit, NULL) 832b14f991SJulian Elischer 84a1c995b6SPoul-Henning Kamp static MALLOC_DEFINE(M_FREE, "free", "should be on free list"); 85a1c995b6SPoul-Henning Kamp 86db669378SPeter Wemm static struct malloc_type *kmemstatistics; 87254c6cb3SPoul-Henning Kamp static char *kmembase; 88043a2f3bSBruce Evans static char *kmemlimit; 891f6889a1SMatthew Dillon 908355f576SJeff Roberson #define KMEM_ZSHIFT 4 918355f576SJeff Roberson #define KMEM_ZBASE 16 928355f576SJeff Roberson #define KMEM_ZMASK (KMEM_ZBASE - 1) 938355f576SJeff Roberson 94bd796eb2SRobert Watson #define KMEM_ZMAX 65536 958355f576SJeff Roberson #define KMEM_ZSIZE (KMEM_ZMAX >> KMEM_ZSHIFT) 966f267175SJeff Roberson static u_int8_t kmemsize[KMEM_ZSIZE + 1]; 976f267175SJeff Roberson 988355f576SJeff Roberson /* These won't be powers of two for long */ 998355f576SJeff Roberson struct { 1006f267175SJeff Roberson int kz_size; 1016f267175SJeff Roberson char *kz_name; 1026f267175SJeff Roberson uma_zone_t kz_zone; 1036f267175SJeff Roberson } kmemzones[] = { 1046f267175SJeff Roberson {16, "16", NULL}, 1056f267175SJeff Roberson {32, "32", NULL}, 1066f267175SJeff Roberson {64, "64", NULL}, 1076f267175SJeff Roberson {128, "128", NULL}, 1086f267175SJeff Roberson {256, "256", NULL}, 1096f267175SJeff Roberson {512, "512", NULL}, 1106f267175SJeff Roberson {1024, "1024", NULL}, 1116f267175SJeff Roberson {2048, "2048", NULL}, 1126f267175SJeff Roberson {4096, "4096", NULL}, 1136f267175SJeff Roberson {8192, "8192", NULL}, 11443a7c4e9SRobert Watson {16384, "16384", NULL}, 115bd796eb2SRobert Watson {32768, "32768", NULL}, 116bd796eb2SRobert Watson {65536, "65536", NULL}, 1178355f576SJeff Roberson {0, NULL}, 1188355f576SJeff Roberson }; 1198355f576SJeff Roberson 1206f267175SJeff Roberson u_int vm_kmem_size; 1215a34a9f0SJeff Roberson 1225a34a9f0SJeff Roberson /* 1235a34a9f0SJeff Roberson * The malloc_mtx protects the kmemstatistics linked list as well as the 1245a34a9f0SJeff Roberson * mallochash. 1255a34a9f0SJeff Roberson */ 1265a34a9f0SJeff Roberson 1275a34a9f0SJeff Roberson struct mtx malloc_mtx; 12869ef67f9SJason Evans 1295e914b96SJeff Roberson #ifdef MALLOC_PROFILE 1305e914b96SJeff Roberson uint64_t krequests[KMEM_ZSIZE + 1]; 1316f267175SJeff Roberson 1325e914b96SJeff Roberson static int sysctl_kern_mprof(SYSCTL_HANDLER_ARGS); 1335e914b96SJeff Roberson #endif 1345e914b96SJeff Roberson 1355e914b96SJeff Roberson static int sysctl_kern_malloc(SYSCTL_HANDLER_ARGS); 136df8bae1dSRodney W. Grimes 137df8bae1dSRodney W. Grimes /* 1381c7c3c6aSMatthew Dillon * malloc: 1391c7c3c6aSMatthew Dillon * 1401c7c3c6aSMatthew Dillon * Allocate a block of memory. 1411c7c3c6aSMatthew Dillon * 1421c7c3c6aSMatthew Dillon * If M_NOWAIT is set, this routine will not block and return NULL if 1431c7c3c6aSMatthew Dillon * the allocation fails. 144df8bae1dSRodney W. Grimes */ 145df8bae1dSRodney W. Grimes void * 146df8bae1dSRodney W. Grimes malloc(size, type, flags) 147df8bae1dSRodney W. Grimes unsigned long size; 14860a513e9SPoul-Henning Kamp struct malloc_type *type; 149254c6cb3SPoul-Henning Kamp int flags; 150df8bae1dSRodney W. Grimes { 1516f267175SJeff Roberson int indx; 1528355f576SJeff Roberson caddr_t va; 1538355f576SJeff Roberson uma_zone_t zone; 15460a513e9SPoul-Henning Kamp register struct malloc_type *ksp = type; 155df8bae1dSRodney W. Grimes 156708da94eSPoul-Henning Kamp #if 0 157708da94eSPoul-Henning Kamp if (size == 0) 158708da94eSPoul-Henning Kamp Debugger("zero size malloc"); 159708da94eSPoul-Henning Kamp #endif 1600fee3d35SPeter Wemm #if defined(INVARIANTS) 161984982d6SPoul-Henning Kamp if (flags == M_WAITOK) 162b40ce416SJulian Elischer KASSERT(curthread->td_intr_nesting_level == 0, 163984982d6SPoul-Henning Kamp ("malloc(M_WAITOK) in interrupt context")); 164984982d6SPoul-Henning Kamp #endif 1658355f576SJeff Roberson if (size <= KMEM_ZMAX) { 1666f267175SJeff Roberson if (size & KMEM_ZMASK) 1676f267175SJeff Roberson size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; 1686f267175SJeff Roberson indx = kmemsize[size >> KMEM_ZSHIFT]; 1696f267175SJeff Roberson zone = kmemzones[indx].kz_zone; 1706f267175SJeff Roberson #ifdef MALLOC_PROFILE 1716f267175SJeff Roberson krequests[size >> KMEM_ZSHIFT]++; 1726f267175SJeff Roberson #endif 1738355f576SJeff Roberson va = uma_zalloc(zone, flags); 1745a34a9f0SJeff Roberson mtx_lock(&ksp->ks_mtx); 1756f267175SJeff Roberson if (va == NULL) 176df8bae1dSRodney W. Grimes goto out; 1776f267175SJeff Roberson 1786f267175SJeff Roberson ksp->ks_size |= 1 << indx; 1796f267175SJeff Roberson size = zone->uz_size; 1808355f576SJeff Roberson } else { 1816f267175SJeff Roberson size = roundup(size, PAGE_SIZE); 1828355f576SJeff Roberson zone = NULL; 1838355f576SJeff Roberson va = uma_large_malloc(size, flags); 1845a34a9f0SJeff Roberson mtx_lock(&ksp->ks_mtx); 1856f267175SJeff Roberson if (va == NULL) 1868355f576SJeff Roberson goto out; 187df8bae1dSRodney W. Grimes } 1886f267175SJeff Roberson ksp->ks_memuse += size; 189df8bae1dSRodney W. Grimes ksp->ks_inuse++; 1908355f576SJeff Roberson out: 191df8bae1dSRodney W. Grimes ksp->ks_calls++; 192df8bae1dSRodney W. Grimes if (ksp->ks_memuse > ksp->ks_maxused) 193df8bae1dSRodney W. Grimes ksp->ks_maxused = ksp->ks_memuse; 1946f267175SJeff Roberson 1955a34a9f0SJeff Roberson mtx_unlock(&ksp->ks_mtx); 196df8bae1dSRodney W. Grimes return ((void *) va); 197df8bae1dSRodney W. Grimes } 198df8bae1dSRodney W. Grimes 199df8bae1dSRodney W. Grimes /* 2001c7c3c6aSMatthew Dillon * free: 2011c7c3c6aSMatthew Dillon * 202df8bae1dSRodney W. Grimes * Free a block of memory allocated by malloc. 2031c7c3c6aSMatthew Dillon * 2041c7c3c6aSMatthew Dillon * This routine may not block. 205df8bae1dSRodney W. Grimes */ 206df8bae1dSRodney W. Grimes void 207df8bae1dSRodney W. Grimes free(addr, type) 208df8bae1dSRodney W. Grimes void *addr; 20960a513e9SPoul-Henning Kamp struct malloc_type *type; 210df8bae1dSRodney W. Grimes { 2118355f576SJeff Roberson uma_slab_t slab; 2128355f576SJeff Roberson void *mem; 2138355f576SJeff Roberson u_long size; 21460a513e9SPoul-Henning Kamp register struct malloc_type *ksp = type; 215254c6cb3SPoul-Henning Kamp 21644a8ff31SArchie Cobbs /* free(NULL, ...) does nothing */ 21744a8ff31SArchie Cobbs if (addr == NULL) 21844a8ff31SArchie Cobbs return; 21944a8ff31SArchie Cobbs 2208355f576SJeff Roberson size = 0; 22169ef67f9SJason Evans 2228355f576SJeff Roberson mem = (void *)((u_long)addr & (~UMA_SLAB_MASK)); 2235a34a9f0SJeff Roberson mtx_lock(&malloc_mtx); 2248355f576SJeff Roberson slab = hash_sfind(mallochash, mem); 2255a34a9f0SJeff Roberson mtx_unlock(&malloc_mtx); 2268355f576SJeff Roberson 2278355f576SJeff Roberson if (slab == NULL) 2286f267175SJeff Roberson panic("free: address %p(%p) has not been allocated.\n", 2296f267175SJeff Roberson addr, mem); 2308355f576SJeff Roberson 2318355f576SJeff Roberson if (!(slab->us_flags & UMA_SLAB_MALLOC)) { 2328355f576SJeff Roberson size = slab->us_zone->uz_size; 2338355f576SJeff Roberson uma_zfree_arg(slab->us_zone, addr, slab); 23414bf02f8SJohn Dyson } else { 2358355f576SJeff Roberson size = slab->us_size; 2368355f576SJeff Roberson uma_large_free(slab); 23714bf02f8SJohn Dyson } 2385a34a9f0SJeff Roberson mtx_lock(&ksp->ks_mtx); 2398355f576SJeff Roberson ksp->ks_memuse -= size; 2408355f576SJeff Roberson ksp->ks_inuse--; 2415a34a9f0SJeff Roberson mtx_unlock(&ksp->ks_mtx); 242df8bae1dSRodney W. Grimes } 243df8bae1dSRodney W. Grimes 244df8bae1dSRodney W. Grimes /* 24544a8ff31SArchie Cobbs * realloc: change the size of a memory block 24644a8ff31SArchie Cobbs */ 24744a8ff31SArchie Cobbs void * 24844a8ff31SArchie Cobbs realloc(addr, size, type, flags) 24944a8ff31SArchie Cobbs void *addr; 25044a8ff31SArchie Cobbs unsigned long size; 25144a8ff31SArchie Cobbs struct malloc_type *type; 25244a8ff31SArchie Cobbs int flags; 25344a8ff31SArchie Cobbs { 2548355f576SJeff Roberson uma_slab_t slab; 25544a8ff31SArchie Cobbs unsigned long alloc; 25644a8ff31SArchie Cobbs void *newaddr; 25744a8ff31SArchie Cobbs 25844a8ff31SArchie Cobbs /* realloc(NULL, ...) is equivalent to malloc(...) */ 25944a8ff31SArchie Cobbs if (addr == NULL) 26044a8ff31SArchie Cobbs return (malloc(size, type, flags)); 26144a8ff31SArchie Cobbs 2625a34a9f0SJeff Roberson mtx_lock(&malloc_mtx); 2638355f576SJeff Roberson slab = hash_sfind(mallochash, 2648355f576SJeff Roberson (void *)((u_long)addr & ~(UMA_SLAB_MASK))); 2655a34a9f0SJeff Roberson mtx_unlock(&malloc_mtx); 2668355f576SJeff Roberson 26744a8ff31SArchie Cobbs /* Sanity check */ 2688355f576SJeff Roberson KASSERT(slab != NULL, 26944a8ff31SArchie Cobbs ("realloc: address %p out of range", (void *)addr)); 27044a8ff31SArchie Cobbs 27144a8ff31SArchie Cobbs /* Get the size of the original block */ 2728355f576SJeff Roberson if (slab->us_zone) 2738355f576SJeff Roberson alloc = slab->us_zone->uz_size; 2748355f576SJeff Roberson else 2758355f576SJeff Roberson alloc = slab->us_size; 27644a8ff31SArchie Cobbs 27744a8ff31SArchie Cobbs /* Reuse the original block if appropriate */ 27844a8ff31SArchie Cobbs if (size <= alloc 27944a8ff31SArchie Cobbs && (size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE)) 28044a8ff31SArchie Cobbs return (addr); 28144a8ff31SArchie Cobbs 28244a8ff31SArchie Cobbs /* Allocate a new, bigger (or smaller) block */ 28344a8ff31SArchie Cobbs if ((newaddr = malloc(size, type, flags)) == NULL) 28444a8ff31SArchie Cobbs return (NULL); 28544a8ff31SArchie Cobbs 28644a8ff31SArchie Cobbs /* Copy over original contents */ 28744a8ff31SArchie Cobbs bcopy(addr, newaddr, min(size, alloc)); 28844a8ff31SArchie Cobbs free(addr, type); 28944a8ff31SArchie Cobbs return (newaddr); 29044a8ff31SArchie Cobbs } 29144a8ff31SArchie Cobbs 29244a8ff31SArchie Cobbs /* 29344a8ff31SArchie Cobbs * reallocf: same as realloc() but free memory on failure. 29444a8ff31SArchie Cobbs */ 29544a8ff31SArchie Cobbs void * 29644a8ff31SArchie Cobbs reallocf(addr, size, type, flags) 29744a8ff31SArchie Cobbs void *addr; 29844a8ff31SArchie Cobbs unsigned long size; 29944a8ff31SArchie Cobbs struct malloc_type *type; 30044a8ff31SArchie Cobbs int flags; 30144a8ff31SArchie Cobbs { 30244a8ff31SArchie Cobbs void *mem; 30344a8ff31SArchie Cobbs 30444a8ff31SArchie Cobbs if ((mem = realloc(addr, size, type, flags)) == NULL) 30544a8ff31SArchie Cobbs free(addr, type); 30644a8ff31SArchie Cobbs return (mem); 30744a8ff31SArchie Cobbs } 30844a8ff31SArchie Cobbs 30944a8ff31SArchie Cobbs /* 310df8bae1dSRodney W. Grimes * Initialize the kernel memory allocator 311df8bae1dSRodney W. Grimes */ 3122b14f991SJulian Elischer /* ARGSUSED*/ 3132b14f991SJulian Elischer static void 314d841aaa7SBruce Evans kmeminit(dummy) 315d841aaa7SBruce Evans void *dummy; 316df8bae1dSRodney W. Grimes { 3176f267175SJeff Roberson u_int8_t indx; 31827b8623fSDavid Greenman u_long npg; 31927b8623fSDavid Greenman u_long mem_size; 3208355f576SJeff Roberson void *hashmem; 3218355f576SJeff Roberson u_long hashsize; 3228355f576SJeff Roberson int highbit; 3238355f576SJeff Roberson int bits; 3248355f576SJeff Roberson int i; 3258a58a9f6SJohn Dyson 3266008862bSJohn Baldwin mtx_init(&malloc_mtx, "malloc", NULL, MTX_DEF); 32769ef67f9SJason Evans 3288a58a9f6SJohn Dyson /* 3298a58a9f6SJohn Dyson * Try to auto-tune the kernel memory size, so that it is 3308a58a9f6SJohn Dyson * more applicable for a wider range of machine sizes. 3318a58a9f6SJohn Dyson * On an X86, a VM_KMEM_SIZE_SCALE value of 4 is good, while 3328a58a9f6SJohn Dyson * a VM_KMEM_SIZE of 12MB is a fair compromise. The 3338a58a9f6SJohn Dyson * VM_KMEM_SIZE_MAX is dependent on the maximum KVA space 3348a58a9f6SJohn Dyson * available, and on an X86 with a total KVA space of 256MB, 3358a58a9f6SJohn Dyson * try to keep VM_KMEM_SIZE_MAX at 80MB or below. 3368a58a9f6SJohn Dyson * 3378a58a9f6SJohn Dyson * Note that the kmem_map is also used by the zone allocator, 3388a58a9f6SJohn Dyson * so make sure that there is enough space. 3398a58a9f6SJohn Dyson */ 34081930014SPeter Wemm vm_kmem_size = VM_KMEM_SIZE; 3418a58a9f6SJohn Dyson mem_size = cnt.v_page_count * PAGE_SIZE; 3428a58a9f6SJohn Dyson 3438a58a9f6SJohn Dyson #if defined(VM_KMEM_SIZE_SCALE) 34481930014SPeter Wemm if ((mem_size / VM_KMEM_SIZE_SCALE) > vm_kmem_size) 34581930014SPeter Wemm vm_kmem_size = mem_size / VM_KMEM_SIZE_SCALE; 3468a58a9f6SJohn Dyson #endif 3478a58a9f6SJohn Dyson 3488a58a9f6SJohn Dyson #if defined(VM_KMEM_SIZE_MAX) 34981930014SPeter Wemm if (vm_kmem_size >= VM_KMEM_SIZE_MAX) 35081930014SPeter Wemm vm_kmem_size = VM_KMEM_SIZE_MAX; 3518a58a9f6SJohn Dyson #endif 3528a58a9f6SJohn Dyson 3538de6e8e1SMike Smith /* Allow final override from the kernel environment */ 35409786698SPeter Wemm TUNABLE_INT_FETCH("kern.vm.kmem.size", &vm_kmem_size); 3558de6e8e1SMike Smith 35627b8623fSDavid Greenman /* 35727b8623fSDavid Greenman * Limit kmem virtual size to twice the physical memory. 35827b8623fSDavid Greenman * This allows for kmem map sparseness, but limits the size 35927b8623fSDavid Greenman * to something sane. Be careful to not overflow the 32bit 36027b8623fSDavid Greenman * ints while doing the check. 36127b8623fSDavid Greenman */ 36227b8623fSDavid Greenman if ((vm_kmem_size / 2) > (cnt.v_page_count * PAGE_SIZE)) 36327b8623fSDavid Greenman vm_kmem_size = 2 * cnt.v_page_count * PAGE_SIZE; 3648a58a9f6SJohn Dyson 36508442f8aSBosko Milekic /* 366ba3e8826SBosko Milekic * In mbuf_init(), we set up submaps for mbufs and clusters, in which 36708442f8aSBosko Milekic * case we rounddown() (nmbufs * MSIZE) and (nmbclusters * MCLBYTES), 36808442f8aSBosko Milekic * respectively. Mathematically, this means that what we do here may 36908442f8aSBosko Milekic * amount to slightly more address space than we need for the submaps, 37008442f8aSBosko Milekic * but it never hurts to have an extra page in kmem_map. 37108442f8aSBosko Milekic */ 372d04d50d1SBosko Milekic npg = (nmbufs * MSIZE + nmbclusters * MCLBYTES + nmbcnt * 37308442f8aSBosko Milekic sizeof(u_int) + vm_kmem_size) / PAGE_SIZE; 3740d94caffSDavid Greenman 375df8bae1dSRodney W. Grimes kmem_map = kmem_suballoc(kernel_map, (vm_offset_t *)&kmembase, 3762d8acc0fSJohn Dyson (vm_offset_t *)&kmemlimit, (vm_size_t)(npg * PAGE_SIZE)); 3773075778bSJohn Dyson kmem_map->system_map = 1; 3788355f576SJeff Roberson 3798355f576SJeff Roberson hashsize = npg * sizeof(void *); 3808355f576SJeff Roberson 3818355f576SJeff Roberson highbit = 0; 3828355f576SJeff Roberson bits = 0; 3838355f576SJeff Roberson /* The hash size must be a power of two */ 3848355f576SJeff Roberson for (i = 0; i < 8 * sizeof(hashsize); i++) 3858355f576SJeff Roberson if (hashsize & (1 << i)) { 3868355f576SJeff Roberson highbit = i; 3878355f576SJeff Roberson bits++; 3888355f576SJeff Roberson } 3898355f576SJeff Roberson if (bits > 1) 3908355f576SJeff Roberson hashsize = 1 << (highbit); 3918355f576SJeff Roberson 3928355f576SJeff Roberson hashmem = (void *)kmem_alloc(kernel_map, (vm_size_t)hashsize); 3938355f576SJeff Roberson uma_startup2(hashmem, hashsize / sizeof(void *)); 3948355f576SJeff Roberson 3956f267175SJeff Roberson for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) { 3966f267175SJeff Roberson int size = kmemzones[indx].kz_size; 3976f267175SJeff Roberson char *name = kmemzones[indx].kz_name; 3988355f576SJeff Roberson 3998efc4effSJeff Roberson kmemzones[indx].kz_zone = uma_zcreate(name, size, 4008efc4effSJeff Roberson #ifdef INVARIANTS 4018efc4effSJeff Roberson trash_ctor, trash_dtor, trash_init, trash_fini, 4028efc4effSJeff Roberson #else 4038efc4effSJeff Roberson NULL, NULL, NULL, NULL, 4048efc4effSJeff Roberson #endif 4058efc4effSJeff Roberson UMA_ALIGN_PTR, UMA_ZONE_MALLOC); 4066f267175SJeff Roberson 4078355f576SJeff Roberson for (;i <= size; i+= KMEM_ZBASE) 4086f267175SJeff Roberson kmemsize[i >> KMEM_ZSHIFT] = indx; 4098355f576SJeff Roberson 410df8bae1dSRodney W. Grimes } 411254c6cb3SPoul-Henning Kamp } 412254c6cb3SPoul-Henning Kamp 413db669378SPeter Wemm void 414db669378SPeter Wemm malloc_init(data) 415db669378SPeter Wemm void *data; 416254c6cb3SPoul-Henning Kamp { 417db669378SPeter Wemm struct malloc_type *type = (struct malloc_type *)data; 418254c6cb3SPoul-Henning Kamp 4196f267175SJeff Roberson mtx_lock(&malloc_mtx); 420d1bbc7ecSPoul-Henning Kamp if (type->ks_magic != M_MAGIC) 421d1bbc7ecSPoul-Henning Kamp panic("malloc type lacks magic"); 422d1bbc7ecSPoul-Henning Kamp 423d4060a87SJohn Dyson if (cnt.v_page_count == 0) 424d4060a87SJohn Dyson panic("malloc_init not allowed before vm init"); 425d4060a87SJohn Dyson 4266f267175SJeff Roberson if (type->ks_next != NULL) 4276f267175SJeff Roberson return; 4286f267175SJeff Roberson 429254c6cb3SPoul-Henning Kamp type->ks_next = kmemstatistics; 430254c6cb3SPoul-Henning Kamp kmemstatistics = type; 4315a34a9f0SJeff Roberson mtx_init(&type->ks_mtx, type->ks_shortdesc, "Malloc Stats", MTX_DEF); 4326f267175SJeff Roberson mtx_unlock(&malloc_mtx); 433df8bae1dSRodney W. Grimes } 434db669378SPeter Wemm 435db669378SPeter Wemm void 436db669378SPeter Wemm malloc_uninit(data) 437db669378SPeter Wemm void *data; 438db669378SPeter Wemm { 439db669378SPeter Wemm struct malloc_type *type = (struct malloc_type *)data; 440db669378SPeter Wemm struct malloc_type *t; 441db669378SPeter Wemm 4426f267175SJeff Roberson mtx_lock(&malloc_mtx); 4435a34a9f0SJeff Roberson mtx_lock(&type->ks_mtx); 444db669378SPeter Wemm if (type->ks_magic != M_MAGIC) 445db669378SPeter Wemm panic("malloc type lacks magic"); 446db669378SPeter Wemm 447db669378SPeter Wemm if (cnt.v_page_count == 0) 448db669378SPeter Wemm panic("malloc_uninit not allowed before vm init"); 449db669378SPeter Wemm 450db669378SPeter Wemm if (type == kmemstatistics) 451db669378SPeter Wemm kmemstatistics = type->ks_next; 452db669378SPeter Wemm else { 453db669378SPeter Wemm for (t = kmemstatistics; t->ks_next != NULL; t = t->ks_next) { 454db669378SPeter Wemm if (t->ks_next == type) { 455db669378SPeter Wemm t->ks_next = type->ks_next; 456db669378SPeter Wemm break; 457db669378SPeter Wemm } 458db669378SPeter Wemm } 459db669378SPeter Wemm } 460ce45b512SBruce Evans type->ks_next = NULL; 4615a34a9f0SJeff Roberson mtx_destroy(&type->ks_mtx); 4626f267175SJeff Roberson mtx_unlock(&malloc_mtx); 463db669378SPeter Wemm } 4646f267175SJeff Roberson 4656f267175SJeff Roberson static int 4666f267175SJeff Roberson sysctl_kern_malloc(SYSCTL_HANDLER_ARGS) 4676f267175SJeff Roberson { 4686f267175SJeff Roberson struct malloc_type *type; 4696f267175SJeff Roberson int linesize = 128; 4706f267175SJeff Roberson int curline; 4716f267175SJeff Roberson int bufsize; 4726f267175SJeff Roberson int first; 4736f267175SJeff Roberson int error; 4746f267175SJeff Roberson char *buf; 4756f267175SJeff Roberson char *p; 4766f267175SJeff Roberson int cnt; 4776f267175SJeff Roberson int len; 4786f267175SJeff Roberson int i; 4796f267175SJeff Roberson 4806f267175SJeff Roberson cnt = 0; 4816f267175SJeff Roberson 4826f267175SJeff Roberson mtx_lock(&malloc_mtx); 4836f267175SJeff Roberson for (type = kmemstatistics; type != NULL; type = type->ks_next) 4846f267175SJeff Roberson cnt++; 4856f267175SJeff Roberson 4865a34a9f0SJeff Roberson mtx_unlock(&malloc_mtx); 4876f267175SJeff Roberson bufsize = linesize * (cnt + 1); 4886f267175SJeff Roberson p = buf = (char *)malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO); 4895a34a9f0SJeff Roberson mtx_lock(&malloc_mtx); 4906f267175SJeff Roberson 4916f267175SJeff Roberson len = snprintf(p, linesize, 4926f267175SJeff Roberson "\n Type InUse MemUse HighUse Requests Size(s)\n"); 4936f267175SJeff Roberson p += len; 4946f267175SJeff Roberson 4956f267175SJeff Roberson for (type = kmemstatistics; cnt != 0 && type != NULL; 4966f267175SJeff Roberson type = type->ks_next, cnt--) { 4976f267175SJeff Roberson if (type->ks_calls == 0) 4986f267175SJeff Roberson continue; 4996f267175SJeff Roberson 5006f267175SJeff Roberson curline = linesize - 2; /* Leave room for the \n */ 501289f207cSJeff Roberson len = snprintf(p, curline, "%13s%6lu%6luK%7luK%9llu", 5026f267175SJeff Roberson type->ks_shortdesc, 5036f267175SJeff Roberson type->ks_inuse, 5046f267175SJeff Roberson (type->ks_memuse + 1023) / 1024, 5056f267175SJeff Roberson (type->ks_maxused + 1023) / 1024, 5066f267175SJeff Roberson (long long unsigned)type->ks_calls); 5076f267175SJeff Roberson curline -= len; 5086f267175SJeff Roberson p += len; 5096f267175SJeff Roberson 5106f267175SJeff Roberson first = 1; 5115e914b96SJeff Roberson for (i = 0; i < 8 * sizeof(type->ks_size); i++) 5126f267175SJeff Roberson if (type->ks_size & (1 << i)) { 5136f267175SJeff Roberson if (first) 5146f267175SJeff Roberson len = snprintf(p, curline, " "); 5156f267175SJeff Roberson else 5166f267175SJeff Roberson len = snprintf(p, curline, ","); 5176f267175SJeff Roberson curline -= len; 5186f267175SJeff Roberson p += len; 5196f267175SJeff Roberson 5206f267175SJeff Roberson len = snprintf(p, curline, 5216f267175SJeff Roberson "%s", kmemzones[i].kz_name); 5226f267175SJeff Roberson curline -= len; 5236f267175SJeff Roberson p += len; 5246f267175SJeff Roberson 5256f267175SJeff Roberson first = 0; 5266f267175SJeff Roberson } 5276f267175SJeff Roberson 5286f267175SJeff Roberson len = snprintf(p, 2, "\n"); 5296f267175SJeff Roberson p += len; 5306f267175SJeff Roberson } 5316f267175SJeff Roberson 5326f267175SJeff Roberson mtx_unlock(&malloc_mtx); 5336f267175SJeff Roberson error = SYSCTL_OUT(req, buf, p - buf); 5346f267175SJeff Roberson 5356f267175SJeff Roberson free(buf, M_TEMP); 5366f267175SJeff Roberson return (error); 5376f267175SJeff Roberson } 5386f267175SJeff Roberson 5396f267175SJeff Roberson SYSCTL_OID(_kern, OID_AUTO, malloc, CTLTYPE_STRING|CTLFLAG_RD, 5406f267175SJeff Roberson NULL, 0, sysctl_kern_malloc, "A", "Malloc Stats"); 5415e914b96SJeff Roberson 5425e914b96SJeff Roberson #ifdef MALLOC_PROFILE 5435e914b96SJeff Roberson 5445e914b96SJeff Roberson static int 5455e914b96SJeff Roberson sysctl_kern_mprof(SYSCTL_HANDLER_ARGS) 5465e914b96SJeff Roberson { 5475e914b96SJeff Roberson int linesize = 64; 5485e914b96SJeff Roberson uint64_t count; 5495e914b96SJeff Roberson uint64_t waste; 5505e914b96SJeff Roberson uint64_t mem; 5515e914b96SJeff Roberson int bufsize; 5525e914b96SJeff Roberson int error; 5535e914b96SJeff Roberson char *buf; 5545e914b96SJeff Roberson int rsize; 5555e914b96SJeff Roberson int size; 5565e914b96SJeff Roberson char *p; 5575e914b96SJeff Roberson int len; 5585e914b96SJeff Roberson int i; 5595e914b96SJeff Roberson 5605e914b96SJeff Roberson bufsize = linesize * (KMEM_ZSIZE + 1); 5615e914b96SJeff Roberson bufsize += 128; /* For the stats line */ 5625e914b96SJeff Roberson bufsize += 128; /* For the banner line */ 5635e914b96SJeff Roberson waste = 0; 5645e914b96SJeff Roberson mem = 0; 5655e914b96SJeff Roberson 5665e914b96SJeff Roberson p = buf = (char *)malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO); 5675e914b96SJeff Roberson len = snprintf(p, bufsize, 5685e914b96SJeff Roberson "\n Size Requests Real Size\n"); 5695e914b96SJeff Roberson bufsize -= len; 5705e914b96SJeff Roberson p += len; 5715e914b96SJeff Roberson 5725e914b96SJeff Roberson for (i = 0; i < KMEM_ZSIZE; i++) { 5735e914b96SJeff Roberson size = i << KMEM_ZSHIFT; 5745e914b96SJeff Roberson rsize = kmemzones[kmemsize[i]].kz_size; 5755e914b96SJeff Roberson count = (long long unsigned)krequests[i]; 5765e914b96SJeff Roberson 5775e914b96SJeff Roberson len = snprintf(p, bufsize, "%6d%28llu%11d\n", 5785e914b96SJeff Roberson size, (unsigned long long)count, rsize); 5795e914b96SJeff Roberson bufsize -= len; 5805e914b96SJeff Roberson p += len; 5815e914b96SJeff Roberson 5825e914b96SJeff Roberson if ((rsize * count) > (size * count)) 5835e914b96SJeff Roberson waste += (rsize * count) - (size * count); 5845e914b96SJeff Roberson mem += (rsize * count); 5855e914b96SJeff Roberson } 5865e914b96SJeff Roberson 5875e914b96SJeff Roberson len = snprintf(p, bufsize, 5885e914b96SJeff Roberson "\nTotal memory used:\t%30llu\nTotal Memory wasted:\t%30llu\n", 5895e914b96SJeff Roberson (unsigned long long)mem, (unsigned long long)waste); 5905e914b96SJeff Roberson p += len; 5915e914b96SJeff Roberson 5925e914b96SJeff Roberson error = SYSCTL_OUT(req, buf, p - buf); 5935e914b96SJeff Roberson 5945e914b96SJeff Roberson free(buf, M_TEMP); 5955e914b96SJeff Roberson return (error); 5965e914b96SJeff Roberson } 5975e914b96SJeff Roberson 5985e914b96SJeff Roberson SYSCTL_OID(_kern, OID_AUTO, mprof, CTLTYPE_STRING|CTLFLAG_RD, 5995e914b96SJeff Roberson NULL, 0, sysctl_kern_mprof, "A", "Malloc Profiling"); 6005e914b96SJeff Roberson #endif /* MALLOC_PROFILE */ 601