xref: /freebsd/sys/kern/kern_malloc.c (revision 5a34a9f0894368264a382b6b15679c42a4a9ca3e)
1df8bae1dSRodney W. Grimes /*
2df8bae1dSRodney W. Grimes  * Copyright (c) 1987, 1991, 1993
3df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
4df8bae1dSRodney W. Grimes  *
5df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
6df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
7df8bae1dSRodney W. Grimes  * are met:
8df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
9df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
10df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
11df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
12df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
13df8bae1dSRodney W. Grimes  * 3. All advertising materials mentioning features or use of this software
14df8bae1dSRodney W. Grimes  *    must display the following acknowledgement:
15df8bae1dSRodney W. Grimes  *	This product includes software developed by the University of
16df8bae1dSRodney W. Grimes  *	California, Berkeley and its contributors.
17df8bae1dSRodney W. Grimes  * 4. Neither the name of the University nor the names of its contributors
18df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
19df8bae1dSRodney W. Grimes  *    without specific prior written permission.
20df8bae1dSRodney W. Grimes  *
21df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
32df8bae1dSRodney W. Grimes  *
33df8bae1dSRodney W. Grimes  *	@(#)kern_malloc.c	8.3 (Berkeley) 1/4/94
34c3aac50fSPeter Wemm  * $FreeBSD$
35df8bae1dSRodney W. Grimes  */
36df8bae1dSRodney W. Grimes 
378a58a9f6SJohn Dyson #include "opt_vm.h"
388a58a9f6SJohn Dyson 
39df8bae1dSRodney W. Grimes #include <sys/param.h>
4026f9a767SRodney W. Grimes #include <sys/systm.h>
41df8bae1dSRodney W. Grimes #include <sys/kernel.h>
42fb919e4dSMark Murray #include <sys/lock.h>
43df8bae1dSRodney W. Grimes #include <sys/malloc.h>
4454e7152cSDavid Greenman #include <sys/mbuf.h>
45eec258d2SJohn Baldwin #include <sys/mutex.h>
46efeaf95aSDavid Greenman #include <sys/vmmeter.h>
47a448b62aSJake Burkholder #include <sys/proc.h>
486f267175SJeff Roberson #include <sys/sysctl.h>
499a02e8c6SJason Evans 
50df8bae1dSRodney W. Grimes #include <vm/vm.h>
51efeaf95aSDavid Greenman #include <vm/vm_param.h>
52df8bae1dSRodney W. Grimes #include <vm/vm_kern.h>
53efeaf95aSDavid Greenman #include <vm/vm_extern.h>
543075778bSJohn Dyson #include <vm/pmap.h>
553075778bSJohn Dyson #include <vm/vm_map.h>
568355f576SJeff Roberson #include <vm/uma.h>
578355f576SJeff Roberson #include <vm/uma_int.h>
588efc4effSJeff Roberson #include <vm/uma_dbg.h>
59df8bae1dSRodney W. Grimes 
60984982d6SPoul-Henning Kamp #if defined(INVARIANTS) && defined(__i386__)
61984982d6SPoul-Henning Kamp #include <machine/cpu.h>
62984982d6SPoul-Henning Kamp #endif
63984982d6SPoul-Henning Kamp 
6444a8ff31SArchie Cobbs /*
6544a8ff31SArchie Cobbs  * When realloc() is called, if the new size is sufficiently smaller than
6644a8ff31SArchie Cobbs  * the old size, realloc() will allocate a new, smaller block to avoid
6744a8ff31SArchie Cobbs  * wasting memory. 'Sufficiently smaller' is defined as: newsize <=
6844a8ff31SArchie Cobbs  * oldsize / 2^n, where REALLOC_FRACTION defines the value of 'n'.
6944a8ff31SArchie Cobbs  */
7044a8ff31SArchie Cobbs #ifndef REALLOC_FRACTION
7144a8ff31SArchie Cobbs #define	REALLOC_FRACTION	1	/* new block if <= half the size */
7244a8ff31SArchie Cobbs #endif
7344a8ff31SArchie Cobbs 
743b6fb885SPoul-Henning Kamp MALLOC_DEFINE(M_CACHE, "cache", "Various Dynamically allocated caches");
759ef246c6SBruce Evans MALLOC_DEFINE(M_DEVBUF, "devbuf", "device driver memory");
769ef246c6SBruce Evans MALLOC_DEFINE(M_TEMP, "temp", "misc temporary data buffers");
779ef246c6SBruce Evans 
7882cd038dSYoshinobu Inoue MALLOC_DEFINE(M_IP6OPT, "ip6opt", "IPv6 options");
7982cd038dSYoshinobu Inoue MALLOC_DEFINE(M_IP6NDP, "ip6ndp", "IPv6 Neighbor Discovery");
8082cd038dSYoshinobu Inoue 
814d77a549SAlfred Perlstein static void kmeminit(void *);
822b14f991SJulian Elischer SYSINIT(kmem, SI_SUB_KMEM, SI_ORDER_FIRST, kmeminit, NULL)
832b14f991SJulian Elischer 
84a1c995b6SPoul-Henning Kamp static MALLOC_DEFINE(M_FREE, "free", "should be on free list");
85a1c995b6SPoul-Henning Kamp 
86db669378SPeter Wemm static struct malloc_type *kmemstatistics;
87254c6cb3SPoul-Henning Kamp static char *kmembase;
88043a2f3bSBruce Evans static char *kmemlimit;
891f6889a1SMatthew Dillon 
908355f576SJeff Roberson #define KMEM_ZSHIFT	4
918355f576SJeff Roberson #define KMEM_ZBASE	16
928355f576SJeff Roberson #define KMEM_ZMASK	(KMEM_ZBASE - 1)
938355f576SJeff Roberson 
94bd796eb2SRobert Watson #define KMEM_ZMAX	65536
958355f576SJeff Roberson #define KMEM_ZSIZE	(KMEM_ZMAX >> KMEM_ZSHIFT)
966f267175SJeff Roberson static u_int8_t kmemsize[KMEM_ZSIZE + 1];
976f267175SJeff Roberson 
988355f576SJeff Roberson /* These won't be powers of two for long */
998355f576SJeff Roberson struct {
1006f267175SJeff Roberson 	int kz_size;
1016f267175SJeff Roberson 	char *kz_name;
1026f267175SJeff Roberson 	uma_zone_t kz_zone;
1036f267175SJeff Roberson } kmemzones[] = {
1046f267175SJeff Roberson 	{16, "16", NULL},
1056f267175SJeff Roberson 	{32, "32", NULL},
1066f267175SJeff Roberson 	{64, "64", NULL},
1076f267175SJeff Roberson 	{128, "128", NULL},
1086f267175SJeff Roberson 	{256, "256", NULL},
1096f267175SJeff Roberson 	{512, "512", NULL},
1106f267175SJeff Roberson 	{1024, "1024", NULL},
1116f267175SJeff Roberson 	{2048, "2048", NULL},
1126f267175SJeff Roberson 	{4096, "4096", NULL},
1136f267175SJeff Roberson 	{8192, "8192", NULL},
11443a7c4e9SRobert Watson 	{16384, "16384", NULL},
115bd796eb2SRobert Watson 	{32768, "32768", NULL},
116bd796eb2SRobert Watson 	{65536, "65536", NULL},
1178355f576SJeff Roberson 	{0, NULL},
1188355f576SJeff Roberson };
1198355f576SJeff Roberson 
1206f267175SJeff Roberson u_int vm_kmem_size;
1215a34a9f0SJeff Roberson 
1225a34a9f0SJeff Roberson /*
1235a34a9f0SJeff Roberson  * The malloc_mtx protects the kmemstatistics linked list as well as the
1245a34a9f0SJeff Roberson  * mallochash.
1255a34a9f0SJeff Roberson  */
1265a34a9f0SJeff Roberson 
1275a34a9f0SJeff Roberson struct mtx malloc_mtx;
12869ef67f9SJason Evans 
1295e914b96SJeff Roberson #ifdef MALLOC_PROFILE
1305e914b96SJeff Roberson uint64_t krequests[KMEM_ZSIZE + 1];
1316f267175SJeff Roberson 
1325e914b96SJeff Roberson static int sysctl_kern_mprof(SYSCTL_HANDLER_ARGS);
1335e914b96SJeff Roberson #endif
1345e914b96SJeff Roberson 
1355e914b96SJeff Roberson static int sysctl_kern_malloc(SYSCTL_HANDLER_ARGS);
136df8bae1dSRodney W. Grimes 
137df8bae1dSRodney W. Grimes /*
1381c7c3c6aSMatthew Dillon  *	malloc:
1391c7c3c6aSMatthew Dillon  *
1401c7c3c6aSMatthew Dillon  *	Allocate a block of memory.
1411c7c3c6aSMatthew Dillon  *
1421c7c3c6aSMatthew Dillon  *	If M_NOWAIT is set, this routine will not block and return NULL if
1431c7c3c6aSMatthew Dillon  *	the allocation fails.
144df8bae1dSRodney W. Grimes  */
145df8bae1dSRodney W. Grimes void *
146df8bae1dSRodney W. Grimes malloc(size, type, flags)
147df8bae1dSRodney W. Grimes 	unsigned long size;
14860a513e9SPoul-Henning Kamp 	struct malloc_type *type;
149254c6cb3SPoul-Henning Kamp 	int flags;
150df8bae1dSRodney W. Grimes {
1516f267175SJeff Roberson 	int indx;
1528355f576SJeff Roberson 	caddr_t va;
1538355f576SJeff Roberson 	uma_zone_t zone;
15460a513e9SPoul-Henning Kamp 	register struct malloc_type *ksp = type;
155df8bae1dSRodney W. Grimes 
156708da94eSPoul-Henning Kamp #if 0
157708da94eSPoul-Henning Kamp 	if (size == 0)
158708da94eSPoul-Henning Kamp 		Debugger("zero size malloc");
159708da94eSPoul-Henning Kamp #endif
1600fee3d35SPeter Wemm #if defined(INVARIANTS)
161984982d6SPoul-Henning Kamp 	if (flags == M_WAITOK)
162b40ce416SJulian Elischer 		KASSERT(curthread->td_intr_nesting_level == 0,
163984982d6SPoul-Henning Kamp 		   ("malloc(M_WAITOK) in interrupt context"));
164984982d6SPoul-Henning Kamp #endif
1658355f576SJeff Roberson 	if (size <= KMEM_ZMAX) {
1666f267175SJeff Roberson 		if (size & KMEM_ZMASK)
1676f267175SJeff Roberson 			size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
1686f267175SJeff Roberson 		indx = kmemsize[size >> KMEM_ZSHIFT];
1696f267175SJeff Roberson 		zone = kmemzones[indx].kz_zone;
1706f267175SJeff Roberson #ifdef MALLOC_PROFILE
1716f267175SJeff Roberson 		krequests[size >> KMEM_ZSHIFT]++;
1726f267175SJeff Roberson #endif
1738355f576SJeff Roberson 		va = uma_zalloc(zone, flags);
1745a34a9f0SJeff Roberson 		mtx_lock(&ksp->ks_mtx);
1756f267175SJeff Roberson 		if (va == NULL)
176df8bae1dSRodney W. Grimes 			goto out;
1776f267175SJeff Roberson 
1786f267175SJeff Roberson 		ksp->ks_size |= 1 << indx;
1796f267175SJeff Roberson 		size = zone->uz_size;
1808355f576SJeff Roberson 	} else {
1816f267175SJeff Roberson 		size = roundup(size, PAGE_SIZE);
1828355f576SJeff Roberson 		zone = NULL;
1838355f576SJeff Roberson 		va = uma_large_malloc(size, flags);
1845a34a9f0SJeff Roberson 		mtx_lock(&ksp->ks_mtx);
1856f267175SJeff Roberson 		if (va == NULL)
1868355f576SJeff Roberson 			goto out;
187df8bae1dSRodney W. Grimes 	}
1886f267175SJeff Roberson 	ksp->ks_memuse += size;
189df8bae1dSRodney W. Grimes 	ksp->ks_inuse++;
1908355f576SJeff Roberson out:
191df8bae1dSRodney W. Grimes 	ksp->ks_calls++;
192df8bae1dSRodney W. Grimes 	if (ksp->ks_memuse > ksp->ks_maxused)
193df8bae1dSRodney W. Grimes 		ksp->ks_maxused = ksp->ks_memuse;
1946f267175SJeff Roberson 
1955a34a9f0SJeff Roberson 	mtx_unlock(&ksp->ks_mtx);
196df8bae1dSRodney W. Grimes 	return ((void *) va);
197df8bae1dSRodney W. Grimes }
198df8bae1dSRodney W. Grimes 
199df8bae1dSRodney W. Grimes /*
2001c7c3c6aSMatthew Dillon  *	free:
2011c7c3c6aSMatthew Dillon  *
202df8bae1dSRodney W. Grimes  *	Free a block of memory allocated by malloc.
2031c7c3c6aSMatthew Dillon  *
2041c7c3c6aSMatthew Dillon  *	This routine may not block.
205df8bae1dSRodney W. Grimes  */
206df8bae1dSRodney W. Grimes void
207df8bae1dSRodney W. Grimes free(addr, type)
208df8bae1dSRodney W. Grimes 	void *addr;
20960a513e9SPoul-Henning Kamp 	struct malloc_type *type;
210df8bae1dSRodney W. Grimes {
2118355f576SJeff Roberson 	uma_slab_t slab;
2128355f576SJeff Roberson 	void *mem;
2138355f576SJeff Roberson 	u_long size;
21460a513e9SPoul-Henning Kamp 	register struct malloc_type *ksp = type;
215254c6cb3SPoul-Henning Kamp 
21644a8ff31SArchie Cobbs 	/* free(NULL, ...) does nothing */
21744a8ff31SArchie Cobbs 	if (addr == NULL)
21844a8ff31SArchie Cobbs 		return;
21944a8ff31SArchie Cobbs 
2208355f576SJeff Roberson 	size = 0;
22169ef67f9SJason Evans 
2228355f576SJeff Roberson 	mem = (void *)((u_long)addr & (~UMA_SLAB_MASK));
2235a34a9f0SJeff Roberson 	mtx_lock(&malloc_mtx);
2248355f576SJeff Roberson 	slab = hash_sfind(mallochash, mem);
2255a34a9f0SJeff Roberson 	mtx_unlock(&malloc_mtx);
2268355f576SJeff Roberson 
2278355f576SJeff Roberson 	if (slab == NULL)
2286f267175SJeff Roberson 		panic("free: address %p(%p) has not been allocated.\n",
2296f267175SJeff Roberson 		    addr, mem);
2308355f576SJeff Roberson 
2318355f576SJeff Roberson 	if (!(slab->us_flags & UMA_SLAB_MALLOC)) {
2328355f576SJeff Roberson 		size = slab->us_zone->uz_size;
2338355f576SJeff Roberson 		uma_zfree_arg(slab->us_zone, addr, slab);
23414bf02f8SJohn Dyson 	} else {
2358355f576SJeff Roberson 		size = slab->us_size;
2368355f576SJeff Roberson 		uma_large_free(slab);
23714bf02f8SJohn Dyson 	}
2385a34a9f0SJeff Roberson 	mtx_lock(&ksp->ks_mtx);
2398355f576SJeff Roberson 	ksp->ks_memuse -= size;
2408355f576SJeff Roberson 	ksp->ks_inuse--;
2415a34a9f0SJeff Roberson 	mtx_unlock(&ksp->ks_mtx);
242df8bae1dSRodney W. Grimes }
243df8bae1dSRodney W. Grimes 
244df8bae1dSRodney W. Grimes /*
24544a8ff31SArchie Cobbs  *	realloc: change the size of a memory block
24644a8ff31SArchie Cobbs  */
24744a8ff31SArchie Cobbs void *
24844a8ff31SArchie Cobbs realloc(addr, size, type, flags)
24944a8ff31SArchie Cobbs 	void *addr;
25044a8ff31SArchie Cobbs 	unsigned long size;
25144a8ff31SArchie Cobbs 	struct malloc_type *type;
25244a8ff31SArchie Cobbs 	int flags;
25344a8ff31SArchie Cobbs {
2548355f576SJeff Roberson 	uma_slab_t slab;
25544a8ff31SArchie Cobbs 	unsigned long alloc;
25644a8ff31SArchie Cobbs 	void *newaddr;
25744a8ff31SArchie Cobbs 
25844a8ff31SArchie Cobbs 	/* realloc(NULL, ...) is equivalent to malloc(...) */
25944a8ff31SArchie Cobbs 	if (addr == NULL)
26044a8ff31SArchie Cobbs 		return (malloc(size, type, flags));
26144a8ff31SArchie Cobbs 
2625a34a9f0SJeff Roberson 	mtx_lock(&malloc_mtx);
2638355f576SJeff Roberson 	slab = hash_sfind(mallochash,
2648355f576SJeff Roberson 	    (void *)((u_long)addr & ~(UMA_SLAB_MASK)));
2655a34a9f0SJeff Roberson 	mtx_unlock(&malloc_mtx);
2668355f576SJeff Roberson 
26744a8ff31SArchie Cobbs 	/* Sanity check */
2688355f576SJeff Roberson 	KASSERT(slab != NULL,
26944a8ff31SArchie Cobbs 	    ("realloc: address %p out of range", (void *)addr));
27044a8ff31SArchie Cobbs 
27144a8ff31SArchie Cobbs 	/* Get the size of the original block */
2728355f576SJeff Roberson 	if (slab->us_zone)
2738355f576SJeff Roberson 		alloc = slab->us_zone->uz_size;
2748355f576SJeff Roberson 	else
2758355f576SJeff Roberson 		alloc = slab->us_size;
27644a8ff31SArchie Cobbs 
27744a8ff31SArchie Cobbs 	/* Reuse the original block if appropriate */
27844a8ff31SArchie Cobbs 	if (size <= alloc
27944a8ff31SArchie Cobbs 	    && (size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE))
28044a8ff31SArchie Cobbs 		return (addr);
28144a8ff31SArchie Cobbs 
28244a8ff31SArchie Cobbs 	/* Allocate a new, bigger (or smaller) block */
28344a8ff31SArchie Cobbs 	if ((newaddr = malloc(size, type, flags)) == NULL)
28444a8ff31SArchie Cobbs 		return (NULL);
28544a8ff31SArchie Cobbs 
28644a8ff31SArchie Cobbs 	/* Copy over original contents */
28744a8ff31SArchie Cobbs 	bcopy(addr, newaddr, min(size, alloc));
28844a8ff31SArchie Cobbs 	free(addr, type);
28944a8ff31SArchie Cobbs 	return (newaddr);
29044a8ff31SArchie Cobbs }
29144a8ff31SArchie Cobbs 
29244a8ff31SArchie Cobbs /*
29344a8ff31SArchie Cobbs  *	reallocf: same as realloc() but free memory on failure.
29444a8ff31SArchie Cobbs  */
29544a8ff31SArchie Cobbs void *
29644a8ff31SArchie Cobbs reallocf(addr, size, type, flags)
29744a8ff31SArchie Cobbs 	void *addr;
29844a8ff31SArchie Cobbs 	unsigned long size;
29944a8ff31SArchie Cobbs 	struct malloc_type *type;
30044a8ff31SArchie Cobbs 	int flags;
30144a8ff31SArchie Cobbs {
30244a8ff31SArchie Cobbs 	void *mem;
30344a8ff31SArchie Cobbs 
30444a8ff31SArchie Cobbs 	if ((mem = realloc(addr, size, type, flags)) == NULL)
30544a8ff31SArchie Cobbs 		free(addr, type);
30644a8ff31SArchie Cobbs 	return (mem);
30744a8ff31SArchie Cobbs }
30844a8ff31SArchie Cobbs 
30944a8ff31SArchie Cobbs /*
310df8bae1dSRodney W. Grimes  * Initialize the kernel memory allocator
311df8bae1dSRodney W. Grimes  */
3122b14f991SJulian Elischer /* ARGSUSED*/
3132b14f991SJulian Elischer static void
314d841aaa7SBruce Evans kmeminit(dummy)
315d841aaa7SBruce Evans 	void *dummy;
316df8bae1dSRodney W. Grimes {
3176f267175SJeff Roberson 	u_int8_t indx;
31827b8623fSDavid Greenman 	u_long npg;
31927b8623fSDavid Greenman 	u_long mem_size;
3208355f576SJeff Roberson 	void *hashmem;
3218355f576SJeff Roberson 	u_long hashsize;
3228355f576SJeff Roberson 	int highbit;
3238355f576SJeff Roberson 	int bits;
3248355f576SJeff Roberson 	int i;
3258a58a9f6SJohn Dyson 
3266008862bSJohn Baldwin 	mtx_init(&malloc_mtx, "malloc", NULL, MTX_DEF);
32769ef67f9SJason Evans 
3288a58a9f6SJohn Dyson 	/*
3298a58a9f6SJohn Dyson 	 * Try to auto-tune the kernel memory size, so that it is
3308a58a9f6SJohn Dyson 	 * more applicable for a wider range of machine sizes.
3318a58a9f6SJohn Dyson 	 * On an X86, a VM_KMEM_SIZE_SCALE value of 4 is good, while
3328a58a9f6SJohn Dyson 	 * a VM_KMEM_SIZE of 12MB is a fair compromise.  The
3338a58a9f6SJohn Dyson 	 * VM_KMEM_SIZE_MAX is dependent on the maximum KVA space
3348a58a9f6SJohn Dyson 	 * available, and on an X86 with a total KVA space of 256MB,
3358a58a9f6SJohn Dyson 	 * try to keep VM_KMEM_SIZE_MAX at 80MB or below.
3368a58a9f6SJohn Dyson 	 *
3378a58a9f6SJohn Dyson 	 * Note that the kmem_map is also used by the zone allocator,
3388a58a9f6SJohn Dyson 	 * so make sure that there is enough space.
3398a58a9f6SJohn Dyson 	 */
34081930014SPeter Wemm 	vm_kmem_size = VM_KMEM_SIZE;
3418a58a9f6SJohn Dyson 	mem_size = cnt.v_page_count * PAGE_SIZE;
3428a58a9f6SJohn Dyson 
3438a58a9f6SJohn Dyson #if defined(VM_KMEM_SIZE_SCALE)
34481930014SPeter Wemm 	if ((mem_size / VM_KMEM_SIZE_SCALE) > vm_kmem_size)
34581930014SPeter Wemm 		vm_kmem_size = mem_size / VM_KMEM_SIZE_SCALE;
3468a58a9f6SJohn Dyson #endif
3478a58a9f6SJohn Dyson 
3488a58a9f6SJohn Dyson #if defined(VM_KMEM_SIZE_MAX)
34981930014SPeter Wemm 	if (vm_kmem_size >= VM_KMEM_SIZE_MAX)
35081930014SPeter Wemm 		vm_kmem_size = VM_KMEM_SIZE_MAX;
3518a58a9f6SJohn Dyson #endif
3528a58a9f6SJohn Dyson 
3538de6e8e1SMike Smith 	/* Allow final override from the kernel environment */
35409786698SPeter Wemm 	TUNABLE_INT_FETCH("kern.vm.kmem.size", &vm_kmem_size);
3558de6e8e1SMike Smith 
35627b8623fSDavid Greenman 	/*
35727b8623fSDavid Greenman 	 * Limit kmem virtual size to twice the physical memory.
35827b8623fSDavid Greenman 	 * This allows for kmem map sparseness, but limits the size
35927b8623fSDavid Greenman 	 * to something sane. Be careful to not overflow the 32bit
36027b8623fSDavid Greenman 	 * ints while doing the check.
36127b8623fSDavid Greenman 	 */
36227b8623fSDavid Greenman 	if ((vm_kmem_size / 2) > (cnt.v_page_count * PAGE_SIZE))
36327b8623fSDavid Greenman 		vm_kmem_size = 2 * cnt.v_page_count * PAGE_SIZE;
3648a58a9f6SJohn Dyson 
36508442f8aSBosko Milekic 	/*
366ba3e8826SBosko Milekic 	 * In mbuf_init(), we set up submaps for mbufs and clusters, in which
36708442f8aSBosko Milekic 	 * case we rounddown() (nmbufs * MSIZE) and (nmbclusters * MCLBYTES),
36808442f8aSBosko Milekic 	 * respectively. Mathematically, this means that what we do here may
36908442f8aSBosko Milekic 	 * amount to slightly more address space than we need for the submaps,
37008442f8aSBosko Milekic 	 * but it never hurts to have an extra page in kmem_map.
37108442f8aSBosko Milekic 	 */
372d04d50d1SBosko Milekic 	npg = (nmbufs * MSIZE + nmbclusters * MCLBYTES + nmbcnt *
37308442f8aSBosko Milekic 	    sizeof(u_int) + vm_kmem_size) / PAGE_SIZE;
3740d94caffSDavid Greenman 
375df8bae1dSRodney W. Grimes 	kmem_map = kmem_suballoc(kernel_map, (vm_offset_t *)&kmembase,
3762d8acc0fSJohn Dyson 		(vm_offset_t *)&kmemlimit, (vm_size_t)(npg * PAGE_SIZE));
3773075778bSJohn Dyson 	kmem_map->system_map = 1;
3788355f576SJeff Roberson 
3798355f576SJeff Roberson 	hashsize = npg * sizeof(void *);
3808355f576SJeff Roberson 
3818355f576SJeff Roberson 	highbit = 0;
3828355f576SJeff Roberson 	bits = 0;
3838355f576SJeff Roberson 	/* The hash size must be a power of two */
3848355f576SJeff Roberson 	for (i = 0; i < 8 * sizeof(hashsize); i++)
3858355f576SJeff Roberson 		if (hashsize & (1 << i)) {
3868355f576SJeff Roberson 			highbit = i;
3878355f576SJeff Roberson 			bits++;
3888355f576SJeff Roberson 		}
3898355f576SJeff Roberson 	if (bits > 1)
3908355f576SJeff Roberson 		hashsize = 1 << (highbit);
3918355f576SJeff Roberson 
3928355f576SJeff Roberson 	hashmem = (void *)kmem_alloc(kernel_map, (vm_size_t)hashsize);
3938355f576SJeff Roberson 	uma_startup2(hashmem, hashsize / sizeof(void *));
3948355f576SJeff Roberson 
3956f267175SJeff Roberson 	for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) {
3966f267175SJeff Roberson 		int size = kmemzones[indx].kz_size;
3976f267175SJeff Roberson 		char *name = kmemzones[indx].kz_name;
3988355f576SJeff Roberson 
3998efc4effSJeff Roberson 		kmemzones[indx].kz_zone = uma_zcreate(name, size,
4008efc4effSJeff Roberson #ifdef INVARIANTS
4018efc4effSJeff Roberson 		    trash_ctor, trash_dtor, trash_init, trash_fini,
4028efc4effSJeff Roberson #else
4038efc4effSJeff Roberson 		    NULL, NULL, NULL, NULL,
4048efc4effSJeff Roberson #endif
4058efc4effSJeff Roberson 		    UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
4066f267175SJeff Roberson 
4078355f576SJeff Roberson 		for (;i <= size; i+= KMEM_ZBASE)
4086f267175SJeff Roberson 			kmemsize[i >> KMEM_ZSHIFT] = indx;
4098355f576SJeff Roberson 
410df8bae1dSRodney W. Grimes 	}
411254c6cb3SPoul-Henning Kamp }
412254c6cb3SPoul-Henning Kamp 
413db669378SPeter Wemm void
414db669378SPeter Wemm malloc_init(data)
415db669378SPeter Wemm 	void *data;
416254c6cb3SPoul-Henning Kamp {
417db669378SPeter Wemm 	struct malloc_type *type = (struct malloc_type *)data;
418254c6cb3SPoul-Henning Kamp 
4196f267175SJeff Roberson 	mtx_lock(&malloc_mtx);
420d1bbc7ecSPoul-Henning Kamp 	if (type->ks_magic != M_MAGIC)
421d1bbc7ecSPoul-Henning Kamp 		panic("malloc type lacks magic");
422d1bbc7ecSPoul-Henning Kamp 
423d4060a87SJohn Dyson 	if (cnt.v_page_count == 0)
424d4060a87SJohn Dyson 		panic("malloc_init not allowed before vm init");
425d4060a87SJohn Dyson 
4266f267175SJeff Roberson 	if (type->ks_next != NULL)
4276f267175SJeff Roberson 		return;
4286f267175SJeff Roberson 
429254c6cb3SPoul-Henning Kamp 	type->ks_next = kmemstatistics;
430254c6cb3SPoul-Henning Kamp 	kmemstatistics = type;
4315a34a9f0SJeff Roberson 	mtx_init(&type->ks_mtx, type->ks_shortdesc, "Malloc Stats", MTX_DEF);
4326f267175SJeff Roberson 	mtx_unlock(&malloc_mtx);
433df8bae1dSRodney W. Grimes }
434db669378SPeter Wemm 
435db669378SPeter Wemm void
436db669378SPeter Wemm malloc_uninit(data)
437db669378SPeter Wemm 	void *data;
438db669378SPeter Wemm {
439db669378SPeter Wemm 	struct malloc_type *type = (struct malloc_type *)data;
440db669378SPeter Wemm 	struct malloc_type *t;
441db669378SPeter Wemm 
4426f267175SJeff Roberson 	mtx_lock(&malloc_mtx);
4435a34a9f0SJeff Roberson 	mtx_lock(&type->ks_mtx);
444db669378SPeter Wemm 	if (type->ks_magic != M_MAGIC)
445db669378SPeter Wemm 		panic("malloc type lacks magic");
446db669378SPeter Wemm 
447db669378SPeter Wemm 	if (cnt.v_page_count == 0)
448db669378SPeter Wemm 		panic("malloc_uninit not allowed before vm init");
449db669378SPeter Wemm 
450db669378SPeter Wemm 	if (type == kmemstatistics)
451db669378SPeter Wemm 		kmemstatistics = type->ks_next;
452db669378SPeter Wemm 	else {
453db669378SPeter Wemm 		for (t = kmemstatistics; t->ks_next != NULL; t = t->ks_next) {
454db669378SPeter Wemm 			if (t->ks_next == type) {
455db669378SPeter Wemm 				t->ks_next = type->ks_next;
456db669378SPeter Wemm 				break;
457db669378SPeter Wemm 			}
458db669378SPeter Wemm 		}
459db669378SPeter Wemm 	}
460ce45b512SBruce Evans 	type->ks_next = NULL;
4615a34a9f0SJeff Roberson 	mtx_destroy(&type->ks_mtx);
4626f267175SJeff Roberson 	mtx_unlock(&malloc_mtx);
463db669378SPeter Wemm }
4646f267175SJeff Roberson 
4656f267175SJeff Roberson static int
4666f267175SJeff Roberson sysctl_kern_malloc(SYSCTL_HANDLER_ARGS)
4676f267175SJeff Roberson {
4686f267175SJeff Roberson 	struct malloc_type *type;
4696f267175SJeff Roberson 	int linesize = 128;
4706f267175SJeff Roberson 	int curline;
4716f267175SJeff Roberson 	int bufsize;
4726f267175SJeff Roberson 	int first;
4736f267175SJeff Roberson 	int error;
4746f267175SJeff Roberson 	char *buf;
4756f267175SJeff Roberson 	char *p;
4766f267175SJeff Roberson 	int cnt;
4776f267175SJeff Roberson 	int len;
4786f267175SJeff Roberson 	int i;
4796f267175SJeff Roberson 
4806f267175SJeff Roberson 	cnt = 0;
4816f267175SJeff Roberson 
4826f267175SJeff Roberson 	mtx_lock(&malloc_mtx);
4836f267175SJeff Roberson 	for (type = kmemstatistics; type != NULL; type = type->ks_next)
4846f267175SJeff Roberson 		cnt++;
4856f267175SJeff Roberson 
4865a34a9f0SJeff Roberson 	mtx_unlock(&malloc_mtx);
4876f267175SJeff Roberson 	bufsize = linesize * (cnt + 1);
4886f267175SJeff Roberson 	p = buf = (char *)malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO);
4895a34a9f0SJeff Roberson 	mtx_lock(&malloc_mtx);
4906f267175SJeff Roberson 
4916f267175SJeff Roberson 	len = snprintf(p, linesize,
4926f267175SJeff Roberson 	    "\n        Type  InUse MemUse HighUse Requests  Size(s)\n");
4936f267175SJeff Roberson 	p += len;
4946f267175SJeff Roberson 
4956f267175SJeff Roberson 	for (type = kmemstatistics; cnt != 0 && type != NULL;
4966f267175SJeff Roberson 	    type = type->ks_next, cnt--) {
4976f267175SJeff Roberson 		if (type->ks_calls == 0)
4986f267175SJeff Roberson 			continue;
4996f267175SJeff Roberson 
5006f267175SJeff Roberson 		curline = linesize - 2;	/* Leave room for the \n */
501289f207cSJeff Roberson 		len = snprintf(p, curline, "%13s%6lu%6luK%7luK%9llu",
5026f267175SJeff Roberson 			type->ks_shortdesc,
5036f267175SJeff Roberson 			type->ks_inuse,
5046f267175SJeff Roberson 			(type->ks_memuse + 1023) / 1024,
5056f267175SJeff Roberson 			(type->ks_maxused + 1023) / 1024,
5066f267175SJeff Roberson 			(long long unsigned)type->ks_calls);
5076f267175SJeff Roberson 		curline -= len;
5086f267175SJeff Roberson 		p += len;
5096f267175SJeff Roberson 
5106f267175SJeff Roberson 		first = 1;
5115e914b96SJeff Roberson 		for (i = 0; i < 8 * sizeof(type->ks_size); i++)
5126f267175SJeff Roberson 			if (type->ks_size & (1 << i)) {
5136f267175SJeff Roberson 				if (first)
5146f267175SJeff Roberson 					len = snprintf(p, curline, "  ");
5156f267175SJeff Roberson 				else
5166f267175SJeff Roberson 					len = snprintf(p, curline, ",");
5176f267175SJeff Roberson 				curline -= len;
5186f267175SJeff Roberson 				p += len;
5196f267175SJeff Roberson 
5206f267175SJeff Roberson 				len = snprintf(p, curline,
5216f267175SJeff Roberson 				    "%s", kmemzones[i].kz_name);
5226f267175SJeff Roberson 				curline -= len;
5236f267175SJeff Roberson 				p += len;
5246f267175SJeff Roberson 
5256f267175SJeff Roberson 				first = 0;
5266f267175SJeff Roberson 			}
5276f267175SJeff Roberson 
5286f267175SJeff Roberson 		len = snprintf(p, 2, "\n");
5296f267175SJeff Roberson 		p += len;
5306f267175SJeff Roberson 	}
5316f267175SJeff Roberson 
5326f267175SJeff Roberson 	mtx_unlock(&malloc_mtx);
5336f267175SJeff Roberson 	error = SYSCTL_OUT(req, buf, p - buf);
5346f267175SJeff Roberson 
5356f267175SJeff Roberson 	free(buf, M_TEMP);
5366f267175SJeff Roberson 	return (error);
5376f267175SJeff Roberson }
5386f267175SJeff Roberson 
5396f267175SJeff Roberson SYSCTL_OID(_kern, OID_AUTO, malloc, CTLTYPE_STRING|CTLFLAG_RD,
5406f267175SJeff Roberson     NULL, 0, sysctl_kern_malloc, "A", "Malloc Stats");
5415e914b96SJeff Roberson 
5425e914b96SJeff Roberson #ifdef MALLOC_PROFILE
5435e914b96SJeff Roberson 
5445e914b96SJeff Roberson static int
5455e914b96SJeff Roberson sysctl_kern_mprof(SYSCTL_HANDLER_ARGS)
5465e914b96SJeff Roberson {
5475e914b96SJeff Roberson 	int linesize = 64;
5485e914b96SJeff Roberson 	uint64_t count;
5495e914b96SJeff Roberson 	uint64_t waste;
5505e914b96SJeff Roberson 	uint64_t mem;
5515e914b96SJeff Roberson 	int bufsize;
5525e914b96SJeff Roberson 	int error;
5535e914b96SJeff Roberson 	char *buf;
5545e914b96SJeff Roberson 	int rsize;
5555e914b96SJeff Roberson 	int size;
5565e914b96SJeff Roberson 	char *p;
5575e914b96SJeff Roberson 	int len;
5585e914b96SJeff Roberson 	int i;
5595e914b96SJeff Roberson 
5605e914b96SJeff Roberson 	bufsize = linesize * (KMEM_ZSIZE + 1);
5615e914b96SJeff Roberson 	bufsize += 128; 	/* For the stats line */
5625e914b96SJeff Roberson 	bufsize += 128; 	/* For the banner line */
5635e914b96SJeff Roberson 	waste = 0;
5645e914b96SJeff Roberson 	mem = 0;
5655e914b96SJeff Roberson 
5665e914b96SJeff Roberson 	p = buf = (char *)malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO);
5675e914b96SJeff Roberson 	len = snprintf(p, bufsize,
5685e914b96SJeff Roberson 	    "\n  Size                    Requests  Real Size\n");
5695e914b96SJeff Roberson 	bufsize -= len;
5705e914b96SJeff Roberson 	p += len;
5715e914b96SJeff Roberson 
5725e914b96SJeff Roberson 	for (i = 0; i < KMEM_ZSIZE; i++) {
5735e914b96SJeff Roberson 		size = i << KMEM_ZSHIFT;
5745e914b96SJeff Roberson 		rsize = kmemzones[kmemsize[i]].kz_size;
5755e914b96SJeff Roberson 		count = (long long unsigned)krequests[i];
5765e914b96SJeff Roberson 
5775e914b96SJeff Roberson 		len = snprintf(p, bufsize, "%6d%28llu%11d\n",
5785e914b96SJeff Roberson 		    size, (unsigned long long)count, rsize);
5795e914b96SJeff Roberson 		bufsize -= len;
5805e914b96SJeff Roberson 		p += len;
5815e914b96SJeff Roberson 
5825e914b96SJeff Roberson 		if ((rsize * count) > (size * count))
5835e914b96SJeff Roberson 			waste += (rsize * count) - (size * count);
5845e914b96SJeff Roberson 		mem += (rsize * count);
5855e914b96SJeff Roberson 	}
5865e914b96SJeff Roberson 
5875e914b96SJeff Roberson 	len = snprintf(p, bufsize,
5885e914b96SJeff Roberson 	    "\nTotal memory used:\t%30llu\nTotal Memory wasted:\t%30llu\n",
5895e914b96SJeff Roberson 	    (unsigned long long)mem, (unsigned long long)waste);
5905e914b96SJeff Roberson 	p += len;
5915e914b96SJeff Roberson 
5925e914b96SJeff Roberson 	error = SYSCTL_OUT(req, buf, p - buf);
5935e914b96SJeff Roberson 
5945e914b96SJeff Roberson 	free(buf, M_TEMP);
5955e914b96SJeff Roberson 	return (error);
5965e914b96SJeff Roberson }
5975e914b96SJeff Roberson 
5985e914b96SJeff Roberson SYSCTL_OID(_kern, OID_AUTO, mprof, CTLTYPE_STRING|CTLFLAG_RD,
5995e914b96SJeff Roberson     NULL, 0, sysctl_kern_mprof, "A", "Malloc Profiling");
6005e914b96SJeff Roberson #endif /* MALLOC_PROFILE */
601