xref: /freebsd/sys/vm/uma_core.c (revision 20d3ab87cd7cb42f44cae9a755d03e87e966c48a)
160727d8bSWarner Losh /*-
2ef72505eSJeff Roberson  * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
308ecce74SRobert Watson  * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4ae4e9636SRobert Watson  * Copyright (c) 2004-2006 Robert N. M. Watson
508ecce74SRobert Watson  * All rights reserved.
68355f576SJeff Roberson  *
78355f576SJeff Roberson  * Redistribution and use in source and binary forms, with or without
88355f576SJeff Roberson  * modification, are permitted provided that the following conditions
98355f576SJeff Roberson  * are met:
108355f576SJeff Roberson  * 1. Redistributions of source code must retain the above copyright
118355f576SJeff Roberson  *    notice unmodified, this list of conditions, and the following
128355f576SJeff Roberson  *    disclaimer.
138355f576SJeff Roberson  * 2. Redistributions in binary form must reproduce the above copyright
148355f576SJeff Roberson  *    notice, this list of conditions and the following disclaimer in the
158355f576SJeff Roberson  *    documentation and/or other materials provided with the distribution.
168355f576SJeff Roberson  *
178355f576SJeff Roberson  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
188355f576SJeff Roberson  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
198355f576SJeff Roberson  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
208355f576SJeff Roberson  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
218355f576SJeff Roberson  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
228355f576SJeff Roberson  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
238355f576SJeff Roberson  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
248355f576SJeff Roberson  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
258355f576SJeff Roberson  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
268355f576SJeff Roberson  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
278355f576SJeff Roberson  */
288355f576SJeff Roberson 
298355f576SJeff Roberson /*
308355f576SJeff Roberson  * uma_core.c  Implementation of the Universal Memory allocator
318355f576SJeff Roberson  *
328355f576SJeff Roberson  * This allocator is intended to replace the multitude of similar object caches
338355f576SJeff Roberson  * in the standard FreeBSD kernel.  The intent is to be flexible as well as
348355f576SJeff Roberson  * effecient.  A primary design goal is to return unused memory to the rest of
358355f576SJeff Roberson  * the system.  This will make the system as a whole more flexible due to the
368355f576SJeff Roberson  * ability to move memory to subsystems which most need it instead of leaving
378355f576SJeff Roberson  * pools of reserved memory unused.
388355f576SJeff Roberson  *
398355f576SJeff Roberson  * The basic ideas stem from similar slab/zone based allocators whose algorithms
408355f576SJeff Roberson  * are well known.
418355f576SJeff Roberson  *
428355f576SJeff Roberson  */
438355f576SJeff Roberson 
448355f576SJeff Roberson /*
458355f576SJeff Roberson  * TODO:
468355f576SJeff Roberson  *	- Improve memory usage for large allocations
478355f576SJeff Roberson  *	- Investigate cache size adjustments
488355f576SJeff Roberson  */
498355f576SJeff Roberson 
50874651b1SDavid E. O'Brien #include <sys/cdefs.h>
51874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$");
52874651b1SDavid E. O'Brien 
538355f576SJeff Roberson /* I should really use ktr.. */
548355f576SJeff Roberson /*
558355f576SJeff Roberson #define UMA_DEBUG 1
568355f576SJeff Roberson #define UMA_DEBUG_ALLOC 1
578355f576SJeff Roberson #define UMA_DEBUG_ALLOC_1 1
588355f576SJeff Roberson */
598355f576SJeff Roberson 
6048c5777eSRobert Watson #include "opt_ddb.h"
618355f576SJeff Roberson #include "opt_param.h"
628d689e04SGleb Smirnoff #include "opt_vm.h"
6348c5777eSRobert Watson 
648355f576SJeff Roberson #include <sys/param.h>
658355f576SJeff Roberson #include <sys/systm.h>
66ef72505eSJeff Roberson #include <sys/bitset.h>
678355f576SJeff Roberson #include <sys/kernel.h>
688355f576SJeff Roberson #include <sys/types.h>
698355f576SJeff Roberson #include <sys/queue.h>
708355f576SJeff Roberson #include <sys/malloc.h>
713659f747SRobert Watson #include <sys/ktr.h>
728355f576SJeff Roberson #include <sys/lock.h>
738355f576SJeff Roberson #include <sys/sysctl.h>
748355f576SJeff Roberson #include <sys/mutex.h>
754c1cc01cSJohn Baldwin #include <sys/proc.h>
7689f6b863SAttilio Rao #include <sys/rwlock.h>
777a52a97eSRobert Watson #include <sys/sbuf.h>
78a2de44abSAlexander Motin #include <sys/sched.h>
798355f576SJeff Roberson #include <sys/smp.h>
8086bbae32SJeff Roberson #include <sys/vmmeter.h>
8186bbae32SJeff Roberson 
828355f576SJeff Roberson #include <vm/vm.h>
838355f576SJeff Roberson #include <vm/vm_object.h>
848355f576SJeff Roberson #include <vm/vm_page.h>
85a4915c21SAttilio Rao #include <vm/vm_pageout.h>
868355f576SJeff Roberson #include <vm/vm_param.h>
878355f576SJeff Roberson #include <vm/vm_map.h>
888355f576SJeff Roberson #include <vm/vm_kern.h>
898355f576SJeff Roberson #include <vm/vm_extern.h>
908355f576SJeff Roberson #include <vm/uma.h>
918355f576SJeff Roberson #include <vm/uma_int.h>
92639c9550SJeff Roberson #include <vm/uma_dbg.h>
938355f576SJeff Roberson 
9448c5777eSRobert Watson #include <ddb/ddb.h>
9548c5777eSRobert Watson 
968d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD
978d689e04SGleb Smirnoff #include <vm/memguard.h>
988d689e04SGleb Smirnoff #endif
998d689e04SGleb Smirnoff 
1008355f576SJeff Roberson /*
101099a0e58SBosko Milekic  * This is the zone and keg from which all zones are spawned.  The idea is that
102099a0e58SBosko Milekic  * even the zone & keg heads are allocated from the allocator, so we use the
103099a0e58SBosko Milekic  * bss section to bootstrap us.
1048355f576SJeff Roberson  */
105099a0e58SBosko Milekic static struct uma_keg masterkeg;
106099a0e58SBosko Milekic static struct uma_zone masterzone_k;
107099a0e58SBosko Milekic static struct uma_zone masterzone_z;
108099a0e58SBosko Milekic static uma_zone_t kegs = &masterzone_k;
109099a0e58SBosko Milekic static uma_zone_t zones = &masterzone_z;
1108355f576SJeff Roberson 
1118355f576SJeff Roberson /* This is the zone from which all of uma_slab_t's are allocated. */
1128355f576SJeff Roberson static uma_zone_t slabzone;
113099a0e58SBosko Milekic static uma_zone_t slabrefzone;	/* With refcounters (for UMA_ZONE_REFCNT) */
1148355f576SJeff Roberson 
1158355f576SJeff Roberson /*
1168355f576SJeff Roberson  * The initial hash tables come out of this zone so they can be allocated
1178355f576SJeff Roberson  * prior to malloc coming up.
1188355f576SJeff Roberson  */
1198355f576SJeff Roberson static uma_zone_t hashzone;
1208355f576SJeff Roberson 
1211e319f6dSRobert Watson /* The boot-time adjusted value for cache line alignment. */
122e4cd31ddSJeff Roberson int uma_align_cache = 64 - 1;
1231e319f6dSRobert Watson 
124961647dfSJeff Roberson static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
125961647dfSJeff Roberson 
1268355f576SJeff Roberson /*
12786bbae32SJeff Roberson  * Are we allowed to allocate buckets?
12886bbae32SJeff Roberson  */
12986bbae32SJeff Roberson static int bucketdisable = 1;
13086bbae32SJeff Roberson 
131099a0e58SBosko Milekic /* Linked list of all kegs in the system */
13213e403fdSAntoine Brodin static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
1338355f576SJeff Roberson 
13403175483SAlexander Motin /* Linked list of all cache-only zones in the system */
13503175483SAlexander Motin static LIST_HEAD(,uma_zone) uma_cachezones =
13603175483SAlexander Motin     LIST_HEAD_INITIALIZER(uma_cachezones);
13703175483SAlexander Motin 
138099a0e58SBosko Milekic /* This mutex protects the keg list */
1390095a784SJeff Roberson static struct mtx_padalign uma_mtx;
1408355f576SJeff Roberson 
1418355f576SJeff Roberson /* Linked list of boot time pages */
1428355f576SJeff Roberson static LIST_HEAD(,uma_slab) uma_boot_pages =
14313e403fdSAntoine Brodin     LIST_HEAD_INITIALIZER(uma_boot_pages);
1448355f576SJeff Roberson 
145f353d338SAlan Cox /* This mutex protects the boot time pages list */
1460095a784SJeff Roberson static struct mtx_padalign uma_boot_pages_mtx;
1478355f576SJeff Roberson 
1488355f576SJeff Roberson /* Is the VM done starting up? */
1498355f576SJeff Roberson static int booted = 0;
150342f1793SAlan Cox #define	UMA_STARTUP	1
151342f1793SAlan Cox #define	UMA_STARTUP2	2
1528355f576SJeff Roberson 
153ef72505eSJeff Roberson /*
154ef72505eSJeff Roberson  * Only mbuf clusters use ref zones.  Just provide enough references
155ef72505eSJeff Roberson  * to support the one user.  New code should not use the ref facility.
156ef72505eSJeff Roberson  */
157ef72505eSJeff Roberson static const u_int uma_max_ipers_ref = PAGE_SIZE / MCLBYTES;
158244f4554SBosko Milekic 
1599643769aSJeff Roberson /*
1609643769aSJeff Roberson  * This is the handle used to schedule events that need to happen
1619643769aSJeff Roberson  * outside of the allocation fast path.
1629643769aSJeff Roberson  */
1638355f576SJeff Roberson static struct callout uma_callout;
1649643769aSJeff Roberson #define	UMA_TIMEOUT	20		/* Seconds for callout interval. */
1658355f576SJeff Roberson 
1668355f576SJeff Roberson /*
1678355f576SJeff Roberson  * This structure is passed as the zone ctor arg so that I don't have to create
1688355f576SJeff Roberson  * a special allocation function just for zones.
1698355f576SJeff Roberson  */
1708355f576SJeff Roberson struct uma_zctor_args {
171bb196eb4SMatthew D Fleming 	const char *name;
172c3bdc05fSAndrew R. Reiter 	size_t size;
1738355f576SJeff Roberson 	uma_ctor ctor;
1748355f576SJeff Roberson 	uma_dtor dtor;
1758355f576SJeff Roberson 	uma_init uminit;
1768355f576SJeff Roberson 	uma_fini fini;
1770095a784SJeff Roberson 	uma_import import;
1780095a784SJeff Roberson 	uma_release release;
1790095a784SJeff Roberson 	void *arg;
180099a0e58SBosko Milekic 	uma_keg_t keg;
181099a0e58SBosko Milekic 	int align;
18285dcf349SGleb Smirnoff 	uint32_t flags;
183099a0e58SBosko Milekic };
184099a0e58SBosko Milekic 
185099a0e58SBosko Milekic struct uma_kctor_args {
186099a0e58SBosko Milekic 	uma_zone_t zone;
187099a0e58SBosko Milekic 	size_t size;
188099a0e58SBosko Milekic 	uma_init uminit;
189099a0e58SBosko Milekic 	uma_fini fini;
1908355f576SJeff Roberson 	int align;
19185dcf349SGleb Smirnoff 	uint32_t flags;
1928355f576SJeff Roberson };
1938355f576SJeff Roberson 
194cae33c14SJeff Roberson struct uma_bucket_zone {
195cae33c14SJeff Roberson 	uma_zone_t	ubz_zone;
196cae33c14SJeff Roberson 	char		*ubz_name;
197fc03d22bSJeff Roberson 	int		ubz_entries;	/* Number of items it can hold. */
198fc03d22bSJeff Roberson 	int		ubz_maxsize;	/* Maximum allocation size per-item. */
199cae33c14SJeff Roberson };
200cae33c14SJeff Roberson 
201f9d27e75SRobert Watson /*
202fc03d22bSJeff Roberson  * Compute the actual number of bucket entries to pack them in power
203fc03d22bSJeff Roberson  * of two sizes for more efficient space utilization.
204f9d27e75SRobert Watson  */
205fc03d22bSJeff Roberson #define	BUCKET_SIZE(n)						\
206fc03d22bSJeff Roberson     (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *))
207fc03d22bSJeff Roberson 
208fc03d22bSJeff Roberson #define	BUCKET_MAX	BUCKET_SIZE(128)
209fc03d22bSJeff Roberson 
210fc03d22bSJeff Roberson struct uma_bucket_zone bucket_zones[] = {
2116fd34d6fSJeff Roberson 	{ NULL, "4 Bucket", BUCKET_SIZE(4), 4096 },
212f3932e90SAlexander Motin 	{ NULL, "6 Bucket", BUCKET_SIZE(6), 3072 },
2136fd34d6fSJeff Roberson 	{ NULL, "8 Bucket", BUCKET_SIZE(8), 2048 },
214f3932e90SAlexander Motin 	{ NULL, "12 Bucket", BUCKET_SIZE(12), 1536 },
2156fd34d6fSJeff Roberson 	{ NULL, "16 Bucket", BUCKET_SIZE(16), 1024 },
216fc03d22bSJeff Roberson 	{ NULL, "32 Bucket", BUCKET_SIZE(32), 512 },
217fc03d22bSJeff Roberson 	{ NULL, "64 Bucket", BUCKET_SIZE(64), 256 },
218fc03d22bSJeff Roberson 	{ NULL, "128 Bucket", BUCKET_SIZE(128), 128 },
219fc03d22bSJeff Roberson 	{ NULL, NULL, 0}
220fc03d22bSJeff Roberson };
221cae33c14SJeff Roberson 
2222019094aSRobert Watson /*
2232019094aSRobert Watson  * Flags and enumerations to be passed to internal functions.
2242019094aSRobert Watson  */
225ef72505eSJeff Roberson enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
226b23f72e9SBrian Feldman 
2278355f576SJeff Roberson /* Prototypes.. */
2288355f576SJeff Roberson 
22985dcf349SGleb Smirnoff static void *noobj_alloc(uma_zone_t, int, uint8_t *, int);
23085dcf349SGleb Smirnoff static void *page_alloc(uma_zone_t, int, uint8_t *, int);
23185dcf349SGleb Smirnoff static void *startup_alloc(uma_zone_t, int, uint8_t *, int);
23285dcf349SGleb Smirnoff static void page_free(void *, int, uint8_t);
233e20a199fSJeff Roberson static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
2349643769aSJeff Roberson static void cache_drain(uma_zone_t);
2358355f576SJeff Roberson static void bucket_drain(uma_zone_t, uma_bucket_t);
236aaa8bb16SJeff Roberson static void bucket_cache_drain(uma_zone_t zone);
237b23f72e9SBrian Feldman static int keg_ctor(void *, int, void *, int);
238099a0e58SBosko Milekic static void keg_dtor(void *, int, void *);
239b23f72e9SBrian Feldman static int zone_ctor(void *, int, void *, int);
2409c2cd7e5SJeff Roberson static void zone_dtor(void *, int, void *);
241b23f72e9SBrian Feldman static int zero_init(void *, int, int);
242e20a199fSJeff Roberson static void keg_small_init(uma_keg_t keg);
243e20a199fSJeff Roberson static void keg_large_init(uma_keg_t keg);
2448355f576SJeff Roberson static void zone_foreach(void (*zfunc)(uma_zone_t));
2458355f576SJeff Roberson static void zone_timeout(uma_zone_t zone);
2460aef6126SJeff Roberson static int hash_alloc(struct uma_hash *);
2470aef6126SJeff Roberson static int hash_expand(struct uma_hash *, struct uma_hash *);
2480aef6126SJeff Roberson static void hash_free(struct uma_hash *hash);
2498355f576SJeff Roberson static void uma_timeout(void *);
2508355f576SJeff Roberson static void uma_startup3(void);
251e20a199fSJeff Roberson static void *zone_alloc_item(uma_zone_t, void *, int);
2520095a784SJeff Roberson static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
25386bbae32SJeff Roberson static void bucket_enable(void);
254cae33c14SJeff Roberson static void bucket_init(void);
2556fd34d6fSJeff Roberson static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
2566fd34d6fSJeff Roberson static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
257cae33c14SJeff Roberson static void bucket_zone_drain(void);
2586fd34d6fSJeff Roberson static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, void *, int flags);
259e20a199fSJeff Roberson static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
260e20a199fSJeff Roberson static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
2610095a784SJeff Roberson static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
2620095a784SJeff Roberson static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
263e20a199fSJeff Roberson static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
26485dcf349SGleb Smirnoff     uma_fini fini, int align, uint32_t flags);
2650095a784SJeff Roberson static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
2660095a784SJeff Roberson static void zone_release(uma_zone_t zone, void **bucket, int cnt);
26748343a2fSGleb Smirnoff static void uma_zero_item(void *item, uma_zone_t zone);
268bbee39c6SJeff Roberson 
2698355f576SJeff Roberson void uma_print_zone(uma_zone_t);
2708355f576SJeff Roberson void uma_print_stats(void);
2717a52a97eSRobert Watson static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
2727a52a97eSRobert Watson static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
2738355f576SJeff Roberson 
2748355f576SJeff Roberson SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL);
2758355f576SJeff Roberson 
2767a52a97eSRobert Watson SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT,
2777a52a97eSRobert Watson     0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones");
2787a52a97eSRobert Watson 
2797a52a97eSRobert Watson SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT,
2807a52a97eSRobert Watson     0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats");
2817a52a97eSRobert Watson 
2822f891cd5SPawel Jakub Dawidek static int zone_warnings = 1;
2832f891cd5SPawel Jakub Dawidek TUNABLE_INT("vm.zone_warnings", &zone_warnings);
2842f891cd5SPawel Jakub Dawidek SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RW, &zone_warnings, 0,
2852f891cd5SPawel Jakub Dawidek     "Warn when UMA zones becomes full");
2862f891cd5SPawel Jakub Dawidek 
28786bbae32SJeff Roberson /*
28886bbae32SJeff Roberson  * This routine checks to see whether or not it's safe to enable buckets.
28986bbae32SJeff Roberson  */
29086bbae32SJeff Roberson static void
29186bbae32SJeff Roberson bucket_enable(void)
29286bbae32SJeff Roberson {
293251386b4SMaksim Yevmenkin 	bucketdisable = vm_page_count_min();
29486bbae32SJeff Roberson }
29586bbae32SJeff Roberson 
296dc2c7965SRobert Watson /*
297dc2c7965SRobert Watson  * Initialize bucket_zones, the array of zones of buckets of various sizes.
298dc2c7965SRobert Watson  *
299dc2c7965SRobert Watson  * For each zone, calculate the memory required for each bucket, consisting
300fc03d22bSJeff Roberson  * of the header and an array of pointers.
301dc2c7965SRobert Watson  */
302cae33c14SJeff Roberson static void
303cae33c14SJeff Roberson bucket_init(void)
304cae33c14SJeff Roberson {
305cae33c14SJeff Roberson 	struct uma_bucket_zone *ubz;
306cae33c14SJeff Roberson 	int size;
307fc03d22bSJeff Roberson 	int i;
308cae33c14SJeff Roberson 
309fc03d22bSJeff Roberson 	for (i = 0, ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) {
310cae33c14SJeff Roberson 		size = roundup(sizeof(struct uma_bucket), sizeof(void *));
311cae33c14SJeff Roberson 		size += sizeof(void *) * ubz->ubz_entries;
312cae33c14SJeff Roberson 		ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
313e20a199fSJeff Roberson 		    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
3146fd34d6fSJeff Roberson 		    UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET);
315cae33c14SJeff Roberson 	}
316cae33c14SJeff Roberson }
317cae33c14SJeff Roberson 
318dc2c7965SRobert Watson /*
319dc2c7965SRobert Watson  * Given a desired number of entries for a bucket, return the zone from which
320dc2c7965SRobert Watson  * to allocate the bucket.
321dc2c7965SRobert Watson  */
322dc2c7965SRobert Watson static struct uma_bucket_zone *
323dc2c7965SRobert Watson bucket_zone_lookup(int entries)
324dc2c7965SRobert Watson {
325fc03d22bSJeff Roberson 	struct uma_bucket_zone *ubz;
326dc2c7965SRobert Watson 
327fc03d22bSJeff Roberson 	for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
328fc03d22bSJeff Roberson 		if (ubz->ubz_entries >= entries)
329fc03d22bSJeff Roberson 			return (ubz);
330fc03d22bSJeff Roberson 	ubz--;
331fc03d22bSJeff Roberson 	return (ubz);
332fc03d22bSJeff Roberson }
333fc03d22bSJeff Roberson 
334fc03d22bSJeff Roberson static int
335fc03d22bSJeff Roberson bucket_select(int size)
336fc03d22bSJeff Roberson {
337fc03d22bSJeff Roberson 	struct uma_bucket_zone *ubz;
338fc03d22bSJeff Roberson 
339fc03d22bSJeff Roberson 	ubz = &bucket_zones[0];
340fc03d22bSJeff Roberson 	if (size > ubz->ubz_maxsize)
341fc03d22bSJeff Roberson 		return MAX((ubz->ubz_maxsize * ubz->ubz_entries) / size, 1);
342fc03d22bSJeff Roberson 
343fc03d22bSJeff Roberson 	for (; ubz->ubz_entries != 0; ubz++)
344fc03d22bSJeff Roberson 		if (ubz->ubz_maxsize < size)
345fc03d22bSJeff Roberson 			break;
346fc03d22bSJeff Roberson 	ubz--;
347fc03d22bSJeff Roberson 	return (ubz->ubz_entries);
348dc2c7965SRobert Watson }
349dc2c7965SRobert Watson 
350cae33c14SJeff Roberson static uma_bucket_t
3516fd34d6fSJeff Roberson bucket_alloc(uma_zone_t zone, void *udata, int flags)
352cae33c14SJeff Roberson {
353cae33c14SJeff Roberson 	struct uma_bucket_zone *ubz;
354cae33c14SJeff Roberson 	uma_bucket_t bucket;
355cae33c14SJeff Roberson 
356cae33c14SJeff Roberson 	/*
357cae33c14SJeff Roberson 	 * This is to stop us from allocating per cpu buckets while we're
3583803b26bSDag-Erling Smørgrav 	 * running out of vm.boot_pages.  Otherwise, we would exhaust the
359cae33c14SJeff Roberson 	 * boot pages.  This also prevents us from allocating buckets in
360cae33c14SJeff Roberson 	 * low memory situations.
361cae33c14SJeff Roberson 	 */
362cae33c14SJeff Roberson 	if (bucketdisable)
363cae33c14SJeff Roberson 		return (NULL);
3646fd34d6fSJeff Roberson 	/*
3656fd34d6fSJeff Roberson 	 * To limit bucket recursion we store the original zone flags
3666fd34d6fSJeff Roberson 	 * in a cookie passed via zalloc_arg/zfree_arg.  This allows the
3676fd34d6fSJeff Roberson 	 * NOVM flag to persist even through deep recursions.  We also
3686fd34d6fSJeff Roberson 	 * store ZFLAG_BUCKET once we have recursed attempting to allocate
3696fd34d6fSJeff Roberson 	 * a bucket for a bucket zone so we do not allow infinite bucket
3706fd34d6fSJeff Roberson 	 * recursion.  This cookie will even persist to frees of unused
3716fd34d6fSJeff Roberson 	 * buckets via the allocation path or bucket allocations in the
3726fd34d6fSJeff Roberson 	 * free path.
3736fd34d6fSJeff Roberson 	 */
3746fd34d6fSJeff Roberson 	if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
3756fd34d6fSJeff Roberson 		udata = (void *)(uintptr_t)zone->uz_flags;
376e8a720feSAlexander Motin 	else {
377e8a720feSAlexander Motin 		if ((uintptr_t)udata & UMA_ZFLAG_BUCKET)
378e8a720feSAlexander Motin 			return (NULL);
3796fd34d6fSJeff Roberson 		udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET);
380e8a720feSAlexander Motin 	}
3816fd34d6fSJeff Roberson 	if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY)
382af526374SJeff Roberson 		flags |= M_NOVM;
383af526374SJeff Roberson 	ubz = bucket_zone_lookup(zone->uz_count);
384*20d3ab87SAlexander Motin 	if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0)
385*20d3ab87SAlexander Motin 		ubz++;
3866fd34d6fSJeff Roberson 	bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags);
387cae33c14SJeff Roberson 	if (bucket) {
388cae33c14SJeff Roberson #ifdef INVARIANTS
389cae33c14SJeff Roberson 		bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
390cae33c14SJeff Roberson #endif
391cae33c14SJeff Roberson 		bucket->ub_cnt = 0;
392cae33c14SJeff Roberson 		bucket->ub_entries = ubz->ubz_entries;
393cae33c14SJeff Roberson 	}
394cae33c14SJeff Roberson 
395cae33c14SJeff Roberson 	return (bucket);
396cae33c14SJeff Roberson }
397cae33c14SJeff Roberson 
398cae33c14SJeff Roberson static void
3996fd34d6fSJeff Roberson bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata)
400cae33c14SJeff Roberson {
401cae33c14SJeff Roberson 	struct uma_bucket_zone *ubz;
402cae33c14SJeff Roberson 
403fc03d22bSJeff Roberson 	KASSERT(bucket->ub_cnt == 0,
404fc03d22bSJeff Roberson 	    ("bucket_free: Freeing a non free bucket."));
4056fd34d6fSJeff Roberson 	if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
4066fd34d6fSJeff Roberson 		udata = (void *)(uintptr_t)zone->uz_flags;
407dc2c7965SRobert Watson 	ubz = bucket_zone_lookup(bucket->ub_entries);
4086fd34d6fSJeff Roberson 	uma_zfree_arg(ubz->ubz_zone, bucket, udata);
409cae33c14SJeff Roberson }
410cae33c14SJeff Roberson 
411cae33c14SJeff Roberson static void
412cae33c14SJeff Roberson bucket_zone_drain(void)
413cae33c14SJeff Roberson {
414cae33c14SJeff Roberson 	struct uma_bucket_zone *ubz;
415cae33c14SJeff Roberson 
416cae33c14SJeff Roberson 	for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
417cae33c14SJeff Roberson 		zone_drain(ubz->ubz_zone);
418cae33c14SJeff Roberson }
419cae33c14SJeff Roberson 
4202f891cd5SPawel Jakub Dawidek static void
4212f891cd5SPawel Jakub Dawidek zone_log_warning(uma_zone_t zone)
4222f891cd5SPawel Jakub Dawidek {
4232f891cd5SPawel Jakub Dawidek 	static const struct timeval warninterval = { 300, 0 };
4242f891cd5SPawel Jakub Dawidek 
4252f891cd5SPawel Jakub Dawidek 	if (!zone_warnings || zone->uz_warning == NULL)
4262f891cd5SPawel Jakub Dawidek 		return;
4272f891cd5SPawel Jakub Dawidek 
4282f891cd5SPawel Jakub Dawidek 	if (ratecheck(&zone->uz_ratecheck, &warninterval))
4292f891cd5SPawel Jakub Dawidek 		printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
4302f891cd5SPawel Jakub Dawidek }
4312f891cd5SPawel Jakub Dawidek 
432e20a199fSJeff Roberson static void
433e20a199fSJeff Roberson zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t))
434e20a199fSJeff Roberson {
435e20a199fSJeff Roberson 	uma_klink_t klink;
436e20a199fSJeff Roberson 
437e20a199fSJeff Roberson 	LIST_FOREACH(klink, &zone->uz_kegs, kl_link)
438e20a199fSJeff Roberson 		kegfn(klink->kl_keg);
439e20a199fSJeff Roberson }
4408355f576SJeff Roberson 
4418355f576SJeff Roberson /*
4428355f576SJeff Roberson  * Routine called by timeout which is used to fire off some time interval
4439643769aSJeff Roberson  * based calculations.  (stats, hash size, etc.)
4448355f576SJeff Roberson  *
4458355f576SJeff Roberson  * Arguments:
4468355f576SJeff Roberson  *	arg   Unused
4478355f576SJeff Roberson  *
4488355f576SJeff Roberson  * Returns:
4498355f576SJeff Roberson  *	Nothing
4508355f576SJeff Roberson  */
4518355f576SJeff Roberson static void
4528355f576SJeff Roberson uma_timeout(void *unused)
4538355f576SJeff Roberson {
45486bbae32SJeff Roberson 	bucket_enable();
4558355f576SJeff Roberson 	zone_foreach(zone_timeout);
4568355f576SJeff Roberson 
4578355f576SJeff Roberson 	/* Reschedule this event */
4589643769aSJeff Roberson 	callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
4598355f576SJeff Roberson }
4608355f576SJeff Roberson 
4618355f576SJeff Roberson /*
4629643769aSJeff Roberson  * Routine to perform timeout driven calculations.  This expands the
4639643769aSJeff Roberson  * hashes and does per cpu statistics aggregation.
4648355f576SJeff Roberson  *
465e20a199fSJeff Roberson  *  Returns nothing.
4668355f576SJeff Roberson  */
4678355f576SJeff Roberson static void
468e20a199fSJeff Roberson keg_timeout(uma_keg_t keg)
4698355f576SJeff Roberson {
4708355f576SJeff Roberson 
471e20a199fSJeff Roberson 	KEG_LOCK(keg);
4728355f576SJeff Roberson 	/*
473e20a199fSJeff Roberson 	 * Expand the keg hash table.
4748355f576SJeff Roberson 	 *
4758355f576SJeff Roberson 	 * This is done if the number of slabs is larger than the hash size.
4768355f576SJeff Roberson 	 * What I'm trying to do here is completely reduce collisions.  This
4778355f576SJeff Roberson 	 * may be a little aggressive.  Should I allow for two collisions max?
4788355f576SJeff Roberson 	 */
479099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_HASH &&
480099a0e58SBosko Milekic 	    keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) {
4810aef6126SJeff Roberson 		struct uma_hash newhash;
4820aef6126SJeff Roberson 		struct uma_hash oldhash;
4830aef6126SJeff Roberson 		int ret;
4845300d9ddSJeff Roberson 
4850aef6126SJeff Roberson 		/*
4860aef6126SJeff Roberson 		 * This is so involved because allocating and freeing
487e20a199fSJeff Roberson 		 * while the keg lock is held will lead to deadlock.
4880aef6126SJeff Roberson 		 * I have to do everything in stages and check for
4890aef6126SJeff Roberson 		 * races.
4900aef6126SJeff Roberson 		 */
491099a0e58SBosko Milekic 		newhash = keg->uk_hash;
492e20a199fSJeff Roberson 		KEG_UNLOCK(keg);
4930aef6126SJeff Roberson 		ret = hash_alloc(&newhash);
494e20a199fSJeff Roberson 		KEG_LOCK(keg);
4950aef6126SJeff Roberson 		if (ret) {
496099a0e58SBosko Milekic 			if (hash_expand(&keg->uk_hash, &newhash)) {
497099a0e58SBosko Milekic 				oldhash = keg->uk_hash;
498099a0e58SBosko Milekic 				keg->uk_hash = newhash;
4990aef6126SJeff Roberson 			} else
5000aef6126SJeff Roberson 				oldhash = newhash;
5010aef6126SJeff Roberson 
502e20a199fSJeff Roberson 			KEG_UNLOCK(keg);
5030aef6126SJeff Roberson 			hash_free(&oldhash);
504a1dff920SDavide Italiano 			return;
5050aef6126SJeff Roberson 		}
5065300d9ddSJeff Roberson 	}
507e20a199fSJeff Roberson 	KEG_UNLOCK(keg);
508e20a199fSJeff Roberson }
509e20a199fSJeff Roberson 
510e20a199fSJeff Roberson static void
511e20a199fSJeff Roberson zone_timeout(uma_zone_t zone)
512e20a199fSJeff Roberson {
513e20a199fSJeff Roberson 
514e20a199fSJeff Roberson 	zone_foreach_keg(zone, &keg_timeout);
5158355f576SJeff Roberson }
5168355f576SJeff Roberson 
5178355f576SJeff Roberson /*
5185300d9ddSJeff Roberson  * Allocate and zero fill the next sized hash table from the appropriate
5195300d9ddSJeff Roberson  * backing store.
5205300d9ddSJeff Roberson  *
5215300d9ddSJeff Roberson  * Arguments:
5220aef6126SJeff Roberson  *	hash  A new hash structure with the old hash size in uh_hashsize
5235300d9ddSJeff Roberson  *
5245300d9ddSJeff Roberson  * Returns:
5250aef6126SJeff Roberson  *	1 on sucess and 0 on failure.
5265300d9ddSJeff Roberson  */
52737c84183SPoul-Henning Kamp static int
5280aef6126SJeff Roberson hash_alloc(struct uma_hash *hash)
5295300d9ddSJeff Roberson {
5300aef6126SJeff Roberson 	int oldsize;
5315300d9ddSJeff Roberson 	int alloc;
5325300d9ddSJeff Roberson 
5330aef6126SJeff Roberson 	oldsize = hash->uh_hashsize;
5340aef6126SJeff Roberson 
5355300d9ddSJeff Roberson 	/* We're just going to go to a power of two greater */
5360aef6126SJeff Roberson 	if (oldsize)  {
5370aef6126SJeff Roberson 		hash->uh_hashsize = oldsize * 2;
5380aef6126SJeff Roberson 		alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize;
5390aef6126SJeff Roberson 		hash->uh_slab_hash = (struct slabhead *)malloc(alloc,
540961647dfSJeff Roberson 		    M_UMAHASH, M_NOWAIT);
5415300d9ddSJeff Roberson 	} else {
5420aef6126SJeff Roberson 		alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
543e20a199fSJeff Roberson 		hash->uh_slab_hash = zone_alloc_item(hashzone, NULL,
544a163d034SWarner Losh 		    M_WAITOK);
5450aef6126SJeff Roberson 		hash->uh_hashsize = UMA_HASH_SIZE_INIT;
5465300d9ddSJeff Roberson 	}
5470aef6126SJeff Roberson 	if (hash->uh_slab_hash) {
5480aef6126SJeff Roberson 		bzero(hash->uh_slab_hash, alloc);
5490aef6126SJeff Roberson 		hash->uh_hashmask = hash->uh_hashsize - 1;
5500aef6126SJeff Roberson 		return (1);
5510aef6126SJeff Roberson 	}
5525300d9ddSJeff Roberson 
5530aef6126SJeff Roberson 	return (0);
5545300d9ddSJeff Roberson }
5555300d9ddSJeff Roberson 
5565300d9ddSJeff Roberson /*
55764f051e9SJeff Roberson  * Expands the hash table for HASH zones.  This is done from zone_timeout
55864f051e9SJeff Roberson  * to reduce collisions.  This must not be done in the regular allocation
55964f051e9SJeff Roberson  * path, otherwise, we can recurse on the vm while allocating pages.
5608355f576SJeff Roberson  *
5618355f576SJeff Roberson  * Arguments:
5620aef6126SJeff Roberson  *	oldhash  The hash you want to expand
5630aef6126SJeff Roberson  *	newhash  The hash structure for the new table
5648355f576SJeff Roberson  *
5658355f576SJeff Roberson  * Returns:
5668355f576SJeff Roberson  *	Nothing
5678355f576SJeff Roberson  *
5688355f576SJeff Roberson  * Discussion:
5698355f576SJeff Roberson  */
5700aef6126SJeff Roberson static int
5710aef6126SJeff Roberson hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash)
5728355f576SJeff Roberson {
5738355f576SJeff Roberson 	uma_slab_t slab;
5748355f576SJeff Roberson 	int hval;
5758355f576SJeff Roberson 	int i;
5768355f576SJeff Roberson 
5770aef6126SJeff Roberson 	if (!newhash->uh_slab_hash)
5780aef6126SJeff Roberson 		return (0);
5798355f576SJeff Roberson 
5800aef6126SJeff Roberson 	if (oldhash->uh_hashsize >= newhash->uh_hashsize)
5810aef6126SJeff Roberson 		return (0);
5828355f576SJeff Roberson 
5838355f576SJeff Roberson 	/*
5848355f576SJeff Roberson 	 * I need to investigate hash algorithms for resizing without a
5858355f576SJeff Roberson 	 * full rehash.
5868355f576SJeff Roberson 	 */
5878355f576SJeff Roberson 
5880aef6126SJeff Roberson 	for (i = 0; i < oldhash->uh_hashsize; i++)
5890aef6126SJeff Roberson 		while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) {
5900aef6126SJeff Roberson 			slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]);
5910aef6126SJeff Roberson 			SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink);
5920aef6126SJeff Roberson 			hval = UMA_HASH(newhash, slab->us_data);
5930aef6126SJeff Roberson 			SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval],
5940aef6126SJeff Roberson 			    slab, us_hlink);
5958355f576SJeff Roberson 		}
5968355f576SJeff Roberson 
5970aef6126SJeff Roberson 	return (1);
5989c2cd7e5SJeff Roberson }
5999c2cd7e5SJeff Roberson 
6005300d9ddSJeff Roberson /*
6015300d9ddSJeff Roberson  * Free the hash bucket to the appropriate backing store.
6025300d9ddSJeff Roberson  *
6035300d9ddSJeff Roberson  * Arguments:
6045300d9ddSJeff Roberson  *	slab_hash  The hash bucket we're freeing
6055300d9ddSJeff Roberson  *	hashsize   The number of entries in that hash bucket
6065300d9ddSJeff Roberson  *
6075300d9ddSJeff Roberson  * Returns:
6085300d9ddSJeff Roberson  *	Nothing
6095300d9ddSJeff Roberson  */
6109c2cd7e5SJeff Roberson static void
6110aef6126SJeff Roberson hash_free(struct uma_hash *hash)
6129c2cd7e5SJeff Roberson {
6130aef6126SJeff Roberson 	if (hash->uh_slab_hash == NULL)
6140aef6126SJeff Roberson 		return;
6150aef6126SJeff Roberson 	if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
6160095a784SJeff Roberson 		zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
6178355f576SJeff Roberson 	else
618961647dfSJeff Roberson 		free(hash->uh_slab_hash, M_UMAHASH);
6198355f576SJeff Roberson }
6208355f576SJeff Roberson 
6218355f576SJeff Roberson /*
6228355f576SJeff Roberson  * Frees all outstanding items in a bucket
6238355f576SJeff Roberson  *
6248355f576SJeff Roberson  * Arguments:
6258355f576SJeff Roberson  *	zone   The zone to free to, must be unlocked.
6268355f576SJeff Roberson  *	bucket The free/alloc bucket with items, cpu queue must be locked.
6278355f576SJeff Roberson  *
6288355f576SJeff Roberson  * Returns:
6298355f576SJeff Roberson  *	Nothing
6308355f576SJeff Roberson  */
6318355f576SJeff Roberson 
6328355f576SJeff Roberson static void
6338355f576SJeff Roberson bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
6348355f576SJeff Roberson {
6350095a784SJeff Roberson 	int i;
6368355f576SJeff Roberson 
6378355f576SJeff Roberson 	if (bucket == NULL)
6388355f576SJeff Roberson 		return;
6398355f576SJeff Roberson 
6400095a784SJeff Roberson 	if (zone->uz_fini)
6410095a784SJeff Roberson 		for (i = 0; i < bucket->ub_cnt; i++)
6420095a784SJeff Roberson 			zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
6430095a784SJeff Roberson 	zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
6440095a784SJeff Roberson 	bucket->ub_cnt = 0;
6458355f576SJeff Roberson }
6468355f576SJeff Roberson 
6478355f576SJeff Roberson /*
6488355f576SJeff Roberson  * Drains the per cpu caches for a zone.
6498355f576SJeff Roberson  *
6505d1ae027SRobert Watson  * NOTE: This may only be called while the zone is being turn down, and not
6515d1ae027SRobert Watson  * during normal operation.  This is necessary in order that we do not have
6525d1ae027SRobert Watson  * to migrate CPUs to drain the per-CPU caches.
6535d1ae027SRobert Watson  *
6548355f576SJeff Roberson  * Arguments:
6558355f576SJeff Roberson  *	zone     The zone to drain, must be unlocked.
6568355f576SJeff Roberson  *
6578355f576SJeff Roberson  * Returns:
6588355f576SJeff Roberson  *	Nothing
6598355f576SJeff Roberson  */
6608355f576SJeff Roberson static void
6619643769aSJeff Roberson cache_drain(uma_zone_t zone)
6628355f576SJeff Roberson {
6638355f576SJeff Roberson 	uma_cache_t cache;
6648355f576SJeff Roberson 	int cpu;
6658355f576SJeff Roberson 
6668355f576SJeff Roberson 	/*
6675d1ae027SRobert Watson 	 * XXX: It is safe to not lock the per-CPU caches, because we're
6685d1ae027SRobert Watson 	 * tearing down the zone anyway.  I.e., there will be no further use
6695d1ae027SRobert Watson 	 * of the caches at this point.
6705d1ae027SRobert Watson 	 *
6715d1ae027SRobert Watson 	 * XXX: It would good to be able to assert that the zone is being
6725d1ae027SRobert Watson 	 * torn down to prevent improper use of cache_drain().
6735d1ae027SRobert Watson 	 *
6745d1ae027SRobert Watson 	 * XXX: We lock the zone before passing into bucket_cache_drain() as
6755d1ae027SRobert Watson 	 * it is used elsewhere.  Should the tear-down path be made special
6765d1ae027SRobert Watson 	 * there in some form?
6778355f576SJeff Roberson 	 */
6783aa6d94eSJohn Baldwin 	CPU_FOREACH(cpu) {
6798355f576SJeff Roberson 		cache = &zone->uz_cpu[cpu];
6808355f576SJeff Roberson 		bucket_drain(zone, cache->uc_allocbucket);
6818355f576SJeff Roberson 		bucket_drain(zone, cache->uc_freebucket);
682174ab450SBosko Milekic 		if (cache->uc_allocbucket != NULL)
6836fd34d6fSJeff Roberson 			bucket_free(zone, cache->uc_allocbucket, NULL);
684174ab450SBosko Milekic 		if (cache->uc_freebucket != NULL)
6856fd34d6fSJeff Roberson 			bucket_free(zone, cache->uc_freebucket, NULL);
686d56368d7SBosko Milekic 		cache->uc_allocbucket = cache->uc_freebucket = NULL;
687d56368d7SBosko Milekic 	}
688aaa8bb16SJeff Roberson 	ZONE_LOCK(zone);
689aaa8bb16SJeff Roberson 	bucket_cache_drain(zone);
690aaa8bb16SJeff Roberson 	ZONE_UNLOCK(zone);
691aaa8bb16SJeff Roberson }
692aaa8bb16SJeff Roberson 
693a2de44abSAlexander Motin static void
694a2de44abSAlexander Motin cache_shrink(uma_zone_t zone)
695a2de44abSAlexander Motin {
696a2de44abSAlexander Motin 
697a2de44abSAlexander Motin 	if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
698a2de44abSAlexander Motin 		return;
699a2de44abSAlexander Motin 
700a2de44abSAlexander Motin 	ZONE_LOCK(zone);
701a2de44abSAlexander Motin 	zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2;
702a2de44abSAlexander Motin 	ZONE_UNLOCK(zone);
703a2de44abSAlexander Motin }
704a2de44abSAlexander Motin 
705a2de44abSAlexander Motin static void
706a2de44abSAlexander Motin cache_drain_safe_cpu(uma_zone_t zone)
707a2de44abSAlexander Motin {
708a2de44abSAlexander Motin 	uma_cache_t cache;
7098a8d9d14SAlexander Motin 	uma_bucket_t b1, b2;
710a2de44abSAlexander Motin 
711a2de44abSAlexander Motin 	if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
712a2de44abSAlexander Motin 		return;
713a2de44abSAlexander Motin 
7148a8d9d14SAlexander Motin 	b1 = b2 = NULL;
715a2de44abSAlexander Motin 	ZONE_LOCK(zone);
716a2de44abSAlexander Motin 	critical_enter();
717a2de44abSAlexander Motin 	cache = &zone->uz_cpu[curcpu];
718a2de44abSAlexander Motin 	if (cache->uc_allocbucket) {
7198a8d9d14SAlexander Motin 		if (cache->uc_allocbucket->ub_cnt != 0)
7208a8d9d14SAlexander Motin 			LIST_INSERT_HEAD(&zone->uz_buckets,
7218a8d9d14SAlexander Motin 			    cache->uc_allocbucket, ub_link);
7228a8d9d14SAlexander Motin 		else
7238a8d9d14SAlexander Motin 			b1 = cache->uc_allocbucket;
724a2de44abSAlexander Motin 		cache->uc_allocbucket = NULL;
725a2de44abSAlexander Motin 	}
726a2de44abSAlexander Motin 	if (cache->uc_freebucket) {
7278a8d9d14SAlexander Motin 		if (cache->uc_freebucket->ub_cnt != 0)
7288a8d9d14SAlexander Motin 			LIST_INSERT_HEAD(&zone->uz_buckets,
7298a8d9d14SAlexander Motin 			    cache->uc_freebucket, ub_link);
7308a8d9d14SAlexander Motin 		else
7318a8d9d14SAlexander Motin 			b2 = cache->uc_freebucket;
732a2de44abSAlexander Motin 		cache->uc_freebucket = NULL;
733a2de44abSAlexander Motin 	}
734a2de44abSAlexander Motin 	critical_exit();
735a2de44abSAlexander Motin 	ZONE_UNLOCK(zone);
7368a8d9d14SAlexander Motin 	if (b1)
7378a8d9d14SAlexander Motin 		bucket_free(zone, b1, NULL);
7388a8d9d14SAlexander Motin 	if (b2)
7398a8d9d14SAlexander Motin 		bucket_free(zone, b2, NULL);
740a2de44abSAlexander Motin }
741a2de44abSAlexander Motin 
742a2de44abSAlexander Motin /*
743a2de44abSAlexander Motin  * Safely drain per-CPU caches of a zone(s) to alloc bucket.
744a2de44abSAlexander Motin  * This is an expensive call because it needs to bind to all CPUs
745a2de44abSAlexander Motin  * one by one and enter a critical section on each of them in order
746a2de44abSAlexander Motin  * to safely access their cache buckets.
747a2de44abSAlexander Motin  * Zone lock must not be held on call this function.
748a2de44abSAlexander Motin  */
749a2de44abSAlexander Motin static void
750a2de44abSAlexander Motin cache_drain_safe(uma_zone_t zone)
751a2de44abSAlexander Motin {
752a2de44abSAlexander Motin 	int cpu;
753a2de44abSAlexander Motin 
754a2de44abSAlexander Motin 	/*
755a2de44abSAlexander Motin 	 * Polite bucket sizes shrinking was not enouth, shrink aggressively.
756a2de44abSAlexander Motin 	 */
757a2de44abSAlexander Motin 	if (zone)
758a2de44abSAlexander Motin 		cache_shrink(zone);
759a2de44abSAlexander Motin 	else
760a2de44abSAlexander Motin 		zone_foreach(cache_shrink);
761a2de44abSAlexander Motin 
762a2de44abSAlexander Motin 	CPU_FOREACH(cpu) {
763a2de44abSAlexander Motin 		thread_lock(curthread);
764a2de44abSAlexander Motin 		sched_bind(curthread, cpu);
765a2de44abSAlexander Motin 		thread_unlock(curthread);
766a2de44abSAlexander Motin 
767a2de44abSAlexander Motin 		if (zone)
768a2de44abSAlexander Motin 			cache_drain_safe_cpu(zone);
769a2de44abSAlexander Motin 		else
770a2de44abSAlexander Motin 			zone_foreach(cache_drain_safe_cpu);
771a2de44abSAlexander Motin 	}
772a2de44abSAlexander Motin 	thread_lock(curthread);
773a2de44abSAlexander Motin 	sched_unbind(curthread);
774a2de44abSAlexander Motin 	thread_unlock(curthread);
775a2de44abSAlexander Motin }
776a2de44abSAlexander Motin 
777aaa8bb16SJeff Roberson /*
778aaa8bb16SJeff Roberson  * Drain the cached buckets from a zone.  Expects a locked zone on entry.
779aaa8bb16SJeff Roberson  */
780aaa8bb16SJeff Roberson static void
781aaa8bb16SJeff Roberson bucket_cache_drain(uma_zone_t zone)
782aaa8bb16SJeff Roberson {
783aaa8bb16SJeff Roberson 	uma_bucket_t bucket;
7848355f576SJeff Roberson 
7858355f576SJeff Roberson 	/*
7868355f576SJeff Roberson 	 * Drain the bucket queues and free the buckets, we just keep two per
7878355f576SJeff Roberson 	 * cpu (alloc/free).
7888355f576SJeff Roberson 	 */
789fc03d22bSJeff Roberson 	while ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
7908355f576SJeff Roberson 		LIST_REMOVE(bucket, ub_link);
7918355f576SJeff Roberson 		ZONE_UNLOCK(zone);
7928355f576SJeff Roberson 		bucket_drain(zone, bucket);
7936fd34d6fSJeff Roberson 		bucket_free(zone, bucket, NULL);
7948355f576SJeff Roberson 		ZONE_LOCK(zone);
7958355f576SJeff Roberson 	}
796ace66b56SAlexander Motin 
797ace66b56SAlexander Motin 	/*
798ace66b56SAlexander Motin 	 * Shrink further bucket sizes.  Price of single zone lock collision
799ace66b56SAlexander Motin 	 * is probably lower then price of global cache drain.
800ace66b56SAlexander Motin 	 */
801ace66b56SAlexander Motin 	if (zone->uz_count > zone->uz_count_min)
802ace66b56SAlexander Motin 		zone->uz_count--;
8038355f576SJeff Roberson }
804fc03d22bSJeff Roberson 
805fc03d22bSJeff Roberson static void
806fc03d22bSJeff Roberson keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)
807fc03d22bSJeff Roberson {
808fc03d22bSJeff Roberson 	uint8_t *mem;
809fc03d22bSJeff Roberson 	int i;
810fc03d22bSJeff Roberson 	uint8_t flags;
811fc03d22bSJeff Roberson 
812fc03d22bSJeff Roberson 	mem = slab->us_data;
813fc03d22bSJeff Roberson 	flags = slab->us_flags;
814fc03d22bSJeff Roberson 	i = start;
815fc03d22bSJeff Roberson 	if (keg->uk_fini != NULL) {
816fc03d22bSJeff Roberson 		for (i--; i > -1; i--)
817fc03d22bSJeff Roberson 			keg->uk_fini(slab->us_data + (keg->uk_rsize * i),
818fc03d22bSJeff Roberson 			    keg->uk_size);
819fc03d22bSJeff Roberson 	}
820fc03d22bSJeff Roberson 	if (keg->uk_flags & UMA_ZONE_OFFPAGE)
821fc03d22bSJeff Roberson 		zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
822fc03d22bSJeff Roberson #ifdef UMA_DEBUG
823fc03d22bSJeff Roberson 	printf("%s: Returning %d bytes.\n", keg->uk_name,
824fc03d22bSJeff Roberson 	    PAGE_SIZE * keg->uk_ppera);
825fc03d22bSJeff Roberson #endif
826fc03d22bSJeff Roberson 	keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
8278355f576SJeff Roberson }
8288355f576SJeff Roberson 
8298355f576SJeff Roberson /*
830e20a199fSJeff Roberson  * Frees pages from a keg back to the system.  This is done on demand from
8318355f576SJeff Roberson  * the pageout daemon.
8328355f576SJeff Roberson  *
833e20a199fSJeff Roberson  * Returns nothing.
8348355f576SJeff Roberson  */
835e20a199fSJeff Roberson static void
836e20a199fSJeff Roberson keg_drain(uma_keg_t keg)
8378355f576SJeff Roberson {
8381e183df2SStefan Farfeleder 	struct slabhead freeslabs = { 0 };
8398355f576SJeff Roberson 	uma_slab_t slab;
8408355f576SJeff Roberson 	uma_slab_t n;
8418355f576SJeff Roberson 
8428355f576SJeff Roberson 	/*
843e20a199fSJeff Roberson 	 * We don't want to take pages from statically allocated kegs at this
8448355f576SJeff Roberson 	 * time
8458355f576SJeff Roberson 	 */
846099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
8478355f576SJeff Roberson 		return;
8488355f576SJeff Roberson 
8498355f576SJeff Roberson #ifdef UMA_DEBUG
850e20a199fSJeff Roberson 	printf("%s free items: %u\n", keg->uk_name, keg->uk_free);
8518355f576SJeff Roberson #endif
852e20a199fSJeff Roberson 	KEG_LOCK(keg);
853099a0e58SBosko Milekic 	if (keg->uk_free == 0)
8548355f576SJeff Roberson 		goto finished;
8558355f576SJeff Roberson 
856099a0e58SBosko Milekic 	slab = LIST_FIRST(&keg->uk_free_slab);
8579643769aSJeff Roberson 	while (slab) {
8588355f576SJeff Roberson 		n = LIST_NEXT(slab, us_link);
8598355f576SJeff Roberson 
8608355f576SJeff Roberson 		/* We have no where to free these to */
8618355f576SJeff Roberson 		if (slab->us_flags & UMA_SLAB_BOOT) {
8628355f576SJeff Roberson 			slab = n;
8638355f576SJeff Roberson 			continue;
8648355f576SJeff Roberson 		}
8658355f576SJeff Roberson 
8668355f576SJeff Roberson 		LIST_REMOVE(slab, us_link);
867099a0e58SBosko Milekic 		keg->uk_pages -= keg->uk_ppera;
868099a0e58SBosko Milekic 		keg->uk_free -= keg->uk_ipers;
869713deb36SJeff Roberson 
870099a0e58SBosko Milekic 		if (keg->uk_flags & UMA_ZONE_HASH)
871099a0e58SBosko Milekic 			UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data);
872713deb36SJeff Roberson 
873713deb36SJeff Roberson 		SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink);
874713deb36SJeff Roberson 
875713deb36SJeff Roberson 		slab = n;
876713deb36SJeff Roberson 	}
877713deb36SJeff Roberson finished:
878e20a199fSJeff Roberson 	KEG_UNLOCK(keg);
879713deb36SJeff Roberson 
880713deb36SJeff Roberson 	while ((slab = SLIST_FIRST(&freeslabs)) != NULL) {
881713deb36SJeff Roberson 		SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink);
8821645995bSKirk McKusick 		keg_free_slab(keg, slab, keg->uk_ipers);
8838355f576SJeff Roberson 	}
8848355f576SJeff Roberson }
8858355f576SJeff Roberson 
886e20a199fSJeff Roberson static void
887e20a199fSJeff Roberson zone_drain_wait(uma_zone_t zone, int waitok)
888e20a199fSJeff Roberson {
889e20a199fSJeff Roberson 
8908355f576SJeff Roberson 	/*
891e20a199fSJeff Roberson 	 * Set draining to interlock with zone_dtor() so we can release our
892e20a199fSJeff Roberson 	 * locks as we go.  Only dtor() should do a WAITOK call since it
893e20a199fSJeff Roberson 	 * is the only call that knows the structure will still be available
894e20a199fSJeff Roberson 	 * when it wakes up.
895e20a199fSJeff Roberson 	 */
896e20a199fSJeff Roberson 	ZONE_LOCK(zone);
897e20a199fSJeff Roberson 	while (zone->uz_flags & UMA_ZFLAG_DRAINING) {
898e20a199fSJeff Roberson 		if (waitok == M_NOWAIT)
899e20a199fSJeff Roberson 			goto out;
900e20a199fSJeff Roberson 		mtx_unlock(&uma_mtx);
901af526374SJeff Roberson 		msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1);
902e20a199fSJeff Roberson 		mtx_lock(&uma_mtx);
903e20a199fSJeff Roberson 	}
904e20a199fSJeff Roberson 	zone->uz_flags |= UMA_ZFLAG_DRAINING;
905e20a199fSJeff Roberson 	bucket_cache_drain(zone);
906e20a199fSJeff Roberson 	ZONE_UNLOCK(zone);
907e20a199fSJeff Roberson 	/*
908e20a199fSJeff Roberson 	 * The DRAINING flag protects us from being freed while
909e20a199fSJeff Roberson 	 * we're running.  Normally the uma_mtx would protect us but we
910e20a199fSJeff Roberson 	 * must be able to release and acquire the right lock for each keg.
911e20a199fSJeff Roberson 	 */
912e20a199fSJeff Roberson 	zone_foreach_keg(zone, &keg_drain);
913e20a199fSJeff Roberson 	ZONE_LOCK(zone);
914e20a199fSJeff Roberson 	zone->uz_flags &= ~UMA_ZFLAG_DRAINING;
915e20a199fSJeff Roberson 	wakeup(zone);
916e20a199fSJeff Roberson out:
917e20a199fSJeff Roberson 	ZONE_UNLOCK(zone);
918e20a199fSJeff Roberson }
919e20a199fSJeff Roberson 
920e20a199fSJeff Roberson void
921e20a199fSJeff Roberson zone_drain(uma_zone_t zone)
922e20a199fSJeff Roberson {
923e20a199fSJeff Roberson 
924e20a199fSJeff Roberson 	zone_drain_wait(zone, M_NOWAIT);
925e20a199fSJeff Roberson }
926e20a199fSJeff Roberson 
927e20a199fSJeff Roberson /*
928e20a199fSJeff Roberson  * Allocate a new slab for a keg.  This does not insert the slab onto a list.
9298355f576SJeff Roberson  *
9308355f576SJeff Roberson  * Arguments:
9318355f576SJeff Roberson  *	wait  Shall we wait?
9328355f576SJeff Roberson  *
9338355f576SJeff Roberson  * Returns:
9348355f576SJeff Roberson  *	The slab that was allocated or NULL if there is no memory and the
9358355f576SJeff Roberson  *	caller specified M_NOWAIT.
9368355f576SJeff Roberson  */
9378355f576SJeff Roberson static uma_slab_t
938e20a199fSJeff Roberson keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
9398355f576SJeff Roberson {
940099a0e58SBosko Milekic 	uma_slabrefcnt_t slabref;
941e20a199fSJeff Roberson 	uma_alloc allocf;
942099a0e58SBosko Milekic 	uma_slab_t slab;
94385dcf349SGleb Smirnoff 	uint8_t *mem;
94485dcf349SGleb Smirnoff 	uint8_t flags;
9458355f576SJeff Roberson 	int i;
9468355f576SJeff Roberson 
947e20a199fSJeff Roberson 	mtx_assert(&keg->uk_lock, MA_OWNED);
948a553d4b8SJeff Roberson 	slab = NULL;
949fc03d22bSJeff Roberson 	mem = NULL;
950a553d4b8SJeff Roberson 
9518355f576SJeff Roberson #ifdef UMA_DEBUG
9520095a784SJeff Roberson 	printf("alloc_slab:  Allocating a new slab for %s\n", keg->uk_name);
9538355f576SJeff Roberson #endif
954e20a199fSJeff Roberson 	allocf = keg->uk_allocf;
955e20a199fSJeff Roberson 	KEG_UNLOCK(keg);
956a553d4b8SJeff Roberson 
957099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
958e20a199fSJeff Roberson 		slab = zone_alloc_item(keg->uk_slabzone, NULL, wait);
959fc03d22bSJeff Roberson 		if (slab == NULL)
960fc03d22bSJeff Roberson 			goto out;
961a553d4b8SJeff Roberson 	}
962a553d4b8SJeff Roberson 
9633370c5bfSJeff Roberson 	/*
9643370c5bfSJeff Roberson 	 * This reproduces the old vm_zone behavior of zero filling pages the
9653370c5bfSJeff Roberson 	 * first time they are added to a zone.
9663370c5bfSJeff Roberson 	 *
9673370c5bfSJeff Roberson 	 * Malloced items are zeroed in uma_zalloc.
9683370c5bfSJeff Roberson 	 */
9693370c5bfSJeff Roberson 
970099a0e58SBosko Milekic 	if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0)
9713370c5bfSJeff Roberson 		wait |= M_ZERO;
9723370c5bfSJeff Roberson 	else
9733370c5bfSJeff Roberson 		wait &= ~M_ZERO;
9743370c5bfSJeff Roberson 
975263811f7SKip Macy 	if (keg->uk_flags & UMA_ZONE_NODUMP)
976263811f7SKip Macy 		wait |= M_NODUMP;
977263811f7SKip Macy 
978e20a199fSJeff Roberson 	/* zone is passed for legacy reasons. */
979ad97af7eSGleb Smirnoff 	mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
980a553d4b8SJeff Roberson 	if (mem == NULL) {
981b23f72e9SBrian Feldman 		if (keg->uk_flags & UMA_ZONE_OFFPAGE)
9820095a784SJeff Roberson 			zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
983fc03d22bSJeff Roberson 		slab = NULL;
984fc03d22bSJeff Roberson 		goto out;
985a553d4b8SJeff Roberson 	}
9868355f576SJeff Roberson 
9875c0e403bSJeff Roberson 	/* Point the slab into the allocated memory */
988099a0e58SBosko Milekic 	if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
989099a0e58SBosko Milekic 		slab = (uma_slab_t )(mem + keg->uk_pgoff);
9905c0e403bSJeff Roberson 
991e20a199fSJeff Roberson 	if (keg->uk_flags & UMA_ZONE_VTOSLAB)
992099a0e58SBosko Milekic 		for (i = 0; i < keg->uk_ppera; i++)
99399571dc3SJeff Roberson 			vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab);
9948355f576SJeff Roberson 
995099a0e58SBosko Milekic 	slab->us_keg = keg;
9968355f576SJeff Roberson 	slab->us_data = mem;
997099a0e58SBosko Milekic 	slab->us_freecount = keg->uk_ipers;
9988355f576SJeff Roberson 	slab->us_flags = flags;
999ef72505eSJeff Roberson 	BIT_FILL(SLAB_SETSIZE, &slab->us_free);
1000ef72505eSJeff Roberson #ifdef INVARIANTS
1001ef72505eSJeff Roberson 	BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree);
1002ef72505eSJeff Roberson #endif
1003099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_REFCNT) {
1004099a0e58SBosko Milekic 		slabref = (uma_slabrefcnt_t)slab;
1005ab14a3f7SBrian Feldman 		for (i = 0; i < keg->uk_ipers; i++)
1006ef72505eSJeff Roberson 			slabref->us_refcnt[i] = 0;
1007099a0e58SBosko Milekic 	}
1008099a0e58SBosko Milekic 
1009b23f72e9SBrian Feldman 	if (keg->uk_init != NULL) {
1010099a0e58SBosko Milekic 		for (i = 0; i < keg->uk_ipers; i++)
1011b23f72e9SBrian Feldman 			if (keg->uk_init(slab->us_data + (keg->uk_rsize * i),
1012b23f72e9SBrian Feldman 			    keg->uk_size, wait) != 0)
1013b23f72e9SBrian Feldman 				break;
1014b23f72e9SBrian Feldman 		if (i != keg->uk_ipers) {
1015fc03d22bSJeff Roberson 			keg_free_slab(keg, slab, i);
1016fc03d22bSJeff Roberson 			slab = NULL;
1017fc03d22bSJeff Roberson 			goto out;
1018b23f72e9SBrian Feldman 		}
1019b23f72e9SBrian Feldman 	}
1020fc03d22bSJeff Roberson out:
1021e20a199fSJeff Roberson 	KEG_LOCK(keg);
10225c0e403bSJeff Roberson 
1023fc03d22bSJeff Roberson 	if (slab != NULL) {
1024099a0e58SBosko Milekic 		if (keg->uk_flags & UMA_ZONE_HASH)
1025099a0e58SBosko Milekic 			UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
10268355f576SJeff Roberson 
1027099a0e58SBosko Milekic 		keg->uk_pages += keg->uk_ppera;
1028099a0e58SBosko Milekic 		keg->uk_free += keg->uk_ipers;
1029fc03d22bSJeff Roberson 	}
10308355f576SJeff Roberson 
10318355f576SJeff Roberson 	return (slab);
10328355f576SJeff Roberson }
10338355f576SJeff Roberson 
10348355f576SJeff Roberson /*
1035009b6fcbSJeff Roberson  * This function is intended to be used early on in place of page_alloc() so
1036009b6fcbSJeff Roberson  * that we may use the boot time page cache to satisfy allocations before
1037009b6fcbSJeff Roberson  * the VM is ready.
1038009b6fcbSJeff Roberson  */
1039009b6fcbSJeff Roberson static void *
104085dcf349SGleb Smirnoff startup_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait)
1041009b6fcbSJeff Roberson {
1042099a0e58SBosko Milekic 	uma_keg_t keg;
1043f353d338SAlan Cox 	uma_slab_t tmps;
1044e9a069d8SJohn Baldwin 	int pages, check_pages;
1045099a0e58SBosko Milekic 
1046e20a199fSJeff Roberson 	keg = zone_first_keg(zone);
1047e9a069d8SJohn Baldwin 	pages = howmany(bytes, PAGE_SIZE);
1048e9a069d8SJohn Baldwin 	check_pages = pages - 1;
1049e9a069d8SJohn Baldwin 	KASSERT(pages > 0, ("startup_alloc can't reserve 0 pages\n"));
1050099a0e58SBosko Milekic 
1051009b6fcbSJeff Roberson 	/*
1052009b6fcbSJeff Roberson 	 * Check our small startup cache to see if it has pages remaining.
1053009b6fcbSJeff Roberson 	 */
1054f353d338SAlan Cox 	mtx_lock(&uma_boot_pages_mtx);
1055e9a069d8SJohn Baldwin 
1056e9a069d8SJohn Baldwin 	/* First check if we have enough room. */
1057e9a069d8SJohn Baldwin 	tmps = LIST_FIRST(&uma_boot_pages);
1058e9a069d8SJohn Baldwin 	while (tmps != NULL && check_pages-- > 0)
1059e9a069d8SJohn Baldwin 		tmps = LIST_NEXT(tmps, us_link);
1060e9a069d8SJohn Baldwin 	if (tmps != NULL) {
1061e9a069d8SJohn Baldwin 		/*
1062e9a069d8SJohn Baldwin 		 * It's ok to lose tmps references.  The last one will
1063e9a069d8SJohn Baldwin 		 * have tmps->us_data pointing to the start address of
1064e9a069d8SJohn Baldwin 		 * "pages" contiguous pages of memory.
1065e9a069d8SJohn Baldwin 		 */
1066e9a069d8SJohn Baldwin 		while (pages-- > 0) {
1067e9a069d8SJohn Baldwin 			tmps = LIST_FIRST(&uma_boot_pages);
1068009b6fcbSJeff Roberson 			LIST_REMOVE(tmps, us_link);
1069e9a069d8SJohn Baldwin 		}
1070f353d338SAlan Cox 		mtx_unlock(&uma_boot_pages_mtx);
1071009b6fcbSJeff Roberson 		*pflag = tmps->us_flags;
1072009b6fcbSJeff Roberson 		return (tmps->us_data);
1073009b6fcbSJeff Roberson 	}
1074f353d338SAlan Cox 	mtx_unlock(&uma_boot_pages_mtx);
1075342f1793SAlan Cox 	if (booted < UMA_STARTUP2)
10763803b26bSDag-Erling Smørgrav 		panic("UMA: Increase vm.boot_pages");
1077009b6fcbSJeff Roberson 	/*
1078009b6fcbSJeff Roberson 	 * Now that we've booted reset these users to their real allocator.
1079009b6fcbSJeff Roberson 	 */
1080009b6fcbSJeff Roberson #ifdef UMA_MD_SMALL_ALLOC
1081e9a069d8SJohn Baldwin 	keg->uk_allocf = (keg->uk_ppera > 1) ? page_alloc : uma_small_alloc;
1082009b6fcbSJeff Roberson #else
1083099a0e58SBosko Milekic 	keg->uk_allocf = page_alloc;
1084009b6fcbSJeff Roberson #endif
1085099a0e58SBosko Milekic 	return keg->uk_allocf(zone, bytes, pflag, wait);
1086009b6fcbSJeff Roberson }
1087009b6fcbSJeff Roberson 
1088009b6fcbSJeff Roberson /*
10898355f576SJeff Roberson  * Allocates a number of pages from the system
10908355f576SJeff Roberson  *
10918355f576SJeff Roberson  * Arguments:
10928355f576SJeff Roberson  *	bytes  The number of bytes requested
10938355f576SJeff Roberson  *	wait  Shall we wait?
10948355f576SJeff Roberson  *
10958355f576SJeff Roberson  * Returns:
10968355f576SJeff Roberson  *	A pointer to the alloced memory or possibly
10978355f576SJeff Roberson  *	NULL if M_NOWAIT is set.
10988355f576SJeff Roberson  */
10998355f576SJeff Roberson static void *
110085dcf349SGleb Smirnoff page_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait)
11018355f576SJeff Roberson {
11028355f576SJeff Roberson 	void *p;	/* Returned page */
11038355f576SJeff Roberson 
11048355f576SJeff Roberson 	*pflag = UMA_SLAB_KMEM;
11055df87b21SJeff Roberson 	p = (void *) kmem_malloc(kmem_arena, bytes, wait);
11068355f576SJeff Roberson 
11078355f576SJeff Roberson 	return (p);
11088355f576SJeff Roberson }
11098355f576SJeff Roberson 
11108355f576SJeff Roberson /*
11118355f576SJeff Roberson  * Allocates a number of pages from within an object
11128355f576SJeff Roberson  *
11138355f576SJeff Roberson  * Arguments:
11148355f576SJeff Roberson  *	bytes  The number of bytes requested
11158355f576SJeff Roberson  *	wait   Shall we wait?
11168355f576SJeff Roberson  *
11178355f576SJeff Roberson  * Returns:
11188355f576SJeff Roberson  *	A pointer to the alloced memory or possibly
11198355f576SJeff Roberson  *	NULL if M_NOWAIT is set.
11208355f576SJeff Roberson  */
11218355f576SJeff Roberson static void *
112285dcf349SGleb Smirnoff noobj_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait)
11238355f576SJeff Roberson {
1124a4915c21SAttilio Rao 	TAILQ_HEAD(, vm_page) alloctail;
1125a4915c21SAttilio Rao 	u_long npages;
1126b245ac95SAlan Cox 	vm_offset_t retkva, zkva;
1127a4915c21SAttilio Rao 	vm_page_t p, p_next;
1128e20a199fSJeff Roberson 	uma_keg_t keg;
11298355f576SJeff Roberson 
1130a4915c21SAttilio Rao 	TAILQ_INIT(&alloctail);
1131e20a199fSJeff Roberson 	keg = zone_first_keg(zone);
1132a4915c21SAttilio Rao 
1133a4915c21SAttilio Rao 	npages = howmany(bytes, PAGE_SIZE);
1134a4915c21SAttilio Rao 	while (npages > 0) {
1135a4915c21SAttilio Rao 		p = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT |
1136a4915c21SAttilio Rao 		    VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
1137a4915c21SAttilio Rao 		if (p != NULL) {
1138a4915c21SAttilio Rao 			/*
1139a4915c21SAttilio Rao 			 * Since the page does not belong to an object, its
1140a4915c21SAttilio Rao 			 * listq is unused.
1141a4915c21SAttilio Rao 			 */
1142a4915c21SAttilio Rao 			TAILQ_INSERT_TAIL(&alloctail, p, listq);
1143a4915c21SAttilio Rao 			npages--;
1144a4915c21SAttilio Rao 			continue;
1145a4915c21SAttilio Rao 		}
1146a4915c21SAttilio Rao 		if (wait & M_WAITOK) {
1147a4915c21SAttilio Rao 			VM_WAIT;
1148a4915c21SAttilio Rao 			continue;
1149a4915c21SAttilio Rao 		}
11508355f576SJeff Roberson 
11518355f576SJeff Roberson 		/*
1152a4915c21SAttilio Rao 		 * Page allocation failed, free intermediate pages and
1153a4915c21SAttilio Rao 		 * exit.
11548355f576SJeff Roberson 		 */
1155a4915c21SAttilio Rao 		TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
1156b245ac95SAlan Cox 			vm_page_unwire(p, 0);
1157b245ac95SAlan Cox 			vm_page_free(p);
1158b245ac95SAlan Cox 		}
1159a4915c21SAttilio Rao 		return (NULL);
1160b245ac95SAlan Cox 	}
11618355f576SJeff Roberson 	*flags = UMA_SLAB_PRIV;
1162a4915c21SAttilio Rao 	zkva = keg->uk_kva +
1163a4915c21SAttilio Rao 	    atomic_fetchadd_long(&keg->uk_offset, round_page(bytes));
1164a4915c21SAttilio Rao 	retkva = zkva;
1165a4915c21SAttilio Rao 	TAILQ_FOREACH(p, &alloctail, listq) {
1166a4915c21SAttilio Rao 		pmap_qenter(zkva, &p, 1);
1167a4915c21SAttilio Rao 		zkva += PAGE_SIZE;
1168a4915c21SAttilio Rao 	}
11698355f576SJeff Roberson 
11708355f576SJeff Roberson 	return ((void *)retkva);
11718355f576SJeff Roberson }
11728355f576SJeff Roberson 
11738355f576SJeff Roberson /*
11748355f576SJeff Roberson  * Frees a number of pages to the system
11758355f576SJeff Roberson  *
11768355f576SJeff Roberson  * Arguments:
11778355f576SJeff Roberson  *	mem   A pointer to the memory to be freed
11788355f576SJeff Roberson  *	size  The size of the memory being freed
11798355f576SJeff Roberson  *	flags The original p->us_flags field
11808355f576SJeff Roberson  *
11818355f576SJeff Roberson  * Returns:
11828355f576SJeff Roberson  *	Nothing
11838355f576SJeff Roberson  */
11848355f576SJeff Roberson static void
118585dcf349SGleb Smirnoff page_free(void *mem, int size, uint8_t flags)
11868355f576SJeff Roberson {
11875df87b21SJeff Roberson 	struct vmem *vmem;
11883370c5bfSJeff Roberson 
11898355f576SJeff Roberson 	if (flags & UMA_SLAB_KMEM)
11905df87b21SJeff Roberson 		vmem = kmem_arena;
1191aea6e893SAlan Cox 	else if (flags & UMA_SLAB_KERNEL)
11925df87b21SJeff Roberson 		vmem = kernel_arena;
11938355f576SJeff Roberson 	else
1194aea6e893SAlan Cox 		panic("UMA: page_free used with invalid flags %d", flags);
11958355f576SJeff Roberson 
11965df87b21SJeff Roberson 	kmem_free(vmem, (vm_offset_t)mem, size);
11978355f576SJeff Roberson }
11988355f576SJeff Roberson 
11998355f576SJeff Roberson /*
12008355f576SJeff Roberson  * Zero fill initializer
12018355f576SJeff Roberson  *
12028355f576SJeff Roberson  * Arguments/Returns follow uma_init specifications
12038355f576SJeff Roberson  */
1204b23f72e9SBrian Feldman static int
1205b23f72e9SBrian Feldman zero_init(void *mem, int size, int flags)
12068355f576SJeff Roberson {
12078355f576SJeff Roberson 	bzero(mem, size);
1208b23f72e9SBrian Feldman 	return (0);
12098355f576SJeff Roberson }
12108355f576SJeff Roberson 
12118355f576SJeff Roberson /*
1212e20a199fSJeff Roberson  * Finish creating a small uma keg.  This calculates ipers, and the keg size.
12138355f576SJeff Roberson  *
12148355f576SJeff Roberson  * Arguments
1215e20a199fSJeff Roberson  *	keg  The zone we should initialize
12168355f576SJeff Roberson  *
12178355f576SJeff Roberson  * Returns
12188355f576SJeff Roberson  *	Nothing
12198355f576SJeff Roberson  */
12208355f576SJeff Roberson static void
1221e20a199fSJeff Roberson keg_small_init(uma_keg_t keg)
12228355f576SJeff Roberson {
1223244f4554SBosko Milekic 	u_int rsize;
1224244f4554SBosko Milekic 	u_int memused;
1225244f4554SBosko Milekic 	u_int wastedspace;
1226244f4554SBosko Milekic 	u_int shsize;
12278355f576SJeff Roberson 
1228ad97af7eSGleb Smirnoff 	if (keg->uk_flags & UMA_ZONE_PCPU) {
1229e28a647dSGleb Smirnoff 		u_int ncpus = mp_ncpus ? mp_ncpus : MAXCPU;
1230e28a647dSGleb Smirnoff 
1231ad97af7eSGleb Smirnoff 		keg->uk_slabsize = sizeof(struct pcpu);
1232e28a647dSGleb Smirnoff 		keg->uk_ppera = howmany(ncpus * sizeof(struct pcpu),
1233ad97af7eSGleb Smirnoff 		    PAGE_SIZE);
1234ad97af7eSGleb Smirnoff 	} else {
1235ad97af7eSGleb Smirnoff 		keg->uk_slabsize = UMA_SLAB_SIZE;
1236ad97af7eSGleb Smirnoff 		keg->uk_ppera = 1;
1237ad97af7eSGleb Smirnoff 	}
1238ad97af7eSGleb Smirnoff 
1239ef72505eSJeff Roberson 	/*
1240ef72505eSJeff Roberson 	 * Calculate the size of each allocation (rsize) according to
1241ef72505eSJeff Roberson 	 * alignment.  If the requested size is smaller than we have
1242ef72505eSJeff Roberson 	 * allocation bits for we round it up.
1243ef72505eSJeff Roberson 	 */
1244099a0e58SBosko Milekic 	rsize = keg->uk_size;
1245ef72505eSJeff Roberson 	if (rsize < keg->uk_slabsize / SLAB_SETSIZE)
1246ef72505eSJeff Roberson 		rsize = keg->uk_slabsize / SLAB_SETSIZE;
1247099a0e58SBosko Milekic 	if (rsize & keg->uk_align)
1248099a0e58SBosko Milekic 		rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
1249099a0e58SBosko Milekic 	keg->uk_rsize = rsize;
1250ad97af7eSGleb Smirnoff 
1251ad97af7eSGleb Smirnoff 	KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
1252ad97af7eSGleb Smirnoff 	    keg->uk_rsize < sizeof(struct pcpu),
1253ad97af7eSGleb Smirnoff 	    ("%s: size %u too large", __func__, keg->uk_rsize));
12548355f576SJeff Roberson 
1255ef72505eSJeff Roberson 	if (keg->uk_flags & UMA_ZONE_REFCNT)
1256ef72505eSJeff Roberson 		rsize += sizeof(uint32_t);
1257ef72505eSJeff Roberson 
1258ef72505eSJeff Roberson 	if (keg->uk_flags & UMA_ZONE_OFFPAGE)
12592864dbbfSGleb Smirnoff 		shsize = 0;
1260ef72505eSJeff Roberson 	else
1261244f4554SBosko Milekic 		shsize = sizeof(struct uma_slab);
12628355f576SJeff Roberson 
1263ad97af7eSGleb Smirnoff 	keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize;
1264ef72505eSJeff Roberson 	KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
1265ad97af7eSGleb Smirnoff 	    ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1266ad97af7eSGleb Smirnoff 
1267244f4554SBosko Milekic 	memused = keg->uk_ipers * rsize + shsize;
1268ad97af7eSGleb Smirnoff 	wastedspace = keg->uk_slabsize - memused;
1269244f4554SBosko Milekic 
127020e8e865SBosko Milekic 	/*
1271244f4554SBosko Milekic 	 * We can't do OFFPAGE if we're internal or if we've been
127220e8e865SBosko Milekic 	 * asked to not go to the VM for buckets.  If we do this we
12736fd34d6fSJeff Roberson 	 * may end up going to the VM  for slabs which we do not
12746fd34d6fSJeff Roberson 	 * want to do if we're UMA_ZFLAG_CACHEONLY as a result
12756fd34d6fSJeff Roberson 	 * of UMA_ZONE_VM, which clearly forbids it.
127620e8e865SBosko Milekic 	 */
1277099a0e58SBosko Milekic 	if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
1278099a0e58SBosko Milekic 	    (keg->uk_flags & UMA_ZFLAG_CACHEONLY))
12798355f576SJeff Roberson 		return;
1280244f4554SBosko Milekic 
1281ef72505eSJeff Roberson 	/*
1282ef72505eSJeff Roberson 	 * See if using an OFFPAGE slab will limit our waste.  Only do
1283ef72505eSJeff Roberson 	 * this if it permits more items per-slab.
1284ef72505eSJeff Roberson 	 *
1285ef72505eSJeff Roberson 	 * XXX We could try growing slabsize to limit max waste as well.
1286ef72505eSJeff Roberson 	 * Historically this was not done because the VM could not
1287ef72505eSJeff Roberson 	 * efficiently handle contiguous allocations.
1288ef72505eSJeff Roberson 	 */
1289ad97af7eSGleb Smirnoff 	if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) &&
1290ad97af7eSGleb Smirnoff 	    (keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) {
1291ad97af7eSGleb Smirnoff 		keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize;
1292ef72505eSJeff Roberson 		KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
1293ad97af7eSGleb Smirnoff 		    ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1294244f4554SBosko Milekic #ifdef UMA_DEBUG
1295244f4554SBosko Milekic 		printf("UMA decided we need offpage slab headers for "
1296e20a199fSJeff Roberson 		    "keg: %s, calculated wastedspace = %d, "
1297244f4554SBosko Milekic 		    "maximum wasted space allowed = %d, "
1298244f4554SBosko Milekic 		    "calculated ipers = %d, "
1299e20a199fSJeff Roberson 		    "new wasted space = %d\n", keg->uk_name, wastedspace,
1300ad97af7eSGleb Smirnoff 		    keg->uk_slabsize / UMA_MAX_WASTE, keg->uk_ipers,
1301ad97af7eSGleb Smirnoff 		    keg->uk_slabsize - keg->uk_ipers * keg->uk_rsize);
1302244f4554SBosko Milekic #endif
1303099a0e58SBosko Milekic 		keg->uk_flags |= UMA_ZONE_OFFPAGE;
13048355f576SJeff Roberson 	}
1305ad97af7eSGleb Smirnoff 
1306ad97af7eSGleb Smirnoff 	if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
1307ad97af7eSGleb Smirnoff 	    (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1308ad97af7eSGleb Smirnoff 		keg->uk_flags |= UMA_ZONE_HASH;
13098355f576SJeff Roberson }
13108355f576SJeff Roberson 
13118355f576SJeff Roberson /*
1312e20a199fSJeff Roberson  * Finish creating a large (> UMA_SLAB_SIZE) uma kegs.  Just give in and do
13138355f576SJeff Roberson  * OFFPAGE for now.  When I can allow for more dynamic slab sizes this will be
13148355f576SJeff Roberson  * more complicated.
13158355f576SJeff Roberson  *
13168355f576SJeff Roberson  * Arguments
1317e20a199fSJeff Roberson  *	keg  The keg we should initialize
13188355f576SJeff Roberson  *
13198355f576SJeff Roberson  * Returns
13208355f576SJeff Roberson  *	Nothing
13218355f576SJeff Roberson  */
13228355f576SJeff Roberson static void
1323e20a199fSJeff Roberson keg_large_init(uma_keg_t keg)
13248355f576SJeff Roberson {
1325cec48e00SAlexander Motin 	u_int shsize;
13268355f576SJeff Roberson 
1327e20a199fSJeff Roberson 	KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
1328099a0e58SBosko Milekic 	KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
1329e20a199fSJeff Roberson 	    ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg"));
1330ad97af7eSGleb Smirnoff 	KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1331ad97af7eSGleb Smirnoff 	    ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__));
133220e8e865SBosko Milekic 
1333ad97af7eSGleb Smirnoff 	keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE);
1334ad97af7eSGleb Smirnoff 	keg->uk_slabsize = keg->uk_ppera * PAGE_SIZE;
1335099a0e58SBosko Milekic 	keg->uk_ipers = 1;
1336e9a069d8SJohn Baldwin 	keg->uk_rsize = keg->uk_size;
1337e9a069d8SJohn Baldwin 
1338e9a069d8SJohn Baldwin 	/* We can't do OFFPAGE if we're internal, bail out here. */
1339e9a069d8SJohn Baldwin 	if (keg->uk_flags & UMA_ZFLAG_INTERNAL)
1340e9a069d8SJohn Baldwin 		return;
13418355f576SJeff Roberson 
1342cec48e00SAlexander Motin 	/* Check whether we have enough space to not do OFFPAGE. */
1343cec48e00SAlexander Motin 	if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0) {
1344cec48e00SAlexander Motin 		shsize = sizeof(struct uma_slab);
1345cec48e00SAlexander Motin 		if (keg->uk_flags & UMA_ZONE_REFCNT)
1346cec48e00SAlexander Motin 			shsize += keg->uk_ipers * sizeof(uint32_t);
1347cec48e00SAlexander Motin 		if (shsize & UMA_ALIGN_PTR)
1348cec48e00SAlexander Motin 			shsize = (shsize & ~UMA_ALIGN_PTR) +
1349cec48e00SAlexander Motin 			    (UMA_ALIGN_PTR + 1);
1350cec48e00SAlexander Motin 
1351cec48e00SAlexander Motin 		if ((PAGE_SIZE * keg->uk_ppera) - keg->uk_rsize < shsize)
1352099a0e58SBosko Milekic 			keg->uk_flags |= UMA_ZONE_OFFPAGE;
1353cec48e00SAlexander Motin 	}
1354cec48e00SAlexander Motin 
1355cec48e00SAlexander Motin 	if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
1356cec48e00SAlexander Motin 	    (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1357099a0e58SBosko Milekic 		keg->uk_flags |= UMA_ZONE_HASH;
13588355f576SJeff Roberson }
13598355f576SJeff Roberson 
1360e20a199fSJeff Roberson static void
1361e20a199fSJeff Roberson keg_cachespread_init(uma_keg_t keg)
1362e20a199fSJeff Roberson {
1363e20a199fSJeff Roberson 	int alignsize;
1364e20a199fSJeff Roberson 	int trailer;
1365e20a199fSJeff Roberson 	int pages;
1366e20a199fSJeff Roberson 	int rsize;
1367e20a199fSJeff Roberson 
1368ad97af7eSGleb Smirnoff 	KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1369ad97af7eSGleb Smirnoff 	    ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__));
1370ad97af7eSGleb Smirnoff 
1371e20a199fSJeff Roberson 	alignsize = keg->uk_align + 1;
1372e20a199fSJeff Roberson 	rsize = keg->uk_size;
1373e20a199fSJeff Roberson 	/*
1374e20a199fSJeff Roberson 	 * We want one item to start on every align boundary in a page.  To
1375e20a199fSJeff Roberson 	 * do this we will span pages.  We will also extend the item by the
1376e20a199fSJeff Roberson 	 * size of align if it is an even multiple of align.  Otherwise, it
1377e20a199fSJeff Roberson 	 * would fall on the same boundary every time.
1378e20a199fSJeff Roberson 	 */
1379e20a199fSJeff Roberson 	if (rsize & keg->uk_align)
1380e20a199fSJeff Roberson 		rsize = (rsize & ~keg->uk_align) + alignsize;
1381e20a199fSJeff Roberson 	if ((rsize & alignsize) == 0)
1382e20a199fSJeff Roberson 		rsize += alignsize;
1383e20a199fSJeff Roberson 	trailer = rsize - keg->uk_size;
1384e20a199fSJeff Roberson 	pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE;
1385e20a199fSJeff Roberson 	pages = MIN(pages, (128 * 1024) / PAGE_SIZE);
1386e20a199fSJeff Roberson 	keg->uk_rsize = rsize;
1387e20a199fSJeff Roberson 	keg->uk_ppera = pages;
1388ad97af7eSGleb Smirnoff 	keg->uk_slabsize = UMA_SLAB_SIZE;
1389e20a199fSJeff Roberson 	keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize;
1390e20a199fSJeff Roberson 	keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB;
13912367b4ddSDimitry Andric 	KASSERT(keg->uk_ipers <= SLAB_SETSIZE,
139242321809SGleb Smirnoff 	    ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__,
1393e20a199fSJeff Roberson 	    keg->uk_ipers));
1394e20a199fSJeff Roberson }
1395e20a199fSJeff Roberson 
13968355f576SJeff Roberson /*
1397099a0e58SBosko Milekic  * Keg header ctor.  This initializes all fields, locks, etc.  And inserts
1398099a0e58SBosko Milekic  * the keg onto the global keg list.
13998355f576SJeff Roberson  *
14008355f576SJeff Roberson  * Arguments/Returns follow uma_ctor specifications
1401099a0e58SBosko Milekic  *	udata  Actually uma_kctor_args
1402099a0e58SBosko Milekic  */
1403b23f72e9SBrian Feldman static int
1404b23f72e9SBrian Feldman keg_ctor(void *mem, int size, void *udata, int flags)
1405099a0e58SBosko Milekic {
1406099a0e58SBosko Milekic 	struct uma_kctor_args *arg = udata;
1407099a0e58SBosko Milekic 	uma_keg_t keg = mem;
1408099a0e58SBosko Milekic 	uma_zone_t zone;
1409099a0e58SBosko Milekic 
1410099a0e58SBosko Milekic 	bzero(keg, size);
1411099a0e58SBosko Milekic 	keg->uk_size = arg->size;
1412099a0e58SBosko Milekic 	keg->uk_init = arg->uminit;
1413099a0e58SBosko Milekic 	keg->uk_fini = arg->fini;
1414099a0e58SBosko Milekic 	keg->uk_align = arg->align;
1415099a0e58SBosko Milekic 	keg->uk_free = 0;
14166fd34d6fSJeff Roberson 	keg->uk_reserve = 0;
1417099a0e58SBosko Milekic 	keg->uk_pages = 0;
1418099a0e58SBosko Milekic 	keg->uk_flags = arg->flags;
1419099a0e58SBosko Milekic 	keg->uk_allocf = page_alloc;
1420099a0e58SBosko Milekic 	keg->uk_freef = page_free;
1421099a0e58SBosko Milekic 	keg->uk_slabzone = NULL;
1422099a0e58SBosko Milekic 
1423099a0e58SBosko Milekic 	/*
1424099a0e58SBosko Milekic 	 * The master zone is passed to us at keg-creation time.
1425099a0e58SBosko Milekic 	 */
1426099a0e58SBosko Milekic 	zone = arg->zone;
1427e20a199fSJeff Roberson 	keg->uk_name = zone->uz_name;
1428099a0e58SBosko Milekic 
1429099a0e58SBosko Milekic 	if (arg->flags & UMA_ZONE_VM)
1430099a0e58SBosko Milekic 		keg->uk_flags |= UMA_ZFLAG_CACHEONLY;
1431099a0e58SBosko Milekic 
1432099a0e58SBosko Milekic 	if (arg->flags & UMA_ZONE_ZINIT)
1433099a0e58SBosko Milekic 		keg->uk_init = zero_init;
1434099a0e58SBosko Milekic 
1435e20a199fSJeff Roberson 	if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC)
1436e20a199fSJeff Roberson 		keg->uk_flags |= UMA_ZONE_VTOSLAB;
1437e20a199fSJeff Roberson 
1438ad97af7eSGleb Smirnoff 	if (arg->flags & UMA_ZONE_PCPU)
1439ad97af7eSGleb Smirnoff #ifdef SMP
1440ad97af7eSGleb Smirnoff 		keg->uk_flags |= UMA_ZONE_OFFPAGE;
1441ad97af7eSGleb Smirnoff #else
1442ad97af7eSGleb Smirnoff 		keg->uk_flags &= ~UMA_ZONE_PCPU;
1443ad97af7eSGleb Smirnoff #endif
1444ad97af7eSGleb Smirnoff 
1445ef72505eSJeff Roberson 	if (keg->uk_flags & UMA_ZONE_CACHESPREAD) {
1446e20a199fSJeff Roberson 		keg_cachespread_init(keg);
1447ef72505eSJeff Roberson 	} else if (keg->uk_flags & UMA_ZONE_REFCNT) {
1448ef72505eSJeff Roberson 		if (keg->uk_size >
1449ef72505eSJeff Roberson 		    (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) -
1450ef72505eSJeff Roberson 		    sizeof(uint32_t)))
1451e20a199fSJeff Roberson 			keg_large_init(keg);
1452099a0e58SBosko Milekic 		else
1453e20a199fSJeff Roberson 			keg_small_init(keg);
1454244f4554SBosko Milekic 	} else {
1455ef72505eSJeff Roberson 		if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab)))
1456e20a199fSJeff Roberson 			keg_large_init(keg);
1457244f4554SBosko Milekic 		else
1458e20a199fSJeff Roberson 			keg_small_init(keg);
1459244f4554SBosko Milekic 	}
1460099a0e58SBosko Milekic 
1461244f4554SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
1462ef72505eSJeff Roberson 		if (keg->uk_flags & UMA_ZONE_REFCNT) {
1463ef72505eSJeff Roberson 			if (keg->uk_ipers > uma_max_ipers_ref)
1464ef72505eSJeff Roberson 				panic("Too many ref items per zone: %d > %d\n",
1465ef72505eSJeff Roberson 				    keg->uk_ipers, uma_max_ipers_ref);
1466099a0e58SBosko Milekic 			keg->uk_slabzone = slabrefzone;
1467ef72505eSJeff Roberson 		} else
1468099a0e58SBosko Milekic 			keg->uk_slabzone = slabzone;
1469244f4554SBosko Milekic 	}
1470099a0e58SBosko Milekic 
1471099a0e58SBosko Milekic 	/*
1472099a0e58SBosko Milekic 	 * If we haven't booted yet we need allocations to go through the
1473099a0e58SBosko Milekic 	 * startup cache until the vm is ready.
1474099a0e58SBosko Milekic 	 */
1475099a0e58SBosko Milekic 	if (keg->uk_ppera == 1) {
1476099a0e58SBosko Milekic #ifdef UMA_MD_SMALL_ALLOC
1477099a0e58SBosko Milekic 		keg->uk_allocf = uma_small_alloc;
1478099a0e58SBosko Milekic 		keg->uk_freef = uma_small_free;
14798cd02d00SAlan Cox 
1480342f1793SAlan Cox 		if (booted < UMA_STARTUP)
1481099a0e58SBosko Milekic 			keg->uk_allocf = startup_alloc;
14828cd02d00SAlan Cox #else
14838cd02d00SAlan Cox 		if (booted < UMA_STARTUP2)
14848cd02d00SAlan Cox 			keg->uk_allocf = startup_alloc;
14858cd02d00SAlan Cox #endif
1486342f1793SAlan Cox 	} else if (booted < UMA_STARTUP2 &&
1487342f1793SAlan Cox 	    (keg->uk_flags & UMA_ZFLAG_INTERNAL))
1488e9a069d8SJohn Baldwin 		keg->uk_allocf = startup_alloc;
1489099a0e58SBosko Milekic 
1490099a0e58SBosko Milekic 	/*
1491af526374SJeff Roberson 	 * Initialize keg's lock
1492099a0e58SBosko Milekic 	 */
1493af526374SJeff Roberson 	KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS));
1494099a0e58SBosko Milekic 
1495099a0e58SBosko Milekic 	/*
1496099a0e58SBosko Milekic 	 * If we're putting the slab header in the actual page we need to
1497099a0e58SBosko Milekic 	 * figure out where in each page it goes.  This calculates a right
1498099a0e58SBosko Milekic 	 * justified offset into the memory on an ALIGN_PTR boundary.
1499099a0e58SBosko Milekic 	 */
1500099a0e58SBosko Milekic 	if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) {
1501244f4554SBosko Milekic 		u_int totsize;
1502099a0e58SBosko Milekic 
1503099a0e58SBosko Milekic 		/* Size of the slab struct and free list */
1504ef72505eSJeff Roberson 		totsize = sizeof(struct uma_slab);
1505ef72505eSJeff Roberson 
1506ef72505eSJeff Roberson 		/* Size of the reference counts. */
1507244f4554SBosko Milekic 		if (keg->uk_flags & UMA_ZONE_REFCNT)
1508ef72505eSJeff Roberson 			totsize += keg->uk_ipers * sizeof(uint32_t);
1509244f4554SBosko Milekic 
1510099a0e58SBosko Milekic 		if (totsize & UMA_ALIGN_PTR)
1511099a0e58SBosko Milekic 			totsize = (totsize & ~UMA_ALIGN_PTR) +
1512099a0e58SBosko Milekic 			    (UMA_ALIGN_PTR + 1);
1513ad97af7eSGleb Smirnoff 		keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize;
1514244f4554SBosko Milekic 
1515244f4554SBosko Milekic 		/*
1516244f4554SBosko Milekic 		 * The only way the following is possible is if with our
1517244f4554SBosko Milekic 		 * UMA_ALIGN_PTR adjustments we are now bigger than
1518244f4554SBosko Milekic 		 * UMA_SLAB_SIZE.  I haven't checked whether this is
1519244f4554SBosko Milekic 		 * mathematically possible for all cases, so we make
1520244f4554SBosko Milekic 		 * sure here anyway.
1521244f4554SBosko Milekic 		 */
1522ef72505eSJeff Roberson 		totsize = keg->uk_pgoff + sizeof(struct uma_slab);
1523ef72505eSJeff Roberson 		if (keg->uk_flags & UMA_ZONE_REFCNT)
1524ef72505eSJeff Roberson 			totsize += keg->uk_ipers * sizeof(uint32_t);
1525ad97af7eSGleb Smirnoff 		if (totsize > PAGE_SIZE * keg->uk_ppera) {
1526099a0e58SBosko Milekic 			printf("zone %s ipers %d rsize %d size %d\n",
1527099a0e58SBosko Milekic 			    zone->uz_name, keg->uk_ipers, keg->uk_rsize,
1528099a0e58SBosko Milekic 			    keg->uk_size);
1529aea6e893SAlan Cox 			panic("UMA slab won't fit.");
1530099a0e58SBosko Milekic 		}
1531099a0e58SBosko Milekic 	}
1532099a0e58SBosko Milekic 
1533099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_HASH)
1534099a0e58SBosko Milekic 		hash_alloc(&keg->uk_hash);
1535099a0e58SBosko Milekic 
1536099a0e58SBosko Milekic #ifdef UMA_DEBUG
15370b80c1e4SEitan Adler 	printf("UMA: %s(%p) size %d(%d) flags %#x ipers %d ppera %d out %d free %d\n",
1538e20a199fSJeff Roberson 	    zone->uz_name, zone, keg->uk_size, keg->uk_rsize, keg->uk_flags,
1539e20a199fSJeff Roberson 	    keg->uk_ipers, keg->uk_ppera,
1540e20a199fSJeff Roberson 	    (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free);
1541099a0e58SBosko Milekic #endif
1542099a0e58SBosko Milekic 
1543099a0e58SBosko Milekic 	LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
1544099a0e58SBosko Milekic 
1545099a0e58SBosko Milekic 	mtx_lock(&uma_mtx);
1546099a0e58SBosko Milekic 	LIST_INSERT_HEAD(&uma_kegs, keg, uk_link);
1547099a0e58SBosko Milekic 	mtx_unlock(&uma_mtx);
1548b23f72e9SBrian Feldman 	return (0);
1549099a0e58SBosko Milekic }
1550099a0e58SBosko Milekic 
1551099a0e58SBosko Milekic /*
1552099a0e58SBosko Milekic  * Zone header ctor.  This initializes all fields, locks, etc.
1553099a0e58SBosko Milekic  *
1554099a0e58SBosko Milekic  * Arguments/Returns follow uma_ctor specifications
1555099a0e58SBosko Milekic  *	udata  Actually uma_zctor_args
15568355f576SJeff Roberson  */
1557b23f72e9SBrian Feldman static int
1558b23f72e9SBrian Feldman zone_ctor(void *mem, int size, void *udata, int flags)
15598355f576SJeff Roberson {
15608355f576SJeff Roberson 	struct uma_zctor_args *arg = udata;
15618355f576SJeff Roberson 	uma_zone_t zone = mem;
1562099a0e58SBosko Milekic 	uma_zone_t z;
1563099a0e58SBosko Milekic 	uma_keg_t keg;
15648355f576SJeff Roberson 
15658355f576SJeff Roberson 	bzero(zone, size);
15668355f576SJeff Roberson 	zone->uz_name = arg->name;
15678355f576SJeff Roberson 	zone->uz_ctor = arg->ctor;
15688355f576SJeff Roberson 	zone->uz_dtor = arg->dtor;
1569e20a199fSJeff Roberson 	zone->uz_slab = zone_fetch_slab;
1570099a0e58SBosko Milekic 	zone->uz_init = NULL;
1571099a0e58SBosko Milekic 	zone->uz_fini = NULL;
1572099a0e58SBosko Milekic 	zone->uz_allocs = 0;
1573773df9abSRobert Watson 	zone->uz_frees = 0;
15742019094aSRobert Watson 	zone->uz_fails = 0;
1575bf965959SSean Bruno 	zone->uz_sleeps = 0;
1576fc03d22bSJeff Roberson 	zone->uz_count = 0;
1577ace66b56SAlexander Motin 	zone->uz_count_min = 0;
1578e20a199fSJeff Roberson 	zone->uz_flags = 0;
15792f891cd5SPawel Jakub Dawidek 	zone->uz_warning = NULL;
15802f891cd5SPawel Jakub Dawidek 	timevalclear(&zone->uz_ratecheck);
1581e20a199fSJeff Roberson 	keg = arg->keg;
1582099a0e58SBosko Milekic 
1583af526374SJeff Roberson 	ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));
1584af526374SJeff Roberson 
15850095a784SJeff Roberson 	/*
15860095a784SJeff Roberson 	 * This is a pure cache zone, no kegs.
15870095a784SJeff Roberson 	 */
15880095a784SJeff Roberson 	if (arg->import) {
15896fd34d6fSJeff Roberson 		if (arg->flags & UMA_ZONE_VM)
15906fd34d6fSJeff Roberson 			arg->flags |= UMA_ZFLAG_CACHEONLY;
15916fd34d6fSJeff Roberson 		zone->uz_flags = arg->flags;
1592af526374SJeff Roberson 		zone->uz_size = arg->size;
15930095a784SJeff Roberson 		zone->uz_import = arg->import;
15940095a784SJeff Roberson 		zone->uz_release = arg->release;
15950095a784SJeff Roberson 		zone->uz_arg = arg->arg;
1596af526374SJeff Roberson 		zone->uz_lockptr = &zone->uz_lock;
159703175483SAlexander Motin 		mtx_lock(&uma_mtx);
159803175483SAlexander Motin 		LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link);
159903175483SAlexander Motin 		mtx_unlock(&uma_mtx);
1600af526374SJeff Roberson 		goto out;
16010095a784SJeff Roberson 	}
16020095a784SJeff Roberson 
16030095a784SJeff Roberson 	/*
16040095a784SJeff Roberson 	 * Use the regular zone/keg/slab allocator.
16050095a784SJeff Roberson 	 */
16060095a784SJeff Roberson 	zone->uz_import = (uma_import)zone_import;
16070095a784SJeff Roberson 	zone->uz_release = (uma_release)zone_release;
16080095a784SJeff Roberson 	zone->uz_arg = zone;
16090095a784SJeff Roberson 
1610099a0e58SBosko Milekic 	if (arg->flags & UMA_ZONE_SECONDARY) {
1611099a0e58SBosko Milekic 		KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
16128355f576SJeff Roberson 		zone->uz_init = arg->uminit;
1613e221e841SJeff Roberson 		zone->uz_fini = arg->fini;
1614af526374SJeff Roberson 		zone->uz_lockptr = &keg->uk_lock;
1615e20a199fSJeff Roberson 		zone->uz_flags |= UMA_ZONE_SECONDARY;
16168355f576SJeff Roberson 		mtx_lock(&uma_mtx);
1617099a0e58SBosko Milekic 		ZONE_LOCK(zone);
1618099a0e58SBosko Milekic 		LIST_FOREACH(z, &keg->uk_zones, uz_link) {
1619099a0e58SBosko Milekic 			if (LIST_NEXT(z, uz_link) == NULL) {
1620099a0e58SBosko Milekic 				LIST_INSERT_AFTER(z, zone, uz_link);
1621099a0e58SBosko Milekic 				break;
1622099a0e58SBosko Milekic 			}
1623099a0e58SBosko Milekic 		}
1624099a0e58SBosko Milekic 		ZONE_UNLOCK(zone);
16258355f576SJeff Roberson 		mtx_unlock(&uma_mtx);
1626e20a199fSJeff Roberson 	} else if (keg == NULL) {
1627e20a199fSJeff Roberson 		if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini,
1628e20a199fSJeff Roberson 		    arg->align, arg->flags)) == NULL)
1629b23f72e9SBrian Feldman 			return (ENOMEM);
1630099a0e58SBosko Milekic 	} else {
1631099a0e58SBosko Milekic 		struct uma_kctor_args karg;
1632b23f72e9SBrian Feldman 		int error;
1633099a0e58SBosko Milekic 
1634099a0e58SBosko Milekic 		/* We should only be here from uma_startup() */
1635099a0e58SBosko Milekic 		karg.size = arg->size;
1636099a0e58SBosko Milekic 		karg.uminit = arg->uminit;
1637099a0e58SBosko Milekic 		karg.fini = arg->fini;
1638099a0e58SBosko Milekic 		karg.align = arg->align;
1639099a0e58SBosko Milekic 		karg.flags = arg->flags;
1640099a0e58SBosko Milekic 		karg.zone = zone;
1641b23f72e9SBrian Feldman 		error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
1642b23f72e9SBrian Feldman 		    flags);
1643b23f72e9SBrian Feldman 		if (error)
1644b23f72e9SBrian Feldman 			return (error);
1645099a0e58SBosko Milekic 	}
16460095a784SJeff Roberson 
1647e20a199fSJeff Roberson 	/*
1648e20a199fSJeff Roberson 	 * Link in the first keg.
1649e20a199fSJeff Roberson 	 */
1650e20a199fSJeff Roberson 	zone->uz_klink.kl_keg = keg;
1651e20a199fSJeff Roberson 	LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
1652af526374SJeff Roberson 	zone->uz_lockptr = &keg->uk_lock;
1653e20a199fSJeff Roberson 	zone->uz_size = keg->uk_size;
1654e20a199fSJeff Roberson 	zone->uz_flags |= (keg->uk_flags &
1655e20a199fSJeff Roberson 	    (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
16568355f576SJeff Roberson 
16578355f576SJeff Roberson 	/*
16588355f576SJeff Roberson 	 * Some internal zones don't have room allocated for the per cpu
16598355f576SJeff Roberson 	 * caches.  If we're internal, bail out here.
16608355f576SJeff Roberson 	 */
1661099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
1662e20a199fSJeff Roberson 		KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
1663099a0e58SBosko Milekic 		    ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
1664b23f72e9SBrian Feldman 		return (0);
1665099a0e58SBosko Milekic 	}
16668355f576SJeff Roberson 
1667af526374SJeff Roberson out:
1668af526374SJeff Roberson 	if ((arg->flags & UMA_ZONE_MAXBUCKET) == 0)
1669af526374SJeff Roberson 		zone->uz_count = bucket_select(zone->uz_size);
16708355f576SJeff Roberson 	else
1671cae33c14SJeff Roberson 		zone->uz_count = BUCKET_MAX;
1672ace66b56SAlexander Motin 	zone->uz_count_min = zone->uz_count;
1673fc03d22bSJeff Roberson 
1674b23f72e9SBrian Feldman 	return (0);
16758355f576SJeff Roberson }
16768355f576SJeff Roberson 
16778355f576SJeff Roberson /*
1678099a0e58SBosko Milekic  * Keg header dtor.  This frees all data, destroys locks, frees the hash
1679099a0e58SBosko Milekic  * table and removes the keg from the global list.
16809c2cd7e5SJeff Roberson  *
16819c2cd7e5SJeff Roberson  * Arguments/Returns follow uma_dtor specifications
16829c2cd7e5SJeff Roberson  *	udata  unused
16839c2cd7e5SJeff Roberson  */
1684099a0e58SBosko Milekic static void
1685099a0e58SBosko Milekic keg_dtor(void *arg, int size, void *udata)
1686099a0e58SBosko Milekic {
1687099a0e58SBosko Milekic 	uma_keg_t keg;
16889c2cd7e5SJeff Roberson 
1689099a0e58SBosko Milekic 	keg = (uma_keg_t)arg;
1690e20a199fSJeff Roberson 	KEG_LOCK(keg);
1691099a0e58SBosko Milekic 	if (keg->uk_free != 0) {
1692a3845534SCraig Rodrigues 		printf("Freed UMA keg (%s) was not empty (%d items). "
1693099a0e58SBosko Milekic 		    " Lost %d pages of memory.\n",
1694a3845534SCraig Rodrigues 		    keg->uk_name ? keg->uk_name : "",
1695099a0e58SBosko Milekic 		    keg->uk_free, keg->uk_pages);
1696099a0e58SBosko Milekic 	}
1697e20a199fSJeff Roberson 	KEG_UNLOCK(keg);
1698099a0e58SBosko Milekic 
1699099a0e58SBosko Milekic 	hash_free(&keg->uk_hash);
1700099a0e58SBosko Milekic 
1701e20a199fSJeff Roberson 	KEG_LOCK_FINI(keg);
1702099a0e58SBosko Milekic }
1703099a0e58SBosko Milekic 
1704099a0e58SBosko Milekic /*
1705099a0e58SBosko Milekic  * Zone header dtor.
1706099a0e58SBosko Milekic  *
1707099a0e58SBosko Milekic  * Arguments/Returns follow uma_dtor specifications
1708099a0e58SBosko Milekic  *	udata  unused
1709099a0e58SBosko Milekic  */
17109c2cd7e5SJeff Roberson static void
17119c2cd7e5SJeff Roberson zone_dtor(void *arg, int size, void *udata)
17129c2cd7e5SJeff Roberson {
1713e20a199fSJeff Roberson 	uma_klink_t klink;
17149c2cd7e5SJeff Roberson 	uma_zone_t zone;
1715099a0e58SBosko Milekic 	uma_keg_t keg;
17169c2cd7e5SJeff Roberson 
17179c2cd7e5SJeff Roberson 	zone = (uma_zone_t)arg;
1718e20a199fSJeff Roberson 	keg = zone_first_keg(zone);
17199643769aSJeff Roberson 
1720e20a199fSJeff Roberson 	if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
17219643769aSJeff Roberson 		cache_drain(zone);
1722099a0e58SBosko Milekic 
172317b9cc49SJeff Roberson 	mtx_lock(&uma_mtx);
1724099a0e58SBosko Milekic 	LIST_REMOVE(zone, uz_link);
1725e20a199fSJeff Roberson 	mtx_unlock(&uma_mtx);
1726099a0e58SBosko Milekic 	/*
1727099a0e58SBosko Milekic 	 * XXX there are some races here where
1728099a0e58SBosko Milekic 	 * the zone can be drained but zone lock
1729099a0e58SBosko Milekic 	 * released and then refilled before we
1730099a0e58SBosko Milekic 	 * remove it... we dont care for now
1731099a0e58SBosko Milekic 	 */
1732e20a199fSJeff Roberson 	zone_drain_wait(zone, M_WAITOK);
1733e20a199fSJeff Roberson 	/*
1734e20a199fSJeff Roberson 	 * Unlink all of our kegs.
1735e20a199fSJeff Roberson 	 */
1736e20a199fSJeff Roberson 	while ((klink = LIST_FIRST(&zone->uz_kegs)) != NULL) {
1737e20a199fSJeff Roberson 		klink->kl_keg = NULL;
1738e20a199fSJeff Roberson 		LIST_REMOVE(klink, kl_link);
1739e20a199fSJeff Roberson 		if (klink == &zone->uz_klink)
1740e20a199fSJeff Roberson 			continue;
1741e20a199fSJeff Roberson 		free(klink, M_TEMP);
1742e20a199fSJeff Roberson 	}
1743e20a199fSJeff Roberson 	/*
1744e20a199fSJeff Roberson 	 * We only destroy kegs from non secondary zones.
1745e20a199fSJeff Roberson 	 */
17460095a784SJeff Roberson 	if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0)  {
1747e20a199fSJeff Roberson 		mtx_lock(&uma_mtx);
1748099a0e58SBosko Milekic 		LIST_REMOVE(keg, uk_link);
1749099a0e58SBosko Milekic 		mtx_unlock(&uma_mtx);
17500095a784SJeff Roberson 		zone_free_item(kegs, keg, NULL, SKIP_NONE);
17519c2cd7e5SJeff Roberson 	}
1752af526374SJeff Roberson 	ZONE_LOCK_FINI(zone);
1753099a0e58SBosko Milekic }
1754099a0e58SBosko Milekic 
17559c2cd7e5SJeff Roberson /*
17568355f576SJeff Roberson  * Traverses every zone in the system and calls a callback
17578355f576SJeff Roberson  *
17588355f576SJeff Roberson  * Arguments:
17598355f576SJeff Roberson  *	zfunc  A pointer to a function which accepts a zone
17608355f576SJeff Roberson  *		as an argument.
17618355f576SJeff Roberson  *
17628355f576SJeff Roberson  * Returns:
17638355f576SJeff Roberson  *	Nothing
17648355f576SJeff Roberson  */
17658355f576SJeff Roberson static void
17668355f576SJeff Roberson zone_foreach(void (*zfunc)(uma_zone_t))
17678355f576SJeff Roberson {
1768099a0e58SBosko Milekic 	uma_keg_t keg;
17698355f576SJeff Roberson 	uma_zone_t zone;
17708355f576SJeff Roberson 
17718355f576SJeff Roberson 	mtx_lock(&uma_mtx);
1772099a0e58SBosko Milekic 	LIST_FOREACH(keg, &uma_kegs, uk_link) {
1773099a0e58SBosko Milekic 		LIST_FOREACH(zone, &keg->uk_zones, uz_link)
17748355f576SJeff Roberson 			zfunc(zone);
1775099a0e58SBosko Milekic 	}
17768355f576SJeff Roberson 	mtx_unlock(&uma_mtx);
17778355f576SJeff Roberson }
17788355f576SJeff Roberson 
17798355f576SJeff Roberson /* Public functions */
17808355f576SJeff Roberson /* See uma.h */
17818355f576SJeff Roberson void
17823803b26bSDag-Erling Smørgrav uma_startup(void *bootmem, int boot_pages)
17838355f576SJeff Roberson {
17848355f576SJeff Roberson 	struct uma_zctor_args args;
17858355f576SJeff Roberson 	uma_slab_t slab;
1786244f4554SBosko Milekic 	u_int slabsize;
17878355f576SJeff Roberson 	int i;
17888355f576SJeff Roberson 
17898355f576SJeff Roberson #ifdef UMA_DEBUG
1790099a0e58SBosko Milekic 	printf("Creating uma keg headers zone and keg.\n");
17918355f576SJeff Roberson #endif
1792f353d338SAlan Cox 	mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
1793099a0e58SBosko Milekic 
1794099a0e58SBosko Milekic 	/* "manually" create the initial zone */
17950095a784SJeff Roberson 	memset(&args, 0, sizeof(args));
1796099a0e58SBosko Milekic 	args.name = "UMA Kegs";
1797099a0e58SBosko Milekic 	args.size = sizeof(struct uma_keg);
1798099a0e58SBosko Milekic 	args.ctor = keg_ctor;
1799099a0e58SBosko Milekic 	args.dtor = keg_dtor;
18008355f576SJeff Roberson 	args.uminit = zero_init;
18018355f576SJeff Roberson 	args.fini = NULL;
1802099a0e58SBosko Milekic 	args.keg = &masterkeg;
18038355f576SJeff Roberson 	args.align = 32 - 1;
1804b60f5b79SJeff Roberson 	args.flags = UMA_ZFLAG_INTERNAL;
18058355f576SJeff Roberson 	/* The initial zone has no Per cpu queues so it's smaller */
1806b23f72e9SBrian Feldman 	zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK);
18078355f576SJeff Roberson 
18088355f576SJeff Roberson #ifdef UMA_DEBUG
18098355f576SJeff Roberson 	printf("Filling boot free list.\n");
18108355f576SJeff Roberson #endif
18113803b26bSDag-Erling Smørgrav 	for (i = 0; i < boot_pages; i++) {
181285dcf349SGleb Smirnoff 		slab = (uma_slab_t)((uint8_t *)bootmem + (i * UMA_SLAB_SIZE));
181385dcf349SGleb Smirnoff 		slab->us_data = (uint8_t *)slab;
18148355f576SJeff Roberson 		slab->us_flags = UMA_SLAB_BOOT;
18158355f576SJeff Roberson 		LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link);
18168355f576SJeff Roberson 	}
1817f353d338SAlan Cox 	mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF);
18188355f576SJeff Roberson 
18198355f576SJeff Roberson #ifdef UMA_DEBUG
1820099a0e58SBosko Milekic 	printf("Creating uma zone headers zone and keg.\n");
1821099a0e58SBosko Milekic #endif
1822099a0e58SBosko Milekic 	args.name = "UMA Zones";
1823099a0e58SBosko Milekic 	args.size = sizeof(struct uma_zone) +
1824099a0e58SBosko Milekic 	    (sizeof(struct uma_cache) * (mp_maxid + 1));
1825099a0e58SBosko Milekic 	args.ctor = zone_ctor;
1826099a0e58SBosko Milekic 	args.dtor = zone_dtor;
1827099a0e58SBosko Milekic 	args.uminit = zero_init;
1828099a0e58SBosko Milekic 	args.fini = NULL;
1829099a0e58SBosko Milekic 	args.keg = NULL;
1830099a0e58SBosko Milekic 	args.align = 32 - 1;
1831099a0e58SBosko Milekic 	args.flags = UMA_ZFLAG_INTERNAL;
1832099a0e58SBosko Milekic 	/* The initial zone has no Per cpu queues so it's smaller */
1833b23f72e9SBrian Feldman 	zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK);
1834099a0e58SBosko Milekic 
1835099a0e58SBosko Milekic #ifdef UMA_DEBUG
1836099a0e58SBosko Milekic 	printf("Initializing pcpu cache locks.\n");
1837099a0e58SBosko Milekic #endif
1838099a0e58SBosko Milekic #ifdef UMA_DEBUG
1839099a0e58SBosko Milekic 	printf("Creating slab and hash zones.\n");
18408355f576SJeff Roberson #endif
18418355f576SJeff Roberson 
18428355f576SJeff Roberson 	/* Now make a zone for slab headers */
18438355f576SJeff Roberson 	slabzone = uma_zcreate("UMA Slabs",
1844ef72505eSJeff Roberson 				sizeof(struct uma_slab),
18458355f576SJeff Roberson 				NULL, NULL, NULL, NULL,
1846b60f5b79SJeff Roberson 				UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
18478355f576SJeff Roberson 
1848099a0e58SBosko Milekic 	/*
1849099a0e58SBosko Milekic 	 * We also create a zone for the bigger slabs with reference
1850099a0e58SBosko Milekic 	 * counts in them, to accomodate UMA_ZONE_REFCNT zones.
1851099a0e58SBosko Milekic 	 */
1852ef72505eSJeff Roberson 	slabsize = sizeof(struct uma_slab_refcnt);
1853ef72505eSJeff Roberson 	slabsize += uma_max_ipers_ref * sizeof(uint32_t);
1854099a0e58SBosko Milekic 	slabrefzone = uma_zcreate("UMA RCntSlabs",
1855099a0e58SBosko Milekic 				  slabsize,
1856099a0e58SBosko Milekic 				  NULL, NULL, NULL, NULL,
1857e66468eaSBosko Milekic 				  UMA_ALIGN_PTR,
18587fd87882SBosko Milekic 				  UMA_ZFLAG_INTERNAL);
1859099a0e58SBosko Milekic 
18608355f576SJeff Roberson 	hashzone = uma_zcreate("UMA Hash",
18618355f576SJeff Roberson 	    sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT,
18628355f576SJeff Roberson 	    NULL, NULL, NULL, NULL,
1863b60f5b79SJeff Roberson 	    UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
18648355f576SJeff Roberson 
1865cae33c14SJeff Roberson 	bucket_init();
18668355f576SJeff Roberson 
1867342f1793SAlan Cox 	booted = UMA_STARTUP;
18688355f576SJeff Roberson 
18698355f576SJeff Roberson #ifdef UMA_DEBUG
18708355f576SJeff Roberson 	printf("UMA startup complete.\n");
18718355f576SJeff Roberson #endif
18728355f576SJeff Roberson }
18738355f576SJeff Roberson 
18748355f576SJeff Roberson /* see uma.h */
18758355f576SJeff Roberson void
187699571dc3SJeff Roberson uma_startup2(void)
18778355f576SJeff Roberson {
1878342f1793SAlan Cox 	booted = UMA_STARTUP2;
187986bbae32SJeff Roberson 	bucket_enable();
18808355f576SJeff Roberson #ifdef UMA_DEBUG
18818355f576SJeff Roberson 	printf("UMA startup2 complete.\n");
18828355f576SJeff Roberson #endif
18838355f576SJeff Roberson }
18848355f576SJeff Roberson 
18858355f576SJeff Roberson /*
18868355f576SJeff Roberson  * Initialize our callout handle
18878355f576SJeff Roberson  *
18888355f576SJeff Roberson  */
18898355f576SJeff Roberson 
18908355f576SJeff Roberson static void
18918355f576SJeff Roberson uma_startup3(void)
18928355f576SJeff Roberson {
18938355f576SJeff Roberson #ifdef UMA_DEBUG
18948355f576SJeff Roberson 	printf("Starting callout.\n");
18958355f576SJeff Roberson #endif
1896a3c07611SRobert Watson 	callout_init(&uma_callout, CALLOUT_MPSAFE);
18979643769aSJeff Roberson 	callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
18988355f576SJeff Roberson #ifdef UMA_DEBUG
18998355f576SJeff Roberson 	printf("UMA startup3 complete.\n");
19008355f576SJeff Roberson #endif
19018355f576SJeff Roberson }
19028355f576SJeff Roberson 
1903e20a199fSJeff Roberson static uma_keg_t
1904099a0e58SBosko Milekic uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
190585dcf349SGleb Smirnoff 		int align, uint32_t flags)
1906099a0e58SBosko Milekic {
1907099a0e58SBosko Milekic 	struct uma_kctor_args args;
1908099a0e58SBosko Milekic 
1909099a0e58SBosko Milekic 	args.size = size;
1910099a0e58SBosko Milekic 	args.uminit = uminit;
1911099a0e58SBosko Milekic 	args.fini = fini;
19121e319f6dSRobert Watson 	args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
1913099a0e58SBosko Milekic 	args.flags = flags;
1914099a0e58SBosko Milekic 	args.zone = zone;
1915e20a199fSJeff Roberson 	return (zone_alloc_item(kegs, &args, M_WAITOK));
1916099a0e58SBosko Milekic }
1917099a0e58SBosko Milekic 
19188355f576SJeff Roberson /* See uma.h */
19191e319f6dSRobert Watson void
19201e319f6dSRobert Watson uma_set_align(int align)
19211e319f6dSRobert Watson {
19221e319f6dSRobert Watson 
19231e319f6dSRobert Watson 	if (align != UMA_ALIGN_CACHE)
19241e319f6dSRobert Watson 		uma_align_cache = align;
19251e319f6dSRobert Watson }
19261e319f6dSRobert Watson 
19271e319f6dSRobert Watson /* See uma.h */
19288355f576SJeff Roberson uma_zone_t
1929bb196eb4SMatthew D Fleming uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
193085dcf349SGleb Smirnoff 		uma_init uminit, uma_fini fini, int align, uint32_t flags)
19318355f576SJeff Roberson 
19328355f576SJeff Roberson {
19338355f576SJeff Roberson 	struct uma_zctor_args args;
19348355f576SJeff Roberson 
19358355f576SJeff Roberson 	/* This stuff is essential for the zone ctor */
19360095a784SJeff Roberson 	memset(&args, 0, sizeof(args));
19378355f576SJeff Roberson 	args.name = name;
19388355f576SJeff Roberson 	args.size = size;
19398355f576SJeff Roberson 	args.ctor = ctor;
19408355f576SJeff Roberson 	args.dtor = dtor;
19418355f576SJeff Roberson 	args.uminit = uminit;
19428355f576SJeff Roberson 	args.fini = fini;
19438355f576SJeff Roberson 	args.align = align;
19448355f576SJeff Roberson 	args.flags = flags;
1945099a0e58SBosko Milekic 	args.keg = NULL;
1946099a0e58SBosko Milekic 
1947e20a199fSJeff Roberson 	return (zone_alloc_item(zones, &args, M_WAITOK));
1948099a0e58SBosko Milekic }
1949099a0e58SBosko Milekic 
1950099a0e58SBosko Milekic /* See uma.h */
1951099a0e58SBosko Milekic uma_zone_t
1952099a0e58SBosko Milekic uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
1953099a0e58SBosko Milekic 		    uma_init zinit, uma_fini zfini, uma_zone_t master)
1954099a0e58SBosko Milekic {
1955099a0e58SBosko Milekic 	struct uma_zctor_args args;
1956e20a199fSJeff Roberson 	uma_keg_t keg;
1957099a0e58SBosko Milekic 
1958e20a199fSJeff Roberson 	keg = zone_first_keg(master);
19590095a784SJeff Roberson 	memset(&args, 0, sizeof(args));
1960099a0e58SBosko Milekic 	args.name = name;
1961e20a199fSJeff Roberson 	args.size = keg->uk_size;
1962099a0e58SBosko Milekic 	args.ctor = ctor;
1963099a0e58SBosko Milekic 	args.dtor = dtor;
1964099a0e58SBosko Milekic 	args.uminit = zinit;
1965099a0e58SBosko Milekic 	args.fini = zfini;
1966e20a199fSJeff Roberson 	args.align = keg->uk_align;
1967e20a199fSJeff Roberson 	args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
1968e20a199fSJeff Roberson 	args.keg = keg;
19698355f576SJeff Roberson 
1970e20a199fSJeff Roberson 	/* XXX Attaches only one keg of potentially many. */
1971e20a199fSJeff Roberson 	return (zone_alloc_item(zones, &args, M_WAITOK));
19728355f576SJeff Roberson }
19738355f576SJeff Roberson 
19740095a784SJeff Roberson /* See uma.h */
19750095a784SJeff Roberson uma_zone_t
1976af526374SJeff Roberson uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor,
1977af526374SJeff Roberson 		    uma_init zinit, uma_fini zfini, uma_import zimport,
1978af526374SJeff Roberson 		    uma_release zrelease, void *arg, int flags)
19790095a784SJeff Roberson {
19800095a784SJeff Roberson 	struct uma_zctor_args args;
19810095a784SJeff Roberson 
19820095a784SJeff Roberson 	memset(&args, 0, sizeof(args));
19830095a784SJeff Roberson 	args.name = name;
1984af526374SJeff Roberson 	args.size = size;
19850095a784SJeff Roberson 	args.ctor = ctor;
19860095a784SJeff Roberson 	args.dtor = dtor;
19870095a784SJeff Roberson 	args.uminit = zinit;
19880095a784SJeff Roberson 	args.fini = zfini;
19890095a784SJeff Roberson 	args.import = zimport;
19900095a784SJeff Roberson 	args.release = zrelease;
19910095a784SJeff Roberson 	args.arg = arg;
19920095a784SJeff Roberson 	args.align = 0;
19930095a784SJeff Roberson 	args.flags = flags;
19940095a784SJeff Roberson 
19950095a784SJeff Roberson 	return (zone_alloc_item(zones, &args, M_WAITOK));
19960095a784SJeff Roberson }
19970095a784SJeff Roberson 
1998e20a199fSJeff Roberson static void
1999e20a199fSJeff Roberson zone_lock_pair(uma_zone_t a, uma_zone_t b)
2000e20a199fSJeff Roberson {
2001e20a199fSJeff Roberson 	if (a < b) {
2002e20a199fSJeff Roberson 		ZONE_LOCK(a);
2003af526374SJeff Roberson 		mtx_lock_flags(b->uz_lockptr, MTX_DUPOK);
2004e20a199fSJeff Roberson 	} else {
2005e20a199fSJeff Roberson 		ZONE_LOCK(b);
2006af526374SJeff Roberson 		mtx_lock_flags(a->uz_lockptr, MTX_DUPOK);
2007e20a199fSJeff Roberson 	}
2008e20a199fSJeff Roberson }
2009e20a199fSJeff Roberson 
2010e20a199fSJeff Roberson static void
2011e20a199fSJeff Roberson zone_unlock_pair(uma_zone_t a, uma_zone_t b)
2012e20a199fSJeff Roberson {
2013e20a199fSJeff Roberson 
2014e20a199fSJeff Roberson 	ZONE_UNLOCK(a);
2015e20a199fSJeff Roberson 	ZONE_UNLOCK(b);
2016e20a199fSJeff Roberson }
2017e20a199fSJeff Roberson 
2018e20a199fSJeff Roberson int
2019e20a199fSJeff Roberson uma_zsecond_add(uma_zone_t zone, uma_zone_t master)
2020e20a199fSJeff Roberson {
2021e20a199fSJeff Roberson 	uma_klink_t klink;
2022e20a199fSJeff Roberson 	uma_klink_t kl;
2023e20a199fSJeff Roberson 	int error;
2024e20a199fSJeff Roberson 
2025e20a199fSJeff Roberson 	error = 0;
2026e20a199fSJeff Roberson 	klink = malloc(sizeof(*klink), M_TEMP, M_WAITOK | M_ZERO);
2027e20a199fSJeff Roberson 
2028e20a199fSJeff Roberson 	zone_lock_pair(zone, master);
2029e20a199fSJeff Roberson 	/*
2030e20a199fSJeff Roberson 	 * zone must use vtoslab() to resolve objects and must already be
2031e20a199fSJeff Roberson 	 * a secondary.
2032e20a199fSJeff Roberson 	 */
2033e20a199fSJeff Roberson 	if ((zone->uz_flags & (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY))
2034e20a199fSJeff Roberson 	    != (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY)) {
2035e20a199fSJeff Roberson 		error = EINVAL;
2036e20a199fSJeff Roberson 		goto out;
2037e20a199fSJeff Roberson 	}
2038e20a199fSJeff Roberson 	/*
2039e20a199fSJeff Roberson 	 * The new master must also use vtoslab().
2040e20a199fSJeff Roberson 	 */
2041e20a199fSJeff Roberson 	if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) {
2042e20a199fSJeff Roberson 		error = EINVAL;
2043e20a199fSJeff Roberson 		goto out;
2044e20a199fSJeff Roberson 	}
2045e20a199fSJeff Roberson 	/*
2046e20a199fSJeff Roberson 	 * Both must either be refcnt, or not be refcnt.
2047e20a199fSJeff Roberson 	 */
2048e20a199fSJeff Roberson 	if ((zone->uz_flags & UMA_ZONE_REFCNT) !=
2049e20a199fSJeff Roberson 	    (master->uz_flags & UMA_ZONE_REFCNT)) {
2050e20a199fSJeff Roberson 		error = EINVAL;
2051e20a199fSJeff Roberson 		goto out;
2052e20a199fSJeff Roberson 	}
2053e20a199fSJeff Roberson 	/*
2054e20a199fSJeff Roberson 	 * The underlying object must be the same size.  rsize
2055e20a199fSJeff Roberson 	 * may be different.
2056e20a199fSJeff Roberson 	 */
2057e20a199fSJeff Roberson 	if (master->uz_size != zone->uz_size) {
2058e20a199fSJeff Roberson 		error = E2BIG;
2059e20a199fSJeff Roberson 		goto out;
2060e20a199fSJeff Roberson 	}
2061e20a199fSJeff Roberson 	/*
2062e20a199fSJeff Roberson 	 * Put it at the end of the list.
2063e20a199fSJeff Roberson 	 */
2064e20a199fSJeff Roberson 	klink->kl_keg = zone_first_keg(master);
2065e20a199fSJeff Roberson 	LIST_FOREACH(kl, &zone->uz_kegs, kl_link) {
2066e20a199fSJeff Roberson 		if (LIST_NEXT(kl, kl_link) == NULL) {
2067e20a199fSJeff Roberson 			LIST_INSERT_AFTER(kl, klink, kl_link);
2068e20a199fSJeff Roberson 			break;
2069e20a199fSJeff Roberson 		}
2070e20a199fSJeff Roberson 	}
2071e20a199fSJeff Roberson 	klink = NULL;
2072e20a199fSJeff Roberson 	zone->uz_flags |= UMA_ZFLAG_MULTI;
2073e20a199fSJeff Roberson 	zone->uz_slab = zone_fetch_slab_multi;
2074e20a199fSJeff Roberson 
2075e20a199fSJeff Roberson out:
2076e20a199fSJeff Roberson 	zone_unlock_pair(zone, master);
2077e20a199fSJeff Roberson 	if (klink != NULL)
2078e20a199fSJeff Roberson 		free(klink, M_TEMP);
2079e20a199fSJeff Roberson 
2080e20a199fSJeff Roberson 	return (error);
2081e20a199fSJeff Roberson }
2082e20a199fSJeff Roberson 
2083e20a199fSJeff Roberson 
20848355f576SJeff Roberson /* See uma.h */
20859c2cd7e5SJeff Roberson void
20869c2cd7e5SJeff Roberson uma_zdestroy(uma_zone_t zone)
20879c2cd7e5SJeff Roberson {
2088f4ff923bSRobert Watson 
20890095a784SJeff Roberson 	zone_free_item(zones, zone, NULL, SKIP_NONE);
20909c2cd7e5SJeff Roberson }
20919c2cd7e5SJeff Roberson 
20929c2cd7e5SJeff Roberson /* See uma.h */
20938355f576SJeff Roberson void *
20942cc35ff9SJeff Roberson uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
20958355f576SJeff Roberson {
20968355f576SJeff Roberson 	void *item;
20978355f576SJeff Roberson 	uma_cache_t cache;
20988355f576SJeff Roberson 	uma_bucket_t bucket;
2099fc03d22bSJeff Roberson 	int lockfail;
21008355f576SJeff Roberson 	int cpu;
21018355f576SJeff Roberson 
21028355f576SJeff Roberson 	/* This is the fast path allocation */
21038355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC_1
21048355f576SJeff Roberson 	printf("Allocating one item from %s(%p)\n", zone->uz_name, zone);
21058355f576SJeff Roberson #endif
21063659f747SRobert Watson 	CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread,
21073659f747SRobert Watson 	    zone->uz_name, flags);
2108a553d4b8SJeff Roberson 
2109635fd505SRobert Watson 	if (flags & M_WAITOK) {
2110b23f72e9SBrian Feldman 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
2111635fd505SRobert Watson 		    "uma_zalloc_arg: zone \"%s\"", zone->uz_name);
21124c1cc01cSJohn Baldwin 	}
21138d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD
21148d689e04SGleb Smirnoff 	if (memguard_cmp_zone(zone)) {
21158d689e04SGleb Smirnoff 		item = memguard_alloc(zone->uz_size, flags);
21168d689e04SGleb Smirnoff 		if (item != NULL) {
21178d689e04SGleb Smirnoff 			/*
21188d689e04SGleb Smirnoff 			 * Avoid conflict with the use-after-free
21198d689e04SGleb Smirnoff 			 * protecting infrastructure from INVARIANTS.
21208d689e04SGleb Smirnoff 			 */
21218d689e04SGleb Smirnoff 			if (zone->uz_init != NULL &&
21228d689e04SGleb Smirnoff 			    zone->uz_init != mtrash_init &&
21238d689e04SGleb Smirnoff 			    zone->uz_init(item, zone->uz_size, flags) != 0)
21248d689e04SGleb Smirnoff 				return (NULL);
21258d689e04SGleb Smirnoff 			if (zone->uz_ctor != NULL &&
21268d689e04SGleb Smirnoff 			    zone->uz_ctor != mtrash_ctor &&
2127fc03d22bSJeff Roberson 			    zone->uz_ctor(item, zone->uz_size, udata,
2128fc03d22bSJeff Roberson 			    flags) != 0) {
21298d689e04SGleb Smirnoff 			    	zone->uz_fini(item, zone->uz_size);
21308d689e04SGleb Smirnoff 				return (NULL);
21318d689e04SGleb Smirnoff 			}
21328d689e04SGleb Smirnoff 			return (item);
21338d689e04SGleb Smirnoff 		}
21348d689e04SGleb Smirnoff 		/* This is unfortunate but should not be fatal. */
21358d689e04SGleb Smirnoff 	}
21368d689e04SGleb Smirnoff #endif
21375d1ae027SRobert Watson 	/*
21385d1ae027SRobert Watson 	 * If possible, allocate from the per-CPU cache.  There are two
21395d1ae027SRobert Watson 	 * requirements for safe access to the per-CPU cache: (1) the thread
21405d1ae027SRobert Watson 	 * accessing the cache must not be preempted or yield during access,
21415d1ae027SRobert Watson 	 * and (2) the thread must not migrate CPUs without switching which
21425d1ae027SRobert Watson 	 * cache it accesses.  We rely on a critical section to prevent
21435d1ae027SRobert Watson 	 * preemption and migration.  We release the critical section in
21445d1ae027SRobert Watson 	 * order to acquire the zone mutex if we are unable to allocate from
21455d1ae027SRobert Watson 	 * the current cache; when we re-acquire the critical section, we
21465d1ae027SRobert Watson 	 * must detect and handle migration if it has occurred.
21475d1ae027SRobert Watson 	 */
21485d1ae027SRobert Watson 	critical_enter();
21495d1ae027SRobert Watson 	cpu = curcpu;
21508355f576SJeff Roberson 	cache = &zone->uz_cpu[cpu];
21518355f576SJeff Roberson 
21528355f576SJeff Roberson zalloc_start:
21538355f576SJeff Roberson 	bucket = cache->uc_allocbucket;
2154fc03d22bSJeff Roberson 	if (bucket != NULL && bucket->ub_cnt > 0) {
2155cae33c14SJeff Roberson 		bucket->ub_cnt--;
2156cae33c14SJeff Roberson 		item = bucket->ub_bucket[bucket->ub_cnt];
21578355f576SJeff Roberson #ifdef INVARIANTS
2158cae33c14SJeff Roberson 		bucket->ub_bucket[bucket->ub_cnt] = NULL;
21598355f576SJeff Roberson #endif
2160fc03d22bSJeff Roberson 		KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled."));
21618355f576SJeff Roberson 		cache->uc_allocs++;
21625d1ae027SRobert Watson 		critical_exit();
2163fc03d22bSJeff Roberson 		if (zone->uz_ctor != NULL &&
2164fc03d22bSJeff Roberson 		    zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
21650095a784SJeff Roberson 			atomic_add_long(&zone->uz_fails, 1);
2166fc03d22bSJeff Roberson 			zone_free_item(zone, item, udata, SKIP_DTOR);
2167b23f72e9SBrian Feldman 			return (NULL);
2168b23f72e9SBrian Feldman 		}
2169ef72505eSJeff Roberson #ifdef INVARIANTS
2170ef72505eSJeff Roberson 		uma_dbg_alloc(zone, NULL, item);
2171ef72505eSJeff Roberson #endif
21722cc35ff9SJeff Roberson 		if (flags & M_ZERO)
217348343a2fSGleb Smirnoff 			uma_zero_item(item, zone);
21748355f576SJeff Roberson 		return (item);
2175fc03d22bSJeff Roberson 	}
2176fc03d22bSJeff Roberson 
21778355f576SJeff Roberson 	/*
21788355f576SJeff Roberson 	 * We have run out of items in our alloc bucket.
21798355f576SJeff Roberson 	 * See if we can switch with our free bucket.
21808355f576SJeff Roberson 	 */
2181b983089aSJeff Roberson 	bucket = cache->uc_freebucket;
2182fc03d22bSJeff Roberson 	if (bucket != NULL && bucket->ub_cnt > 0) {
2183fc03d22bSJeff Roberson #ifdef UMA_DEBUG_ALLOC
2184fc03d22bSJeff Roberson 		printf("uma_zalloc: Swapping empty with alloc.\n");
2185fc03d22bSJeff Roberson #endif
21868355f576SJeff Roberson 		cache->uc_freebucket = cache->uc_allocbucket;
2187b983089aSJeff Roberson 		cache->uc_allocbucket = bucket;
21888355f576SJeff Roberson 		goto zalloc_start;
21898355f576SJeff Roberson 	}
2190fc03d22bSJeff Roberson 
2191fc03d22bSJeff Roberson 	/*
2192fc03d22bSJeff Roberson 	 * Discard any empty allocation bucket while we hold no locks.
2193fc03d22bSJeff Roberson 	 */
2194fc03d22bSJeff Roberson 	bucket = cache->uc_allocbucket;
2195fc03d22bSJeff Roberson 	cache->uc_allocbucket = NULL;
2196fc03d22bSJeff Roberson 	critical_exit();
2197fc03d22bSJeff Roberson 	if (bucket != NULL)
21986fd34d6fSJeff Roberson 		bucket_free(zone, bucket, udata);
2199fc03d22bSJeff Roberson 
2200fc03d22bSJeff Roberson 	/* Short-circuit for zones without buckets and low memory. */
2201fc03d22bSJeff Roberson 	if (zone->uz_count == 0 || bucketdisable)
2202fc03d22bSJeff Roberson 		goto zalloc_item;
2203fc03d22bSJeff Roberson 
22045d1ae027SRobert Watson 	/*
22055d1ae027SRobert Watson 	 * Attempt to retrieve the item from the per-CPU cache has failed, so
22065d1ae027SRobert Watson 	 * we must go back to the zone.  This requires the zone lock, so we
22075d1ae027SRobert Watson 	 * must drop the critical section, then re-acquire it when we go back
22085d1ae027SRobert Watson 	 * to the cache.  Since the critical section is released, we may be
22095d1ae027SRobert Watson 	 * preempted or migrate.  As such, make sure not to maintain any
22105d1ae027SRobert Watson 	 * thread-local state specific to the cache from prior to releasing
22115d1ae027SRobert Watson 	 * the critical section.
22125d1ae027SRobert Watson 	 */
2213fc03d22bSJeff Roberson 	lockfail = 0;
2214fc03d22bSJeff Roberson 	if (ZONE_TRYLOCK(zone) == 0) {
2215fc03d22bSJeff Roberson 		/* Record contention to size the buckets. */
2216a553d4b8SJeff Roberson 		ZONE_LOCK(zone);
2217fc03d22bSJeff Roberson 		lockfail = 1;
2218fc03d22bSJeff Roberson 	}
22195d1ae027SRobert Watson 	critical_enter();
22205d1ae027SRobert Watson 	cpu = curcpu;
22215d1ae027SRobert Watson 	cache = &zone->uz_cpu[cpu];
22225d1ae027SRobert Watson 
2223fc03d22bSJeff Roberson 	/*
2224fc03d22bSJeff Roberson 	 * Since we have locked the zone we may as well send back our stats.
2225fc03d22bSJeff Roberson 	 */
22260095a784SJeff Roberson 	atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
22270095a784SJeff Roberson 	atomic_add_long(&zone->uz_frees, cache->uc_frees);
2228a553d4b8SJeff Roberson 	cache->uc_allocs = 0;
2229773df9abSRobert Watson 	cache->uc_frees = 0;
22308355f576SJeff Roberson 
2231fc03d22bSJeff Roberson 	/* See if we lost the race to fill the cache. */
2232fc03d22bSJeff Roberson 	if (cache->uc_allocbucket != NULL) {
2233fc03d22bSJeff Roberson 		ZONE_UNLOCK(zone);
2234fc03d22bSJeff Roberson 		goto zalloc_start;
2235a553d4b8SJeff Roberson 	}
22368355f576SJeff Roberson 
2237fc03d22bSJeff Roberson 	/*
2238fc03d22bSJeff Roberson 	 * Check the zone's cache of buckets.
2239fc03d22bSJeff Roberson 	 */
2240fc03d22bSJeff Roberson 	if ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
2241cae33c14SJeff Roberson 		KASSERT(bucket->ub_cnt != 0,
2242a553d4b8SJeff Roberson 		    ("uma_zalloc_arg: Returning an empty bucket."));
22438355f576SJeff Roberson 
2244a553d4b8SJeff Roberson 		LIST_REMOVE(bucket, ub_link);
2245a553d4b8SJeff Roberson 		cache->uc_allocbucket = bucket;
2246a553d4b8SJeff Roberson 		ZONE_UNLOCK(zone);
22478355f576SJeff Roberson 		goto zalloc_start;
2248a553d4b8SJeff Roberson 	}
22495d1ae027SRobert Watson 	/* We are no longer associated with this CPU. */
22505d1ae027SRobert Watson 	critical_exit();
2251bbee39c6SJeff Roberson 
2252fc03d22bSJeff Roberson 	/*
2253fc03d22bSJeff Roberson 	 * We bump the uz count when the cache size is insufficient to
2254fc03d22bSJeff Roberson 	 * handle the working set.
2255fc03d22bSJeff Roberson 	 */
22566fd34d6fSJeff Roberson 	if (lockfail && zone->uz_count < BUCKET_MAX)
2257a553d4b8SJeff Roberson 		zone->uz_count++;
2258fc03d22bSJeff Roberson 	ZONE_UNLOCK(zone);
2259099a0e58SBosko Milekic 
22608355f576SJeff Roberson 	/*
2261a553d4b8SJeff Roberson 	 * Now lets just fill a bucket and put it on the free list.  If that
2262fc03d22bSJeff Roberson 	 * works we'll restart the allocation from the begining and it
2263fc03d22bSJeff Roberson 	 * will use the just filled bucket.
2264bbee39c6SJeff Roberson 	 */
22656fd34d6fSJeff Roberson 	bucket = zone_alloc_bucket(zone, udata, flags);
2266fc03d22bSJeff Roberson 	if (bucket != NULL) {
2267fc03d22bSJeff Roberson 		ZONE_LOCK(zone);
2268fc03d22bSJeff Roberson 		critical_enter();
2269fc03d22bSJeff Roberson 		cpu = curcpu;
2270fc03d22bSJeff Roberson 		cache = &zone->uz_cpu[cpu];
2271fc03d22bSJeff Roberson 		/*
2272fc03d22bSJeff Roberson 		 * See if we lost the race or were migrated.  Cache the
2273fc03d22bSJeff Roberson 		 * initialized bucket to make this less likely or claim
2274fc03d22bSJeff Roberson 		 * the memory directly.
2275fc03d22bSJeff Roberson 		 */
2276fc03d22bSJeff Roberson 		if (cache->uc_allocbucket == NULL)
2277fc03d22bSJeff Roberson 			cache->uc_allocbucket = bucket;
2278fc03d22bSJeff Roberson 		else
2279fc03d22bSJeff Roberson 			LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link);
2280bbee39c6SJeff Roberson 		ZONE_UNLOCK(zone);
2281fc03d22bSJeff Roberson 		goto zalloc_start;
2282bbee39c6SJeff Roberson 	}
2283fc03d22bSJeff Roberson 
2284bbee39c6SJeff Roberson 	/*
2285bbee39c6SJeff Roberson 	 * We may not be able to get a bucket so return an actual item.
2286bbee39c6SJeff Roberson 	 */
2287bbee39c6SJeff Roberson #ifdef UMA_DEBUG
2288bbee39c6SJeff Roberson 	printf("uma_zalloc_arg: Bucketzone returned NULL\n");
2289bbee39c6SJeff Roberson #endif
2290bbee39c6SJeff Roberson 
2291fc03d22bSJeff Roberson zalloc_item:
2292e20a199fSJeff Roberson 	item = zone_alloc_item(zone, udata, flags);
2293fc03d22bSJeff Roberson 
2294e20a199fSJeff Roberson 	return (item);
2295bbee39c6SJeff Roberson }
2296bbee39c6SJeff Roberson 
2297bbee39c6SJeff Roberson static uma_slab_t
2298e20a199fSJeff Roberson keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int flags)
2299bbee39c6SJeff Roberson {
2300bbee39c6SJeff Roberson 	uma_slab_t slab;
23016fd34d6fSJeff Roberson 	int reserve;
2302099a0e58SBosko Milekic 
2303e20a199fSJeff Roberson 	mtx_assert(&keg->uk_lock, MA_OWNED);
2304bbee39c6SJeff Roberson 	slab = NULL;
23056fd34d6fSJeff Roberson 	reserve = 0;
23066fd34d6fSJeff Roberson 	if ((flags & M_USE_RESERVE) == 0)
23076fd34d6fSJeff Roberson 		reserve = keg->uk_reserve;
2308bbee39c6SJeff Roberson 
2309bbee39c6SJeff Roberson 	for (;;) {
2310bbee39c6SJeff Roberson 		/*
2311bbee39c6SJeff Roberson 		 * Find a slab with some space.  Prefer slabs that are partially
2312bbee39c6SJeff Roberson 		 * used over those that are totally full.  This helps to reduce
2313bbee39c6SJeff Roberson 		 * fragmentation.
2314bbee39c6SJeff Roberson 		 */
23156fd34d6fSJeff Roberson 		if (keg->uk_free > reserve) {
2316099a0e58SBosko Milekic 			if (!LIST_EMPTY(&keg->uk_part_slab)) {
2317099a0e58SBosko Milekic 				slab = LIST_FIRST(&keg->uk_part_slab);
2318bbee39c6SJeff Roberson 			} else {
2319099a0e58SBosko Milekic 				slab = LIST_FIRST(&keg->uk_free_slab);
2320bbee39c6SJeff Roberson 				LIST_REMOVE(slab, us_link);
2321099a0e58SBosko Milekic 				LIST_INSERT_HEAD(&keg->uk_part_slab, slab,
2322bbee39c6SJeff Roberson 				    us_link);
2323bbee39c6SJeff Roberson 			}
2324e20a199fSJeff Roberson 			MPASS(slab->us_keg == keg);
2325bbee39c6SJeff Roberson 			return (slab);
2326bbee39c6SJeff Roberson 		}
2327bbee39c6SJeff Roberson 
2328bbee39c6SJeff Roberson 		/*
2329bbee39c6SJeff Roberson 		 * M_NOVM means don't ask at all!
2330bbee39c6SJeff Roberson 		 */
2331bbee39c6SJeff Roberson 		if (flags & M_NOVM)
2332bbee39c6SJeff Roberson 			break;
2333bbee39c6SJeff Roberson 
2334e20a199fSJeff Roberson 		if (keg->uk_maxpages && keg->uk_pages >= keg->uk_maxpages) {
2335099a0e58SBosko Milekic 			keg->uk_flags |= UMA_ZFLAG_FULL;
2336e20a199fSJeff Roberson 			/*
2337e20a199fSJeff Roberson 			 * If this is not a multi-zone, set the FULL bit.
2338e20a199fSJeff Roberson 			 * Otherwise slab_multi() takes care of it.
2339e20a199fSJeff Roberson 			 */
23402f891cd5SPawel Jakub Dawidek 			if ((zone->uz_flags & UMA_ZFLAG_MULTI) == 0) {
2341e20a199fSJeff Roberson 				zone->uz_flags |= UMA_ZFLAG_FULL;
23422f891cd5SPawel Jakub Dawidek 				zone_log_warning(zone);
23432f891cd5SPawel Jakub Dawidek 			}
2344ebc85edfSJeff Roberson 			if (flags & M_NOWAIT)
2345bbee39c6SJeff Roberson 				break;
2346c288b548SEitan Adler 			zone->uz_sleeps++;
2347e20a199fSJeff Roberson 			msleep(keg, &keg->uk_lock, PVM, "keglimit", 0);
2348bbee39c6SJeff Roberson 			continue;
2349bbee39c6SJeff Roberson 		}
2350e20a199fSJeff Roberson 		slab = keg_alloc_slab(keg, zone, flags);
2351bbee39c6SJeff Roberson 		/*
2352bbee39c6SJeff Roberson 		 * If we got a slab here it's safe to mark it partially used
2353bbee39c6SJeff Roberson 		 * and return.  We assume that the caller is going to remove
2354bbee39c6SJeff Roberson 		 * at least one item.
2355bbee39c6SJeff Roberson 		 */
2356bbee39c6SJeff Roberson 		if (slab) {
2357e20a199fSJeff Roberson 			MPASS(slab->us_keg == keg);
2358099a0e58SBosko Milekic 			LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
2359bbee39c6SJeff Roberson 			return (slab);
2360bbee39c6SJeff Roberson 		}
2361bbee39c6SJeff Roberson 		/*
2362bbee39c6SJeff Roberson 		 * We might not have been able to get a slab but another cpu
2363bbee39c6SJeff Roberson 		 * could have while we were unlocked.  Check again before we
2364bbee39c6SJeff Roberson 		 * fail.
2365bbee39c6SJeff Roberson 		 */
2366bbee39c6SJeff Roberson 		flags |= M_NOVM;
2367bbee39c6SJeff Roberson 	}
2368bbee39c6SJeff Roberson 	return (slab);
2369bbee39c6SJeff Roberson }
2370bbee39c6SJeff Roberson 
2371e20a199fSJeff Roberson static uma_slab_t
2372e20a199fSJeff Roberson zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int flags)
2373e20a199fSJeff Roberson {
2374e20a199fSJeff Roberson 	uma_slab_t slab;
2375e20a199fSJeff Roberson 
2376af526374SJeff Roberson 	if (keg == NULL) {
2377e20a199fSJeff Roberson 		keg = zone_first_keg(zone);
2378af526374SJeff Roberson 		KEG_LOCK(keg);
2379af526374SJeff Roberson 	}
2380e20a199fSJeff Roberson 
2381e20a199fSJeff Roberson 	for (;;) {
2382e20a199fSJeff Roberson 		slab = keg_fetch_slab(keg, zone, flags);
2383e20a199fSJeff Roberson 		if (slab)
2384e20a199fSJeff Roberson 			return (slab);
2385e20a199fSJeff Roberson 		if (flags & (M_NOWAIT | M_NOVM))
2386e20a199fSJeff Roberson 			break;
2387e20a199fSJeff Roberson 	}
2388af526374SJeff Roberson 	KEG_UNLOCK(keg);
2389e20a199fSJeff Roberson 	return (NULL);
2390e20a199fSJeff Roberson }
2391e20a199fSJeff Roberson 
2392e20a199fSJeff Roberson /*
2393e20a199fSJeff Roberson  * uma_zone_fetch_slab_multi:  Fetches a slab from one available keg.  Returns
2394af526374SJeff Roberson  * with the keg locked.  On NULL no lock is held.
2395e20a199fSJeff Roberson  *
2396e20a199fSJeff Roberson  * The last pointer is used to seed the search.  It is not required.
2397e20a199fSJeff Roberson  */
2398e20a199fSJeff Roberson static uma_slab_t
2399e20a199fSJeff Roberson zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags)
2400e20a199fSJeff Roberson {
2401e20a199fSJeff Roberson 	uma_klink_t klink;
2402e20a199fSJeff Roberson 	uma_slab_t slab;
2403e20a199fSJeff Roberson 	uma_keg_t keg;
2404e20a199fSJeff Roberson 	int flags;
2405e20a199fSJeff Roberson 	int empty;
2406e20a199fSJeff Roberson 	int full;
2407e20a199fSJeff Roberson 
2408e20a199fSJeff Roberson 	/*
2409e20a199fSJeff Roberson 	 * Don't wait on the first pass.  This will skip limit tests
2410e20a199fSJeff Roberson 	 * as well.  We don't want to block if we can find a provider
2411e20a199fSJeff Roberson 	 * without blocking.
2412e20a199fSJeff Roberson 	 */
2413e20a199fSJeff Roberson 	flags = (rflags & ~M_WAITOK) | M_NOWAIT;
2414e20a199fSJeff Roberson 	/*
2415e20a199fSJeff Roberson 	 * Use the last slab allocated as a hint for where to start
2416e20a199fSJeff Roberson 	 * the search.
2417e20a199fSJeff Roberson 	 */
2418af526374SJeff Roberson 	if (last != NULL) {
2419e20a199fSJeff Roberson 		slab = keg_fetch_slab(last, zone, flags);
2420e20a199fSJeff Roberson 		if (slab)
2421e20a199fSJeff Roberson 			return (slab);
2422af526374SJeff Roberson 		KEG_UNLOCK(last);
2423e20a199fSJeff Roberson 	}
2424e20a199fSJeff Roberson 	/*
2425e20a199fSJeff Roberson 	 * Loop until we have a slab incase of transient failures
2426e20a199fSJeff Roberson 	 * while M_WAITOK is specified.  I'm not sure this is 100%
2427e20a199fSJeff Roberson 	 * required but we've done it for so long now.
2428e20a199fSJeff Roberson 	 */
2429e20a199fSJeff Roberson 	for (;;) {
2430e20a199fSJeff Roberson 		empty = 0;
2431e20a199fSJeff Roberson 		full = 0;
2432e20a199fSJeff Roberson 		/*
2433e20a199fSJeff Roberson 		 * Search the available kegs for slabs.  Be careful to hold the
2434e20a199fSJeff Roberson 		 * correct lock while calling into the keg layer.
2435e20a199fSJeff Roberson 		 */
2436e20a199fSJeff Roberson 		LIST_FOREACH(klink, &zone->uz_kegs, kl_link) {
2437e20a199fSJeff Roberson 			keg = klink->kl_keg;
2438af526374SJeff Roberson 			KEG_LOCK(keg);
2439e20a199fSJeff Roberson 			if ((keg->uk_flags & UMA_ZFLAG_FULL) == 0) {
2440e20a199fSJeff Roberson 				slab = keg_fetch_slab(keg, zone, flags);
2441e20a199fSJeff Roberson 				if (slab)
2442e20a199fSJeff Roberson 					return (slab);
2443e20a199fSJeff Roberson 			}
2444e20a199fSJeff Roberson 			if (keg->uk_flags & UMA_ZFLAG_FULL)
2445e20a199fSJeff Roberson 				full++;
2446e20a199fSJeff Roberson 			else
2447e20a199fSJeff Roberson 				empty++;
2448af526374SJeff Roberson 			KEG_UNLOCK(keg);
2449e20a199fSJeff Roberson 		}
2450e20a199fSJeff Roberson 		if (rflags & (M_NOWAIT | M_NOVM))
2451e20a199fSJeff Roberson 			break;
2452e20a199fSJeff Roberson 		flags = rflags;
2453e20a199fSJeff Roberson 		/*
2454e20a199fSJeff Roberson 		 * All kegs are full.  XXX We can't atomically check all kegs
2455e20a199fSJeff Roberson 		 * and sleep so just sleep for a short period and retry.
2456e20a199fSJeff Roberson 		 */
2457e20a199fSJeff Roberson 		if (full && !empty) {
2458af526374SJeff Roberson 			ZONE_LOCK(zone);
2459e20a199fSJeff Roberson 			zone->uz_flags |= UMA_ZFLAG_FULL;
2460bf965959SSean Bruno 			zone->uz_sleeps++;
24612f891cd5SPawel Jakub Dawidek 			zone_log_warning(zone);
2462af526374SJeff Roberson 			msleep(zone, zone->uz_lockptr, PVM,
2463af526374SJeff Roberson 			    "zonelimit", hz/100);
2464e20a199fSJeff Roberson 			zone->uz_flags &= ~UMA_ZFLAG_FULL;
2465af526374SJeff Roberson 			ZONE_UNLOCK(zone);
2466e20a199fSJeff Roberson 			continue;
2467e20a199fSJeff Roberson 		}
2468e20a199fSJeff Roberson 	}
2469e20a199fSJeff Roberson 	return (NULL);
2470e20a199fSJeff Roberson }
2471e20a199fSJeff Roberson 
2472d56368d7SBosko Milekic static void *
24730095a784SJeff Roberson slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
2474bbee39c6SJeff Roberson {
2475bbee39c6SJeff Roberson 	void *item;
247685dcf349SGleb Smirnoff 	uint8_t freei;
2477bbee39c6SJeff Roberson 
24780095a784SJeff Roberson 	MPASS(keg == slab->us_keg);
2479e20a199fSJeff Roberson 	mtx_assert(&keg->uk_lock, MA_OWNED);
2480099a0e58SBosko Milekic 
2481ef72505eSJeff Roberson 	freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
2482ef72505eSJeff Roberson 	BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free);
2483099a0e58SBosko Milekic 	item = slab->us_data + (keg->uk_rsize * freei);
2484bbee39c6SJeff Roberson 	slab->us_freecount--;
2485099a0e58SBosko Milekic 	keg->uk_free--;
2486ef72505eSJeff Roberson 
2487bbee39c6SJeff Roberson 	/* Move this slab to the full list */
2488bbee39c6SJeff Roberson 	if (slab->us_freecount == 0) {
2489bbee39c6SJeff Roberson 		LIST_REMOVE(slab, us_link);
2490099a0e58SBosko Milekic 		LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link);
2491bbee39c6SJeff Roberson 	}
2492bbee39c6SJeff Roberson 
2493bbee39c6SJeff Roberson 	return (item);
2494bbee39c6SJeff Roberson }
2495bbee39c6SJeff Roberson 
2496bbee39c6SJeff Roberson static int
24970095a784SJeff Roberson zone_import(uma_zone_t zone, void **bucket, int max, int flags)
24980095a784SJeff Roberson {
24990095a784SJeff Roberson 	uma_slab_t slab;
25000095a784SJeff Roberson 	uma_keg_t keg;
25010095a784SJeff Roberson 	int i;
25020095a784SJeff Roberson 
25030095a784SJeff Roberson 	slab = NULL;
25040095a784SJeff Roberson 	keg = NULL;
2505af526374SJeff Roberson 	/* Try to keep the buckets totally full */
25060095a784SJeff Roberson 	for (i = 0; i < max; ) {
25070095a784SJeff Roberson 		if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
25080095a784SJeff Roberson 			break;
25090095a784SJeff Roberson 		keg = slab->us_keg;
25106fd34d6fSJeff Roberson 		while (slab->us_freecount && i < max) {
25110095a784SJeff Roberson 			bucket[i++] = slab_alloc_item(keg, slab);
25126fd34d6fSJeff Roberson 			if (keg->uk_free <= keg->uk_reserve)
25136fd34d6fSJeff Roberson 				break;
25146fd34d6fSJeff Roberson 		}
25156fd34d6fSJeff Roberson 		/* Don't grab more than one slab at a time. */
25160095a784SJeff Roberson 		flags &= ~M_WAITOK;
25170095a784SJeff Roberson 		flags |= M_NOWAIT;
25180095a784SJeff Roberson 	}
25190095a784SJeff Roberson 	if (slab != NULL)
25200095a784SJeff Roberson 		KEG_UNLOCK(keg);
25210095a784SJeff Roberson 
25220095a784SJeff Roberson 	return i;
25230095a784SJeff Roberson }
25240095a784SJeff Roberson 
2525fc03d22bSJeff Roberson static uma_bucket_t
25266fd34d6fSJeff Roberson zone_alloc_bucket(uma_zone_t zone, void *udata, int flags)
2527bbee39c6SJeff Roberson {
2528bbee39c6SJeff Roberson 	uma_bucket_t bucket;
25290095a784SJeff Roberson 	int max;
2530bbee39c6SJeff Roberson 
25316fd34d6fSJeff Roberson 	/* Don't wait for buckets, preserve caller's NOVM setting. */
25326fd34d6fSJeff Roberson 	bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM));
25330095a784SJeff Roberson 	if (bucket == NULL)
2534f7104ccdSAlexander Motin 		return (NULL);
25350095a784SJeff Roberson 
2536af526374SJeff Roberson 	max = MIN(bucket->ub_entries, zone->uz_count);
25370095a784SJeff Roberson 	bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
25380095a784SJeff Roberson 	    max, flags);
25390095a784SJeff Roberson 
25400095a784SJeff Roberson 	/*
25410095a784SJeff Roberson 	 * Initialize the memory if necessary.
25420095a784SJeff Roberson 	 */
25430095a784SJeff Roberson 	if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
2544099a0e58SBosko Milekic 		int i;
2545bbee39c6SJeff Roberson 
25460095a784SJeff Roberson 		for (i = 0; i < bucket->ub_cnt; i++)
2547e20a199fSJeff Roberson 			if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
25480095a784SJeff Roberson 			    flags) != 0)
2549b23f72e9SBrian Feldman 				break;
2550b23f72e9SBrian Feldman 		/*
2551b23f72e9SBrian Feldman 		 * If we couldn't initialize the whole bucket, put the
2552b23f72e9SBrian Feldman 		 * rest back onto the freelist.
2553b23f72e9SBrian Feldman 		 */
2554b23f72e9SBrian Feldman 		if (i != bucket->ub_cnt) {
2555af526374SJeff Roberson 			zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i],
25560095a784SJeff Roberson 			    bucket->ub_cnt - i);
2557a5a262c6SBosko Milekic #ifdef INVARIANTS
25580095a784SJeff Roberson 			bzero(&bucket->ub_bucket[i],
25590095a784SJeff Roberson 			    sizeof(void *) * (bucket->ub_cnt - i));
2560a5a262c6SBosko Milekic #endif
2561b23f72e9SBrian Feldman 			bucket->ub_cnt = i;
2562b23f72e9SBrian Feldman 		}
2563099a0e58SBosko Milekic 	}
2564099a0e58SBosko Milekic 
2565f7104ccdSAlexander Motin 	if (bucket->ub_cnt == 0) {
25666fd34d6fSJeff Roberson 		bucket_free(zone, bucket, udata);
2567fc03d22bSJeff Roberson 		atomic_add_long(&zone->uz_fails, 1);
2568fc03d22bSJeff Roberson 		return (NULL);
2569bbee39c6SJeff Roberson 	}
2570fc03d22bSJeff Roberson 
2571fc03d22bSJeff Roberson 	return (bucket);
2572fc03d22bSJeff Roberson }
2573fc03d22bSJeff Roberson 
25748355f576SJeff Roberson /*
25750095a784SJeff Roberson  * Allocates a single item from a zone.
25768355f576SJeff Roberson  *
25778355f576SJeff Roberson  * Arguments
25788355f576SJeff Roberson  *	zone   The zone to alloc for.
25798355f576SJeff Roberson  *	udata  The data to be passed to the constructor.
2580a163d034SWarner Losh  *	flags  M_WAITOK, M_NOWAIT, M_ZERO.
25818355f576SJeff Roberson  *
25828355f576SJeff Roberson  * Returns
25838355f576SJeff Roberson  *	NULL if there is no memory and M_NOWAIT is set
2584bbee39c6SJeff Roberson  *	An item if successful
25858355f576SJeff Roberson  */
25868355f576SJeff Roberson 
25878355f576SJeff Roberson static void *
2588e20a199fSJeff Roberson zone_alloc_item(uma_zone_t zone, void *udata, int flags)
25898355f576SJeff Roberson {
25908355f576SJeff Roberson 	void *item;
25918355f576SJeff Roberson 
25928355f576SJeff Roberson 	item = NULL;
25938355f576SJeff Roberson 
25948355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC
25958355f576SJeff Roberson 	printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
25968355f576SJeff Roberson #endif
25970095a784SJeff Roberson 	if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
25980095a784SJeff Roberson 		goto fail;
25990095a784SJeff Roberson 	atomic_add_long(&zone->uz_allocs, 1);
26008355f576SJeff Roberson 
2601099a0e58SBosko Milekic 	/*
2602099a0e58SBosko Milekic 	 * We have to call both the zone's init (not the keg's init)
2603099a0e58SBosko Milekic 	 * and the zone's ctor.  This is because the item is going from
2604099a0e58SBosko Milekic 	 * a keg slab directly to the user, and the user is expecting it
2605099a0e58SBosko Milekic 	 * to be both zone-init'd as well as zone-ctor'd.
2606099a0e58SBosko Milekic 	 */
2607b23f72e9SBrian Feldman 	if (zone->uz_init != NULL) {
2608e20a199fSJeff Roberson 		if (zone->uz_init(item, zone->uz_size, flags) != 0) {
26090095a784SJeff Roberson 			zone_free_item(zone, item, udata, SKIP_FINI);
26100095a784SJeff Roberson 			goto fail;
2611b23f72e9SBrian Feldman 		}
2612b23f72e9SBrian Feldman 	}
2613b23f72e9SBrian Feldman 	if (zone->uz_ctor != NULL) {
2614e20a199fSJeff Roberson 		if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
26150095a784SJeff Roberson 			zone_free_item(zone, item, udata, SKIP_DTOR);
26160095a784SJeff Roberson 			goto fail;
2617b23f72e9SBrian Feldman 		}
2618b23f72e9SBrian Feldman 	}
2619ef72505eSJeff Roberson #ifdef INVARIANTS
26200095a784SJeff Roberson 	uma_dbg_alloc(zone, NULL, item);
2621ef72505eSJeff Roberson #endif
26222cc35ff9SJeff Roberson 	if (flags & M_ZERO)
262348343a2fSGleb Smirnoff 		uma_zero_item(item, zone);
26248355f576SJeff Roberson 
26258355f576SJeff Roberson 	return (item);
26260095a784SJeff Roberson 
26270095a784SJeff Roberson fail:
26280095a784SJeff Roberson 	atomic_add_long(&zone->uz_fails, 1);
26290095a784SJeff Roberson 	return (NULL);
26308355f576SJeff Roberson }
26318355f576SJeff Roberson 
26328355f576SJeff Roberson /* See uma.h */
26338355f576SJeff Roberson void
26348355f576SJeff Roberson uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
26358355f576SJeff Roberson {
26368355f576SJeff Roberson 	uma_cache_t cache;
26378355f576SJeff Roberson 	uma_bucket_t bucket;
26384d104ba0SAlexander Motin 	int lockfail;
26398355f576SJeff Roberson 	int cpu;
26408355f576SJeff Roberson 
26418355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC_1
26428355f576SJeff Roberson 	printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone);
26438355f576SJeff Roberson #endif
26443659f747SRobert Watson 	CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread,
26453659f747SRobert Watson 	    zone->uz_name);
26463659f747SRobert Watson 
264720ed0cb0SMatthew D Fleming         /* uma_zfree(..., NULL) does nothing, to match free(9). */
264820ed0cb0SMatthew D Fleming         if (item == NULL)
264920ed0cb0SMatthew D Fleming                 return;
26508d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD
26518d689e04SGleb Smirnoff 	if (is_memguard_addr(item)) {
26528d689e04SGleb Smirnoff 		if (zone->uz_dtor != NULL && zone->uz_dtor != mtrash_dtor)
26538d689e04SGleb Smirnoff 			zone->uz_dtor(item, zone->uz_size, udata);
26548d689e04SGleb Smirnoff 		if (zone->uz_fini != NULL && zone->uz_fini != mtrash_fini)
26558d689e04SGleb Smirnoff 			zone->uz_fini(item, zone->uz_size);
26568d689e04SGleb Smirnoff 		memguard_free(item);
26578d689e04SGleb Smirnoff 		return;
26588d689e04SGleb Smirnoff 	}
26598d689e04SGleb Smirnoff #endif
26605d1ae027SRobert Watson #ifdef INVARIANTS
2661e20a199fSJeff Roberson 	if (zone->uz_flags & UMA_ZONE_MALLOC)
26625d1ae027SRobert Watson 		uma_dbg_free(zone, udata, item);
26635d1ae027SRobert Watson 	else
26645d1ae027SRobert Watson 		uma_dbg_free(zone, NULL, item);
26655d1ae027SRobert Watson #endif
2666fc03d22bSJeff Roberson 	if (zone->uz_dtor != NULL)
2667ef72505eSJeff Roberson 		zone->uz_dtor(item, zone->uz_size, udata);
2668ef72505eSJeff Roberson 
2669af7f9b97SJeff Roberson 	/*
2670af7f9b97SJeff Roberson 	 * The race here is acceptable.  If we miss it we'll just have to wait
2671af7f9b97SJeff Roberson 	 * a little longer for the limits to be reset.
2672af7f9b97SJeff Roberson 	 */
2673e20a199fSJeff Roberson 	if (zone->uz_flags & UMA_ZFLAG_FULL)
2674fc03d22bSJeff Roberson 		goto zfree_item;
2675af7f9b97SJeff Roberson 
26765d1ae027SRobert Watson 	/*
26775d1ae027SRobert Watson 	 * If possible, free to the per-CPU cache.  There are two
26785d1ae027SRobert Watson 	 * requirements for safe access to the per-CPU cache: (1) the thread
26795d1ae027SRobert Watson 	 * accessing the cache must not be preempted or yield during access,
26805d1ae027SRobert Watson 	 * and (2) the thread must not migrate CPUs without switching which
26815d1ae027SRobert Watson 	 * cache it accesses.  We rely on a critical section to prevent
26825d1ae027SRobert Watson 	 * preemption and migration.  We release the critical section in
26835d1ae027SRobert Watson 	 * order to acquire the zone mutex if we are unable to free to the
26845d1ae027SRobert Watson 	 * current cache; when we re-acquire the critical section, we must
26855d1ae027SRobert Watson 	 * detect and handle migration if it has occurred.
26865d1ae027SRobert Watson 	 */
2687a553d4b8SJeff Roberson zfree_restart:
26885d1ae027SRobert Watson 	critical_enter();
26895d1ae027SRobert Watson 	cpu = curcpu;
26908355f576SJeff Roberson 	cache = &zone->uz_cpu[cpu];
26918355f576SJeff Roberson 
26928355f576SJeff Roberson zfree_start:
2693a553d4b8SJeff Roberson 	/*
2694fc03d22bSJeff Roberson 	 * Try to free into the allocbucket first to give LIFO ordering
2695fc03d22bSJeff Roberson 	 * for cache-hot datastructures.  Spill over into the freebucket
2696fc03d22bSJeff Roberson 	 * if necessary.  Alloc will swap them if one runs dry.
2697a553d4b8SJeff Roberson 	 */
2698fc03d22bSJeff Roberson 	bucket = cache->uc_allocbucket;
2699fc03d22bSJeff Roberson 	if (bucket == NULL || bucket->ub_cnt >= bucket->ub_entries)
2700fc03d22bSJeff Roberson 		bucket = cache->uc_freebucket;
2701fc03d22bSJeff Roberson 	if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) {
2702cae33c14SJeff Roberson 		KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL,
27038355f576SJeff Roberson 		    ("uma_zfree: Freeing to non free bucket index."));
2704cae33c14SJeff Roberson 		bucket->ub_bucket[bucket->ub_cnt] = item;
2705cae33c14SJeff Roberson 		bucket->ub_cnt++;
2706773df9abSRobert Watson 		cache->uc_frees++;
27075d1ae027SRobert Watson 		critical_exit();
27088355f576SJeff Roberson 		return;
2709fc03d22bSJeff Roberson 	}
2710fc03d22bSJeff Roberson 
27118355f576SJeff Roberson 	/*
27125d1ae027SRobert Watson 	 * We must go back the zone, which requires acquiring the zone lock,
27135d1ae027SRobert Watson 	 * which in turn means we must release and re-acquire the critical
27145d1ae027SRobert Watson 	 * section.  Since the critical section is released, we may be
27155d1ae027SRobert Watson 	 * preempted or migrate.  As such, make sure not to maintain any
27165d1ae027SRobert Watson 	 * thread-local state specific to the cache from prior to releasing
27175d1ae027SRobert Watson 	 * the critical section.
27188355f576SJeff Roberson 	 */
27195d1ae027SRobert Watson 	critical_exit();
2720fc03d22bSJeff Roberson 	if (zone->uz_count == 0 || bucketdisable)
2721fc03d22bSJeff Roberson 		goto zfree_item;
2722fc03d22bSJeff Roberson 
27234d104ba0SAlexander Motin 	lockfail = 0;
27244d104ba0SAlexander Motin 	if (ZONE_TRYLOCK(zone) == 0) {
27254d104ba0SAlexander Motin 		/* Record contention to size the buckets. */
27268355f576SJeff Roberson 		ZONE_LOCK(zone);
27274d104ba0SAlexander Motin 		lockfail = 1;
27284d104ba0SAlexander Motin 	}
27295d1ae027SRobert Watson 	critical_enter();
27305d1ae027SRobert Watson 	cpu = curcpu;
27315d1ae027SRobert Watson 	cache = &zone->uz_cpu[cpu];
27328355f576SJeff Roberson 
2733fc03d22bSJeff Roberson 	/*
2734fc03d22bSJeff Roberson 	 * Since we have locked the zone we may as well send back our stats.
2735fc03d22bSJeff Roberson 	 */
27360095a784SJeff Roberson 	atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
27370095a784SJeff Roberson 	atomic_add_long(&zone->uz_frees, cache->uc_frees);
2738f4ff923bSRobert Watson 	cache->uc_allocs = 0;
2739f4ff923bSRobert Watson 	cache->uc_frees = 0;
2740f4ff923bSRobert Watson 
27418355f576SJeff Roberson 	bucket = cache->uc_freebucket;
2742fc03d22bSJeff Roberson 	if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) {
2743fc03d22bSJeff Roberson 		ZONE_UNLOCK(zone);
2744fc03d22bSJeff Roberson 		goto zfree_start;
2745fc03d22bSJeff Roberson 	}
27468355f576SJeff Roberson 	cache->uc_freebucket = NULL;
27478355f576SJeff Roberson 
27488355f576SJeff Roberson 	/* Can we throw this on the zone full list? */
27498355f576SJeff Roberson 	if (bucket != NULL) {
27508355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC
27518355f576SJeff Roberson 		printf("uma_zfree: Putting old bucket on the free list.\n");
27528355f576SJeff Roberson #endif
2753cae33c14SJeff Roberson 		/* ub_cnt is pointing to the last free item */
2754cae33c14SJeff Roberson 		KASSERT(bucket->ub_cnt != 0,
27558355f576SJeff Roberson 		    ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n"));
2756fc03d22bSJeff Roberson 		LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link);
27578355f576SJeff Roberson 	}
2758fc03d22bSJeff Roberson 
27595d1ae027SRobert Watson 	/* We are no longer associated with this CPU. */
27605d1ae027SRobert Watson 	critical_exit();
2761a553d4b8SJeff Roberson 
27624d104ba0SAlexander Motin 	/*
27634d104ba0SAlexander Motin 	 * We bump the uz count when the cache size is insufficient to
27644d104ba0SAlexander Motin 	 * handle the working set.
27654d104ba0SAlexander Motin 	 */
27664d104ba0SAlexander Motin 	if (lockfail && zone->uz_count < BUCKET_MAX)
27674d104ba0SAlexander Motin 		zone->uz_count++;
2768a553d4b8SJeff Roberson 	ZONE_UNLOCK(zone);
2769a553d4b8SJeff Roberson 
27708355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC
27718355f576SJeff Roberson 	printf("uma_zfree: Allocating new free bucket.\n");
27728355f576SJeff Roberson #endif
27736fd34d6fSJeff Roberson 	bucket = bucket_alloc(zone, udata, M_NOWAIT);
27744741dcbfSJeff Roberson 	if (bucket) {
2775fc03d22bSJeff Roberson 		critical_enter();
2776fc03d22bSJeff Roberson 		cpu = curcpu;
2777fc03d22bSJeff Roberson 		cache = &zone->uz_cpu[cpu];
2778fc03d22bSJeff Roberson 		if (cache->uc_freebucket == NULL) {
2779fc03d22bSJeff Roberson 			cache->uc_freebucket = bucket;
2780fc03d22bSJeff Roberson 			goto zfree_start;
2781fc03d22bSJeff Roberson 		}
2782fc03d22bSJeff Roberson 		/*
2783fc03d22bSJeff Roberson 		 * We lost the race, start over.  We have to drop our
2784fc03d22bSJeff Roberson 		 * critical section to free the bucket.
2785fc03d22bSJeff Roberson 		 */
2786fc03d22bSJeff Roberson 		critical_exit();
27876fd34d6fSJeff Roberson 		bucket_free(zone, bucket, udata);
2788a553d4b8SJeff Roberson 		goto zfree_restart;
27898355f576SJeff Roberson 	}
27908355f576SJeff Roberson 
2791a553d4b8SJeff Roberson 	/*
2792a553d4b8SJeff Roberson 	 * If nothing else caught this, we'll just do an internal free.
2793a553d4b8SJeff Roberson 	 */
2794fc03d22bSJeff Roberson zfree_item:
27950095a784SJeff Roberson 	zone_free_item(zone, item, udata, SKIP_DTOR);
27968355f576SJeff Roberson 
27978355f576SJeff Roberson 	return;
27988355f576SJeff Roberson }
27998355f576SJeff Roberson 
28008355f576SJeff Roberson static void
28010095a784SJeff Roberson slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
28028355f576SJeff Roberson {
280385dcf349SGleb Smirnoff 	uint8_t freei;
2804099a0e58SBosko Milekic 
28050095a784SJeff Roberson 	mtx_assert(&keg->uk_lock, MA_OWNED);
2806e20a199fSJeff Roberson 	MPASS(keg == slab->us_keg);
28078355f576SJeff Roberson 
28088355f576SJeff Roberson 	/* Do we need to remove from any lists? */
2809099a0e58SBosko Milekic 	if (slab->us_freecount+1 == keg->uk_ipers) {
28108355f576SJeff Roberson 		LIST_REMOVE(slab, us_link);
2811099a0e58SBosko Milekic 		LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
28128355f576SJeff Roberson 	} else if (slab->us_freecount == 0) {
28138355f576SJeff Roberson 		LIST_REMOVE(slab, us_link);
2814099a0e58SBosko Milekic 		LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
28158355f576SJeff Roberson 	}
28168355f576SJeff Roberson 
2817ef72505eSJeff Roberson 	/* Slab management. */
2818ef72505eSJeff Roberson 	freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
2819ef72505eSJeff Roberson 	BIT_SET(SLAB_SETSIZE, freei, &slab->us_free);
28208355f576SJeff Roberson 	slab->us_freecount++;
28218355f576SJeff Roberson 
2822ef72505eSJeff Roberson 	/* Keg statistics. */
2823099a0e58SBosko Milekic 	keg->uk_free++;
28240095a784SJeff Roberson }
28250095a784SJeff Roberson 
28260095a784SJeff Roberson static void
28270095a784SJeff Roberson zone_release(uma_zone_t zone, void **bucket, int cnt)
28280095a784SJeff Roberson {
28290095a784SJeff Roberson 	void *item;
28300095a784SJeff Roberson 	uma_slab_t slab;
28310095a784SJeff Roberson 	uma_keg_t keg;
28320095a784SJeff Roberson 	uint8_t *mem;
28330095a784SJeff Roberson 	int clearfull;
28340095a784SJeff Roberson 	int i;
28358355f576SJeff Roberson 
2836e20a199fSJeff Roberson 	clearfull = 0;
28370095a784SJeff Roberson 	keg = zone_first_keg(zone);
2838af526374SJeff Roberson 	KEG_LOCK(keg);
28390095a784SJeff Roberson 	for (i = 0; i < cnt; i++) {
28400095a784SJeff Roberson 		item = bucket[i];
28410095a784SJeff Roberson 		if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
28420095a784SJeff Roberson 			mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
28430095a784SJeff Roberson 			if (zone->uz_flags & UMA_ZONE_HASH) {
28440095a784SJeff Roberson 				slab = hash_sfind(&keg->uk_hash, mem);
28450095a784SJeff Roberson 			} else {
28460095a784SJeff Roberson 				mem += keg->uk_pgoff;
28470095a784SJeff Roberson 				slab = (uma_slab_t)mem;
28480095a784SJeff Roberson 			}
28490095a784SJeff Roberson 		} else {
28500095a784SJeff Roberson 			slab = vtoslab((vm_offset_t)item);
28510095a784SJeff Roberson 			if (slab->us_keg != keg) {
28520095a784SJeff Roberson 				KEG_UNLOCK(keg);
28530095a784SJeff Roberson 				keg = slab->us_keg;
28540095a784SJeff Roberson 				KEG_LOCK(keg);
28550095a784SJeff Roberson 			}
28560095a784SJeff Roberson 		}
28570095a784SJeff Roberson 		slab_free_item(keg, slab, item);
2858099a0e58SBosko Milekic 		if (keg->uk_flags & UMA_ZFLAG_FULL) {
2859e20a199fSJeff Roberson 			if (keg->uk_pages < keg->uk_maxpages) {
2860099a0e58SBosko Milekic 				keg->uk_flags &= ~UMA_ZFLAG_FULL;
2861e20a199fSJeff Roberson 				clearfull = 1;
2862e20a199fSJeff Roberson 			}
2863af7f9b97SJeff Roberson 
286477380291SMohan Srinivasan 			/*
2865ef72505eSJeff Roberson 			 * We can handle one more allocation. Since we're
2866ef72505eSJeff Roberson 			 * clearing ZFLAG_FULL, wake up all procs blocked
2867ef72505eSJeff Roberson 			 * on pages. This should be uncommon, so keeping this
2868ef72505eSJeff Roberson 			 * simple for now (rather than adding count of blocked
286977380291SMohan Srinivasan 			 * threads etc).
287077380291SMohan Srinivasan 			 */
287177380291SMohan Srinivasan 			wakeup(keg);
2872af7f9b97SJeff Roberson 		}
28730095a784SJeff Roberson 	}
2874af526374SJeff Roberson 	KEG_UNLOCK(keg);
28750095a784SJeff Roberson 	if (clearfull) {
2876af526374SJeff Roberson 		ZONE_LOCK(zone);
2877e20a199fSJeff Roberson 		zone->uz_flags &= ~UMA_ZFLAG_FULL;
2878e20a199fSJeff Roberson 		wakeup(zone);
2879605cbd6aSJeff Roberson 		ZONE_UNLOCK(zone);
2880af526374SJeff Roberson 	}
2881ef72505eSJeff Roberson 
28828355f576SJeff Roberson }
28838355f576SJeff Roberson 
28840095a784SJeff Roberson /*
28850095a784SJeff Roberson  * Frees a single item to any zone.
28860095a784SJeff Roberson  *
28870095a784SJeff Roberson  * Arguments:
28880095a784SJeff Roberson  *	zone   The zone to free to
28890095a784SJeff Roberson  *	item   The item we're freeing
28900095a784SJeff Roberson  *	udata  User supplied data for the dtor
28910095a784SJeff Roberson  *	skip   Skip dtors and finis
28920095a784SJeff Roberson  */
28930095a784SJeff Roberson static void
28940095a784SJeff Roberson zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
28950095a784SJeff Roberson {
28960095a784SJeff Roberson 
28970095a784SJeff Roberson #ifdef INVARIANTS
28980095a784SJeff Roberson 	if (skip == SKIP_NONE) {
28990095a784SJeff Roberson 		if (zone->uz_flags & UMA_ZONE_MALLOC)
29000095a784SJeff Roberson 			uma_dbg_free(zone, udata, item);
29010095a784SJeff Roberson 		else
29020095a784SJeff Roberson 			uma_dbg_free(zone, NULL, item);
29030095a784SJeff Roberson 	}
29040095a784SJeff Roberson #endif
29050095a784SJeff Roberson 	if (skip < SKIP_DTOR && zone->uz_dtor)
29060095a784SJeff Roberson 		zone->uz_dtor(item, zone->uz_size, udata);
29070095a784SJeff Roberson 
29080095a784SJeff Roberson 	if (skip < SKIP_FINI && zone->uz_fini)
29090095a784SJeff Roberson 		zone->uz_fini(item, zone->uz_size);
29100095a784SJeff Roberson 
29110095a784SJeff Roberson 	atomic_add_long(&zone->uz_frees, 1);
29120095a784SJeff Roberson 	zone->uz_release(zone->uz_arg, &item, 1);
29130095a784SJeff Roberson }
29140095a784SJeff Roberson 
29158355f576SJeff Roberson /* See uma.h */
29161c6cae97SLawrence Stewart int
2917736ee590SJeff Roberson uma_zone_set_max(uma_zone_t zone, int nitems)
2918736ee590SJeff Roberson {
2919099a0e58SBosko Milekic 	uma_keg_t keg;
2920099a0e58SBosko Milekic 
2921e20a199fSJeff Roberson 	keg = zone_first_keg(zone);
29220095a784SJeff Roberson 	if (keg == NULL)
29230095a784SJeff Roberson 		return (0);
2924af526374SJeff Roberson 	KEG_LOCK(keg);
2925e20a199fSJeff Roberson 	keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
2926099a0e58SBosko Milekic 	if (keg->uk_maxpages * keg->uk_ipers < nitems)
2927e20a199fSJeff Roberson 		keg->uk_maxpages += keg->uk_ppera;
29281c6cae97SLawrence Stewart 	nitems = keg->uk_maxpages * keg->uk_ipers;
2929af526374SJeff Roberson 	KEG_UNLOCK(keg);
29301c6cae97SLawrence Stewart 
29311c6cae97SLawrence Stewart 	return (nitems);
2932736ee590SJeff Roberson }
2933736ee590SJeff Roberson 
2934736ee590SJeff Roberson /* See uma.h */
2935e49471b0SAndre Oppermann int
2936e49471b0SAndre Oppermann uma_zone_get_max(uma_zone_t zone)
2937e49471b0SAndre Oppermann {
2938e49471b0SAndre Oppermann 	int nitems;
2939e49471b0SAndre Oppermann 	uma_keg_t keg;
2940e49471b0SAndre Oppermann 
2941e49471b0SAndre Oppermann 	keg = zone_first_keg(zone);
29420095a784SJeff Roberson 	if (keg == NULL)
29430095a784SJeff Roberson 		return (0);
2944af526374SJeff Roberson 	KEG_LOCK(keg);
2945e49471b0SAndre Oppermann 	nitems = keg->uk_maxpages * keg->uk_ipers;
2946af526374SJeff Roberson 	KEG_UNLOCK(keg);
2947e49471b0SAndre Oppermann 
2948e49471b0SAndre Oppermann 	return (nitems);
2949e49471b0SAndre Oppermann }
2950e49471b0SAndre Oppermann 
2951e49471b0SAndre Oppermann /* See uma.h */
29522f891cd5SPawel Jakub Dawidek void
29532f891cd5SPawel Jakub Dawidek uma_zone_set_warning(uma_zone_t zone, const char *warning)
29542f891cd5SPawel Jakub Dawidek {
29552f891cd5SPawel Jakub Dawidek 
29562f891cd5SPawel Jakub Dawidek 	ZONE_LOCK(zone);
29572f891cd5SPawel Jakub Dawidek 	zone->uz_warning = warning;
29582f891cd5SPawel Jakub Dawidek 	ZONE_UNLOCK(zone);
29592f891cd5SPawel Jakub Dawidek }
29602f891cd5SPawel Jakub Dawidek 
29612f891cd5SPawel Jakub Dawidek /* See uma.h */
2962c4ae7908SLawrence Stewart int
2963c4ae7908SLawrence Stewart uma_zone_get_cur(uma_zone_t zone)
2964c4ae7908SLawrence Stewart {
2965c4ae7908SLawrence Stewart 	int64_t nitems;
2966c4ae7908SLawrence Stewart 	u_int i;
2967c4ae7908SLawrence Stewart 
2968c4ae7908SLawrence Stewart 	ZONE_LOCK(zone);
2969c4ae7908SLawrence Stewart 	nitems = zone->uz_allocs - zone->uz_frees;
2970c4ae7908SLawrence Stewart 	CPU_FOREACH(i) {
2971c4ae7908SLawrence Stewart 		/*
2972c4ae7908SLawrence Stewart 		 * See the comment in sysctl_vm_zone_stats() regarding the
2973c4ae7908SLawrence Stewart 		 * safety of accessing the per-cpu caches. With the zone lock
2974c4ae7908SLawrence Stewart 		 * held, it is safe, but can potentially result in stale data.
2975c4ae7908SLawrence Stewart 		 */
2976c4ae7908SLawrence Stewart 		nitems += zone->uz_cpu[i].uc_allocs -
2977c4ae7908SLawrence Stewart 		    zone->uz_cpu[i].uc_frees;
2978c4ae7908SLawrence Stewart 	}
2979c4ae7908SLawrence Stewart 	ZONE_UNLOCK(zone);
2980c4ae7908SLawrence Stewart 
2981c4ae7908SLawrence Stewart 	return (nitems < 0 ? 0 : nitems);
2982c4ae7908SLawrence Stewart }
2983c4ae7908SLawrence Stewart 
2984c4ae7908SLawrence Stewart /* See uma.h */
2985736ee590SJeff Roberson void
2986099a0e58SBosko Milekic uma_zone_set_init(uma_zone_t zone, uma_init uminit)
2987099a0e58SBosko Milekic {
2988e20a199fSJeff Roberson 	uma_keg_t keg;
2989e20a199fSJeff Roberson 
2990e20a199fSJeff Roberson 	keg = zone_first_keg(zone);
29910095a784SJeff Roberson 	KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2992af526374SJeff Roberson 	KEG_LOCK(keg);
2993e20a199fSJeff Roberson 	KASSERT(keg->uk_pages == 0,
2994099a0e58SBosko Milekic 	    ("uma_zone_set_init on non-empty keg"));
2995e20a199fSJeff Roberson 	keg->uk_init = uminit;
2996af526374SJeff Roberson 	KEG_UNLOCK(keg);
2997099a0e58SBosko Milekic }
2998099a0e58SBosko Milekic 
2999099a0e58SBosko Milekic /* See uma.h */
3000099a0e58SBosko Milekic void
3001099a0e58SBosko Milekic uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
3002099a0e58SBosko Milekic {
3003e20a199fSJeff Roberson 	uma_keg_t keg;
3004e20a199fSJeff Roberson 
3005e20a199fSJeff Roberson 	keg = zone_first_keg(zone);
30060095a784SJeff Roberson 	KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
3007af526374SJeff Roberson 	KEG_LOCK(keg);
3008e20a199fSJeff Roberson 	KASSERT(keg->uk_pages == 0,
3009099a0e58SBosko Milekic 	    ("uma_zone_set_fini on non-empty keg"));
3010e20a199fSJeff Roberson 	keg->uk_fini = fini;
3011af526374SJeff Roberson 	KEG_UNLOCK(keg);
3012099a0e58SBosko Milekic }
3013099a0e58SBosko Milekic 
3014099a0e58SBosko Milekic /* See uma.h */
3015099a0e58SBosko Milekic void
3016099a0e58SBosko Milekic uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
3017099a0e58SBosko Milekic {
3018af526374SJeff Roberson 
3019099a0e58SBosko Milekic 	ZONE_LOCK(zone);
3020e20a199fSJeff Roberson 	KASSERT(zone_first_keg(zone)->uk_pages == 0,
3021099a0e58SBosko Milekic 	    ("uma_zone_set_zinit on non-empty keg"));
3022099a0e58SBosko Milekic 	zone->uz_init = zinit;
3023099a0e58SBosko Milekic 	ZONE_UNLOCK(zone);
3024099a0e58SBosko Milekic }
3025099a0e58SBosko Milekic 
3026099a0e58SBosko Milekic /* See uma.h */
3027099a0e58SBosko Milekic void
3028099a0e58SBosko Milekic uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
3029099a0e58SBosko Milekic {
3030af526374SJeff Roberson 
3031099a0e58SBosko Milekic 	ZONE_LOCK(zone);
3032e20a199fSJeff Roberson 	KASSERT(zone_first_keg(zone)->uk_pages == 0,
3033099a0e58SBosko Milekic 	    ("uma_zone_set_zfini on non-empty keg"));
3034099a0e58SBosko Milekic 	zone->uz_fini = zfini;
3035099a0e58SBosko Milekic 	ZONE_UNLOCK(zone);
3036099a0e58SBosko Milekic }
3037099a0e58SBosko Milekic 
3038099a0e58SBosko Milekic /* See uma.h */
3039b23f72e9SBrian Feldman /* XXX uk_freef is not actually used with the zone locked */
3040099a0e58SBosko Milekic void
30418355f576SJeff Roberson uma_zone_set_freef(uma_zone_t zone, uma_free freef)
30428355f576SJeff Roberson {
30430095a784SJeff Roberson 	uma_keg_t keg;
3044e20a199fSJeff Roberson 
30450095a784SJeff Roberson 	keg = zone_first_keg(zone);
30460095a784SJeff Roberson 	KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
3047af526374SJeff Roberson 	KEG_LOCK(keg);
30480095a784SJeff Roberson 	keg->uk_freef = freef;
3049af526374SJeff Roberson 	KEG_UNLOCK(keg);
30508355f576SJeff Roberson }
30518355f576SJeff Roberson 
30528355f576SJeff Roberson /* See uma.h */
3053b23f72e9SBrian Feldman /* XXX uk_allocf is not actually used with the zone locked */
30548355f576SJeff Roberson void
30558355f576SJeff Roberson uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
30568355f576SJeff Roberson {
3057e20a199fSJeff Roberson 	uma_keg_t keg;
3058e20a199fSJeff Roberson 
3059e20a199fSJeff Roberson 	keg = zone_first_keg(zone);
3060af526374SJeff Roberson 	KEG_LOCK(keg);
3061e20a199fSJeff Roberson 	keg->uk_allocf = allocf;
3062af526374SJeff Roberson 	KEG_UNLOCK(keg);
30638355f576SJeff Roberson }
30648355f576SJeff Roberson 
30658355f576SJeff Roberson /* See uma.h */
30666fd34d6fSJeff Roberson void
30676fd34d6fSJeff Roberson uma_zone_reserve(uma_zone_t zone, int items)
30686fd34d6fSJeff Roberson {
30696fd34d6fSJeff Roberson 	uma_keg_t keg;
30706fd34d6fSJeff Roberson 
30716fd34d6fSJeff Roberson 	keg = zone_first_keg(zone);
30726fd34d6fSJeff Roberson 	if (keg == NULL)
30736fd34d6fSJeff Roberson 		return;
30746fd34d6fSJeff Roberson 	KEG_LOCK(keg);
30756fd34d6fSJeff Roberson 	keg->uk_reserve = items;
30766fd34d6fSJeff Roberson 	KEG_UNLOCK(keg);
30776fd34d6fSJeff Roberson 
30786fd34d6fSJeff Roberson 	return;
30796fd34d6fSJeff Roberson }
30806fd34d6fSJeff Roberson 
30816fd34d6fSJeff Roberson /* See uma.h */
30828355f576SJeff Roberson int
3083a4915c21SAttilio Rao uma_zone_reserve_kva(uma_zone_t zone, int count)
30848355f576SJeff Roberson {
3085099a0e58SBosko Milekic 	uma_keg_t keg;
30868355f576SJeff Roberson 	vm_offset_t kva;
3087099a0e58SBosko Milekic 	int pages;
30888355f576SJeff Roberson 
3089e20a199fSJeff Roberson 	keg = zone_first_keg(zone);
30900095a784SJeff Roberson 	if (keg == NULL)
30910095a784SJeff Roberson 		return (0);
3092099a0e58SBosko Milekic 	pages = count / keg->uk_ipers;
30938355f576SJeff Roberson 
3094099a0e58SBosko Milekic 	if (pages * keg->uk_ipers < count)
30958355f576SJeff Roberson 		pages++;
3096a553d4b8SJeff Roberson 
3097a4915c21SAttilio Rao #ifdef UMA_MD_SMALL_ALLOC
3098a4915c21SAttilio Rao 	if (keg->uk_ppera > 1) {
3099a4915c21SAttilio Rao #else
3100a4915c21SAttilio Rao 	if (1) {
3101a4915c21SAttilio Rao #endif
31025df87b21SJeff Roberson 		kva = kva_alloc(pages * UMA_SLAB_SIZE);
3103d1f42ac2SAlan Cox 		if (kva == 0)
31048355f576SJeff Roberson 			return (0);
3105a4915c21SAttilio Rao 	} else
3106a4915c21SAttilio Rao 		kva = 0;
3107af526374SJeff Roberson 	KEG_LOCK(keg);
3108099a0e58SBosko Milekic 	keg->uk_kva = kva;
3109a4915c21SAttilio Rao 	keg->uk_offset = 0;
3110099a0e58SBosko Milekic 	keg->uk_maxpages = pages;
3111a4915c21SAttilio Rao #ifdef UMA_MD_SMALL_ALLOC
3112a4915c21SAttilio Rao 	keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
3113a4915c21SAttilio Rao #else
3114a4915c21SAttilio Rao 	keg->uk_allocf = noobj_alloc;
3115a4915c21SAttilio Rao #endif
31166fd34d6fSJeff Roberson 	keg->uk_flags |= UMA_ZONE_NOFREE;
3117af526374SJeff Roberson 	KEG_UNLOCK(keg);
3118af526374SJeff Roberson 
31198355f576SJeff Roberson 	return (1);
31208355f576SJeff Roberson }
31218355f576SJeff Roberson 
31228355f576SJeff Roberson /* See uma.h */
31238355f576SJeff Roberson void
31248355f576SJeff Roberson uma_prealloc(uma_zone_t zone, int items)
31258355f576SJeff Roberson {
31268355f576SJeff Roberson 	int slabs;
31278355f576SJeff Roberson 	uma_slab_t slab;
3128099a0e58SBosko Milekic 	uma_keg_t keg;
31298355f576SJeff Roberson 
3130e20a199fSJeff Roberson 	keg = zone_first_keg(zone);
31310095a784SJeff Roberson 	if (keg == NULL)
31320095a784SJeff Roberson 		return;
3133af526374SJeff Roberson 	KEG_LOCK(keg);
3134099a0e58SBosko Milekic 	slabs = items / keg->uk_ipers;
3135099a0e58SBosko Milekic 	if (slabs * keg->uk_ipers < items)
31368355f576SJeff Roberson 		slabs++;
31378355f576SJeff Roberson 	while (slabs > 0) {
3138e20a199fSJeff Roberson 		slab = keg_alloc_slab(keg, zone, M_WAITOK);
3139e20a199fSJeff Roberson 		if (slab == NULL)
3140e20a199fSJeff Roberson 			break;
3141e20a199fSJeff Roberson 		MPASS(slab->us_keg == keg);
3142099a0e58SBosko Milekic 		LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
31438355f576SJeff Roberson 		slabs--;
31448355f576SJeff Roberson 	}
3145af526374SJeff Roberson 	KEG_UNLOCK(keg);
31468355f576SJeff Roberson }
31478355f576SJeff Roberson 
31488355f576SJeff Roberson /* See uma.h */
314985dcf349SGleb Smirnoff uint32_t *
3150099a0e58SBosko Milekic uma_find_refcnt(uma_zone_t zone, void *item)
3151099a0e58SBosko Milekic {
3152ab14a3f7SBrian Feldman 	uma_slabrefcnt_t slabref;
3153ef72505eSJeff Roberson 	uma_slab_t slab;
3154099a0e58SBosko Milekic 	uma_keg_t keg;
315585dcf349SGleb Smirnoff 	uint32_t *refcnt;
3156099a0e58SBosko Milekic 	int idx;
3157099a0e58SBosko Milekic 
3158ef72505eSJeff Roberson 	slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
3159ef72505eSJeff Roberson 	slabref = (uma_slabrefcnt_t)slab;
3160ef72505eSJeff Roberson 	keg = slab->us_keg;
3161ef72505eSJeff Roberson 	KASSERT(keg->uk_flags & UMA_ZONE_REFCNT,
3162099a0e58SBosko Milekic 	    ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT"));
3163ef72505eSJeff Roberson 	idx = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
3164ef72505eSJeff Roberson 	refcnt = &slabref->us_refcnt[idx];
3165099a0e58SBosko Milekic 	return refcnt;
3166099a0e58SBosko Milekic }
3167099a0e58SBosko Milekic 
3168099a0e58SBosko Milekic /* See uma.h */
31698355f576SJeff Roberson void
31708355f576SJeff Roberson uma_reclaim(void)
31718355f576SJeff Roberson {
31728355f576SJeff Roberson #ifdef UMA_DEBUG
31738355f576SJeff Roberson 	printf("UMA: vm asked us to release pages!\n");
31748355f576SJeff Roberson #endif
317586bbae32SJeff Roberson 	bucket_enable();
31768355f576SJeff Roberson 	zone_foreach(zone_drain);
3177a2de44abSAlexander Motin 	if (vm_page_count_min()) {
3178a2de44abSAlexander Motin 		cache_drain_safe(NULL);
3179a2de44abSAlexander Motin 		zone_foreach(zone_drain);
3180a2de44abSAlexander Motin 	}
31818355f576SJeff Roberson 	/*
31828355f576SJeff Roberson 	 * Some slabs may have been freed but this zone will be visited early
31838355f576SJeff Roberson 	 * we visit again so that we can free pages that are empty once other
31848355f576SJeff Roberson 	 * zones are drained.  We have to do the same for buckets.
31858355f576SJeff Roberson 	 */
31869643769aSJeff Roberson 	zone_drain(slabzone);
3187099a0e58SBosko Milekic 	zone_drain(slabrefzone);
3188cae33c14SJeff Roberson 	bucket_zone_drain();
31898355f576SJeff Roberson }
31908355f576SJeff Roberson 
3191663b416fSJohn Baldwin /* See uma.h */
3192663b416fSJohn Baldwin int
3193663b416fSJohn Baldwin uma_zone_exhausted(uma_zone_t zone)
3194663b416fSJohn Baldwin {
3195663b416fSJohn Baldwin 	int full;
3196663b416fSJohn Baldwin 
3197663b416fSJohn Baldwin 	ZONE_LOCK(zone);
3198e20a199fSJeff Roberson 	full = (zone->uz_flags & UMA_ZFLAG_FULL);
3199663b416fSJohn Baldwin 	ZONE_UNLOCK(zone);
3200663b416fSJohn Baldwin 	return (full);
3201663b416fSJohn Baldwin }
3202663b416fSJohn Baldwin 
32036c125b8dSMohan Srinivasan int
32046c125b8dSMohan Srinivasan uma_zone_exhausted_nolock(uma_zone_t zone)
32056c125b8dSMohan Srinivasan {
3206e20a199fSJeff Roberson 	return (zone->uz_flags & UMA_ZFLAG_FULL);
32076c125b8dSMohan Srinivasan }
32086c125b8dSMohan Srinivasan 
32098355f576SJeff Roberson void *
32108355f576SJeff Roberson uma_large_malloc(int size, int wait)
32118355f576SJeff Roberson {
32128355f576SJeff Roberson 	void *mem;
32138355f576SJeff Roberson 	uma_slab_t slab;
321485dcf349SGleb Smirnoff 	uint8_t flags;
32158355f576SJeff Roberson 
3216e20a199fSJeff Roberson 	slab = zone_alloc_item(slabzone, NULL, wait);
32178355f576SJeff Roberson 	if (slab == NULL)
32188355f576SJeff Roberson 		return (NULL);
32198355f576SJeff Roberson 	mem = page_alloc(NULL, size, &flags, wait);
32208355f576SJeff Roberson 	if (mem) {
322199571dc3SJeff Roberson 		vsetslab((vm_offset_t)mem, slab);
32228355f576SJeff Roberson 		slab->us_data = mem;
32238355f576SJeff Roberson 		slab->us_flags = flags | UMA_SLAB_MALLOC;
32248355f576SJeff Roberson 		slab->us_size = size;
32258355f576SJeff Roberson 	} else {
32260095a784SJeff Roberson 		zone_free_item(slabzone, slab, NULL, SKIP_NONE);
32278355f576SJeff Roberson 	}
32288355f576SJeff Roberson 
32298355f576SJeff Roberson 	return (mem);
32308355f576SJeff Roberson }
32318355f576SJeff Roberson 
32328355f576SJeff Roberson void
32338355f576SJeff Roberson uma_large_free(uma_slab_t slab)
32348355f576SJeff Roberson {
3235c325e866SKonstantin Belousov 
32368355f576SJeff Roberson 	page_free(slab->us_data, slab->us_size, slab->us_flags);
32370095a784SJeff Roberson 	zone_free_item(slabzone, slab, NULL, SKIP_NONE);
32388355f576SJeff Roberson }
32398355f576SJeff Roberson 
324048343a2fSGleb Smirnoff static void
324148343a2fSGleb Smirnoff uma_zero_item(void *item, uma_zone_t zone)
324248343a2fSGleb Smirnoff {
324348343a2fSGleb Smirnoff 
324448343a2fSGleb Smirnoff 	if (zone->uz_flags & UMA_ZONE_PCPU) {
324548343a2fSGleb Smirnoff 		for (int i = 0; i < mp_ncpus; i++)
324648343a2fSGleb Smirnoff 			bzero(zpcpu_get_cpu(item, i), zone->uz_size);
324748343a2fSGleb Smirnoff 	} else
324848343a2fSGleb Smirnoff 		bzero(item, zone->uz_size);
324948343a2fSGleb Smirnoff }
325048343a2fSGleb Smirnoff 
32518355f576SJeff Roberson void
32528355f576SJeff Roberson uma_print_stats(void)
32538355f576SJeff Roberson {
32548355f576SJeff Roberson 	zone_foreach(uma_print_zone);
32558355f576SJeff Roberson }
32568355f576SJeff Roberson 
3257504d5de3SJeff Roberson static void
3258504d5de3SJeff Roberson slab_print(uma_slab_t slab)
3259504d5de3SJeff Roberson {
3260ef72505eSJeff Roberson 	printf("slab: keg %p, data %p, freecount %d\n",
3261ef72505eSJeff Roberson 		slab->us_keg, slab->us_data, slab->us_freecount);
3262504d5de3SJeff Roberson }
3263504d5de3SJeff Roberson 
3264504d5de3SJeff Roberson static void
3265504d5de3SJeff Roberson cache_print(uma_cache_t cache)
3266504d5de3SJeff Roberson {
3267504d5de3SJeff Roberson 	printf("alloc: %p(%d), free: %p(%d)\n",
3268504d5de3SJeff Roberson 		cache->uc_allocbucket,
3269504d5de3SJeff Roberson 		cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
3270504d5de3SJeff Roberson 		cache->uc_freebucket,
3271504d5de3SJeff Roberson 		cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0);
3272504d5de3SJeff Roberson }
3273504d5de3SJeff Roberson 
3274e20a199fSJeff Roberson static void
3275e20a199fSJeff Roberson uma_print_keg(uma_keg_t keg)
32768355f576SJeff Roberson {
3277504d5de3SJeff Roberson 	uma_slab_t slab;
3278504d5de3SJeff Roberson 
32790b80c1e4SEitan Adler 	printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
3280e20a199fSJeff Roberson 	    "out %d free %d limit %d\n",
3281e20a199fSJeff Roberson 	    keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
3282099a0e58SBosko Milekic 	    keg->uk_ipers, keg->uk_ppera,
3283e20a199fSJeff Roberson 	    (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free,
3284e20a199fSJeff Roberson 	    (keg->uk_maxpages / keg->uk_ppera) * keg->uk_ipers);
3285504d5de3SJeff Roberson 	printf("Part slabs:\n");
3286099a0e58SBosko Milekic 	LIST_FOREACH(slab, &keg->uk_part_slab, us_link)
3287504d5de3SJeff Roberson 		slab_print(slab);
3288504d5de3SJeff Roberson 	printf("Free slabs:\n");
3289099a0e58SBosko Milekic 	LIST_FOREACH(slab, &keg->uk_free_slab, us_link)
3290504d5de3SJeff Roberson 		slab_print(slab);
3291504d5de3SJeff Roberson 	printf("Full slabs:\n");
3292099a0e58SBosko Milekic 	LIST_FOREACH(slab, &keg->uk_full_slab, us_link)
3293504d5de3SJeff Roberson 		slab_print(slab);
3294e20a199fSJeff Roberson }
3295e20a199fSJeff Roberson 
3296e20a199fSJeff Roberson void
3297e20a199fSJeff Roberson uma_print_zone(uma_zone_t zone)
3298e20a199fSJeff Roberson {
3299e20a199fSJeff Roberson 	uma_cache_t cache;
3300e20a199fSJeff Roberson 	uma_klink_t kl;
3301e20a199fSJeff Roberson 	int i;
3302e20a199fSJeff Roberson 
33030b80c1e4SEitan Adler 	printf("zone: %s(%p) size %d flags %#x\n",
3304e20a199fSJeff Roberson 	    zone->uz_name, zone, zone->uz_size, zone->uz_flags);
3305e20a199fSJeff Roberson 	LIST_FOREACH(kl, &zone->uz_kegs, kl_link)
3306e20a199fSJeff Roberson 		uma_print_keg(kl->kl_keg);
33073aa6d94eSJohn Baldwin 	CPU_FOREACH(i) {
3308504d5de3SJeff Roberson 		cache = &zone->uz_cpu[i];
3309504d5de3SJeff Roberson 		printf("CPU %d Cache:\n", i);
3310504d5de3SJeff Roberson 		cache_print(cache);
3311504d5de3SJeff Roberson 	}
33128355f576SJeff Roberson }
33138355f576SJeff Roberson 
3314a0d4b0aeSRobert Watson #ifdef DDB
33158355f576SJeff Roberson /*
33167a52a97eSRobert Watson  * Generate statistics across both the zone and its per-cpu cache's.  Return
33177a52a97eSRobert Watson  * desired statistics if the pointer is non-NULL for that statistic.
33187a52a97eSRobert Watson  *
33197a52a97eSRobert Watson  * Note: does not update the zone statistics, as it can't safely clear the
33207a52a97eSRobert Watson  * per-CPU cache statistic.
33217a52a97eSRobert Watson  *
33227a52a97eSRobert Watson  * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't
33237a52a97eSRobert Watson  * safe from off-CPU; we should modify the caches to track this information
33247a52a97eSRobert Watson  * directly so that we don't have to.
33257a52a97eSRobert Watson  */
33267a52a97eSRobert Watson static void
332785dcf349SGleb Smirnoff uma_zone_sumstat(uma_zone_t z, int *cachefreep, uint64_t *allocsp,
332885dcf349SGleb Smirnoff     uint64_t *freesp, uint64_t *sleepsp)
33297a52a97eSRobert Watson {
33307a52a97eSRobert Watson 	uma_cache_t cache;
333185dcf349SGleb Smirnoff 	uint64_t allocs, frees, sleeps;
33327a52a97eSRobert Watson 	int cachefree, cpu;
33337a52a97eSRobert Watson 
3334bf965959SSean Bruno 	allocs = frees = sleeps = 0;
33357a52a97eSRobert Watson 	cachefree = 0;
33363aa6d94eSJohn Baldwin 	CPU_FOREACH(cpu) {
33377a52a97eSRobert Watson 		cache = &z->uz_cpu[cpu];
33387a52a97eSRobert Watson 		if (cache->uc_allocbucket != NULL)
33397a52a97eSRobert Watson 			cachefree += cache->uc_allocbucket->ub_cnt;
33407a52a97eSRobert Watson 		if (cache->uc_freebucket != NULL)
33417a52a97eSRobert Watson 			cachefree += cache->uc_freebucket->ub_cnt;
33427a52a97eSRobert Watson 		allocs += cache->uc_allocs;
33437a52a97eSRobert Watson 		frees += cache->uc_frees;
33447a52a97eSRobert Watson 	}
33457a52a97eSRobert Watson 	allocs += z->uz_allocs;
33467a52a97eSRobert Watson 	frees += z->uz_frees;
3347bf965959SSean Bruno 	sleeps += z->uz_sleeps;
33487a52a97eSRobert Watson 	if (cachefreep != NULL)
33497a52a97eSRobert Watson 		*cachefreep = cachefree;
33507a52a97eSRobert Watson 	if (allocsp != NULL)
33517a52a97eSRobert Watson 		*allocsp = allocs;
33527a52a97eSRobert Watson 	if (freesp != NULL)
33537a52a97eSRobert Watson 		*freesp = frees;
3354bf965959SSean Bruno 	if (sleepsp != NULL)
3355bf965959SSean Bruno 		*sleepsp = sleeps;
33567a52a97eSRobert Watson }
3357a0d4b0aeSRobert Watson #endif /* DDB */
33587a52a97eSRobert Watson 
33597a52a97eSRobert Watson static int
33607a52a97eSRobert Watson sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS)
33617a52a97eSRobert Watson {
33627a52a97eSRobert Watson 	uma_keg_t kz;
33637a52a97eSRobert Watson 	uma_zone_t z;
33647a52a97eSRobert Watson 	int count;
33657a52a97eSRobert Watson 
33667a52a97eSRobert Watson 	count = 0;
33677a52a97eSRobert Watson 	mtx_lock(&uma_mtx);
33687a52a97eSRobert Watson 	LIST_FOREACH(kz, &uma_kegs, uk_link) {
33697a52a97eSRobert Watson 		LIST_FOREACH(z, &kz->uk_zones, uz_link)
33707a52a97eSRobert Watson 			count++;
33717a52a97eSRobert Watson 	}
33727a52a97eSRobert Watson 	mtx_unlock(&uma_mtx);
33737a52a97eSRobert Watson 	return (sysctl_handle_int(oidp, &count, 0, req));
33747a52a97eSRobert Watson }
33757a52a97eSRobert Watson 
33767a52a97eSRobert Watson static int
33777a52a97eSRobert Watson sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
33787a52a97eSRobert Watson {
33797a52a97eSRobert Watson 	struct uma_stream_header ush;
33807a52a97eSRobert Watson 	struct uma_type_header uth;
33817a52a97eSRobert Watson 	struct uma_percpu_stat ups;
33827a52a97eSRobert Watson 	uma_bucket_t bucket;
33837a52a97eSRobert Watson 	struct sbuf sbuf;
33847a52a97eSRobert Watson 	uma_cache_t cache;
3385e20a199fSJeff Roberson 	uma_klink_t kl;
33867a52a97eSRobert Watson 	uma_keg_t kz;
33877a52a97eSRobert Watson 	uma_zone_t z;
3388e20a199fSJeff Roberson 	uma_keg_t k;
33894e657159SMatthew D Fleming 	int count, error, i;
33907a52a97eSRobert Watson 
339100f0e671SMatthew D Fleming 	error = sysctl_wire_old_buffer(req, 0);
339200f0e671SMatthew D Fleming 	if (error != 0)
339300f0e671SMatthew D Fleming 		return (error);
33944e657159SMatthew D Fleming 	sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
33954e657159SMatthew D Fleming 
3396404a593eSMatthew D Fleming 	count = 0;
33974e657159SMatthew D Fleming 	mtx_lock(&uma_mtx);
33987a52a97eSRobert Watson 	LIST_FOREACH(kz, &uma_kegs, uk_link) {
33997a52a97eSRobert Watson 		LIST_FOREACH(z, &kz->uk_zones, uz_link)
34007a52a97eSRobert Watson 			count++;
34017a52a97eSRobert Watson 	}
34027a52a97eSRobert Watson 
34037a52a97eSRobert Watson 	/*
34047a52a97eSRobert Watson 	 * Insert stream header.
34057a52a97eSRobert Watson 	 */
34067a52a97eSRobert Watson 	bzero(&ush, sizeof(ush));
34077a52a97eSRobert Watson 	ush.ush_version = UMA_STREAM_VERSION;
3408ab3a57c0SRobert Watson 	ush.ush_maxcpus = (mp_maxid + 1);
34097a52a97eSRobert Watson 	ush.ush_count = count;
34104e657159SMatthew D Fleming 	(void)sbuf_bcat(&sbuf, &ush, sizeof(ush));
34117a52a97eSRobert Watson 
34127a52a97eSRobert Watson 	LIST_FOREACH(kz, &uma_kegs, uk_link) {
34137a52a97eSRobert Watson 		LIST_FOREACH(z, &kz->uk_zones, uz_link) {
34147a52a97eSRobert Watson 			bzero(&uth, sizeof(uth));
34157a52a97eSRobert Watson 			ZONE_LOCK(z);
3416cbbb4a00SRobert Watson 			strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME);
34177a52a97eSRobert Watson 			uth.uth_align = kz->uk_align;
34187a52a97eSRobert Watson 			uth.uth_size = kz->uk_size;
34197a52a97eSRobert Watson 			uth.uth_rsize = kz->uk_rsize;
3420e20a199fSJeff Roberson 			LIST_FOREACH(kl, &z->uz_kegs, kl_link) {
3421e20a199fSJeff Roberson 				k = kl->kl_keg;
3422e20a199fSJeff Roberson 				uth.uth_maxpages += k->uk_maxpages;
3423e20a199fSJeff Roberson 				uth.uth_pages += k->uk_pages;
3424e20a199fSJeff Roberson 				uth.uth_keg_free += k->uk_free;
3425e20a199fSJeff Roberson 				uth.uth_limit = (k->uk_maxpages / k->uk_ppera)
3426e20a199fSJeff Roberson 				    * k->uk_ipers;
3427e20a199fSJeff Roberson 			}
3428cbbb4a00SRobert Watson 
3429cbbb4a00SRobert Watson 			/*
3430cbbb4a00SRobert Watson 			 * A zone is secondary is it is not the first entry
3431cbbb4a00SRobert Watson 			 * on the keg's zone list.
3432cbbb4a00SRobert Watson 			 */
3433e20a199fSJeff Roberson 			if ((z->uz_flags & UMA_ZONE_SECONDARY) &&
3434cbbb4a00SRobert Watson 			    (LIST_FIRST(&kz->uk_zones) != z))
3435cbbb4a00SRobert Watson 				uth.uth_zone_flags = UTH_ZONE_SECONDARY;
3436cbbb4a00SRobert Watson 
3437fc03d22bSJeff Roberson 			LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
34387a52a97eSRobert Watson 				uth.uth_zone_free += bucket->ub_cnt;
34397a52a97eSRobert Watson 			uth.uth_allocs = z->uz_allocs;
34407a52a97eSRobert Watson 			uth.uth_frees = z->uz_frees;
34412019094aSRobert Watson 			uth.uth_fails = z->uz_fails;
3442bf965959SSean Bruno 			uth.uth_sleeps = z->uz_sleeps;
34434e657159SMatthew D Fleming 			(void)sbuf_bcat(&sbuf, &uth, sizeof(uth));
34447a52a97eSRobert Watson 			/*
34452450bbb8SRobert Watson 			 * While it is not normally safe to access the cache
34462450bbb8SRobert Watson 			 * bucket pointers while not on the CPU that owns the
34472450bbb8SRobert Watson 			 * cache, we only allow the pointers to be exchanged
34482450bbb8SRobert Watson 			 * without the zone lock held, not invalidated, so
34492450bbb8SRobert Watson 			 * accept the possible race associated with bucket
34502450bbb8SRobert Watson 			 * exchange during monitoring.
34517a52a97eSRobert Watson 			 */
3452ab3a57c0SRobert Watson 			for (i = 0; i < (mp_maxid + 1); i++) {
34537a52a97eSRobert Watson 				bzero(&ups, sizeof(ups));
34547a52a97eSRobert Watson 				if (kz->uk_flags & UMA_ZFLAG_INTERNAL)
34557a52a97eSRobert Watson 					goto skip;
3456082dc776SRobert Watson 				if (CPU_ABSENT(i))
3457082dc776SRobert Watson 					goto skip;
34587a52a97eSRobert Watson 				cache = &z->uz_cpu[i];
34597a52a97eSRobert Watson 				if (cache->uc_allocbucket != NULL)
34607a52a97eSRobert Watson 					ups.ups_cache_free +=
34617a52a97eSRobert Watson 					    cache->uc_allocbucket->ub_cnt;
34627a52a97eSRobert Watson 				if (cache->uc_freebucket != NULL)
34637a52a97eSRobert Watson 					ups.ups_cache_free +=
34647a52a97eSRobert Watson 					    cache->uc_freebucket->ub_cnt;
34657a52a97eSRobert Watson 				ups.ups_allocs = cache->uc_allocs;
34667a52a97eSRobert Watson 				ups.ups_frees = cache->uc_frees;
34677a52a97eSRobert Watson skip:
34684e657159SMatthew D Fleming 				(void)sbuf_bcat(&sbuf, &ups, sizeof(ups));
34697a52a97eSRobert Watson 			}
34702450bbb8SRobert Watson 			ZONE_UNLOCK(z);
34717a52a97eSRobert Watson 		}
34727a52a97eSRobert Watson 	}
34737a52a97eSRobert Watson 	mtx_unlock(&uma_mtx);
34744e657159SMatthew D Fleming 	error = sbuf_finish(&sbuf);
34754e657159SMatthew D Fleming 	sbuf_delete(&sbuf);
34767a52a97eSRobert Watson 	return (error);
34777a52a97eSRobert Watson }
347848c5777eSRobert Watson 
34790a5a3ccbSGleb Smirnoff int
34800a5a3ccbSGleb Smirnoff sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS)
34810a5a3ccbSGleb Smirnoff {
34820a5a3ccbSGleb Smirnoff 	uma_zone_t zone = *(uma_zone_t *)arg1;
34830a5a3ccbSGleb Smirnoff 	int error, max, old;
34840a5a3ccbSGleb Smirnoff 
34850a5a3ccbSGleb Smirnoff 	old = max = uma_zone_get_max(zone);
34860a5a3ccbSGleb Smirnoff 	error = sysctl_handle_int(oidp, &max, 0, req);
34870a5a3ccbSGleb Smirnoff 	if (error || !req->newptr)
34880a5a3ccbSGleb Smirnoff 		return (error);
34890a5a3ccbSGleb Smirnoff 
34900a5a3ccbSGleb Smirnoff 	if (max < old)
34910a5a3ccbSGleb Smirnoff 		return (EINVAL);
34920a5a3ccbSGleb Smirnoff 
34930a5a3ccbSGleb Smirnoff 	uma_zone_set_max(zone, max);
34940a5a3ccbSGleb Smirnoff 
34950a5a3ccbSGleb Smirnoff 	return (0);
34960a5a3ccbSGleb Smirnoff }
34970a5a3ccbSGleb Smirnoff 
34980a5a3ccbSGleb Smirnoff int
34990a5a3ccbSGleb Smirnoff sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS)
35000a5a3ccbSGleb Smirnoff {
35010a5a3ccbSGleb Smirnoff 	uma_zone_t zone = *(uma_zone_t *)arg1;
35020a5a3ccbSGleb Smirnoff 	int cur;
35030a5a3ccbSGleb Smirnoff 
35040a5a3ccbSGleb Smirnoff 	cur = uma_zone_get_cur(zone);
35050a5a3ccbSGleb Smirnoff 	return (sysctl_handle_int(oidp, &cur, 0, req));
35060a5a3ccbSGleb Smirnoff }
35070a5a3ccbSGleb Smirnoff 
350848c5777eSRobert Watson #ifdef DDB
350948c5777eSRobert Watson DB_SHOW_COMMAND(uma, db_show_uma)
351048c5777eSRobert Watson {
351185dcf349SGleb Smirnoff 	uint64_t allocs, frees, sleeps;
351248c5777eSRobert Watson 	uma_bucket_t bucket;
351348c5777eSRobert Watson 	uma_keg_t kz;
351448c5777eSRobert Watson 	uma_zone_t z;
351548c5777eSRobert Watson 	int cachefree;
351648c5777eSRobert Watson 
351703175483SAlexander Motin 	db_printf("%18s %8s %8s %8s %12s %8s %8s\n", "Zone", "Size", "Used",
351803175483SAlexander Motin 	    "Free", "Requests", "Sleeps", "Bucket");
351948c5777eSRobert Watson 	LIST_FOREACH(kz, &uma_kegs, uk_link) {
352048c5777eSRobert Watson 		LIST_FOREACH(z, &kz->uk_zones, uz_link) {
352148c5777eSRobert Watson 			if (kz->uk_flags & UMA_ZFLAG_INTERNAL) {
352248c5777eSRobert Watson 				allocs = z->uz_allocs;
352348c5777eSRobert Watson 				frees = z->uz_frees;
3524bf965959SSean Bruno 				sleeps = z->uz_sleeps;
352548c5777eSRobert Watson 				cachefree = 0;
352648c5777eSRobert Watson 			} else
352748c5777eSRobert Watson 				uma_zone_sumstat(z, &cachefree, &allocs,
3528bf965959SSean Bruno 				    &frees, &sleeps);
3529e20a199fSJeff Roberson 			if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
353048c5777eSRobert Watson 			    (LIST_FIRST(&kz->uk_zones) != z)))
353148c5777eSRobert Watson 				cachefree += kz->uk_free;
3532fc03d22bSJeff Roberson 			LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
353348c5777eSRobert Watson 				cachefree += bucket->ub_cnt;
353403175483SAlexander Motin 			db_printf("%18s %8ju %8jd %8d %12ju %8ju %8u\n",
353503175483SAlexander Motin 			    z->uz_name, (uintmax_t)kz->uk_size,
3536ae4e9636SRobert Watson 			    (intmax_t)(allocs - frees), cachefree,
353703175483SAlexander Motin 			    (uintmax_t)allocs, sleeps, z->uz_count);
3538687c94aaSJohn Baldwin 			if (db_pager_quit)
3539687c94aaSJohn Baldwin 				return;
354048c5777eSRobert Watson 		}
354148c5777eSRobert Watson 	}
354248c5777eSRobert Watson }
354303175483SAlexander Motin 
354403175483SAlexander Motin DB_SHOW_COMMAND(umacache, db_show_umacache)
354503175483SAlexander Motin {
354603175483SAlexander Motin 	uint64_t allocs, frees;
354703175483SAlexander Motin 	uma_bucket_t bucket;
354803175483SAlexander Motin 	uma_zone_t z;
354903175483SAlexander Motin 	int cachefree;
355003175483SAlexander Motin 
355103175483SAlexander Motin 	db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",
355203175483SAlexander Motin 	    "Requests", "Bucket");
355303175483SAlexander Motin 	LIST_FOREACH(z, &uma_cachezones, uz_link) {
355403175483SAlexander Motin 		uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL);
355503175483SAlexander Motin 		LIST_FOREACH(bucket, &z->uz_buckets, ub_link)
355603175483SAlexander Motin 			cachefree += bucket->ub_cnt;
355703175483SAlexander Motin 		db_printf("%18s %8ju %8jd %8d %12ju %8u\n",
355803175483SAlexander Motin 		    z->uz_name, (uintmax_t)z->uz_size,
355903175483SAlexander Motin 		    (intmax_t)(allocs - frees), cachefree,
356003175483SAlexander Motin 		    (uintmax_t)allocs, z->uz_count);
356103175483SAlexander Motin 		if (db_pager_quit)
356203175483SAlexander Motin 			return;
356303175483SAlexander Motin 	}
356403175483SAlexander Motin }
356548c5777eSRobert Watson #endif
3566