xref: /freebsd/sys/vm/uma_core.c (revision 058f0f74643e7d9897e3455eed77c4cdbdf11d89)
160727d8bSWarner Losh /*-
2fe267a55SPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3fe267a55SPedro F. Giffuni  *
4ef72505eSJeff Roberson  * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
508ecce74SRobert Watson  * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
6ae4e9636SRobert Watson  * Copyright (c) 2004-2006 Robert N. M. Watson
708ecce74SRobert Watson  * All rights reserved.
88355f576SJeff Roberson  *
98355f576SJeff Roberson  * Redistribution and use in source and binary forms, with or without
108355f576SJeff Roberson  * modification, are permitted provided that the following conditions
118355f576SJeff Roberson  * are met:
128355f576SJeff Roberson  * 1. Redistributions of source code must retain the above copyright
138355f576SJeff Roberson  *    notice unmodified, this list of conditions, and the following
148355f576SJeff Roberson  *    disclaimer.
158355f576SJeff Roberson  * 2. Redistributions in binary form must reproduce the above copyright
168355f576SJeff Roberson  *    notice, this list of conditions and the following disclaimer in the
178355f576SJeff Roberson  *    documentation and/or other materials provided with the distribution.
188355f576SJeff Roberson  *
198355f576SJeff Roberson  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
208355f576SJeff Roberson  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
218355f576SJeff Roberson  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
228355f576SJeff Roberson  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
238355f576SJeff Roberson  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
248355f576SJeff Roberson  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
258355f576SJeff Roberson  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
268355f576SJeff Roberson  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
278355f576SJeff Roberson  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
288355f576SJeff Roberson  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
298355f576SJeff Roberson  */
308355f576SJeff Roberson 
318355f576SJeff Roberson /*
328355f576SJeff Roberson  * uma_core.c  Implementation of the Universal Memory allocator
338355f576SJeff Roberson  *
348355f576SJeff Roberson  * This allocator is intended to replace the multitude of similar object caches
358355f576SJeff Roberson  * in the standard FreeBSD kernel.  The intent is to be flexible as well as
36763df3ecSPedro F. Giffuni  * efficient.  A primary design goal is to return unused memory to the rest of
378355f576SJeff Roberson  * the system.  This will make the system as a whole more flexible due to the
388355f576SJeff Roberson  * ability to move memory to subsystems which most need it instead of leaving
398355f576SJeff Roberson  * pools of reserved memory unused.
408355f576SJeff Roberson  *
418355f576SJeff Roberson  * The basic ideas stem from similar slab/zone based allocators whose algorithms
428355f576SJeff Roberson  * are well known.
438355f576SJeff Roberson  *
448355f576SJeff Roberson  */
458355f576SJeff Roberson 
468355f576SJeff Roberson /*
478355f576SJeff Roberson  * TODO:
488355f576SJeff Roberson  *	- Improve memory usage for large allocations
498355f576SJeff Roberson  *	- Investigate cache size adjustments
508355f576SJeff Roberson  */
518355f576SJeff Roberson 
52874651b1SDavid E. O'Brien #include <sys/cdefs.h>
53874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$");
54874651b1SDavid E. O'Brien 
5548c5777eSRobert Watson #include "opt_ddb.h"
568355f576SJeff Roberson #include "opt_param.h"
578d689e04SGleb Smirnoff #include "opt_vm.h"
5848c5777eSRobert Watson 
598355f576SJeff Roberson #include <sys/param.h>
608355f576SJeff Roberson #include <sys/systm.h>
61ef72505eSJeff Roberson #include <sys/bitset.h>
62194a979eSMark Johnston #include <sys/domainset.h>
639b43bc27SAndriy Gapon #include <sys/eventhandler.h>
648355f576SJeff Roberson #include <sys/kernel.h>
658355f576SJeff Roberson #include <sys/types.h>
66ad5b0f5bSJeff Roberson #include <sys/limits.h>
678355f576SJeff Roberson #include <sys/queue.h>
688355f576SJeff Roberson #include <sys/malloc.h>
693659f747SRobert Watson #include <sys/ktr.h>
708355f576SJeff Roberson #include <sys/lock.h>
718355f576SJeff Roberson #include <sys/sysctl.h>
728355f576SJeff Roberson #include <sys/mutex.h>
734c1cc01cSJohn Baldwin #include <sys/proc.h>
7410cb2424SMark Murray #include <sys/random.h>
7589f6b863SAttilio Rao #include <sys/rwlock.h>
767a52a97eSRobert Watson #include <sys/sbuf.h>
77a2de44abSAlexander Motin #include <sys/sched.h>
788355f576SJeff Roberson #include <sys/smp.h>
79e60b2fcbSGleb Smirnoff #include <sys/taskqueue.h>
8086bbae32SJeff Roberson #include <sys/vmmeter.h>
8186bbae32SJeff Roberson 
828355f576SJeff Roberson #include <vm/vm.h>
83194a979eSMark Johnston #include <vm/vm_domainset.h>
848355f576SJeff Roberson #include <vm/vm_object.h>
858355f576SJeff Roberson #include <vm/vm_page.h>
86a4915c21SAttilio Rao #include <vm/vm_pageout.h>
878355f576SJeff Roberson #include <vm/vm_param.h>
88ab3185d1SJeff Roberson #include <vm/vm_phys.h>
8930c5525bSAndrew Gallatin #include <vm/vm_pagequeue.h>
908355f576SJeff Roberson #include <vm/vm_map.h>
918355f576SJeff Roberson #include <vm/vm_kern.h>
928355f576SJeff Roberson #include <vm/vm_extern.h>
938355f576SJeff Roberson #include <vm/uma.h>
948355f576SJeff Roberson #include <vm/uma_int.h>
95639c9550SJeff Roberson #include <vm/uma_dbg.h>
968355f576SJeff Roberson 
9748c5777eSRobert Watson #include <ddb/ddb.h>
9848c5777eSRobert Watson 
998d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD
1008d689e04SGleb Smirnoff #include <vm/memguard.h>
1018d689e04SGleb Smirnoff #endif
1028d689e04SGleb Smirnoff 
1038355f576SJeff Roberson /*
104ab3185d1SJeff Roberson  * This is the zone and keg from which all zones are spawned.
1058355f576SJeff Roberson  */
106ab3185d1SJeff Roberson static uma_zone_t kegs;
107ab3185d1SJeff Roberson static uma_zone_t zones;
1088355f576SJeff Roberson 
109ab3185d1SJeff Roberson /* This is the zone from which all offpage uma_slab_ts are allocated. */
1108355f576SJeff Roberson static uma_zone_t slabzone;
1118355f576SJeff Roberson 
1128355f576SJeff Roberson /*
1138355f576SJeff Roberson  * The initial hash tables come out of this zone so they can be allocated
1148355f576SJeff Roberson  * prior to malloc coming up.
1158355f576SJeff Roberson  */
1168355f576SJeff Roberson static uma_zone_t hashzone;
1178355f576SJeff Roberson 
1181e319f6dSRobert Watson /* The boot-time adjusted value for cache line alignment. */
119e4cd31ddSJeff Roberson int uma_align_cache = 64 - 1;
1201e319f6dSRobert Watson 
121961647dfSJeff Roberson static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
122961647dfSJeff Roberson 
1238355f576SJeff Roberson /*
12486bbae32SJeff Roberson  * Are we allowed to allocate buckets?
12586bbae32SJeff Roberson  */
12686bbae32SJeff Roberson static int bucketdisable = 1;
12786bbae32SJeff Roberson 
128099a0e58SBosko Milekic /* Linked list of all kegs in the system */
12913e403fdSAntoine Brodin static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
1308355f576SJeff Roberson 
13103175483SAlexander Motin /* Linked list of all cache-only zones in the system */
13203175483SAlexander Motin static LIST_HEAD(,uma_zone) uma_cachezones =
13303175483SAlexander Motin     LIST_HEAD_INITIALIZER(uma_cachezones);
13403175483SAlexander Motin 
135111fbcd5SBryan Venteicher /* This RW lock protects the keg list */
136fe933c1dSMateusz Guzik static struct rwlock_padalign __exclusive_cache_line uma_rwlock;
1378355f576SJeff Roberson 
138ac0a6fd0SGleb Smirnoff /*
139ac0a6fd0SGleb Smirnoff  * Pointer and counter to pool of pages, that is preallocated at
140f7d35785SGleb Smirnoff  * startup to bootstrap UMA.
141ac0a6fd0SGleb Smirnoff  */
142ac0a6fd0SGleb Smirnoff static char *bootmem;
143ac0a6fd0SGleb Smirnoff static int boot_pages;
1448355f576SJeff Roberson 
14595c4bf75SKonstantin Belousov static struct sx uma_drain_lock;
14695c4bf75SKonstantin Belousov 
1472e47807cSJeff Roberson /* kmem soft limit. */
148ad5b0f5bSJeff Roberson static unsigned long uma_kmem_limit = LONG_MAX;
149*058f0f74SMark Johnston static unsigned long uma_kmem_total;
1502e47807cSJeff Roberson 
1518355f576SJeff Roberson /* Is the VM done starting up? */
152f4bef67cSGleb Smirnoff static enum { BOOT_COLD = 0, BOOT_STRAPPED, BOOT_PAGEALLOC, BOOT_BUCKETS,
153f4bef67cSGleb Smirnoff     BOOT_RUNNING } booted = BOOT_COLD;
1548355f576SJeff Roberson 
155ef72505eSJeff Roberson /*
1569643769aSJeff Roberson  * This is the handle used to schedule events that need to happen
1579643769aSJeff Roberson  * outside of the allocation fast path.
1589643769aSJeff Roberson  */
1598355f576SJeff Roberson static struct callout uma_callout;
1609643769aSJeff Roberson #define	UMA_TIMEOUT	20		/* Seconds for callout interval. */
1618355f576SJeff Roberson 
1628355f576SJeff Roberson /*
1638355f576SJeff Roberson  * This structure is passed as the zone ctor arg so that I don't have to create
1648355f576SJeff Roberson  * a special allocation function just for zones.
1658355f576SJeff Roberson  */
1668355f576SJeff Roberson struct uma_zctor_args {
167bb196eb4SMatthew D Fleming 	const char *name;
168c3bdc05fSAndrew R. Reiter 	size_t size;
1698355f576SJeff Roberson 	uma_ctor ctor;
1708355f576SJeff Roberson 	uma_dtor dtor;
1718355f576SJeff Roberson 	uma_init uminit;
1728355f576SJeff Roberson 	uma_fini fini;
1730095a784SJeff Roberson 	uma_import import;
1740095a784SJeff Roberson 	uma_release release;
1750095a784SJeff Roberson 	void *arg;
176099a0e58SBosko Milekic 	uma_keg_t keg;
177099a0e58SBosko Milekic 	int align;
17885dcf349SGleb Smirnoff 	uint32_t flags;
179099a0e58SBosko Milekic };
180099a0e58SBosko Milekic 
181099a0e58SBosko Milekic struct uma_kctor_args {
182099a0e58SBosko Milekic 	uma_zone_t zone;
183099a0e58SBosko Milekic 	size_t size;
184099a0e58SBosko Milekic 	uma_init uminit;
185099a0e58SBosko Milekic 	uma_fini fini;
1868355f576SJeff Roberson 	int align;
18785dcf349SGleb Smirnoff 	uint32_t flags;
1888355f576SJeff Roberson };
1898355f576SJeff Roberson 
190cae33c14SJeff Roberson struct uma_bucket_zone {
191cae33c14SJeff Roberson 	uma_zone_t	ubz_zone;
192cae33c14SJeff Roberson 	char		*ubz_name;
193fc03d22bSJeff Roberson 	int		ubz_entries;	/* Number of items it can hold. */
194fc03d22bSJeff Roberson 	int		ubz_maxsize;	/* Maximum allocation size per-item. */
195cae33c14SJeff Roberson };
196cae33c14SJeff Roberson 
197f9d27e75SRobert Watson /*
198fc03d22bSJeff Roberson  * Compute the actual number of bucket entries to pack them in power
199fc03d22bSJeff Roberson  * of two sizes for more efficient space utilization.
200f9d27e75SRobert Watson  */
201fc03d22bSJeff Roberson #define	BUCKET_SIZE(n)						\
202fc03d22bSJeff Roberson     (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *))
203fc03d22bSJeff Roberson 
2041aa6c758SAlexander Motin #define	BUCKET_MAX	BUCKET_SIZE(256)
205fc03d22bSJeff Roberson 
206fc03d22bSJeff Roberson struct uma_bucket_zone bucket_zones[] = {
2076fd34d6fSJeff Roberson 	{ NULL, "4 Bucket", BUCKET_SIZE(4), 4096 },
208f3932e90SAlexander Motin 	{ NULL, "6 Bucket", BUCKET_SIZE(6), 3072 },
2096fd34d6fSJeff Roberson 	{ NULL, "8 Bucket", BUCKET_SIZE(8), 2048 },
210f3932e90SAlexander Motin 	{ NULL, "12 Bucket", BUCKET_SIZE(12), 1536 },
2116fd34d6fSJeff Roberson 	{ NULL, "16 Bucket", BUCKET_SIZE(16), 1024 },
212fc03d22bSJeff Roberson 	{ NULL, "32 Bucket", BUCKET_SIZE(32), 512 },
213fc03d22bSJeff Roberson 	{ NULL, "64 Bucket", BUCKET_SIZE(64), 256 },
214fc03d22bSJeff Roberson 	{ NULL, "128 Bucket", BUCKET_SIZE(128), 128 },
2151aa6c758SAlexander Motin 	{ NULL, "256 Bucket", BUCKET_SIZE(256), 64 },
216fc03d22bSJeff Roberson 	{ NULL, NULL, 0}
217fc03d22bSJeff Roberson };
218cae33c14SJeff Roberson 
2192019094aSRobert Watson /*
2202019094aSRobert Watson  * Flags and enumerations to be passed to internal functions.
2212019094aSRobert Watson  */
222bb15d1c7SGleb Smirnoff enum zfreeskip {
223bb15d1c7SGleb Smirnoff 	SKIP_NONE =	0,
224bb15d1c7SGleb Smirnoff 	SKIP_CNT =	0x00000001,
225bb15d1c7SGleb Smirnoff 	SKIP_DTOR =	0x00010000,
226bb15d1c7SGleb Smirnoff 	SKIP_FINI =	0x00020000,
227bb15d1c7SGleb Smirnoff };
228b23f72e9SBrian Feldman 
229ab3185d1SJeff Roberson #define	UMA_ANYDOMAIN	-1	/* Special value for domain search. */
230ab3185d1SJeff Roberson 
2318355f576SJeff Roberson /* Prototypes.. */
2328355f576SJeff Roberson 
233f4bef67cSGleb Smirnoff int	uma_startup_count(int);
234f4bef67cSGleb Smirnoff void	uma_startup(void *, int);
235f4bef67cSGleb Smirnoff void	uma_startup1(void);
236f4bef67cSGleb Smirnoff void	uma_startup2(void);
237f4bef67cSGleb Smirnoff 
238ab3185d1SJeff Roberson static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
239ab3185d1SJeff Roberson static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
240ab3059a8SMatt Macy static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
241ab3185d1SJeff Roberson static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
242f2c2231eSRyan Stone static void page_free(void *, vm_size_t, uint8_t);
243ab3059a8SMatt Macy static void pcpu_page_free(void *, vm_size_t, uint8_t);
24486220393SMark Johnston static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int);
2459643769aSJeff Roberson static void cache_drain(uma_zone_t);
2468355f576SJeff Roberson static void bucket_drain(uma_zone_t, uma_bucket_t);
247aaa8bb16SJeff Roberson static void bucket_cache_drain(uma_zone_t zone);
248b23f72e9SBrian Feldman static int keg_ctor(void *, int, void *, int);
249099a0e58SBosko Milekic static void keg_dtor(void *, int, void *);
250b23f72e9SBrian Feldman static int zone_ctor(void *, int, void *, int);
2519c2cd7e5SJeff Roberson static void zone_dtor(void *, int, void *);
252b23f72e9SBrian Feldman static int zero_init(void *, int, int);
253e20a199fSJeff Roberson static void keg_small_init(uma_keg_t keg);
254e20a199fSJeff Roberson static void keg_large_init(uma_keg_t keg);
2558355f576SJeff Roberson static void zone_foreach(void (*zfunc)(uma_zone_t));
2568355f576SJeff Roberson static void zone_timeout(uma_zone_t zone);
2570aef6126SJeff Roberson static int hash_alloc(struct uma_hash *);
2580aef6126SJeff Roberson static int hash_expand(struct uma_hash *, struct uma_hash *);
2590aef6126SJeff Roberson static void hash_free(struct uma_hash *hash);
2608355f576SJeff Roberson static void uma_timeout(void *);
2618355f576SJeff Roberson static void uma_startup3(void);
262ab3185d1SJeff Roberson static void *zone_alloc_item(uma_zone_t, void *, int, int);
263bb15d1c7SGleb Smirnoff static void *zone_alloc_item_locked(uma_zone_t, void *, int, int);
2640095a784SJeff Roberson static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
26586bbae32SJeff Roberson static void bucket_enable(void);
266cae33c14SJeff Roberson static void bucket_init(void);
2676fd34d6fSJeff Roberson static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
2686fd34d6fSJeff Roberson static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
269cae33c14SJeff Roberson static void bucket_zone_drain(void);
270bb15d1c7SGleb Smirnoff static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int, int);
271ab3185d1SJeff Roberson static uma_slab_t zone_fetch_slab(uma_zone_t, uma_keg_t, int, int);
2720095a784SJeff Roberson static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
273bb15d1c7SGleb Smirnoff static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item);
274e20a199fSJeff Roberson static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
27585dcf349SGleb Smirnoff     uma_fini fini, int align, uint32_t flags);
276ab3185d1SJeff Roberson static int zone_import(uma_zone_t, void **, int, int, int);
277ab3185d1SJeff Roberson static void zone_release(uma_zone_t, void **, int);
278ab3185d1SJeff Roberson static void uma_zero_item(void *, uma_zone_t);
279bbee39c6SJeff Roberson 
2808355f576SJeff Roberson void uma_print_zone(uma_zone_t);
2818355f576SJeff Roberson void uma_print_stats(void);
2827a52a97eSRobert Watson static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
2837a52a97eSRobert Watson static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
2848355f576SJeff Roberson 
2859542ea7bSGleb Smirnoff #ifdef INVARIANTS
286c5deaf04SGleb Smirnoff static bool uma_dbg_kskip(uma_keg_t keg, void *mem);
287c5deaf04SGleb Smirnoff static bool uma_dbg_zskip(uma_zone_t zone, void *mem);
2889542ea7bSGleb Smirnoff static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
2899542ea7bSGleb Smirnoff static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);
290c5deaf04SGleb Smirnoff 
291c5deaf04SGleb Smirnoff static SYSCTL_NODE(_vm, OID_AUTO, debug, CTLFLAG_RD, 0,
292c5deaf04SGleb Smirnoff     "Memory allocation debugging");
293c5deaf04SGleb Smirnoff 
294c5deaf04SGleb Smirnoff static u_int dbg_divisor = 1;
295c5deaf04SGleb Smirnoff SYSCTL_UINT(_vm_debug, OID_AUTO, divisor,
296c5deaf04SGleb Smirnoff     CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &dbg_divisor, 0,
297c5deaf04SGleb Smirnoff     "Debug & thrash every this item in memory allocator");
298c5deaf04SGleb Smirnoff 
299c5deaf04SGleb Smirnoff static counter_u64_t uma_dbg_cnt = EARLY_COUNTER;
300c5deaf04SGleb Smirnoff static counter_u64_t uma_skip_cnt = EARLY_COUNTER;
301c5deaf04SGleb Smirnoff SYSCTL_COUNTER_U64(_vm_debug, OID_AUTO, trashed, CTLFLAG_RD,
302c5deaf04SGleb Smirnoff     &uma_dbg_cnt, "memory items debugged");
303c5deaf04SGleb Smirnoff SYSCTL_COUNTER_U64(_vm_debug, OID_AUTO, skipped, CTLFLAG_RD,
304c5deaf04SGleb Smirnoff     &uma_skip_cnt, "memory items skipped, not debugged");
3059542ea7bSGleb Smirnoff #endif
3069542ea7bSGleb Smirnoff 
3078355f576SJeff Roberson SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL);
3088355f576SJeff Roberson 
3097a52a97eSRobert Watson SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT,
3107a52a97eSRobert Watson     0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones");
3117a52a97eSRobert Watson 
3127a52a97eSRobert Watson SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT,
3137a52a97eSRobert Watson     0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats");
3147a52a97eSRobert Watson 
3152f891cd5SPawel Jakub Dawidek static int zone_warnings = 1;
316af3b2549SHans Petter Selasky SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RWTUN, &zone_warnings, 0,
3172f891cd5SPawel Jakub Dawidek     "Warn when UMA zones becomes full");
3182f891cd5SPawel Jakub Dawidek 
3192e47807cSJeff Roberson /* Adjust bytes under management by UMA. */
3202e47807cSJeff Roberson static inline void
3212e47807cSJeff Roberson uma_total_dec(unsigned long size)
3222e47807cSJeff Roberson {
3232e47807cSJeff Roberson 
3242e47807cSJeff Roberson 	atomic_subtract_long(&uma_kmem_total, size);
3252e47807cSJeff Roberson }
3262e47807cSJeff Roberson 
3272e47807cSJeff Roberson static inline void
3282e47807cSJeff Roberson uma_total_inc(unsigned long size)
3292e47807cSJeff Roberson {
3302e47807cSJeff Roberson 
3312e47807cSJeff Roberson 	if (atomic_fetchadd_long(&uma_kmem_total, size) > uma_kmem_limit)
3322e47807cSJeff Roberson 		uma_reclaim_wakeup();
3332e47807cSJeff Roberson }
3342e47807cSJeff Roberson 
33586bbae32SJeff Roberson /*
33686bbae32SJeff Roberson  * This routine checks to see whether or not it's safe to enable buckets.
33786bbae32SJeff Roberson  */
33886bbae32SJeff Roberson static void
33986bbae32SJeff Roberson bucket_enable(void)
34086bbae32SJeff Roberson {
341251386b4SMaksim Yevmenkin 	bucketdisable = vm_page_count_min();
34286bbae32SJeff Roberson }
34386bbae32SJeff Roberson 
344dc2c7965SRobert Watson /*
345dc2c7965SRobert Watson  * Initialize bucket_zones, the array of zones of buckets of various sizes.
346dc2c7965SRobert Watson  *
347dc2c7965SRobert Watson  * For each zone, calculate the memory required for each bucket, consisting
348fc03d22bSJeff Roberson  * of the header and an array of pointers.
349dc2c7965SRobert Watson  */
350cae33c14SJeff Roberson static void
351cae33c14SJeff Roberson bucket_init(void)
352cae33c14SJeff Roberson {
353cae33c14SJeff Roberson 	struct uma_bucket_zone *ubz;
354cae33c14SJeff Roberson 	int size;
355cae33c14SJeff Roberson 
356d74e6a1dSAlan Cox 	for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) {
357cae33c14SJeff Roberson 		size = roundup(sizeof(struct uma_bucket), sizeof(void *));
358cae33c14SJeff Roberson 		size += sizeof(void *) * ubz->ubz_entries;
359cae33c14SJeff Roberson 		ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
360e20a199fSJeff Roberson 		    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
361ab3185d1SJeff Roberson 		    UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET | UMA_ZONE_NUMA);
362cae33c14SJeff Roberson 	}
363cae33c14SJeff Roberson }
364cae33c14SJeff Roberson 
365dc2c7965SRobert Watson /*
366dc2c7965SRobert Watson  * Given a desired number of entries for a bucket, return the zone from which
367dc2c7965SRobert Watson  * to allocate the bucket.
368dc2c7965SRobert Watson  */
369dc2c7965SRobert Watson static struct uma_bucket_zone *
370dc2c7965SRobert Watson bucket_zone_lookup(int entries)
371dc2c7965SRobert Watson {
372fc03d22bSJeff Roberson 	struct uma_bucket_zone *ubz;
373dc2c7965SRobert Watson 
374fc03d22bSJeff Roberson 	for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
375fc03d22bSJeff Roberson 		if (ubz->ubz_entries >= entries)
376fc03d22bSJeff Roberson 			return (ubz);
377fc03d22bSJeff Roberson 	ubz--;
378fc03d22bSJeff Roberson 	return (ubz);
379fc03d22bSJeff Roberson }
380fc03d22bSJeff Roberson 
381fc03d22bSJeff Roberson static int
382fc03d22bSJeff Roberson bucket_select(int size)
383fc03d22bSJeff Roberson {
384fc03d22bSJeff Roberson 	struct uma_bucket_zone *ubz;
385fc03d22bSJeff Roberson 
386fc03d22bSJeff Roberson 	ubz = &bucket_zones[0];
387fc03d22bSJeff Roberson 	if (size > ubz->ubz_maxsize)
388fc03d22bSJeff Roberson 		return MAX((ubz->ubz_maxsize * ubz->ubz_entries) / size, 1);
389fc03d22bSJeff Roberson 
390fc03d22bSJeff Roberson 	for (; ubz->ubz_entries != 0; ubz++)
391fc03d22bSJeff Roberson 		if (ubz->ubz_maxsize < size)
392fc03d22bSJeff Roberson 			break;
393fc03d22bSJeff Roberson 	ubz--;
394fc03d22bSJeff Roberson 	return (ubz->ubz_entries);
395dc2c7965SRobert Watson }
396dc2c7965SRobert Watson 
397cae33c14SJeff Roberson static uma_bucket_t
3986fd34d6fSJeff Roberson bucket_alloc(uma_zone_t zone, void *udata, int flags)
399cae33c14SJeff Roberson {
400cae33c14SJeff Roberson 	struct uma_bucket_zone *ubz;
401cae33c14SJeff Roberson 	uma_bucket_t bucket;
402cae33c14SJeff Roberson 
403cae33c14SJeff Roberson 	/*
404cae33c14SJeff Roberson 	 * This is to stop us from allocating per cpu buckets while we're
4053803b26bSDag-Erling Smørgrav 	 * running out of vm.boot_pages.  Otherwise, we would exhaust the
406cae33c14SJeff Roberson 	 * boot pages.  This also prevents us from allocating buckets in
407cae33c14SJeff Roberson 	 * low memory situations.
408cae33c14SJeff Roberson 	 */
409cae33c14SJeff Roberson 	if (bucketdisable)
410cae33c14SJeff Roberson 		return (NULL);
4116fd34d6fSJeff Roberson 	/*
4126fd34d6fSJeff Roberson 	 * To limit bucket recursion we store the original zone flags
4136fd34d6fSJeff Roberson 	 * in a cookie passed via zalloc_arg/zfree_arg.  This allows the
4146fd34d6fSJeff Roberson 	 * NOVM flag to persist even through deep recursions.  We also
4156fd34d6fSJeff Roberson 	 * store ZFLAG_BUCKET once we have recursed attempting to allocate
4166fd34d6fSJeff Roberson 	 * a bucket for a bucket zone so we do not allow infinite bucket
4176fd34d6fSJeff Roberson 	 * recursion.  This cookie will even persist to frees of unused
4186fd34d6fSJeff Roberson 	 * buckets via the allocation path or bucket allocations in the
4196fd34d6fSJeff Roberson 	 * free path.
4206fd34d6fSJeff Roberson 	 */
4216fd34d6fSJeff Roberson 	if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
4226fd34d6fSJeff Roberson 		udata = (void *)(uintptr_t)zone->uz_flags;
423e8a720feSAlexander Motin 	else {
424e8a720feSAlexander Motin 		if ((uintptr_t)udata & UMA_ZFLAG_BUCKET)
425e8a720feSAlexander Motin 			return (NULL);
4266fd34d6fSJeff Roberson 		udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET);
427e8a720feSAlexander Motin 	}
4286fd34d6fSJeff Roberson 	if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY)
429af526374SJeff Roberson 		flags |= M_NOVM;
430af526374SJeff Roberson 	ubz = bucket_zone_lookup(zone->uz_count);
43120d3ab87SAlexander Motin 	if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0)
43220d3ab87SAlexander Motin 		ubz++;
4336fd34d6fSJeff Roberson 	bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags);
434cae33c14SJeff Roberson 	if (bucket) {
435cae33c14SJeff Roberson #ifdef INVARIANTS
436cae33c14SJeff Roberson 		bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
437cae33c14SJeff Roberson #endif
438cae33c14SJeff Roberson 		bucket->ub_cnt = 0;
439cae33c14SJeff Roberson 		bucket->ub_entries = ubz->ubz_entries;
440cae33c14SJeff Roberson 	}
441cae33c14SJeff Roberson 
442cae33c14SJeff Roberson 	return (bucket);
443cae33c14SJeff Roberson }
444cae33c14SJeff Roberson 
445cae33c14SJeff Roberson static void
4466fd34d6fSJeff Roberson bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata)
447cae33c14SJeff Roberson {
448cae33c14SJeff Roberson 	struct uma_bucket_zone *ubz;
449cae33c14SJeff Roberson 
450fc03d22bSJeff Roberson 	KASSERT(bucket->ub_cnt == 0,
451fc03d22bSJeff Roberson 	    ("bucket_free: Freeing a non free bucket."));
4526fd34d6fSJeff Roberson 	if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
4536fd34d6fSJeff Roberson 		udata = (void *)(uintptr_t)zone->uz_flags;
454dc2c7965SRobert Watson 	ubz = bucket_zone_lookup(bucket->ub_entries);
4556fd34d6fSJeff Roberson 	uma_zfree_arg(ubz->ubz_zone, bucket, udata);
456cae33c14SJeff Roberson }
457cae33c14SJeff Roberson 
458cae33c14SJeff Roberson static void
459cae33c14SJeff Roberson bucket_zone_drain(void)
460cae33c14SJeff Roberson {
461cae33c14SJeff Roberson 	struct uma_bucket_zone *ubz;
462cae33c14SJeff Roberson 
463cae33c14SJeff Roberson 	for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
464cae33c14SJeff Roberson 		zone_drain(ubz->ubz_zone);
465cae33c14SJeff Roberson }
466cae33c14SJeff Roberson 
4670f9b7bf3SMark Johnston static uma_bucket_t
4680f9b7bf3SMark Johnston zone_try_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, const bool ws)
4690f9b7bf3SMark Johnston {
4700f9b7bf3SMark Johnston 	uma_bucket_t bucket;
4710f9b7bf3SMark Johnston 
4720f9b7bf3SMark Johnston 	ZONE_LOCK_ASSERT(zone);
4730f9b7bf3SMark Johnston 
4740f9b7bf3SMark Johnston 	if ((bucket = LIST_FIRST(&zdom->uzd_buckets)) != NULL) {
4750f9b7bf3SMark Johnston 		MPASS(zdom->uzd_nitems >= bucket->ub_cnt);
4760f9b7bf3SMark Johnston 		LIST_REMOVE(bucket, ub_link);
4770f9b7bf3SMark Johnston 		zdom->uzd_nitems -= bucket->ub_cnt;
4780f9b7bf3SMark Johnston 		if (ws && zdom->uzd_imin > zdom->uzd_nitems)
4790f9b7bf3SMark Johnston 			zdom->uzd_imin = zdom->uzd_nitems;
480bb15d1c7SGleb Smirnoff 		zone->uz_bkt_count -= bucket->ub_cnt;
4810f9b7bf3SMark Johnston 	}
4820f9b7bf3SMark Johnston 	return (bucket);
4830f9b7bf3SMark Johnston }
4840f9b7bf3SMark Johnston 
4850f9b7bf3SMark Johnston static void
4860f9b7bf3SMark Johnston zone_put_bucket(uma_zone_t zone, uma_zone_domain_t zdom, uma_bucket_t bucket,
4870f9b7bf3SMark Johnston     const bool ws)
4880f9b7bf3SMark Johnston {
4890f9b7bf3SMark Johnston 
4900f9b7bf3SMark Johnston 	ZONE_LOCK_ASSERT(zone);
491bb15d1c7SGleb Smirnoff 	KASSERT(zone->uz_bkt_count < zone->uz_bkt_max, ("%s: zone %p overflow",
492bb15d1c7SGleb Smirnoff 	    __func__, zone));
4930f9b7bf3SMark Johnston 
4940f9b7bf3SMark Johnston 	LIST_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link);
4950f9b7bf3SMark Johnston 	zdom->uzd_nitems += bucket->ub_cnt;
4960f9b7bf3SMark Johnston 	if (ws && zdom->uzd_imax < zdom->uzd_nitems)
4970f9b7bf3SMark Johnston 		zdom->uzd_imax = zdom->uzd_nitems;
498bb15d1c7SGleb Smirnoff 	zone->uz_bkt_count += bucket->ub_cnt;
4990f9b7bf3SMark Johnston }
5000f9b7bf3SMark Johnston 
5012f891cd5SPawel Jakub Dawidek static void
5022f891cd5SPawel Jakub Dawidek zone_log_warning(uma_zone_t zone)
5032f891cd5SPawel Jakub Dawidek {
5042f891cd5SPawel Jakub Dawidek 	static const struct timeval warninterval = { 300, 0 };
5052f891cd5SPawel Jakub Dawidek 
5062f891cd5SPawel Jakub Dawidek 	if (!zone_warnings || zone->uz_warning == NULL)
5072f891cd5SPawel Jakub Dawidek 		return;
5082f891cd5SPawel Jakub Dawidek 
5092f891cd5SPawel Jakub Dawidek 	if (ratecheck(&zone->uz_ratecheck, &warninterval))
5102f891cd5SPawel Jakub Dawidek 		printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
5112f891cd5SPawel Jakub Dawidek }
5122f891cd5SPawel Jakub Dawidek 
51354503a13SJonathan T. Looney static inline void
51454503a13SJonathan T. Looney zone_maxaction(uma_zone_t zone)
51554503a13SJonathan T. Looney {
516e60b2fcbSGleb Smirnoff 
517e60b2fcbSGleb Smirnoff 	if (zone->uz_maxaction.ta_func != NULL)
518e60b2fcbSGleb Smirnoff 		taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction);
51954503a13SJonathan T. Looney }
52054503a13SJonathan T. Looney 
5218355f576SJeff Roberson /*
5228355f576SJeff Roberson  * Routine called by timeout which is used to fire off some time interval
5239643769aSJeff Roberson  * based calculations.  (stats, hash size, etc.)
5248355f576SJeff Roberson  *
5258355f576SJeff Roberson  * Arguments:
5268355f576SJeff Roberson  *	arg   Unused
5278355f576SJeff Roberson  *
5288355f576SJeff Roberson  * Returns:
5298355f576SJeff Roberson  *	Nothing
5308355f576SJeff Roberson  */
5318355f576SJeff Roberson static void
5328355f576SJeff Roberson uma_timeout(void *unused)
5338355f576SJeff Roberson {
53486bbae32SJeff Roberson 	bucket_enable();
5358355f576SJeff Roberson 	zone_foreach(zone_timeout);
5368355f576SJeff Roberson 
5378355f576SJeff Roberson 	/* Reschedule this event */
5389643769aSJeff Roberson 	callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
5398355f576SJeff Roberson }
5408355f576SJeff Roberson 
5418355f576SJeff Roberson /*
5420f9b7bf3SMark Johnston  * Update the working set size estimate for the zone's bucket cache.
5430f9b7bf3SMark Johnston  * The constants chosen here are somewhat arbitrary.  With an update period of
5440f9b7bf3SMark Johnston  * 20s (UMA_TIMEOUT), this estimate is dominated by zone activity over the
5450f9b7bf3SMark Johnston  * last 100s.
5460f9b7bf3SMark Johnston  */
5470f9b7bf3SMark Johnston static void
5480f9b7bf3SMark Johnston zone_domain_update_wss(uma_zone_domain_t zdom)
5490f9b7bf3SMark Johnston {
5500f9b7bf3SMark Johnston 	long wss;
5510f9b7bf3SMark Johnston 
5520f9b7bf3SMark Johnston 	MPASS(zdom->uzd_imax >= zdom->uzd_imin);
5530f9b7bf3SMark Johnston 	wss = zdom->uzd_imax - zdom->uzd_imin;
5540f9b7bf3SMark Johnston 	zdom->uzd_imax = zdom->uzd_imin = zdom->uzd_nitems;
5550f9b7bf3SMark Johnston 	zdom->uzd_wss = (3 * wss + 2 * zdom->uzd_wss) / 5;
5560f9b7bf3SMark Johnston }
5570f9b7bf3SMark Johnston 
5580f9b7bf3SMark Johnston /*
5599643769aSJeff Roberson  * Routine to perform timeout driven calculations.  This expands the
5609643769aSJeff Roberson  * hashes and does per cpu statistics aggregation.
5618355f576SJeff Roberson  *
562e20a199fSJeff Roberson  *  Returns nothing.
5638355f576SJeff Roberson  */
5648355f576SJeff Roberson static void
565bb15d1c7SGleb Smirnoff zone_timeout(uma_zone_t zone)
5668355f576SJeff Roberson {
567bb15d1c7SGleb Smirnoff 	uma_keg_t keg = zone->uz_keg;
5688355f576SJeff Roberson 
569e20a199fSJeff Roberson 	KEG_LOCK(keg);
5708355f576SJeff Roberson 	/*
571e20a199fSJeff Roberson 	 * Expand the keg hash table.
5728355f576SJeff Roberson 	 *
5738355f576SJeff Roberson 	 * This is done if the number of slabs is larger than the hash size.
5748355f576SJeff Roberson 	 * What I'm trying to do here is completely reduce collisions.  This
5758355f576SJeff Roberson 	 * may be a little aggressive.  Should I allow for two collisions max?
5768355f576SJeff Roberson 	 */
577099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_HASH &&
578099a0e58SBosko Milekic 	    keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) {
5790aef6126SJeff Roberson 		struct uma_hash newhash;
5800aef6126SJeff Roberson 		struct uma_hash oldhash;
5810aef6126SJeff Roberson 		int ret;
5825300d9ddSJeff Roberson 
5830aef6126SJeff Roberson 		/*
5840aef6126SJeff Roberson 		 * This is so involved because allocating and freeing
585e20a199fSJeff Roberson 		 * while the keg lock is held will lead to deadlock.
5860aef6126SJeff Roberson 		 * I have to do everything in stages and check for
5870aef6126SJeff Roberson 		 * races.
5880aef6126SJeff Roberson 		 */
589099a0e58SBosko Milekic 		newhash = keg->uk_hash;
590e20a199fSJeff Roberson 		KEG_UNLOCK(keg);
5910aef6126SJeff Roberson 		ret = hash_alloc(&newhash);
592e20a199fSJeff Roberson 		KEG_LOCK(keg);
5930aef6126SJeff Roberson 		if (ret) {
594099a0e58SBosko Milekic 			if (hash_expand(&keg->uk_hash, &newhash)) {
595099a0e58SBosko Milekic 				oldhash = keg->uk_hash;
596099a0e58SBosko Milekic 				keg->uk_hash = newhash;
5970aef6126SJeff Roberson 			} else
5980aef6126SJeff Roberson 				oldhash = newhash;
5990aef6126SJeff Roberson 
600e20a199fSJeff Roberson 			KEG_UNLOCK(keg);
6010aef6126SJeff Roberson 			hash_free(&oldhash);
602a1dff920SDavide Italiano 			return;
6030aef6126SJeff Roberson 		}
6045300d9ddSJeff Roberson 	}
605e20a199fSJeff Roberson 
606bb15d1c7SGleb Smirnoff 	for (int i = 0; i < vm_ndomains; i++)
6070f9b7bf3SMark Johnston 		zone_domain_update_wss(&zone->uz_domain[i]);
608bb15d1c7SGleb Smirnoff 
609bb15d1c7SGleb Smirnoff 	KEG_UNLOCK(keg);
6108355f576SJeff Roberson }
6118355f576SJeff Roberson 
6128355f576SJeff Roberson /*
6135300d9ddSJeff Roberson  * Allocate and zero fill the next sized hash table from the appropriate
6145300d9ddSJeff Roberson  * backing store.
6155300d9ddSJeff Roberson  *
6165300d9ddSJeff Roberson  * Arguments:
6170aef6126SJeff Roberson  *	hash  A new hash structure with the old hash size in uh_hashsize
6185300d9ddSJeff Roberson  *
6195300d9ddSJeff Roberson  * Returns:
620763df3ecSPedro F. Giffuni  *	1 on success and 0 on failure.
6215300d9ddSJeff Roberson  */
62237c84183SPoul-Henning Kamp static int
6230aef6126SJeff Roberson hash_alloc(struct uma_hash *hash)
6245300d9ddSJeff Roberson {
6256929b7d1SPedro F. Giffuni 	u_int oldsize;
62659568a0eSAlexander Motin 	size_t alloc;
6275300d9ddSJeff Roberson 
6280aef6126SJeff Roberson 	oldsize = hash->uh_hashsize;
6290aef6126SJeff Roberson 
6305300d9ddSJeff Roberson 	/* We're just going to go to a power of two greater */
6310aef6126SJeff Roberson 	if (oldsize)  {
6320aef6126SJeff Roberson 		hash->uh_hashsize = oldsize * 2;
6330aef6126SJeff Roberson 		alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize;
6340aef6126SJeff Roberson 		hash->uh_slab_hash = (struct slabhead *)malloc(alloc,
635961647dfSJeff Roberson 		    M_UMAHASH, M_NOWAIT);
6365300d9ddSJeff Roberson 	} else {
6370aef6126SJeff Roberson 		alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
638e20a199fSJeff Roberson 		hash->uh_slab_hash = zone_alloc_item(hashzone, NULL,
639ab3185d1SJeff Roberson 		    UMA_ANYDOMAIN, M_WAITOK);
6400aef6126SJeff Roberson 		hash->uh_hashsize = UMA_HASH_SIZE_INIT;
6415300d9ddSJeff Roberson 	}
6420aef6126SJeff Roberson 	if (hash->uh_slab_hash) {
6430aef6126SJeff Roberson 		bzero(hash->uh_slab_hash, alloc);
6440aef6126SJeff Roberson 		hash->uh_hashmask = hash->uh_hashsize - 1;
6450aef6126SJeff Roberson 		return (1);
6460aef6126SJeff Roberson 	}
6475300d9ddSJeff Roberson 
6480aef6126SJeff Roberson 	return (0);
6495300d9ddSJeff Roberson }
6505300d9ddSJeff Roberson 
6515300d9ddSJeff Roberson /*
65264f051e9SJeff Roberson  * Expands the hash table for HASH zones.  This is done from zone_timeout
65364f051e9SJeff Roberson  * to reduce collisions.  This must not be done in the regular allocation
65464f051e9SJeff Roberson  * path, otherwise, we can recurse on the vm while allocating pages.
6558355f576SJeff Roberson  *
6568355f576SJeff Roberson  * Arguments:
6570aef6126SJeff Roberson  *	oldhash  The hash you want to expand
6580aef6126SJeff Roberson  *	newhash  The hash structure for the new table
6598355f576SJeff Roberson  *
6608355f576SJeff Roberson  * Returns:
6618355f576SJeff Roberson  *	Nothing
6628355f576SJeff Roberson  *
6638355f576SJeff Roberson  * Discussion:
6648355f576SJeff Roberson  */
6650aef6126SJeff Roberson static int
6660aef6126SJeff Roberson hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash)
6678355f576SJeff Roberson {
6688355f576SJeff Roberson 	uma_slab_t slab;
6696929b7d1SPedro F. Giffuni 	u_int hval;
6706929b7d1SPedro F. Giffuni 	u_int idx;
6718355f576SJeff Roberson 
6720aef6126SJeff Roberson 	if (!newhash->uh_slab_hash)
6730aef6126SJeff Roberson 		return (0);
6748355f576SJeff Roberson 
6750aef6126SJeff Roberson 	if (oldhash->uh_hashsize >= newhash->uh_hashsize)
6760aef6126SJeff Roberson 		return (0);
6778355f576SJeff Roberson 
6788355f576SJeff Roberson 	/*
6798355f576SJeff Roberson 	 * I need to investigate hash algorithms for resizing without a
6808355f576SJeff Roberson 	 * full rehash.
6818355f576SJeff Roberson 	 */
6828355f576SJeff Roberson 
6836929b7d1SPedro F. Giffuni 	for (idx = 0; idx < oldhash->uh_hashsize; idx++)
6846929b7d1SPedro F. Giffuni 		while (!SLIST_EMPTY(&oldhash->uh_slab_hash[idx])) {
6856929b7d1SPedro F. Giffuni 			slab = SLIST_FIRST(&oldhash->uh_slab_hash[idx]);
6866929b7d1SPedro F. Giffuni 			SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[idx], us_hlink);
6870aef6126SJeff Roberson 			hval = UMA_HASH(newhash, slab->us_data);
6880aef6126SJeff Roberson 			SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval],
6890aef6126SJeff Roberson 			    slab, us_hlink);
6908355f576SJeff Roberson 		}
6918355f576SJeff Roberson 
6920aef6126SJeff Roberson 	return (1);
6939c2cd7e5SJeff Roberson }
6949c2cd7e5SJeff Roberson 
6955300d9ddSJeff Roberson /*
6965300d9ddSJeff Roberson  * Free the hash bucket to the appropriate backing store.
6975300d9ddSJeff Roberson  *
6985300d9ddSJeff Roberson  * Arguments:
6995300d9ddSJeff Roberson  *	slab_hash  The hash bucket we're freeing
7005300d9ddSJeff Roberson  *	hashsize   The number of entries in that hash bucket
7015300d9ddSJeff Roberson  *
7025300d9ddSJeff Roberson  * Returns:
7035300d9ddSJeff Roberson  *	Nothing
7045300d9ddSJeff Roberson  */
7059c2cd7e5SJeff Roberson static void
7060aef6126SJeff Roberson hash_free(struct uma_hash *hash)
7079c2cd7e5SJeff Roberson {
7080aef6126SJeff Roberson 	if (hash->uh_slab_hash == NULL)
7090aef6126SJeff Roberson 		return;
7100aef6126SJeff Roberson 	if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
7110095a784SJeff Roberson 		zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
7128355f576SJeff Roberson 	else
713961647dfSJeff Roberson 		free(hash->uh_slab_hash, M_UMAHASH);
7148355f576SJeff Roberson }
7158355f576SJeff Roberson 
7168355f576SJeff Roberson /*
7178355f576SJeff Roberson  * Frees all outstanding items in a bucket
7188355f576SJeff Roberson  *
7198355f576SJeff Roberson  * Arguments:
7208355f576SJeff Roberson  *	zone   The zone to free to, must be unlocked.
7218355f576SJeff Roberson  *	bucket The free/alloc bucket with items, cpu queue must be locked.
7228355f576SJeff Roberson  *
7238355f576SJeff Roberson  * Returns:
7248355f576SJeff Roberson  *	Nothing
7258355f576SJeff Roberson  */
7268355f576SJeff Roberson 
7278355f576SJeff Roberson static void
7288355f576SJeff Roberson bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
7298355f576SJeff Roberson {
7300095a784SJeff Roberson 	int i;
7318355f576SJeff Roberson 
7328355f576SJeff Roberson 	if (bucket == NULL)
7338355f576SJeff Roberson 		return;
7348355f576SJeff Roberson 
7350095a784SJeff Roberson 	if (zone->uz_fini)
7360095a784SJeff Roberson 		for (i = 0; i < bucket->ub_cnt; i++)
7370095a784SJeff Roberson 			zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
7380095a784SJeff Roberson 	zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
739bb45b411SGleb Smirnoff 	if (zone->uz_max_items > 0) {
740bb15d1c7SGleb Smirnoff 		ZONE_LOCK(zone);
741bb15d1c7SGleb Smirnoff 		zone->uz_items -= bucket->ub_cnt;
742bb15d1c7SGleb Smirnoff 		if (zone->uz_sleepers && zone->uz_items < zone->uz_max_items)
743bb15d1c7SGleb Smirnoff 			wakeup_one(zone);
744bb15d1c7SGleb Smirnoff 		ZONE_UNLOCK(zone);
745bb45b411SGleb Smirnoff 	}
7460095a784SJeff Roberson 	bucket->ub_cnt = 0;
7478355f576SJeff Roberson }
7488355f576SJeff Roberson 
7498355f576SJeff Roberson /*
7508355f576SJeff Roberson  * Drains the per cpu caches for a zone.
7518355f576SJeff Roberson  *
7525d1ae027SRobert Watson  * NOTE: This may only be called while the zone is being turn down, and not
7535d1ae027SRobert Watson  * during normal operation.  This is necessary in order that we do not have
7545d1ae027SRobert Watson  * to migrate CPUs to drain the per-CPU caches.
7555d1ae027SRobert Watson  *
7568355f576SJeff Roberson  * Arguments:
7578355f576SJeff Roberson  *	zone     The zone to drain, must be unlocked.
7588355f576SJeff Roberson  *
7598355f576SJeff Roberson  * Returns:
7608355f576SJeff Roberson  *	Nothing
7618355f576SJeff Roberson  */
7628355f576SJeff Roberson static void
7639643769aSJeff Roberson cache_drain(uma_zone_t zone)
7648355f576SJeff Roberson {
7658355f576SJeff Roberson 	uma_cache_t cache;
7668355f576SJeff Roberson 	int cpu;
7678355f576SJeff Roberson 
7688355f576SJeff Roberson 	/*
7695d1ae027SRobert Watson 	 * XXX: It is safe to not lock the per-CPU caches, because we're
7705d1ae027SRobert Watson 	 * tearing down the zone anyway.  I.e., there will be no further use
7715d1ae027SRobert Watson 	 * of the caches at this point.
7725d1ae027SRobert Watson 	 *
7735d1ae027SRobert Watson 	 * XXX: It would good to be able to assert that the zone is being
7745d1ae027SRobert Watson 	 * torn down to prevent improper use of cache_drain().
7755d1ae027SRobert Watson 	 *
7765d1ae027SRobert Watson 	 * XXX: We lock the zone before passing into bucket_cache_drain() as
7775d1ae027SRobert Watson 	 * it is used elsewhere.  Should the tear-down path be made special
7785d1ae027SRobert Watson 	 * there in some form?
7798355f576SJeff Roberson 	 */
7803aa6d94eSJohn Baldwin 	CPU_FOREACH(cpu) {
7818355f576SJeff Roberson 		cache = &zone->uz_cpu[cpu];
7828355f576SJeff Roberson 		bucket_drain(zone, cache->uc_allocbucket);
7838355f576SJeff Roberson 		bucket_drain(zone, cache->uc_freebucket);
784174ab450SBosko Milekic 		if (cache->uc_allocbucket != NULL)
7856fd34d6fSJeff Roberson 			bucket_free(zone, cache->uc_allocbucket, NULL);
786174ab450SBosko Milekic 		if (cache->uc_freebucket != NULL)
7876fd34d6fSJeff Roberson 			bucket_free(zone, cache->uc_freebucket, NULL);
788d56368d7SBosko Milekic 		cache->uc_allocbucket = cache->uc_freebucket = NULL;
789d56368d7SBosko Milekic 	}
790aaa8bb16SJeff Roberson 	ZONE_LOCK(zone);
791aaa8bb16SJeff Roberson 	bucket_cache_drain(zone);
792aaa8bb16SJeff Roberson 	ZONE_UNLOCK(zone);
793aaa8bb16SJeff Roberson }
794aaa8bb16SJeff Roberson 
795a2de44abSAlexander Motin static void
796a2de44abSAlexander Motin cache_shrink(uma_zone_t zone)
797a2de44abSAlexander Motin {
798a2de44abSAlexander Motin 
799a2de44abSAlexander Motin 	if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
800a2de44abSAlexander Motin 		return;
801a2de44abSAlexander Motin 
802a2de44abSAlexander Motin 	ZONE_LOCK(zone);
803a2de44abSAlexander Motin 	zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2;
804a2de44abSAlexander Motin 	ZONE_UNLOCK(zone);
805a2de44abSAlexander Motin }
806a2de44abSAlexander Motin 
807a2de44abSAlexander Motin static void
808a2de44abSAlexander Motin cache_drain_safe_cpu(uma_zone_t zone)
809a2de44abSAlexander Motin {
810a2de44abSAlexander Motin 	uma_cache_t cache;
8118a8d9d14SAlexander Motin 	uma_bucket_t b1, b2;
812ab3185d1SJeff Roberson 	int domain;
813a2de44abSAlexander Motin 
814a2de44abSAlexander Motin 	if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
815a2de44abSAlexander Motin 		return;
816a2de44abSAlexander Motin 
8178a8d9d14SAlexander Motin 	b1 = b2 = NULL;
818a2de44abSAlexander Motin 	ZONE_LOCK(zone);
819a2de44abSAlexander Motin 	critical_enter();
820ab3185d1SJeff Roberson 	if (zone->uz_flags & UMA_ZONE_NUMA)
821ab3185d1SJeff Roberson 		domain = PCPU_GET(domain);
822ab3185d1SJeff Roberson 	else
823ab3185d1SJeff Roberson 		domain = 0;
824a2de44abSAlexander Motin 	cache = &zone->uz_cpu[curcpu];
825a2de44abSAlexander Motin 	if (cache->uc_allocbucket) {
8268a8d9d14SAlexander Motin 		if (cache->uc_allocbucket->ub_cnt != 0)
8270f9b7bf3SMark Johnston 			zone_put_bucket(zone, &zone->uz_domain[domain],
8280f9b7bf3SMark Johnston 			    cache->uc_allocbucket, false);
8298a8d9d14SAlexander Motin 		else
8308a8d9d14SAlexander Motin 			b1 = cache->uc_allocbucket;
831a2de44abSAlexander Motin 		cache->uc_allocbucket = NULL;
832a2de44abSAlexander Motin 	}
833a2de44abSAlexander Motin 	if (cache->uc_freebucket) {
8348a8d9d14SAlexander Motin 		if (cache->uc_freebucket->ub_cnt != 0)
8350f9b7bf3SMark Johnston 			zone_put_bucket(zone, &zone->uz_domain[domain],
8360f9b7bf3SMark Johnston 			    cache->uc_freebucket, false);
8378a8d9d14SAlexander Motin 		else
8388a8d9d14SAlexander Motin 			b2 = cache->uc_freebucket;
839a2de44abSAlexander Motin 		cache->uc_freebucket = NULL;
840a2de44abSAlexander Motin 	}
841a2de44abSAlexander Motin 	critical_exit();
842a2de44abSAlexander Motin 	ZONE_UNLOCK(zone);
8438a8d9d14SAlexander Motin 	if (b1)
8448a8d9d14SAlexander Motin 		bucket_free(zone, b1, NULL);
8458a8d9d14SAlexander Motin 	if (b2)
8468a8d9d14SAlexander Motin 		bucket_free(zone, b2, NULL);
847a2de44abSAlexander Motin }
848a2de44abSAlexander Motin 
849a2de44abSAlexander Motin /*
850a2de44abSAlexander Motin  * Safely drain per-CPU caches of a zone(s) to alloc bucket.
851a2de44abSAlexander Motin  * This is an expensive call because it needs to bind to all CPUs
852a2de44abSAlexander Motin  * one by one and enter a critical section on each of them in order
853a2de44abSAlexander Motin  * to safely access their cache buckets.
854a2de44abSAlexander Motin  * Zone lock must not be held on call this function.
855a2de44abSAlexander Motin  */
856a2de44abSAlexander Motin static void
857a2de44abSAlexander Motin cache_drain_safe(uma_zone_t zone)
858a2de44abSAlexander Motin {
859a2de44abSAlexander Motin 	int cpu;
860a2de44abSAlexander Motin 
861a2de44abSAlexander Motin 	/*
862a2de44abSAlexander Motin 	 * Polite bucket sizes shrinking was not enouth, shrink aggressively.
863a2de44abSAlexander Motin 	 */
864a2de44abSAlexander Motin 	if (zone)
865a2de44abSAlexander Motin 		cache_shrink(zone);
866a2de44abSAlexander Motin 	else
867a2de44abSAlexander Motin 		zone_foreach(cache_shrink);
868a2de44abSAlexander Motin 
869a2de44abSAlexander Motin 	CPU_FOREACH(cpu) {
870a2de44abSAlexander Motin 		thread_lock(curthread);
871a2de44abSAlexander Motin 		sched_bind(curthread, cpu);
872a2de44abSAlexander Motin 		thread_unlock(curthread);
873a2de44abSAlexander Motin 
874a2de44abSAlexander Motin 		if (zone)
875a2de44abSAlexander Motin 			cache_drain_safe_cpu(zone);
876a2de44abSAlexander Motin 		else
877a2de44abSAlexander Motin 			zone_foreach(cache_drain_safe_cpu);
878a2de44abSAlexander Motin 	}
879a2de44abSAlexander Motin 	thread_lock(curthread);
880a2de44abSAlexander Motin 	sched_unbind(curthread);
881a2de44abSAlexander Motin 	thread_unlock(curthread);
882a2de44abSAlexander Motin }
883a2de44abSAlexander Motin 
884aaa8bb16SJeff Roberson /*
885aaa8bb16SJeff Roberson  * Drain the cached buckets from a zone.  Expects a locked zone on entry.
886aaa8bb16SJeff Roberson  */
887aaa8bb16SJeff Roberson static void
888aaa8bb16SJeff Roberson bucket_cache_drain(uma_zone_t zone)
889aaa8bb16SJeff Roberson {
890ab3185d1SJeff Roberson 	uma_zone_domain_t zdom;
891aaa8bb16SJeff Roberson 	uma_bucket_t bucket;
892ab3185d1SJeff Roberson 	int i;
8938355f576SJeff Roberson 
8948355f576SJeff Roberson 	/*
895ab3185d1SJeff Roberson 	 * Drain the bucket queues and free the buckets.
8968355f576SJeff Roberson 	 */
897ab3185d1SJeff Roberson 	for (i = 0; i < vm_ndomains; i++) {
898ab3185d1SJeff Roberson 		zdom = &zone->uz_domain[i];
8990f9b7bf3SMark Johnston 		while ((bucket = zone_try_fetch_bucket(zone, zdom, false)) !=
9000f9b7bf3SMark Johnston 		    NULL) {
9018355f576SJeff Roberson 			ZONE_UNLOCK(zone);
9028355f576SJeff Roberson 			bucket_drain(zone, bucket);
9036fd34d6fSJeff Roberson 			bucket_free(zone, bucket, NULL);
9048355f576SJeff Roberson 			ZONE_LOCK(zone);
9058355f576SJeff Roberson 		}
906ab3185d1SJeff Roberson 	}
907ace66b56SAlexander Motin 
908ace66b56SAlexander Motin 	/*
909ace66b56SAlexander Motin 	 * Shrink further bucket sizes.  Price of single zone lock collision
910ace66b56SAlexander Motin 	 * is probably lower then price of global cache drain.
911ace66b56SAlexander Motin 	 */
912ace66b56SAlexander Motin 	if (zone->uz_count > zone->uz_count_min)
913ace66b56SAlexander Motin 		zone->uz_count--;
9148355f576SJeff Roberson }
915fc03d22bSJeff Roberson 
916fc03d22bSJeff Roberson static void
917fc03d22bSJeff Roberson keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)
918fc03d22bSJeff Roberson {
919fc03d22bSJeff Roberson 	uint8_t *mem;
920fc03d22bSJeff Roberson 	int i;
921fc03d22bSJeff Roberson 	uint8_t flags;
922fc03d22bSJeff Roberson 
9231431a748SGleb Smirnoff 	CTR4(KTR_UMA, "keg_free_slab keg %s(%p) slab %p, returning %d bytes",
9241431a748SGleb Smirnoff 	    keg->uk_name, keg, slab, PAGE_SIZE * keg->uk_ppera);
9251431a748SGleb Smirnoff 
926fc03d22bSJeff Roberson 	mem = slab->us_data;
927fc03d22bSJeff Roberson 	flags = slab->us_flags;
928fc03d22bSJeff Roberson 	i = start;
929fc03d22bSJeff Roberson 	if (keg->uk_fini != NULL) {
930fc03d22bSJeff Roberson 		for (i--; i > -1; i--)
931c5deaf04SGleb Smirnoff #ifdef INVARIANTS
932c5deaf04SGleb Smirnoff 		/*
933c5deaf04SGleb Smirnoff 		 * trash_fini implies that dtor was trash_dtor. trash_fini
934c5deaf04SGleb Smirnoff 		 * would check that memory hasn't been modified since free,
935c5deaf04SGleb Smirnoff 		 * which executed trash_dtor.
936c5deaf04SGleb Smirnoff 		 * That's why we need to run uma_dbg_kskip() check here,
937c5deaf04SGleb Smirnoff 		 * albeit we don't make skip check for other init/fini
938c5deaf04SGleb Smirnoff 		 * invocations.
939c5deaf04SGleb Smirnoff 		 */
940c5deaf04SGleb Smirnoff 		if (!uma_dbg_kskip(keg, slab->us_data + (keg->uk_rsize * i)) ||
941c5deaf04SGleb Smirnoff 		    keg->uk_fini != trash_fini)
942c5deaf04SGleb Smirnoff #endif
943fc03d22bSJeff Roberson 			keg->uk_fini(slab->us_data + (keg->uk_rsize * i),
944fc03d22bSJeff Roberson 			    keg->uk_size);
945fc03d22bSJeff Roberson 	}
946fc03d22bSJeff Roberson 	if (keg->uk_flags & UMA_ZONE_OFFPAGE)
947fc03d22bSJeff Roberson 		zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
948fc03d22bSJeff Roberson 	keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
9492e47807cSJeff Roberson 	uma_total_dec(PAGE_SIZE * keg->uk_ppera);
9508355f576SJeff Roberson }
9518355f576SJeff Roberson 
9528355f576SJeff Roberson /*
953e20a199fSJeff Roberson  * Frees pages from a keg back to the system.  This is done on demand from
9548355f576SJeff Roberson  * the pageout daemon.
9558355f576SJeff Roberson  *
956e20a199fSJeff Roberson  * Returns nothing.
9578355f576SJeff Roberson  */
958e20a199fSJeff Roberson static void
959e20a199fSJeff Roberson keg_drain(uma_keg_t keg)
9608355f576SJeff Roberson {
9611e183df2SStefan Farfeleder 	struct slabhead freeslabs = { 0 };
962ab3185d1SJeff Roberson 	uma_domain_t dom;
963829be516SMark Johnston 	uma_slab_t slab, tmp;
964ab3185d1SJeff Roberson 	int i;
9658355f576SJeff Roberson 
9668355f576SJeff Roberson 	/*
967e20a199fSJeff Roberson 	 * We don't want to take pages from statically allocated kegs at this
9688355f576SJeff Roberson 	 * time
9698355f576SJeff Roberson 	 */
970099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
9718355f576SJeff Roberson 		return;
9728355f576SJeff Roberson 
9731431a748SGleb Smirnoff 	CTR3(KTR_UMA, "keg_drain %s(%p) free items: %u",
9741431a748SGleb Smirnoff 	    keg->uk_name, keg, keg->uk_free);
975e20a199fSJeff Roberson 	KEG_LOCK(keg);
976099a0e58SBosko Milekic 	if (keg->uk_free == 0)
9778355f576SJeff Roberson 		goto finished;
9788355f576SJeff Roberson 
979ab3185d1SJeff Roberson 	for (i = 0; i < vm_ndomains; i++) {
980ab3185d1SJeff Roberson 		dom = &keg->uk_domain[i];
981ab3185d1SJeff Roberson 		LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) {
982829be516SMark Johnston 			/* We have nowhere to free these to. */
983829be516SMark Johnston 			if (slab->us_flags & UMA_SLAB_BOOT)
9848355f576SJeff Roberson 				continue;
9858355f576SJeff Roberson 
9868355f576SJeff Roberson 			LIST_REMOVE(slab, us_link);
987099a0e58SBosko Milekic 			keg->uk_pages -= keg->uk_ppera;
988099a0e58SBosko Milekic 			keg->uk_free -= keg->uk_ipers;
989713deb36SJeff Roberson 
990099a0e58SBosko Milekic 			if (keg->uk_flags & UMA_ZONE_HASH)
991ab3185d1SJeff Roberson 				UMA_HASH_REMOVE(&keg->uk_hash, slab,
992ab3185d1SJeff Roberson 				    slab->us_data);
993713deb36SJeff Roberson 
994713deb36SJeff Roberson 			SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink);
995713deb36SJeff Roberson 		}
996ab3185d1SJeff Roberson 	}
997ab3185d1SJeff Roberson 
998713deb36SJeff Roberson finished:
999e20a199fSJeff Roberson 	KEG_UNLOCK(keg);
1000713deb36SJeff Roberson 
1001713deb36SJeff Roberson 	while ((slab = SLIST_FIRST(&freeslabs)) != NULL) {
1002713deb36SJeff Roberson 		SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink);
10031645995bSKirk McKusick 		keg_free_slab(keg, slab, keg->uk_ipers);
10048355f576SJeff Roberson 	}
10058355f576SJeff Roberson }
10068355f576SJeff Roberson 
1007e20a199fSJeff Roberson static void
1008e20a199fSJeff Roberson zone_drain_wait(uma_zone_t zone, int waitok)
1009e20a199fSJeff Roberson {
1010e20a199fSJeff Roberson 
10118355f576SJeff Roberson 	/*
1012e20a199fSJeff Roberson 	 * Set draining to interlock with zone_dtor() so we can release our
1013e20a199fSJeff Roberson 	 * locks as we go.  Only dtor() should do a WAITOK call since it
1014e20a199fSJeff Roberson 	 * is the only call that knows the structure will still be available
1015e20a199fSJeff Roberson 	 * when it wakes up.
1016e20a199fSJeff Roberson 	 */
1017e20a199fSJeff Roberson 	ZONE_LOCK(zone);
1018e20a199fSJeff Roberson 	while (zone->uz_flags & UMA_ZFLAG_DRAINING) {
1019e20a199fSJeff Roberson 		if (waitok == M_NOWAIT)
1020e20a199fSJeff Roberson 			goto out;
1021af526374SJeff Roberson 		msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1);
1022e20a199fSJeff Roberson 	}
1023e20a199fSJeff Roberson 	zone->uz_flags |= UMA_ZFLAG_DRAINING;
1024e20a199fSJeff Roberson 	bucket_cache_drain(zone);
1025e20a199fSJeff Roberson 	ZONE_UNLOCK(zone);
1026e20a199fSJeff Roberson 	/*
1027e20a199fSJeff Roberson 	 * The DRAINING flag protects us from being freed while
1028111fbcd5SBryan Venteicher 	 * we're running.  Normally the uma_rwlock would protect us but we
1029e20a199fSJeff Roberson 	 * must be able to release and acquire the right lock for each keg.
1030e20a199fSJeff Roberson 	 */
1031bb15d1c7SGleb Smirnoff 	keg_drain(zone->uz_keg);
1032e20a199fSJeff Roberson 	ZONE_LOCK(zone);
1033e20a199fSJeff Roberson 	zone->uz_flags &= ~UMA_ZFLAG_DRAINING;
1034e20a199fSJeff Roberson 	wakeup(zone);
1035e20a199fSJeff Roberson out:
1036e20a199fSJeff Roberson 	ZONE_UNLOCK(zone);
1037e20a199fSJeff Roberson }
1038e20a199fSJeff Roberson 
1039e20a199fSJeff Roberson void
1040e20a199fSJeff Roberson zone_drain(uma_zone_t zone)
1041e20a199fSJeff Roberson {
1042e20a199fSJeff Roberson 
1043e20a199fSJeff Roberson 	zone_drain_wait(zone, M_NOWAIT);
1044e20a199fSJeff Roberson }
1045e20a199fSJeff Roberson 
1046e20a199fSJeff Roberson /*
1047e20a199fSJeff Roberson  * Allocate a new slab for a keg.  This does not insert the slab onto a list.
1048194a979eSMark Johnston  * If the allocation was successful, the keg lock will be held upon return,
1049194a979eSMark Johnston  * otherwise the keg will be left unlocked.
10508355f576SJeff Roberson  *
10518355f576SJeff Roberson  * Arguments:
105286220393SMark Johnston  *	flags   Wait flags for the item initialization routine
105386220393SMark Johnston  *	aflags  Wait flags for the slab allocation
10548355f576SJeff Roberson  *
10558355f576SJeff Roberson  * Returns:
10568355f576SJeff Roberson  *	The slab that was allocated or NULL if there is no memory and the
10578355f576SJeff Roberson  *	caller specified M_NOWAIT.
10588355f576SJeff Roberson  */
10598355f576SJeff Roberson static uma_slab_t
106086220393SMark Johnston keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags,
106186220393SMark Johnston     int aflags)
10628355f576SJeff Roberson {
1063e20a199fSJeff Roberson 	uma_alloc allocf;
1064099a0e58SBosko Milekic 	uma_slab_t slab;
10652e47807cSJeff Roberson 	unsigned long size;
106685dcf349SGleb Smirnoff 	uint8_t *mem;
106786220393SMark Johnston 	uint8_t sflags;
10688355f576SJeff Roberson 	int i;
10698355f576SJeff Roberson 
1070ab3185d1SJeff Roberson 	KASSERT(domain >= 0 && domain < vm_ndomains,
1071ab3185d1SJeff Roberson 	    ("keg_alloc_slab: domain %d out of range", domain));
1072bb15d1c7SGleb Smirnoff 	KEG_LOCK_ASSERT(keg);
1073bb15d1c7SGleb Smirnoff 	MPASS(zone->uz_lockptr == &keg->uk_lock);
1074a553d4b8SJeff Roberson 
1075e20a199fSJeff Roberson 	allocf = keg->uk_allocf;
1076e20a199fSJeff Roberson 	KEG_UNLOCK(keg);
1077a553d4b8SJeff Roberson 
1078194a979eSMark Johnston 	slab = NULL;
1079194a979eSMark Johnston 	mem = NULL;
1080099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
108186220393SMark Johnston 		slab = zone_alloc_item(keg->uk_slabzone, NULL, domain, aflags);
1082fc03d22bSJeff Roberson 		if (slab == NULL)
1083fc03d22bSJeff Roberson 			goto out;
1084a553d4b8SJeff Roberson 	}
1085a553d4b8SJeff Roberson 
10863370c5bfSJeff Roberson 	/*
10873370c5bfSJeff Roberson 	 * This reproduces the old vm_zone behavior of zero filling pages the
10883370c5bfSJeff Roberson 	 * first time they are added to a zone.
10893370c5bfSJeff Roberson 	 *
10903370c5bfSJeff Roberson 	 * Malloced items are zeroed in uma_zalloc.
10913370c5bfSJeff Roberson 	 */
10923370c5bfSJeff Roberson 
1093099a0e58SBosko Milekic 	if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0)
109486220393SMark Johnston 		aflags |= M_ZERO;
10953370c5bfSJeff Roberson 	else
109686220393SMark Johnston 		aflags &= ~M_ZERO;
10973370c5bfSJeff Roberson 
1098263811f7SKip Macy 	if (keg->uk_flags & UMA_ZONE_NODUMP)
109986220393SMark Johnston 		aflags |= M_NODUMP;
1100263811f7SKip Macy 
1101e20a199fSJeff Roberson 	/* zone is passed for legacy reasons. */
1102194a979eSMark Johnston 	size = keg->uk_ppera * PAGE_SIZE;
110386220393SMark Johnston 	mem = allocf(zone, size, domain, &sflags, aflags);
1104a553d4b8SJeff Roberson 	if (mem == NULL) {
1105b23f72e9SBrian Feldman 		if (keg->uk_flags & UMA_ZONE_OFFPAGE)
11060095a784SJeff Roberson 			zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
1107fc03d22bSJeff Roberson 		slab = NULL;
1108fc03d22bSJeff Roberson 		goto out;
1109a553d4b8SJeff Roberson 	}
11102e47807cSJeff Roberson 	uma_total_inc(size);
11118355f576SJeff Roberson 
11125c0e403bSJeff Roberson 	/* Point the slab into the allocated memory */
1113099a0e58SBosko Milekic 	if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
1114099a0e58SBosko Milekic 		slab = (uma_slab_t )(mem + keg->uk_pgoff);
11155c0e403bSJeff Roberson 
1116e20a199fSJeff Roberson 	if (keg->uk_flags & UMA_ZONE_VTOSLAB)
1117099a0e58SBosko Milekic 		for (i = 0; i < keg->uk_ppera; i++)
111899571dc3SJeff Roberson 			vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab);
11198355f576SJeff Roberson 
1120099a0e58SBosko Milekic 	slab->us_keg = keg;
11218355f576SJeff Roberson 	slab->us_data = mem;
1122099a0e58SBosko Milekic 	slab->us_freecount = keg->uk_ipers;
112386220393SMark Johnston 	slab->us_flags = sflags;
1124ab3185d1SJeff Roberson 	slab->us_domain = domain;
1125ef72505eSJeff Roberson 	BIT_FILL(SLAB_SETSIZE, &slab->us_free);
1126ef72505eSJeff Roberson #ifdef INVARIANTS
1127ef72505eSJeff Roberson 	BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree);
1128ef72505eSJeff Roberson #endif
1129099a0e58SBosko Milekic 
1130b23f72e9SBrian Feldman 	if (keg->uk_init != NULL) {
1131099a0e58SBosko Milekic 		for (i = 0; i < keg->uk_ipers; i++)
1132b23f72e9SBrian Feldman 			if (keg->uk_init(slab->us_data + (keg->uk_rsize * i),
113386220393SMark Johnston 			    keg->uk_size, flags) != 0)
1134b23f72e9SBrian Feldman 				break;
1135b23f72e9SBrian Feldman 		if (i != keg->uk_ipers) {
1136fc03d22bSJeff Roberson 			keg_free_slab(keg, slab, i);
1137fc03d22bSJeff Roberson 			slab = NULL;
1138fc03d22bSJeff Roberson 			goto out;
1139b23f72e9SBrian Feldman 		}
1140b23f72e9SBrian Feldman 	}
1141e20a199fSJeff Roberson 	KEG_LOCK(keg);
11425c0e403bSJeff Roberson 
11431431a748SGleb Smirnoff 	CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)",
11441431a748SGleb Smirnoff 	    slab, keg->uk_name, keg);
11451431a748SGleb Smirnoff 
1146099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_HASH)
1147099a0e58SBosko Milekic 		UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
11488355f576SJeff Roberson 
1149099a0e58SBosko Milekic 	keg->uk_pages += keg->uk_ppera;
1150099a0e58SBosko Milekic 	keg->uk_free += keg->uk_ipers;
11518355f576SJeff Roberson 
1152194a979eSMark Johnston out:
11538355f576SJeff Roberson 	return (slab);
11548355f576SJeff Roberson }
11558355f576SJeff Roberson 
11568355f576SJeff Roberson /*
1157009b6fcbSJeff Roberson  * This function is intended to be used early on in place of page_alloc() so
1158009b6fcbSJeff Roberson  * that we may use the boot time page cache to satisfy allocations before
1159009b6fcbSJeff Roberson  * the VM is ready.
1160009b6fcbSJeff Roberson  */
1161009b6fcbSJeff Roberson static void *
1162ab3185d1SJeff Roberson startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
1163ab3185d1SJeff Roberson     int wait)
1164009b6fcbSJeff Roberson {
1165099a0e58SBosko Milekic 	uma_keg_t keg;
1166ac0a6fd0SGleb Smirnoff 	void *mem;
1167ac0a6fd0SGleb Smirnoff 	int pages;
1168099a0e58SBosko Milekic 
1169bb15d1c7SGleb Smirnoff 	keg = zone->uz_keg;
1170009b6fcbSJeff Roberson 	/*
1171f7d35785SGleb Smirnoff 	 * If we are in BOOT_BUCKETS or higher, than switch to real
1172f7d35785SGleb Smirnoff 	 * allocator.  Zones with page sized slabs switch at BOOT_PAGEALLOC.
1173009b6fcbSJeff Roberson 	 */
1174f7d35785SGleb Smirnoff 	switch (booted) {
1175f7d35785SGleb Smirnoff 		case BOOT_COLD:
1176f7d35785SGleb Smirnoff 		case BOOT_STRAPPED:
1177f7d35785SGleb Smirnoff 			break;
1178f7d35785SGleb Smirnoff 		case BOOT_PAGEALLOC:
1179f7d35785SGleb Smirnoff 			if (keg->uk_ppera > 1)
1180f7d35785SGleb Smirnoff 				break;
1181f7d35785SGleb Smirnoff 		case BOOT_BUCKETS:
1182f7d35785SGleb Smirnoff 		case BOOT_RUNNING:
1183009b6fcbSJeff Roberson #ifdef UMA_MD_SMALL_ALLOC
1184f7d35785SGleb Smirnoff 			keg->uk_allocf = (keg->uk_ppera > 1) ?
1185f7d35785SGleb Smirnoff 			    page_alloc : uma_small_alloc;
1186009b6fcbSJeff Roberson #else
1187099a0e58SBosko Milekic 			keg->uk_allocf = page_alloc;
1188009b6fcbSJeff Roberson #endif
1189ab3185d1SJeff Roberson 			return keg->uk_allocf(zone, bytes, domain, pflag, wait);
1190009b6fcbSJeff Roberson 	}
1191009b6fcbSJeff Roberson 
1192009b6fcbSJeff Roberson 	/*
1193f7d35785SGleb Smirnoff 	 * Check our small startup cache to see if it has pages remaining.
1194f7d35785SGleb Smirnoff 	 */
1195f7d35785SGleb Smirnoff 	pages = howmany(bytes, PAGE_SIZE);
1196f7d35785SGleb Smirnoff 	KASSERT(pages > 0, ("%s can't reserve 0 pages", __func__));
1197f7d35785SGleb Smirnoff 	if (pages > boot_pages)
1198f7d35785SGleb Smirnoff 		panic("UMA zone \"%s\": Increase vm.boot_pages", zone->uz_name);
1199f7d35785SGleb Smirnoff #ifdef DIAGNOSTIC
1200f7d35785SGleb Smirnoff 	printf("%s from \"%s\", %d boot pages left\n", __func__, zone->uz_name,
1201f7d35785SGleb Smirnoff 	    boot_pages);
1202f7d35785SGleb Smirnoff #endif
1203f7d35785SGleb Smirnoff 	mem = bootmem;
1204f7d35785SGleb Smirnoff 	boot_pages -= pages;
1205f7d35785SGleb Smirnoff 	bootmem += pages * PAGE_SIZE;
1206f7d35785SGleb Smirnoff 	*pflag = UMA_SLAB_BOOT;
1207f7d35785SGleb Smirnoff 
1208f7d35785SGleb Smirnoff 	return (mem);
1209f7d35785SGleb Smirnoff }
1210f7d35785SGleb Smirnoff 
1211f7d35785SGleb Smirnoff /*
12128355f576SJeff Roberson  * Allocates a number of pages from the system
12138355f576SJeff Roberson  *
12148355f576SJeff Roberson  * Arguments:
12158355f576SJeff Roberson  *	bytes  The number of bytes requested
12168355f576SJeff Roberson  *	wait  Shall we wait?
12178355f576SJeff Roberson  *
12188355f576SJeff Roberson  * Returns:
12198355f576SJeff Roberson  *	A pointer to the alloced memory or possibly
12208355f576SJeff Roberson  *	NULL if M_NOWAIT is set.
12218355f576SJeff Roberson  */
12228355f576SJeff Roberson static void *
1223ab3185d1SJeff Roberson page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
1224ab3185d1SJeff Roberson     int wait)
12258355f576SJeff Roberson {
12268355f576SJeff Roberson 	void *p;	/* Returned page */
12278355f576SJeff Roberson 
12282e47807cSJeff Roberson 	*pflag = UMA_SLAB_KERNEL;
12299978bd99SMark Johnston 	p = (void *)kmem_malloc_domainset(DOMAINSET_FIXED(domain), bytes, wait);
12308355f576SJeff Roberson 
12318355f576SJeff Roberson 	return (p);
12328355f576SJeff Roberson }
12338355f576SJeff Roberson 
1234ab3059a8SMatt Macy static void *
1235ab3059a8SMatt Macy pcpu_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag,
1236ab3059a8SMatt Macy     int wait)
1237ab3059a8SMatt Macy {
1238ab3059a8SMatt Macy 	struct pglist alloctail;
1239ab3059a8SMatt Macy 	vm_offset_t addr, zkva;
1240ab3059a8SMatt Macy 	int cpu, flags;
1241ab3059a8SMatt Macy 	vm_page_t p, p_next;
1242ab3059a8SMatt Macy #ifdef NUMA
1243ab3059a8SMatt Macy 	struct pcpu *pc;
1244ab3059a8SMatt Macy #endif
1245ab3059a8SMatt Macy 
1246ab3059a8SMatt Macy 	MPASS(bytes == (mp_maxid + 1) * PAGE_SIZE);
1247ab3059a8SMatt Macy 
1248013072f0SMark Johnston 	TAILQ_INIT(&alloctail);
1249ab3059a8SMatt Macy 	flags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ |
1250013072f0SMark Johnston 	    malloc2vm_flags(wait);
1251013072f0SMark Johnston 	*pflag = UMA_SLAB_KERNEL;
1252ab3059a8SMatt Macy 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
1253ab3059a8SMatt Macy 		if (CPU_ABSENT(cpu)) {
1254ab3059a8SMatt Macy 			p = vm_page_alloc(NULL, 0, flags);
1255ab3059a8SMatt Macy 		} else {
1256ab3059a8SMatt Macy #ifndef NUMA
1257ab3059a8SMatt Macy 			p = vm_page_alloc(NULL, 0, flags);
1258ab3059a8SMatt Macy #else
1259ab3059a8SMatt Macy 			pc = pcpu_find(cpu);
1260ab3059a8SMatt Macy 			p = vm_page_alloc_domain(NULL, 0, pc->pc_domain, flags);
1261ab3059a8SMatt Macy 			if (__predict_false(p == NULL))
1262ab3059a8SMatt Macy 				p = vm_page_alloc(NULL, 0, flags);
1263ab3059a8SMatt Macy #endif
1264ab3059a8SMatt Macy 		}
1265ab3059a8SMatt Macy 		if (__predict_false(p == NULL))
1266ab3059a8SMatt Macy 			goto fail;
1267ab3059a8SMatt Macy 		TAILQ_INSERT_TAIL(&alloctail, p, listq);
1268ab3059a8SMatt Macy 	}
1269ab3059a8SMatt Macy 	if ((addr = kva_alloc(bytes)) == 0)
1270ab3059a8SMatt Macy 		goto fail;
1271ab3059a8SMatt Macy 	zkva = addr;
1272ab3059a8SMatt Macy 	TAILQ_FOREACH(p, &alloctail, listq) {
1273ab3059a8SMatt Macy 		pmap_qenter(zkva, &p, 1);
1274ab3059a8SMatt Macy 		zkva += PAGE_SIZE;
1275ab3059a8SMatt Macy 	}
1276ab3059a8SMatt Macy 	return ((void*)addr);
1277ab3059a8SMatt Macy  fail:
1278ab3059a8SMatt Macy 	TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
1279ab3059a8SMatt Macy 		vm_page_unwire(p, PQ_NONE);
1280ab3059a8SMatt Macy 		vm_page_free(p);
1281ab3059a8SMatt Macy 	}
1282ab3059a8SMatt Macy 	return (NULL);
1283ab3059a8SMatt Macy }
1284ab3059a8SMatt Macy 
12858355f576SJeff Roberson /*
12868355f576SJeff Roberson  * Allocates a number of pages from within an object
12878355f576SJeff Roberson  *
12888355f576SJeff Roberson  * Arguments:
12898355f576SJeff Roberson  *	bytes  The number of bytes requested
12908355f576SJeff Roberson  *	wait   Shall we wait?
12918355f576SJeff Roberson  *
12928355f576SJeff Roberson  * Returns:
12938355f576SJeff Roberson  *	A pointer to the alloced memory or possibly
12948355f576SJeff Roberson  *	NULL if M_NOWAIT is set.
12958355f576SJeff Roberson  */
12968355f576SJeff Roberson static void *
1297ab3185d1SJeff Roberson noobj_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
1298ab3185d1SJeff Roberson     int wait)
12998355f576SJeff Roberson {
1300a4915c21SAttilio Rao 	TAILQ_HEAD(, vm_page) alloctail;
1301a4915c21SAttilio Rao 	u_long npages;
1302b245ac95SAlan Cox 	vm_offset_t retkva, zkva;
1303a4915c21SAttilio Rao 	vm_page_t p, p_next;
1304e20a199fSJeff Roberson 	uma_keg_t keg;
13058355f576SJeff Roberson 
1306a4915c21SAttilio Rao 	TAILQ_INIT(&alloctail);
1307bb15d1c7SGleb Smirnoff 	keg = zone->uz_keg;
1308a4915c21SAttilio Rao 
1309a4915c21SAttilio Rao 	npages = howmany(bytes, PAGE_SIZE);
1310a4915c21SAttilio Rao 	while (npages > 0) {
1311ab3185d1SJeff Roberson 		p = vm_page_alloc_domain(NULL, 0, domain, VM_ALLOC_INTERRUPT |
13128d6fbbb8SJeff Roberson 		    VM_ALLOC_WIRED | VM_ALLOC_NOOBJ |
1313772c8b67SKonstantin Belousov 		    ((wait & M_WAITOK) != 0 ? VM_ALLOC_WAITOK :
1314772c8b67SKonstantin Belousov 		    VM_ALLOC_NOWAIT));
1315a4915c21SAttilio Rao 		if (p != NULL) {
1316a4915c21SAttilio Rao 			/*
1317a4915c21SAttilio Rao 			 * Since the page does not belong to an object, its
1318a4915c21SAttilio Rao 			 * listq is unused.
1319a4915c21SAttilio Rao 			 */
1320a4915c21SAttilio Rao 			TAILQ_INSERT_TAIL(&alloctail, p, listq);
1321a4915c21SAttilio Rao 			npages--;
1322a4915c21SAttilio Rao 			continue;
1323a4915c21SAttilio Rao 		}
13248355f576SJeff Roberson 		/*
1325a4915c21SAttilio Rao 		 * Page allocation failed, free intermediate pages and
1326a4915c21SAttilio Rao 		 * exit.
13278355f576SJeff Roberson 		 */
1328a4915c21SAttilio Rao 		TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
1329087a6132SAlan Cox 			vm_page_unwire(p, PQ_NONE);
1330b245ac95SAlan Cox 			vm_page_free(p);
1331b245ac95SAlan Cox 		}
1332a4915c21SAttilio Rao 		return (NULL);
1333b245ac95SAlan Cox 	}
13348355f576SJeff Roberson 	*flags = UMA_SLAB_PRIV;
1335a4915c21SAttilio Rao 	zkva = keg->uk_kva +
1336a4915c21SAttilio Rao 	    atomic_fetchadd_long(&keg->uk_offset, round_page(bytes));
1337a4915c21SAttilio Rao 	retkva = zkva;
1338a4915c21SAttilio Rao 	TAILQ_FOREACH(p, &alloctail, listq) {
1339a4915c21SAttilio Rao 		pmap_qenter(zkva, &p, 1);
1340a4915c21SAttilio Rao 		zkva += PAGE_SIZE;
1341a4915c21SAttilio Rao 	}
13428355f576SJeff Roberson 
13438355f576SJeff Roberson 	return ((void *)retkva);
13448355f576SJeff Roberson }
13458355f576SJeff Roberson 
13468355f576SJeff Roberson /*
13478355f576SJeff Roberson  * Frees a number of pages to the system
13488355f576SJeff Roberson  *
13498355f576SJeff Roberson  * Arguments:
13508355f576SJeff Roberson  *	mem   A pointer to the memory to be freed
13518355f576SJeff Roberson  *	size  The size of the memory being freed
13528355f576SJeff Roberson  *	flags The original p->us_flags field
13538355f576SJeff Roberson  *
13548355f576SJeff Roberson  * Returns:
13558355f576SJeff Roberson  *	Nothing
13568355f576SJeff Roberson  */
13578355f576SJeff Roberson static void
1358f2c2231eSRyan Stone page_free(void *mem, vm_size_t size, uint8_t flags)
13598355f576SJeff Roberson {
13603370c5bfSJeff Roberson 
136149bfa624SAlan Cox 	if ((flags & UMA_SLAB_KERNEL) == 0)
1362b5345ef1SJustin Hibbits 		panic("UMA: page_free used with invalid flags %x", flags);
13638355f576SJeff Roberson 
136449bfa624SAlan Cox 	kmem_free((vm_offset_t)mem, size);
13658355f576SJeff Roberson }
13668355f576SJeff Roberson 
13678355f576SJeff Roberson /*
1368ab3059a8SMatt Macy  * Frees pcpu zone allocations
1369ab3059a8SMatt Macy  *
1370ab3059a8SMatt Macy  * Arguments:
1371ab3059a8SMatt Macy  *	mem   A pointer to the memory to be freed
1372ab3059a8SMatt Macy  *	size  The size of the memory being freed
1373ab3059a8SMatt Macy  *	flags The original p->us_flags field
1374ab3059a8SMatt Macy  *
1375ab3059a8SMatt Macy  * Returns:
1376ab3059a8SMatt Macy  *	Nothing
1377ab3059a8SMatt Macy  */
1378ab3059a8SMatt Macy static void
1379ab3059a8SMatt Macy pcpu_page_free(void *mem, vm_size_t size, uint8_t flags)
1380ab3059a8SMatt Macy {
1381ab3059a8SMatt Macy 	vm_offset_t sva, curva;
1382ab3059a8SMatt Macy 	vm_paddr_t paddr;
1383ab3059a8SMatt Macy 	vm_page_t m;
1384ab3059a8SMatt Macy 
1385ab3059a8SMatt Macy 	MPASS(size == (mp_maxid+1)*PAGE_SIZE);
1386ab3059a8SMatt Macy 	sva = (vm_offset_t)mem;
1387ab3059a8SMatt Macy 	for (curva = sva; curva < sva + size; curva += PAGE_SIZE) {
1388ab3059a8SMatt Macy 		paddr = pmap_kextract(curva);
1389ab3059a8SMatt Macy 		m = PHYS_TO_VM_PAGE(paddr);
1390ab3059a8SMatt Macy 		vm_page_unwire(m, PQ_NONE);
1391ab3059a8SMatt Macy 		vm_page_free(m);
1392ab3059a8SMatt Macy 	}
1393ab3059a8SMatt Macy 	pmap_qremove(sva, size >> PAGE_SHIFT);
1394ab3059a8SMatt Macy 	kva_free(sva, size);
1395ab3059a8SMatt Macy }
1396ab3059a8SMatt Macy 
1397ab3059a8SMatt Macy 
1398ab3059a8SMatt Macy /*
13998355f576SJeff Roberson  * Zero fill initializer
14008355f576SJeff Roberson  *
14018355f576SJeff Roberson  * Arguments/Returns follow uma_init specifications
14028355f576SJeff Roberson  */
1403b23f72e9SBrian Feldman static int
1404b23f72e9SBrian Feldman zero_init(void *mem, int size, int flags)
14058355f576SJeff Roberson {
14068355f576SJeff Roberson 	bzero(mem, size);
1407b23f72e9SBrian Feldman 	return (0);
14088355f576SJeff Roberson }
14098355f576SJeff Roberson 
14108355f576SJeff Roberson /*
1411e20a199fSJeff Roberson  * Finish creating a small uma keg.  This calculates ipers, and the keg size.
14128355f576SJeff Roberson  *
14138355f576SJeff Roberson  * Arguments
1414e20a199fSJeff Roberson  *	keg  The zone we should initialize
14158355f576SJeff Roberson  *
14168355f576SJeff Roberson  * Returns
14178355f576SJeff Roberson  *	Nothing
14188355f576SJeff Roberson  */
14198355f576SJeff Roberson static void
1420e20a199fSJeff Roberson keg_small_init(uma_keg_t keg)
14218355f576SJeff Roberson {
1422244f4554SBosko Milekic 	u_int rsize;
1423244f4554SBosko Milekic 	u_int memused;
1424244f4554SBosko Milekic 	u_int wastedspace;
1425244f4554SBosko Milekic 	u_int shsize;
1426a55ebb7cSAndriy Gapon 	u_int slabsize;
14278355f576SJeff Roberson 
1428ad97af7eSGleb Smirnoff 	if (keg->uk_flags & UMA_ZONE_PCPU) {
142996c85efbSNathan Whitehorn 		u_int ncpus = (mp_maxid + 1) ? (mp_maxid + 1) : MAXCPU;
1430e28a647dSGleb Smirnoff 
1431ab3059a8SMatt Macy 		slabsize = UMA_PCPU_ALLOC_SIZE;
1432ab3059a8SMatt Macy 		keg->uk_ppera = ncpus;
1433ad97af7eSGleb Smirnoff 	} else {
1434a55ebb7cSAndriy Gapon 		slabsize = UMA_SLAB_SIZE;
1435ad97af7eSGleb Smirnoff 		keg->uk_ppera = 1;
1436ad97af7eSGleb Smirnoff 	}
1437ad97af7eSGleb Smirnoff 
1438ef72505eSJeff Roberson 	/*
1439ef72505eSJeff Roberson 	 * Calculate the size of each allocation (rsize) according to
1440ef72505eSJeff Roberson 	 * alignment.  If the requested size is smaller than we have
1441ef72505eSJeff Roberson 	 * allocation bits for we round it up.
1442ef72505eSJeff Roberson 	 */
1443099a0e58SBosko Milekic 	rsize = keg->uk_size;
1444a55ebb7cSAndriy Gapon 	if (rsize < slabsize / SLAB_SETSIZE)
1445a55ebb7cSAndriy Gapon 		rsize = slabsize / SLAB_SETSIZE;
1446099a0e58SBosko Milekic 	if (rsize & keg->uk_align)
1447099a0e58SBosko Milekic 		rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
1448099a0e58SBosko Milekic 	keg->uk_rsize = rsize;
1449ad97af7eSGleb Smirnoff 
1450ad97af7eSGleb Smirnoff 	KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
1451ab3059a8SMatt Macy 	    keg->uk_rsize < UMA_PCPU_ALLOC_SIZE,
1452ad97af7eSGleb Smirnoff 	    ("%s: size %u too large", __func__, keg->uk_rsize));
14538355f576SJeff Roberson 
1454ef72505eSJeff Roberson 	if (keg->uk_flags & UMA_ZONE_OFFPAGE)
14552864dbbfSGleb Smirnoff 		shsize = 0;
1456ef72505eSJeff Roberson 	else
14573d5e3df7SGleb Smirnoff 		shsize = SIZEOF_UMA_SLAB;
14588355f576SJeff Roberson 
14591ca6ed45SGleb Smirnoff 	if (rsize <= slabsize - shsize)
1460a55ebb7cSAndriy Gapon 		keg->uk_ipers = (slabsize - shsize) / rsize;
14611ca6ed45SGleb Smirnoff 	else {
14621ca6ed45SGleb Smirnoff 		/* Handle special case when we have 1 item per slab, so
14631ca6ed45SGleb Smirnoff 		 * alignment requirement can be relaxed. */
14641ca6ed45SGleb Smirnoff 		KASSERT(keg->uk_size <= slabsize - shsize,
14651ca6ed45SGleb Smirnoff 		    ("%s: size %u greater than slab", __func__, keg->uk_size));
14661ca6ed45SGleb Smirnoff 		keg->uk_ipers = 1;
14671ca6ed45SGleb Smirnoff 	}
1468ef72505eSJeff Roberson 	KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
1469ad97af7eSGleb Smirnoff 	    ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1470ad97af7eSGleb Smirnoff 
1471244f4554SBosko Milekic 	memused = keg->uk_ipers * rsize + shsize;
1472a55ebb7cSAndriy Gapon 	wastedspace = slabsize - memused;
1473244f4554SBosko Milekic 
147420e8e865SBosko Milekic 	/*
1475244f4554SBosko Milekic 	 * We can't do OFFPAGE if we're internal or if we've been
147620e8e865SBosko Milekic 	 * asked to not go to the VM for buckets.  If we do this we
14776fd34d6fSJeff Roberson 	 * may end up going to the VM  for slabs which we do not
14786fd34d6fSJeff Roberson 	 * want to do if we're UMA_ZFLAG_CACHEONLY as a result
14796fd34d6fSJeff Roberson 	 * of UMA_ZONE_VM, which clearly forbids it.
148020e8e865SBosko Milekic 	 */
1481099a0e58SBosko Milekic 	if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
1482099a0e58SBosko Milekic 	    (keg->uk_flags & UMA_ZFLAG_CACHEONLY))
14838355f576SJeff Roberson 		return;
1484244f4554SBosko Milekic 
1485ef72505eSJeff Roberson 	/*
1486ef72505eSJeff Roberson 	 * See if using an OFFPAGE slab will limit our waste.  Only do
1487ef72505eSJeff Roberson 	 * this if it permits more items per-slab.
1488ef72505eSJeff Roberson 	 *
1489ef72505eSJeff Roberson 	 * XXX We could try growing slabsize to limit max waste as well.
1490ef72505eSJeff Roberson 	 * Historically this was not done because the VM could not
1491ef72505eSJeff Roberson 	 * efficiently handle contiguous allocations.
1492ef72505eSJeff Roberson 	 */
1493a55ebb7cSAndriy Gapon 	if ((wastedspace >= slabsize / UMA_MAX_WASTE) &&
1494a55ebb7cSAndriy Gapon 	    (keg->uk_ipers < (slabsize / keg->uk_rsize))) {
1495a55ebb7cSAndriy Gapon 		keg->uk_ipers = slabsize / keg->uk_rsize;
1496ef72505eSJeff Roberson 		KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
1497ad97af7eSGleb Smirnoff 		    ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
14981431a748SGleb Smirnoff 		CTR6(KTR_UMA, "UMA decided we need offpage slab headers for "
14991431a748SGleb Smirnoff 		    "keg: %s(%p), calculated wastedspace = %d, "
1500244f4554SBosko Milekic 		    "maximum wasted space allowed = %d, "
1501244f4554SBosko Milekic 		    "calculated ipers = %d, "
15021431a748SGleb Smirnoff 		    "new wasted space = %d\n", keg->uk_name, keg, wastedspace,
1503a55ebb7cSAndriy Gapon 		    slabsize / UMA_MAX_WASTE, keg->uk_ipers,
1504a55ebb7cSAndriy Gapon 		    slabsize - keg->uk_ipers * keg->uk_rsize);
1505099a0e58SBosko Milekic 		keg->uk_flags |= UMA_ZONE_OFFPAGE;
15068355f576SJeff Roberson 	}
1507ad97af7eSGleb Smirnoff 
1508ad97af7eSGleb Smirnoff 	if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
1509ad97af7eSGleb Smirnoff 	    (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1510ad97af7eSGleb Smirnoff 		keg->uk_flags |= UMA_ZONE_HASH;
15118355f576SJeff Roberson }
15128355f576SJeff Roberson 
15138355f576SJeff Roberson /*
1514e20a199fSJeff Roberson  * Finish creating a large (> UMA_SLAB_SIZE) uma kegs.  Just give in and do
15158355f576SJeff Roberson  * OFFPAGE for now.  When I can allow for more dynamic slab sizes this will be
15168355f576SJeff Roberson  * more complicated.
15178355f576SJeff Roberson  *
15188355f576SJeff Roberson  * Arguments
1519e20a199fSJeff Roberson  *	keg  The keg we should initialize
15208355f576SJeff Roberson  *
15218355f576SJeff Roberson  * Returns
15228355f576SJeff Roberson  *	Nothing
15238355f576SJeff Roberson  */
15248355f576SJeff Roberson static void
1525e20a199fSJeff Roberson keg_large_init(uma_keg_t keg)
15268355f576SJeff Roberson {
15278355f576SJeff Roberson 
1528e20a199fSJeff Roberson 	KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
1529ad97af7eSGleb Smirnoff 	KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1530ad97af7eSGleb Smirnoff 	    ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__));
153120e8e865SBosko Milekic 
1532ad97af7eSGleb Smirnoff 	keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE);
1533099a0e58SBosko Milekic 	keg->uk_ipers = 1;
1534e9a069d8SJohn Baldwin 	keg->uk_rsize = keg->uk_size;
1535e9a069d8SJohn Baldwin 
1536cec48e00SAlexander Motin 	/* Check whether we have enough space to not do OFFPAGE. */
15373d5e3df7SGleb Smirnoff 	if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0 &&
15383d5e3df7SGleb Smirnoff 	    PAGE_SIZE * keg->uk_ppera - keg->uk_rsize < SIZEOF_UMA_SLAB) {
15392934eb8aSMark Johnston 		/*
15402934eb8aSMark Johnston 		 * We can't do OFFPAGE if we're internal, in which case
15412934eb8aSMark Johnston 		 * we need an extra page per allocation to contain the
15422934eb8aSMark Johnston 		 * slab header.
15432934eb8aSMark Johnston 		 */
15442934eb8aSMark Johnston 		if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) == 0)
1545099a0e58SBosko Milekic 			keg->uk_flags |= UMA_ZONE_OFFPAGE;
15462934eb8aSMark Johnston 		else
15472934eb8aSMark Johnston 			keg->uk_ppera++;
15482934eb8aSMark Johnston 	}
1549cec48e00SAlexander Motin 
1550cec48e00SAlexander Motin 	if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
1551cec48e00SAlexander Motin 	    (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1552099a0e58SBosko Milekic 		keg->uk_flags |= UMA_ZONE_HASH;
15538355f576SJeff Roberson }
15548355f576SJeff Roberson 
1555e20a199fSJeff Roberson static void
1556e20a199fSJeff Roberson keg_cachespread_init(uma_keg_t keg)
1557e20a199fSJeff Roberson {
1558e20a199fSJeff Roberson 	int alignsize;
1559e20a199fSJeff Roberson 	int trailer;
1560e20a199fSJeff Roberson 	int pages;
1561e20a199fSJeff Roberson 	int rsize;
1562e20a199fSJeff Roberson 
1563ad97af7eSGleb Smirnoff 	KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1564ad97af7eSGleb Smirnoff 	    ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__));
1565ad97af7eSGleb Smirnoff 
1566e20a199fSJeff Roberson 	alignsize = keg->uk_align + 1;
1567e20a199fSJeff Roberson 	rsize = keg->uk_size;
1568e20a199fSJeff Roberson 	/*
1569e20a199fSJeff Roberson 	 * We want one item to start on every align boundary in a page.  To
1570e20a199fSJeff Roberson 	 * do this we will span pages.  We will also extend the item by the
1571e20a199fSJeff Roberson 	 * size of align if it is an even multiple of align.  Otherwise, it
1572e20a199fSJeff Roberson 	 * would fall on the same boundary every time.
1573e20a199fSJeff Roberson 	 */
1574e20a199fSJeff Roberson 	if (rsize & keg->uk_align)
1575e20a199fSJeff Roberson 		rsize = (rsize & ~keg->uk_align) + alignsize;
1576e20a199fSJeff Roberson 	if ((rsize & alignsize) == 0)
1577e20a199fSJeff Roberson 		rsize += alignsize;
1578e20a199fSJeff Roberson 	trailer = rsize - keg->uk_size;
1579e20a199fSJeff Roberson 	pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE;
1580e20a199fSJeff Roberson 	pages = MIN(pages, (128 * 1024) / PAGE_SIZE);
1581e20a199fSJeff Roberson 	keg->uk_rsize = rsize;
1582e20a199fSJeff Roberson 	keg->uk_ppera = pages;
1583e20a199fSJeff Roberson 	keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize;
1584e20a199fSJeff Roberson 	keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB;
15852367b4ddSDimitry Andric 	KASSERT(keg->uk_ipers <= SLAB_SETSIZE,
158642321809SGleb Smirnoff 	    ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__,
1587e20a199fSJeff Roberson 	    keg->uk_ipers));
1588e20a199fSJeff Roberson }
1589e20a199fSJeff Roberson 
15908355f576SJeff Roberson /*
1591099a0e58SBosko Milekic  * Keg header ctor.  This initializes all fields, locks, etc.  And inserts
1592099a0e58SBosko Milekic  * the keg onto the global keg list.
15938355f576SJeff Roberson  *
15948355f576SJeff Roberson  * Arguments/Returns follow uma_ctor specifications
1595099a0e58SBosko Milekic  *	udata  Actually uma_kctor_args
1596099a0e58SBosko Milekic  */
1597b23f72e9SBrian Feldman static int
1598b23f72e9SBrian Feldman keg_ctor(void *mem, int size, void *udata, int flags)
1599099a0e58SBosko Milekic {
1600099a0e58SBosko Milekic 	struct uma_kctor_args *arg = udata;
1601099a0e58SBosko Milekic 	uma_keg_t keg = mem;
1602099a0e58SBosko Milekic 	uma_zone_t zone;
1603099a0e58SBosko Milekic 
1604099a0e58SBosko Milekic 	bzero(keg, size);
1605099a0e58SBosko Milekic 	keg->uk_size = arg->size;
1606099a0e58SBosko Milekic 	keg->uk_init = arg->uminit;
1607099a0e58SBosko Milekic 	keg->uk_fini = arg->fini;
1608099a0e58SBosko Milekic 	keg->uk_align = arg->align;
1609099a0e58SBosko Milekic 	keg->uk_free = 0;
16106fd34d6fSJeff Roberson 	keg->uk_reserve = 0;
1611099a0e58SBosko Milekic 	keg->uk_pages = 0;
1612099a0e58SBosko Milekic 	keg->uk_flags = arg->flags;
1613099a0e58SBosko Milekic 	keg->uk_slabzone = NULL;
1614099a0e58SBosko Milekic 
1615099a0e58SBosko Milekic 	/*
1616194a979eSMark Johnston 	 * We use a global round-robin policy by default.  Zones with
1617194a979eSMark Johnston 	 * UMA_ZONE_NUMA set will use first-touch instead, in which case the
1618194a979eSMark Johnston 	 * iterator is never run.
1619194a979eSMark Johnston 	 */
1620194a979eSMark Johnston 	keg->uk_dr.dr_policy = DOMAINSET_RR();
1621194a979eSMark Johnston 	keg->uk_dr.dr_iter = 0;
1622194a979eSMark Johnston 
1623194a979eSMark Johnston 	/*
1624099a0e58SBosko Milekic 	 * The master zone is passed to us at keg-creation time.
1625099a0e58SBosko Milekic 	 */
1626099a0e58SBosko Milekic 	zone = arg->zone;
1627e20a199fSJeff Roberson 	keg->uk_name = zone->uz_name;
1628099a0e58SBosko Milekic 
1629099a0e58SBosko Milekic 	if (arg->flags & UMA_ZONE_VM)
1630099a0e58SBosko Milekic 		keg->uk_flags |= UMA_ZFLAG_CACHEONLY;
1631099a0e58SBosko Milekic 
1632099a0e58SBosko Milekic 	if (arg->flags & UMA_ZONE_ZINIT)
1633099a0e58SBosko Milekic 		keg->uk_init = zero_init;
1634099a0e58SBosko Milekic 
1635cfcae3f8SGleb Smirnoff 	if (arg->flags & UMA_ZONE_MALLOC)
1636e20a199fSJeff Roberson 		keg->uk_flags |= UMA_ZONE_VTOSLAB;
1637e20a199fSJeff Roberson 
1638ad97af7eSGleb Smirnoff 	if (arg->flags & UMA_ZONE_PCPU)
1639ad97af7eSGleb Smirnoff #ifdef SMP
1640ad97af7eSGleb Smirnoff 		keg->uk_flags |= UMA_ZONE_OFFPAGE;
1641ad97af7eSGleb Smirnoff #else
1642ad97af7eSGleb Smirnoff 		keg->uk_flags &= ~UMA_ZONE_PCPU;
1643ad97af7eSGleb Smirnoff #endif
1644ad97af7eSGleb Smirnoff 
1645ef72505eSJeff Roberson 	if (keg->uk_flags & UMA_ZONE_CACHESPREAD) {
1646e20a199fSJeff Roberson 		keg_cachespread_init(keg);
1647244f4554SBosko Milekic 	} else {
1648b92b26adSGleb Smirnoff 		if (keg->uk_size > UMA_SLAB_SPACE)
1649e20a199fSJeff Roberson 			keg_large_init(keg);
1650244f4554SBosko Milekic 		else
1651e20a199fSJeff Roberson 			keg_small_init(keg);
1652244f4554SBosko Milekic 	}
1653099a0e58SBosko Milekic 
1654cfcae3f8SGleb Smirnoff 	if (keg->uk_flags & UMA_ZONE_OFFPAGE)
1655099a0e58SBosko Milekic 		keg->uk_slabzone = slabzone;
1656099a0e58SBosko Milekic 
1657099a0e58SBosko Milekic 	/*
1658099a0e58SBosko Milekic 	 * If we haven't booted yet we need allocations to go through the
1659099a0e58SBosko Milekic 	 * startup cache until the vm is ready.
1660099a0e58SBosko Milekic 	 */
1661f4bef67cSGleb Smirnoff 	if (booted < BOOT_PAGEALLOC)
16628cd02d00SAlan Cox 		keg->uk_allocf = startup_alloc;
166377e19437SGleb Smirnoff #ifdef UMA_MD_SMALL_ALLOC
166477e19437SGleb Smirnoff 	else if (keg->uk_ppera == 1)
166577e19437SGleb Smirnoff 		keg->uk_allocf = uma_small_alloc;
16668cd02d00SAlan Cox #endif
1667ab3059a8SMatt Macy 	else if (keg->uk_flags & UMA_ZONE_PCPU)
1668ab3059a8SMatt Macy 		keg->uk_allocf = pcpu_page_alloc;
166977e19437SGleb Smirnoff 	else
167077e19437SGleb Smirnoff 		keg->uk_allocf = page_alloc;
167177e19437SGleb Smirnoff #ifdef UMA_MD_SMALL_ALLOC
167277e19437SGleb Smirnoff 	if (keg->uk_ppera == 1)
167377e19437SGleb Smirnoff 		keg->uk_freef = uma_small_free;
167477e19437SGleb Smirnoff 	else
167577e19437SGleb Smirnoff #endif
1676ab3059a8SMatt Macy 	if (keg->uk_flags & UMA_ZONE_PCPU)
1677ab3059a8SMatt Macy 		keg->uk_freef = pcpu_page_free;
1678ab3059a8SMatt Macy 	else
167977e19437SGleb Smirnoff 		keg->uk_freef = page_free;
1680099a0e58SBosko Milekic 
1681099a0e58SBosko Milekic 	/*
1682af526374SJeff Roberson 	 * Initialize keg's lock
1683099a0e58SBosko Milekic 	 */
1684af526374SJeff Roberson 	KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS));
1685099a0e58SBosko Milekic 
1686099a0e58SBosko Milekic 	/*
1687099a0e58SBosko Milekic 	 * If we're putting the slab header in the actual page we need to
16883d5e3df7SGleb Smirnoff 	 * figure out where in each page it goes.  See SIZEOF_UMA_SLAB
16893d5e3df7SGleb Smirnoff 	 * macro definition.
1690099a0e58SBosko Milekic 	 */
1691099a0e58SBosko Milekic 	if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) {
16923d5e3df7SGleb Smirnoff 		keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - SIZEOF_UMA_SLAB;
1693244f4554SBosko Milekic 		/*
1694244f4554SBosko Milekic 		 * The only way the following is possible is if with our
1695244f4554SBosko Milekic 		 * UMA_ALIGN_PTR adjustments we are now bigger than
1696244f4554SBosko Milekic 		 * UMA_SLAB_SIZE.  I haven't checked whether this is
1697244f4554SBosko Milekic 		 * mathematically possible for all cases, so we make
1698244f4554SBosko Milekic 		 * sure here anyway.
1699244f4554SBosko Milekic 		 */
17003d5e3df7SGleb Smirnoff 		KASSERT(keg->uk_pgoff + sizeof(struct uma_slab) <=
17013d5e3df7SGleb Smirnoff 		    PAGE_SIZE * keg->uk_ppera,
17023d5e3df7SGleb Smirnoff 		    ("zone %s ipers %d rsize %d size %d slab won't fit",
17033d5e3df7SGleb Smirnoff 		    zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size));
1704099a0e58SBosko Milekic 	}
1705099a0e58SBosko Milekic 
1706099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZONE_HASH)
1707099a0e58SBosko Milekic 		hash_alloc(&keg->uk_hash);
1708099a0e58SBosko Milekic 
17091431a748SGleb Smirnoff 	CTR5(KTR_UMA, "keg_ctor %p zone %s(%p) out %d free %d\n",
17101431a748SGleb Smirnoff 	    keg, zone->uz_name, zone,
171157223e99SAndriy Gapon 	    (keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free,
171257223e99SAndriy Gapon 	    keg->uk_free);
1713099a0e58SBosko Milekic 
1714099a0e58SBosko Milekic 	LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
1715099a0e58SBosko Milekic 
1716111fbcd5SBryan Venteicher 	rw_wlock(&uma_rwlock);
1717099a0e58SBosko Milekic 	LIST_INSERT_HEAD(&uma_kegs, keg, uk_link);
1718111fbcd5SBryan Venteicher 	rw_wunlock(&uma_rwlock);
1719b23f72e9SBrian Feldman 	return (0);
1720099a0e58SBosko Milekic }
1721099a0e58SBosko Milekic 
17222efcc8cbSGleb Smirnoff static void
17232efcc8cbSGleb Smirnoff zone_alloc_counters(uma_zone_t zone)
17242efcc8cbSGleb Smirnoff {
17252efcc8cbSGleb Smirnoff 
17262efcc8cbSGleb Smirnoff 	zone->uz_allocs = counter_u64_alloc(M_WAITOK);
17272efcc8cbSGleb Smirnoff 	zone->uz_frees = counter_u64_alloc(M_WAITOK);
17282efcc8cbSGleb Smirnoff 	zone->uz_fails = counter_u64_alloc(M_WAITOK);
17292efcc8cbSGleb Smirnoff }
17302efcc8cbSGleb Smirnoff 
1731099a0e58SBosko Milekic /*
1732099a0e58SBosko Milekic  * Zone header ctor.  This initializes all fields, locks, etc.
1733099a0e58SBosko Milekic  *
1734099a0e58SBosko Milekic  * Arguments/Returns follow uma_ctor specifications
1735099a0e58SBosko Milekic  *	udata  Actually uma_zctor_args
17368355f576SJeff Roberson  */
1737b23f72e9SBrian Feldman static int
1738b23f72e9SBrian Feldman zone_ctor(void *mem, int size, void *udata, int flags)
17398355f576SJeff Roberson {
17408355f576SJeff Roberson 	struct uma_zctor_args *arg = udata;
17418355f576SJeff Roberson 	uma_zone_t zone = mem;
1742099a0e58SBosko Milekic 	uma_zone_t z;
1743099a0e58SBosko Milekic 	uma_keg_t keg;
17448355f576SJeff Roberson 
17458355f576SJeff Roberson 	bzero(zone, size);
17468355f576SJeff Roberson 	zone->uz_name = arg->name;
17478355f576SJeff Roberson 	zone->uz_ctor = arg->ctor;
17488355f576SJeff Roberson 	zone->uz_dtor = arg->dtor;
1749099a0e58SBosko Milekic 	zone->uz_init = NULL;
1750099a0e58SBosko Milekic 	zone->uz_fini = NULL;
1751bf965959SSean Bruno 	zone->uz_sleeps = 0;
1752fc03d22bSJeff Roberson 	zone->uz_count = 0;
1753ace66b56SAlexander Motin 	zone->uz_count_min = 0;
1754bb15d1c7SGleb Smirnoff 	zone->uz_count_max = BUCKET_MAX;
1755e20a199fSJeff Roberson 	zone->uz_flags = 0;
17562f891cd5SPawel Jakub Dawidek 	zone->uz_warning = NULL;
1757ab3185d1SJeff Roberson 	/* The domain structures follow the cpu structures. */
1758ab3185d1SJeff Roberson 	zone->uz_domain = (struct uma_zone_domain *)&zone->uz_cpu[mp_ncpus];
1759bb15d1c7SGleb Smirnoff 	zone->uz_bkt_max = ULONG_MAX;
17602f891cd5SPawel Jakub Dawidek 	timevalclear(&zone->uz_ratecheck);
1761af526374SJeff Roberson 
17622efcc8cbSGleb Smirnoff 	if (__predict_true(booted == BOOT_RUNNING))
17632efcc8cbSGleb Smirnoff 		zone_alloc_counters(zone);
17642efcc8cbSGleb Smirnoff 	else {
17652efcc8cbSGleb Smirnoff 		zone->uz_allocs = EARLY_COUNTER;
17662efcc8cbSGleb Smirnoff 		zone->uz_frees = EARLY_COUNTER;
17672efcc8cbSGleb Smirnoff 		zone->uz_fails = EARLY_COUNTER;
17682efcc8cbSGleb Smirnoff 	}
17692efcc8cbSGleb Smirnoff 
17700095a784SJeff Roberson 	/*
17710095a784SJeff Roberson 	 * This is a pure cache zone, no kegs.
17720095a784SJeff Roberson 	 */
17730095a784SJeff Roberson 	if (arg->import) {
17746fd34d6fSJeff Roberson 		if (arg->flags & UMA_ZONE_VM)
17756fd34d6fSJeff Roberson 			arg->flags |= UMA_ZFLAG_CACHEONLY;
17766fd34d6fSJeff Roberson 		zone->uz_flags = arg->flags;
1777af526374SJeff Roberson 		zone->uz_size = arg->size;
17780095a784SJeff Roberson 		zone->uz_import = arg->import;
17790095a784SJeff Roberson 		zone->uz_release = arg->release;
17800095a784SJeff Roberson 		zone->uz_arg = arg->arg;
1781af526374SJeff Roberson 		zone->uz_lockptr = &zone->uz_lock;
1782bb15d1c7SGleb Smirnoff 		ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));
1783111fbcd5SBryan Venteicher 		rw_wlock(&uma_rwlock);
178403175483SAlexander Motin 		LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link);
1785111fbcd5SBryan Venteicher 		rw_wunlock(&uma_rwlock);
1786af526374SJeff Roberson 		goto out;
17870095a784SJeff Roberson 	}
17880095a784SJeff Roberson 
17890095a784SJeff Roberson 	/*
17900095a784SJeff Roberson 	 * Use the regular zone/keg/slab allocator.
17910095a784SJeff Roberson 	 */
17920095a784SJeff Roberson 	zone->uz_import = (uma_import)zone_import;
17930095a784SJeff Roberson 	zone->uz_release = (uma_release)zone_release;
17940095a784SJeff Roberson 	zone->uz_arg = zone;
1795bb15d1c7SGleb Smirnoff 	keg = arg->keg;
17960095a784SJeff Roberson 
1797099a0e58SBosko Milekic 	if (arg->flags & UMA_ZONE_SECONDARY) {
1798099a0e58SBosko Milekic 		KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
17998355f576SJeff Roberson 		zone->uz_init = arg->uminit;
1800e221e841SJeff Roberson 		zone->uz_fini = arg->fini;
1801af526374SJeff Roberson 		zone->uz_lockptr = &keg->uk_lock;
1802e20a199fSJeff Roberson 		zone->uz_flags |= UMA_ZONE_SECONDARY;
1803111fbcd5SBryan Venteicher 		rw_wlock(&uma_rwlock);
1804099a0e58SBosko Milekic 		ZONE_LOCK(zone);
1805099a0e58SBosko Milekic 		LIST_FOREACH(z, &keg->uk_zones, uz_link) {
1806099a0e58SBosko Milekic 			if (LIST_NEXT(z, uz_link) == NULL) {
1807099a0e58SBosko Milekic 				LIST_INSERT_AFTER(z, zone, uz_link);
1808099a0e58SBosko Milekic 				break;
1809099a0e58SBosko Milekic 			}
1810099a0e58SBosko Milekic 		}
1811099a0e58SBosko Milekic 		ZONE_UNLOCK(zone);
1812111fbcd5SBryan Venteicher 		rw_wunlock(&uma_rwlock);
1813e20a199fSJeff Roberson 	} else if (keg == NULL) {
1814e20a199fSJeff Roberson 		if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini,
1815e20a199fSJeff Roberson 		    arg->align, arg->flags)) == NULL)
1816b23f72e9SBrian Feldman 			return (ENOMEM);
1817099a0e58SBosko Milekic 	} else {
1818099a0e58SBosko Milekic 		struct uma_kctor_args karg;
1819b23f72e9SBrian Feldman 		int error;
1820099a0e58SBosko Milekic 
1821099a0e58SBosko Milekic 		/* We should only be here from uma_startup() */
1822099a0e58SBosko Milekic 		karg.size = arg->size;
1823099a0e58SBosko Milekic 		karg.uminit = arg->uminit;
1824099a0e58SBosko Milekic 		karg.fini = arg->fini;
1825099a0e58SBosko Milekic 		karg.align = arg->align;
1826099a0e58SBosko Milekic 		karg.flags = arg->flags;
1827099a0e58SBosko Milekic 		karg.zone = zone;
1828b23f72e9SBrian Feldman 		error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
1829b23f72e9SBrian Feldman 		    flags);
1830b23f72e9SBrian Feldman 		if (error)
1831b23f72e9SBrian Feldman 			return (error);
1832099a0e58SBosko Milekic 	}
18330095a784SJeff Roberson 
1834bb15d1c7SGleb Smirnoff 	zone->uz_keg = keg;
1835e20a199fSJeff Roberson 	zone->uz_size = keg->uk_size;
1836e20a199fSJeff Roberson 	zone->uz_flags |= (keg->uk_flags &
1837e20a199fSJeff Roberson 	    (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
18388355f576SJeff Roberson 
18398355f576SJeff Roberson 	/*
18408355f576SJeff Roberson 	 * Some internal zones don't have room allocated for the per cpu
18418355f576SJeff Roberson 	 * caches.  If we're internal, bail out here.
18428355f576SJeff Roberson 	 */
1843099a0e58SBosko Milekic 	if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
1844e20a199fSJeff Roberson 		KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
1845099a0e58SBosko Milekic 		    ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
1846b23f72e9SBrian Feldman 		return (0);
1847099a0e58SBosko Milekic 	}
18488355f576SJeff Roberson 
1849af526374SJeff Roberson out:
18507e28037aSMark Johnston 	KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) !=
18517e28037aSMark Johnston 	    (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET),
18527e28037aSMark Johnston 	    ("Invalid zone flag combination"));
18537e28037aSMark Johnston 	if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0)
1854cae33c14SJeff Roberson 		zone->uz_count = BUCKET_MAX;
18557e28037aSMark Johnston 	else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0)
18567e28037aSMark Johnston 		zone->uz_count = 0;
18577e28037aSMark Johnston 	else
18587e28037aSMark Johnston 		zone->uz_count = bucket_select(zone->uz_size);
1859ace66b56SAlexander Motin 	zone->uz_count_min = zone->uz_count;
1860fc03d22bSJeff Roberson 
1861b23f72e9SBrian Feldman 	return (0);
18628355f576SJeff Roberson }
18638355f576SJeff Roberson 
18648355f576SJeff Roberson /*
1865099a0e58SBosko Milekic  * Keg header dtor.  This frees all data, destroys locks, frees the hash
1866099a0e58SBosko Milekic  * table and removes the keg from the global list.
18679c2cd7e5SJeff Roberson  *
18689c2cd7e5SJeff Roberson  * Arguments/Returns follow uma_dtor specifications
18699c2cd7e5SJeff Roberson  *	udata  unused
18709c2cd7e5SJeff Roberson  */
1871099a0e58SBosko Milekic static void
1872099a0e58SBosko Milekic keg_dtor(void *arg, int size, void *udata)
1873099a0e58SBosko Milekic {
1874099a0e58SBosko Milekic 	uma_keg_t keg;
18759c2cd7e5SJeff Roberson 
1876099a0e58SBosko Milekic 	keg = (uma_keg_t)arg;
1877e20a199fSJeff Roberson 	KEG_LOCK(keg);
1878099a0e58SBosko Milekic 	if (keg->uk_free != 0) {
1879a3845534SCraig Rodrigues 		printf("Freed UMA keg (%s) was not empty (%d items). "
1880099a0e58SBosko Milekic 		    " Lost %d pages of memory.\n",
1881a3845534SCraig Rodrigues 		    keg->uk_name ? keg->uk_name : "",
1882099a0e58SBosko Milekic 		    keg->uk_free, keg->uk_pages);
1883099a0e58SBosko Milekic 	}
1884e20a199fSJeff Roberson 	KEG_UNLOCK(keg);
1885099a0e58SBosko Milekic 
1886099a0e58SBosko Milekic 	hash_free(&keg->uk_hash);
1887099a0e58SBosko Milekic 
1888e20a199fSJeff Roberson 	KEG_LOCK_FINI(keg);
1889099a0e58SBosko Milekic }
1890099a0e58SBosko Milekic 
1891099a0e58SBosko Milekic /*
1892099a0e58SBosko Milekic  * Zone header dtor.
1893099a0e58SBosko Milekic  *
1894099a0e58SBosko Milekic  * Arguments/Returns follow uma_dtor specifications
1895099a0e58SBosko Milekic  *	udata  unused
1896099a0e58SBosko Milekic  */
18979c2cd7e5SJeff Roberson static void
18989c2cd7e5SJeff Roberson zone_dtor(void *arg, int size, void *udata)
18999c2cd7e5SJeff Roberson {
19009c2cd7e5SJeff Roberson 	uma_zone_t zone;
1901099a0e58SBosko Milekic 	uma_keg_t keg;
19029c2cd7e5SJeff Roberson 
19039c2cd7e5SJeff Roberson 	zone = (uma_zone_t)arg;
19049643769aSJeff Roberson 
1905e20a199fSJeff Roberson 	if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
19069643769aSJeff Roberson 		cache_drain(zone);
1907099a0e58SBosko Milekic 
1908111fbcd5SBryan Venteicher 	rw_wlock(&uma_rwlock);
1909099a0e58SBosko Milekic 	LIST_REMOVE(zone, uz_link);
1910111fbcd5SBryan Venteicher 	rw_wunlock(&uma_rwlock);
1911099a0e58SBosko Milekic 	/*
1912099a0e58SBosko Milekic 	 * XXX there are some races here where
1913099a0e58SBosko Milekic 	 * the zone can be drained but zone lock
1914099a0e58SBosko Milekic 	 * released and then refilled before we
1915099a0e58SBosko Milekic 	 * remove it... we dont care for now
1916099a0e58SBosko Milekic 	 */
1917e20a199fSJeff Roberson 	zone_drain_wait(zone, M_WAITOK);
1918e20a199fSJeff Roberson 	/*
1919323ad386STycho Nightingale 	 * We only destroy kegs from non secondary/non cache zones.
1920e20a199fSJeff Roberson 	 */
1921323ad386STycho Nightingale 	if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) {
1922323ad386STycho Nightingale 		keg = zone->uz_keg;
1923111fbcd5SBryan Venteicher 		rw_wlock(&uma_rwlock);
1924099a0e58SBosko Milekic 		LIST_REMOVE(keg, uk_link);
1925111fbcd5SBryan Venteicher 		rw_wunlock(&uma_rwlock);
19260095a784SJeff Roberson 		zone_free_item(kegs, keg, NULL, SKIP_NONE);
19279c2cd7e5SJeff Roberson 	}
19282efcc8cbSGleb Smirnoff 	counter_u64_free(zone->uz_allocs);
19292efcc8cbSGleb Smirnoff 	counter_u64_free(zone->uz_frees);
19302efcc8cbSGleb Smirnoff 	counter_u64_free(zone->uz_fails);
1931bb15d1c7SGleb Smirnoff 	if (zone->uz_lockptr == &zone->uz_lock)
1932af526374SJeff Roberson 		ZONE_LOCK_FINI(zone);
1933099a0e58SBosko Milekic }
1934099a0e58SBosko Milekic 
19359c2cd7e5SJeff Roberson /*
19368355f576SJeff Roberson  * Traverses every zone in the system and calls a callback
19378355f576SJeff Roberson  *
19388355f576SJeff Roberson  * Arguments:
19398355f576SJeff Roberson  *	zfunc  A pointer to a function which accepts a zone
19408355f576SJeff Roberson  *		as an argument.
19418355f576SJeff Roberson  *
19428355f576SJeff Roberson  * Returns:
19438355f576SJeff Roberson  *	Nothing
19448355f576SJeff Roberson  */
19458355f576SJeff Roberson static void
19468355f576SJeff Roberson zone_foreach(void (*zfunc)(uma_zone_t))
19478355f576SJeff Roberson {
1948099a0e58SBosko Milekic 	uma_keg_t keg;
19498355f576SJeff Roberson 	uma_zone_t zone;
19508355f576SJeff Roberson 
19512efcc8cbSGleb Smirnoff 	/*
19522efcc8cbSGleb Smirnoff 	 * Before BOOT_RUNNING we are guaranteed to be single
19532efcc8cbSGleb Smirnoff 	 * threaded, so locking isn't needed. Startup functions
19542efcc8cbSGleb Smirnoff 	 * are allowed to use M_WAITOK.
19552efcc8cbSGleb Smirnoff 	 */
19562efcc8cbSGleb Smirnoff 	if (__predict_true(booted == BOOT_RUNNING))
1957111fbcd5SBryan Venteicher 		rw_rlock(&uma_rwlock);
1958099a0e58SBosko Milekic 	LIST_FOREACH(keg, &uma_kegs, uk_link) {
1959099a0e58SBosko Milekic 		LIST_FOREACH(zone, &keg->uk_zones, uz_link)
19608355f576SJeff Roberson 			zfunc(zone);
1961099a0e58SBosko Milekic 	}
19622efcc8cbSGleb Smirnoff 	if (__predict_true(booted == BOOT_RUNNING))
1963111fbcd5SBryan Venteicher 		rw_runlock(&uma_rwlock);
19648355f576SJeff Roberson }
19658355f576SJeff Roberson 
1966f4bef67cSGleb Smirnoff /*
1967f4bef67cSGleb Smirnoff  * Count how many pages do we need to bootstrap.  VM supplies
1968f4bef67cSGleb Smirnoff  * its need in early zones in the argument, we add up our zones,
1969f4bef67cSGleb Smirnoff  * which consist of: UMA Slabs, UMA Hash and 9 Bucket zones. The
1970f4bef67cSGleb Smirnoff  * zone of zones and zone of kegs are accounted separately.
1971f4bef67cSGleb Smirnoff  */
1972f4bef67cSGleb Smirnoff #define	UMA_BOOT_ZONES	11
19735073a083SGleb Smirnoff /* Zone of zones and zone of kegs have arbitrary alignment. */
19745073a083SGleb Smirnoff #define	UMA_BOOT_ALIGN	32
1975f4bef67cSGleb Smirnoff static int zsize, ksize;
1976f4bef67cSGleb Smirnoff int
1977f7d35785SGleb Smirnoff uma_startup_count(int vm_zones)
1978f4bef67cSGleb Smirnoff {
1979f7d35785SGleb Smirnoff 	int zones, pages;
1980f4bef67cSGleb Smirnoff 
1981f4bef67cSGleb Smirnoff 	ksize = sizeof(struct uma_keg) +
1982f4bef67cSGleb Smirnoff 	    (sizeof(struct uma_domain) * vm_ndomains);
1983f4bef67cSGleb Smirnoff 	zsize = sizeof(struct uma_zone) +
1984f4bef67cSGleb Smirnoff 	    (sizeof(struct uma_cache) * (mp_maxid + 1)) +
1985f4bef67cSGleb Smirnoff 	    (sizeof(struct uma_zone_domain) * vm_ndomains);
1986f4bef67cSGleb Smirnoff 
19875073a083SGleb Smirnoff 	/*
19885073a083SGleb Smirnoff 	 * Memory for the zone of kegs and its keg,
19895073a083SGleb Smirnoff 	 * and for zone of zones.
19905073a083SGleb Smirnoff 	 */
1991f4bef67cSGleb Smirnoff 	pages = howmany(roundup(zsize, CACHE_LINE_SIZE) * 2 +
1992f4bef67cSGleb Smirnoff 	    roundup(ksize, CACHE_LINE_SIZE), PAGE_SIZE);
1993f4bef67cSGleb Smirnoff 
1994f7d35785SGleb Smirnoff #ifdef	UMA_MD_SMALL_ALLOC
1995f7d35785SGleb Smirnoff 	zones = UMA_BOOT_ZONES;
1996f7d35785SGleb Smirnoff #else
1997f7d35785SGleb Smirnoff 	zones = UMA_BOOT_ZONES + vm_zones;
1998f7d35785SGleb Smirnoff 	vm_zones = 0;
1999f7d35785SGleb Smirnoff #endif
2000f4bef67cSGleb Smirnoff 
20015073a083SGleb Smirnoff 	/* Memory for the rest of startup zones, UMA and VM, ... */
20020b2e3aeaSGleb Smirnoff 	if (zsize > UMA_SLAB_SPACE) {
20030b2e3aeaSGleb Smirnoff 		/* See keg_large_init(). */
20040b2e3aeaSGleb Smirnoff 		u_int ppera;
20050b2e3aeaSGleb Smirnoff 
20060b2e3aeaSGleb Smirnoff 		ppera = howmany(roundup2(zsize, UMA_BOOT_ALIGN), PAGE_SIZE);
20070b2e3aeaSGleb Smirnoff 		if (PAGE_SIZE * ppera - roundup2(zsize, UMA_BOOT_ALIGN) <
20080b2e3aeaSGleb Smirnoff 		    SIZEOF_UMA_SLAB)
20090b2e3aeaSGleb Smirnoff 			ppera++;
20100b2e3aeaSGleb Smirnoff 		pages += (zones + vm_zones) * ppera;
20110b2e3aeaSGleb Smirnoff 	} else if (roundup2(zsize, UMA_BOOT_ALIGN) > UMA_SLAB_SPACE)
20120b2e3aeaSGleb Smirnoff 		/* See keg_small_init() special case for uk_ppera = 1. */
201396a10340SGleb Smirnoff 		pages += zones;
2014f4bef67cSGleb Smirnoff 	else
20155073a083SGleb Smirnoff 		pages += howmany(zones,
20165073a083SGleb Smirnoff 		    UMA_SLAB_SPACE / roundup2(zsize, UMA_BOOT_ALIGN));
2017f4bef67cSGleb Smirnoff 
20185073a083SGleb Smirnoff 	/* ... and their kegs. Note that zone of zones allocates a keg! */
20195073a083SGleb Smirnoff 	pages += howmany(zones + 1,
20205073a083SGleb Smirnoff 	    UMA_SLAB_SPACE / roundup2(ksize, UMA_BOOT_ALIGN));
2021f4bef67cSGleb Smirnoff 
2022f4bef67cSGleb Smirnoff 	/*
20235073a083SGleb Smirnoff 	 * Most of startup zones are not going to be offpages, that's
20245073a083SGleb Smirnoff 	 * why we use UMA_SLAB_SPACE instead of UMA_SLAB_SIZE in all
20255073a083SGleb Smirnoff 	 * calculations.  Some large bucket zones will be offpage, and
20265073a083SGleb Smirnoff 	 * thus will allocate hashes.  We take conservative approach
20275073a083SGleb Smirnoff 	 * and assume that all zones may allocate hash.  This may give
20285073a083SGleb Smirnoff 	 * us some positive inaccuracy, usually an extra single page.
2029f4bef67cSGleb Smirnoff 	 */
20305073a083SGleb Smirnoff 	pages += howmany(zones, UMA_SLAB_SPACE /
2031d2be4a1eSGleb Smirnoff 	    (sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT));
2032f4bef67cSGleb Smirnoff 
2033f4bef67cSGleb Smirnoff 	return (pages);
2034f4bef67cSGleb Smirnoff }
2035f4bef67cSGleb Smirnoff 
20368355f576SJeff Roberson void
2037ac0a6fd0SGleb Smirnoff uma_startup(void *mem, int npages)
20388355f576SJeff Roberson {
20398355f576SJeff Roberson 	struct uma_zctor_args args;
2040ab3185d1SJeff Roberson 	uma_keg_t masterkeg;
2041ab3185d1SJeff Roberson 	uintptr_t m;
2042f4bef67cSGleb Smirnoff 
2043f4bef67cSGleb Smirnoff #ifdef DIAGNOSTIC
2044f4bef67cSGleb Smirnoff 	printf("Entering %s with %d boot pages configured\n", __func__, npages);
2045f4bef67cSGleb Smirnoff #endif
20468355f576SJeff Roberson 
2047111fbcd5SBryan Venteicher 	rw_init(&uma_rwlock, "UMA lock");
2048099a0e58SBosko Milekic 
2049ab3185d1SJeff Roberson 	/* Use bootpages memory for the zone of zones and zone of kegs. */
2050ab3185d1SJeff Roberson 	m = (uintptr_t)mem;
2051ab3185d1SJeff Roberson 	zones = (uma_zone_t)m;
2052ab3185d1SJeff Roberson 	m += roundup(zsize, CACHE_LINE_SIZE);
2053ab3185d1SJeff Roberson 	kegs = (uma_zone_t)m;
2054ab3185d1SJeff Roberson 	m += roundup(zsize, CACHE_LINE_SIZE);
2055ab3185d1SJeff Roberson 	masterkeg = (uma_keg_t)m;
2056ab3185d1SJeff Roberson 	m += roundup(ksize, CACHE_LINE_SIZE);
2057ab3185d1SJeff Roberson 	m = roundup(m, PAGE_SIZE);
2058ab3185d1SJeff Roberson 	npages -= (m - (uintptr_t)mem) / PAGE_SIZE;
2059ab3185d1SJeff Roberson 	mem = (void *)m;
2060ab3185d1SJeff Roberson 
2061099a0e58SBosko Milekic 	/* "manually" create the initial zone */
20620095a784SJeff Roberson 	memset(&args, 0, sizeof(args));
2063099a0e58SBosko Milekic 	args.name = "UMA Kegs";
2064ab3185d1SJeff Roberson 	args.size = ksize;
2065099a0e58SBosko Milekic 	args.ctor = keg_ctor;
2066099a0e58SBosko Milekic 	args.dtor = keg_dtor;
20678355f576SJeff Roberson 	args.uminit = zero_init;
20688355f576SJeff Roberson 	args.fini = NULL;
2069ab3185d1SJeff Roberson 	args.keg = masterkeg;
20705073a083SGleb Smirnoff 	args.align = UMA_BOOT_ALIGN - 1;
2071b60f5b79SJeff Roberson 	args.flags = UMA_ZFLAG_INTERNAL;
2072ab3185d1SJeff Roberson 	zone_ctor(kegs, zsize, &args, M_WAITOK);
20738355f576SJeff Roberson 
2074ac0a6fd0SGleb Smirnoff 	bootmem = mem;
2075ac0a6fd0SGleb Smirnoff 	boot_pages = npages;
20768355f576SJeff Roberson 
2077099a0e58SBosko Milekic 	args.name = "UMA Zones";
2078f4bef67cSGleb Smirnoff 	args.size = zsize;
2079099a0e58SBosko Milekic 	args.ctor = zone_ctor;
2080099a0e58SBosko Milekic 	args.dtor = zone_dtor;
2081099a0e58SBosko Milekic 	args.uminit = zero_init;
2082099a0e58SBosko Milekic 	args.fini = NULL;
2083099a0e58SBosko Milekic 	args.keg = NULL;
20845073a083SGleb Smirnoff 	args.align = UMA_BOOT_ALIGN - 1;
2085099a0e58SBosko Milekic 	args.flags = UMA_ZFLAG_INTERNAL;
2086ab3185d1SJeff Roberson 	zone_ctor(zones, zsize, &args, M_WAITOK);
2087099a0e58SBosko Milekic 
20888355f576SJeff Roberson 	/* Now make a zone for slab headers */
20898355f576SJeff Roberson 	slabzone = uma_zcreate("UMA Slabs",
2090ef72505eSJeff Roberson 				sizeof(struct uma_slab),
20918355f576SJeff Roberson 				NULL, NULL, NULL, NULL,
2092b60f5b79SJeff Roberson 				UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
20938355f576SJeff Roberson 
20948355f576SJeff Roberson 	hashzone = uma_zcreate("UMA Hash",
20958355f576SJeff Roberson 	    sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT,
20968355f576SJeff Roberson 	    NULL, NULL, NULL, NULL,
2097b60f5b79SJeff Roberson 	    UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
20988355f576SJeff Roberson 
2099cae33c14SJeff Roberson 	bucket_init();
21008355f576SJeff Roberson 
2101f4bef67cSGleb Smirnoff 	booted = BOOT_STRAPPED;
21028355f576SJeff Roberson }
21038355f576SJeff Roberson 
2104f4bef67cSGleb Smirnoff void
2105f4bef67cSGleb Smirnoff uma_startup1(void)
2106f4bef67cSGleb Smirnoff {
2107f4bef67cSGleb Smirnoff 
2108f4bef67cSGleb Smirnoff #ifdef DIAGNOSTIC
2109f4bef67cSGleb Smirnoff 	printf("Entering %s with %d boot pages left\n", __func__, boot_pages);
2110f4bef67cSGleb Smirnoff #endif
2111f4bef67cSGleb Smirnoff 	booted = BOOT_PAGEALLOC;
2112f4bef67cSGleb Smirnoff }
2113f4bef67cSGleb Smirnoff 
21148355f576SJeff Roberson void
211599571dc3SJeff Roberson uma_startup2(void)
21168355f576SJeff Roberson {
2117f4bef67cSGleb Smirnoff 
2118f7d35785SGleb Smirnoff #ifdef DIAGNOSTIC
2119f7d35785SGleb Smirnoff 	printf("Entering %s with %d boot pages left\n", __func__, boot_pages);
2120f7d35785SGleb Smirnoff #endif
2121f4bef67cSGleb Smirnoff 	booted = BOOT_BUCKETS;
212295c4bf75SKonstantin Belousov 	sx_init(&uma_drain_lock, "umadrain");
2123f4bef67cSGleb Smirnoff 	bucket_enable();
21248355f576SJeff Roberson }
21258355f576SJeff Roberson 
21268355f576SJeff Roberson /*
21278355f576SJeff Roberson  * Initialize our callout handle
21288355f576SJeff Roberson  *
21298355f576SJeff Roberson  */
21308355f576SJeff Roberson static void
21318355f576SJeff Roberson uma_startup3(void)
21328355f576SJeff Roberson {
21331431a748SGleb Smirnoff 
2134c5deaf04SGleb Smirnoff #ifdef INVARIANTS
2135c5deaf04SGleb Smirnoff 	TUNABLE_INT_FETCH("vm.debug.divisor", &dbg_divisor);
2136c5deaf04SGleb Smirnoff 	uma_dbg_cnt = counter_u64_alloc(M_WAITOK);
2137c5deaf04SGleb Smirnoff 	uma_skip_cnt = counter_u64_alloc(M_WAITOK);
2138c5deaf04SGleb Smirnoff #endif
21392efcc8cbSGleb Smirnoff 	zone_foreach(zone_alloc_counters);
2140fd90e2edSJung-uk Kim 	callout_init(&uma_callout, 1);
21419643769aSJeff Roberson 	callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
2142c5deaf04SGleb Smirnoff 	booted = BOOT_RUNNING;
21438355f576SJeff Roberson }
21448355f576SJeff Roberson 
2145e20a199fSJeff Roberson static uma_keg_t
2146099a0e58SBosko Milekic uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
214785dcf349SGleb Smirnoff 		int align, uint32_t flags)
2148099a0e58SBosko Milekic {
2149099a0e58SBosko Milekic 	struct uma_kctor_args args;
2150099a0e58SBosko Milekic 
2151099a0e58SBosko Milekic 	args.size = size;
2152099a0e58SBosko Milekic 	args.uminit = uminit;
2153099a0e58SBosko Milekic 	args.fini = fini;
21541e319f6dSRobert Watson 	args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
2155099a0e58SBosko Milekic 	args.flags = flags;
2156099a0e58SBosko Milekic 	args.zone = zone;
2157ab3185d1SJeff Roberson 	return (zone_alloc_item(kegs, &args, UMA_ANYDOMAIN, M_WAITOK));
2158099a0e58SBosko Milekic }
2159099a0e58SBosko Milekic 
2160f4bef67cSGleb Smirnoff /* Public functions */
21618355f576SJeff Roberson /* See uma.h */
21621e319f6dSRobert Watson void
21631e319f6dSRobert Watson uma_set_align(int align)
21641e319f6dSRobert Watson {
21651e319f6dSRobert Watson 
21661e319f6dSRobert Watson 	if (align != UMA_ALIGN_CACHE)
21671e319f6dSRobert Watson 		uma_align_cache = align;
21681e319f6dSRobert Watson }
21691e319f6dSRobert Watson 
21701e319f6dSRobert Watson /* See uma.h */
21718355f576SJeff Roberson uma_zone_t
2172bb196eb4SMatthew D Fleming uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
217385dcf349SGleb Smirnoff 		uma_init uminit, uma_fini fini, int align, uint32_t flags)
21748355f576SJeff Roberson 
21758355f576SJeff Roberson {
21768355f576SJeff Roberson 	struct uma_zctor_args args;
217795c4bf75SKonstantin Belousov 	uma_zone_t res;
217895c4bf75SKonstantin Belousov 	bool locked;
21798355f576SJeff Roberson 
2180a5a35578SJohn Baldwin 	KASSERT(powerof2(align + 1), ("invalid zone alignment %d for \"%s\"",
2181a5a35578SJohn Baldwin 	    align, name));
2182a5a35578SJohn Baldwin 
21838355f576SJeff Roberson 	/* This stuff is essential for the zone ctor */
21840095a784SJeff Roberson 	memset(&args, 0, sizeof(args));
21858355f576SJeff Roberson 	args.name = name;
21868355f576SJeff Roberson 	args.size = size;
21878355f576SJeff Roberson 	args.ctor = ctor;
21888355f576SJeff Roberson 	args.dtor = dtor;
21898355f576SJeff Roberson 	args.uminit = uminit;
21908355f576SJeff Roberson 	args.fini = fini;
2191afc6dc36SJohn-Mark Gurney #ifdef  INVARIANTS
2192afc6dc36SJohn-Mark Gurney 	/*
2193afc6dc36SJohn-Mark Gurney 	 * If a zone is being created with an empty constructor and
2194afc6dc36SJohn-Mark Gurney 	 * destructor, pass UMA constructor/destructor which checks for
2195afc6dc36SJohn-Mark Gurney 	 * memory use after free.
2196afc6dc36SJohn-Mark Gurney 	 */
219719c591bfSMateusz Guzik 	if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOFREE))) &&
219819c591bfSMateusz Guzik 	    ctor == NULL && dtor == NULL && uminit == NULL && fini == NULL) {
2199afc6dc36SJohn-Mark Gurney 		args.ctor = trash_ctor;
2200afc6dc36SJohn-Mark Gurney 		args.dtor = trash_dtor;
2201afc6dc36SJohn-Mark Gurney 		args.uminit = trash_init;
2202afc6dc36SJohn-Mark Gurney 		args.fini = trash_fini;
2203afc6dc36SJohn-Mark Gurney 	}
2204afc6dc36SJohn-Mark Gurney #endif
22058355f576SJeff Roberson 	args.align = align;
22068355f576SJeff Roberson 	args.flags = flags;
2207099a0e58SBosko Milekic 	args.keg = NULL;
2208099a0e58SBosko Milekic 
2209f4bef67cSGleb Smirnoff 	if (booted < BOOT_BUCKETS) {
221095c4bf75SKonstantin Belousov 		locked = false;
221195c4bf75SKonstantin Belousov 	} else {
221295c4bf75SKonstantin Belousov 		sx_slock(&uma_drain_lock);
221395c4bf75SKonstantin Belousov 		locked = true;
221495c4bf75SKonstantin Belousov 	}
2215ab3185d1SJeff Roberson 	res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK);
221695c4bf75SKonstantin Belousov 	if (locked)
221795c4bf75SKonstantin Belousov 		sx_sunlock(&uma_drain_lock);
221895c4bf75SKonstantin Belousov 	return (res);
2219099a0e58SBosko Milekic }
2220099a0e58SBosko Milekic 
2221099a0e58SBosko Milekic /* See uma.h */
2222099a0e58SBosko Milekic uma_zone_t
2223099a0e58SBosko Milekic uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
2224099a0e58SBosko Milekic 		    uma_init zinit, uma_fini zfini, uma_zone_t master)
2225099a0e58SBosko Milekic {
2226099a0e58SBosko Milekic 	struct uma_zctor_args args;
2227e20a199fSJeff Roberson 	uma_keg_t keg;
222895c4bf75SKonstantin Belousov 	uma_zone_t res;
222995c4bf75SKonstantin Belousov 	bool locked;
2230099a0e58SBosko Milekic 
2231bb15d1c7SGleb Smirnoff 	keg = master->uz_keg;
22320095a784SJeff Roberson 	memset(&args, 0, sizeof(args));
2233099a0e58SBosko Milekic 	args.name = name;
2234e20a199fSJeff Roberson 	args.size = keg->uk_size;
2235099a0e58SBosko Milekic 	args.ctor = ctor;
2236099a0e58SBosko Milekic 	args.dtor = dtor;
2237099a0e58SBosko Milekic 	args.uminit = zinit;
2238099a0e58SBosko Milekic 	args.fini = zfini;
2239e20a199fSJeff Roberson 	args.align = keg->uk_align;
2240e20a199fSJeff Roberson 	args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
2241e20a199fSJeff Roberson 	args.keg = keg;
22428355f576SJeff Roberson 
2243f4bef67cSGleb Smirnoff 	if (booted < BOOT_BUCKETS) {
224495c4bf75SKonstantin Belousov 		locked = false;
224595c4bf75SKonstantin Belousov 	} else {
224695c4bf75SKonstantin Belousov 		sx_slock(&uma_drain_lock);
224795c4bf75SKonstantin Belousov 		locked = true;
224895c4bf75SKonstantin Belousov 	}
2249e20a199fSJeff Roberson 	/* XXX Attaches only one keg of potentially many. */
2250ab3185d1SJeff Roberson 	res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK);
225195c4bf75SKonstantin Belousov 	if (locked)
225295c4bf75SKonstantin Belousov 		sx_sunlock(&uma_drain_lock);
225395c4bf75SKonstantin Belousov 	return (res);
22548355f576SJeff Roberson }
22558355f576SJeff Roberson 
22560095a784SJeff Roberson /* See uma.h */
22570095a784SJeff Roberson uma_zone_t
2258af526374SJeff Roberson uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor,
2259af526374SJeff Roberson 		    uma_init zinit, uma_fini zfini, uma_import zimport,
2260af526374SJeff Roberson 		    uma_release zrelease, void *arg, int flags)
22610095a784SJeff Roberson {
22620095a784SJeff Roberson 	struct uma_zctor_args args;
22630095a784SJeff Roberson 
22640095a784SJeff Roberson 	memset(&args, 0, sizeof(args));
22650095a784SJeff Roberson 	args.name = name;
2266af526374SJeff Roberson 	args.size = size;
22670095a784SJeff Roberson 	args.ctor = ctor;
22680095a784SJeff Roberson 	args.dtor = dtor;
22690095a784SJeff Roberson 	args.uminit = zinit;
22700095a784SJeff Roberson 	args.fini = zfini;
22710095a784SJeff Roberson 	args.import = zimport;
22720095a784SJeff Roberson 	args.release = zrelease;
22730095a784SJeff Roberson 	args.arg = arg;
22740095a784SJeff Roberson 	args.align = 0;
2275bb15d1c7SGleb Smirnoff 	args.flags = flags | UMA_ZFLAG_CACHE;
22760095a784SJeff Roberson 
2277ab3185d1SJeff Roberson 	return (zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK));
22780095a784SJeff Roberson }
22790095a784SJeff Roberson 
22808355f576SJeff Roberson /* See uma.h */
22819c2cd7e5SJeff Roberson void
22829c2cd7e5SJeff Roberson uma_zdestroy(uma_zone_t zone)
22839c2cd7e5SJeff Roberson {
2284f4ff923bSRobert Watson 
228595c4bf75SKonstantin Belousov 	sx_slock(&uma_drain_lock);
22860095a784SJeff Roberson 	zone_free_item(zones, zone, NULL, SKIP_NONE);
228795c4bf75SKonstantin Belousov 	sx_sunlock(&uma_drain_lock);
22889c2cd7e5SJeff Roberson }
22899c2cd7e5SJeff Roberson 
22908d6fbbb8SJeff Roberson void
22918d6fbbb8SJeff Roberson uma_zwait(uma_zone_t zone)
22928d6fbbb8SJeff Roberson {
22938d6fbbb8SJeff Roberson 	void *item;
22948d6fbbb8SJeff Roberson 
22958d6fbbb8SJeff Roberson 	item = uma_zalloc_arg(zone, NULL, M_WAITOK);
22968d6fbbb8SJeff Roberson 	uma_zfree(zone, item);
22978d6fbbb8SJeff Roberson }
22988d6fbbb8SJeff Roberson 
22994e180881SMateusz Guzik void *
23004e180881SMateusz Guzik uma_zalloc_pcpu_arg(uma_zone_t zone, void *udata, int flags)
23014e180881SMateusz Guzik {
23024e180881SMateusz Guzik 	void *item;
2303b4799947SRuslan Bukin #ifdef SMP
23044e180881SMateusz Guzik 	int i;
23054e180881SMateusz Guzik 
23064e180881SMateusz Guzik 	MPASS(zone->uz_flags & UMA_ZONE_PCPU);
2307b4799947SRuslan Bukin #endif
23084e180881SMateusz Guzik 	item = uma_zalloc_arg(zone, udata, flags & ~M_ZERO);
23094e180881SMateusz Guzik 	if (item != NULL && (flags & M_ZERO)) {
2310b4799947SRuslan Bukin #ifdef SMP
2311013072f0SMark Johnston 		for (i = 0; i <= mp_maxid; i++)
23124e180881SMateusz Guzik 			bzero(zpcpu_get_cpu(item, i), zone->uz_size);
2313b4799947SRuslan Bukin #else
2314b4799947SRuslan Bukin 		bzero(item, zone->uz_size);
2315b4799947SRuslan Bukin #endif
23164e180881SMateusz Guzik 	}
23174e180881SMateusz Guzik 	return (item);
23184e180881SMateusz Guzik }
23194e180881SMateusz Guzik 
23204e180881SMateusz Guzik /*
23214e180881SMateusz Guzik  * A stub while both regular and pcpu cases are identical.
23224e180881SMateusz Guzik  */
23234e180881SMateusz Guzik void
23244e180881SMateusz Guzik uma_zfree_pcpu_arg(uma_zone_t zone, void *item, void *udata)
23254e180881SMateusz Guzik {
23264e180881SMateusz Guzik 
2327c5b7751fSIan Lepore #ifdef SMP
23284e180881SMateusz Guzik 	MPASS(zone->uz_flags & UMA_ZONE_PCPU);
2329c5b7751fSIan Lepore #endif
23304e180881SMateusz Guzik 	uma_zfree_arg(zone, item, udata);
23314e180881SMateusz Guzik }
23324e180881SMateusz Guzik 
23339c2cd7e5SJeff Roberson /* See uma.h */
23348355f576SJeff Roberson void *
23352cc35ff9SJeff Roberson uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
23368355f576SJeff Roberson {
2337ab3185d1SJeff Roberson 	uma_zone_domain_t zdom;
23388355f576SJeff Roberson 	uma_bucket_t bucket;
2339ab3185d1SJeff Roberson 	uma_cache_t cache;
2340ab3185d1SJeff Roberson 	void *item;
2341bb15d1c7SGleb Smirnoff 	int cpu, domain, lockfail, maxbucket;
2342c5deaf04SGleb Smirnoff #ifdef INVARIANTS
2343c5deaf04SGleb Smirnoff 	bool skipdbg;
2344c5deaf04SGleb Smirnoff #endif
23458355f576SJeff Roberson 
2346e866d8f0SMark Murray 	/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
234719fa89e9SMark Murray 	random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
234810cb2424SMark Murray 
23498355f576SJeff Roberson 	/* This is the fast path allocation */
23501431a748SGleb Smirnoff 	CTR4(KTR_UMA, "uma_zalloc_arg thread %x zone %s(%p) flags %d",
23511431a748SGleb Smirnoff 	    curthread, zone->uz_name, zone, flags);
2352a553d4b8SJeff Roberson 
2353635fd505SRobert Watson 	if (flags & M_WAITOK) {
2354b23f72e9SBrian Feldman 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
2355635fd505SRobert Watson 		    "uma_zalloc_arg: zone \"%s\"", zone->uz_name);
23564c1cc01cSJohn Baldwin 	}
23570766f278SJonathan T. Looney 	KASSERT((flags & M_EXEC) == 0, ("uma_zalloc_arg: called with M_EXEC"));
2358d9e2e68dSMark Johnston 	KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
23591067a2baSJonathan T. Looney 	    ("uma_zalloc_arg: called with spinlock or critical section held"));
2360ea99223eSMateusz Guzik 	if (zone->uz_flags & UMA_ZONE_PCPU)
2361b8af2820SMateusz Guzik 		KASSERT((flags & M_ZERO) == 0, ("allocating from a pcpu zone "
2362b8af2820SMateusz Guzik 		    "with M_ZERO passed"));
23631067a2baSJonathan T. Looney 
23648d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD
23658d689e04SGleb Smirnoff 	if (memguard_cmp_zone(zone)) {
23668d689e04SGleb Smirnoff 		item = memguard_alloc(zone->uz_size, flags);
23678d689e04SGleb Smirnoff 		if (item != NULL) {
23688d689e04SGleb Smirnoff 			if (zone->uz_init != NULL &&
23698d689e04SGleb Smirnoff 			    zone->uz_init(item, zone->uz_size, flags) != 0)
23708d689e04SGleb Smirnoff 				return (NULL);
23718d689e04SGleb Smirnoff 			if (zone->uz_ctor != NULL &&
2372fc03d22bSJeff Roberson 			    zone->uz_ctor(item, zone->uz_size, udata,
2373fc03d22bSJeff Roberson 			    flags) != 0) {
23748d689e04SGleb Smirnoff 			    	zone->uz_fini(item, zone->uz_size);
23758d689e04SGleb Smirnoff 				return (NULL);
23768d689e04SGleb Smirnoff 			}
23778d689e04SGleb Smirnoff 			return (item);
23788d689e04SGleb Smirnoff 		}
23798d689e04SGleb Smirnoff 		/* This is unfortunate but should not be fatal. */
23808d689e04SGleb Smirnoff 	}
23818d689e04SGleb Smirnoff #endif
23825d1ae027SRobert Watson 	/*
23835d1ae027SRobert Watson 	 * If possible, allocate from the per-CPU cache.  There are two
23845d1ae027SRobert Watson 	 * requirements for safe access to the per-CPU cache: (1) the thread
23855d1ae027SRobert Watson 	 * accessing the cache must not be preempted or yield during access,
23865d1ae027SRobert Watson 	 * and (2) the thread must not migrate CPUs without switching which
23875d1ae027SRobert Watson 	 * cache it accesses.  We rely on a critical section to prevent
23885d1ae027SRobert Watson 	 * preemption and migration.  We release the critical section in
23895d1ae027SRobert Watson 	 * order to acquire the zone mutex if we are unable to allocate from
23905d1ae027SRobert Watson 	 * the current cache; when we re-acquire the critical section, we
23915d1ae027SRobert Watson 	 * must detect and handle migration if it has occurred.
23925d1ae027SRobert Watson 	 */
239381c0d72cSGleb Smirnoff zalloc_restart:
23945d1ae027SRobert Watson 	critical_enter();
23955d1ae027SRobert Watson 	cpu = curcpu;
23968355f576SJeff Roberson 	cache = &zone->uz_cpu[cpu];
23978355f576SJeff Roberson 
23988355f576SJeff Roberson zalloc_start:
23998355f576SJeff Roberson 	bucket = cache->uc_allocbucket;
2400fc03d22bSJeff Roberson 	if (bucket != NULL && bucket->ub_cnt > 0) {
2401cae33c14SJeff Roberson 		bucket->ub_cnt--;
2402cae33c14SJeff Roberson 		item = bucket->ub_bucket[bucket->ub_cnt];
24038355f576SJeff Roberson #ifdef INVARIANTS
2404cae33c14SJeff Roberson 		bucket->ub_bucket[bucket->ub_cnt] = NULL;
24058355f576SJeff Roberson #endif
2406fc03d22bSJeff Roberson 		KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled."));
24078355f576SJeff Roberson 		cache->uc_allocs++;
24085d1ae027SRobert Watson 		critical_exit();
2409c5deaf04SGleb Smirnoff #ifdef INVARIANTS
2410c5deaf04SGleb Smirnoff 		skipdbg = uma_dbg_zskip(zone, item);
2411c5deaf04SGleb Smirnoff #endif
2412fc03d22bSJeff Roberson 		if (zone->uz_ctor != NULL &&
2413c5deaf04SGleb Smirnoff #ifdef INVARIANTS
2414c5deaf04SGleb Smirnoff 		    (!skipdbg || zone->uz_ctor != trash_ctor ||
2415c5deaf04SGleb Smirnoff 		    zone->uz_dtor != trash_dtor) &&
2416c5deaf04SGleb Smirnoff #endif
2417fc03d22bSJeff Roberson 		    zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
24182efcc8cbSGleb Smirnoff 			counter_u64_add(zone->uz_fails, 1);
2419bb15d1c7SGleb Smirnoff 			zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT);
2420b23f72e9SBrian Feldman 			return (NULL);
2421b23f72e9SBrian Feldman 		}
2422ef72505eSJeff Roberson #ifdef INVARIANTS
2423c5deaf04SGleb Smirnoff 		if (!skipdbg)
2424ef72505eSJeff Roberson 			uma_dbg_alloc(zone, NULL, item);
2425ef72505eSJeff Roberson #endif
24262cc35ff9SJeff Roberson 		if (flags & M_ZERO)
242748343a2fSGleb Smirnoff 			uma_zero_item(item, zone);
24288355f576SJeff Roberson 		return (item);
2429fc03d22bSJeff Roberson 	}
2430fc03d22bSJeff Roberson 
24318355f576SJeff Roberson 	/*
24328355f576SJeff Roberson 	 * We have run out of items in our alloc bucket.
24338355f576SJeff Roberson 	 * See if we can switch with our free bucket.
24348355f576SJeff Roberson 	 */
2435b983089aSJeff Roberson 	bucket = cache->uc_freebucket;
2436fc03d22bSJeff Roberson 	if (bucket != NULL && bucket->ub_cnt > 0) {
24371431a748SGleb Smirnoff 		CTR2(KTR_UMA,
24381431a748SGleb Smirnoff 		    "uma_zalloc: zone %s(%p) swapping empty with alloc",
24391431a748SGleb Smirnoff 		    zone->uz_name, zone);
24408355f576SJeff Roberson 		cache->uc_freebucket = cache->uc_allocbucket;
2441b983089aSJeff Roberson 		cache->uc_allocbucket = bucket;
24428355f576SJeff Roberson 		goto zalloc_start;
24438355f576SJeff Roberson 	}
2444fc03d22bSJeff Roberson 
2445fc03d22bSJeff Roberson 	/*
2446fc03d22bSJeff Roberson 	 * Discard any empty allocation bucket while we hold no locks.
2447fc03d22bSJeff Roberson 	 */
2448fc03d22bSJeff Roberson 	bucket = cache->uc_allocbucket;
2449fc03d22bSJeff Roberson 	cache->uc_allocbucket = NULL;
2450fc03d22bSJeff Roberson 	critical_exit();
2451fc03d22bSJeff Roberson 	if (bucket != NULL)
24526fd34d6fSJeff Roberson 		bucket_free(zone, bucket, udata);
2453fc03d22bSJeff Roberson 
245430c5525bSAndrew Gallatin 	if (zone->uz_flags & UMA_ZONE_NUMA) {
2455ab3185d1SJeff Roberson 		domain = PCPU_GET(domain);
245630c5525bSAndrew Gallatin 		if (VM_DOMAIN_EMPTY(domain))
245730c5525bSAndrew Gallatin 			domain = UMA_ANYDOMAIN;
245830c5525bSAndrew Gallatin 	} else
2459ab3185d1SJeff Roberson 		domain = UMA_ANYDOMAIN;
2460ab3185d1SJeff Roberson 
2461fc03d22bSJeff Roberson 	/* Short-circuit for zones without buckets and low memory. */
2462bb15d1c7SGleb Smirnoff 	if (zone->uz_count == 0 || bucketdisable) {
2463bb15d1c7SGleb Smirnoff 		ZONE_LOCK(zone);
2464fc03d22bSJeff Roberson 		goto zalloc_item;
2465bb15d1c7SGleb Smirnoff 	}
2466fc03d22bSJeff Roberson 
24675d1ae027SRobert Watson 	/*
24685d1ae027SRobert Watson 	 * Attempt to retrieve the item from the per-CPU cache has failed, so
24695d1ae027SRobert Watson 	 * we must go back to the zone.  This requires the zone lock, so we
24705d1ae027SRobert Watson 	 * must drop the critical section, then re-acquire it when we go back
24715d1ae027SRobert Watson 	 * to the cache.  Since the critical section is released, we may be
24725d1ae027SRobert Watson 	 * preempted or migrate.  As such, make sure not to maintain any
24735d1ae027SRobert Watson 	 * thread-local state specific to the cache from prior to releasing
24745d1ae027SRobert Watson 	 * the critical section.
24755d1ae027SRobert Watson 	 */
2476fc03d22bSJeff Roberson 	lockfail = 0;
2477fc03d22bSJeff Roberson 	if (ZONE_TRYLOCK(zone) == 0) {
2478fc03d22bSJeff Roberson 		/* Record contention to size the buckets. */
2479a553d4b8SJeff Roberson 		ZONE_LOCK(zone);
2480fc03d22bSJeff Roberson 		lockfail = 1;
2481fc03d22bSJeff Roberson 	}
24825d1ae027SRobert Watson 	critical_enter();
24835d1ae027SRobert Watson 	cpu = curcpu;
24845d1ae027SRobert Watson 	cache = &zone->uz_cpu[cpu];
24855d1ae027SRobert Watson 
2486fc03d22bSJeff Roberson 	/* See if we lost the race to fill the cache. */
2487fc03d22bSJeff Roberson 	if (cache->uc_allocbucket != NULL) {
2488fc03d22bSJeff Roberson 		ZONE_UNLOCK(zone);
2489fc03d22bSJeff Roberson 		goto zalloc_start;
2490a553d4b8SJeff Roberson 	}
24918355f576SJeff Roberson 
2492fc03d22bSJeff Roberson 	/*
2493fc03d22bSJeff Roberson 	 * Check the zone's cache of buckets.
2494fc03d22bSJeff Roberson 	 */
2495ab3185d1SJeff Roberson 	if (domain == UMA_ANYDOMAIN)
2496ab3185d1SJeff Roberson 		zdom = &zone->uz_domain[0];
2497ab3185d1SJeff Roberson 	else
2498ab3185d1SJeff Roberson 		zdom = &zone->uz_domain[domain];
24990f9b7bf3SMark Johnston 	if ((bucket = zone_try_fetch_bucket(zone, zdom, true)) != NULL) {
2500cae33c14SJeff Roberson 		KASSERT(bucket->ub_cnt != 0,
2501a553d4b8SJeff Roberson 		    ("uma_zalloc_arg: Returning an empty bucket."));
2502a553d4b8SJeff Roberson 		cache->uc_allocbucket = bucket;
2503a553d4b8SJeff Roberson 		ZONE_UNLOCK(zone);
25048355f576SJeff Roberson 		goto zalloc_start;
2505a553d4b8SJeff Roberson 	}
25065d1ae027SRobert Watson 	/* We are no longer associated with this CPU. */
25075d1ae027SRobert Watson 	critical_exit();
2508bbee39c6SJeff Roberson 
2509fc03d22bSJeff Roberson 	/*
2510fc03d22bSJeff Roberson 	 * We bump the uz count when the cache size is insufficient to
2511fc03d22bSJeff Roberson 	 * handle the working set.
2512fc03d22bSJeff Roberson 	 */
2513bb15d1c7SGleb Smirnoff 	if (lockfail && zone->uz_count < zone->uz_count_max)
2514a553d4b8SJeff Roberson 		zone->uz_count++;
2515bb15d1c7SGleb Smirnoff 
2516bb15d1c7SGleb Smirnoff 	if (zone->uz_max_items > 0) {
2517bb15d1c7SGleb Smirnoff 		if (zone->uz_items >= zone->uz_max_items)
2518bb15d1c7SGleb Smirnoff 			goto zalloc_item;
2519bb15d1c7SGleb Smirnoff 		maxbucket = MIN(zone->uz_count,
2520bb15d1c7SGleb Smirnoff 		    zone->uz_max_items - zone->uz_items);
2521bb45b411SGleb Smirnoff 		zone->uz_items += maxbucket;
2522bb15d1c7SGleb Smirnoff 	} else
2523bb15d1c7SGleb Smirnoff 		maxbucket = zone->uz_count;
2524fc03d22bSJeff Roberson 	ZONE_UNLOCK(zone);
2525099a0e58SBosko Milekic 
25268355f576SJeff Roberson 	/*
2527a553d4b8SJeff Roberson 	 * Now lets just fill a bucket and put it on the free list.  If that
2528763df3ecSPedro F. Giffuni 	 * works we'll restart the allocation from the beginning and it
2529fc03d22bSJeff Roberson 	 * will use the just filled bucket.
2530bbee39c6SJeff Roberson 	 */
2531bb15d1c7SGleb Smirnoff 	bucket = zone_alloc_bucket(zone, udata, domain, flags, maxbucket);
25321431a748SGleb Smirnoff 	CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p",
25331431a748SGleb Smirnoff 	    zone->uz_name, zone, bucket);
2534fc03d22bSJeff Roberson 	ZONE_LOCK(zone);
2535bb15d1c7SGleb Smirnoff 	if (bucket != NULL) {
2536bb45b411SGleb Smirnoff 		if (zone->uz_max_items > 0 && bucket->ub_cnt < maxbucket) {
2537bb45b411SGleb Smirnoff 			MPASS(zone->uz_items >= maxbucket - bucket->ub_cnt);
2538bb15d1c7SGleb Smirnoff 			zone->uz_items -= maxbucket - bucket->ub_cnt;
2539bb15d1c7SGleb Smirnoff 			if (zone->uz_sleepers > 0 &&
2540bb15d1c7SGleb Smirnoff 			    zone->uz_items < zone->uz_max_items)
2541bb15d1c7SGleb Smirnoff 				wakeup_one(zone);
2542bb15d1c7SGleb Smirnoff 		}
2543fc03d22bSJeff Roberson 		critical_enter();
2544fc03d22bSJeff Roberson 		cpu = curcpu;
2545fc03d22bSJeff Roberson 		cache = &zone->uz_cpu[cpu];
25460f9b7bf3SMark Johnston 
2547fc03d22bSJeff Roberson 		/*
2548fc03d22bSJeff Roberson 		 * See if we lost the race or were migrated.  Cache the
2549fc03d22bSJeff Roberson 		 * initialized bucket to make this less likely or claim
2550fc03d22bSJeff Roberson 		 * the memory directly.
2551fc03d22bSJeff Roberson 		 */
255281c0d72cSGleb Smirnoff 		if (cache->uc_allocbucket == NULL &&
255381c0d72cSGleb Smirnoff 		    ((zone->uz_flags & UMA_ZONE_NUMA) == 0 ||
255481c0d72cSGleb Smirnoff 		    domain == PCPU_GET(domain))) {
2555ab3185d1SJeff Roberson 			cache->uc_allocbucket = bucket;
25560f9b7bf3SMark Johnston 			zdom->uzd_imax += bucket->ub_cnt;
2557bb15d1c7SGleb Smirnoff 		} else if (zone->uz_bkt_count >= zone->uz_bkt_max) {
255881c0d72cSGleb Smirnoff 			critical_exit();
255981c0d72cSGleb Smirnoff 			ZONE_UNLOCK(zone);
256081c0d72cSGleb Smirnoff 			bucket_drain(zone, bucket);
256181c0d72cSGleb Smirnoff 			bucket_free(zone, bucket, udata);
256281c0d72cSGleb Smirnoff 			goto zalloc_restart;
256381c0d72cSGleb Smirnoff 		} else
25640f9b7bf3SMark Johnston 			zone_put_bucket(zone, zdom, bucket, false);
2565bbee39c6SJeff Roberson 		ZONE_UNLOCK(zone);
2566fc03d22bSJeff Roberson 		goto zalloc_start;
2567bb45b411SGleb Smirnoff 	} else if (zone->uz_max_items > 0) {
2568bb15d1c7SGleb Smirnoff 		zone->uz_items -= maxbucket;
2569bb15d1c7SGleb Smirnoff 		if (zone->uz_sleepers > 0 &&
2570bb15d1c7SGleb Smirnoff 		    zone->uz_items + 1 < zone->uz_max_items)
2571bb15d1c7SGleb Smirnoff 			wakeup_one(zone);
2572bbee39c6SJeff Roberson 	}
2573fc03d22bSJeff Roberson 
2574bbee39c6SJeff Roberson 	/*
2575bbee39c6SJeff Roberson 	 * We may not be able to get a bucket so return an actual item.
2576bbee39c6SJeff Roberson 	 */
2577fc03d22bSJeff Roberson zalloc_item:
2578bb15d1c7SGleb Smirnoff 	item = zone_alloc_item_locked(zone, udata, domain, flags);
2579fc03d22bSJeff Roberson 
2580e20a199fSJeff Roberson 	return (item);
2581bbee39c6SJeff Roberson }
2582bbee39c6SJeff Roberson 
2583ab3185d1SJeff Roberson void *
2584ab3185d1SJeff Roberson uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags)
2585bbee39c6SJeff Roberson {
2586ab3185d1SJeff Roberson 
2587ab3185d1SJeff Roberson 	/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
258819fa89e9SMark Murray 	random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
2589ab3185d1SJeff Roberson 
2590ab3185d1SJeff Roberson 	/* This is the fast path allocation */
2591ab3185d1SJeff Roberson 	CTR5(KTR_UMA,
2592ab3185d1SJeff Roberson 	    "uma_zalloc_domain thread %x zone %s(%p) domain %d flags %d",
2593ab3185d1SJeff Roberson 	    curthread, zone->uz_name, zone, domain, flags);
2594ab3185d1SJeff Roberson 
2595ab3185d1SJeff Roberson 	if (flags & M_WAITOK) {
2596ab3185d1SJeff Roberson 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
2597ab3185d1SJeff Roberson 		    "uma_zalloc_domain: zone \"%s\"", zone->uz_name);
2598ab3185d1SJeff Roberson 	}
2599ab3185d1SJeff Roberson 	KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
2600ab3185d1SJeff Roberson 	    ("uma_zalloc_domain: called with spinlock or critical section held"));
2601ab3185d1SJeff Roberson 
2602ab3185d1SJeff Roberson 	return (zone_alloc_item(zone, udata, domain, flags));
2603ab3185d1SJeff Roberson }
2604ab3185d1SJeff Roberson 
2605ab3185d1SJeff Roberson /*
2606ab3185d1SJeff Roberson  * Find a slab with some space.  Prefer slabs that are partially used over those
2607ab3185d1SJeff Roberson  * that are totally full.  This helps to reduce fragmentation.
2608ab3185d1SJeff Roberson  *
2609ab3185d1SJeff Roberson  * If 'rr' is 1, search all domains starting from 'domain'.  Otherwise check
2610ab3185d1SJeff Roberson  * only 'domain'.
2611ab3185d1SJeff Roberson  */
2612ab3185d1SJeff Roberson static uma_slab_t
2613194a979eSMark Johnston keg_first_slab(uma_keg_t keg, int domain, bool rr)
2614ab3185d1SJeff Roberson {
2615ab3185d1SJeff Roberson 	uma_domain_t dom;
2616bbee39c6SJeff Roberson 	uma_slab_t slab;
2617ab3185d1SJeff Roberson 	int start;
2618ab3185d1SJeff Roberson 
2619ab3185d1SJeff Roberson 	KASSERT(domain >= 0 && domain < vm_ndomains,
2620ab3185d1SJeff Roberson 	    ("keg_first_slab: domain %d out of range", domain));
2621bb15d1c7SGleb Smirnoff 	KEG_LOCK_ASSERT(keg);
2622ab3185d1SJeff Roberson 
2623ab3185d1SJeff Roberson 	slab = NULL;
2624ab3185d1SJeff Roberson 	start = domain;
2625ab3185d1SJeff Roberson 	do {
2626ab3185d1SJeff Roberson 		dom = &keg->uk_domain[domain];
2627ab3185d1SJeff Roberson 		if (!LIST_EMPTY(&dom->ud_part_slab))
2628ab3185d1SJeff Roberson 			return (LIST_FIRST(&dom->ud_part_slab));
2629ab3185d1SJeff Roberson 		if (!LIST_EMPTY(&dom->ud_free_slab)) {
2630ab3185d1SJeff Roberson 			slab = LIST_FIRST(&dom->ud_free_slab);
2631ab3185d1SJeff Roberson 			LIST_REMOVE(slab, us_link);
2632ab3185d1SJeff Roberson 			LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);
2633ab3185d1SJeff Roberson 			return (slab);
2634ab3185d1SJeff Roberson 		}
2635ab3185d1SJeff Roberson 		if (rr)
2636ab3185d1SJeff Roberson 			domain = (domain + 1) % vm_ndomains;
2637ab3185d1SJeff Roberson 	} while (domain != start);
2638ab3185d1SJeff Roberson 
2639ab3185d1SJeff Roberson 	return (NULL);
2640ab3185d1SJeff Roberson }
2641ab3185d1SJeff Roberson 
2642ab3185d1SJeff Roberson static uma_slab_t
2643194a979eSMark Johnston keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags)
2644ab3185d1SJeff Roberson {
2645194a979eSMark Johnston 	uint32_t reserve;
2646099a0e58SBosko Milekic 
2647bb15d1c7SGleb Smirnoff 	KEG_LOCK_ASSERT(keg);
2648194a979eSMark Johnston 
2649194a979eSMark Johnston 	reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve;
2650194a979eSMark Johnston 	if (keg->uk_free <= reserve)
2651194a979eSMark Johnston 		return (NULL);
2652194a979eSMark Johnston 	return (keg_first_slab(keg, domain, rr));
2653194a979eSMark Johnston }
2654194a979eSMark Johnston 
2655194a979eSMark Johnston static uma_slab_t
2656194a979eSMark Johnston keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags)
2657194a979eSMark Johnston {
2658194a979eSMark Johnston 	struct vm_domainset_iter di;
2659194a979eSMark Johnston 	uma_domain_t dom;
2660194a979eSMark Johnston 	uma_slab_t slab;
2661194a979eSMark Johnston 	int aflags, domain;
2662194a979eSMark Johnston 	bool rr;
2663194a979eSMark Johnston 
2664194a979eSMark Johnston restart:
2665bb15d1c7SGleb Smirnoff 	KEG_LOCK_ASSERT(keg);
2666bbee39c6SJeff Roberson 
2667bbee39c6SJeff Roberson 	/*
2668194a979eSMark Johnston 	 * Use the keg's policy if upper layers haven't already specified a
2669194a979eSMark Johnston 	 * domain (as happens with first-touch zones).
2670194a979eSMark Johnston 	 *
2671194a979eSMark Johnston 	 * To avoid races we run the iterator with the keg lock held, but that
2672194a979eSMark Johnston 	 * means that we cannot allow the vm_domainset layer to sleep.  Thus,
2673194a979eSMark Johnston 	 * clear M_WAITOK and handle low memory conditions locally.
2674bbee39c6SJeff Roberson 	 */
2675ab3185d1SJeff Roberson 	rr = rdomain == UMA_ANYDOMAIN;
2676ab3185d1SJeff Roberson 	if (rr) {
2677194a979eSMark Johnston 		aflags = (flags & ~M_WAITOK) | M_NOWAIT;
2678194a979eSMark Johnston 		vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain,
2679194a979eSMark Johnston 		    &aflags);
2680194a979eSMark Johnston 	} else {
2681194a979eSMark Johnston 		aflags = flags;
2682194a979eSMark Johnston 		domain = rdomain;
2683194a979eSMark Johnston 	}
2684ab3185d1SJeff Roberson 
2685194a979eSMark Johnston 	for (;;) {
2686194a979eSMark Johnston 		slab = keg_fetch_free_slab(keg, domain, rr, flags);
2687194a979eSMark Johnston 		if (slab != NULL) {
2688e20a199fSJeff Roberson 			MPASS(slab->us_keg == keg);
2689bbee39c6SJeff Roberson 			return (slab);
2690bbee39c6SJeff Roberson 		}
2691bbee39c6SJeff Roberson 
2692bbee39c6SJeff Roberson 		/*
2693bbee39c6SJeff Roberson 		 * M_NOVM means don't ask at all!
2694bbee39c6SJeff Roberson 		 */
2695bbee39c6SJeff Roberson 		if (flags & M_NOVM)
2696bbee39c6SJeff Roberson 			break;
2697bbee39c6SJeff Roberson 
2698bb15d1c7SGleb Smirnoff 		KASSERT(zone->uz_max_items == 0 ||
2699bb15d1c7SGleb Smirnoff 		    zone->uz_items <= zone->uz_max_items,
2700bb15d1c7SGleb Smirnoff 		    ("%s: zone %p overflow", __func__, zone));
2701bb15d1c7SGleb Smirnoff 
270286220393SMark Johnston 		slab = keg_alloc_slab(keg, zone, domain, flags, aflags);
2703bbee39c6SJeff Roberson 		/*
2704bbee39c6SJeff Roberson 		 * If we got a slab here it's safe to mark it partially used
2705bbee39c6SJeff Roberson 		 * and return.  We assume that the caller is going to remove
2706bbee39c6SJeff Roberson 		 * at least one item.
2707bbee39c6SJeff Roberson 		 */
2708bbee39c6SJeff Roberson 		if (slab) {
2709e20a199fSJeff Roberson 			MPASS(slab->us_keg == keg);
2710ab3185d1SJeff Roberson 			dom = &keg->uk_domain[slab->us_domain];
2711ab3185d1SJeff Roberson 			LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);
2712bbee39c6SJeff Roberson 			return (slab);
2713bbee39c6SJeff Roberson 		}
2714194a979eSMark Johnston 		KEG_LOCK(keg);
2715194a979eSMark Johnston 		if (rr && vm_domainset_iter_policy(&di, &domain) != 0) {
2716194a979eSMark Johnston 			if ((flags & M_WAITOK) != 0) {
2717194a979eSMark Johnston 				KEG_UNLOCK(keg);
2718194a979eSMark Johnston 				vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask);
2719194a979eSMark Johnston 				KEG_LOCK(keg);
2720194a979eSMark Johnston 				goto restart;
272130c5525bSAndrew Gallatin 			}
2722194a979eSMark Johnston 			break;
2723194a979eSMark Johnston 		}
2724ab3185d1SJeff Roberson 	}
2725ab3185d1SJeff Roberson 
2726bbee39c6SJeff Roberson 	/*
2727bbee39c6SJeff Roberson 	 * We might not have been able to get a slab but another cpu
2728bbee39c6SJeff Roberson 	 * could have while we were unlocked.  Check again before we
2729bbee39c6SJeff Roberson 	 * fail.
2730bbee39c6SJeff Roberson 	 */
2731194a979eSMark Johnston 	if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) {
2732ab3185d1SJeff Roberson 		MPASS(slab->us_keg == keg);
2733bbee39c6SJeff Roberson 		return (slab);
2734bbee39c6SJeff Roberson 	}
2735ab3185d1SJeff Roberson 	return (NULL);
2736ab3185d1SJeff Roberson }
2737bbee39c6SJeff Roberson 
2738e20a199fSJeff Roberson static uma_slab_t
2739ab3185d1SJeff Roberson zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int domain, int flags)
2740e20a199fSJeff Roberson {
2741e20a199fSJeff Roberson 	uma_slab_t slab;
2742e20a199fSJeff Roberson 
2743af526374SJeff Roberson 	if (keg == NULL) {
2744bb15d1c7SGleb Smirnoff 		keg = zone->uz_keg;
2745af526374SJeff Roberson 		KEG_LOCK(keg);
2746af526374SJeff Roberson 	}
2747e20a199fSJeff Roberson 
2748e20a199fSJeff Roberson 	for (;;) {
2749ab3185d1SJeff Roberson 		slab = keg_fetch_slab(keg, zone, domain, flags);
2750e20a199fSJeff Roberson 		if (slab)
2751e20a199fSJeff Roberson 			return (slab);
2752e20a199fSJeff Roberson 		if (flags & (M_NOWAIT | M_NOVM))
2753e20a199fSJeff Roberson 			break;
2754e20a199fSJeff Roberson 	}
2755af526374SJeff Roberson 	KEG_UNLOCK(keg);
2756e20a199fSJeff Roberson 	return (NULL);
2757e20a199fSJeff Roberson }
2758e20a199fSJeff Roberson 
2759d56368d7SBosko Milekic static void *
27600095a784SJeff Roberson slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
2761bbee39c6SJeff Roberson {
2762ab3185d1SJeff Roberson 	uma_domain_t dom;
2763bbee39c6SJeff Roberson 	void *item;
276485dcf349SGleb Smirnoff 	uint8_t freei;
2765bbee39c6SJeff Roberson 
27660095a784SJeff Roberson 	MPASS(keg == slab->us_keg);
2767bb15d1c7SGleb Smirnoff 	KEG_LOCK_ASSERT(keg);
2768099a0e58SBosko Milekic 
2769ef72505eSJeff Roberson 	freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
2770ef72505eSJeff Roberson 	BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free);
2771099a0e58SBosko Milekic 	item = slab->us_data + (keg->uk_rsize * freei);
2772bbee39c6SJeff Roberson 	slab->us_freecount--;
2773099a0e58SBosko Milekic 	keg->uk_free--;
2774ef72505eSJeff Roberson 
2775bbee39c6SJeff Roberson 	/* Move this slab to the full list */
2776bbee39c6SJeff Roberson 	if (slab->us_freecount == 0) {
2777bbee39c6SJeff Roberson 		LIST_REMOVE(slab, us_link);
2778ab3185d1SJeff Roberson 		dom = &keg->uk_domain[slab->us_domain];
2779ab3185d1SJeff Roberson 		LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link);
2780bbee39c6SJeff Roberson 	}
2781bbee39c6SJeff Roberson 
2782bbee39c6SJeff Roberson 	return (item);
2783bbee39c6SJeff Roberson }
2784bbee39c6SJeff Roberson 
2785bbee39c6SJeff Roberson static int
2786ab3185d1SJeff Roberson zone_import(uma_zone_t zone, void **bucket, int max, int domain, int flags)
27870095a784SJeff Roberson {
27880095a784SJeff Roberson 	uma_slab_t slab;
27890095a784SJeff Roberson 	uma_keg_t keg;
2790a03af342SSean Bruno #ifdef NUMA
2791ab3185d1SJeff Roberson 	int stripe;
2792a03af342SSean Bruno #endif
27930095a784SJeff Roberson 	int i;
27940095a784SJeff Roberson 
27950095a784SJeff Roberson 	slab = NULL;
27960095a784SJeff Roberson 	keg = NULL;
2797af526374SJeff Roberson 	/* Try to keep the buckets totally full */
27980095a784SJeff Roberson 	for (i = 0; i < max; ) {
2799ad66f958SGleb Smirnoff 		if ((slab = zone_fetch_slab(zone, keg, domain, flags)) == NULL)
28000095a784SJeff Roberson 			break;
28010095a784SJeff Roberson 		keg = slab->us_keg;
2802a03af342SSean Bruno #ifdef NUMA
2803ab3185d1SJeff Roberson 		stripe = howmany(max, vm_ndomains);
2804a03af342SSean Bruno #endif
28056fd34d6fSJeff Roberson 		while (slab->us_freecount && i < max) {
28060095a784SJeff Roberson 			bucket[i++] = slab_alloc_item(keg, slab);
28076fd34d6fSJeff Roberson 			if (keg->uk_free <= keg->uk_reserve)
28086fd34d6fSJeff Roberson 				break;
2809b6715dabSJeff Roberson #ifdef NUMA
2810ab3185d1SJeff Roberson 			/*
2811ab3185d1SJeff Roberson 			 * If the zone is striped we pick a new slab for every
2812ab3185d1SJeff Roberson 			 * N allocations.  Eliminating this conditional will
2813ab3185d1SJeff Roberson 			 * instead pick a new domain for each bucket rather
2814ab3185d1SJeff Roberson 			 * than stripe within each bucket.  The current option
2815ab3185d1SJeff Roberson 			 * produces more fragmentation and requires more cpu
2816ab3185d1SJeff Roberson 			 * time but yields better distribution.
2817ab3185d1SJeff Roberson 			 */
2818ab3185d1SJeff Roberson 			if ((zone->uz_flags & UMA_ZONE_NUMA) == 0 &&
2819ab3185d1SJeff Roberson 			    vm_ndomains > 1 && --stripe == 0)
2820ab3185d1SJeff Roberson 				break;
2821ab3185d1SJeff Roberson #endif
28226fd34d6fSJeff Roberson 		}
2823ab3185d1SJeff Roberson 		/* Don't block if we allocated any successfully. */
28240095a784SJeff Roberson 		flags &= ~M_WAITOK;
28250095a784SJeff Roberson 		flags |= M_NOWAIT;
28260095a784SJeff Roberson 	}
28270095a784SJeff Roberson 	if (slab != NULL)
28280095a784SJeff Roberson 		KEG_UNLOCK(keg);
28290095a784SJeff Roberson 
28300095a784SJeff Roberson 	return i;
28310095a784SJeff Roberson }
28320095a784SJeff Roberson 
2833fc03d22bSJeff Roberson static uma_bucket_t
2834bb15d1c7SGleb Smirnoff zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags, int max)
2835bbee39c6SJeff Roberson {
2836bbee39c6SJeff Roberson 	uma_bucket_t bucket;
2837bbee39c6SJeff Roberson 
283830c5525bSAndrew Gallatin 	CTR1(KTR_UMA, "zone_alloc:_bucket domain %d)", domain);
283930c5525bSAndrew Gallatin 
28406fd34d6fSJeff Roberson 	/* Don't wait for buckets, preserve caller's NOVM setting. */
28416fd34d6fSJeff Roberson 	bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM));
28420095a784SJeff Roberson 	if (bucket == NULL)
2843f7104ccdSAlexander Motin 		return (NULL);
28440095a784SJeff Roberson 
28450095a784SJeff Roberson 	bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
284637125720SGleb Smirnoff 	    MIN(max, bucket->ub_entries), domain, flags);
28470095a784SJeff Roberson 
28480095a784SJeff Roberson 	/*
28490095a784SJeff Roberson 	 * Initialize the memory if necessary.
28500095a784SJeff Roberson 	 */
28510095a784SJeff Roberson 	if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
2852099a0e58SBosko Milekic 		int i;
2853bbee39c6SJeff Roberson 
28540095a784SJeff Roberson 		for (i = 0; i < bucket->ub_cnt; i++)
2855e20a199fSJeff Roberson 			if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
28560095a784SJeff Roberson 			    flags) != 0)
2857b23f72e9SBrian Feldman 				break;
2858b23f72e9SBrian Feldman 		/*
2859b23f72e9SBrian Feldman 		 * If we couldn't initialize the whole bucket, put the
2860b23f72e9SBrian Feldman 		 * rest back onto the freelist.
2861b23f72e9SBrian Feldman 		 */
2862b23f72e9SBrian Feldman 		if (i != bucket->ub_cnt) {
2863af526374SJeff Roberson 			zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i],
28640095a784SJeff Roberson 			    bucket->ub_cnt - i);
2865a5a262c6SBosko Milekic #ifdef INVARIANTS
28660095a784SJeff Roberson 			bzero(&bucket->ub_bucket[i],
28670095a784SJeff Roberson 			    sizeof(void *) * (bucket->ub_cnt - i));
2868a5a262c6SBosko Milekic #endif
2869b23f72e9SBrian Feldman 			bucket->ub_cnt = i;
2870b23f72e9SBrian Feldman 		}
2871099a0e58SBosko Milekic 	}
2872099a0e58SBosko Milekic 
2873f7104ccdSAlexander Motin 	if (bucket->ub_cnt == 0) {
28746fd34d6fSJeff Roberson 		bucket_free(zone, bucket, udata);
28752efcc8cbSGleb Smirnoff 		counter_u64_add(zone->uz_fails, 1);
2876fc03d22bSJeff Roberson 		return (NULL);
2877bbee39c6SJeff Roberson 	}
2878fc03d22bSJeff Roberson 
2879fc03d22bSJeff Roberson 	return (bucket);
2880fc03d22bSJeff Roberson }
2881fc03d22bSJeff Roberson 
28828355f576SJeff Roberson /*
28830095a784SJeff Roberson  * Allocates a single item from a zone.
28848355f576SJeff Roberson  *
28858355f576SJeff Roberson  * Arguments
28868355f576SJeff Roberson  *	zone   The zone to alloc for.
28878355f576SJeff Roberson  *	udata  The data to be passed to the constructor.
2888ab3185d1SJeff Roberson  *	domain The domain to allocate from or UMA_ANYDOMAIN.
2889a163d034SWarner Losh  *	flags  M_WAITOK, M_NOWAIT, M_ZERO.
28908355f576SJeff Roberson  *
28918355f576SJeff Roberson  * Returns
28928355f576SJeff Roberson  *	NULL if there is no memory and M_NOWAIT is set
2893bbee39c6SJeff Roberson  *	An item if successful
28948355f576SJeff Roberson  */
28958355f576SJeff Roberson 
28968355f576SJeff Roberson static void *
2897ab3185d1SJeff Roberson zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags)
28988355f576SJeff Roberson {
2899bb15d1c7SGleb Smirnoff 
2900bb15d1c7SGleb Smirnoff 	ZONE_LOCK(zone);
2901bb15d1c7SGleb Smirnoff 	return (zone_alloc_item_locked(zone, udata, domain, flags));
2902bb15d1c7SGleb Smirnoff }
2903bb15d1c7SGleb Smirnoff 
2904bb15d1c7SGleb Smirnoff /*
2905bb15d1c7SGleb Smirnoff  * Returns with zone unlocked.
2906bb15d1c7SGleb Smirnoff  */
2907bb15d1c7SGleb Smirnoff static void *
2908bb15d1c7SGleb Smirnoff zone_alloc_item_locked(uma_zone_t zone, void *udata, int domain, int flags)
2909bb15d1c7SGleb Smirnoff {
29108355f576SJeff Roberson 	void *item;
2911c5deaf04SGleb Smirnoff #ifdef INVARIANTS
2912c5deaf04SGleb Smirnoff 	bool skipdbg;
2913c5deaf04SGleb Smirnoff #endif
29148355f576SJeff Roberson 
2915bb15d1c7SGleb Smirnoff 	ZONE_LOCK_ASSERT(zone);
2916bb15d1c7SGleb Smirnoff 
2917bb45b411SGleb Smirnoff 	if (zone->uz_max_items > 0) {
2918bb45b411SGleb Smirnoff 		if (zone->uz_items >= zone->uz_max_items) {
2919bb15d1c7SGleb Smirnoff 			zone_log_warning(zone);
2920bb15d1c7SGleb Smirnoff 			zone_maxaction(zone);
2921bb15d1c7SGleb Smirnoff 			if (flags & M_NOWAIT) {
2922bb15d1c7SGleb Smirnoff 				ZONE_UNLOCK(zone);
2923bb15d1c7SGleb Smirnoff 				return (NULL);
2924bb15d1c7SGleb Smirnoff 			}
2925bb15d1c7SGleb Smirnoff 			zone->uz_sleeps++;
2926bb15d1c7SGleb Smirnoff 			zone->uz_sleepers++;
2927bb15d1c7SGleb Smirnoff 			while (zone->uz_items >= zone->uz_max_items)
2928e7e4bcd8SGleb Smirnoff 				mtx_sleep(zone, zone->uz_lockptr, PVM,
2929e7e4bcd8SGleb Smirnoff 				    "zonelimit", 0);
2930bb15d1c7SGleb Smirnoff 			zone->uz_sleepers--;
2931bb15d1c7SGleb Smirnoff 			if (zone->uz_sleepers > 0 &&
2932bb15d1c7SGleb Smirnoff 			    zone->uz_items + 1 < zone->uz_max_items)
2933bb15d1c7SGleb Smirnoff 				wakeup_one(zone);
2934bb15d1c7SGleb Smirnoff 		}
2935bb15d1c7SGleb Smirnoff 		zone->uz_items++;
2936bb45b411SGleb Smirnoff 	}
2937bb15d1c7SGleb Smirnoff 	ZONE_UNLOCK(zone);
29388355f576SJeff Roberson 
293930c5525bSAndrew Gallatin 	if (domain != UMA_ANYDOMAIN) {
294030c5525bSAndrew Gallatin 		/* avoid allocs targeting empty domains */
294130c5525bSAndrew Gallatin 		if (VM_DOMAIN_EMPTY(domain))
294230c5525bSAndrew Gallatin 			domain = UMA_ANYDOMAIN;
294330c5525bSAndrew Gallatin 	}
2944ab3185d1SJeff Roberson 	if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1)
29450095a784SJeff Roberson 		goto fail;
29468355f576SJeff Roberson 
2947c5deaf04SGleb Smirnoff #ifdef INVARIANTS
2948c5deaf04SGleb Smirnoff 	skipdbg = uma_dbg_zskip(zone, item);
2949c5deaf04SGleb Smirnoff #endif
2950099a0e58SBosko Milekic 	/*
2951099a0e58SBosko Milekic 	 * We have to call both the zone's init (not the keg's init)
2952099a0e58SBosko Milekic 	 * and the zone's ctor.  This is because the item is going from
2953099a0e58SBosko Milekic 	 * a keg slab directly to the user, and the user is expecting it
2954099a0e58SBosko Milekic 	 * to be both zone-init'd as well as zone-ctor'd.
2955099a0e58SBosko Milekic 	 */
2956b23f72e9SBrian Feldman 	if (zone->uz_init != NULL) {
2957e20a199fSJeff Roberson 		if (zone->uz_init(item, zone->uz_size, flags) != 0) {
2958bb15d1c7SGleb Smirnoff 			zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT);
29590095a784SJeff Roberson 			goto fail;
2960b23f72e9SBrian Feldman 		}
2961b23f72e9SBrian Feldman 	}
2962c5deaf04SGleb Smirnoff 	if (zone->uz_ctor != NULL &&
2963c5deaf04SGleb Smirnoff #ifdef INVARIANTS
2964c5deaf04SGleb Smirnoff 	    (!skipdbg || zone->uz_ctor != trash_ctor ||
2965c5deaf04SGleb Smirnoff 	    zone->uz_dtor != trash_dtor) &&
2966c5deaf04SGleb Smirnoff #endif
2967c5deaf04SGleb Smirnoff 	    zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
2968bb15d1c7SGleb Smirnoff 		zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT);
29690095a784SJeff Roberson 		goto fail;
2970b23f72e9SBrian Feldman 	}
2971ef72505eSJeff Roberson #ifdef INVARIANTS
2972c5deaf04SGleb Smirnoff 	if (!skipdbg)
29730095a784SJeff Roberson 		uma_dbg_alloc(zone, NULL, item);
2974ef72505eSJeff Roberson #endif
29752cc35ff9SJeff Roberson 	if (flags & M_ZERO)
297648343a2fSGleb Smirnoff 		uma_zero_item(item, zone);
29778355f576SJeff Roberson 
29782efcc8cbSGleb Smirnoff 	counter_u64_add(zone->uz_allocs, 1);
29791431a748SGleb Smirnoff 	CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item,
29801431a748SGleb Smirnoff 	    zone->uz_name, zone);
29811431a748SGleb Smirnoff 
29828355f576SJeff Roberson 	return (item);
29830095a784SJeff Roberson 
29840095a784SJeff Roberson fail:
2985bb45b411SGleb Smirnoff 	if (zone->uz_max_items > 0) {
2986bb15d1c7SGleb Smirnoff 		ZONE_LOCK(zone);
2987bb15d1c7SGleb Smirnoff 		zone->uz_items--;
2988bb15d1c7SGleb Smirnoff 		ZONE_UNLOCK(zone);
2989bb45b411SGleb Smirnoff 	}
29902efcc8cbSGleb Smirnoff 	counter_u64_add(zone->uz_fails, 1);
29911431a748SGleb Smirnoff 	CTR2(KTR_UMA, "zone_alloc_item failed from %s(%p)",
29921431a748SGleb Smirnoff 	    zone->uz_name, zone);
29930095a784SJeff Roberson 	return (NULL);
29948355f576SJeff Roberson }
29958355f576SJeff Roberson 
29968355f576SJeff Roberson /* See uma.h */
29978355f576SJeff Roberson void
29988355f576SJeff Roberson uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
29998355f576SJeff Roberson {
30008355f576SJeff Roberson 	uma_cache_t cache;
30018355f576SJeff Roberson 	uma_bucket_t bucket;
3002ab3185d1SJeff Roberson 	uma_zone_domain_t zdom;
3003bb15d1c7SGleb Smirnoff 	int cpu, domain;
3004bb15d1c7SGleb Smirnoff 	bool lockfail;
3005c5deaf04SGleb Smirnoff #ifdef INVARIANTS
3006c5deaf04SGleb Smirnoff 	bool skipdbg;
3007c5deaf04SGleb Smirnoff #endif
30088355f576SJeff Roberson 
3009e866d8f0SMark Murray 	/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
301019fa89e9SMark Murray 	random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
301110cb2424SMark Murray 
30123659f747SRobert Watson 	CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread,
30133659f747SRobert Watson 	    zone->uz_name);
30143659f747SRobert Watson 
3015d9e2e68dSMark Johnston 	KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
30161067a2baSJonathan T. Looney 	    ("uma_zfree_arg: called with spinlock or critical section held"));
30171067a2baSJonathan T. Looney 
301820ed0cb0SMatthew D Fleming         /* uma_zfree(..., NULL) does nothing, to match free(9). */
301920ed0cb0SMatthew D Fleming         if (item == NULL)
302020ed0cb0SMatthew D Fleming                 return;
30218d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD
30228d689e04SGleb Smirnoff 	if (is_memguard_addr(item)) {
3023bc9d08e1SMark Johnston 		if (zone->uz_dtor != NULL)
30248d689e04SGleb Smirnoff 			zone->uz_dtor(item, zone->uz_size, udata);
3025bc9d08e1SMark Johnston 		if (zone->uz_fini != NULL)
30268d689e04SGleb Smirnoff 			zone->uz_fini(item, zone->uz_size);
30278d689e04SGleb Smirnoff 		memguard_free(item);
30288d689e04SGleb Smirnoff 		return;
30298d689e04SGleb Smirnoff 	}
30308d689e04SGleb Smirnoff #endif
30315d1ae027SRobert Watson #ifdef INVARIANTS
3032c5deaf04SGleb Smirnoff 	skipdbg = uma_dbg_zskip(zone, item);
3033c5deaf04SGleb Smirnoff 	if (skipdbg == false) {
3034e20a199fSJeff Roberson 		if (zone->uz_flags & UMA_ZONE_MALLOC)
30355d1ae027SRobert Watson 			uma_dbg_free(zone, udata, item);
30365d1ae027SRobert Watson 		else
30375d1ae027SRobert Watson 			uma_dbg_free(zone, NULL, item);
3038c5deaf04SGleb Smirnoff 	}
3039c5deaf04SGleb Smirnoff 	if (zone->uz_dtor != NULL && (!skipdbg ||
3040c5deaf04SGleb Smirnoff 	    zone->uz_dtor != trash_dtor || zone->uz_ctor != trash_ctor))
3041c5deaf04SGleb Smirnoff #else
3042fc03d22bSJeff Roberson 	if (zone->uz_dtor != NULL)
3043c5deaf04SGleb Smirnoff #endif
3044ef72505eSJeff Roberson 		zone->uz_dtor(item, zone->uz_size, udata);
3045ef72505eSJeff Roberson 
3046af7f9b97SJeff Roberson 	/*
3047af7f9b97SJeff Roberson 	 * The race here is acceptable.  If we miss it we'll just have to wait
3048af7f9b97SJeff Roberson 	 * a little longer for the limits to be reset.
3049af7f9b97SJeff Roberson 	 */
3050bb15d1c7SGleb Smirnoff 	if (zone->uz_sleepers > 0)
3051fc03d22bSJeff Roberson 		goto zfree_item;
3052af7f9b97SJeff Roberson 
30535d1ae027SRobert Watson 	/*
30545d1ae027SRobert Watson 	 * If possible, free to the per-CPU cache.  There are two
30555d1ae027SRobert Watson 	 * requirements for safe access to the per-CPU cache: (1) the thread
30565d1ae027SRobert Watson 	 * accessing the cache must not be preempted or yield during access,
30575d1ae027SRobert Watson 	 * and (2) the thread must not migrate CPUs without switching which
30585d1ae027SRobert Watson 	 * cache it accesses.  We rely on a critical section to prevent
30595d1ae027SRobert Watson 	 * preemption and migration.  We release the critical section in
30605d1ae027SRobert Watson 	 * order to acquire the zone mutex if we are unable to free to the
30615d1ae027SRobert Watson 	 * current cache; when we re-acquire the critical section, we must
30625d1ae027SRobert Watson 	 * detect and handle migration if it has occurred.
30635d1ae027SRobert Watson 	 */
3064a553d4b8SJeff Roberson zfree_restart:
30655d1ae027SRobert Watson 	critical_enter();
30665d1ae027SRobert Watson 	cpu = curcpu;
30678355f576SJeff Roberson 	cache = &zone->uz_cpu[cpu];
30688355f576SJeff Roberson 
30698355f576SJeff Roberson zfree_start:
3070a553d4b8SJeff Roberson 	/*
3071fc03d22bSJeff Roberson 	 * Try to free into the allocbucket first to give LIFO ordering
3072fc03d22bSJeff Roberson 	 * for cache-hot datastructures.  Spill over into the freebucket
3073fc03d22bSJeff Roberson 	 * if necessary.  Alloc will swap them if one runs dry.
3074a553d4b8SJeff Roberson 	 */
3075fc03d22bSJeff Roberson 	bucket = cache->uc_allocbucket;
3076fc03d22bSJeff Roberson 	if (bucket == NULL || bucket->ub_cnt >= bucket->ub_entries)
3077fc03d22bSJeff Roberson 		bucket = cache->uc_freebucket;
3078fc03d22bSJeff Roberson 	if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) {
3079cae33c14SJeff Roberson 		KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL,
30808355f576SJeff Roberson 		    ("uma_zfree: Freeing to non free bucket index."));
3081cae33c14SJeff Roberson 		bucket->ub_bucket[bucket->ub_cnt] = item;
3082cae33c14SJeff Roberson 		bucket->ub_cnt++;
3083773df9abSRobert Watson 		cache->uc_frees++;
30845d1ae027SRobert Watson 		critical_exit();
30858355f576SJeff Roberson 		return;
3086fc03d22bSJeff Roberson 	}
3087fc03d22bSJeff Roberson 
30888355f576SJeff Roberson 	/*
30895d1ae027SRobert Watson 	 * We must go back the zone, which requires acquiring the zone lock,
30905d1ae027SRobert Watson 	 * which in turn means we must release and re-acquire the critical
30915d1ae027SRobert Watson 	 * section.  Since the critical section is released, we may be
30925d1ae027SRobert Watson 	 * preempted or migrate.  As such, make sure not to maintain any
30935d1ae027SRobert Watson 	 * thread-local state specific to the cache from prior to releasing
30945d1ae027SRobert Watson 	 * the critical section.
30958355f576SJeff Roberson 	 */
30965d1ae027SRobert Watson 	critical_exit();
3097fc03d22bSJeff Roberson 	if (zone->uz_count == 0 || bucketdisable)
3098fc03d22bSJeff Roberson 		goto zfree_item;
3099fc03d22bSJeff Roberson 
3100bb15d1c7SGleb Smirnoff 	lockfail = false;
31014d104ba0SAlexander Motin 	if (ZONE_TRYLOCK(zone) == 0) {
31024d104ba0SAlexander Motin 		/* Record contention to size the buckets. */
31038355f576SJeff Roberson 		ZONE_LOCK(zone);
3104bb15d1c7SGleb Smirnoff 		lockfail = true;
31054d104ba0SAlexander Motin 	}
31065d1ae027SRobert Watson 	critical_enter();
31075d1ae027SRobert Watson 	cpu = curcpu;
31085d1ae027SRobert Watson 	cache = &zone->uz_cpu[cpu];
31098355f576SJeff Roberson 
31108355f576SJeff Roberson 	bucket = cache->uc_freebucket;
3111fc03d22bSJeff Roberson 	if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) {
3112fc03d22bSJeff Roberson 		ZONE_UNLOCK(zone);
3113fc03d22bSJeff Roberson 		goto zfree_start;
3114fc03d22bSJeff Roberson 	}
31158355f576SJeff Roberson 	cache->uc_freebucket = NULL;
3116afa5d703SMark Johnston 	/* We are no longer associated with this CPU. */
3117afa5d703SMark Johnston 	critical_exit();
31188355f576SJeff Roberson 
311930c5525bSAndrew Gallatin 	if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) {
3120ab3185d1SJeff Roberson 		domain = PCPU_GET(domain);
312130c5525bSAndrew Gallatin 		if (VM_DOMAIN_EMPTY(domain))
312230c5525bSAndrew Gallatin 			domain = UMA_ANYDOMAIN;
312330c5525bSAndrew Gallatin 	} else
3124ab3185d1SJeff Roberson 		domain = 0;
3125ab3185d1SJeff Roberson 	zdom = &zone->uz_domain[0];
3126ab3185d1SJeff Roberson 
31278355f576SJeff Roberson 	/* Can we throw this on the zone full list? */
31288355f576SJeff Roberson 	if (bucket != NULL) {
31291431a748SGleb Smirnoff 		CTR3(KTR_UMA,
31301431a748SGleb Smirnoff 		    "uma_zfree: zone %s(%p) putting bucket %p on free list",
31311431a748SGleb Smirnoff 		    zone->uz_name, zone, bucket);
3132cae33c14SJeff Roberson 		/* ub_cnt is pointing to the last free item */
3133bb15d1c7SGleb Smirnoff 		KASSERT(bucket->ub_cnt == bucket->ub_entries,
3134bb15d1c7SGleb Smirnoff 		    ("uma_zfree: Attempting to insert not full bucket onto the full list.\n"));
3135bb15d1c7SGleb Smirnoff 		if (zone->uz_bkt_count >= zone->uz_bkt_max) {
3136e8bb2dc7SJeff Roberson 			ZONE_UNLOCK(zone);
3137e8bb2dc7SJeff Roberson 			bucket_drain(zone, bucket);
3138e8bb2dc7SJeff Roberson 			bucket_free(zone, bucket, udata);
3139e8bb2dc7SJeff Roberson 			goto zfree_restart;
3140e8bb2dc7SJeff Roberson 		} else
31410f9b7bf3SMark Johnston 			zone_put_bucket(zone, zdom, bucket, true);
31428355f576SJeff Roberson 	}
3143fc03d22bSJeff Roberson 
31444d104ba0SAlexander Motin 	/*
31454d104ba0SAlexander Motin 	 * We bump the uz count when the cache size is insufficient to
31464d104ba0SAlexander Motin 	 * handle the working set.
31474d104ba0SAlexander Motin 	 */
3148bb15d1c7SGleb Smirnoff 	if (lockfail && zone->uz_count < zone->uz_count_max)
31494d104ba0SAlexander Motin 		zone->uz_count++;
3150a553d4b8SJeff Roberson 	ZONE_UNLOCK(zone);
3151a553d4b8SJeff Roberson 
31526fd34d6fSJeff Roberson 	bucket = bucket_alloc(zone, udata, M_NOWAIT);
31531431a748SGleb Smirnoff 	CTR3(KTR_UMA, "uma_zfree: zone %s(%p) allocated bucket %p",
31541431a748SGleb Smirnoff 	    zone->uz_name, zone, bucket);
31554741dcbfSJeff Roberson 	if (bucket) {
3156fc03d22bSJeff Roberson 		critical_enter();
3157fc03d22bSJeff Roberson 		cpu = curcpu;
3158fc03d22bSJeff Roberson 		cache = &zone->uz_cpu[cpu];
3159ab3185d1SJeff Roberson 		if (cache->uc_freebucket == NULL &&
3160ab3185d1SJeff Roberson 		    ((zone->uz_flags & UMA_ZONE_NUMA) == 0 ||
3161ab3185d1SJeff Roberson 		    domain == PCPU_GET(domain))) {
3162fc03d22bSJeff Roberson 			cache->uc_freebucket = bucket;
3163fc03d22bSJeff Roberson 			goto zfree_start;
3164fc03d22bSJeff Roberson 		}
3165fc03d22bSJeff Roberson 		/*
3166fc03d22bSJeff Roberson 		 * We lost the race, start over.  We have to drop our
3167fc03d22bSJeff Roberson 		 * critical section to free the bucket.
3168fc03d22bSJeff Roberson 		 */
3169fc03d22bSJeff Roberson 		critical_exit();
31706fd34d6fSJeff Roberson 		bucket_free(zone, bucket, udata);
3171a553d4b8SJeff Roberson 		goto zfree_restart;
31728355f576SJeff Roberson 	}
31738355f576SJeff Roberson 
3174a553d4b8SJeff Roberson 	/*
3175a553d4b8SJeff Roberson 	 * If nothing else caught this, we'll just do an internal free.
3176a553d4b8SJeff Roberson 	 */
3177fc03d22bSJeff Roberson zfree_item:
31780095a784SJeff Roberson 	zone_free_item(zone, item, udata, SKIP_DTOR);
31798355f576SJeff Roberson }
31808355f576SJeff Roberson 
3181ab3185d1SJeff Roberson void
3182ab3185d1SJeff Roberson uma_zfree_domain(uma_zone_t zone, void *item, void *udata)
3183ab3185d1SJeff Roberson {
3184ab3185d1SJeff Roberson 
3185ab3185d1SJeff Roberson 	/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
318619fa89e9SMark Murray 	random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
3187ab3185d1SJeff Roberson 
3188ab3185d1SJeff Roberson 	CTR2(KTR_UMA, "uma_zfree_domain thread %x zone %s", curthread,
3189ab3185d1SJeff Roberson 	    zone->uz_name);
3190ab3185d1SJeff Roberson 
3191ab3185d1SJeff Roberson 	KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
3192ab3185d1SJeff Roberson 	    ("uma_zfree_domain: called with spinlock or critical section held"));
3193ab3185d1SJeff Roberson 
3194ab3185d1SJeff Roberson         /* uma_zfree(..., NULL) does nothing, to match free(9). */
3195ab3185d1SJeff Roberson         if (item == NULL)
3196ab3185d1SJeff Roberson                 return;
3197ab3185d1SJeff Roberson 	zone_free_item(zone, item, udata, SKIP_NONE);
3198ab3185d1SJeff Roberson }
3199ab3185d1SJeff Roberson 
32008355f576SJeff Roberson static void
3201bb15d1c7SGleb Smirnoff slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item)
32028355f576SJeff Roberson {
3203bb15d1c7SGleb Smirnoff 	uma_keg_t keg;
3204ab3185d1SJeff Roberson 	uma_domain_t dom;
320585dcf349SGleb Smirnoff 	uint8_t freei;
3206099a0e58SBosko Milekic 
3207bb15d1c7SGleb Smirnoff 	keg = zone->uz_keg;
3208bb15d1c7SGleb Smirnoff 	MPASS(zone->uz_lockptr == &keg->uk_lock);
3209bb15d1c7SGleb Smirnoff 	KEG_LOCK_ASSERT(keg);
3210e20a199fSJeff Roberson 	MPASS(keg == slab->us_keg);
32118355f576SJeff Roberson 
3212ab3185d1SJeff Roberson 	dom = &keg->uk_domain[slab->us_domain];
3213ab3185d1SJeff Roberson 
32148355f576SJeff Roberson 	/* Do we need to remove from any lists? */
3215099a0e58SBosko Milekic 	if (slab->us_freecount+1 == keg->uk_ipers) {
32168355f576SJeff Roberson 		LIST_REMOVE(slab, us_link);
3217ab3185d1SJeff Roberson 		LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link);
32188355f576SJeff Roberson 	} else if (slab->us_freecount == 0) {
32198355f576SJeff Roberson 		LIST_REMOVE(slab, us_link);
3220ab3185d1SJeff Roberson 		LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);
32218355f576SJeff Roberson 	}
32228355f576SJeff Roberson 
3223ef72505eSJeff Roberson 	/* Slab management. */
3224ef72505eSJeff Roberson 	freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
3225ef72505eSJeff Roberson 	BIT_SET(SLAB_SETSIZE, freei, &slab->us_free);
32268355f576SJeff Roberson 	slab->us_freecount++;
32278355f576SJeff Roberson 
3228ef72505eSJeff Roberson 	/* Keg statistics. */
3229099a0e58SBosko Milekic 	keg->uk_free++;
32300095a784SJeff Roberson }
32310095a784SJeff Roberson 
32320095a784SJeff Roberson static void
32330095a784SJeff Roberson zone_release(uma_zone_t zone, void **bucket, int cnt)
32340095a784SJeff Roberson {
32350095a784SJeff Roberson 	void *item;
32360095a784SJeff Roberson 	uma_slab_t slab;
32370095a784SJeff Roberson 	uma_keg_t keg;
32380095a784SJeff Roberson 	uint8_t *mem;
32390095a784SJeff Roberson 	int i;
32408355f576SJeff Roberson 
3241bb15d1c7SGleb Smirnoff 	keg = zone->uz_keg;
3242af526374SJeff Roberson 	KEG_LOCK(keg);
32430095a784SJeff Roberson 	for (i = 0; i < cnt; i++) {
32440095a784SJeff Roberson 		item = bucket[i];
32450095a784SJeff Roberson 		if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
32460095a784SJeff Roberson 			mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
32470095a784SJeff Roberson 			if (zone->uz_flags & UMA_ZONE_HASH) {
32480095a784SJeff Roberson 				slab = hash_sfind(&keg->uk_hash, mem);
32490095a784SJeff Roberson 			} else {
32500095a784SJeff Roberson 				mem += keg->uk_pgoff;
32510095a784SJeff Roberson 				slab = (uma_slab_t)mem;
32520095a784SJeff Roberson 			}
32530095a784SJeff Roberson 		} else {
32540095a784SJeff Roberson 			slab = vtoslab((vm_offset_t)item);
3255bb15d1c7SGleb Smirnoff 			MPASS(slab->us_keg == keg);
32560095a784SJeff Roberson 		}
3257bb15d1c7SGleb Smirnoff 		slab_free_item(zone, slab, item);
32580095a784SJeff Roberson 	}
3259af526374SJeff Roberson 	KEG_UNLOCK(keg);
32608355f576SJeff Roberson }
32618355f576SJeff Roberson 
32620095a784SJeff Roberson /*
32630095a784SJeff Roberson  * Frees a single item to any zone.
32640095a784SJeff Roberson  *
32650095a784SJeff Roberson  * Arguments:
32660095a784SJeff Roberson  *	zone   The zone to free to
32670095a784SJeff Roberson  *	item   The item we're freeing
32680095a784SJeff Roberson  *	udata  User supplied data for the dtor
32690095a784SJeff Roberson  *	skip   Skip dtors and finis
32700095a784SJeff Roberson  */
32710095a784SJeff Roberson static void
32720095a784SJeff Roberson zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
32730095a784SJeff Roberson {
32740095a784SJeff Roberson #ifdef INVARIANTS
3275c5deaf04SGleb Smirnoff 	bool skipdbg;
3276c5deaf04SGleb Smirnoff 
3277c5deaf04SGleb Smirnoff 	skipdbg = uma_dbg_zskip(zone, item);
3278c5deaf04SGleb Smirnoff 	if (skip == SKIP_NONE && !skipdbg) {
32790095a784SJeff Roberson 		if (zone->uz_flags & UMA_ZONE_MALLOC)
32800095a784SJeff Roberson 			uma_dbg_free(zone, udata, item);
32810095a784SJeff Roberson 		else
32820095a784SJeff Roberson 			uma_dbg_free(zone, NULL, item);
32830095a784SJeff Roberson 	}
3284c5deaf04SGleb Smirnoff 
3285c5deaf04SGleb Smirnoff 	if (skip < SKIP_DTOR && zone->uz_dtor != NULL &&
3286c5deaf04SGleb Smirnoff 	    (!skipdbg || zone->uz_dtor != trash_dtor ||
3287c5deaf04SGleb Smirnoff 	    zone->uz_ctor != trash_ctor))
3288c5deaf04SGleb Smirnoff #else
3289c5deaf04SGleb Smirnoff 	if (skip < SKIP_DTOR && zone->uz_dtor != NULL)
32900095a784SJeff Roberson #endif
32910095a784SJeff Roberson 		zone->uz_dtor(item, zone->uz_size, udata);
32920095a784SJeff Roberson 
32930095a784SJeff Roberson 	if (skip < SKIP_FINI && zone->uz_fini)
32940095a784SJeff Roberson 		zone->uz_fini(item, zone->uz_size);
32950095a784SJeff Roberson 
32960095a784SJeff Roberson 	zone->uz_release(zone->uz_arg, &item, 1);
3297bb15d1c7SGleb Smirnoff 
3298bb15d1c7SGleb Smirnoff 	if (skip & SKIP_CNT)
3299bb15d1c7SGleb Smirnoff 		return;
3300bb15d1c7SGleb Smirnoff 
33012efcc8cbSGleb Smirnoff 	counter_u64_add(zone->uz_frees, 1);
33022efcc8cbSGleb Smirnoff 
3303bb45b411SGleb Smirnoff 	if (zone->uz_max_items > 0) {
3304bb15d1c7SGleb Smirnoff 		ZONE_LOCK(zone);
3305bb15d1c7SGleb Smirnoff 		zone->uz_items--;
3306bb45b411SGleb Smirnoff 		if (zone->uz_sleepers > 0 &&
3307bb45b411SGleb Smirnoff 		    zone->uz_items < zone->uz_max_items)
3308bb15d1c7SGleb Smirnoff 			wakeup_one(zone);
3309bb15d1c7SGleb Smirnoff 		ZONE_UNLOCK(zone);
33100095a784SJeff Roberson 	}
3311bb45b411SGleb Smirnoff }
33120095a784SJeff Roberson 
33138355f576SJeff Roberson /* See uma.h */
33141c6cae97SLawrence Stewart int
3315736ee590SJeff Roberson uma_zone_set_max(uma_zone_t zone, int nitems)
3316736ee590SJeff Roberson {
3317bb15d1c7SGleb Smirnoff 	struct uma_bucket_zone *ubz;
3318099a0e58SBosko Milekic 
3319bb15d1c7SGleb Smirnoff 	/*
3320bb15d1c7SGleb Smirnoff 	 * If limit is very low we may need to limit how
3321bb15d1c7SGleb Smirnoff 	 * much items are allowed in CPU caches.
3322bb15d1c7SGleb Smirnoff 	 */
3323bb15d1c7SGleb Smirnoff 	ubz = &bucket_zones[0];
3324bb15d1c7SGleb Smirnoff 	for (; ubz->ubz_entries != 0; ubz++)
3325bb15d1c7SGleb Smirnoff 		if (ubz->ubz_entries * 2 * mp_ncpus > nitems)
3326bb15d1c7SGleb Smirnoff 			break;
3327bb15d1c7SGleb Smirnoff 	if (ubz == &bucket_zones[0])
3328bb15d1c7SGleb Smirnoff 		nitems = ubz->ubz_entries * 2 * mp_ncpus;
3329bb15d1c7SGleb Smirnoff 	else
3330bb15d1c7SGleb Smirnoff 		ubz--;
3331bb15d1c7SGleb Smirnoff 
3332bb15d1c7SGleb Smirnoff 	ZONE_LOCK(zone);
3333bb15d1c7SGleb Smirnoff 	zone->uz_count_max = zone->uz_count = ubz->ubz_entries;
3334bb15d1c7SGleb Smirnoff 	if (zone->uz_count_min > zone->uz_count_max)
3335bb15d1c7SGleb Smirnoff 		zone->uz_count_min = zone->uz_count_max;
3336bb15d1c7SGleb Smirnoff 	zone->uz_max_items = nitems;
3337bb15d1c7SGleb Smirnoff 	ZONE_UNLOCK(zone);
3338bb15d1c7SGleb Smirnoff 
3339bb15d1c7SGleb Smirnoff 	return (nitems);
3340bb15d1c7SGleb Smirnoff }
3341bb15d1c7SGleb Smirnoff 
3342bb15d1c7SGleb Smirnoff /* See uma.h */
3343bb15d1c7SGleb Smirnoff int
3344bb15d1c7SGleb Smirnoff uma_zone_set_maxcache(uma_zone_t zone, int nitems)
3345bb15d1c7SGleb Smirnoff {
3346bb15d1c7SGleb Smirnoff 
3347bb15d1c7SGleb Smirnoff 	ZONE_LOCK(zone);
3348bb15d1c7SGleb Smirnoff 	zone->uz_bkt_max = nitems;
3349bb15d1c7SGleb Smirnoff 	ZONE_UNLOCK(zone);
33501c6cae97SLawrence Stewart 
33511c6cae97SLawrence Stewart 	return (nitems);
3352736ee590SJeff Roberson }
3353736ee590SJeff Roberson 
3354736ee590SJeff Roberson /* See uma.h */
3355e49471b0SAndre Oppermann int
3356e49471b0SAndre Oppermann uma_zone_get_max(uma_zone_t zone)
3357e49471b0SAndre Oppermann {
3358e49471b0SAndre Oppermann 	int nitems;
3359e49471b0SAndre Oppermann 
3360bb15d1c7SGleb Smirnoff 	ZONE_LOCK(zone);
3361bb15d1c7SGleb Smirnoff 	nitems = zone->uz_max_items;
3362bb15d1c7SGleb Smirnoff 	ZONE_UNLOCK(zone);
3363e49471b0SAndre Oppermann 
3364e49471b0SAndre Oppermann 	return (nitems);
3365e49471b0SAndre Oppermann }
3366e49471b0SAndre Oppermann 
3367e49471b0SAndre Oppermann /* See uma.h */
33682f891cd5SPawel Jakub Dawidek void
33692f891cd5SPawel Jakub Dawidek uma_zone_set_warning(uma_zone_t zone, const char *warning)
33702f891cd5SPawel Jakub Dawidek {
33712f891cd5SPawel Jakub Dawidek 
33722f891cd5SPawel Jakub Dawidek 	ZONE_LOCK(zone);
33732f891cd5SPawel Jakub Dawidek 	zone->uz_warning = warning;
33742f891cd5SPawel Jakub Dawidek 	ZONE_UNLOCK(zone);
33752f891cd5SPawel Jakub Dawidek }
33762f891cd5SPawel Jakub Dawidek 
33772f891cd5SPawel Jakub Dawidek /* See uma.h */
337854503a13SJonathan T. Looney void
337954503a13SJonathan T. Looney uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction)
338054503a13SJonathan T. Looney {
338154503a13SJonathan T. Looney 
338254503a13SJonathan T. Looney 	ZONE_LOCK(zone);
3383e60b2fcbSGleb Smirnoff 	TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone);
338454503a13SJonathan T. Looney 	ZONE_UNLOCK(zone);
338554503a13SJonathan T. Looney }
338654503a13SJonathan T. Looney 
338754503a13SJonathan T. Looney /* See uma.h */
3388c4ae7908SLawrence Stewart int
3389c4ae7908SLawrence Stewart uma_zone_get_cur(uma_zone_t zone)
3390c4ae7908SLawrence Stewart {
3391c4ae7908SLawrence Stewart 	int64_t nitems;
3392c4ae7908SLawrence Stewart 	u_int i;
3393c4ae7908SLawrence Stewart 
3394c4ae7908SLawrence Stewart 	ZONE_LOCK(zone);
33952efcc8cbSGleb Smirnoff 	nitems = counter_u64_fetch(zone->uz_allocs) -
33962efcc8cbSGleb Smirnoff 	    counter_u64_fetch(zone->uz_frees);
3397c4ae7908SLawrence Stewart 	CPU_FOREACH(i) {
3398c4ae7908SLawrence Stewart 		/*
33994a9f6ba7SGleb Smirnoff 		 * See the comment in uma_vm_zone_stats() regarding the
3400c4ae7908SLawrence Stewart 		 * safety of accessing the per-cpu caches. With the zone lock
3401c4ae7908SLawrence Stewart 		 * held, it is safe, but can potentially result in stale data.
3402c4ae7908SLawrence Stewart 		 */
3403c4ae7908SLawrence Stewart 		nitems += zone->uz_cpu[i].uc_allocs -
3404c4ae7908SLawrence Stewart 		    zone->uz_cpu[i].uc_frees;
3405c4ae7908SLawrence Stewart 	}
3406c4ae7908SLawrence Stewart 	ZONE_UNLOCK(zone);
3407c4ae7908SLawrence Stewart 
3408c4ae7908SLawrence Stewart 	return (nitems < 0 ? 0 : nitems);
3409c4ae7908SLawrence Stewart }
3410c4ae7908SLawrence Stewart 
3411c4ae7908SLawrence Stewart /* See uma.h */
3412736ee590SJeff Roberson void
3413099a0e58SBosko Milekic uma_zone_set_init(uma_zone_t zone, uma_init uminit)
3414099a0e58SBosko Milekic {
3415e20a199fSJeff Roberson 	uma_keg_t keg;
3416e20a199fSJeff Roberson 
3417bb15d1c7SGleb Smirnoff 	KEG_GET(zone, keg);
3418af526374SJeff Roberson 	KEG_LOCK(keg);
3419e20a199fSJeff Roberson 	KASSERT(keg->uk_pages == 0,
3420099a0e58SBosko Milekic 	    ("uma_zone_set_init on non-empty keg"));
3421e20a199fSJeff Roberson 	keg->uk_init = uminit;
3422af526374SJeff Roberson 	KEG_UNLOCK(keg);
3423099a0e58SBosko Milekic }
3424099a0e58SBosko Milekic 
3425099a0e58SBosko Milekic /* See uma.h */
3426099a0e58SBosko Milekic void
3427099a0e58SBosko Milekic uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
3428099a0e58SBosko Milekic {
3429e20a199fSJeff Roberson 	uma_keg_t keg;
3430e20a199fSJeff Roberson 
3431bb15d1c7SGleb Smirnoff 	KEG_GET(zone, keg);
3432af526374SJeff Roberson 	KEG_LOCK(keg);
3433e20a199fSJeff Roberson 	KASSERT(keg->uk_pages == 0,
3434099a0e58SBosko Milekic 	    ("uma_zone_set_fini on non-empty keg"));
3435e20a199fSJeff Roberson 	keg->uk_fini = fini;
3436af526374SJeff Roberson 	KEG_UNLOCK(keg);
3437099a0e58SBosko Milekic }
3438099a0e58SBosko Milekic 
3439099a0e58SBosko Milekic /* See uma.h */
3440099a0e58SBosko Milekic void
3441099a0e58SBosko Milekic uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
3442099a0e58SBosko Milekic {
3443af526374SJeff Roberson 
3444099a0e58SBosko Milekic 	ZONE_LOCK(zone);
3445bb15d1c7SGleb Smirnoff 	KASSERT(zone->uz_keg->uk_pages == 0,
3446099a0e58SBosko Milekic 	    ("uma_zone_set_zinit on non-empty keg"));
3447099a0e58SBosko Milekic 	zone->uz_init = zinit;
3448099a0e58SBosko Milekic 	ZONE_UNLOCK(zone);
3449099a0e58SBosko Milekic }
3450099a0e58SBosko Milekic 
3451099a0e58SBosko Milekic /* See uma.h */
3452099a0e58SBosko Milekic void
3453099a0e58SBosko Milekic uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
3454099a0e58SBosko Milekic {
3455af526374SJeff Roberson 
3456099a0e58SBosko Milekic 	ZONE_LOCK(zone);
3457bb15d1c7SGleb Smirnoff 	KASSERT(zone->uz_keg->uk_pages == 0,
3458099a0e58SBosko Milekic 	    ("uma_zone_set_zfini on non-empty keg"));
3459099a0e58SBosko Milekic 	zone->uz_fini = zfini;
3460099a0e58SBosko Milekic 	ZONE_UNLOCK(zone);
3461099a0e58SBosko Milekic }
3462099a0e58SBosko Milekic 
3463099a0e58SBosko Milekic /* See uma.h */
3464b23f72e9SBrian Feldman /* XXX uk_freef is not actually used with the zone locked */
3465099a0e58SBosko Milekic void
34668355f576SJeff Roberson uma_zone_set_freef(uma_zone_t zone, uma_free freef)
34678355f576SJeff Roberson {
34680095a784SJeff Roberson 	uma_keg_t keg;
3469e20a199fSJeff Roberson 
3470bb15d1c7SGleb Smirnoff 	KEG_GET(zone, keg);
34711d2c0c46SDmitry Chagin 	KASSERT(keg != NULL, ("uma_zone_set_freef: Invalid zone type"));
3472af526374SJeff Roberson 	KEG_LOCK(keg);
34730095a784SJeff Roberson 	keg->uk_freef = freef;
3474af526374SJeff Roberson 	KEG_UNLOCK(keg);
34758355f576SJeff Roberson }
34768355f576SJeff Roberson 
34778355f576SJeff Roberson /* See uma.h */
3478b23f72e9SBrian Feldman /* XXX uk_allocf is not actually used with the zone locked */
34798355f576SJeff Roberson void
34808355f576SJeff Roberson uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
34818355f576SJeff Roberson {
3482e20a199fSJeff Roberson 	uma_keg_t keg;
3483e20a199fSJeff Roberson 
3484bb15d1c7SGleb Smirnoff 	KEG_GET(zone, keg);
3485af526374SJeff Roberson 	KEG_LOCK(keg);
3486e20a199fSJeff Roberson 	keg->uk_allocf = allocf;
3487af526374SJeff Roberson 	KEG_UNLOCK(keg);
34888355f576SJeff Roberson }
34898355f576SJeff Roberson 
34908355f576SJeff Roberson /* See uma.h */
34916fd34d6fSJeff Roberson void
34926fd34d6fSJeff Roberson uma_zone_reserve(uma_zone_t zone, int items)
34936fd34d6fSJeff Roberson {
34946fd34d6fSJeff Roberson 	uma_keg_t keg;
34956fd34d6fSJeff Roberson 
3496bb15d1c7SGleb Smirnoff 	KEG_GET(zone, keg);
34976fd34d6fSJeff Roberson 	KEG_LOCK(keg);
34986fd34d6fSJeff Roberson 	keg->uk_reserve = items;
34996fd34d6fSJeff Roberson 	KEG_UNLOCK(keg);
35006fd34d6fSJeff Roberson }
35016fd34d6fSJeff Roberson 
35026fd34d6fSJeff Roberson /* See uma.h */
35038355f576SJeff Roberson int
3504a4915c21SAttilio Rao uma_zone_reserve_kva(uma_zone_t zone, int count)
35058355f576SJeff Roberson {
3506099a0e58SBosko Milekic 	uma_keg_t keg;
35078355f576SJeff Roberson 	vm_offset_t kva;
35089ba30bcbSZbigniew Bodek 	u_int pages;
35098355f576SJeff Roberson 
3510bb15d1c7SGleb Smirnoff 	KEG_GET(zone, keg);
35118355f576SJeff Roberson 
3512bb15d1c7SGleb Smirnoff 	pages = count / keg->uk_ipers;
3513099a0e58SBosko Milekic 	if (pages * keg->uk_ipers < count)
35148355f576SJeff Roberson 		pages++;
351557223e99SAndriy Gapon 	pages *= keg->uk_ppera;
3516a553d4b8SJeff Roberson 
3517a4915c21SAttilio Rao #ifdef UMA_MD_SMALL_ALLOC
3518a4915c21SAttilio Rao 	if (keg->uk_ppera > 1) {
3519a4915c21SAttilio Rao #else
3520a4915c21SAttilio Rao 	if (1) {
3521a4915c21SAttilio Rao #endif
352257223e99SAndriy Gapon 		kva = kva_alloc((vm_size_t)pages * PAGE_SIZE);
3523d1f42ac2SAlan Cox 		if (kva == 0)
35248355f576SJeff Roberson 			return (0);
3525a4915c21SAttilio Rao 	} else
3526a4915c21SAttilio Rao 		kva = 0;
3527bb15d1c7SGleb Smirnoff 
3528bb15d1c7SGleb Smirnoff 	ZONE_LOCK(zone);
3529bb15d1c7SGleb Smirnoff 	MPASS(keg->uk_kva == 0);
3530099a0e58SBosko Milekic 	keg->uk_kva = kva;
3531a4915c21SAttilio Rao 	keg->uk_offset = 0;
3532bb15d1c7SGleb Smirnoff 	zone->uz_max_items = pages * keg->uk_ipers;
3533a4915c21SAttilio Rao #ifdef UMA_MD_SMALL_ALLOC
3534a4915c21SAttilio Rao 	keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
3535a4915c21SAttilio Rao #else
3536a4915c21SAttilio Rao 	keg->uk_allocf = noobj_alloc;
3537a4915c21SAttilio Rao #endif
35386fd34d6fSJeff Roberson 	keg->uk_flags |= UMA_ZONE_NOFREE;
3539bb15d1c7SGleb Smirnoff 	ZONE_UNLOCK(zone);
3540af526374SJeff Roberson 
35418355f576SJeff Roberson 	return (1);
35428355f576SJeff Roberson }
35438355f576SJeff Roberson 
35448355f576SJeff Roberson /* See uma.h */
35458355f576SJeff Roberson void
35468355f576SJeff Roberson uma_prealloc(uma_zone_t zone, int items)
35478355f576SJeff Roberson {
3548920239efSMark Johnston 	struct vm_domainset_iter di;
3549ab3185d1SJeff Roberson 	uma_domain_t dom;
35508355f576SJeff Roberson 	uma_slab_t slab;
3551099a0e58SBosko Milekic 	uma_keg_t keg;
355286220393SMark Johnston 	int aflags, domain, slabs;
35538355f576SJeff Roberson 
3554bb15d1c7SGleb Smirnoff 	KEG_GET(zone, keg);
3555af526374SJeff Roberson 	KEG_LOCK(keg);
3556099a0e58SBosko Milekic 	slabs = items / keg->uk_ipers;
3557099a0e58SBosko Milekic 	if (slabs * keg->uk_ipers < items)
35588355f576SJeff Roberson 		slabs++;
3559194a979eSMark Johnston 	while (slabs-- > 0) {
356086220393SMark Johnston 		aflags = M_NOWAIT;
356186220393SMark Johnston 		vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain,
356286220393SMark Johnston 		    &aflags);
356386220393SMark Johnston 		for (;;) {
356486220393SMark Johnston 			slab = keg_alloc_slab(keg, zone, domain, M_WAITOK,
356586220393SMark Johnston 			    aflags);
356686220393SMark Johnston 			if (slab != NULL) {
3567e20a199fSJeff Roberson 				MPASS(slab->us_keg == keg);
3568ab3185d1SJeff Roberson 				dom = &keg->uk_domain[slab->us_domain];
356986220393SMark Johnston 				LIST_INSERT_HEAD(&dom->ud_free_slab, slab,
357086220393SMark Johnston 				    us_link);
3571920239efSMark Johnston 				break;
35728355f576SJeff Roberson 			}
357386220393SMark Johnston 			KEG_LOCK(keg);
357486220393SMark Johnston 			if (vm_domainset_iter_policy(&di, &domain) != 0) {
357586220393SMark Johnston 				KEG_UNLOCK(keg);
357686220393SMark Johnston 				vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask);
357786220393SMark Johnston 				KEG_LOCK(keg);
357886220393SMark Johnston 			}
357986220393SMark Johnston 		}
358086220393SMark Johnston 	}
3581af526374SJeff Roberson 	KEG_UNLOCK(keg);
35828355f576SJeff Roberson }
35838355f576SJeff Roberson 
35848355f576SJeff Roberson /* See uma.h */
358544ec2b63SKonstantin Belousov static void
358644ec2b63SKonstantin Belousov uma_reclaim_locked(bool kmem_danger)
35878355f576SJeff Roberson {
358844ec2b63SKonstantin Belousov 
35891431a748SGleb Smirnoff 	CTR0(KTR_UMA, "UMA: vm asked us to release pages!");
359044ec2b63SKonstantin Belousov 	sx_assert(&uma_drain_lock, SA_XLOCKED);
359186bbae32SJeff Roberson 	bucket_enable();
35928355f576SJeff Roberson 	zone_foreach(zone_drain);
359344ec2b63SKonstantin Belousov 	if (vm_page_count_min() || kmem_danger) {
3594a2de44abSAlexander Motin 		cache_drain_safe(NULL);
3595a2de44abSAlexander Motin 		zone_foreach(zone_drain);
3596a2de44abSAlexander Motin 	}
35970f9b7bf3SMark Johnston 
35988355f576SJeff Roberson 	/*
35998355f576SJeff Roberson 	 * Some slabs may have been freed but this zone will be visited early
36008355f576SJeff Roberson 	 * we visit again so that we can free pages that are empty once other
36018355f576SJeff Roberson 	 * zones are drained.  We have to do the same for buckets.
36028355f576SJeff Roberson 	 */
36039643769aSJeff Roberson 	zone_drain(slabzone);
3604cae33c14SJeff Roberson 	bucket_zone_drain();
360544ec2b63SKonstantin Belousov }
360644ec2b63SKonstantin Belousov 
360744ec2b63SKonstantin Belousov void
360844ec2b63SKonstantin Belousov uma_reclaim(void)
360944ec2b63SKonstantin Belousov {
361044ec2b63SKonstantin Belousov 
361144ec2b63SKonstantin Belousov 	sx_xlock(&uma_drain_lock);
361244ec2b63SKonstantin Belousov 	uma_reclaim_locked(false);
361395c4bf75SKonstantin Belousov 	sx_xunlock(&uma_drain_lock);
36148355f576SJeff Roberson }
36158355f576SJeff Roberson 
36162e47807cSJeff Roberson static volatile int uma_reclaim_needed;
361744ec2b63SKonstantin Belousov 
361844ec2b63SKonstantin Belousov void
361944ec2b63SKonstantin Belousov uma_reclaim_wakeup(void)
362044ec2b63SKonstantin Belousov {
362144ec2b63SKonstantin Belousov 
36222e47807cSJeff Roberson 	if (atomic_fetchadd_int(&uma_reclaim_needed, 1) == 0)
36232e47807cSJeff Roberson 		wakeup(uma_reclaim);
362444ec2b63SKonstantin Belousov }
362544ec2b63SKonstantin Belousov 
362644ec2b63SKonstantin Belousov void
362744ec2b63SKonstantin Belousov uma_reclaim_worker(void *arg __unused)
362844ec2b63SKonstantin Belousov {
362944ec2b63SKonstantin Belousov 
363044ec2b63SKonstantin Belousov 	for (;;) {
36312e47807cSJeff Roberson 		sx_xlock(&uma_drain_lock);
3632200f8117SKonstantin Belousov 		while (atomic_load_int(&uma_reclaim_needed) == 0)
36332e47807cSJeff Roberson 			sx_sleep(uma_reclaim, &uma_drain_lock, PVM, "umarcl",
36342e47807cSJeff Roberson 			    hz);
36359b43bc27SAndriy Gapon 		sx_xunlock(&uma_drain_lock);
36369b43bc27SAndriy Gapon 		EVENTHANDLER_INVOKE(vm_lowmem, VM_LOW_KMEM);
36379b43bc27SAndriy Gapon 		sx_xlock(&uma_drain_lock);
363844ec2b63SKonstantin Belousov 		uma_reclaim_locked(true);
3639200f8117SKonstantin Belousov 		atomic_store_int(&uma_reclaim_needed, 0);
36402e47807cSJeff Roberson 		sx_xunlock(&uma_drain_lock);
36412e47807cSJeff Roberson 		/* Don't fire more than once per-second. */
36422e47807cSJeff Roberson 		pause("umarclslp", hz);
364344ec2b63SKonstantin Belousov 	}
364444ec2b63SKonstantin Belousov }
364544ec2b63SKonstantin Belousov 
3646663b416fSJohn Baldwin /* See uma.h */
3647663b416fSJohn Baldwin int
3648663b416fSJohn Baldwin uma_zone_exhausted(uma_zone_t zone)
3649663b416fSJohn Baldwin {
3650663b416fSJohn Baldwin 	int full;
3651663b416fSJohn Baldwin 
3652663b416fSJohn Baldwin 	ZONE_LOCK(zone);
3653bb15d1c7SGleb Smirnoff 	full = zone->uz_sleepers > 0;
3654663b416fSJohn Baldwin 	ZONE_UNLOCK(zone);
3655663b416fSJohn Baldwin 	return (full);
3656663b416fSJohn Baldwin }
3657663b416fSJohn Baldwin 
36586c125b8dSMohan Srinivasan int
36596c125b8dSMohan Srinivasan uma_zone_exhausted_nolock(uma_zone_t zone)
36606c125b8dSMohan Srinivasan {
3661bb15d1c7SGleb Smirnoff 	return (zone->uz_sleepers > 0);
36626c125b8dSMohan Srinivasan }
36636c125b8dSMohan Srinivasan 
36648355f576SJeff Roberson void *
3665ab3185d1SJeff Roberson uma_large_malloc_domain(vm_size_t size, int domain, int wait)
36668355f576SJeff Roberson {
36679978bd99SMark Johnston 	struct domainset *policy;
3668ab3185d1SJeff Roberson 	vm_offset_t addr;
36698355f576SJeff Roberson 	uma_slab_t slab;
36708355f576SJeff Roberson 
367130c5525bSAndrew Gallatin 	if (domain != UMA_ANYDOMAIN) {
367230c5525bSAndrew Gallatin 		/* avoid allocs targeting empty domains */
367330c5525bSAndrew Gallatin 		if (VM_DOMAIN_EMPTY(domain))
367430c5525bSAndrew Gallatin 			domain = UMA_ANYDOMAIN;
367530c5525bSAndrew Gallatin 	}
3676ab3185d1SJeff Roberson 	slab = zone_alloc_item(slabzone, NULL, domain, wait);
36778355f576SJeff Roberson 	if (slab == NULL)
36788355f576SJeff Roberson 		return (NULL);
36799978bd99SMark Johnston 	policy = (domain == UMA_ANYDOMAIN) ? DOMAINSET_RR() :
36809978bd99SMark Johnston 	    DOMAINSET_FIXED(domain);
36819978bd99SMark Johnston 	addr = kmem_malloc_domainset(policy, size, wait);
3682ab3185d1SJeff Roberson 	if (addr != 0) {
3683ab3185d1SJeff Roberson 		vsetslab(addr, slab);
3684ab3185d1SJeff Roberson 		slab->us_data = (void *)addr;
3685ab3185d1SJeff Roberson 		slab->us_flags = UMA_SLAB_KERNEL | UMA_SLAB_MALLOC;
36868355f576SJeff Roberson 		slab->us_size = size;
3687e2068d0bSJeff Roberson 		slab->us_domain = vm_phys_domain(PHYS_TO_VM_PAGE(
3688ab3185d1SJeff Roberson 		    pmap_kextract(addr)));
36892e47807cSJeff Roberson 		uma_total_inc(size);
36908355f576SJeff Roberson 	} else {
36910095a784SJeff Roberson 		zone_free_item(slabzone, slab, NULL, SKIP_NONE);
36928355f576SJeff Roberson 	}
36938355f576SJeff Roberson 
3694ab3185d1SJeff Roberson 	return ((void *)addr);
3695ab3185d1SJeff Roberson }
3696ab3185d1SJeff Roberson 
3697ab3185d1SJeff Roberson void *
3698ab3185d1SJeff Roberson uma_large_malloc(vm_size_t size, int wait)
3699ab3185d1SJeff Roberson {
3700ab3185d1SJeff Roberson 
3701ab3185d1SJeff Roberson 	return uma_large_malloc_domain(size, UMA_ANYDOMAIN, wait);
37028355f576SJeff Roberson }
37038355f576SJeff Roberson 
37048355f576SJeff Roberson void
37058355f576SJeff Roberson uma_large_free(uma_slab_t slab)
37068355f576SJeff Roberson {
3707c325e866SKonstantin Belousov 
3708ab3185d1SJeff Roberson 	KASSERT((slab->us_flags & UMA_SLAB_KERNEL) != 0,
3709ab3185d1SJeff Roberson 	    ("uma_large_free:  Memory not allocated with uma_large_malloc."));
371049bfa624SAlan Cox 	kmem_free((vm_offset_t)slab->us_data, slab->us_size);
37112e47807cSJeff Roberson 	uma_total_dec(slab->us_size);
37120095a784SJeff Roberson 	zone_free_item(slabzone, slab, NULL, SKIP_NONE);
37138355f576SJeff Roberson }
37148355f576SJeff Roberson 
371548343a2fSGleb Smirnoff static void
371648343a2fSGleb Smirnoff uma_zero_item(void *item, uma_zone_t zone)
371748343a2fSGleb Smirnoff {
371848343a2fSGleb Smirnoff 
371948343a2fSGleb Smirnoff 	bzero(item, zone->uz_size);
372048343a2fSGleb Smirnoff }
372148343a2fSGleb Smirnoff 
37222e47807cSJeff Roberson unsigned long
37232e47807cSJeff Roberson uma_limit(void)
37242e47807cSJeff Roberson {
37252e47807cSJeff Roberson 
37262e47807cSJeff Roberson 	return (uma_kmem_limit);
37272e47807cSJeff Roberson }
37282e47807cSJeff Roberson 
37292e47807cSJeff Roberson void
37302e47807cSJeff Roberson uma_set_limit(unsigned long limit)
37312e47807cSJeff Roberson {
37322e47807cSJeff Roberson 
37332e47807cSJeff Roberson 	uma_kmem_limit = limit;
37342e47807cSJeff Roberson }
37352e47807cSJeff Roberson 
37362e47807cSJeff Roberson unsigned long
37372e47807cSJeff Roberson uma_size(void)
37382e47807cSJeff Roberson {
37392e47807cSJeff Roberson 
3740*058f0f74SMark Johnston 	return (atomic_load_long(&uma_kmem_total));
3741ad5b0f5bSJeff Roberson }
3742ad5b0f5bSJeff Roberson 
3743ad5b0f5bSJeff Roberson long
3744ad5b0f5bSJeff Roberson uma_avail(void)
3745ad5b0f5bSJeff Roberson {
3746ad5b0f5bSJeff Roberson 
3747*058f0f74SMark Johnston 	return (uma_kmem_limit - uma_size());
37482e47807cSJeff Roberson }
37492e47807cSJeff Roberson 
37508355f576SJeff Roberson void
37518355f576SJeff Roberson uma_print_stats(void)
37528355f576SJeff Roberson {
37538355f576SJeff Roberson 	zone_foreach(uma_print_zone);
37548355f576SJeff Roberson }
37558355f576SJeff Roberson 
3756504d5de3SJeff Roberson static void
3757504d5de3SJeff Roberson slab_print(uma_slab_t slab)
3758504d5de3SJeff Roberson {
3759ef72505eSJeff Roberson 	printf("slab: keg %p, data %p, freecount %d\n",
3760ef72505eSJeff Roberson 		slab->us_keg, slab->us_data, slab->us_freecount);
3761504d5de3SJeff Roberson }
3762504d5de3SJeff Roberson 
3763504d5de3SJeff Roberson static void
3764504d5de3SJeff Roberson cache_print(uma_cache_t cache)
3765504d5de3SJeff Roberson {
3766504d5de3SJeff Roberson 	printf("alloc: %p(%d), free: %p(%d)\n",
3767504d5de3SJeff Roberson 		cache->uc_allocbucket,
3768504d5de3SJeff Roberson 		cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
3769504d5de3SJeff Roberson 		cache->uc_freebucket,
3770504d5de3SJeff Roberson 		cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0);
3771504d5de3SJeff Roberson }
3772504d5de3SJeff Roberson 
3773e20a199fSJeff Roberson static void
3774e20a199fSJeff Roberson uma_print_keg(uma_keg_t keg)
37758355f576SJeff Roberson {
3776ab3185d1SJeff Roberson 	uma_domain_t dom;
3777504d5de3SJeff Roberson 	uma_slab_t slab;
3778ab3185d1SJeff Roberson 	int i;
3779504d5de3SJeff Roberson 
37800b80c1e4SEitan Adler 	printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
3781bb15d1c7SGleb Smirnoff 	    "out %d free %d\n",
3782e20a199fSJeff Roberson 	    keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
3783099a0e58SBosko Milekic 	    keg->uk_ipers, keg->uk_ppera,
378457223e99SAndriy Gapon 	    (keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free,
3785bb15d1c7SGleb Smirnoff 	    keg->uk_free);
3786ab3185d1SJeff Roberson 	for (i = 0; i < vm_ndomains; i++) {
3787ab3185d1SJeff Roberson 		dom = &keg->uk_domain[i];
3788504d5de3SJeff Roberson 		printf("Part slabs:\n");
3789ab3185d1SJeff Roberson 		LIST_FOREACH(slab, &dom->ud_part_slab, us_link)
3790504d5de3SJeff Roberson 			slab_print(slab);
3791504d5de3SJeff Roberson 		printf("Free slabs:\n");
3792ab3185d1SJeff Roberson 		LIST_FOREACH(slab, &dom->ud_free_slab, us_link)
3793504d5de3SJeff Roberson 			slab_print(slab);
3794504d5de3SJeff Roberson 		printf("Full slabs:\n");
3795ab3185d1SJeff Roberson 		LIST_FOREACH(slab, &dom->ud_full_slab, us_link)
3796504d5de3SJeff Roberson 			slab_print(slab);
3797e20a199fSJeff Roberson 	}
3798ab3185d1SJeff Roberson }
3799e20a199fSJeff Roberson 
3800e20a199fSJeff Roberson void
3801e20a199fSJeff Roberson uma_print_zone(uma_zone_t zone)
3802e20a199fSJeff Roberson {
3803e20a199fSJeff Roberson 	uma_cache_t cache;
3804e20a199fSJeff Roberson 	int i;
3805e20a199fSJeff Roberson 
38065a8eee2bSGleb Smirnoff 	printf("zone: %s(%p) size %d maxitems %ju flags %#x\n",
38075a8eee2bSGleb Smirnoff 	    zone->uz_name, zone, zone->uz_size, (uintmax_t)zone->uz_max_items,
3808bb15d1c7SGleb Smirnoff 	    zone->uz_flags);
3809bb15d1c7SGleb Smirnoff 	if (zone->uz_lockptr != &zone->uz_lock)
3810bb15d1c7SGleb Smirnoff 		uma_print_keg(zone->uz_keg);
38113aa6d94eSJohn Baldwin 	CPU_FOREACH(i) {
3812504d5de3SJeff Roberson 		cache = &zone->uz_cpu[i];
3813504d5de3SJeff Roberson 		printf("CPU %d Cache:\n", i);
3814504d5de3SJeff Roberson 		cache_print(cache);
3815504d5de3SJeff Roberson 	}
38168355f576SJeff Roberson }
38178355f576SJeff Roberson 
3818a0d4b0aeSRobert Watson #ifdef DDB
38198355f576SJeff Roberson /*
38207a52a97eSRobert Watson  * Generate statistics across both the zone and its per-cpu cache's.  Return
38217a52a97eSRobert Watson  * desired statistics if the pointer is non-NULL for that statistic.
38227a52a97eSRobert Watson  *
38237a52a97eSRobert Watson  * Note: does not update the zone statistics, as it can't safely clear the
38247a52a97eSRobert Watson  * per-CPU cache statistic.
38257a52a97eSRobert Watson  *
38267a52a97eSRobert Watson  * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't
38277a52a97eSRobert Watson  * safe from off-CPU; we should modify the caches to track this information
38287a52a97eSRobert Watson  * directly so that we don't have to.
38297a52a97eSRobert Watson  */
38307a52a97eSRobert Watson static void
38310f9b7bf3SMark Johnston uma_zone_sumstat(uma_zone_t z, long *cachefreep, uint64_t *allocsp,
383285dcf349SGleb Smirnoff     uint64_t *freesp, uint64_t *sleepsp)
38337a52a97eSRobert Watson {
38347a52a97eSRobert Watson 	uma_cache_t cache;
383585dcf349SGleb Smirnoff 	uint64_t allocs, frees, sleeps;
38367a52a97eSRobert Watson 	int cachefree, cpu;
38377a52a97eSRobert Watson 
3838bf965959SSean Bruno 	allocs = frees = sleeps = 0;
38397a52a97eSRobert Watson 	cachefree = 0;
38403aa6d94eSJohn Baldwin 	CPU_FOREACH(cpu) {
38417a52a97eSRobert Watson 		cache = &z->uz_cpu[cpu];
38427a52a97eSRobert Watson 		if (cache->uc_allocbucket != NULL)
38437a52a97eSRobert Watson 			cachefree += cache->uc_allocbucket->ub_cnt;
38447a52a97eSRobert Watson 		if (cache->uc_freebucket != NULL)
38457a52a97eSRobert Watson 			cachefree += cache->uc_freebucket->ub_cnt;
38467a52a97eSRobert Watson 		allocs += cache->uc_allocs;
38477a52a97eSRobert Watson 		frees += cache->uc_frees;
38487a52a97eSRobert Watson 	}
38492efcc8cbSGleb Smirnoff 	allocs += counter_u64_fetch(z->uz_allocs);
38502efcc8cbSGleb Smirnoff 	frees += counter_u64_fetch(z->uz_frees);
3851bf965959SSean Bruno 	sleeps += z->uz_sleeps;
38527a52a97eSRobert Watson 	if (cachefreep != NULL)
38537a52a97eSRobert Watson 		*cachefreep = cachefree;
38547a52a97eSRobert Watson 	if (allocsp != NULL)
38557a52a97eSRobert Watson 		*allocsp = allocs;
38567a52a97eSRobert Watson 	if (freesp != NULL)
38577a52a97eSRobert Watson 		*freesp = frees;
3858bf965959SSean Bruno 	if (sleepsp != NULL)
3859bf965959SSean Bruno 		*sleepsp = sleeps;
38607a52a97eSRobert Watson }
3861a0d4b0aeSRobert Watson #endif /* DDB */
38627a52a97eSRobert Watson 
38637a52a97eSRobert Watson static int
38647a52a97eSRobert Watson sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS)
38657a52a97eSRobert Watson {
38667a52a97eSRobert Watson 	uma_keg_t kz;
38677a52a97eSRobert Watson 	uma_zone_t z;
38687a52a97eSRobert Watson 	int count;
38697a52a97eSRobert Watson 
38707a52a97eSRobert Watson 	count = 0;
3871111fbcd5SBryan Venteicher 	rw_rlock(&uma_rwlock);
38727a52a97eSRobert Watson 	LIST_FOREACH(kz, &uma_kegs, uk_link) {
38737a52a97eSRobert Watson 		LIST_FOREACH(z, &kz->uk_zones, uz_link)
38747a52a97eSRobert Watson 			count++;
38757a52a97eSRobert Watson 	}
3876b47acb0aSGleb Smirnoff 	LIST_FOREACH(z, &uma_cachezones, uz_link)
3877b47acb0aSGleb Smirnoff 		count++;
3878b47acb0aSGleb Smirnoff 
3879111fbcd5SBryan Venteicher 	rw_runlock(&uma_rwlock);
38807a52a97eSRobert Watson 	return (sysctl_handle_int(oidp, &count, 0, req));
38817a52a97eSRobert Watson }
38827a52a97eSRobert Watson 
3883b47acb0aSGleb Smirnoff static void
3884b47acb0aSGleb Smirnoff uma_vm_zone_stats(struct uma_type_header *uth, uma_zone_t z, struct sbuf *sbuf,
3885b47acb0aSGleb Smirnoff     struct uma_percpu_stat *ups, bool internal)
3886b47acb0aSGleb Smirnoff {
3887b47acb0aSGleb Smirnoff 	uma_zone_domain_t zdom;
3888b47acb0aSGleb Smirnoff 	uma_cache_t cache;
3889b47acb0aSGleb Smirnoff 	int i;
3890b47acb0aSGleb Smirnoff 
3891b47acb0aSGleb Smirnoff 
3892b47acb0aSGleb Smirnoff 	for (i = 0; i < vm_ndomains; i++) {
3893b47acb0aSGleb Smirnoff 		zdom = &z->uz_domain[i];
3894b47acb0aSGleb Smirnoff 		uth->uth_zone_free += zdom->uzd_nitems;
3895b47acb0aSGleb Smirnoff 	}
3896b47acb0aSGleb Smirnoff 	uth->uth_allocs = counter_u64_fetch(z->uz_allocs);
3897b47acb0aSGleb Smirnoff 	uth->uth_frees = counter_u64_fetch(z->uz_frees);
3898b47acb0aSGleb Smirnoff 	uth->uth_fails = counter_u64_fetch(z->uz_fails);
3899b47acb0aSGleb Smirnoff 	uth->uth_sleeps = z->uz_sleeps;
3900b47acb0aSGleb Smirnoff 	/*
3901b47acb0aSGleb Smirnoff 	 * While it is not normally safe to access the cache
3902b47acb0aSGleb Smirnoff 	 * bucket pointers while not on the CPU that owns the
3903b47acb0aSGleb Smirnoff 	 * cache, we only allow the pointers to be exchanged
3904b47acb0aSGleb Smirnoff 	 * without the zone lock held, not invalidated, so
3905b47acb0aSGleb Smirnoff 	 * accept the possible race associated with bucket
3906b47acb0aSGleb Smirnoff 	 * exchange during monitoring.
3907b47acb0aSGleb Smirnoff 	 */
3908b47acb0aSGleb Smirnoff 	for (i = 0; i < mp_maxid + 1; i++) {
3909b47acb0aSGleb Smirnoff 		bzero(&ups[i], sizeof(*ups));
3910b47acb0aSGleb Smirnoff 		if (internal || CPU_ABSENT(i))
3911b47acb0aSGleb Smirnoff 			continue;
3912b47acb0aSGleb Smirnoff 		cache = &z->uz_cpu[i];
3913b47acb0aSGleb Smirnoff 		if (cache->uc_allocbucket != NULL)
3914b47acb0aSGleb Smirnoff 			ups[i].ups_cache_free +=
3915b47acb0aSGleb Smirnoff 			    cache->uc_allocbucket->ub_cnt;
3916b47acb0aSGleb Smirnoff 		if (cache->uc_freebucket != NULL)
3917b47acb0aSGleb Smirnoff 			ups[i].ups_cache_free +=
3918b47acb0aSGleb Smirnoff 			    cache->uc_freebucket->ub_cnt;
3919b47acb0aSGleb Smirnoff 		ups[i].ups_allocs = cache->uc_allocs;
3920b47acb0aSGleb Smirnoff 		ups[i].ups_frees = cache->uc_frees;
3921b47acb0aSGleb Smirnoff 	}
3922b47acb0aSGleb Smirnoff }
3923b47acb0aSGleb Smirnoff 
39247a52a97eSRobert Watson static int
39257a52a97eSRobert Watson sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
39267a52a97eSRobert Watson {
39277a52a97eSRobert Watson 	struct uma_stream_header ush;
39287a52a97eSRobert Watson 	struct uma_type_header uth;
392963b5d112SKonstantin Belousov 	struct uma_percpu_stat *ups;
39307a52a97eSRobert Watson 	struct sbuf sbuf;
39317a52a97eSRobert Watson 	uma_keg_t kz;
39327a52a97eSRobert Watson 	uma_zone_t z;
39334e657159SMatthew D Fleming 	int count, error, i;
39347a52a97eSRobert Watson 
393500f0e671SMatthew D Fleming 	error = sysctl_wire_old_buffer(req, 0);
393600f0e671SMatthew D Fleming 	if (error != 0)
393700f0e671SMatthew D Fleming 		return (error);
39384e657159SMatthew D Fleming 	sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
39391eafc078SIan Lepore 	sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL);
394063b5d112SKonstantin Belousov 	ups = malloc((mp_maxid + 1) * sizeof(*ups), M_TEMP, M_WAITOK);
39414e657159SMatthew D Fleming 
3942404a593eSMatthew D Fleming 	count = 0;
3943111fbcd5SBryan Venteicher 	rw_rlock(&uma_rwlock);
39447a52a97eSRobert Watson 	LIST_FOREACH(kz, &uma_kegs, uk_link) {
39457a52a97eSRobert Watson 		LIST_FOREACH(z, &kz->uk_zones, uz_link)
39467a52a97eSRobert Watson 			count++;
39477a52a97eSRobert Watson 	}
39487a52a97eSRobert Watson 
3949b47acb0aSGleb Smirnoff 	LIST_FOREACH(z, &uma_cachezones, uz_link)
3950b47acb0aSGleb Smirnoff 		count++;
3951b47acb0aSGleb Smirnoff 
39527a52a97eSRobert Watson 	/*
39537a52a97eSRobert Watson 	 * Insert stream header.
39547a52a97eSRobert Watson 	 */
39557a52a97eSRobert Watson 	bzero(&ush, sizeof(ush));
39567a52a97eSRobert Watson 	ush.ush_version = UMA_STREAM_VERSION;
3957ab3a57c0SRobert Watson 	ush.ush_maxcpus = (mp_maxid + 1);
39587a52a97eSRobert Watson 	ush.ush_count = count;
39594e657159SMatthew D Fleming 	(void)sbuf_bcat(&sbuf, &ush, sizeof(ush));
39607a52a97eSRobert Watson 
39617a52a97eSRobert Watson 	LIST_FOREACH(kz, &uma_kegs, uk_link) {
39627a52a97eSRobert Watson 		LIST_FOREACH(z, &kz->uk_zones, uz_link) {
39637a52a97eSRobert Watson 			bzero(&uth, sizeof(uth));
39647a52a97eSRobert Watson 			ZONE_LOCK(z);
3965cbbb4a00SRobert Watson 			strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME);
39667a52a97eSRobert Watson 			uth.uth_align = kz->uk_align;
39677a52a97eSRobert Watson 			uth.uth_size = kz->uk_size;
39687a52a97eSRobert Watson 			uth.uth_rsize = kz->uk_rsize;
3969bb45b411SGleb Smirnoff 			if (z->uz_max_items > 0)
3970bb45b411SGleb Smirnoff 				uth.uth_pages = (z->uz_items / kz->uk_ipers) *
3971bb15d1c7SGleb Smirnoff 					kz->uk_ppera;
3972bb45b411SGleb Smirnoff 			else
3973bb45b411SGleb Smirnoff 				uth.uth_pages = kz->uk_pages;
3974f8c86a5fSGleb Smirnoff 			uth.uth_maxpages = (z->uz_max_items / kz->uk_ipers) *
3975bb15d1c7SGleb Smirnoff 			    kz->uk_ppera;
3976bb15d1c7SGleb Smirnoff 			uth.uth_limit = z->uz_max_items;
3977f8c86a5fSGleb Smirnoff 			uth.uth_keg_free = z->uz_keg->uk_free;
3978cbbb4a00SRobert Watson 
3979cbbb4a00SRobert Watson 			/*
3980cbbb4a00SRobert Watson 			 * A zone is secondary is it is not the first entry
3981cbbb4a00SRobert Watson 			 * on the keg's zone list.
3982cbbb4a00SRobert Watson 			 */
3983e20a199fSJeff Roberson 			if ((z->uz_flags & UMA_ZONE_SECONDARY) &&
3984cbbb4a00SRobert Watson 			    (LIST_FIRST(&kz->uk_zones) != z))
3985cbbb4a00SRobert Watson 				uth.uth_zone_flags = UTH_ZONE_SECONDARY;
3986b47acb0aSGleb Smirnoff 			uma_vm_zone_stats(&uth, z, &sbuf, ups,
3987b47acb0aSGleb Smirnoff 			    kz->uk_flags & UMA_ZFLAG_INTERNAL);
39882450bbb8SRobert Watson 			ZONE_UNLOCK(z);
398963b5d112SKonstantin Belousov 			(void)sbuf_bcat(&sbuf, &uth, sizeof(uth));
399063b5d112SKonstantin Belousov 			for (i = 0; i < mp_maxid + 1; i++)
399163b5d112SKonstantin Belousov 				(void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i]));
39927a52a97eSRobert Watson 		}
39937a52a97eSRobert Watson 	}
3994b47acb0aSGleb Smirnoff 	LIST_FOREACH(z, &uma_cachezones, uz_link) {
3995b47acb0aSGleb Smirnoff 		bzero(&uth, sizeof(uth));
3996b47acb0aSGleb Smirnoff 		ZONE_LOCK(z);
3997b47acb0aSGleb Smirnoff 		strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME);
3998b47acb0aSGleb Smirnoff 		uth.uth_size = z->uz_size;
3999b47acb0aSGleb Smirnoff 		uma_vm_zone_stats(&uth, z, &sbuf, ups, false);
4000b47acb0aSGleb Smirnoff 		ZONE_UNLOCK(z);
4001b47acb0aSGleb Smirnoff 		(void)sbuf_bcat(&sbuf, &uth, sizeof(uth));
4002b47acb0aSGleb Smirnoff 		for (i = 0; i < mp_maxid + 1; i++)
4003b47acb0aSGleb Smirnoff 			(void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i]));
4004b47acb0aSGleb Smirnoff 	}
4005b47acb0aSGleb Smirnoff 
4006111fbcd5SBryan Venteicher 	rw_runlock(&uma_rwlock);
40074e657159SMatthew D Fleming 	error = sbuf_finish(&sbuf);
40084e657159SMatthew D Fleming 	sbuf_delete(&sbuf);
400963b5d112SKonstantin Belousov 	free(ups, M_TEMP);
40107a52a97eSRobert Watson 	return (error);
40117a52a97eSRobert Watson }
401248c5777eSRobert Watson 
40130a5a3ccbSGleb Smirnoff int
40140a5a3ccbSGleb Smirnoff sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS)
40150a5a3ccbSGleb Smirnoff {
40160a5a3ccbSGleb Smirnoff 	uma_zone_t zone = *(uma_zone_t *)arg1;
401716be9f54SGleb Smirnoff 	int error, max;
40180a5a3ccbSGleb Smirnoff 
401916be9f54SGleb Smirnoff 	max = uma_zone_get_max(zone);
40200a5a3ccbSGleb Smirnoff 	error = sysctl_handle_int(oidp, &max, 0, req);
40210a5a3ccbSGleb Smirnoff 	if (error || !req->newptr)
40220a5a3ccbSGleb Smirnoff 		return (error);
40230a5a3ccbSGleb Smirnoff 
40240a5a3ccbSGleb Smirnoff 	uma_zone_set_max(zone, max);
40250a5a3ccbSGleb Smirnoff 
40260a5a3ccbSGleb Smirnoff 	return (0);
40270a5a3ccbSGleb Smirnoff }
40280a5a3ccbSGleb Smirnoff 
40290a5a3ccbSGleb Smirnoff int
40300a5a3ccbSGleb Smirnoff sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS)
40310a5a3ccbSGleb Smirnoff {
40320a5a3ccbSGleb Smirnoff 	uma_zone_t zone = *(uma_zone_t *)arg1;
40330a5a3ccbSGleb Smirnoff 	int cur;
40340a5a3ccbSGleb Smirnoff 
40350a5a3ccbSGleb Smirnoff 	cur = uma_zone_get_cur(zone);
40360a5a3ccbSGleb Smirnoff 	return (sysctl_handle_int(oidp, &cur, 0, req));
40370a5a3ccbSGleb Smirnoff }
40380a5a3ccbSGleb Smirnoff 
40399542ea7bSGleb Smirnoff #ifdef INVARIANTS
40409542ea7bSGleb Smirnoff static uma_slab_t
40419542ea7bSGleb Smirnoff uma_dbg_getslab(uma_zone_t zone, void *item)
40429542ea7bSGleb Smirnoff {
40439542ea7bSGleb Smirnoff 	uma_slab_t slab;
40449542ea7bSGleb Smirnoff 	uma_keg_t keg;
40459542ea7bSGleb Smirnoff 	uint8_t *mem;
40469542ea7bSGleb Smirnoff 
40479542ea7bSGleb Smirnoff 	mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
40489542ea7bSGleb Smirnoff 	if (zone->uz_flags & UMA_ZONE_VTOSLAB) {
40499542ea7bSGleb Smirnoff 		slab = vtoslab((vm_offset_t)mem);
40509542ea7bSGleb Smirnoff 	} else {
40519542ea7bSGleb Smirnoff 		/*
40529542ea7bSGleb Smirnoff 		 * It is safe to return the slab here even though the
40539542ea7bSGleb Smirnoff 		 * zone is unlocked because the item's allocation state
40549542ea7bSGleb Smirnoff 		 * essentially holds a reference.
40559542ea7bSGleb Smirnoff 		 */
4056bb15d1c7SGleb Smirnoff 		if (zone->uz_lockptr == &zone->uz_lock)
4057bb15d1c7SGleb Smirnoff 			return (NULL);
40589542ea7bSGleb Smirnoff 		ZONE_LOCK(zone);
4059bb15d1c7SGleb Smirnoff 		keg = zone->uz_keg;
40609542ea7bSGleb Smirnoff 		if (keg->uk_flags & UMA_ZONE_HASH)
40619542ea7bSGleb Smirnoff 			slab = hash_sfind(&keg->uk_hash, mem);
40629542ea7bSGleb Smirnoff 		else
40639542ea7bSGleb Smirnoff 			slab = (uma_slab_t)(mem + keg->uk_pgoff);
40649542ea7bSGleb Smirnoff 		ZONE_UNLOCK(zone);
40659542ea7bSGleb Smirnoff 	}
40669542ea7bSGleb Smirnoff 
40679542ea7bSGleb Smirnoff 	return (slab);
40689542ea7bSGleb Smirnoff }
40699542ea7bSGleb Smirnoff 
4070c5deaf04SGleb Smirnoff static bool
4071c5deaf04SGleb Smirnoff uma_dbg_zskip(uma_zone_t zone, void *mem)
4072c5deaf04SGleb Smirnoff {
4073c5deaf04SGleb Smirnoff 
4074bb15d1c7SGleb Smirnoff 	if (zone->uz_lockptr == &zone->uz_lock)
4075c5deaf04SGleb Smirnoff 		return (true);
4076c5deaf04SGleb Smirnoff 
4077bb15d1c7SGleb Smirnoff 	return (uma_dbg_kskip(zone->uz_keg, mem));
4078c5deaf04SGleb Smirnoff }
4079c5deaf04SGleb Smirnoff 
4080c5deaf04SGleb Smirnoff static bool
4081c5deaf04SGleb Smirnoff uma_dbg_kskip(uma_keg_t keg, void *mem)
4082c5deaf04SGleb Smirnoff {
4083c5deaf04SGleb Smirnoff 	uintptr_t idx;
4084c5deaf04SGleb Smirnoff 
4085c5deaf04SGleb Smirnoff 	if (dbg_divisor == 0)
4086c5deaf04SGleb Smirnoff 		return (true);
4087c5deaf04SGleb Smirnoff 
4088c5deaf04SGleb Smirnoff 	if (dbg_divisor == 1)
4089c5deaf04SGleb Smirnoff 		return (false);
4090c5deaf04SGleb Smirnoff 
4091c5deaf04SGleb Smirnoff 	idx = (uintptr_t)mem >> PAGE_SHIFT;
4092c5deaf04SGleb Smirnoff 	if (keg->uk_ipers > 1) {
4093c5deaf04SGleb Smirnoff 		idx *= keg->uk_ipers;
4094c5deaf04SGleb Smirnoff 		idx += ((uintptr_t)mem & PAGE_MASK) / keg->uk_rsize;
4095c5deaf04SGleb Smirnoff 	}
4096c5deaf04SGleb Smirnoff 
4097c5deaf04SGleb Smirnoff 	if ((idx / dbg_divisor) * dbg_divisor != idx) {
4098c5deaf04SGleb Smirnoff 		counter_u64_add(uma_skip_cnt, 1);
4099c5deaf04SGleb Smirnoff 		return (true);
4100c5deaf04SGleb Smirnoff 	}
4101c5deaf04SGleb Smirnoff 	counter_u64_add(uma_dbg_cnt, 1);
4102c5deaf04SGleb Smirnoff 
4103c5deaf04SGleb Smirnoff 	return (false);
4104c5deaf04SGleb Smirnoff }
4105c5deaf04SGleb Smirnoff 
41069542ea7bSGleb Smirnoff /*
41079542ea7bSGleb Smirnoff  * Set up the slab's freei data such that uma_dbg_free can function.
41089542ea7bSGleb Smirnoff  *
41099542ea7bSGleb Smirnoff  */
41109542ea7bSGleb Smirnoff static void
41119542ea7bSGleb Smirnoff uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item)
41129542ea7bSGleb Smirnoff {
41139542ea7bSGleb Smirnoff 	uma_keg_t keg;
41149542ea7bSGleb Smirnoff 	int freei;
41159542ea7bSGleb Smirnoff 
41169542ea7bSGleb Smirnoff 	if (slab == NULL) {
41179542ea7bSGleb Smirnoff 		slab = uma_dbg_getslab(zone, item);
41189542ea7bSGleb Smirnoff 		if (slab == NULL)
41199542ea7bSGleb Smirnoff 			panic("uma: item %p did not belong to zone %s\n",
41209542ea7bSGleb Smirnoff 			    item, zone->uz_name);
41219542ea7bSGleb Smirnoff 	}
41229542ea7bSGleb Smirnoff 	keg = slab->us_keg;
41239542ea7bSGleb Smirnoff 	freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
41249542ea7bSGleb Smirnoff 
41259542ea7bSGleb Smirnoff 	if (BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree))
41269542ea7bSGleb Smirnoff 		panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)\n",
41279542ea7bSGleb Smirnoff 		    item, zone, zone->uz_name, slab, freei);
41289542ea7bSGleb Smirnoff 	BIT_SET_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree);
41299542ea7bSGleb Smirnoff 
41309542ea7bSGleb Smirnoff 	return;
41319542ea7bSGleb Smirnoff }
41329542ea7bSGleb Smirnoff 
41339542ea7bSGleb Smirnoff /*
41349542ea7bSGleb Smirnoff  * Verifies freed addresses.  Checks for alignment, valid slab membership
41359542ea7bSGleb Smirnoff  * and duplicate frees.
41369542ea7bSGleb Smirnoff  *
41379542ea7bSGleb Smirnoff  */
41389542ea7bSGleb Smirnoff static void
41399542ea7bSGleb Smirnoff uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item)
41409542ea7bSGleb Smirnoff {
41419542ea7bSGleb Smirnoff 	uma_keg_t keg;
41429542ea7bSGleb Smirnoff 	int freei;
41439542ea7bSGleb Smirnoff 
41449542ea7bSGleb Smirnoff 	if (slab == NULL) {
41459542ea7bSGleb Smirnoff 		slab = uma_dbg_getslab(zone, item);
41469542ea7bSGleb Smirnoff 		if (slab == NULL)
41479542ea7bSGleb Smirnoff 			panic("uma: Freed item %p did not belong to zone %s\n",
41489542ea7bSGleb Smirnoff 			    item, zone->uz_name);
41499542ea7bSGleb Smirnoff 	}
41509542ea7bSGleb Smirnoff 	keg = slab->us_keg;
41519542ea7bSGleb Smirnoff 	freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
41529542ea7bSGleb Smirnoff 
41539542ea7bSGleb Smirnoff 	if (freei >= keg->uk_ipers)
41549542ea7bSGleb Smirnoff 		panic("Invalid free of %p from zone %p(%s) slab %p(%d)\n",
41559542ea7bSGleb Smirnoff 		    item, zone, zone->uz_name, slab, freei);
41569542ea7bSGleb Smirnoff 
41579542ea7bSGleb Smirnoff 	if (((freei * keg->uk_rsize) + slab->us_data) != item)
41589542ea7bSGleb Smirnoff 		panic("Unaligned free of %p from zone %p(%s) slab %p(%d)\n",
41599542ea7bSGleb Smirnoff 		    item, zone, zone->uz_name, slab, freei);
41609542ea7bSGleb Smirnoff 
41619542ea7bSGleb Smirnoff 	if (!BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree))
41629542ea7bSGleb Smirnoff 		panic("Duplicate free of %p from zone %p(%s) slab %p(%d)\n",
41639542ea7bSGleb Smirnoff 		    item, zone, zone->uz_name, slab, freei);
41649542ea7bSGleb Smirnoff 
41659542ea7bSGleb Smirnoff 	BIT_CLR_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree);
41669542ea7bSGleb Smirnoff }
41679542ea7bSGleb Smirnoff #endif /* INVARIANTS */
41689542ea7bSGleb Smirnoff 
416948c5777eSRobert Watson #ifdef DDB
417048c5777eSRobert Watson DB_SHOW_COMMAND(uma, db_show_uma)
417148c5777eSRobert Watson {
417248c5777eSRobert Watson 	uma_keg_t kz;
417348c5777eSRobert Watson 	uma_zone_t z;
4174ab3185d1SJeff Roberson 	uint64_t allocs, frees, sleeps;
41750f9b7bf3SMark Johnston 	long cachefree;
41760f9b7bf3SMark Johnston 	int i;
417748c5777eSRobert Watson 
417803175483SAlexander Motin 	db_printf("%18s %8s %8s %8s %12s %8s %8s\n", "Zone", "Size", "Used",
417903175483SAlexander Motin 	    "Free", "Requests", "Sleeps", "Bucket");
418048c5777eSRobert Watson 	LIST_FOREACH(kz, &uma_kegs, uk_link) {
418148c5777eSRobert Watson 		LIST_FOREACH(z, &kz->uk_zones, uz_link) {
418248c5777eSRobert Watson 			if (kz->uk_flags & UMA_ZFLAG_INTERNAL) {
41832efcc8cbSGleb Smirnoff 				allocs = counter_u64_fetch(z->uz_allocs);
41842efcc8cbSGleb Smirnoff 				frees = counter_u64_fetch(z->uz_frees);
4185bf965959SSean Bruno 				sleeps = z->uz_sleeps;
418648c5777eSRobert Watson 				cachefree = 0;
418748c5777eSRobert Watson 			} else
418848c5777eSRobert Watson 				uma_zone_sumstat(z, &cachefree, &allocs,
4189bf965959SSean Bruno 				    &frees, &sleeps);
4190e20a199fSJeff Roberson 			if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
419148c5777eSRobert Watson 			    (LIST_FIRST(&kz->uk_zones) != z)))
419248c5777eSRobert Watson 				cachefree += kz->uk_free;
41930f9b7bf3SMark Johnston 			for (i = 0; i < vm_ndomains; i++)
41940f9b7bf3SMark Johnston 				cachefree += z->uz_domain[i].uzd_nitems;
41950f9b7bf3SMark Johnston 
41960f9b7bf3SMark Johnston 			db_printf("%18s %8ju %8jd %8ld %12ju %8ju %8u\n",
419703175483SAlexander Motin 			    z->uz_name, (uintmax_t)kz->uk_size,
4198ae4e9636SRobert Watson 			    (intmax_t)(allocs - frees), cachefree,
419903175483SAlexander Motin 			    (uintmax_t)allocs, sleeps, z->uz_count);
4200687c94aaSJohn Baldwin 			if (db_pager_quit)
4201687c94aaSJohn Baldwin 				return;
420248c5777eSRobert Watson 		}
420348c5777eSRobert Watson 	}
420448c5777eSRobert Watson }
420503175483SAlexander Motin 
420603175483SAlexander Motin DB_SHOW_COMMAND(umacache, db_show_umacache)
420703175483SAlexander Motin {
420803175483SAlexander Motin 	uma_zone_t z;
4209ab3185d1SJeff Roberson 	uint64_t allocs, frees;
42100f9b7bf3SMark Johnston 	long cachefree;
42110f9b7bf3SMark Johnston 	int i;
421203175483SAlexander Motin 
421303175483SAlexander Motin 	db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",
421403175483SAlexander Motin 	    "Requests", "Bucket");
421503175483SAlexander Motin 	LIST_FOREACH(z, &uma_cachezones, uz_link) {
421603175483SAlexander Motin 		uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL);
42170f9b7bf3SMark Johnston 		for (i = 0; i < vm_ndomains; i++)
42180f9b7bf3SMark Johnston 			cachefree += z->uz_domain[i].uzd_nitems;
42190f9b7bf3SMark Johnston 		db_printf("%18s %8ju %8jd %8ld %12ju %8u\n",
422003175483SAlexander Motin 		    z->uz_name, (uintmax_t)z->uz_size,
422103175483SAlexander Motin 		    (intmax_t)(allocs - frees), cachefree,
422203175483SAlexander Motin 		    (uintmax_t)allocs, z->uz_count);
422303175483SAlexander Motin 		if (db_pager_quit)
422403175483SAlexander Motin 			return;
422503175483SAlexander Motin 	}
422603175483SAlexander Motin }
42279542ea7bSGleb Smirnoff #endif	/* DDB */
4228