160727d8bSWarner Losh /*- 2fe267a55SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3fe267a55SPedro F. Giffuni * 4ef72505eSJeff Roberson * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org> 508ecce74SRobert Watson * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 6ae4e9636SRobert Watson * Copyright (c) 2004-2006 Robert N. M. Watson 708ecce74SRobert Watson * All rights reserved. 88355f576SJeff Roberson * 98355f576SJeff Roberson * Redistribution and use in source and binary forms, with or without 108355f576SJeff Roberson * modification, are permitted provided that the following conditions 118355f576SJeff Roberson * are met: 128355f576SJeff Roberson * 1. Redistributions of source code must retain the above copyright 138355f576SJeff Roberson * notice unmodified, this list of conditions, and the following 148355f576SJeff Roberson * disclaimer. 158355f576SJeff Roberson * 2. Redistributions in binary form must reproduce the above copyright 168355f576SJeff Roberson * notice, this list of conditions and the following disclaimer in the 178355f576SJeff Roberson * documentation and/or other materials provided with the distribution. 188355f576SJeff Roberson * 198355f576SJeff Roberson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 208355f576SJeff Roberson * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 218355f576SJeff Roberson * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 228355f576SJeff Roberson * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 238355f576SJeff Roberson * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 248355f576SJeff Roberson * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 258355f576SJeff Roberson * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 268355f576SJeff Roberson * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 278355f576SJeff Roberson * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 288355f576SJeff Roberson * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 298355f576SJeff Roberson */ 308355f576SJeff Roberson 318355f576SJeff Roberson /* 328355f576SJeff Roberson * uma_core.c Implementation of the Universal Memory allocator 338355f576SJeff Roberson * 348355f576SJeff Roberson * This allocator is intended to replace the multitude of similar object caches 358355f576SJeff Roberson * in the standard FreeBSD kernel. The intent is to be flexible as well as 36763df3ecSPedro F. Giffuni * efficient. A primary design goal is to return unused memory to the rest of 378355f576SJeff Roberson * the system. This will make the system as a whole more flexible due to the 388355f576SJeff Roberson * ability to move memory to subsystems which most need it instead of leaving 398355f576SJeff Roberson * pools of reserved memory unused. 408355f576SJeff Roberson * 418355f576SJeff Roberson * The basic ideas stem from similar slab/zone based allocators whose algorithms 428355f576SJeff Roberson * are well known. 438355f576SJeff Roberson * 448355f576SJeff Roberson */ 458355f576SJeff Roberson 468355f576SJeff Roberson /* 478355f576SJeff Roberson * TODO: 488355f576SJeff Roberson * - Improve memory usage for large allocations 498355f576SJeff Roberson * - Investigate cache size adjustments 508355f576SJeff Roberson */ 518355f576SJeff Roberson 52874651b1SDavid E. O'Brien #include <sys/cdefs.h> 53874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$"); 54874651b1SDavid E. O'Brien 5548c5777eSRobert Watson #include "opt_ddb.h" 568355f576SJeff Roberson #include "opt_param.h" 578d689e04SGleb Smirnoff #include "opt_vm.h" 5848c5777eSRobert Watson 598355f576SJeff Roberson #include <sys/param.h> 608355f576SJeff Roberson #include <sys/systm.h> 61ef72505eSJeff Roberson #include <sys/bitset.h> 62194a979eSMark Johnston #include <sys/domainset.h> 639b43bc27SAndriy Gapon #include <sys/eventhandler.h> 648355f576SJeff Roberson #include <sys/kernel.h> 658355f576SJeff Roberson #include <sys/types.h> 66ad5b0f5bSJeff Roberson #include <sys/limits.h> 678355f576SJeff Roberson #include <sys/queue.h> 688355f576SJeff Roberson #include <sys/malloc.h> 693659f747SRobert Watson #include <sys/ktr.h> 708355f576SJeff Roberson #include <sys/lock.h> 718355f576SJeff Roberson #include <sys/sysctl.h> 728355f576SJeff Roberson #include <sys/mutex.h> 734c1cc01cSJohn Baldwin #include <sys/proc.h> 7410cb2424SMark Murray #include <sys/random.h> 7589f6b863SAttilio Rao #include <sys/rwlock.h> 767a52a97eSRobert Watson #include <sys/sbuf.h> 77a2de44abSAlexander Motin #include <sys/sched.h> 788355f576SJeff Roberson #include <sys/smp.h> 79e60b2fcbSGleb Smirnoff #include <sys/taskqueue.h> 8086bbae32SJeff Roberson #include <sys/vmmeter.h> 8186bbae32SJeff Roberson 828355f576SJeff Roberson #include <vm/vm.h> 83194a979eSMark Johnston #include <vm/vm_domainset.h> 848355f576SJeff Roberson #include <vm/vm_object.h> 858355f576SJeff Roberson #include <vm/vm_page.h> 86a4915c21SAttilio Rao #include <vm/vm_pageout.h> 878355f576SJeff Roberson #include <vm/vm_param.h> 88ab3185d1SJeff Roberson #include <vm/vm_phys.h> 8930c5525bSAndrew Gallatin #include <vm/vm_pagequeue.h> 908355f576SJeff Roberson #include <vm/vm_map.h> 918355f576SJeff Roberson #include <vm/vm_kern.h> 928355f576SJeff Roberson #include <vm/vm_extern.h> 938355f576SJeff Roberson #include <vm/uma.h> 948355f576SJeff Roberson #include <vm/uma_int.h> 95639c9550SJeff Roberson #include <vm/uma_dbg.h> 968355f576SJeff Roberson 9748c5777eSRobert Watson #include <ddb/ddb.h> 9848c5777eSRobert Watson 998d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD 1008d689e04SGleb Smirnoff #include <vm/memguard.h> 1018d689e04SGleb Smirnoff #endif 1028d689e04SGleb Smirnoff 1038355f576SJeff Roberson /* 104ab3185d1SJeff Roberson * This is the zone and keg from which all zones are spawned. 1058355f576SJeff Roberson */ 106ab3185d1SJeff Roberson static uma_zone_t kegs; 107ab3185d1SJeff Roberson static uma_zone_t zones; 1088355f576SJeff Roberson 109ab3185d1SJeff Roberson /* This is the zone from which all offpage uma_slab_ts are allocated. */ 1108355f576SJeff Roberson static uma_zone_t slabzone; 1118355f576SJeff Roberson 1128355f576SJeff Roberson /* 1138355f576SJeff Roberson * The initial hash tables come out of this zone so they can be allocated 1148355f576SJeff Roberson * prior to malloc coming up. 1158355f576SJeff Roberson */ 1168355f576SJeff Roberson static uma_zone_t hashzone; 1178355f576SJeff Roberson 1181e319f6dSRobert Watson /* The boot-time adjusted value for cache line alignment. */ 119e4cd31ddSJeff Roberson int uma_align_cache = 64 - 1; 1201e319f6dSRobert Watson 121961647dfSJeff Roberson static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets"); 122961647dfSJeff Roberson 1238355f576SJeff Roberson /* 12486bbae32SJeff Roberson * Are we allowed to allocate buckets? 12586bbae32SJeff Roberson */ 12686bbae32SJeff Roberson static int bucketdisable = 1; 12786bbae32SJeff Roberson 128099a0e58SBosko Milekic /* Linked list of all kegs in the system */ 12913e403fdSAntoine Brodin static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs); 1308355f576SJeff Roberson 13103175483SAlexander Motin /* Linked list of all cache-only zones in the system */ 13203175483SAlexander Motin static LIST_HEAD(,uma_zone) uma_cachezones = 13303175483SAlexander Motin LIST_HEAD_INITIALIZER(uma_cachezones); 13403175483SAlexander Motin 135111fbcd5SBryan Venteicher /* This RW lock protects the keg list */ 136fe933c1dSMateusz Guzik static struct rwlock_padalign __exclusive_cache_line uma_rwlock; 1378355f576SJeff Roberson 138ac0a6fd0SGleb Smirnoff /* 139ac0a6fd0SGleb Smirnoff * Pointer and counter to pool of pages, that is preallocated at 140f7d35785SGleb Smirnoff * startup to bootstrap UMA. 141ac0a6fd0SGleb Smirnoff */ 142ac0a6fd0SGleb Smirnoff static char *bootmem; 143ac0a6fd0SGleb Smirnoff static int boot_pages; 1448355f576SJeff Roberson 14595c4bf75SKonstantin Belousov static struct sx uma_drain_lock; 14695c4bf75SKonstantin Belousov 1472e47807cSJeff Roberson /* kmem soft limit. */ 148ad5b0f5bSJeff Roberson static unsigned long uma_kmem_limit = LONG_MAX; 149*058f0f74SMark Johnston static unsigned long uma_kmem_total; 1502e47807cSJeff Roberson 1518355f576SJeff Roberson /* Is the VM done starting up? */ 152f4bef67cSGleb Smirnoff static enum { BOOT_COLD = 0, BOOT_STRAPPED, BOOT_PAGEALLOC, BOOT_BUCKETS, 153f4bef67cSGleb Smirnoff BOOT_RUNNING } booted = BOOT_COLD; 1548355f576SJeff Roberson 155ef72505eSJeff Roberson /* 1569643769aSJeff Roberson * This is the handle used to schedule events that need to happen 1579643769aSJeff Roberson * outside of the allocation fast path. 1589643769aSJeff Roberson */ 1598355f576SJeff Roberson static struct callout uma_callout; 1609643769aSJeff Roberson #define UMA_TIMEOUT 20 /* Seconds for callout interval. */ 1618355f576SJeff Roberson 1628355f576SJeff Roberson /* 1638355f576SJeff Roberson * This structure is passed as the zone ctor arg so that I don't have to create 1648355f576SJeff Roberson * a special allocation function just for zones. 1658355f576SJeff Roberson */ 1668355f576SJeff Roberson struct uma_zctor_args { 167bb196eb4SMatthew D Fleming const char *name; 168c3bdc05fSAndrew R. Reiter size_t size; 1698355f576SJeff Roberson uma_ctor ctor; 1708355f576SJeff Roberson uma_dtor dtor; 1718355f576SJeff Roberson uma_init uminit; 1728355f576SJeff Roberson uma_fini fini; 1730095a784SJeff Roberson uma_import import; 1740095a784SJeff Roberson uma_release release; 1750095a784SJeff Roberson void *arg; 176099a0e58SBosko Milekic uma_keg_t keg; 177099a0e58SBosko Milekic int align; 17885dcf349SGleb Smirnoff uint32_t flags; 179099a0e58SBosko Milekic }; 180099a0e58SBosko Milekic 181099a0e58SBosko Milekic struct uma_kctor_args { 182099a0e58SBosko Milekic uma_zone_t zone; 183099a0e58SBosko Milekic size_t size; 184099a0e58SBosko Milekic uma_init uminit; 185099a0e58SBosko Milekic uma_fini fini; 1868355f576SJeff Roberson int align; 18785dcf349SGleb Smirnoff uint32_t flags; 1888355f576SJeff Roberson }; 1898355f576SJeff Roberson 190cae33c14SJeff Roberson struct uma_bucket_zone { 191cae33c14SJeff Roberson uma_zone_t ubz_zone; 192cae33c14SJeff Roberson char *ubz_name; 193fc03d22bSJeff Roberson int ubz_entries; /* Number of items it can hold. */ 194fc03d22bSJeff Roberson int ubz_maxsize; /* Maximum allocation size per-item. */ 195cae33c14SJeff Roberson }; 196cae33c14SJeff Roberson 197f9d27e75SRobert Watson /* 198fc03d22bSJeff Roberson * Compute the actual number of bucket entries to pack them in power 199fc03d22bSJeff Roberson * of two sizes for more efficient space utilization. 200f9d27e75SRobert Watson */ 201fc03d22bSJeff Roberson #define BUCKET_SIZE(n) \ 202fc03d22bSJeff Roberson (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *)) 203fc03d22bSJeff Roberson 2041aa6c758SAlexander Motin #define BUCKET_MAX BUCKET_SIZE(256) 205fc03d22bSJeff Roberson 206fc03d22bSJeff Roberson struct uma_bucket_zone bucket_zones[] = { 2076fd34d6fSJeff Roberson { NULL, "4 Bucket", BUCKET_SIZE(4), 4096 }, 208f3932e90SAlexander Motin { NULL, "6 Bucket", BUCKET_SIZE(6), 3072 }, 2096fd34d6fSJeff Roberson { NULL, "8 Bucket", BUCKET_SIZE(8), 2048 }, 210f3932e90SAlexander Motin { NULL, "12 Bucket", BUCKET_SIZE(12), 1536 }, 2116fd34d6fSJeff Roberson { NULL, "16 Bucket", BUCKET_SIZE(16), 1024 }, 212fc03d22bSJeff Roberson { NULL, "32 Bucket", BUCKET_SIZE(32), 512 }, 213fc03d22bSJeff Roberson { NULL, "64 Bucket", BUCKET_SIZE(64), 256 }, 214fc03d22bSJeff Roberson { NULL, "128 Bucket", BUCKET_SIZE(128), 128 }, 2151aa6c758SAlexander Motin { NULL, "256 Bucket", BUCKET_SIZE(256), 64 }, 216fc03d22bSJeff Roberson { NULL, NULL, 0} 217fc03d22bSJeff Roberson }; 218cae33c14SJeff Roberson 2192019094aSRobert Watson /* 2202019094aSRobert Watson * Flags and enumerations to be passed to internal functions. 2212019094aSRobert Watson */ 222bb15d1c7SGleb Smirnoff enum zfreeskip { 223bb15d1c7SGleb Smirnoff SKIP_NONE = 0, 224bb15d1c7SGleb Smirnoff SKIP_CNT = 0x00000001, 225bb15d1c7SGleb Smirnoff SKIP_DTOR = 0x00010000, 226bb15d1c7SGleb Smirnoff SKIP_FINI = 0x00020000, 227bb15d1c7SGleb Smirnoff }; 228b23f72e9SBrian Feldman 229ab3185d1SJeff Roberson #define UMA_ANYDOMAIN -1 /* Special value for domain search. */ 230ab3185d1SJeff Roberson 2318355f576SJeff Roberson /* Prototypes.. */ 2328355f576SJeff Roberson 233f4bef67cSGleb Smirnoff int uma_startup_count(int); 234f4bef67cSGleb Smirnoff void uma_startup(void *, int); 235f4bef67cSGleb Smirnoff void uma_startup1(void); 236f4bef67cSGleb Smirnoff void uma_startup2(void); 237f4bef67cSGleb Smirnoff 238ab3185d1SJeff Roberson static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 239ab3185d1SJeff Roberson static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 240ab3059a8SMatt Macy static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 241ab3185d1SJeff Roberson static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 242f2c2231eSRyan Stone static void page_free(void *, vm_size_t, uint8_t); 243ab3059a8SMatt Macy static void pcpu_page_free(void *, vm_size_t, uint8_t); 24486220393SMark Johnston static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int); 2459643769aSJeff Roberson static void cache_drain(uma_zone_t); 2468355f576SJeff Roberson static void bucket_drain(uma_zone_t, uma_bucket_t); 247aaa8bb16SJeff Roberson static void bucket_cache_drain(uma_zone_t zone); 248b23f72e9SBrian Feldman static int keg_ctor(void *, int, void *, int); 249099a0e58SBosko Milekic static void keg_dtor(void *, int, void *); 250b23f72e9SBrian Feldman static int zone_ctor(void *, int, void *, int); 2519c2cd7e5SJeff Roberson static void zone_dtor(void *, int, void *); 252b23f72e9SBrian Feldman static int zero_init(void *, int, int); 253e20a199fSJeff Roberson static void keg_small_init(uma_keg_t keg); 254e20a199fSJeff Roberson static void keg_large_init(uma_keg_t keg); 2558355f576SJeff Roberson static void zone_foreach(void (*zfunc)(uma_zone_t)); 2568355f576SJeff Roberson static void zone_timeout(uma_zone_t zone); 2570aef6126SJeff Roberson static int hash_alloc(struct uma_hash *); 2580aef6126SJeff Roberson static int hash_expand(struct uma_hash *, struct uma_hash *); 2590aef6126SJeff Roberson static void hash_free(struct uma_hash *hash); 2608355f576SJeff Roberson static void uma_timeout(void *); 2618355f576SJeff Roberson static void uma_startup3(void); 262ab3185d1SJeff Roberson static void *zone_alloc_item(uma_zone_t, void *, int, int); 263bb15d1c7SGleb Smirnoff static void *zone_alloc_item_locked(uma_zone_t, void *, int, int); 2640095a784SJeff Roberson static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip); 26586bbae32SJeff Roberson static void bucket_enable(void); 266cae33c14SJeff Roberson static void bucket_init(void); 2676fd34d6fSJeff Roberson static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int); 2686fd34d6fSJeff Roberson static void bucket_free(uma_zone_t zone, uma_bucket_t, void *); 269cae33c14SJeff Roberson static void bucket_zone_drain(void); 270bb15d1c7SGleb Smirnoff static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int, int); 271ab3185d1SJeff Roberson static uma_slab_t zone_fetch_slab(uma_zone_t, uma_keg_t, int, int); 2720095a784SJeff Roberson static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); 273bb15d1c7SGleb Smirnoff static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item); 274e20a199fSJeff Roberson static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, 27585dcf349SGleb Smirnoff uma_fini fini, int align, uint32_t flags); 276ab3185d1SJeff Roberson static int zone_import(uma_zone_t, void **, int, int, int); 277ab3185d1SJeff Roberson static void zone_release(uma_zone_t, void **, int); 278ab3185d1SJeff Roberson static void uma_zero_item(void *, uma_zone_t); 279bbee39c6SJeff Roberson 2808355f576SJeff Roberson void uma_print_zone(uma_zone_t); 2818355f576SJeff Roberson void uma_print_stats(void); 2827a52a97eSRobert Watson static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); 2837a52a97eSRobert Watson static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); 2848355f576SJeff Roberson 2859542ea7bSGleb Smirnoff #ifdef INVARIANTS 286c5deaf04SGleb Smirnoff static bool uma_dbg_kskip(uma_keg_t keg, void *mem); 287c5deaf04SGleb Smirnoff static bool uma_dbg_zskip(uma_zone_t zone, void *mem); 2889542ea7bSGleb Smirnoff static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item); 2899542ea7bSGleb Smirnoff static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item); 290c5deaf04SGleb Smirnoff 291c5deaf04SGleb Smirnoff static SYSCTL_NODE(_vm, OID_AUTO, debug, CTLFLAG_RD, 0, 292c5deaf04SGleb Smirnoff "Memory allocation debugging"); 293c5deaf04SGleb Smirnoff 294c5deaf04SGleb Smirnoff static u_int dbg_divisor = 1; 295c5deaf04SGleb Smirnoff SYSCTL_UINT(_vm_debug, OID_AUTO, divisor, 296c5deaf04SGleb Smirnoff CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &dbg_divisor, 0, 297c5deaf04SGleb Smirnoff "Debug & thrash every this item in memory allocator"); 298c5deaf04SGleb Smirnoff 299c5deaf04SGleb Smirnoff static counter_u64_t uma_dbg_cnt = EARLY_COUNTER; 300c5deaf04SGleb Smirnoff static counter_u64_t uma_skip_cnt = EARLY_COUNTER; 301c5deaf04SGleb Smirnoff SYSCTL_COUNTER_U64(_vm_debug, OID_AUTO, trashed, CTLFLAG_RD, 302c5deaf04SGleb Smirnoff &uma_dbg_cnt, "memory items debugged"); 303c5deaf04SGleb Smirnoff SYSCTL_COUNTER_U64(_vm_debug, OID_AUTO, skipped, CTLFLAG_RD, 304c5deaf04SGleb Smirnoff &uma_skip_cnt, "memory items skipped, not debugged"); 3059542ea7bSGleb Smirnoff #endif 3069542ea7bSGleb Smirnoff 3078355f576SJeff Roberson SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 3088355f576SJeff Roberson 3097a52a97eSRobert Watson SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT, 3107a52a97eSRobert Watson 0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones"); 3117a52a97eSRobert Watson 3127a52a97eSRobert Watson SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT, 3137a52a97eSRobert Watson 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats"); 3147a52a97eSRobert Watson 3152f891cd5SPawel Jakub Dawidek static int zone_warnings = 1; 316af3b2549SHans Petter Selasky SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RWTUN, &zone_warnings, 0, 3172f891cd5SPawel Jakub Dawidek "Warn when UMA zones becomes full"); 3182f891cd5SPawel Jakub Dawidek 3192e47807cSJeff Roberson /* Adjust bytes under management by UMA. */ 3202e47807cSJeff Roberson static inline void 3212e47807cSJeff Roberson uma_total_dec(unsigned long size) 3222e47807cSJeff Roberson { 3232e47807cSJeff Roberson 3242e47807cSJeff Roberson atomic_subtract_long(&uma_kmem_total, size); 3252e47807cSJeff Roberson } 3262e47807cSJeff Roberson 3272e47807cSJeff Roberson static inline void 3282e47807cSJeff Roberson uma_total_inc(unsigned long size) 3292e47807cSJeff Roberson { 3302e47807cSJeff Roberson 3312e47807cSJeff Roberson if (atomic_fetchadd_long(&uma_kmem_total, size) > uma_kmem_limit) 3322e47807cSJeff Roberson uma_reclaim_wakeup(); 3332e47807cSJeff Roberson } 3342e47807cSJeff Roberson 33586bbae32SJeff Roberson /* 33686bbae32SJeff Roberson * This routine checks to see whether or not it's safe to enable buckets. 33786bbae32SJeff Roberson */ 33886bbae32SJeff Roberson static void 33986bbae32SJeff Roberson bucket_enable(void) 34086bbae32SJeff Roberson { 341251386b4SMaksim Yevmenkin bucketdisable = vm_page_count_min(); 34286bbae32SJeff Roberson } 34386bbae32SJeff Roberson 344dc2c7965SRobert Watson /* 345dc2c7965SRobert Watson * Initialize bucket_zones, the array of zones of buckets of various sizes. 346dc2c7965SRobert Watson * 347dc2c7965SRobert Watson * For each zone, calculate the memory required for each bucket, consisting 348fc03d22bSJeff Roberson * of the header and an array of pointers. 349dc2c7965SRobert Watson */ 350cae33c14SJeff Roberson static void 351cae33c14SJeff Roberson bucket_init(void) 352cae33c14SJeff Roberson { 353cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 354cae33c14SJeff Roberson int size; 355cae33c14SJeff Roberson 356d74e6a1dSAlan Cox for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) { 357cae33c14SJeff Roberson size = roundup(sizeof(struct uma_bucket), sizeof(void *)); 358cae33c14SJeff Roberson size += sizeof(void *) * ubz->ubz_entries; 359cae33c14SJeff Roberson ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size, 360e20a199fSJeff Roberson NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 361ab3185d1SJeff Roberson UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET | UMA_ZONE_NUMA); 362cae33c14SJeff Roberson } 363cae33c14SJeff Roberson } 364cae33c14SJeff Roberson 365dc2c7965SRobert Watson /* 366dc2c7965SRobert Watson * Given a desired number of entries for a bucket, return the zone from which 367dc2c7965SRobert Watson * to allocate the bucket. 368dc2c7965SRobert Watson */ 369dc2c7965SRobert Watson static struct uma_bucket_zone * 370dc2c7965SRobert Watson bucket_zone_lookup(int entries) 371dc2c7965SRobert Watson { 372fc03d22bSJeff Roberson struct uma_bucket_zone *ubz; 373dc2c7965SRobert Watson 374fc03d22bSJeff Roberson for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 375fc03d22bSJeff Roberson if (ubz->ubz_entries >= entries) 376fc03d22bSJeff Roberson return (ubz); 377fc03d22bSJeff Roberson ubz--; 378fc03d22bSJeff Roberson return (ubz); 379fc03d22bSJeff Roberson } 380fc03d22bSJeff Roberson 381fc03d22bSJeff Roberson static int 382fc03d22bSJeff Roberson bucket_select(int size) 383fc03d22bSJeff Roberson { 384fc03d22bSJeff Roberson struct uma_bucket_zone *ubz; 385fc03d22bSJeff Roberson 386fc03d22bSJeff Roberson ubz = &bucket_zones[0]; 387fc03d22bSJeff Roberson if (size > ubz->ubz_maxsize) 388fc03d22bSJeff Roberson return MAX((ubz->ubz_maxsize * ubz->ubz_entries) / size, 1); 389fc03d22bSJeff Roberson 390fc03d22bSJeff Roberson for (; ubz->ubz_entries != 0; ubz++) 391fc03d22bSJeff Roberson if (ubz->ubz_maxsize < size) 392fc03d22bSJeff Roberson break; 393fc03d22bSJeff Roberson ubz--; 394fc03d22bSJeff Roberson return (ubz->ubz_entries); 395dc2c7965SRobert Watson } 396dc2c7965SRobert Watson 397cae33c14SJeff Roberson static uma_bucket_t 3986fd34d6fSJeff Roberson bucket_alloc(uma_zone_t zone, void *udata, int flags) 399cae33c14SJeff Roberson { 400cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 401cae33c14SJeff Roberson uma_bucket_t bucket; 402cae33c14SJeff Roberson 403cae33c14SJeff Roberson /* 404cae33c14SJeff Roberson * This is to stop us from allocating per cpu buckets while we're 4053803b26bSDag-Erling Smørgrav * running out of vm.boot_pages. Otherwise, we would exhaust the 406cae33c14SJeff Roberson * boot pages. This also prevents us from allocating buckets in 407cae33c14SJeff Roberson * low memory situations. 408cae33c14SJeff Roberson */ 409cae33c14SJeff Roberson if (bucketdisable) 410cae33c14SJeff Roberson return (NULL); 4116fd34d6fSJeff Roberson /* 4126fd34d6fSJeff Roberson * To limit bucket recursion we store the original zone flags 4136fd34d6fSJeff Roberson * in a cookie passed via zalloc_arg/zfree_arg. This allows the 4146fd34d6fSJeff Roberson * NOVM flag to persist even through deep recursions. We also 4156fd34d6fSJeff Roberson * store ZFLAG_BUCKET once we have recursed attempting to allocate 4166fd34d6fSJeff Roberson * a bucket for a bucket zone so we do not allow infinite bucket 4176fd34d6fSJeff Roberson * recursion. This cookie will even persist to frees of unused 4186fd34d6fSJeff Roberson * buckets via the allocation path or bucket allocations in the 4196fd34d6fSJeff Roberson * free path. 4206fd34d6fSJeff Roberson */ 4216fd34d6fSJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) 4226fd34d6fSJeff Roberson udata = (void *)(uintptr_t)zone->uz_flags; 423e8a720feSAlexander Motin else { 424e8a720feSAlexander Motin if ((uintptr_t)udata & UMA_ZFLAG_BUCKET) 425e8a720feSAlexander Motin return (NULL); 4266fd34d6fSJeff Roberson udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET); 427e8a720feSAlexander Motin } 4286fd34d6fSJeff Roberson if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY) 429af526374SJeff Roberson flags |= M_NOVM; 430af526374SJeff Roberson ubz = bucket_zone_lookup(zone->uz_count); 43120d3ab87SAlexander Motin if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) 43220d3ab87SAlexander Motin ubz++; 4336fd34d6fSJeff Roberson bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags); 434cae33c14SJeff Roberson if (bucket) { 435cae33c14SJeff Roberson #ifdef INVARIANTS 436cae33c14SJeff Roberson bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); 437cae33c14SJeff Roberson #endif 438cae33c14SJeff Roberson bucket->ub_cnt = 0; 439cae33c14SJeff Roberson bucket->ub_entries = ubz->ubz_entries; 440cae33c14SJeff Roberson } 441cae33c14SJeff Roberson 442cae33c14SJeff Roberson return (bucket); 443cae33c14SJeff Roberson } 444cae33c14SJeff Roberson 445cae33c14SJeff Roberson static void 4466fd34d6fSJeff Roberson bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata) 447cae33c14SJeff Roberson { 448cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 449cae33c14SJeff Roberson 450fc03d22bSJeff Roberson KASSERT(bucket->ub_cnt == 0, 451fc03d22bSJeff Roberson ("bucket_free: Freeing a non free bucket.")); 4526fd34d6fSJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) 4536fd34d6fSJeff Roberson udata = (void *)(uintptr_t)zone->uz_flags; 454dc2c7965SRobert Watson ubz = bucket_zone_lookup(bucket->ub_entries); 4556fd34d6fSJeff Roberson uma_zfree_arg(ubz->ubz_zone, bucket, udata); 456cae33c14SJeff Roberson } 457cae33c14SJeff Roberson 458cae33c14SJeff Roberson static void 459cae33c14SJeff Roberson bucket_zone_drain(void) 460cae33c14SJeff Roberson { 461cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 462cae33c14SJeff Roberson 463cae33c14SJeff Roberson for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 464cae33c14SJeff Roberson zone_drain(ubz->ubz_zone); 465cae33c14SJeff Roberson } 466cae33c14SJeff Roberson 4670f9b7bf3SMark Johnston static uma_bucket_t 4680f9b7bf3SMark Johnston zone_try_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, const bool ws) 4690f9b7bf3SMark Johnston { 4700f9b7bf3SMark Johnston uma_bucket_t bucket; 4710f9b7bf3SMark Johnston 4720f9b7bf3SMark Johnston ZONE_LOCK_ASSERT(zone); 4730f9b7bf3SMark Johnston 4740f9b7bf3SMark Johnston if ((bucket = LIST_FIRST(&zdom->uzd_buckets)) != NULL) { 4750f9b7bf3SMark Johnston MPASS(zdom->uzd_nitems >= bucket->ub_cnt); 4760f9b7bf3SMark Johnston LIST_REMOVE(bucket, ub_link); 4770f9b7bf3SMark Johnston zdom->uzd_nitems -= bucket->ub_cnt; 4780f9b7bf3SMark Johnston if (ws && zdom->uzd_imin > zdom->uzd_nitems) 4790f9b7bf3SMark Johnston zdom->uzd_imin = zdom->uzd_nitems; 480bb15d1c7SGleb Smirnoff zone->uz_bkt_count -= bucket->ub_cnt; 4810f9b7bf3SMark Johnston } 4820f9b7bf3SMark Johnston return (bucket); 4830f9b7bf3SMark Johnston } 4840f9b7bf3SMark Johnston 4850f9b7bf3SMark Johnston static void 4860f9b7bf3SMark Johnston zone_put_bucket(uma_zone_t zone, uma_zone_domain_t zdom, uma_bucket_t bucket, 4870f9b7bf3SMark Johnston const bool ws) 4880f9b7bf3SMark Johnston { 4890f9b7bf3SMark Johnston 4900f9b7bf3SMark Johnston ZONE_LOCK_ASSERT(zone); 491bb15d1c7SGleb Smirnoff KASSERT(zone->uz_bkt_count < zone->uz_bkt_max, ("%s: zone %p overflow", 492bb15d1c7SGleb Smirnoff __func__, zone)); 4930f9b7bf3SMark Johnston 4940f9b7bf3SMark Johnston LIST_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link); 4950f9b7bf3SMark Johnston zdom->uzd_nitems += bucket->ub_cnt; 4960f9b7bf3SMark Johnston if (ws && zdom->uzd_imax < zdom->uzd_nitems) 4970f9b7bf3SMark Johnston zdom->uzd_imax = zdom->uzd_nitems; 498bb15d1c7SGleb Smirnoff zone->uz_bkt_count += bucket->ub_cnt; 4990f9b7bf3SMark Johnston } 5000f9b7bf3SMark Johnston 5012f891cd5SPawel Jakub Dawidek static void 5022f891cd5SPawel Jakub Dawidek zone_log_warning(uma_zone_t zone) 5032f891cd5SPawel Jakub Dawidek { 5042f891cd5SPawel Jakub Dawidek static const struct timeval warninterval = { 300, 0 }; 5052f891cd5SPawel Jakub Dawidek 5062f891cd5SPawel Jakub Dawidek if (!zone_warnings || zone->uz_warning == NULL) 5072f891cd5SPawel Jakub Dawidek return; 5082f891cd5SPawel Jakub Dawidek 5092f891cd5SPawel Jakub Dawidek if (ratecheck(&zone->uz_ratecheck, &warninterval)) 5102f891cd5SPawel Jakub Dawidek printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning); 5112f891cd5SPawel Jakub Dawidek } 5122f891cd5SPawel Jakub Dawidek 51354503a13SJonathan T. Looney static inline void 51454503a13SJonathan T. Looney zone_maxaction(uma_zone_t zone) 51554503a13SJonathan T. Looney { 516e60b2fcbSGleb Smirnoff 517e60b2fcbSGleb Smirnoff if (zone->uz_maxaction.ta_func != NULL) 518e60b2fcbSGleb Smirnoff taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction); 51954503a13SJonathan T. Looney } 52054503a13SJonathan T. Looney 5218355f576SJeff Roberson /* 5228355f576SJeff Roberson * Routine called by timeout which is used to fire off some time interval 5239643769aSJeff Roberson * based calculations. (stats, hash size, etc.) 5248355f576SJeff Roberson * 5258355f576SJeff Roberson * Arguments: 5268355f576SJeff Roberson * arg Unused 5278355f576SJeff Roberson * 5288355f576SJeff Roberson * Returns: 5298355f576SJeff Roberson * Nothing 5308355f576SJeff Roberson */ 5318355f576SJeff Roberson static void 5328355f576SJeff Roberson uma_timeout(void *unused) 5338355f576SJeff Roberson { 53486bbae32SJeff Roberson bucket_enable(); 5358355f576SJeff Roberson zone_foreach(zone_timeout); 5368355f576SJeff Roberson 5378355f576SJeff Roberson /* Reschedule this event */ 5389643769aSJeff Roberson callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 5398355f576SJeff Roberson } 5408355f576SJeff Roberson 5418355f576SJeff Roberson /* 5420f9b7bf3SMark Johnston * Update the working set size estimate for the zone's bucket cache. 5430f9b7bf3SMark Johnston * The constants chosen here are somewhat arbitrary. With an update period of 5440f9b7bf3SMark Johnston * 20s (UMA_TIMEOUT), this estimate is dominated by zone activity over the 5450f9b7bf3SMark Johnston * last 100s. 5460f9b7bf3SMark Johnston */ 5470f9b7bf3SMark Johnston static void 5480f9b7bf3SMark Johnston zone_domain_update_wss(uma_zone_domain_t zdom) 5490f9b7bf3SMark Johnston { 5500f9b7bf3SMark Johnston long wss; 5510f9b7bf3SMark Johnston 5520f9b7bf3SMark Johnston MPASS(zdom->uzd_imax >= zdom->uzd_imin); 5530f9b7bf3SMark Johnston wss = zdom->uzd_imax - zdom->uzd_imin; 5540f9b7bf3SMark Johnston zdom->uzd_imax = zdom->uzd_imin = zdom->uzd_nitems; 5550f9b7bf3SMark Johnston zdom->uzd_wss = (3 * wss + 2 * zdom->uzd_wss) / 5; 5560f9b7bf3SMark Johnston } 5570f9b7bf3SMark Johnston 5580f9b7bf3SMark Johnston /* 5599643769aSJeff Roberson * Routine to perform timeout driven calculations. This expands the 5609643769aSJeff Roberson * hashes and does per cpu statistics aggregation. 5618355f576SJeff Roberson * 562e20a199fSJeff Roberson * Returns nothing. 5638355f576SJeff Roberson */ 5648355f576SJeff Roberson static void 565bb15d1c7SGleb Smirnoff zone_timeout(uma_zone_t zone) 5668355f576SJeff Roberson { 567bb15d1c7SGleb Smirnoff uma_keg_t keg = zone->uz_keg; 5688355f576SJeff Roberson 569e20a199fSJeff Roberson KEG_LOCK(keg); 5708355f576SJeff Roberson /* 571e20a199fSJeff Roberson * Expand the keg hash table. 5728355f576SJeff Roberson * 5738355f576SJeff Roberson * This is done if the number of slabs is larger than the hash size. 5748355f576SJeff Roberson * What I'm trying to do here is completely reduce collisions. This 5758355f576SJeff Roberson * may be a little aggressive. Should I allow for two collisions max? 5768355f576SJeff Roberson */ 577099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_HASH && 578099a0e58SBosko Milekic keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) { 5790aef6126SJeff Roberson struct uma_hash newhash; 5800aef6126SJeff Roberson struct uma_hash oldhash; 5810aef6126SJeff Roberson int ret; 5825300d9ddSJeff Roberson 5830aef6126SJeff Roberson /* 5840aef6126SJeff Roberson * This is so involved because allocating and freeing 585e20a199fSJeff Roberson * while the keg lock is held will lead to deadlock. 5860aef6126SJeff Roberson * I have to do everything in stages and check for 5870aef6126SJeff Roberson * races. 5880aef6126SJeff Roberson */ 589099a0e58SBosko Milekic newhash = keg->uk_hash; 590e20a199fSJeff Roberson KEG_UNLOCK(keg); 5910aef6126SJeff Roberson ret = hash_alloc(&newhash); 592e20a199fSJeff Roberson KEG_LOCK(keg); 5930aef6126SJeff Roberson if (ret) { 594099a0e58SBosko Milekic if (hash_expand(&keg->uk_hash, &newhash)) { 595099a0e58SBosko Milekic oldhash = keg->uk_hash; 596099a0e58SBosko Milekic keg->uk_hash = newhash; 5970aef6126SJeff Roberson } else 5980aef6126SJeff Roberson oldhash = newhash; 5990aef6126SJeff Roberson 600e20a199fSJeff Roberson KEG_UNLOCK(keg); 6010aef6126SJeff Roberson hash_free(&oldhash); 602a1dff920SDavide Italiano return; 6030aef6126SJeff Roberson } 6045300d9ddSJeff Roberson } 605e20a199fSJeff Roberson 606bb15d1c7SGleb Smirnoff for (int i = 0; i < vm_ndomains; i++) 6070f9b7bf3SMark Johnston zone_domain_update_wss(&zone->uz_domain[i]); 608bb15d1c7SGleb Smirnoff 609bb15d1c7SGleb Smirnoff KEG_UNLOCK(keg); 6108355f576SJeff Roberson } 6118355f576SJeff Roberson 6128355f576SJeff Roberson /* 6135300d9ddSJeff Roberson * Allocate and zero fill the next sized hash table from the appropriate 6145300d9ddSJeff Roberson * backing store. 6155300d9ddSJeff Roberson * 6165300d9ddSJeff Roberson * Arguments: 6170aef6126SJeff Roberson * hash A new hash structure with the old hash size in uh_hashsize 6185300d9ddSJeff Roberson * 6195300d9ddSJeff Roberson * Returns: 620763df3ecSPedro F. Giffuni * 1 on success and 0 on failure. 6215300d9ddSJeff Roberson */ 62237c84183SPoul-Henning Kamp static int 6230aef6126SJeff Roberson hash_alloc(struct uma_hash *hash) 6245300d9ddSJeff Roberson { 6256929b7d1SPedro F. Giffuni u_int oldsize; 62659568a0eSAlexander Motin size_t alloc; 6275300d9ddSJeff Roberson 6280aef6126SJeff Roberson oldsize = hash->uh_hashsize; 6290aef6126SJeff Roberson 6305300d9ddSJeff Roberson /* We're just going to go to a power of two greater */ 6310aef6126SJeff Roberson if (oldsize) { 6320aef6126SJeff Roberson hash->uh_hashsize = oldsize * 2; 6330aef6126SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; 6340aef6126SJeff Roberson hash->uh_slab_hash = (struct slabhead *)malloc(alloc, 635961647dfSJeff Roberson M_UMAHASH, M_NOWAIT); 6365300d9ddSJeff Roberson } else { 6370aef6126SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; 638e20a199fSJeff Roberson hash->uh_slab_hash = zone_alloc_item(hashzone, NULL, 639ab3185d1SJeff Roberson UMA_ANYDOMAIN, M_WAITOK); 6400aef6126SJeff Roberson hash->uh_hashsize = UMA_HASH_SIZE_INIT; 6415300d9ddSJeff Roberson } 6420aef6126SJeff Roberson if (hash->uh_slab_hash) { 6430aef6126SJeff Roberson bzero(hash->uh_slab_hash, alloc); 6440aef6126SJeff Roberson hash->uh_hashmask = hash->uh_hashsize - 1; 6450aef6126SJeff Roberson return (1); 6460aef6126SJeff Roberson } 6475300d9ddSJeff Roberson 6480aef6126SJeff Roberson return (0); 6495300d9ddSJeff Roberson } 6505300d9ddSJeff Roberson 6515300d9ddSJeff Roberson /* 65264f051e9SJeff Roberson * Expands the hash table for HASH zones. This is done from zone_timeout 65364f051e9SJeff Roberson * to reduce collisions. This must not be done in the regular allocation 65464f051e9SJeff Roberson * path, otherwise, we can recurse on the vm while allocating pages. 6558355f576SJeff Roberson * 6568355f576SJeff Roberson * Arguments: 6570aef6126SJeff Roberson * oldhash The hash you want to expand 6580aef6126SJeff Roberson * newhash The hash structure for the new table 6598355f576SJeff Roberson * 6608355f576SJeff Roberson * Returns: 6618355f576SJeff Roberson * Nothing 6628355f576SJeff Roberson * 6638355f576SJeff Roberson * Discussion: 6648355f576SJeff Roberson */ 6650aef6126SJeff Roberson static int 6660aef6126SJeff Roberson hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash) 6678355f576SJeff Roberson { 6688355f576SJeff Roberson uma_slab_t slab; 6696929b7d1SPedro F. Giffuni u_int hval; 6706929b7d1SPedro F. Giffuni u_int idx; 6718355f576SJeff Roberson 6720aef6126SJeff Roberson if (!newhash->uh_slab_hash) 6730aef6126SJeff Roberson return (0); 6748355f576SJeff Roberson 6750aef6126SJeff Roberson if (oldhash->uh_hashsize >= newhash->uh_hashsize) 6760aef6126SJeff Roberson return (0); 6778355f576SJeff Roberson 6788355f576SJeff Roberson /* 6798355f576SJeff Roberson * I need to investigate hash algorithms for resizing without a 6808355f576SJeff Roberson * full rehash. 6818355f576SJeff Roberson */ 6828355f576SJeff Roberson 6836929b7d1SPedro F. Giffuni for (idx = 0; idx < oldhash->uh_hashsize; idx++) 6846929b7d1SPedro F. Giffuni while (!SLIST_EMPTY(&oldhash->uh_slab_hash[idx])) { 6856929b7d1SPedro F. Giffuni slab = SLIST_FIRST(&oldhash->uh_slab_hash[idx]); 6866929b7d1SPedro F. Giffuni SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[idx], us_hlink); 6870aef6126SJeff Roberson hval = UMA_HASH(newhash, slab->us_data); 6880aef6126SJeff Roberson SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval], 6890aef6126SJeff Roberson slab, us_hlink); 6908355f576SJeff Roberson } 6918355f576SJeff Roberson 6920aef6126SJeff Roberson return (1); 6939c2cd7e5SJeff Roberson } 6949c2cd7e5SJeff Roberson 6955300d9ddSJeff Roberson /* 6965300d9ddSJeff Roberson * Free the hash bucket to the appropriate backing store. 6975300d9ddSJeff Roberson * 6985300d9ddSJeff Roberson * Arguments: 6995300d9ddSJeff Roberson * slab_hash The hash bucket we're freeing 7005300d9ddSJeff Roberson * hashsize The number of entries in that hash bucket 7015300d9ddSJeff Roberson * 7025300d9ddSJeff Roberson * Returns: 7035300d9ddSJeff Roberson * Nothing 7045300d9ddSJeff Roberson */ 7059c2cd7e5SJeff Roberson static void 7060aef6126SJeff Roberson hash_free(struct uma_hash *hash) 7079c2cd7e5SJeff Roberson { 7080aef6126SJeff Roberson if (hash->uh_slab_hash == NULL) 7090aef6126SJeff Roberson return; 7100aef6126SJeff Roberson if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) 7110095a784SJeff Roberson zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE); 7128355f576SJeff Roberson else 713961647dfSJeff Roberson free(hash->uh_slab_hash, M_UMAHASH); 7148355f576SJeff Roberson } 7158355f576SJeff Roberson 7168355f576SJeff Roberson /* 7178355f576SJeff Roberson * Frees all outstanding items in a bucket 7188355f576SJeff Roberson * 7198355f576SJeff Roberson * Arguments: 7208355f576SJeff Roberson * zone The zone to free to, must be unlocked. 7218355f576SJeff Roberson * bucket The free/alloc bucket with items, cpu queue must be locked. 7228355f576SJeff Roberson * 7238355f576SJeff Roberson * Returns: 7248355f576SJeff Roberson * Nothing 7258355f576SJeff Roberson */ 7268355f576SJeff Roberson 7278355f576SJeff Roberson static void 7288355f576SJeff Roberson bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 7298355f576SJeff Roberson { 7300095a784SJeff Roberson int i; 7318355f576SJeff Roberson 7328355f576SJeff Roberson if (bucket == NULL) 7338355f576SJeff Roberson return; 7348355f576SJeff Roberson 7350095a784SJeff Roberson if (zone->uz_fini) 7360095a784SJeff Roberson for (i = 0; i < bucket->ub_cnt; i++) 7370095a784SJeff Roberson zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); 7380095a784SJeff Roberson zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); 739bb45b411SGleb Smirnoff if (zone->uz_max_items > 0) { 740bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 741bb15d1c7SGleb Smirnoff zone->uz_items -= bucket->ub_cnt; 742bb15d1c7SGleb Smirnoff if (zone->uz_sleepers && zone->uz_items < zone->uz_max_items) 743bb15d1c7SGleb Smirnoff wakeup_one(zone); 744bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 745bb45b411SGleb Smirnoff } 7460095a784SJeff Roberson bucket->ub_cnt = 0; 7478355f576SJeff Roberson } 7488355f576SJeff Roberson 7498355f576SJeff Roberson /* 7508355f576SJeff Roberson * Drains the per cpu caches for a zone. 7518355f576SJeff Roberson * 7525d1ae027SRobert Watson * NOTE: This may only be called while the zone is being turn down, and not 7535d1ae027SRobert Watson * during normal operation. This is necessary in order that we do not have 7545d1ae027SRobert Watson * to migrate CPUs to drain the per-CPU caches. 7555d1ae027SRobert Watson * 7568355f576SJeff Roberson * Arguments: 7578355f576SJeff Roberson * zone The zone to drain, must be unlocked. 7588355f576SJeff Roberson * 7598355f576SJeff Roberson * Returns: 7608355f576SJeff Roberson * Nothing 7618355f576SJeff Roberson */ 7628355f576SJeff Roberson static void 7639643769aSJeff Roberson cache_drain(uma_zone_t zone) 7648355f576SJeff Roberson { 7658355f576SJeff Roberson uma_cache_t cache; 7668355f576SJeff Roberson int cpu; 7678355f576SJeff Roberson 7688355f576SJeff Roberson /* 7695d1ae027SRobert Watson * XXX: It is safe to not lock the per-CPU caches, because we're 7705d1ae027SRobert Watson * tearing down the zone anyway. I.e., there will be no further use 7715d1ae027SRobert Watson * of the caches at this point. 7725d1ae027SRobert Watson * 7735d1ae027SRobert Watson * XXX: It would good to be able to assert that the zone is being 7745d1ae027SRobert Watson * torn down to prevent improper use of cache_drain(). 7755d1ae027SRobert Watson * 7765d1ae027SRobert Watson * XXX: We lock the zone before passing into bucket_cache_drain() as 7775d1ae027SRobert Watson * it is used elsewhere. Should the tear-down path be made special 7785d1ae027SRobert Watson * there in some form? 7798355f576SJeff Roberson */ 7803aa6d94eSJohn Baldwin CPU_FOREACH(cpu) { 7818355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 7828355f576SJeff Roberson bucket_drain(zone, cache->uc_allocbucket); 7838355f576SJeff Roberson bucket_drain(zone, cache->uc_freebucket); 784174ab450SBosko Milekic if (cache->uc_allocbucket != NULL) 7856fd34d6fSJeff Roberson bucket_free(zone, cache->uc_allocbucket, NULL); 786174ab450SBosko Milekic if (cache->uc_freebucket != NULL) 7876fd34d6fSJeff Roberson bucket_free(zone, cache->uc_freebucket, NULL); 788d56368d7SBosko Milekic cache->uc_allocbucket = cache->uc_freebucket = NULL; 789d56368d7SBosko Milekic } 790aaa8bb16SJeff Roberson ZONE_LOCK(zone); 791aaa8bb16SJeff Roberson bucket_cache_drain(zone); 792aaa8bb16SJeff Roberson ZONE_UNLOCK(zone); 793aaa8bb16SJeff Roberson } 794aaa8bb16SJeff Roberson 795a2de44abSAlexander Motin static void 796a2de44abSAlexander Motin cache_shrink(uma_zone_t zone) 797a2de44abSAlexander Motin { 798a2de44abSAlexander Motin 799a2de44abSAlexander Motin if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 800a2de44abSAlexander Motin return; 801a2de44abSAlexander Motin 802a2de44abSAlexander Motin ZONE_LOCK(zone); 803a2de44abSAlexander Motin zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2; 804a2de44abSAlexander Motin ZONE_UNLOCK(zone); 805a2de44abSAlexander Motin } 806a2de44abSAlexander Motin 807a2de44abSAlexander Motin static void 808a2de44abSAlexander Motin cache_drain_safe_cpu(uma_zone_t zone) 809a2de44abSAlexander Motin { 810a2de44abSAlexander Motin uma_cache_t cache; 8118a8d9d14SAlexander Motin uma_bucket_t b1, b2; 812ab3185d1SJeff Roberson int domain; 813a2de44abSAlexander Motin 814a2de44abSAlexander Motin if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 815a2de44abSAlexander Motin return; 816a2de44abSAlexander Motin 8178a8d9d14SAlexander Motin b1 = b2 = NULL; 818a2de44abSAlexander Motin ZONE_LOCK(zone); 819a2de44abSAlexander Motin critical_enter(); 820ab3185d1SJeff Roberson if (zone->uz_flags & UMA_ZONE_NUMA) 821ab3185d1SJeff Roberson domain = PCPU_GET(domain); 822ab3185d1SJeff Roberson else 823ab3185d1SJeff Roberson domain = 0; 824a2de44abSAlexander Motin cache = &zone->uz_cpu[curcpu]; 825a2de44abSAlexander Motin if (cache->uc_allocbucket) { 8268a8d9d14SAlexander Motin if (cache->uc_allocbucket->ub_cnt != 0) 8270f9b7bf3SMark Johnston zone_put_bucket(zone, &zone->uz_domain[domain], 8280f9b7bf3SMark Johnston cache->uc_allocbucket, false); 8298a8d9d14SAlexander Motin else 8308a8d9d14SAlexander Motin b1 = cache->uc_allocbucket; 831a2de44abSAlexander Motin cache->uc_allocbucket = NULL; 832a2de44abSAlexander Motin } 833a2de44abSAlexander Motin if (cache->uc_freebucket) { 8348a8d9d14SAlexander Motin if (cache->uc_freebucket->ub_cnt != 0) 8350f9b7bf3SMark Johnston zone_put_bucket(zone, &zone->uz_domain[domain], 8360f9b7bf3SMark Johnston cache->uc_freebucket, false); 8378a8d9d14SAlexander Motin else 8388a8d9d14SAlexander Motin b2 = cache->uc_freebucket; 839a2de44abSAlexander Motin cache->uc_freebucket = NULL; 840a2de44abSAlexander Motin } 841a2de44abSAlexander Motin critical_exit(); 842a2de44abSAlexander Motin ZONE_UNLOCK(zone); 8438a8d9d14SAlexander Motin if (b1) 8448a8d9d14SAlexander Motin bucket_free(zone, b1, NULL); 8458a8d9d14SAlexander Motin if (b2) 8468a8d9d14SAlexander Motin bucket_free(zone, b2, NULL); 847a2de44abSAlexander Motin } 848a2de44abSAlexander Motin 849a2de44abSAlexander Motin /* 850a2de44abSAlexander Motin * Safely drain per-CPU caches of a zone(s) to alloc bucket. 851a2de44abSAlexander Motin * This is an expensive call because it needs to bind to all CPUs 852a2de44abSAlexander Motin * one by one and enter a critical section on each of them in order 853a2de44abSAlexander Motin * to safely access their cache buckets. 854a2de44abSAlexander Motin * Zone lock must not be held on call this function. 855a2de44abSAlexander Motin */ 856a2de44abSAlexander Motin static void 857a2de44abSAlexander Motin cache_drain_safe(uma_zone_t zone) 858a2de44abSAlexander Motin { 859a2de44abSAlexander Motin int cpu; 860a2de44abSAlexander Motin 861a2de44abSAlexander Motin /* 862a2de44abSAlexander Motin * Polite bucket sizes shrinking was not enouth, shrink aggressively. 863a2de44abSAlexander Motin */ 864a2de44abSAlexander Motin if (zone) 865a2de44abSAlexander Motin cache_shrink(zone); 866a2de44abSAlexander Motin else 867a2de44abSAlexander Motin zone_foreach(cache_shrink); 868a2de44abSAlexander Motin 869a2de44abSAlexander Motin CPU_FOREACH(cpu) { 870a2de44abSAlexander Motin thread_lock(curthread); 871a2de44abSAlexander Motin sched_bind(curthread, cpu); 872a2de44abSAlexander Motin thread_unlock(curthread); 873a2de44abSAlexander Motin 874a2de44abSAlexander Motin if (zone) 875a2de44abSAlexander Motin cache_drain_safe_cpu(zone); 876a2de44abSAlexander Motin else 877a2de44abSAlexander Motin zone_foreach(cache_drain_safe_cpu); 878a2de44abSAlexander Motin } 879a2de44abSAlexander Motin thread_lock(curthread); 880a2de44abSAlexander Motin sched_unbind(curthread); 881a2de44abSAlexander Motin thread_unlock(curthread); 882a2de44abSAlexander Motin } 883a2de44abSAlexander Motin 884aaa8bb16SJeff Roberson /* 885aaa8bb16SJeff Roberson * Drain the cached buckets from a zone. Expects a locked zone on entry. 886aaa8bb16SJeff Roberson */ 887aaa8bb16SJeff Roberson static void 888aaa8bb16SJeff Roberson bucket_cache_drain(uma_zone_t zone) 889aaa8bb16SJeff Roberson { 890ab3185d1SJeff Roberson uma_zone_domain_t zdom; 891aaa8bb16SJeff Roberson uma_bucket_t bucket; 892ab3185d1SJeff Roberson int i; 8938355f576SJeff Roberson 8948355f576SJeff Roberson /* 895ab3185d1SJeff Roberson * Drain the bucket queues and free the buckets. 8968355f576SJeff Roberson */ 897ab3185d1SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 898ab3185d1SJeff Roberson zdom = &zone->uz_domain[i]; 8990f9b7bf3SMark Johnston while ((bucket = zone_try_fetch_bucket(zone, zdom, false)) != 9000f9b7bf3SMark Johnston NULL) { 9018355f576SJeff Roberson ZONE_UNLOCK(zone); 9028355f576SJeff Roberson bucket_drain(zone, bucket); 9036fd34d6fSJeff Roberson bucket_free(zone, bucket, NULL); 9048355f576SJeff Roberson ZONE_LOCK(zone); 9058355f576SJeff Roberson } 906ab3185d1SJeff Roberson } 907ace66b56SAlexander Motin 908ace66b56SAlexander Motin /* 909ace66b56SAlexander Motin * Shrink further bucket sizes. Price of single zone lock collision 910ace66b56SAlexander Motin * is probably lower then price of global cache drain. 911ace66b56SAlexander Motin */ 912ace66b56SAlexander Motin if (zone->uz_count > zone->uz_count_min) 913ace66b56SAlexander Motin zone->uz_count--; 9148355f576SJeff Roberson } 915fc03d22bSJeff Roberson 916fc03d22bSJeff Roberson static void 917fc03d22bSJeff Roberson keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) 918fc03d22bSJeff Roberson { 919fc03d22bSJeff Roberson uint8_t *mem; 920fc03d22bSJeff Roberson int i; 921fc03d22bSJeff Roberson uint8_t flags; 922fc03d22bSJeff Roberson 9231431a748SGleb Smirnoff CTR4(KTR_UMA, "keg_free_slab keg %s(%p) slab %p, returning %d bytes", 9241431a748SGleb Smirnoff keg->uk_name, keg, slab, PAGE_SIZE * keg->uk_ppera); 9251431a748SGleb Smirnoff 926fc03d22bSJeff Roberson mem = slab->us_data; 927fc03d22bSJeff Roberson flags = slab->us_flags; 928fc03d22bSJeff Roberson i = start; 929fc03d22bSJeff Roberson if (keg->uk_fini != NULL) { 930fc03d22bSJeff Roberson for (i--; i > -1; i--) 931c5deaf04SGleb Smirnoff #ifdef INVARIANTS 932c5deaf04SGleb Smirnoff /* 933c5deaf04SGleb Smirnoff * trash_fini implies that dtor was trash_dtor. trash_fini 934c5deaf04SGleb Smirnoff * would check that memory hasn't been modified since free, 935c5deaf04SGleb Smirnoff * which executed trash_dtor. 936c5deaf04SGleb Smirnoff * That's why we need to run uma_dbg_kskip() check here, 937c5deaf04SGleb Smirnoff * albeit we don't make skip check for other init/fini 938c5deaf04SGleb Smirnoff * invocations. 939c5deaf04SGleb Smirnoff */ 940c5deaf04SGleb Smirnoff if (!uma_dbg_kskip(keg, slab->us_data + (keg->uk_rsize * i)) || 941c5deaf04SGleb Smirnoff keg->uk_fini != trash_fini) 942c5deaf04SGleb Smirnoff #endif 943fc03d22bSJeff Roberson keg->uk_fini(slab->us_data + (keg->uk_rsize * i), 944fc03d22bSJeff Roberson keg->uk_size); 945fc03d22bSJeff Roberson } 946fc03d22bSJeff Roberson if (keg->uk_flags & UMA_ZONE_OFFPAGE) 947fc03d22bSJeff Roberson zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); 948fc03d22bSJeff Roberson keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags); 9492e47807cSJeff Roberson uma_total_dec(PAGE_SIZE * keg->uk_ppera); 9508355f576SJeff Roberson } 9518355f576SJeff Roberson 9528355f576SJeff Roberson /* 953e20a199fSJeff Roberson * Frees pages from a keg back to the system. This is done on demand from 9548355f576SJeff Roberson * the pageout daemon. 9558355f576SJeff Roberson * 956e20a199fSJeff Roberson * Returns nothing. 9578355f576SJeff Roberson */ 958e20a199fSJeff Roberson static void 959e20a199fSJeff Roberson keg_drain(uma_keg_t keg) 9608355f576SJeff Roberson { 9611e183df2SStefan Farfeleder struct slabhead freeslabs = { 0 }; 962ab3185d1SJeff Roberson uma_domain_t dom; 963829be516SMark Johnston uma_slab_t slab, tmp; 964ab3185d1SJeff Roberson int i; 9658355f576SJeff Roberson 9668355f576SJeff Roberson /* 967e20a199fSJeff Roberson * We don't want to take pages from statically allocated kegs at this 9688355f576SJeff Roberson * time 9698355f576SJeff Roberson */ 970099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) 9718355f576SJeff Roberson return; 9728355f576SJeff Roberson 9731431a748SGleb Smirnoff CTR3(KTR_UMA, "keg_drain %s(%p) free items: %u", 9741431a748SGleb Smirnoff keg->uk_name, keg, keg->uk_free); 975e20a199fSJeff Roberson KEG_LOCK(keg); 976099a0e58SBosko Milekic if (keg->uk_free == 0) 9778355f576SJeff Roberson goto finished; 9788355f576SJeff Roberson 979ab3185d1SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 980ab3185d1SJeff Roberson dom = &keg->uk_domain[i]; 981ab3185d1SJeff Roberson LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) { 982829be516SMark Johnston /* We have nowhere to free these to. */ 983829be516SMark Johnston if (slab->us_flags & UMA_SLAB_BOOT) 9848355f576SJeff Roberson continue; 9858355f576SJeff Roberson 9868355f576SJeff Roberson LIST_REMOVE(slab, us_link); 987099a0e58SBosko Milekic keg->uk_pages -= keg->uk_ppera; 988099a0e58SBosko Milekic keg->uk_free -= keg->uk_ipers; 989713deb36SJeff Roberson 990099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_HASH) 991ab3185d1SJeff Roberson UMA_HASH_REMOVE(&keg->uk_hash, slab, 992ab3185d1SJeff Roberson slab->us_data); 993713deb36SJeff Roberson 994713deb36SJeff Roberson SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink); 995713deb36SJeff Roberson } 996ab3185d1SJeff Roberson } 997ab3185d1SJeff Roberson 998713deb36SJeff Roberson finished: 999e20a199fSJeff Roberson KEG_UNLOCK(keg); 1000713deb36SJeff Roberson 1001713deb36SJeff Roberson while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { 1002713deb36SJeff Roberson SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); 10031645995bSKirk McKusick keg_free_slab(keg, slab, keg->uk_ipers); 10048355f576SJeff Roberson } 10058355f576SJeff Roberson } 10068355f576SJeff Roberson 1007e20a199fSJeff Roberson static void 1008e20a199fSJeff Roberson zone_drain_wait(uma_zone_t zone, int waitok) 1009e20a199fSJeff Roberson { 1010e20a199fSJeff Roberson 10118355f576SJeff Roberson /* 1012e20a199fSJeff Roberson * Set draining to interlock with zone_dtor() so we can release our 1013e20a199fSJeff Roberson * locks as we go. Only dtor() should do a WAITOK call since it 1014e20a199fSJeff Roberson * is the only call that knows the structure will still be available 1015e20a199fSJeff Roberson * when it wakes up. 1016e20a199fSJeff Roberson */ 1017e20a199fSJeff Roberson ZONE_LOCK(zone); 1018e20a199fSJeff Roberson while (zone->uz_flags & UMA_ZFLAG_DRAINING) { 1019e20a199fSJeff Roberson if (waitok == M_NOWAIT) 1020e20a199fSJeff Roberson goto out; 1021af526374SJeff Roberson msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1); 1022e20a199fSJeff Roberson } 1023e20a199fSJeff Roberson zone->uz_flags |= UMA_ZFLAG_DRAINING; 1024e20a199fSJeff Roberson bucket_cache_drain(zone); 1025e20a199fSJeff Roberson ZONE_UNLOCK(zone); 1026e20a199fSJeff Roberson /* 1027e20a199fSJeff Roberson * The DRAINING flag protects us from being freed while 1028111fbcd5SBryan Venteicher * we're running. Normally the uma_rwlock would protect us but we 1029e20a199fSJeff Roberson * must be able to release and acquire the right lock for each keg. 1030e20a199fSJeff Roberson */ 1031bb15d1c7SGleb Smirnoff keg_drain(zone->uz_keg); 1032e20a199fSJeff Roberson ZONE_LOCK(zone); 1033e20a199fSJeff Roberson zone->uz_flags &= ~UMA_ZFLAG_DRAINING; 1034e20a199fSJeff Roberson wakeup(zone); 1035e20a199fSJeff Roberson out: 1036e20a199fSJeff Roberson ZONE_UNLOCK(zone); 1037e20a199fSJeff Roberson } 1038e20a199fSJeff Roberson 1039e20a199fSJeff Roberson void 1040e20a199fSJeff Roberson zone_drain(uma_zone_t zone) 1041e20a199fSJeff Roberson { 1042e20a199fSJeff Roberson 1043e20a199fSJeff Roberson zone_drain_wait(zone, M_NOWAIT); 1044e20a199fSJeff Roberson } 1045e20a199fSJeff Roberson 1046e20a199fSJeff Roberson /* 1047e20a199fSJeff Roberson * Allocate a new slab for a keg. This does not insert the slab onto a list. 1048194a979eSMark Johnston * If the allocation was successful, the keg lock will be held upon return, 1049194a979eSMark Johnston * otherwise the keg will be left unlocked. 10508355f576SJeff Roberson * 10518355f576SJeff Roberson * Arguments: 105286220393SMark Johnston * flags Wait flags for the item initialization routine 105386220393SMark Johnston * aflags Wait flags for the slab allocation 10548355f576SJeff Roberson * 10558355f576SJeff Roberson * Returns: 10568355f576SJeff Roberson * The slab that was allocated or NULL if there is no memory and the 10578355f576SJeff Roberson * caller specified M_NOWAIT. 10588355f576SJeff Roberson */ 10598355f576SJeff Roberson static uma_slab_t 106086220393SMark Johnston keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, 106186220393SMark Johnston int aflags) 10628355f576SJeff Roberson { 1063e20a199fSJeff Roberson uma_alloc allocf; 1064099a0e58SBosko Milekic uma_slab_t slab; 10652e47807cSJeff Roberson unsigned long size; 106685dcf349SGleb Smirnoff uint8_t *mem; 106786220393SMark Johnston uint8_t sflags; 10688355f576SJeff Roberson int i; 10698355f576SJeff Roberson 1070ab3185d1SJeff Roberson KASSERT(domain >= 0 && domain < vm_ndomains, 1071ab3185d1SJeff Roberson ("keg_alloc_slab: domain %d out of range", domain)); 1072bb15d1c7SGleb Smirnoff KEG_LOCK_ASSERT(keg); 1073bb15d1c7SGleb Smirnoff MPASS(zone->uz_lockptr == &keg->uk_lock); 1074a553d4b8SJeff Roberson 1075e20a199fSJeff Roberson allocf = keg->uk_allocf; 1076e20a199fSJeff Roberson KEG_UNLOCK(keg); 1077a553d4b8SJeff Roberson 1078194a979eSMark Johnston slab = NULL; 1079194a979eSMark Johnston mem = NULL; 1080099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_OFFPAGE) { 108186220393SMark Johnston slab = zone_alloc_item(keg->uk_slabzone, NULL, domain, aflags); 1082fc03d22bSJeff Roberson if (slab == NULL) 1083fc03d22bSJeff Roberson goto out; 1084a553d4b8SJeff Roberson } 1085a553d4b8SJeff Roberson 10863370c5bfSJeff Roberson /* 10873370c5bfSJeff Roberson * This reproduces the old vm_zone behavior of zero filling pages the 10883370c5bfSJeff Roberson * first time they are added to a zone. 10893370c5bfSJeff Roberson * 10903370c5bfSJeff Roberson * Malloced items are zeroed in uma_zalloc. 10913370c5bfSJeff Roberson */ 10923370c5bfSJeff Roberson 1093099a0e58SBosko Milekic if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) 109486220393SMark Johnston aflags |= M_ZERO; 10953370c5bfSJeff Roberson else 109686220393SMark Johnston aflags &= ~M_ZERO; 10973370c5bfSJeff Roberson 1098263811f7SKip Macy if (keg->uk_flags & UMA_ZONE_NODUMP) 109986220393SMark Johnston aflags |= M_NODUMP; 1100263811f7SKip Macy 1101e20a199fSJeff Roberson /* zone is passed for legacy reasons. */ 1102194a979eSMark Johnston size = keg->uk_ppera * PAGE_SIZE; 110386220393SMark Johnston mem = allocf(zone, size, domain, &sflags, aflags); 1104a553d4b8SJeff Roberson if (mem == NULL) { 1105b23f72e9SBrian Feldman if (keg->uk_flags & UMA_ZONE_OFFPAGE) 11060095a784SJeff Roberson zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); 1107fc03d22bSJeff Roberson slab = NULL; 1108fc03d22bSJeff Roberson goto out; 1109a553d4b8SJeff Roberson } 11102e47807cSJeff Roberson uma_total_inc(size); 11118355f576SJeff Roberson 11125c0e403bSJeff Roberson /* Point the slab into the allocated memory */ 1113099a0e58SBosko Milekic if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) 1114099a0e58SBosko Milekic slab = (uma_slab_t )(mem + keg->uk_pgoff); 11155c0e403bSJeff Roberson 1116e20a199fSJeff Roberson if (keg->uk_flags & UMA_ZONE_VTOSLAB) 1117099a0e58SBosko Milekic for (i = 0; i < keg->uk_ppera; i++) 111899571dc3SJeff Roberson vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab); 11198355f576SJeff Roberson 1120099a0e58SBosko Milekic slab->us_keg = keg; 11218355f576SJeff Roberson slab->us_data = mem; 1122099a0e58SBosko Milekic slab->us_freecount = keg->uk_ipers; 112386220393SMark Johnston slab->us_flags = sflags; 1124ab3185d1SJeff Roberson slab->us_domain = domain; 1125ef72505eSJeff Roberson BIT_FILL(SLAB_SETSIZE, &slab->us_free); 1126ef72505eSJeff Roberson #ifdef INVARIANTS 1127ef72505eSJeff Roberson BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree); 1128ef72505eSJeff Roberson #endif 1129099a0e58SBosko Milekic 1130b23f72e9SBrian Feldman if (keg->uk_init != NULL) { 1131099a0e58SBosko Milekic for (i = 0; i < keg->uk_ipers; i++) 1132b23f72e9SBrian Feldman if (keg->uk_init(slab->us_data + (keg->uk_rsize * i), 113386220393SMark Johnston keg->uk_size, flags) != 0) 1134b23f72e9SBrian Feldman break; 1135b23f72e9SBrian Feldman if (i != keg->uk_ipers) { 1136fc03d22bSJeff Roberson keg_free_slab(keg, slab, i); 1137fc03d22bSJeff Roberson slab = NULL; 1138fc03d22bSJeff Roberson goto out; 1139b23f72e9SBrian Feldman } 1140b23f72e9SBrian Feldman } 1141e20a199fSJeff Roberson KEG_LOCK(keg); 11425c0e403bSJeff Roberson 11431431a748SGleb Smirnoff CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)", 11441431a748SGleb Smirnoff slab, keg->uk_name, keg); 11451431a748SGleb Smirnoff 1146099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_HASH) 1147099a0e58SBosko Milekic UMA_HASH_INSERT(&keg->uk_hash, slab, mem); 11488355f576SJeff Roberson 1149099a0e58SBosko Milekic keg->uk_pages += keg->uk_ppera; 1150099a0e58SBosko Milekic keg->uk_free += keg->uk_ipers; 11518355f576SJeff Roberson 1152194a979eSMark Johnston out: 11538355f576SJeff Roberson return (slab); 11548355f576SJeff Roberson } 11558355f576SJeff Roberson 11568355f576SJeff Roberson /* 1157009b6fcbSJeff Roberson * This function is intended to be used early on in place of page_alloc() so 1158009b6fcbSJeff Roberson * that we may use the boot time page cache to satisfy allocations before 1159009b6fcbSJeff Roberson * the VM is ready. 1160009b6fcbSJeff Roberson */ 1161009b6fcbSJeff Roberson static void * 1162ab3185d1SJeff Roberson startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 1163ab3185d1SJeff Roberson int wait) 1164009b6fcbSJeff Roberson { 1165099a0e58SBosko Milekic uma_keg_t keg; 1166ac0a6fd0SGleb Smirnoff void *mem; 1167ac0a6fd0SGleb Smirnoff int pages; 1168099a0e58SBosko Milekic 1169bb15d1c7SGleb Smirnoff keg = zone->uz_keg; 1170009b6fcbSJeff Roberson /* 1171f7d35785SGleb Smirnoff * If we are in BOOT_BUCKETS or higher, than switch to real 1172f7d35785SGleb Smirnoff * allocator. Zones with page sized slabs switch at BOOT_PAGEALLOC. 1173009b6fcbSJeff Roberson */ 1174f7d35785SGleb Smirnoff switch (booted) { 1175f7d35785SGleb Smirnoff case BOOT_COLD: 1176f7d35785SGleb Smirnoff case BOOT_STRAPPED: 1177f7d35785SGleb Smirnoff break; 1178f7d35785SGleb Smirnoff case BOOT_PAGEALLOC: 1179f7d35785SGleb Smirnoff if (keg->uk_ppera > 1) 1180f7d35785SGleb Smirnoff break; 1181f7d35785SGleb Smirnoff case BOOT_BUCKETS: 1182f7d35785SGleb Smirnoff case BOOT_RUNNING: 1183009b6fcbSJeff Roberson #ifdef UMA_MD_SMALL_ALLOC 1184f7d35785SGleb Smirnoff keg->uk_allocf = (keg->uk_ppera > 1) ? 1185f7d35785SGleb Smirnoff page_alloc : uma_small_alloc; 1186009b6fcbSJeff Roberson #else 1187099a0e58SBosko Milekic keg->uk_allocf = page_alloc; 1188009b6fcbSJeff Roberson #endif 1189ab3185d1SJeff Roberson return keg->uk_allocf(zone, bytes, domain, pflag, wait); 1190009b6fcbSJeff Roberson } 1191009b6fcbSJeff Roberson 1192009b6fcbSJeff Roberson /* 1193f7d35785SGleb Smirnoff * Check our small startup cache to see if it has pages remaining. 1194f7d35785SGleb Smirnoff */ 1195f7d35785SGleb Smirnoff pages = howmany(bytes, PAGE_SIZE); 1196f7d35785SGleb Smirnoff KASSERT(pages > 0, ("%s can't reserve 0 pages", __func__)); 1197f7d35785SGleb Smirnoff if (pages > boot_pages) 1198f7d35785SGleb Smirnoff panic("UMA zone \"%s\": Increase vm.boot_pages", zone->uz_name); 1199f7d35785SGleb Smirnoff #ifdef DIAGNOSTIC 1200f7d35785SGleb Smirnoff printf("%s from \"%s\", %d boot pages left\n", __func__, zone->uz_name, 1201f7d35785SGleb Smirnoff boot_pages); 1202f7d35785SGleb Smirnoff #endif 1203f7d35785SGleb Smirnoff mem = bootmem; 1204f7d35785SGleb Smirnoff boot_pages -= pages; 1205f7d35785SGleb Smirnoff bootmem += pages * PAGE_SIZE; 1206f7d35785SGleb Smirnoff *pflag = UMA_SLAB_BOOT; 1207f7d35785SGleb Smirnoff 1208f7d35785SGleb Smirnoff return (mem); 1209f7d35785SGleb Smirnoff } 1210f7d35785SGleb Smirnoff 1211f7d35785SGleb Smirnoff /* 12128355f576SJeff Roberson * Allocates a number of pages from the system 12138355f576SJeff Roberson * 12148355f576SJeff Roberson * Arguments: 12158355f576SJeff Roberson * bytes The number of bytes requested 12168355f576SJeff Roberson * wait Shall we wait? 12178355f576SJeff Roberson * 12188355f576SJeff Roberson * Returns: 12198355f576SJeff Roberson * A pointer to the alloced memory or possibly 12208355f576SJeff Roberson * NULL if M_NOWAIT is set. 12218355f576SJeff Roberson */ 12228355f576SJeff Roberson static void * 1223ab3185d1SJeff Roberson page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 1224ab3185d1SJeff Roberson int wait) 12258355f576SJeff Roberson { 12268355f576SJeff Roberson void *p; /* Returned page */ 12278355f576SJeff Roberson 12282e47807cSJeff Roberson *pflag = UMA_SLAB_KERNEL; 12299978bd99SMark Johnston p = (void *)kmem_malloc_domainset(DOMAINSET_FIXED(domain), bytes, wait); 12308355f576SJeff Roberson 12318355f576SJeff Roberson return (p); 12328355f576SJeff Roberson } 12338355f576SJeff Roberson 1234ab3059a8SMatt Macy static void * 1235ab3059a8SMatt Macy pcpu_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 1236ab3059a8SMatt Macy int wait) 1237ab3059a8SMatt Macy { 1238ab3059a8SMatt Macy struct pglist alloctail; 1239ab3059a8SMatt Macy vm_offset_t addr, zkva; 1240ab3059a8SMatt Macy int cpu, flags; 1241ab3059a8SMatt Macy vm_page_t p, p_next; 1242ab3059a8SMatt Macy #ifdef NUMA 1243ab3059a8SMatt Macy struct pcpu *pc; 1244ab3059a8SMatt Macy #endif 1245ab3059a8SMatt Macy 1246ab3059a8SMatt Macy MPASS(bytes == (mp_maxid + 1) * PAGE_SIZE); 1247ab3059a8SMatt Macy 1248013072f0SMark Johnston TAILQ_INIT(&alloctail); 1249ab3059a8SMatt Macy flags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ | 1250013072f0SMark Johnston malloc2vm_flags(wait); 1251013072f0SMark Johnston *pflag = UMA_SLAB_KERNEL; 1252ab3059a8SMatt Macy for (cpu = 0; cpu <= mp_maxid; cpu++) { 1253ab3059a8SMatt Macy if (CPU_ABSENT(cpu)) { 1254ab3059a8SMatt Macy p = vm_page_alloc(NULL, 0, flags); 1255ab3059a8SMatt Macy } else { 1256ab3059a8SMatt Macy #ifndef NUMA 1257ab3059a8SMatt Macy p = vm_page_alloc(NULL, 0, flags); 1258ab3059a8SMatt Macy #else 1259ab3059a8SMatt Macy pc = pcpu_find(cpu); 1260ab3059a8SMatt Macy p = vm_page_alloc_domain(NULL, 0, pc->pc_domain, flags); 1261ab3059a8SMatt Macy if (__predict_false(p == NULL)) 1262ab3059a8SMatt Macy p = vm_page_alloc(NULL, 0, flags); 1263ab3059a8SMatt Macy #endif 1264ab3059a8SMatt Macy } 1265ab3059a8SMatt Macy if (__predict_false(p == NULL)) 1266ab3059a8SMatt Macy goto fail; 1267ab3059a8SMatt Macy TAILQ_INSERT_TAIL(&alloctail, p, listq); 1268ab3059a8SMatt Macy } 1269ab3059a8SMatt Macy if ((addr = kva_alloc(bytes)) == 0) 1270ab3059a8SMatt Macy goto fail; 1271ab3059a8SMatt Macy zkva = addr; 1272ab3059a8SMatt Macy TAILQ_FOREACH(p, &alloctail, listq) { 1273ab3059a8SMatt Macy pmap_qenter(zkva, &p, 1); 1274ab3059a8SMatt Macy zkva += PAGE_SIZE; 1275ab3059a8SMatt Macy } 1276ab3059a8SMatt Macy return ((void*)addr); 1277ab3059a8SMatt Macy fail: 1278ab3059a8SMatt Macy TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) { 1279ab3059a8SMatt Macy vm_page_unwire(p, PQ_NONE); 1280ab3059a8SMatt Macy vm_page_free(p); 1281ab3059a8SMatt Macy } 1282ab3059a8SMatt Macy return (NULL); 1283ab3059a8SMatt Macy } 1284ab3059a8SMatt Macy 12858355f576SJeff Roberson /* 12868355f576SJeff Roberson * Allocates a number of pages from within an object 12878355f576SJeff Roberson * 12888355f576SJeff Roberson * Arguments: 12898355f576SJeff Roberson * bytes The number of bytes requested 12908355f576SJeff Roberson * wait Shall we wait? 12918355f576SJeff Roberson * 12928355f576SJeff Roberson * Returns: 12938355f576SJeff Roberson * A pointer to the alloced memory or possibly 12948355f576SJeff Roberson * NULL if M_NOWAIT is set. 12958355f576SJeff Roberson */ 12968355f576SJeff Roberson static void * 1297ab3185d1SJeff Roberson noobj_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, 1298ab3185d1SJeff Roberson int wait) 12998355f576SJeff Roberson { 1300a4915c21SAttilio Rao TAILQ_HEAD(, vm_page) alloctail; 1301a4915c21SAttilio Rao u_long npages; 1302b245ac95SAlan Cox vm_offset_t retkva, zkva; 1303a4915c21SAttilio Rao vm_page_t p, p_next; 1304e20a199fSJeff Roberson uma_keg_t keg; 13058355f576SJeff Roberson 1306a4915c21SAttilio Rao TAILQ_INIT(&alloctail); 1307bb15d1c7SGleb Smirnoff keg = zone->uz_keg; 1308a4915c21SAttilio Rao 1309a4915c21SAttilio Rao npages = howmany(bytes, PAGE_SIZE); 1310a4915c21SAttilio Rao while (npages > 0) { 1311ab3185d1SJeff Roberson p = vm_page_alloc_domain(NULL, 0, domain, VM_ALLOC_INTERRUPT | 13128d6fbbb8SJeff Roberson VM_ALLOC_WIRED | VM_ALLOC_NOOBJ | 1313772c8b67SKonstantin Belousov ((wait & M_WAITOK) != 0 ? VM_ALLOC_WAITOK : 1314772c8b67SKonstantin Belousov VM_ALLOC_NOWAIT)); 1315a4915c21SAttilio Rao if (p != NULL) { 1316a4915c21SAttilio Rao /* 1317a4915c21SAttilio Rao * Since the page does not belong to an object, its 1318a4915c21SAttilio Rao * listq is unused. 1319a4915c21SAttilio Rao */ 1320a4915c21SAttilio Rao TAILQ_INSERT_TAIL(&alloctail, p, listq); 1321a4915c21SAttilio Rao npages--; 1322a4915c21SAttilio Rao continue; 1323a4915c21SAttilio Rao } 13248355f576SJeff Roberson /* 1325a4915c21SAttilio Rao * Page allocation failed, free intermediate pages and 1326a4915c21SAttilio Rao * exit. 13278355f576SJeff Roberson */ 1328a4915c21SAttilio Rao TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) { 1329087a6132SAlan Cox vm_page_unwire(p, PQ_NONE); 1330b245ac95SAlan Cox vm_page_free(p); 1331b245ac95SAlan Cox } 1332a4915c21SAttilio Rao return (NULL); 1333b245ac95SAlan Cox } 13348355f576SJeff Roberson *flags = UMA_SLAB_PRIV; 1335a4915c21SAttilio Rao zkva = keg->uk_kva + 1336a4915c21SAttilio Rao atomic_fetchadd_long(&keg->uk_offset, round_page(bytes)); 1337a4915c21SAttilio Rao retkva = zkva; 1338a4915c21SAttilio Rao TAILQ_FOREACH(p, &alloctail, listq) { 1339a4915c21SAttilio Rao pmap_qenter(zkva, &p, 1); 1340a4915c21SAttilio Rao zkva += PAGE_SIZE; 1341a4915c21SAttilio Rao } 13428355f576SJeff Roberson 13438355f576SJeff Roberson return ((void *)retkva); 13448355f576SJeff Roberson } 13458355f576SJeff Roberson 13468355f576SJeff Roberson /* 13478355f576SJeff Roberson * Frees a number of pages to the system 13488355f576SJeff Roberson * 13498355f576SJeff Roberson * Arguments: 13508355f576SJeff Roberson * mem A pointer to the memory to be freed 13518355f576SJeff Roberson * size The size of the memory being freed 13528355f576SJeff Roberson * flags The original p->us_flags field 13538355f576SJeff Roberson * 13548355f576SJeff Roberson * Returns: 13558355f576SJeff Roberson * Nothing 13568355f576SJeff Roberson */ 13578355f576SJeff Roberson static void 1358f2c2231eSRyan Stone page_free(void *mem, vm_size_t size, uint8_t flags) 13598355f576SJeff Roberson { 13603370c5bfSJeff Roberson 136149bfa624SAlan Cox if ((flags & UMA_SLAB_KERNEL) == 0) 1362b5345ef1SJustin Hibbits panic("UMA: page_free used with invalid flags %x", flags); 13638355f576SJeff Roberson 136449bfa624SAlan Cox kmem_free((vm_offset_t)mem, size); 13658355f576SJeff Roberson } 13668355f576SJeff Roberson 13678355f576SJeff Roberson /* 1368ab3059a8SMatt Macy * Frees pcpu zone allocations 1369ab3059a8SMatt Macy * 1370ab3059a8SMatt Macy * Arguments: 1371ab3059a8SMatt Macy * mem A pointer to the memory to be freed 1372ab3059a8SMatt Macy * size The size of the memory being freed 1373ab3059a8SMatt Macy * flags The original p->us_flags field 1374ab3059a8SMatt Macy * 1375ab3059a8SMatt Macy * Returns: 1376ab3059a8SMatt Macy * Nothing 1377ab3059a8SMatt Macy */ 1378ab3059a8SMatt Macy static void 1379ab3059a8SMatt Macy pcpu_page_free(void *mem, vm_size_t size, uint8_t flags) 1380ab3059a8SMatt Macy { 1381ab3059a8SMatt Macy vm_offset_t sva, curva; 1382ab3059a8SMatt Macy vm_paddr_t paddr; 1383ab3059a8SMatt Macy vm_page_t m; 1384ab3059a8SMatt Macy 1385ab3059a8SMatt Macy MPASS(size == (mp_maxid+1)*PAGE_SIZE); 1386ab3059a8SMatt Macy sva = (vm_offset_t)mem; 1387ab3059a8SMatt Macy for (curva = sva; curva < sva + size; curva += PAGE_SIZE) { 1388ab3059a8SMatt Macy paddr = pmap_kextract(curva); 1389ab3059a8SMatt Macy m = PHYS_TO_VM_PAGE(paddr); 1390ab3059a8SMatt Macy vm_page_unwire(m, PQ_NONE); 1391ab3059a8SMatt Macy vm_page_free(m); 1392ab3059a8SMatt Macy } 1393ab3059a8SMatt Macy pmap_qremove(sva, size >> PAGE_SHIFT); 1394ab3059a8SMatt Macy kva_free(sva, size); 1395ab3059a8SMatt Macy } 1396ab3059a8SMatt Macy 1397ab3059a8SMatt Macy 1398ab3059a8SMatt Macy /* 13998355f576SJeff Roberson * Zero fill initializer 14008355f576SJeff Roberson * 14018355f576SJeff Roberson * Arguments/Returns follow uma_init specifications 14028355f576SJeff Roberson */ 1403b23f72e9SBrian Feldman static int 1404b23f72e9SBrian Feldman zero_init(void *mem, int size, int flags) 14058355f576SJeff Roberson { 14068355f576SJeff Roberson bzero(mem, size); 1407b23f72e9SBrian Feldman return (0); 14088355f576SJeff Roberson } 14098355f576SJeff Roberson 14108355f576SJeff Roberson /* 1411e20a199fSJeff Roberson * Finish creating a small uma keg. This calculates ipers, and the keg size. 14128355f576SJeff Roberson * 14138355f576SJeff Roberson * Arguments 1414e20a199fSJeff Roberson * keg The zone we should initialize 14158355f576SJeff Roberson * 14168355f576SJeff Roberson * Returns 14178355f576SJeff Roberson * Nothing 14188355f576SJeff Roberson */ 14198355f576SJeff Roberson static void 1420e20a199fSJeff Roberson keg_small_init(uma_keg_t keg) 14218355f576SJeff Roberson { 1422244f4554SBosko Milekic u_int rsize; 1423244f4554SBosko Milekic u_int memused; 1424244f4554SBosko Milekic u_int wastedspace; 1425244f4554SBosko Milekic u_int shsize; 1426a55ebb7cSAndriy Gapon u_int slabsize; 14278355f576SJeff Roberson 1428ad97af7eSGleb Smirnoff if (keg->uk_flags & UMA_ZONE_PCPU) { 142996c85efbSNathan Whitehorn u_int ncpus = (mp_maxid + 1) ? (mp_maxid + 1) : MAXCPU; 1430e28a647dSGleb Smirnoff 1431ab3059a8SMatt Macy slabsize = UMA_PCPU_ALLOC_SIZE; 1432ab3059a8SMatt Macy keg->uk_ppera = ncpus; 1433ad97af7eSGleb Smirnoff } else { 1434a55ebb7cSAndriy Gapon slabsize = UMA_SLAB_SIZE; 1435ad97af7eSGleb Smirnoff keg->uk_ppera = 1; 1436ad97af7eSGleb Smirnoff } 1437ad97af7eSGleb Smirnoff 1438ef72505eSJeff Roberson /* 1439ef72505eSJeff Roberson * Calculate the size of each allocation (rsize) according to 1440ef72505eSJeff Roberson * alignment. If the requested size is smaller than we have 1441ef72505eSJeff Roberson * allocation bits for we round it up. 1442ef72505eSJeff Roberson */ 1443099a0e58SBosko Milekic rsize = keg->uk_size; 1444a55ebb7cSAndriy Gapon if (rsize < slabsize / SLAB_SETSIZE) 1445a55ebb7cSAndriy Gapon rsize = slabsize / SLAB_SETSIZE; 1446099a0e58SBosko Milekic if (rsize & keg->uk_align) 1447099a0e58SBosko Milekic rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1); 1448099a0e58SBosko Milekic keg->uk_rsize = rsize; 1449ad97af7eSGleb Smirnoff 1450ad97af7eSGleb Smirnoff KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || 1451ab3059a8SMatt Macy keg->uk_rsize < UMA_PCPU_ALLOC_SIZE, 1452ad97af7eSGleb Smirnoff ("%s: size %u too large", __func__, keg->uk_rsize)); 14538355f576SJeff Roberson 1454ef72505eSJeff Roberson if (keg->uk_flags & UMA_ZONE_OFFPAGE) 14552864dbbfSGleb Smirnoff shsize = 0; 1456ef72505eSJeff Roberson else 14573d5e3df7SGleb Smirnoff shsize = SIZEOF_UMA_SLAB; 14588355f576SJeff Roberson 14591ca6ed45SGleb Smirnoff if (rsize <= slabsize - shsize) 1460a55ebb7cSAndriy Gapon keg->uk_ipers = (slabsize - shsize) / rsize; 14611ca6ed45SGleb Smirnoff else { 14621ca6ed45SGleb Smirnoff /* Handle special case when we have 1 item per slab, so 14631ca6ed45SGleb Smirnoff * alignment requirement can be relaxed. */ 14641ca6ed45SGleb Smirnoff KASSERT(keg->uk_size <= slabsize - shsize, 14651ca6ed45SGleb Smirnoff ("%s: size %u greater than slab", __func__, keg->uk_size)); 14661ca6ed45SGleb Smirnoff keg->uk_ipers = 1; 14671ca6ed45SGleb Smirnoff } 1468ef72505eSJeff Roberson KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE, 1469ad97af7eSGleb Smirnoff ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); 1470ad97af7eSGleb Smirnoff 1471244f4554SBosko Milekic memused = keg->uk_ipers * rsize + shsize; 1472a55ebb7cSAndriy Gapon wastedspace = slabsize - memused; 1473244f4554SBosko Milekic 147420e8e865SBosko Milekic /* 1475244f4554SBosko Milekic * We can't do OFFPAGE if we're internal or if we've been 147620e8e865SBosko Milekic * asked to not go to the VM for buckets. If we do this we 14776fd34d6fSJeff Roberson * may end up going to the VM for slabs which we do not 14786fd34d6fSJeff Roberson * want to do if we're UMA_ZFLAG_CACHEONLY as a result 14796fd34d6fSJeff Roberson * of UMA_ZONE_VM, which clearly forbids it. 148020e8e865SBosko Milekic */ 1481099a0e58SBosko Milekic if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) || 1482099a0e58SBosko Milekic (keg->uk_flags & UMA_ZFLAG_CACHEONLY)) 14838355f576SJeff Roberson return; 1484244f4554SBosko Milekic 1485ef72505eSJeff Roberson /* 1486ef72505eSJeff Roberson * See if using an OFFPAGE slab will limit our waste. Only do 1487ef72505eSJeff Roberson * this if it permits more items per-slab. 1488ef72505eSJeff Roberson * 1489ef72505eSJeff Roberson * XXX We could try growing slabsize to limit max waste as well. 1490ef72505eSJeff Roberson * Historically this was not done because the VM could not 1491ef72505eSJeff Roberson * efficiently handle contiguous allocations. 1492ef72505eSJeff Roberson */ 1493a55ebb7cSAndriy Gapon if ((wastedspace >= slabsize / UMA_MAX_WASTE) && 1494a55ebb7cSAndriy Gapon (keg->uk_ipers < (slabsize / keg->uk_rsize))) { 1495a55ebb7cSAndriy Gapon keg->uk_ipers = slabsize / keg->uk_rsize; 1496ef72505eSJeff Roberson KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE, 1497ad97af7eSGleb Smirnoff ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); 14981431a748SGleb Smirnoff CTR6(KTR_UMA, "UMA decided we need offpage slab headers for " 14991431a748SGleb Smirnoff "keg: %s(%p), calculated wastedspace = %d, " 1500244f4554SBosko Milekic "maximum wasted space allowed = %d, " 1501244f4554SBosko Milekic "calculated ipers = %d, " 15021431a748SGleb Smirnoff "new wasted space = %d\n", keg->uk_name, keg, wastedspace, 1503a55ebb7cSAndriy Gapon slabsize / UMA_MAX_WASTE, keg->uk_ipers, 1504a55ebb7cSAndriy Gapon slabsize - keg->uk_ipers * keg->uk_rsize); 1505099a0e58SBosko Milekic keg->uk_flags |= UMA_ZONE_OFFPAGE; 15068355f576SJeff Roberson } 1507ad97af7eSGleb Smirnoff 1508ad97af7eSGleb Smirnoff if ((keg->uk_flags & UMA_ZONE_OFFPAGE) && 1509ad97af7eSGleb Smirnoff (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0) 1510ad97af7eSGleb Smirnoff keg->uk_flags |= UMA_ZONE_HASH; 15118355f576SJeff Roberson } 15128355f576SJeff Roberson 15138355f576SJeff Roberson /* 1514e20a199fSJeff Roberson * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do 15158355f576SJeff Roberson * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be 15168355f576SJeff Roberson * more complicated. 15178355f576SJeff Roberson * 15188355f576SJeff Roberson * Arguments 1519e20a199fSJeff Roberson * keg The keg we should initialize 15208355f576SJeff Roberson * 15218355f576SJeff Roberson * Returns 15228355f576SJeff Roberson * Nothing 15238355f576SJeff Roberson */ 15248355f576SJeff Roberson static void 1525e20a199fSJeff Roberson keg_large_init(uma_keg_t keg) 15268355f576SJeff Roberson { 15278355f576SJeff Roberson 1528e20a199fSJeff Roberson KASSERT(keg != NULL, ("Keg is null in keg_large_init")); 1529ad97af7eSGleb Smirnoff KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0, 1530ad97af7eSGleb Smirnoff ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__)); 153120e8e865SBosko Milekic 1532ad97af7eSGleb Smirnoff keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE); 1533099a0e58SBosko Milekic keg->uk_ipers = 1; 1534e9a069d8SJohn Baldwin keg->uk_rsize = keg->uk_size; 1535e9a069d8SJohn Baldwin 1536cec48e00SAlexander Motin /* Check whether we have enough space to not do OFFPAGE. */ 15373d5e3df7SGleb Smirnoff if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0 && 15383d5e3df7SGleb Smirnoff PAGE_SIZE * keg->uk_ppera - keg->uk_rsize < SIZEOF_UMA_SLAB) { 15392934eb8aSMark Johnston /* 15402934eb8aSMark Johnston * We can't do OFFPAGE if we're internal, in which case 15412934eb8aSMark Johnston * we need an extra page per allocation to contain the 15422934eb8aSMark Johnston * slab header. 15432934eb8aSMark Johnston */ 15442934eb8aSMark Johnston if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) == 0) 1545099a0e58SBosko Milekic keg->uk_flags |= UMA_ZONE_OFFPAGE; 15462934eb8aSMark Johnston else 15472934eb8aSMark Johnston keg->uk_ppera++; 15482934eb8aSMark Johnston } 1549cec48e00SAlexander Motin 1550cec48e00SAlexander Motin if ((keg->uk_flags & UMA_ZONE_OFFPAGE) && 1551cec48e00SAlexander Motin (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0) 1552099a0e58SBosko Milekic keg->uk_flags |= UMA_ZONE_HASH; 15538355f576SJeff Roberson } 15548355f576SJeff Roberson 1555e20a199fSJeff Roberson static void 1556e20a199fSJeff Roberson keg_cachespread_init(uma_keg_t keg) 1557e20a199fSJeff Roberson { 1558e20a199fSJeff Roberson int alignsize; 1559e20a199fSJeff Roberson int trailer; 1560e20a199fSJeff Roberson int pages; 1561e20a199fSJeff Roberson int rsize; 1562e20a199fSJeff Roberson 1563ad97af7eSGleb Smirnoff KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0, 1564ad97af7eSGleb Smirnoff ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__)); 1565ad97af7eSGleb Smirnoff 1566e20a199fSJeff Roberson alignsize = keg->uk_align + 1; 1567e20a199fSJeff Roberson rsize = keg->uk_size; 1568e20a199fSJeff Roberson /* 1569e20a199fSJeff Roberson * We want one item to start on every align boundary in a page. To 1570e20a199fSJeff Roberson * do this we will span pages. We will also extend the item by the 1571e20a199fSJeff Roberson * size of align if it is an even multiple of align. Otherwise, it 1572e20a199fSJeff Roberson * would fall on the same boundary every time. 1573e20a199fSJeff Roberson */ 1574e20a199fSJeff Roberson if (rsize & keg->uk_align) 1575e20a199fSJeff Roberson rsize = (rsize & ~keg->uk_align) + alignsize; 1576e20a199fSJeff Roberson if ((rsize & alignsize) == 0) 1577e20a199fSJeff Roberson rsize += alignsize; 1578e20a199fSJeff Roberson trailer = rsize - keg->uk_size; 1579e20a199fSJeff Roberson pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE; 1580e20a199fSJeff Roberson pages = MIN(pages, (128 * 1024) / PAGE_SIZE); 1581e20a199fSJeff Roberson keg->uk_rsize = rsize; 1582e20a199fSJeff Roberson keg->uk_ppera = pages; 1583e20a199fSJeff Roberson keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize; 1584e20a199fSJeff Roberson keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB; 15852367b4ddSDimitry Andric KASSERT(keg->uk_ipers <= SLAB_SETSIZE, 158642321809SGleb Smirnoff ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__, 1587e20a199fSJeff Roberson keg->uk_ipers)); 1588e20a199fSJeff Roberson } 1589e20a199fSJeff Roberson 15908355f576SJeff Roberson /* 1591099a0e58SBosko Milekic * Keg header ctor. This initializes all fields, locks, etc. And inserts 1592099a0e58SBosko Milekic * the keg onto the global keg list. 15938355f576SJeff Roberson * 15948355f576SJeff Roberson * Arguments/Returns follow uma_ctor specifications 1595099a0e58SBosko Milekic * udata Actually uma_kctor_args 1596099a0e58SBosko Milekic */ 1597b23f72e9SBrian Feldman static int 1598b23f72e9SBrian Feldman keg_ctor(void *mem, int size, void *udata, int flags) 1599099a0e58SBosko Milekic { 1600099a0e58SBosko Milekic struct uma_kctor_args *arg = udata; 1601099a0e58SBosko Milekic uma_keg_t keg = mem; 1602099a0e58SBosko Milekic uma_zone_t zone; 1603099a0e58SBosko Milekic 1604099a0e58SBosko Milekic bzero(keg, size); 1605099a0e58SBosko Milekic keg->uk_size = arg->size; 1606099a0e58SBosko Milekic keg->uk_init = arg->uminit; 1607099a0e58SBosko Milekic keg->uk_fini = arg->fini; 1608099a0e58SBosko Milekic keg->uk_align = arg->align; 1609099a0e58SBosko Milekic keg->uk_free = 0; 16106fd34d6fSJeff Roberson keg->uk_reserve = 0; 1611099a0e58SBosko Milekic keg->uk_pages = 0; 1612099a0e58SBosko Milekic keg->uk_flags = arg->flags; 1613099a0e58SBosko Milekic keg->uk_slabzone = NULL; 1614099a0e58SBosko Milekic 1615099a0e58SBosko Milekic /* 1616194a979eSMark Johnston * We use a global round-robin policy by default. Zones with 1617194a979eSMark Johnston * UMA_ZONE_NUMA set will use first-touch instead, in which case the 1618194a979eSMark Johnston * iterator is never run. 1619194a979eSMark Johnston */ 1620194a979eSMark Johnston keg->uk_dr.dr_policy = DOMAINSET_RR(); 1621194a979eSMark Johnston keg->uk_dr.dr_iter = 0; 1622194a979eSMark Johnston 1623194a979eSMark Johnston /* 1624099a0e58SBosko Milekic * The master zone is passed to us at keg-creation time. 1625099a0e58SBosko Milekic */ 1626099a0e58SBosko Milekic zone = arg->zone; 1627e20a199fSJeff Roberson keg->uk_name = zone->uz_name; 1628099a0e58SBosko Milekic 1629099a0e58SBosko Milekic if (arg->flags & UMA_ZONE_VM) 1630099a0e58SBosko Milekic keg->uk_flags |= UMA_ZFLAG_CACHEONLY; 1631099a0e58SBosko Milekic 1632099a0e58SBosko Milekic if (arg->flags & UMA_ZONE_ZINIT) 1633099a0e58SBosko Milekic keg->uk_init = zero_init; 1634099a0e58SBosko Milekic 1635cfcae3f8SGleb Smirnoff if (arg->flags & UMA_ZONE_MALLOC) 1636e20a199fSJeff Roberson keg->uk_flags |= UMA_ZONE_VTOSLAB; 1637e20a199fSJeff Roberson 1638ad97af7eSGleb Smirnoff if (arg->flags & UMA_ZONE_PCPU) 1639ad97af7eSGleb Smirnoff #ifdef SMP 1640ad97af7eSGleb Smirnoff keg->uk_flags |= UMA_ZONE_OFFPAGE; 1641ad97af7eSGleb Smirnoff #else 1642ad97af7eSGleb Smirnoff keg->uk_flags &= ~UMA_ZONE_PCPU; 1643ad97af7eSGleb Smirnoff #endif 1644ad97af7eSGleb Smirnoff 1645ef72505eSJeff Roberson if (keg->uk_flags & UMA_ZONE_CACHESPREAD) { 1646e20a199fSJeff Roberson keg_cachespread_init(keg); 1647244f4554SBosko Milekic } else { 1648b92b26adSGleb Smirnoff if (keg->uk_size > UMA_SLAB_SPACE) 1649e20a199fSJeff Roberson keg_large_init(keg); 1650244f4554SBosko Milekic else 1651e20a199fSJeff Roberson keg_small_init(keg); 1652244f4554SBosko Milekic } 1653099a0e58SBosko Milekic 1654cfcae3f8SGleb Smirnoff if (keg->uk_flags & UMA_ZONE_OFFPAGE) 1655099a0e58SBosko Milekic keg->uk_slabzone = slabzone; 1656099a0e58SBosko Milekic 1657099a0e58SBosko Milekic /* 1658099a0e58SBosko Milekic * If we haven't booted yet we need allocations to go through the 1659099a0e58SBosko Milekic * startup cache until the vm is ready. 1660099a0e58SBosko Milekic */ 1661f4bef67cSGleb Smirnoff if (booted < BOOT_PAGEALLOC) 16628cd02d00SAlan Cox keg->uk_allocf = startup_alloc; 166377e19437SGleb Smirnoff #ifdef UMA_MD_SMALL_ALLOC 166477e19437SGleb Smirnoff else if (keg->uk_ppera == 1) 166577e19437SGleb Smirnoff keg->uk_allocf = uma_small_alloc; 16668cd02d00SAlan Cox #endif 1667ab3059a8SMatt Macy else if (keg->uk_flags & UMA_ZONE_PCPU) 1668ab3059a8SMatt Macy keg->uk_allocf = pcpu_page_alloc; 166977e19437SGleb Smirnoff else 167077e19437SGleb Smirnoff keg->uk_allocf = page_alloc; 167177e19437SGleb Smirnoff #ifdef UMA_MD_SMALL_ALLOC 167277e19437SGleb Smirnoff if (keg->uk_ppera == 1) 167377e19437SGleb Smirnoff keg->uk_freef = uma_small_free; 167477e19437SGleb Smirnoff else 167577e19437SGleb Smirnoff #endif 1676ab3059a8SMatt Macy if (keg->uk_flags & UMA_ZONE_PCPU) 1677ab3059a8SMatt Macy keg->uk_freef = pcpu_page_free; 1678ab3059a8SMatt Macy else 167977e19437SGleb Smirnoff keg->uk_freef = page_free; 1680099a0e58SBosko Milekic 1681099a0e58SBosko Milekic /* 1682af526374SJeff Roberson * Initialize keg's lock 1683099a0e58SBosko Milekic */ 1684af526374SJeff Roberson KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS)); 1685099a0e58SBosko Milekic 1686099a0e58SBosko Milekic /* 1687099a0e58SBosko Milekic * If we're putting the slab header in the actual page we need to 16883d5e3df7SGleb Smirnoff * figure out where in each page it goes. See SIZEOF_UMA_SLAB 16893d5e3df7SGleb Smirnoff * macro definition. 1690099a0e58SBosko Milekic */ 1691099a0e58SBosko Milekic if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) { 16923d5e3df7SGleb Smirnoff keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - SIZEOF_UMA_SLAB; 1693244f4554SBosko Milekic /* 1694244f4554SBosko Milekic * The only way the following is possible is if with our 1695244f4554SBosko Milekic * UMA_ALIGN_PTR adjustments we are now bigger than 1696244f4554SBosko Milekic * UMA_SLAB_SIZE. I haven't checked whether this is 1697244f4554SBosko Milekic * mathematically possible for all cases, so we make 1698244f4554SBosko Milekic * sure here anyway. 1699244f4554SBosko Milekic */ 17003d5e3df7SGleb Smirnoff KASSERT(keg->uk_pgoff + sizeof(struct uma_slab) <= 17013d5e3df7SGleb Smirnoff PAGE_SIZE * keg->uk_ppera, 17023d5e3df7SGleb Smirnoff ("zone %s ipers %d rsize %d size %d slab won't fit", 17033d5e3df7SGleb Smirnoff zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); 1704099a0e58SBosko Milekic } 1705099a0e58SBosko Milekic 1706099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_HASH) 1707099a0e58SBosko Milekic hash_alloc(&keg->uk_hash); 1708099a0e58SBosko Milekic 17091431a748SGleb Smirnoff CTR5(KTR_UMA, "keg_ctor %p zone %s(%p) out %d free %d\n", 17101431a748SGleb Smirnoff keg, zone->uz_name, zone, 171157223e99SAndriy Gapon (keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free, 171257223e99SAndriy Gapon keg->uk_free); 1713099a0e58SBosko Milekic 1714099a0e58SBosko Milekic LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); 1715099a0e58SBosko Milekic 1716111fbcd5SBryan Venteicher rw_wlock(&uma_rwlock); 1717099a0e58SBosko Milekic LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); 1718111fbcd5SBryan Venteicher rw_wunlock(&uma_rwlock); 1719b23f72e9SBrian Feldman return (0); 1720099a0e58SBosko Milekic } 1721099a0e58SBosko Milekic 17222efcc8cbSGleb Smirnoff static void 17232efcc8cbSGleb Smirnoff zone_alloc_counters(uma_zone_t zone) 17242efcc8cbSGleb Smirnoff { 17252efcc8cbSGleb Smirnoff 17262efcc8cbSGleb Smirnoff zone->uz_allocs = counter_u64_alloc(M_WAITOK); 17272efcc8cbSGleb Smirnoff zone->uz_frees = counter_u64_alloc(M_WAITOK); 17282efcc8cbSGleb Smirnoff zone->uz_fails = counter_u64_alloc(M_WAITOK); 17292efcc8cbSGleb Smirnoff } 17302efcc8cbSGleb Smirnoff 1731099a0e58SBosko Milekic /* 1732099a0e58SBosko Milekic * Zone header ctor. This initializes all fields, locks, etc. 1733099a0e58SBosko Milekic * 1734099a0e58SBosko Milekic * Arguments/Returns follow uma_ctor specifications 1735099a0e58SBosko Milekic * udata Actually uma_zctor_args 17368355f576SJeff Roberson */ 1737b23f72e9SBrian Feldman static int 1738b23f72e9SBrian Feldman zone_ctor(void *mem, int size, void *udata, int flags) 17398355f576SJeff Roberson { 17408355f576SJeff Roberson struct uma_zctor_args *arg = udata; 17418355f576SJeff Roberson uma_zone_t zone = mem; 1742099a0e58SBosko Milekic uma_zone_t z; 1743099a0e58SBosko Milekic uma_keg_t keg; 17448355f576SJeff Roberson 17458355f576SJeff Roberson bzero(zone, size); 17468355f576SJeff Roberson zone->uz_name = arg->name; 17478355f576SJeff Roberson zone->uz_ctor = arg->ctor; 17488355f576SJeff Roberson zone->uz_dtor = arg->dtor; 1749099a0e58SBosko Milekic zone->uz_init = NULL; 1750099a0e58SBosko Milekic zone->uz_fini = NULL; 1751bf965959SSean Bruno zone->uz_sleeps = 0; 1752fc03d22bSJeff Roberson zone->uz_count = 0; 1753ace66b56SAlexander Motin zone->uz_count_min = 0; 1754bb15d1c7SGleb Smirnoff zone->uz_count_max = BUCKET_MAX; 1755e20a199fSJeff Roberson zone->uz_flags = 0; 17562f891cd5SPawel Jakub Dawidek zone->uz_warning = NULL; 1757ab3185d1SJeff Roberson /* The domain structures follow the cpu structures. */ 1758ab3185d1SJeff Roberson zone->uz_domain = (struct uma_zone_domain *)&zone->uz_cpu[mp_ncpus]; 1759bb15d1c7SGleb Smirnoff zone->uz_bkt_max = ULONG_MAX; 17602f891cd5SPawel Jakub Dawidek timevalclear(&zone->uz_ratecheck); 1761af526374SJeff Roberson 17622efcc8cbSGleb Smirnoff if (__predict_true(booted == BOOT_RUNNING)) 17632efcc8cbSGleb Smirnoff zone_alloc_counters(zone); 17642efcc8cbSGleb Smirnoff else { 17652efcc8cbSGleb Smirnoff zone->uz_allocs = EARLY_COUNTER; 17662efcc8cbSGleb Smirnoff zone->uz_frees = EARLY_COUNTER; 17672efcc8cbSGleb Smirnoff zone->uz_fails = EARLY_COUNTER; 17682efcc8cbSGleb Smirnoff } 17692efcc8cbSGleb Smirnoff 17700095a784SJeff Roberson /* 17710095a784SJeff Roberson * This is a pure cache zone, no kegs. 17720095a784SJeff Roberson */ 17730095a784SJeff Roberson if (arg->import) { 17746fd34d6fSJeff Roberson if (arg->flags & UMA_ZONE_VM) 17756fd34d6fSJeff Roberson arg->flags |= UMA_ZFLAG_CACHEONLY; 17766fd34d6fSJeff Roberson zone->uz_flags = arg->flags; 1777af526374SJeff Roberson zone->uz_size = arg->size; 17780095a784SJeff Roberson zone->uz_import = arg->import; 17790095a784SJeff Roberson zone->uz_release = arg->release; 17800095a784SJeff Roberson zone->uz_arg = arg->arg; 1781af526374SJeff Roberson zone->uz_lockptr = &zone->uz_lock; 1782bb15d1c7SGleb Smirnoff ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS)); 1783111fbcd5SBryan Venteicher rw_wlock(&uma_rwlock); 178403175483SAlexander Motin LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link); 1785111fbcd5SBryan Venteicher rw_wunlock(&uma_rwlock); 1786af526374SJeff Roberson goto out; 17870095a784SJeff Roberson } 17880095a784SJeff Roberson 17890095a784SJeff Roberson /* 17900095a784SJeff Roberson * Use the regular zone/keg/slab allocator. 17910095a784SJeff Roberson */ 17920095a784SJeff Roberson zone->uz_import = (uma_import)zone_import; 17930095a784SJeff Roberson zone->uz_release = (uma_release)zone_release; 17940095a784SJeff Roberson zone->uz_arg = zone; 1795bb15d1c7SGleb Smirnoff keg = arg->keg; 17960095a784SJeff Roberson 1797099a0e58SBosko Milekic if (arg->flags & UMA_ZONE_SECONDARY) { 1798099a0e58SBosko Milekic KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); 17998355f576SJeff Roberson zone->uz_init = arg->uminit; 1800e221e841SJeff Roberson zone->uz_fini = arg->fini; 1801af526374SJeff Roberson zone->uz_lockptr = &keg->uk_lock; 1802e20a199fSJeff Roberson zone->uz_flags |= UMA_ZONE_SECONDARY; 1803111fbcd5SBryan Venteicher rw_wlock(&uma_rwlock); 1804099a0e58SBosko Milekic ZONE_LOCK(zone); 1805099a0e58SBosko Milekic LIST_FOREACH(z, &keg->uk_zones, uz_link) { 1806099a0e58SBosko Milekic if (LIST_NEXT(z, uz_link) == NULL) { 1807099a0e58SBosko Milekic LIST_INSERT_AFTER(z, zone, uz_link); 1808099a0e58SBosko Milekic break; 1809099a0e58SBosko Milekic } 1810099a0e58SBosko Milekic } 1811099a0e58SBosko Milekic ZONE_UNLOCK(zone); 1812111fbcd5SBryan Venteicher rw_wunlock(&uma_rwlock); 1813e20a199fSJeff Roberson } else if (keg == NULL) { 1814e20a199fSJeff Roberson if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini, 1815e20a199fSJeff Roberson arg->align, arg->flags)) == NULL) 1816b23f72e9SBrian Feldman return (ENOMEM); 1817099a0e58SBosko Milekic } else { 1818099a0e58SBosko Milekic struct uma_kctor_args karg; 1819b23f72e9SBrian Feldman int error; 1820099a0e58SBosko Milekic 1821099a0e58SBosko Milekic /* We should only be here from uma_startup() */ 1822099a0e58SBosko Milekic karg.size = arg->size; 1823099a0e58SBosko Milekic karg.uminit = arg->uminit; 1824099a0e58SBosko Milekic karg.fini = arg->fini; 1825099a0e58SBosko Milekic karg.align = arg->align; 1826099a0e58SBosko Milekic karg.flags = arg->flags; 1827099a0e58SBosko Milekic karg.zone = zone; 1828b23f72e9SBrian Feldman error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, 1829b23f72e9SBrian Feldman flags); 1830b23f72e9SBrian Feldman if (error) 1831b23f72e9SBrian Feldman return (error); 1832099a0e58SBosko Milekic } 18330095a784SJeff Roberson 1834bb15d1c7SGleb Smirnoff zone->uz_keg = keg; 1835e20a199fSJeff Roberson zone->uz_size = keg->uk_size; 1836e20a199fSJeff Roberson zone->uz_flags |= (keg->uk_flags & 1837e20a199fSJeff Roberson (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT)); 18388355f576SJeff Roberson 18398355f576SJeff Roberson /* 18408355f576SJeff Roberson * Some internal zones don't have room allocated for the per cpu 18418355f576SJeff Roberson * caches. If we're internal, bail out here. 18428355f576SJeff Roberson */ 1843099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZFLAG_INTERNAL) { 1844e20a199fSJeff Roberson KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0, 1845099a0e58SBosko Milekic ("Secondary zone requested UMA_ZFLAG_INTERNAL")); 1846b23f72e9SBrian Feldman return (0); 1847099a0e58SBosko Milekic } 18488355f576SJeff Roberson 1849af526374SJeff Roberson out: 18507e28037aSMark Johnston KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) != 18517e28037aSMark Johnston (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET), 18527e28037aSMark Johnston ("Invalid zone flag combination")); 18537e28037aSMark Johnston if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) 1854cae33c14SJeff Roberson zone->uz_count = BUCKET_MAX; 18557e28037aSMark Johnston else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0) 18567e28037aSMark Johnston zone->uz_count = 0; 18577e28037aSMark Johnston else 18587e28037aSMark Johnston zone->uz_count = bucket_select(zone->uz_size); 1859ace66b56SAlexander Motin zone->uz_count_min = zone->uz_count; 1860fc03d22bSJeff Roberson 1861b23f72e9SBrian Feldman return (0); 18628355f576SJeff Roberson } 18638355f576SJeff Roberson 18648355f576SJeff Roberson /* 1865099a0e58SBosko Milekic * Keg header dtor. This frees all data, destroys locks, frees the hash 1866099a0e58SBosko Milekic * table and removes the keg from the global list. 18679c2cd7e5SJeff Roberson * 18689c2cd7e5SJeff Roberson * Arguments/Returns follow uma_dtor specifications 18699c2cd7e5SJeff Roberson * udata unused 18709c2cd7e5SJeff Roberson */ 1871099a0e58SBosko Milekic static void 1872099a0e58SBosko Milekic keg_dtor(void *arg, int size, void *udata) 1873099a0e58SBosko Milekic { 1874099a0e58SBosko Milekic uma_keg_t keg; 18759c2cd7e5SJeff Roberson 1876099a0e58SBosko Milekic keg = (uma_keg_t)arg; 1877e20a199fSJeff Roberson KEG_LOCK(keg); 1878099a0e58SBosko Milekic if (keg->uk_free != 0) { 1879a3845534SCraig Rodrigues printf("Freed UMA keg (%s) was not empty (%d items). " 1880099a0e58SBosko Milekic " Lost %d pages of memory.\n", 1881a3845534SCraig Rodrigues keg->uk_name ? keg->uk_name : "", 1882099a0e58SBosko Milekic keg->uk_free, keg->uk_pages); 1883099a0e58SBosko Milekic } 1884e20a199fSJeff Roberson KEG_UNLOCK(keg); 1885099a0e58SBosko Milekic 1886099a0e58SBosko Milekic hash_free(&keg->uk_hash); 1887099a0e58SBosko Milekic 1888e20a199fSJeff Roberson KEG_LOCK_FINI(keg); 1889099a0e58SBosko Milekic } 1890099a0e58SBosko Milekic 1891099a0e58SBosko Milekic /* 1892099a0e58SBosko Milekic * Zone header dtor. 1893099a0e58SBosko Milekic * 1894099a0e58SBosko Milekic * Arguments/Returns follow uma_dtor specifications 1895099a0e58SBosko Milekic * udata unused 1896099a0e58SBosko Milekic */ 18979c2cd7e5SJeff Roberson static void 18989c2cd7e5SJeff Roberson zone_dtor(void *arg, int size, void *udata) 18999c2cd7e5SJeff Roberson { 19009c2cd7e5SJeff Roberson uma_zone_t zone; 1901099a0e58SBosko Milekic uma_keg_t keg; 19029c2cd7e5SJeff Roberson 19039c2cd7e5SJeff Roberson zone = (uma_zone_t)arg; 19049643769aSJeff Roberson 1905e20a199fSJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) 19069643769aSJeff Roberson cache_drain(zone); 1907099a0e58SBosko Milekic 1908111fbcd5SBryan Venteicher rw_wlock(&uma_rwlock); 1909099a0e58SBosko Milekic LIST_REMOVE(zone, uz_link); 1910111fbcd5SBryan Venteicher rw_wunlock(&uma_rwlock); 1911099a0e58SBosko Milekic /* 1912099a0e58SBosko Milekic * XXX there are some races here where 1913099a0e58SBosko Milekic * the zone can be drained but zone lock 1914099a0e58SBosko Milekic * released and then refilled before we 1915099a0e58SBosko Milekic * remove it... we dont care for now 1916099a0e58SBosko Milekic */ 1917e20a199fSJeff Roberson zone_drain_wait(zone, M_WAITOK); 1918e20a199fSJeff Roberson /* 1919323ad386STycho Nightingale * We only destroy kegs from non secondary/non cache zones. 1920e20a199fSJeff Roberson */ 1921323ad386STycho Nightingale if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) { 1922323ad386STycho Nightingale keg = zone->uz_keg; 1923111fbcd5SBryan Venteicher rw_wlock(&uma_rwlock); 1924099a0e58SBosko Milekic LIST_REMOVE(keg, uk_link); 1925111fbcd5SBryan Venteicher rw_wunlock(&uma_rwlock); 19260095a784SJeff Roberson zone_free_item(kegs, keg, NULL, SKIP_NONE); 19279c2cd7e5SJeff Roberson } 19282efcc8cbSGleb Smirnoff counter_u64_free(zone->uz_allocs); 19292efcc8cbSGleb Smirnoff counter_u64_free(zone->uz_frees); 19302efcc8cbSGleb Smirnoff counter_u64_free(zone->uz_fails); 1931bb15d1c7SGleb Smirnoff if (zone->uz_lockptr == &zone->uz_lock) 1932af526374SJeff Roberson ZONE_LOCK_FINI(zone); 1933099a0e58SBosko Milekic } 1934099a0e58SBosko Milekic 19359c2cd7e5SJeff Roberson /* 19368355f576SJeff Roberson * Traverses every zone in the system and calls a callback 19378355f576SJeff Roberson * 19388355f576SJeff Roberson * Arguments: 19398355f576SJeff Roberson * zfunc A pointer to a function which accepts a zone 19408355f576SJeff Roberson * as an argument. 19418355f576SJeff Roberson * 19428355f576SJeff Roberson * Returns: 19438355f576SJeff Roberson * Nothing 19448355f576SJeff Roberson */ 19458355f576SJeff Roberson static void 19468355f576SJeff Roberson zone_foreach(void (*zfunc)(uma_zone_t)) 19478355f576SJeff Roberson { 1948099a0e58SBosko Milekic uma_keg_t keg; 19498355f576SJeff Roberson uma_zone_t zone; 19508355f576SJeff Roberson 19512efcc8cbSGleb Smirnoff /* 19522efcc8cbSGleb Smirnoff * Before BOOT_RUNNING we are guaranteed to be single 19532efcc8cbSGleb Smirnoff * threaded, so locking isn't needed. Startup functions 19542efcc8cbSGleb Smirnoff * are allowed to use M_WAITOK. 19552efcc8cbSGleb Smirnoff */ 19562efcc8cbSGleb Smirnoff if (__predict_true(booted == BOOT_RUNNING)) 1957111fbcd5SBryan Venteicher rw_rlock(&uma_rwlock); 1958099a0e58SBosko Milekic LIST_FOREACH(keg, &uma_kegs, uk_link) { 1959099a0e58SBosko Milekic LIST_FOREACH(zone, &keg->uk_zones, uz_link) 19608355f576SJeff Roberson zfunc(zone); 1961099a0e58SBosko Milekic } 19622efcc8cbSGleb Smirnoff if (__predict_true(booted == BOOT_RUNNING)) 1963111fbcd5SBryan Venteicher rw_runlock(&uma_rwlock); 19648355f576SJeff Roberson } 19658355f576SJeff Roberson 1966f4bef67cSGleb Smirnoff /* 1967f4bef67cSGleb Smirnoff * Count how many pages do we need to bootstrap. VM supplies 1968f4bef67cSGleb Smirnoff * its need in early zones in the argument, we add up our zones, 1969f4bef67cSGleb Smirnoff * which consist of: UMA Slabs, UMA Hash and 9 Bucket zones. The 1970f4bef67cSGleb Smirnoff * zone of zones and zone of kegs are accounted separately. 1971f4bef67cSGleb Smirnoff */ 1972f4bef67cSGleb Smirnoff #define UMA_BOOT_ZONES 11 19735073a083SGleb Smirnoff /* Zone of zones and zone of kegs have arbitrary alignment. */ 19745073a083SGleb Smirnoff #define UMA_BOOT_ALIGN 32 1975f4bef67cSGleb Smirnoff static int zsize, ksize; 1976f4bef67cSGleb Smirnoff int 1977f7d35785SGleb Smirnoff uma_startup_count(int vm_zones) 1978f4bef67cSGleb Smirnoff { 1979f7d35785SGleb Smirnoff int zones, pages; 1980f4bef67cSGleb Smirnoff 1981f4bef67cSGleb Smirnoff ksize = sizeof(struct uma_keg) + 1982f4bef67cSGleb Smirnoff (sizeof(struct uma_domain) * vm_ndomains); 1983f4bef67cSGleb Smirnoff zsize = sizeof(struct uma_zone) + 1984f4bef67cSGleb Smirnoff (sizeof(struct uma_cache) * (mp_maxid + 1)) + 1985f4bef67cSGleb Smirnoff (sizeof(struct uma_zone_domain) * vm_ndomains); 1986f4bef67cSGleb Smirnoff 19875073a083SGleb Smirnoff /* 19885073a083SGleb Smirnoff * Memory for the zone of kegs and its keg, 19895073a083SGleb Smirnoff * and for zone of zones. 19905073a083SGleb Smirnoff */ 1991f4bef67cSGleb Smirnoff pages = howmany(roundup(zsize, CACHE_LINE_SIZE) * 2 + 1992f4bef67cSGleb Smirnoff roundup(ksize, CACHE_LINE_SIZE), PAGE_SIZE); 1993f4bef67cSGleb Smirnoff 1994f7d35785SGleb Smirnoff #ifdef UMA_MD_SMALL_ALLOC 1995f7d35785SGleb Smirnoff zones = UMA_BOOT_ZONES; 1996f7d35785SGleb Smirnoff #else 1997f7d35785SGleb Smirnoff zones = UMA_BOOT_ZONES + vm_zones; 1998f7d35785SGleb Smirnoff vm_zones = 0; 1999f7d35785SGleb Smirnoff #endif 2000f4bef67cSGleb Smirnoff 20015073a083SGleb Smirnoff /* Memory for the rest of startup zones, UMA and VM, ... */ 20020b2e3aeaSGleb Smirnoff if (zsize > UMA_SLAB_SPACE) { 20030b2e3aeaSGleb Smirnoff /* See keg_large_init(). */ 20040b2e3aeaSGleb Smirnoff u_int ppera; 20050b2e3aeaSGleb Smirnoff 20060b2e3aeaSGleb Smirnoff ppera = howmany(roundup2(zsize, UMA_BOOT_ALIGN), PAGE_SIZE); 20070b2e3aeaSGleb Smirnoff if (PAGE_SIZE * ppera - roundup2(zsize, UMA_BOOT_ALIGN) < 20080b2e3aeaSGleb Smirnoff SIZEOF_UMA_SLAB) 20090b2e3aeaSGleb Smirnoff ppera++; 20100b2e3aeaSGleb Smirnoff pages += (zones + vm_zones) * ppera; 20110b2e3aeaSGleb Smirnoff } else if (roundup2(zsize, UMA_BOOT_ALIGN) > UMA_SLAB_SPACE) 20120b2e3aeaSGleb Smirnoff /* See keg_small_init() special case for uk_ppera = 1. */ 201396a10340SGleb Smirnoff pages += zones; 2014f4bef67cSGleb Smirnoff else 20155073a083SGleb Smirnoff pages += howmany(zones, 20165073a083SGleb Smirnoff UMA_SLAB_SPACE / roundup2(zsize, UMA_BOOT_ALIGN)); 2017f4bef67cSGleb Smirnoff 20185073a083SGleb Smirnoff /* ... and their kegs. Note that zone of zones allocates a keg! */ 20195073a083SGleb Smirnoff pages += howmany(zones + 1, 20205073a083SGleb Smirnoff UMA_SLAB_SPACE / roundup2(ksize, UMA_BOOT_ALIGN)); 2021f4bef67cSGleb Smirnoff 2022f4bef67cSGleb Smirnoff /* 20235073a083SGleb Smirnoff * Most of startup zones are not going to be offpages, that's 20245073a083SGleb Smirnoff * why we use UMA_SLAB_SPACE instead of UMA_SLAB_SIZE in all 20255073a083SGleb Smirnoff * calculations. Some large bucket zones will be offpage, and 20265073a083SGleb Smirnoff * thus will allocate hashes. We take conservative approach 20275073a083SGleb Smirnoff * and assume that all zones may allocate hash. This may give 20285073a083SGleb Smirnoff * us some positive inaccuracy, usually an extra single page. 2029f4bef67cSGleb Smirnoff */ 20305073a083SGleb Smirnoff pages += howmany(zones, UMA_SLAB_SPACE / 2031d2be4a1eSGleb Smirnoff (sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT)); 2032f4bef67cSGleb Smirnoff 2033f4bef67cSGleb Smirnoff return (pages); 2034f4bef67cSGleb Smirnoff } 2035f4bef67cSGleb Smirnoff 20368355f576SJeff Roberson void 2037ac0a6fd0SGleb Smirnoff uma_startup(void *mem, int npages) 20388355f576SJeff Roberson { 20398355f576SJeff Roberson struct uma_zctor_args args; 2040ab3185d1SJeff Roberson uma_keg_t masterkeg; 2041ab3185d1SJeff Roberson uintptr_t m; 2042f4bef67cSGleb Smirnoff 2043f4bef67cSGleb Smirnoff #ifdef DIAGNOSTIC 2044f4bef67cSGleb Smirnoff printf("Entering %s with %d boot pages configured\n", __func__, npages); 2045f4bef67cSGleb Smirnoff #endif 20468355f576SJeff Roberson 2047111fbcd5SBryan Venteicher rw_init(&uma_rwlock, "UMA lock"); 2048099a0e58SBosko Milekic 2049ab3185d1SJeff Roberson /* Use bootpages memory for the zone of zones and zone of kegs. */ 2050ab3185d1SJeff Roberson m = (uintptr_t)mem; 2051ab3185d1SJeff Roberson zones = (uma_zone_t)m; 2052ab3185d1SJeff Roberson m += roundup(zsize, CACHE_LINE_SIZE); 2053ab3185d1SJeff Roberson kegs = (uma_zone_t)m; 2054ab3185d1SJeff Roberson m += roundup(zsize, CACHE_LINE_SIZE); 2055ab3185d1SJeff Roberson masterkeg = (uma_keg_t)m; 2056ab3185d1SJeff Roberson m += roundup(ksize, CACHE_LINE_SIZE); 2057ab3185d1SJeff Roberson m = roundup(m, PAGE_SIZE); 2058ab3185d1SJeff Roberson npages -= (m - (uintptr_t)mem) / PAGE_SIZE; 2059ab3185d1SJeff Roberson mem = (void *)m; 2060ab3185d1SJeff Roberson 2061099a0e58SBosko Milekic /* "manually" create the initial zone */ 20620095a784SJeff Roberson memset(&args, 0, sizeof(args)); 2063099a0e58SBosko Milekic args.name = "UMA Kegs"; 2064ab3185d1SJeff Roberson args.size = ksize; 2065099a0e58SBosko Milekic args.ctor = keg_ctor; 2066099a0e58SBosko Milekic args.dtor = keg_dtor; 20678355f576SJeff Roberson args.uminit = zero_init; 20688355f576SJeff Roberson args.fini = NULL; 2069ab3185d1SJeff Roberson args.keg = masterkeg; 20705073a083SGleb Smirnoff args.align = UMA_BOOT_ALIGN - 1; 2071b60f5b79SJeff Roberson args.flags = UMA_ZFLAG_INTERNAL; 2072ab3185d1SJeff Roberson zone_ctor(kegs, zsize, &args, M_WAITOK); 20738355f576SJeff Roberson 2074ac0a6fd0SGleb Smirnoff bootmem = mem; 2075ac0a6fd0SGleb Smirnoff boot_pages = npages; 20768355f576SJeff Roberson 2077099a0e58SBosko Milekic args.name = "UMA Zones"; 2078f4bef67cSGleb Smirnoff args.size = zsize; 2079099a0e58SBosko Milekic args.ctor = zone_ctor; 2080099a0e58SBosko Milekic args.dtor = zone_dtor; 2081099a0e58SBosko Milekic args.uminit = zero_init; 2082099a0e58SBosko Milekic args.fini = NULL; 2083099a0e58SBosko Milekic args.keg = NULL; 20845073a083SGleb Smirnoff args.align = UMA_BOOT_ALIGN - 1; 2085099a0e58SBosko Milekic args.flags = UMA_ZFLAG_INTERNAL; 2086ab3185d1SJeff Roberson zone_ctor(zones, zsize, &args, M_WAITOK); 2087099a0e58SBosko Milekic 20888355f576SJeff Roberson /* Now make a zone for slab headers */ 20898355f576SJeff Roberson slabzone = uma_zcreate("UMA Slabs", 2090ef72505eSJeff Roberson sizeof(struct uma_slab), 20918355f576SJeff Roberson NULL, NULL, NULL, NULL, 2092b60f5b79SJeff Roberson UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 20938355f576SJeff Roberson 20948355f576SJeff Roberson hashzone = uma_zcreate("UMA Hash", 20958355f576SJeff Roberson sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 20968355f576SJeff Roberson NULL, NULL, NULL, NULL, 2097b60f5b79SJeff Roberson UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 20988355f576SJeff Roberson 2099cae33c14SJeff Roberson bucket_init(); 21008355f576SJeff Roberson 2101f4bef67cSGleb Smirnoff booted = BOOT_STRAPPED; 21028355f576SJeff Roberson } 21038355f576SJeff Roberson 2104f4bef67cSGleb Smirnoff void 2105f4bef67cSGleb Smirnoff uma_startup1(void) 2106f4bef67cSGleb Smirnoff { 2107f4bef67cSGleb Smirnoff 2108f4bef67cSGleb Smirnoff #ifdef DIAGNOSTIC 2109f4bef67cSGleb Smirnoff printf("Entering %s with %d boot pages left\n", __func__, boot_pages); 2110f4bef67cSGleb Smirnoff #endif 2111f4bef67cSGleb Smirnoff booted = BOOT_PAGEALLOC; 2112f4bef67cSGleb Smirnoff } 2113f4bef67cSGleb Smirnoff 21148355f576SJeff Roberson void 211599571dc3SJeff Roberson uma_startup2(void) 21168355f576SJeff Roberson { 2117f4bef67cSGleb Smirnoff 2118f7d35785SGleb Smirnoff #ifdef DIAGNOSTIC 2119f7d35785SGleb Smirnoff printf("Entering %s with %d boot pages left\n", __func__, boot_pages); 2120f7d35785SGleb Smirnoff #endif 2121f4bef67cSGleb Smirnoff booted = BOOT_BUCKETS; 212295c4bf75SKonstantin Belousov sx_init(&uma_drain_lock, "umadrain"); 2123f4bef67cSGleb Smirnoff bucket_enable(); 21248355f576SJeff Roberson } 21258355f576SJeff Roberson 21268355f576SJeff Roberson /* 21278355f576SJeff Roberson * Initialize our callout handle 21288355f576SJeff Roberson * 21298355f576SJeff Roberson */ 21308355f576SJeff Roberson static void 21318355f576SJeff Roberson uma_startup3(void) 21328355f576SJeff Roberson { 21331431a748SGleb Smirnoff 2134c5deaf04SGleb Smirnoff #ifdef INVARIANTS 2135c5deaf04SGleb Smirnoff TUNABLE_INT_FETCH("vm.debug.divisor", &dbg_divisor); 2136c5deaf04SGleb Smirnoff uma_dbg_cnt = counter_u64_alloc(M_WAITOK); 2137c5deaf04SGleb Smirnoff uma_skip_cnt = counter_u64_alloc(M_WAITOK); 2138c5deaf04SGleb Smirnoff #endif 21392efcc8cbSGleb Smirnoff zone_foreach(zone_alloc_counters); 2140fd90e2edSJung-uk Kim callout_init(&uma_callout, 1); 21419643769aSJeff Roberson callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 2142c5deaf04SGleb Smirnoff booted = BOOT_RUNNING; 21438355f576SJeff Roberson } 21448355f576SJeff Roberson 2145e20a199fSJeff Roberson static uma_keg_t 2146099a0e58SBosko Milekic uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, 214785dcf349SGleb Smirnoff int align, uint32_t flags) 2148099a0e58SBosko Milekic { 2149099a0e58SBosko Milekic struct uma_kctor_args args; 2150099a0e58SBosko Milekic 2151099a0e58SBosko Milekic args.size = size; 2152099a0e58SBosko Milekic args.uminit = uminit; 2153099a0e58SBosko Milekic args.fini = fini; 21541e319f6dSRobert Watson args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align; 2155099a0e58SBosko Milekic args.flags = flags; 2156099a0e58SBosko Milekic args.zone = zone; 2157ab3185d1SJeff Roberson return (zone_alloc_item(kegs, &args, UMA_ANYDOMAIN, M_WAITOK)); 2158099a0e58SBosko Milekic } 2159099a0e58SBosko Milekic 2160f4bef67cSGleb Smirnoff /* Public functions */ 21618355f576SJeff Roberson /* See uma.h */ 21621e319f6dSRobert Watson void 21631e319f6dSRobert Watson uma_set_align(int align) 21641e319f6dSRobert Watson { 21651e319f6dSRobert Watson 21661e319f6dSRobert Watson if (align != UMA_ALIGN_CACHE) 21671e319f6dSRobert Watson uma_align_cache = align; 21681e319f6dSRobert Watson } 21691e319f6dSRobert Watson 21701e319f6dSRobert Watson /* See uma.h */ 21718355f576SJeff Roberson uma_zone_t 2172bb196eb4SMatthew D Fleming uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, 217385dcf349SGleb Smirnoff uma_init uminit, uma_fini fini, int align, uint32_t flags) 21748355f576SJeff Roberson 21758355f576SJeff Roberson { 21768355f576SJeff Roberson struct uma_zctor_args args; 217795c4bf75SKonstantin Belousov uma_zone_t res; 217895c4bf75SKonstantin Belousov bool locked; 21798355f576SJeff Roberson 2180a5a35578SJohn Baldwin KASSERT(powerof2(align + 1), ("invalid zone alignment %d for \"%s\"", 2181a5a35578SJohn Baldwin align, name)); 2182a5a35578SJohn Baldwin 21838355f576SJeff Roberson /* This stuff is essential for the zone ctor */ 21840095a784SJeff Roberson memset(&args, 0, sizeof(args)); 21858355f576SJeff Roberson args.name = name; 21868355f576SJeff Roberson args.size = size; 21878355f576SJeff Roberson args.ctor = ctor; 21888355f576SJeff Roberson args.dtor = dtor; 21898355f576SJeff Roberson args.uminit = uminit; 21908355f576SJeff Roberson args.fini = fini; 2191afc6dc36SJohn-Mark Gurney #ifdef INVARIANTS 2192afc6dc36SJohn-Mark Gurney /* 2193afc6dc36SJohn-Mark Gurney * If a zone is being created with an empty constructor and 2194afc6dc36SJohn-Mark Gurney * destructor, pass UMA constructor/destructor which checks for 2195afc6dc36SJohn-Mark Gurney * memory use after free. 2196afc6dc36SJohn-Mark Gurney */ 219719c591bfSMateusz Guzik if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOFREE))) && 219819c591bfSMateusz Guzik ctor == NULL && dtor == NULL && uminit == NULL && fini == NULL) { 2199afc6dc36SJohn-Mark Gurney args.ctor = trash_ctor; 2200afc6dc36SJohn-Mark Gurney args.dtor = trash_dtor; 2201afc6dc36SJohn-Mark Gurney args.uminit = trash_init; 2202afc6dc36SJohn-Mark Gurney args.fini = trash_fini; 2203afc6dc36SJohn-Mark Gurney } 2204afc6dc36SJohn-Mark Gurney #endif 22058355f576SJeff Roberson args.align = align; 22068355f576SJeff Roberson args.flags = flags; 2207099a0e58SBosko Milekic args.keg = NULL; 2208099a0e58SBosko Milekic 2209f4bef67cSGleb Smirnoff if (booted < BOOT_BUCKETS) { 221095c4bf75SKonstantin Belousov locked = false; 221195c4bf75SKonstantin Belousov } else { 221295c4bf75SKonstantin Belousov sx_slock(&uma_drain_lock); 221395c4bf75SKonstantin Belousov locked = true; 221495c4bf75SKonstantin Belousov } 2215ab3185d1SJeff Roberson res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); 221695c4bf75SKonstantin Belousov if (locked) 221795c4bf75SKonstantin Belousov sx_sunlock(&uma_drain_lock); 221895c4bf75SKonstantin Belousov return (res); 2219099a0e58SBosko Milekic } 2220099a0e58SBosko Milekic 2221099a0e58SBosko Milekic /* See uma.h */ 2222099a0e58SBosko Milekic uma_zone_t 2223099a0e58SBosko Milekic uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, 2224099a0e58SBosko Milekic uma_init zinit, uma_fini zfini, uma_zone_t master) 2225099a0e58SBosko Milekic { 2226099a0e58SBosko Milekic struct uma_zctor_args args; 2227e20a199fSJeff Roberson uma_keg_t keg; 222895c4bf75SKonstantin Belousov uma_zone_t res; 222995c4bf75SKonstantin Belousov bool locked; 2230099a0e58SBosko Milekic 2231bb15d1c7SGleb Smirnoff keg = master->uz_keg; 22320095a784SJeff Roberson memset(&args, 0, sizeof(args)); 2233099a0e58SBosko Milekic args.name = name; 2234e20a199fSJeff Roberson args.size = keg->uk_size; 2235099a0e58SBosko Milekic args.ctor = ctor; 2236099a0e58SBosko Milekic args.dtor = dtor; 2237099a0e58SBosko Milekic args.uminit = zinit; 2238099a0e58SBosko Milekic args.fini = zfini; 2239e20a199fSJeff Roberson args.align = keg->uk_align; 2240e20a199fSJeff Roberson args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; 2241e20a199fSJeff Roberson args.keg = keg; 22428355f576SJeff Roberson 2243f4bef67cSGleb Smirnoff if (booted < BOOT_BUCKETS) { 224495c4bf75SKonstantin Belousov locked = false; 224595c4bf75SKonstantin Belousov } else { 224695c4bf75SKonstantin Belousov sx_slock(&uma_drain_lock); 224795c4bf75SKonstantin Belousov locked = true; 224895c4bf75SKonstantin Belousov } 2249e20a199fSJeff Roberson /* XXX Attaches only one keg of potentially many. */ 2250ab3185d1SJeff Roberson res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); 225195c4bf75SKonstantin Belousov if (locked) 225295c4bf75SKonstantin Belousov sx_sunlock(&uma_drain_lock); 225395c4bf75SKonstantin Belousov return (res); 22548355f576SJeff Roberson } 22558355f576SJeff Roberson 22560095a784SJeff Roberson /* See uma.h */ 22570095a784SJeff Roberson uma_zone_t 2258af526374SJeff Roberson uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor, 2259af526374SJeff Roberson uma_init zinit, uma_fini zfini, uma_import zimport, 2260af526374SJeff Roberson uma_release zrelease, void *arg, int flags) 22610095a784SJeff Roberson { 22620095a784SJeff Roberson struct uma_zctor_args args; 22630095a784SJeff Roberson 22640095a784SJeff Roberson memset(&args, 0, sizeof(args)); 22650095a784SJeff Roberson args.name = name; 2266af526374SJeff Roberson args.size = size; 22670095a784SJeff Roberson args.ctor = ctor; 22680095a784SJeff Roberson args.dtor = dtor; 22690095a784SJeff Roberson args.uminit = zinit; 22700095a784SJeff Roberson args.fini = zfini; 22710095a784SJeff Roberson args.import = zimport; 22720095a784SJeff Roberson args.release = zrelease; 22730095a784SJeff Roberson args.arg = arg; 22740095a784SJeff Roberson args.align = 0; 2275bb15d1c7SGleb Smirnoff args.flags = flags | UMA_ZFLAG_CACHE; 22760095a784SJeff Roberson 2277ab3185d1SJeff Roberson return (zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK)); 22780095a784SJeff Roberson } 22790095a784SJeff Roberson 22808355f576SJeff Roberson /* See uma.h */ 22819c2cd7e5SJeff Roberson void 22829c2cd7e5SJeff Roberson uma_zdestroy(uma_zone_t zone) 22839c2cd7e5SJeff Roberson { 2284f4ff923bSRobert Watson 228595c4bf75SKonstantin Belousov sx_slock(&uma_drain_lock); 22860095a784SJeff Roberson zone_free_item(zones, zone, NULL, SKIP_NONE); 228795c4bf75SKonstantin Belousov sx_sunlock(&uma_drain_lock); 22889c2cd7e5SJeff Roberson } 22899c2cd7e5SJeff Roberson 22908d6fbbb8SJeff Roberson void 22918d6fbbb8SJeff Roberson uma_zwait(uma_zone_t zone) 22928d6fbbb8SJeff Roberson { 22938d6fbbb8SJeff Roberson void *item; 22948d6fbbb8SJeff Roberson 22958d6fbbb8SJeff Roberson item = uma_zalloc_arg(zone, NULL, M_WAITOK); 22968d6fbbb8SJeff Roberson uma_zfree(zone, item); 22978d6fbbb8SJeff Roberson } 22988d6fbbb8SJeff Roberson 22994e180881SMateusz Guzik void * 23004e180881SMateusz Guzik uma_zalloc_pcpu_arg(uma_zone_t zone, void *udata, int flags) 23014e180881SMateusz Guzik { 23024e180881SMateusz Guzik void *item; 2303b4799947SRuslan Bukin #ifdef SMP 23044e180881SMateusz Guzik int i; 23054e180881SMateusz Guzik 23064e180881SMateusz Guzik MPASS(zone->uz_flags & UMA_ZONE_PCPU); 2307b4799947SRuslan Bukin #endif 23084e180881SMateusz Guzik item = uma_zalloc_arg(zone, udata, flags & ~M_ZERO); 23094e180881SMateusz Guzik if (item != NULL && (flags & M_ZERO)) { 2310b4799947SRuslan Bukin #ifdef SMP 2311013072f0SMark Johnston for (i = 0; i <= mp_maxid; i++) 23124e180881SMateusz Guzik bzero(zpcpu_get_cpu(item, i), zone->uz_size); 2313b4799947SRuslan Bukin #else 2314b4799947SRuslan Bukin bzero(item, zone->uz_size); 2315b4799947SRuslan Bukin #endif 23164e180881SMateusz Guzik } 23174e180881SMateusz Guzik return (item); 23184e180881SMateusz Guzik } 23194e180881SMateusz Guzik 23204e180881SMateusz Guzik /* 23214e180881SMateusz Guzik * A stub while both regular and pcpu cases are identical. 23224e180881SMateusz Guzik */ 23234e180881SMateusz Guzik void 23244e180881SMateusz Guzik uma_zfree_pcpu_arg(uma_zone_t zone, void *item, void *udata) 23254e180881SMateusz Guzik { 23264e180881SMateusz Guzik 2327c5b7751fSIan Lepore #ifdef SMP 23284e180881SMateusz Guzik MPASS(zone->uz_flags & UMA_ZONE_PCPU); 2329c5b7751fSIan Lepore #endif 23304e180881SMateusz Guzik uma_zfree_arg(zone, item, udata); 23314e180881SMateusz Guzik } 23324e180881SMateusz Guzik 23339c2cd7e5SJeff Roberson /* See uma.h */ 23348355f576SJeff Roberson void * 23352cc35ff9SJeff Roberson uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 23368355f576SJeff Roberson { 2337ab3185d1SJeff Roberson uma_zone_domain_t zdom; 23388355f576SJeff Roberson uma_bucket_t bucket; 2339ab3185d1SJeff Roberson uma_cache_t cache; 2340ab3185d1SJeff Roberson void *item; 2341bb15d1c7SGleb Smirnoff int cpu, domain, lockfail, maxbucket; 2342c5deaf04SGleb Smirnoff #ifdef INVARIANTS 2343c5deaf04SGleb Smirnoff bool skipdbg; 2344c5deaf04SGleb Smirnoff #endif 23458355f576SJeff Roberson 2346e866d8f0SMark Murray /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 234719fa89e9SMark Murray random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 234810cb2424SMark Murray 23498355f576SJeff Roberson /* This is the fast path allocation */ 23501431a748SGleb Smirnoff CTR4(KTR_UMA, "uma_zalloc_arg thread %x zone %s(%p) flags %d", 23511431a748SGleb Smirnoff curthread, zone->uz_name, zone, flags); 2352a553d4b8SJeff Roberson 2353635fd505SRobert Watson if (flags & M_WAITOK) { 2354b23f72e9SBrian Feldman WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 2355635fd505SRobert Watson "uma_zalloc_arg: zone \"%s\"", zone->uz_name); 23564c1cc01cSJohn Baldwin } 23570766f278SJonathan T. Looney KASSERT((flags & M_EXEC) == 0, ("uma_zalloc_arg: called with M_EXEC")); 2358d9e2e68dSMark Johnston KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), 23591067a2baSJonathan T. Looney ("uma_zalloc_arg: called with spinlock or critical section held")); 2360ea99223eSMateusz Guzik if (zone->uz_flags & UMA_ZONE_PCPU) 2361b8af2820SMateusz Guzik KASSERT((flags & M_ZERO) == 0, ("allocating from a pcpu zone " 2362b8af2820SMateusz Guzik "with M_ZERO passed")); 23631067a2baSJonathan T. Looney 23648d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD 23658d689e04SGleb Smirnoff if (memguard_cmp_zone(zone)) { 23668d689e04SGleb Smirnoff item = memguard_alloc(zone->uz_size, flags); 23678d689e04SGleb Smirnoff if (item != NULL) { 23688d689e04SGleb Smirnoff if (zone->uz_init != NULL && 23698d689e04SGleb Smirnoff zone->uz_init(item, zone->uz_size, flags) != 0) 23708d689e04SGleb Smirnoff return (NULL); 23718d689e04SGleb Smirnoff if (zone->uz_ctor != NULL && 2372fc03d22bSJeff Roberson zone->uz_ctor(item, zone->uz_size, udata, 2373fc03d22bSJeff Roberson flags) != 0) { 23748d689e04SGleb Smirnoff zone->uz_fini(item, zone->uz_size); 23758d689e04SGleb Smirnoff return (NULL); 23768d689e04SGleb Smirnoff } 23778d689e04SGleb Smirnoff return (item); 23788d689e04SGleb Smirnoff } 23798d689e04SGleb Smirnoff /* This is unfortunate but should not be fatal. */ 23808d689e04SGleb Smirnoff } 23818d689e04SGleb Smirnoff #endif 23825d1ae027SRobert Watson /* 23835d1ae027SRobert Watson * If possible, allocate from the per-CPU cache. There are two 23845d1ae027SRobert Watson * requirements for safe access to the per-CPU cache: (1) the thread 23855d1ae027SRobert Watson * accessing the cache must not be preempted or yield during access, 23865d1ae027SRobert Watson * and (2) the thread must not migrate CPUs without switching which 23875d1ae027SRobert Watson * cache it accesses. We rely on a critical section to prevent 23885d1ae027SRobert Watson * preemption and migration. We release the critical section in 23895d1ae027SRobert Watson * order to acquire the zone mutex if we are unable to allocate from 23905d1ae027SRobert Watson * the current cache; when we re-acquire the critical section, we 23915d1ae027SRobert Watson * must detect and handle migration if it has occurred. 23925d1ae027SRobert Watson */ 239381c0d72cSGleb Smirnoff zalloc_restart: 23945d1ae027SRobert Watson critical_enter(); 23955d1ae027SRobert Watson cpu = curcpu; 23968355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 23978355f576SJeff Roberson 23988355f576SJeff Roberson zalloc_start: 23998355f576SJeff Roberson bucket = cache->uc_allocbucket; 2400fc03d22bSJeff Roberson if (bucket != NULL && bucket->ub_cnt > 0) { 2401cae33c14SJeff Roberson bucket->ub_cnt--; 2402cae33c14SJeff Roberson item = bucket->ub_bucket[bucket->ub_cnt]; 24038355f576SJeff Roberson #ifdef INVARIANTS 2404cae33c14SJeff Roberson bucket->ub_bucket[bucket->ub_cnt] = NULL; 24058355f576SJeff Roberson #endif 2406fc03d22bSJeff Roberson KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled.")); 24078355f576SJeff Roberson cache->uc_allocs++; 24085d1ae027SRobert Watson critical_exit(); 2409c5deaf04SGleb Smirnoff #ifdef INVARIANTS 2410c5deaf04SGleb Smirnoff skipdbg = uma_dbg_zskip(zone, item); 2411c5deaf04SGleb Smirnoff #endif 2412fc03d22bSJeff Roberson if (zone->uz_ctor != NULL && 2413c5deaf04SGleb Smirnoff #ifdef INVARIANTS 2414c5deaf04SGleb Smirnoff (!skipdbg || zone->uz_ctor != trash_ctor || 2415c5deaf04SGleb Smirnoff zone->uz_dtor != trash_dtor) && 2416c5deaf04SGleb Smirnoff #endif 2417fc03d22bSJeff Roberson zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { 24182efcc8cbSGleb Smirnoff counter_u64_add(zone->uz_fails, 1); 2419bb15d1c7SGleb Smirnoff zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); 2420b23f72e9SBrian Feldman return (NULL); 2421b23f72e9SBrian Feldman } 2422ef72505eSJeff Roberson #ifdef INVARIANTS 2423c5deaf04SGleb Smirnoff if (!skipdbg) 2424ef72505eSJeff Roberson uma_dbg_alloc(zone, NULL, item); 2425ef72505eSJeff Roberson #endif 24262cc35ff9SJeff Roberson if (flags & M_ZERO) 242748343a2fSGleb Smirnoff uma_zero_item(item, zone); 24288355f576SJeff Roberson return (item); 2429fc03d22bSJeff Roberson } 2430fc03d22bSJeff Roberson 24318355f576SJeff Roberson /* 24328355f576SJeff Roberson * We have run out of items in our alloc bucket. 24338355f576SJeff Roberson * See if we can switch with our free bucket. 24348355f576SJeff Roberson */ 2435b983089aSJeff Roberson bucket = cache->uc_freebucket; 2436fc03d22bSJeff Roberson if (bucket != NULL && bucket->ub_cnt > 0) { 24371431a748SGleb Smirnoff CTR2(KTR_UMA, 24381431a748SGleb Smirnoff "uma_zalloc: zone %s(%p) swapping empty with alloc", 24391431a748SGleb Smirnoff zone->uz_name, zone); 24408355f576SJeff Roberson cache->uc_freebucket = cache->uc_allocbucket; 2441b983089aSJeff Roberson cache->uc_allocbucket = bucket; 24428355f576SJeff Roberson goto zalloc_start; 24438355f576SJeff Roberson } 2444fc03d22bSJeff Roberson 2445fc03d22bSJeff Roberson /* 2446fc03d22bSJeff Roberson * Discard any empty allocation bucket while we hold no locks. 2447fc03d22bSJeff Roberson */ 2448fc03d22bSJeff Roberson bucket = cache->uc_allocbucket; 2449fc03d22bSJeff Roberson cache->uc_allocbucket = NULL; 2450fc03d22bSJeff Roberson critical_exit(); 2451fc03d22bSJeff Roberson if (bucket != NULL) 24526fd34d6fSJeff Roberson bucket_free(zone, bucket, udata); 2453fc03d22bSJeff Roberson 245430c5525bSAndrew Gallatin if (zone->uz_flags & UMA_ZONE_NUMA) { 2455ab3185d1SJeff Roberson domain = PCPU_GET(domain); 245630c5525bSAndrew Gallatin if (VM_DOMAIN_EMPTY(domain)) 245730c5525bSAndrew Gallatin domain = UMA_ANYDOMAIN; 245830c5525bSAndrew Gallatin } else 2459ab3185d1SJeff Roberson domain = UMA_ANYDOMAIN; 2460ab3185d1SJeff Roberson 2461fc03d22bSJeff Roberson /* Short-circuit for zones without buckets and low memory. */ 2462bb15d1c7SGleb Smirnoff if (zone->uz_count == 0 || bucketdisable) { 2463bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 2464fc03d22bSJeff Roberson goto zalloc_item; 2465bb15d1c7SGleb Smirnoff } 2466fc03d22bSJeff Roberson 24675d1ae027SRobert Watson /* 24685d1ae027SRobert Watson * Attempt to retrieve the item from the per-CPU cache has failed, so 24695d1ae027SRobert Watson * we must go back to the zone. This requires the zone lock, so we 24705d1ae027SRobert Watson * must drop the critical section, then re-acquire it when we go back 24715d1ae027SRobert Watson * to the cache. Since the critical section is released, we may be 24725d1ae027SRobert Watson * preempted or migrate. As such, make sure not to maintain any 24735d1ae027SRobert Watson * thread-local state specific to the cache from prior to releasing 24745d1ae027SRobert Watson * the critical section. 24755d1ae027SRobert Watson */ 2476fc03d22bSJeff Roberson lockfail = 0; 2477fc03d22bSJeff Roberson if (ZONE_TRYLOCK(zone) == 0) { 2478fc03d22bSJeff Roberson /* Record contention to size the buckets. */ 2479a553d4b8SJeff Roberson ZONE_LOCK(zone); 2480fc03d22bSJeff Roberson lockfail = 1; 2481fc03d22bSJeff Roberson } 24825d1ae027SRobert Watson critical_enter(); 24835d1ae027SRobert Watson cpu = curcpu; 24845d1ae027SRobert Watson cache = &zone->uz_cpu[cpu]; 24855d1ae027SRobert Watson 2486fc03d22bSJeff Roberson /* See if we lost the race to fill the cache. */ 2487fc03d22bSJeff Roberson if (cache->uc_allocbucket != NULL) { 2488fc03d22bSJeff Roberson ZONE_UNLOCK(zone); 2489fc03d22bSJeff Roberson goto zalloc_start; 2490a553d4b8SJeff Roberson } 24918355f576SJeff Roberson 2492fc03d22bSJeff Roberson /* 2493fc03d22bSJeff Roberson * Check the zone's cache of buckets. 2494fc03d22bSJeff Roberson */ 2495ab3185d1SJeff Roberson if (domain == UMA_ANYDOMAIN) 2496ab3185d1SJeff Roberson zdom = &zone->uz_domain[0]; 2497ab3185d1SJeff Roberson else 2498ab3185d1SJeff Roberson zdom = &zone->uz_domain[domain]; 24990f9b7bf3SMark Johnston if ((bucket = zone_try_fetch_bucket(zone, zdom, true)) != NULL) { 2500cae33c14SJeff Roberson KASSERT(bucket->ub_cnt != 0, 2501a553d4b8SJeff Roberson ("uma_zalloc_arg: Returning an empty bucket.")); 2502a553d4b8SJeff Roberson cache->uc_allocbucket = bucket; 2503a553d4b8SJeff Roberson ZONE_UNLOCK(zone); 25048355f576SJeff Roberson goto zalloc_start; 2505a553d4b8SJeff Roberson } 25065d1ae027SRobert Watson /* We are no longer associated with this CPU. */ 25075d1ae027SRobert Watson critical_exit(); 2508bbee39c6SJeff Roberson 2509fc03d22bSJeff Roberson /* 2510fc03d22bSJeff Roberson * We bump the uz count when the cache size is insufficient to 2511fc03d22bSJeff Roberson * handle the working set. 2512fc03d22bSJeff Roberson */ 2513bb15d1c7SGleb Smirnoff if (lockfail && zone->uz_count < zone->uz_count_max) 2514a553d4b8SJeff Roberson zone->uz_count++; 2515bb15d1c7SGleb Smirnoff 2516bb15d1c7SGleb Smirnoff if (zone->uz_max_items > 0) { 2517bb15d1c7SGleb Smirnoff if (zone->uz_items >= zone->uz_max_items) 2518bb15d1c7SGleb Smirnoff goto zalloc_item; 2519bb15d1c7SGleb Smirnoff maxbucket = MIN(zone->uz_count, 2520bb15d1c7SGleb Smirnoff zone->uz_max_items - zone->uz_items); 2521bb45b411SGleb Smirnoff zone->uz_items += maxbucket; 2522bb15d1c7SGleb Smirnoff } else 2523bb15d1c7SGleb Smirnoff maxbucket = zone->uz_count; 2524fc03d22bSJeff Roberson ZONE_UNLOCK(zone); 2525099a0e58SBosko Milekic 25268355f576SJeff Roberson /* 2527a553d4b8SJeff Roberson * Now lets just fill a bucket and put it on the free list. If that 2528763df3ecSPedro F. Giffuni * works we'll restart the allocation from the beginning and it 2529fc03d22bSJeff Roberson * will use the just filled bucket. 2530bbee39c6SJeff Roberson */ 2531bb15d1c7SGleb Smirnoff bucket = zone_alloc_bucket(zone, udata, domain, flags, maxbucket); 25321431a748SGleb Smirnoff CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p", 25331431a748SGleb Smirnoff zone->uz_name, zone, bucket); 2534fc03d22bSJeff Roberson ZONE_LOCK(zone); 2535bb15d1c7SGleb Smirnoff if (bucket != NULL) { 2536bb45b411SGleb Smirnoff if (zone->uz_max_items > 0 && bucket->ub_cnt < maxbucket) { 2537bb45b411SGleb Smirnoff MPASS(zone->uz_items >= maxbucket - bucket->ub_cnt); 2538bb15d1c7SGleb Smirnoff zone->uz_items -= maxbucket - bucket->ub_cnt; 2539bb15d1c7SGleb Smirnoff if (zone->uz_sleepers > 0 && 2540bb15d1c7SGleb Smirnoff zone->uz_items < zone->uz_max_items) 2541bb15d1c7SGleb Smirnoff wakeup_one(zone); 2542bb15d1c7SGleb Smirnoff } 2543fc03d22bSJeff Roberson critical_enter(); 2544fc03d22bSJeff Roberson cpu = curcpu; 2545fc03d22bSJeff Roberson cache = &zone->uz_cpu[cpu]; 25460f9b7bf3SMark Johnston 2547fc03d22bSJeff Roberson /* 2548fc03d22bSJeff Roberson * See if we lost the race or were migrated. Cache the 2549fc03d22bSJeff Roberson * initialized bucket to make this less likely or claim 2550fc03d22bSJeff Roberson * the memory directly. 2551fc03d22bSJeff Roberson */ 255281c0d72cSGleb Smirnoff if (cache->uc_allocbucket == NULL && 255381c0d72cSGleb Smirnoff ((zone->uz_flags & UMA_ZONE_NUMA) == 0 || 255481c0d72cSGleb Smirnoff domain == PCPU_GET(domain))) { 2555ab3185d1SJeff Roberson cache->uc_allocbucket = bucket; 25560f9b7bf3SMark Johnston zdom->uzd_imax += bucket->ub_cnt; 2557bb15d1c7SGleb Smirnoff } else if (zone->uz_bkt_count >= zone->uz_bkt_max) { 255881c0d72cSGleb Smirnoff critical_exit(); 255981c0d72cSGleb Smirnoff ZONE_UNLOCK(zone); 256081c0d72cSGleb Smirnoff bucket_drain(zone, bucket); 256181c0d72cSGleb Smirnoff bucket_free(zone, bucket, udata); 256281c0d72cSGleb Smirnoff goto zalloc_restart; 256381c0d72cSGleb Smirnoff } else 25640f9b7bf3SMark Johnston zone_put_bucket(zone, zdom, bucket, false); 2565bbee39c6SJeff Roberson ZONE_UNLOCK(zone); 2566fc03d22bSJeff Roberson goto zalloc_start; 2567bb45b411SGleb Smirnoff } else if (zone->uz_max_items > 0) { 2568bb15d1c7SGleb Smirnoff zone->uz_items -= maxbucket; 2569bb15d1c7SGleb Smirnoff if (zone->uz_sleepers > 0 && 2570bb15d1c7SGleb Smirnoff zone->uz_items + 1 < zone->uz_max_items) 2571bb15d1c7SGleb Smirnoff wakeup_one(zone); 2572bbee39c6SJeff Roberson } 2573fc03d22bSJeff Roberson 2574bbee39c6SJeff Roberson /* 2575bbee39c6SJeff Roberson * We may not be able to get a bucket so return an actual item. 2576bbee39c6SJeff Roberson */ 2577fc03d22bSJeff Roberson zalloc_item: 2578bb15d1c7SGleb Smirnoff item = zone_alloc_item_locked(zone, udata, domain, flags); 2579fc03d22bSJeff Roberson 2580e20a199fSJeff Roberson return (item); 2581bbee39c6SJeff Roberson } 2582bbee39c6SJeff Roberson 2583ab3185d1SJeff Roberson void * 2584ab3185d1SJeff Roberson uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) 2585bbee39c6SJeff Roberson { 2586ab3185d1SJeff Roberson 2587ab3185d1SJeff Roberson /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 258819fa89e9SMark Murray random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 2589ab3185d1SJeff Roberson 2590ab3185d1SJeff Roberson /* This is the fast path allocation */ 2591ab3185d1SJeff Roberson CTR5(KTR_UMA, 2592ab3185d1SJeff Roberson "uma_zalloc_domain thread %x zone %s(%p) domain %d flags %d", 2593ab3185d1SJeff Roberson curthread, zone->uz_name, zone, domain, flags); 2594ab3185d1SJeff Roberson 2595ab3185d1SJeff Roberson if (flags & M_WAITOK) { 2596ab3185d1SJeff Roberson WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 2597ab3185d1SJeff Roberson "uma_zalloc_domain: zone \"%s\"", zone->uz_name); 2598ab3185d1SJeff Roberson } 2599ab3185d1SJeff Roberson KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), 2600ab3185d1SJeff Roberson ("uma_zalloc_domain: called with spinlock or critical section held")); 2601ab3185d1SJeff Roberson 2602ab3185d1SJeff Roberson return (zone_alloc_item(zone, udata, domain, flags)); 2603ab3185d1SJeff Roberson } 2604ab3185d1SJeff Roberson 2605ab3185d1SJeff Roberson /* 2606ab3185d1SJeff Roberson * Find a slab with some space. Prefer slabs that are partially used over those 2607ab3185d1SJeff Roberson * that are totally full. This helps to reduce fragmentation. 2608ab3185d1SJeff Roberson * 2609ab3185d1SJeff Roberson * If 'rr' is 1, search all domains starting from 'domain'. Otherwise check 2610ab3185d1SJeff Roberson * only 'domain'. 2611ab3185d1SJeff Roberson */ 2612ab3185d1SJeff Roberson static uma_slab_t 2613194a979eSMark Johnston keg_first_slab(uma_keg_t keg, int domain, bool rr) 2614ab3185d1SJeff Roberson { 2615ab3185d1SJeff Roberson uma_domain_t dom; 2616bbee39c6SJeff Roberson uma_slab_t slab; 2617ab3185d1SJeff Roberson int start; 2618ab3185d1SJeff Roberson 2619ab3185d1SJeff Roberson KASSERT(domain >= 0 && domain < vm_ndomains, 2620ab3185d1SJeff Roberson ("keg_first_slab: domain %d out of range", domain)); 2621bb15d1c7SGleb Smirnoff KEG_LOCK_ASSERT(keg); 2622ab3185d1SJeff Roberson 2623ab3185d1SJeff Roberson slab = NULL; 2624ab3185d1SJeff Roberson start = domain; 2625ab3185d1SJeff Roberson do { 2626ab3185d1SJeff Roberson dom = &keg->uk_domain[domain]; 2627ab3185d1SJeff Roberson if (!LIST_EMPTY(&dom->ud_part_slab)) 2628ab3185d1SJeff Roberson return (LIST_FIRST(&dom->ud_part_slab)); 2629ab3185d1SJeff Roberson if (!LIST_EMPTY(&dom->ud_free_slab)) { 2630ab3185d1SJeff Roberson slab = LIST_FIRST(&dom->ud_free_slab); 2631ab3185d1SJeff Roberson LIST_REMOVE(slab, us_link); 2632ab3185d1SJeff Roberson LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); 2633ab3185d1SJeff Roberson return (slab); 2634ab3185d1SJeff Roberson } 2635ab3185d1SJeff Roberson if (rr) 2636ab3185d1SJeff Roberson domain = (domain + 1) % vm_ndomains; 2637ab3185d1SJeff Roberson } while (domain != start); 2638ab3185d1SJeff Roberson 2639ab3185d1SJeff Roberson return (NULL); 2640ab3185d1SJeff Roberson } 2641ab3185d1SJeff Roberson 2642ab3185d1SJeff Roberson static uma_slab_t 2643194a979eSMark Johnston keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags) 2644ab3185d1SJeff Roberson { 2645194a979eSMark Johnston uint32_t reserve; 2646099a0e58SBosko Milekic 2647bb15d1c7SGleb Smirnoff KEG_LOCK_ASSERT(keg); 2648194a979eSMark Johnston 2649194a979eSMark Johnston reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; 2650194a979eSMark Johnston if (keg->uk_free <= reserve) 2651194a979eSMark Johnston return (NULL); 2652194a979eSMark Johnston return (keg_first_slab(keg, domain, rr)); 2653194a979eSMark Johnston } 2654194a979eSMark Johnston 2655194a979eSMark Johnston static uma_slab_t 2656194a979eSMark Johnston keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) 2657194a979eSMark Johnston { 2658194a979eSMark Johnston struct vm_domainset_iter di; 2659194a979eSMark Johnston uma_domain_t dom; 2660194a979eSMark Johnston uma_slab_t slab; 2661194a979eSMark Johnston int aflags, domain; 2662194a979eSMark Johnston bool rr; 2663194a979eSMark Johnston 2664194a979eSMark Johnston restart: 2665bb15d1c7SGleb Smirnoff KEG_LOCK_ASSERT(keg); 2666bbee39c6SJeff Roberson 2667bbee39c6SJeff Roberson /* 2668194a979eSMark Johnston * Use the keg's policy if upper layers haven't already specified a 2669194a979eSMark Johnston * domain (as happens with first-touch zones). 2670194a979eSMark Johnston * 2671194a979eSMark Johnston * To avoid races we run the iterator with the keg lock held, but that 2672194a979eSMark Johnston * means that we cannot allow the vm_domainset layer to sleep. Thus, 2673194a979eSMark Johnston * clear M_WAITOK and handle low memory conditions locally. 2674bbee39c6SJeff Roberson */ 2675ab3185d1SJeff Roberson rr = rdomain == UMA_ANYDOMAIN; 2676ab3185d1SJeff Roberson if (rr) { 2677194a979eSMark Johnston aflags = (flags & ~M_WAITOK) | M_NOWAIT; 2678194a979eSMark Johnston vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, 2679194a979eSMark Johnston &aflags); 2680194a979eSMark Johnston } else { 2681194a979eSMark Johnston aflags = flags; 2682194a979eSMark Johnston domain = rdomain; 2683194a979eSMark Johnston } 2684ab3185d1SJeff Roberson 2685194a979eSMark Johnston for (;;) { 2686194a979eSMark Johnston slab = keg_fetch_free_slab(keg, domain, rr, flags); 2687194a979eSMark Johnston if (slab != NULL) { 2688e20a199fSJeff Roberson MPASS(slab->us_keg == keg); 2689bbee39c6SJeff Roberson return (slab); 2690bbee39c6SJeff Roberson } 2691bbee39c6SJeff Roberson 2692bbee39c6SJeff Roberson /* 2693bbee39c6SJeff Roberson * M_NOVM means don't ask at all! 2694bbee39c6SJeff Roberson */ 2695bbee39c6SJeff Roberson if (flags & M_NOVM) 2696bbee39c6SJeff Roberson break; 2697bbee39c6SJeff Roberson 2698bb15d1c7SGleb Smirnoff KASSERT(zone->uz_max_items == 0 || 2699bb15d1c7SGleb Smirnoff zone->uz_items <= zone->uz_max_items, 2700bb15d1c7SGleb Smirnoff ("%s: zone %p overflow", __func__, zone)); 2701bb15d1c7SGleb Smirnoff 270286220393SMark Johnston slab = keg_alloc_slab(keg, zone, domain, flags, aflags); 2703bbee39c6SJeff Roberson /* 2704bbee39c6SJeff Roberson * If we got a slab here it's safe to mark it partially used 2705bbee39c6SJeff Roberson * and return. We assume that the caller is going to remove 2706bbee39c6SJeff Roberson * at least one item. 2707bbee39c6SJeff Roberson */ 2708bbee39c6SJeff Roberson if (slab) { 2709e20a199fSJeff Roberson MPASS(slab->us_keg == keg); 2710ab3185d1SJeff Roberson dom = &keg->uk_domain[slab->us_domain]; 2711ab3185d1SJeff Roberson LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); 2712bbee39c6SJeff Roberson return (slab); 2713bbee39c6SJeff Roberson } 2714194a979eSMark Johnston KEG_LOCK(keg); 2715194a979eSMark Johnston if (rr && vm_domainset_iter_policy(&di, &domain) != 0) { 2716194a979eSMark Johnston if ((flags & M_WAITOK) != 0) { 2717194a979eSMark Johnston KEG_UNLOCK(keg); 2718194a979eSMark Johnston vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask); 2719194a979eSMark Johnston KEG_LOCK(keg); 2720194a979eSMark Johnston goto restart; 272130c5525bSAndrew Gallatin } 2722194a979eSMark Johnston break; 2723194a979eSMark Johnston } 2724ab3185d1SJeff Roberson } 2725ab3185d1SJeff Roberson 2726bbee39c6SJeff Roberson /* 2727bbee39c6SJeff Roberson * We might not have been able to get a slab but another cpu 2728bbee39c6SJeff Roberson * could have while we were unlocked. Check again before we 2729bbee39c6SJeff Roberson * fail. 2730bbee39c6SJeff Roberson */ 2731194a979eSMark Johnston if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) { 2732ab3185d1SJeff Roberson MPASS(slab->us_keg == keg); 2733bbee39c6SJeff Roberson return (slab); 2734bbee39c6SJeff Roberson } 2735ab3185d1SJeff Roberson return (NULL); 2736ab3185d1SJeff Roberson } 2737bbee39c6SJeff Roberson 2738e20a199fSJeff Roberson static uma_slab_t 2739ab3185d1SJeff Roberson zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int domain, int flags) 2740e20a199fSJeff Roberson { 2741e20a199fSJeff Roberson uma_slab_t slab; 2742e20a199fSJeff Roberson 2743af526374SJeff Roberson if (keg == NULL) { 2744bb15d1c7SGleb Smirnoff keg = zone->uz_keg; 2745af526374SJeff Roberson KEG_LOCK(keg); 2746af526374SJeff Roberson } 2747e20a199fSJeff Roberson 2748e20a199fSJeff Roberson for (;;) { 2749ab3185d1SJeff Roberson slab = keg_fetch_slab(keg, zone, domain, flags); 2750e20a199fSJeff Roberson if (slab) 2751e20a199fSJeff Roberson return (slab); 2752e20a199fSJeff Roberson if (flags & (M_NOWAIT | M_NOVM)) 2753e20a199fSJeff Roberson break; 2754e20a199fSJeff Roberson } 2755af526374SJeff Roberson KEG_UNLOCK(keg); 2756e20a199fSJeff Roberson return (NULL); 2757e20a199fSJeff Roberson } 2758e20a199fSJeff Roberson 2759d56368d7SBosko Milekic static void * 27600095a784SJeff Roberson slab_alloc_item(uma_keg_t keg, uma_slab_t slab) 2761bbee39c6SJeff Roberson { 2762ab3185d1SJeff Roberson uma_domain_t dom; 2763bbee39c6SJeff Roberson void *item; 276485dcf349SGleb Smirnoff uint8_t freei; 2765bbee39c6SJeff Roberson 27660095a784SJeff Roberson MPASS(keg == slab->us_keg); 2767bb15d1c7SGleb Smirnoff KEG_LOCK_ASSERT(keg); 2768099a0e58SBosko Milekic 2769ef72505eSJeff Roberson freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1; 2770ef72505eSJeff Roberson BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free); 2771099a0e58SBosko Milekic item = slab->us_data + (keg->uk_rsize * freei); 2772bbee39c6SJeff Roberson slab->us_freecount--; 2773099a0e58SBosko Milekic keg->uk_free--; 2774ef72505eSJeff Roberson 2775bbee39c6SJeff Roberson /* Move this slab to the full list */ 2776bbee39c6SJeff Roberson if (slab->us_freecount == 0) { 2777bbee39c6SJeff Roberson LIST_REMOVE(slab, us_link); 2778ab3185d1SJeff Roberson dom = &keg->uk_domain[slab->us_domain]; 2779ab3185d1SJeff Roberson LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link); 2780bbee39c6SJeff Roberson } 2781bbee39c6SJeff Roberson 2782bbee39c6SJeff Roberson return (item); 2783bbee39c6SJeff Roberson } 2784bbee39c6SJeff Roberson 2785bbee39c6SJeff Roberson static int 2786ab3185d1SJeff Roberson zone_import(uma_zone_t zone, void **bucket, int max, int domain, int flags) 27870095a784SJeff Roberson { 27880095a784SJeff Roberson uma_slab_t slab; 27890095a784SJeff Roberson uma_keg_t keg; 2790a03af342SSean Bruno #ifdef NUMA 2791ab3185d1SJeff Roberson int stripe; 2792a03af342SSean Bruno #endif 27930095a784SJeff Roberson int i; 27940095a784SJeff Roberson 27950095a784SJeff Roberson slab = NULL; 27960095a784SJeff Roberson keg = NULL; 2797af526374SJeff Roberson /* Try to keep the buckets totally full */ 27980095a784SJeff Roberson for (i = 0; i < max; ) { 2799ad66f958SGleb Smirnoff if ((slab = zone_fetch_slab(zone, keg, domain, flags)) == NULL) 28000095a784SJeff Roberson break; 28010095a784SJeff Roberson keg = slab->us_keg; 2802a03af342SSean Bruno #ifdef NUMA 2803ab3185d1SJeff Roberson stripe = howmany(max, vm_ndomains); 2804a03af342SSean Bruno #endif 28056fd34d6fSJeff Roberson while (slab->us_freecount && i < max) { 28060095a784SJeff Roberson bucket[i++] = slab_alloc_item(keg, slab); 28076fd34d6fSJeff Roberson if (keg->uk_free <= keg->uk_reserve) 28086fd34d6fSJeff Roberson break; 2809b6715dabSJeff Roberson #ifdef NUMA 2810ab3185d1SJeff Roberson /* 2811ab3185d1SJeff Roberson * If the zone is striped we pick a new slab for every 2812ab3185d1SJeff Roberson * N allocations. Eliminating this conditional will 2813ab3185d1SJeff Roberson * instead pick a new domain for each bucket rather 2814ab3185d1SJeff Roberson * than stripe within each bucket. The current option 2815ab3185d1SJeff Roberson * produces more fragmentation and requires more cpu 2816ab3185d1SJeff Roberson * time but yields better distribution. 2817ab3185d1SJeff Roberson */ 2818ab3185d1SJeff Roberson if ((zone->uz_flags & UMA_ZONE_NUMA) == 0 && 2819ab3185d1SJeff Roberson vm_ndomains > 1 && --stripe == 0) 2820ab3185d1SJeff Roberson break; 2821ab3185d1SJeff Roberson #endif 28226fd34d6fSJeff Roberson } 2823ab3185d1SJeff Roberson /* Don't block if we allocated any successfully. */ 28240095a784SJeff Roberson flags &= ~M_WAITOK; 28250095a784SJeff Roberson flags |= M_NOWAIT; 28260095a784SJeff Roberson } 28270095a784SJeff Roberson if (slab != NULL) 28280095a784SJeff Roberson KEG_UNLOCK(keg); 28290095a784SJeff Roberson 28300095a784SJeff Roberson return i; 28310095a784SJeff Roberson } 28320095a784SJeff Roberson 2833fc03d22bSJeff Roberson static uma_bucket_t 2834bb15d1c7SGleb Smirnoff zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags, int max) 2835bbee39c6SJeff Roberson { 2836bbee39c6SJeff Roberson uma_bucket_t bucket; 2837bbee39c6SJeff Roberson 283830c5525bSAndrew Gallatin CTR1(KTR_UMA, "zone_alloc:_bucket domain %d)", domain); 283930c5525bSAndrew Gallatin 28406fd34d6fSJeff Roberson /* Don't wait for buckets, preserve caller's NOVM setting. */ 28416fd34d6fSJeff Roberson bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); 28420095a784SJeff Roberson if (bucket == NULL) 2843f7104ccdSAlexander Motin return (NULL); 28440095a784SJeff Roberson 28450095a784SJeff Roberson bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, 284637125720SGleb Smirnoff MIN(max, bucket->ub_entries), domain, flags); 28470095a784SJeff Roberson 28480095a784SJeff Roberson /* 28490095a784SJeff Roberson * Initialize the memory if necessary. 28500095a784SJeff Roberson */ 28510095a784SJeff Roberson if (bucket->ub_cnt != 0 && zone->uz_init != NULL) { 2852099a0e58SBosko Milekic int i; 2853bbee39c6SJeff Roberson 28540095a784SJeff Roberson for (i = 0; i < bucket->ub_cnt; i++) 2855e20a199fSJeff Roberson if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size, 28560095a784SJeff Roberson flags) != 0) 2857b23f72e9SBrian Feldman break; 2858b23f72e9SBrian Feldman /* 2859b23f72e9SBrian Feldman * If we couldn't initialize the whole bucket, put the 2860b23f72e9SBrian Feldman * rest back onto the freelist. 2861b23f72e9SBrian Feldman */ 2862b23f72e9SBrian Feldman if (i != bucket->ub_cnt) { 2863af526374SJeff Roberson zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i], 28640095a784SJeff Roberson bucket->ub_cnt - i); 2865a5a262c6SBosko Milekic #ifdef INVARIANTS 28660095a784SJeff Roberson bzero(&bucket->ub_bucket[i], 28670095a784SJeff Roberson sizeof(void *) * (bucket->ub_cnt - i)); 2868a5a262c6SBosko Milekic #endif 2869b23f72e9SBrian Feldman bucket->ub_cnt = i; 2870b23f72e9SBrian Feldman } 2871099a0e58SBosko Milekic } 2872099a0e58SBosko Milekic 2873f7104ccdSAlexander Motin if (bucket->ub_cnt == 0) { 28746fd34d6fSJeff Roberson bucket_free(zone, bucket, udata); 28752efcc8cbSGleb Smirnoff counter_u64_add(zone->uz_fails, 1); 2876fc03d22bSJeff Roberson return (NULL); 2877bbee39c6SJeff Roberson } 2878fc03d22bSJeff Roberson 2879fc03d22bSJeff Roberson return (bucket); 2880fc03d22bSJeff Roberson } 2881fc03d22bSJeff Roberson 28828355f576SJeff Roberson /* 28830095a784SJeff Roberson * Allocates a single item from a zone. 28848355f576SJeff Roberson * 28858355f576SJeff Roberson * Arguments 28868355f576SJeff Roberson * zone The zone to alloc for. 28878355f576SJeff Roberson * udata The data to be passed to the constructor. 2888ab3185d1SJeff Roberson * domain The domain to allocate from or UMA_ANYDOMAIN. 2889a163d034SWarner Losh * flags M_WAITOK, M_NOWAIT, M_ZERO. 28908355f576SJeff Roberson * 28918355f576SJeff Roberson * Returns 28928355f576SJeff Roberson * NULL if there is no memory and M_NOWAIT is set 2893bbee39c6SJeff Roberson * An item if successful 28948355f576SJeff Roberson */ 28958355f576SJeff Roberson 28968355f576SJeff Roberson static void * 2897ab3185d1SJeff Roberson zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags) 28988355f576SJeff Roberson { 2899bb15d1c7SGleb Smirnoff 2900bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 2901bb15d1c7SGleb Smirnoff return (zone_alloc_item_locked(zone, udata, domain, flags)); 2902bb15d1c7SGleb Smirnoff } 2903bb15d1c7SGleb Smirnoff 2904bb15d1c7SGleb Smirnoff /* 2905bb15d1c7SGleb Smirnoff * Returns with zone unlocked. 2906bb15d1c7SGleb Smirnoff */ 2907bb15d1c7SGleb Smirnoff static void * 2908bb15d1c7SGleb Smirnoff zone_alloc_item_locked(uma_zone_t zone, void *udata, int domain, int flags) 2909bb15d1c7SGleb Smirnoff { 29108355f576SJeff Roberson void *item; 2911c5deaf04SGleb Smirnoff #ifdef INVARIANTS 2912c5deaf04SGleb Smirnoff bool skipdbg; 2913c5deaf04SGleb Smirnoff #endif 29148355f576SJeff Roberson 2915bb15d1c7SGleb Smirnoff ZONE_LOCK_ASSERT(zone); 2916bb15d1c7SGleb Smirnoff 2917bb45b411SGleb Smirnoff if (zone->uz_max_items > 0) { 2918bb45b411SGleb Smirnoff if (zone->uz_items >= zone->uz_max_items) { 2919bb15d1c7SGleb Smirnoff zone_log_warning(zone); 2920bb15d1c7SGleb Smirnoff zone_maxaction(zone); 2921bb15d1c7SGleb Smirnoff if (flags & M_NOWAIT) { 2922bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 2923bb15d1c7SGleb Smirnoff return (NULL); 2924bb15d1c7SGleb Smirnoff } 2925bb15d1c7SGleb Smirnoff zone->uz_sleeps++; 2926bb15d1c7SGleb Smirnoff zone->uz_sleepers++; 2927bb15d1c7SGleb Smirnoff while (zone->uz_items >= zone->uz_max_items) 2928e7e4bcd8SGleb Smirnoff mtx_sleep(zone, zone->uz_lockptr, PVM, 2929e7e4bcd8SGleb Smirnoff "zonelimit", 0); 2930bb15d1c7SGleb Smirnoff zone->uz_sleepers--; 2931bb15d1c7SGleb Smirnoff if (zone->uz_sleepers > 0 && 2932bb15d1c7SGleb Smirnoff zone->uz_items + 1 < zone->uz_max_items) 2933bb15d1c7SGleb Smirnoff wakeup_one(zone); 2934bb15d1c7SGleb Smirnoff } 2935bb15d1c7SGleb Smirnoff zone->uz_items++; 2936bb45b411SGleb Smirnoff } 2937bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 29388355f576SJeff Roberson 293930c5525bSAndrew Gallatin if (domain != UMA_ANYDOMAIN) { 294030c5525bSAndrew Gallatin /* avoid allocs targeting empty domains */ 294130c5525bSAndrew Gallatin if (VM_DOMAIN_EMPTY(domain)) 294230c5525bSAndrew Gallatin domain = UMA_ANYDOMAIN; 294330c5525bSAndrew Gallatin } 2944ab3185d1SJeff Roberson if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) 29450095a784SJeff Roberson goto fail; 29468355f576SJeff Roberson 2947c5deaf04SGleb Smirnoff #ifdef INVARIANTS 2948c5deaf04SGleb Smirnoff skipdbg = uma_dbg_zskip(zone, item); 2949c5deaf04SGleb Smirnoff #endif 2950099a0e58SBosko Milekic /* 2951099a0e58SBosko Milekic * We have to call both the zone's init (not the keg's init) 2952099a0e58SBosko Milekic * and the zone's ctor. This is because the item is going from 2953099a0e58SBosko Milekic * a keg slab directly to the user, and the user is expecting it 2954099a0e58SBosko Milekic * to be both zone-init'd as well as zone-ctor'd. 2955099a0e58SBosko Milekic */ 2956b23f72e9SBrian Feldman if (zone->uz_init != NULL) { 2957e20a199fSJeff Roberson if (zone->uz_init(item, zone->uz_size, flags) != 0) { 2958bb15d1c7SGleb Smirnoff zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT); 29590095a784SJeff Roberson goto fail; 2960b23f72e9SBrian Feldman } 2961b23f72e9SBrian Feldman } 2962c5deaf04SGleb Smirnoff if (zone->uz_ctor != NULL && 2963c5deaf04SGleb Smirnoff #ifdef INVARIANTS 2964c5deaf04SGleb Smirnoff (!skipdbg || zone->uz_ctor != trash_ctor || 2965c5deaf04SGleb Smirnoff zone->uz_dtor != trash_dtor) && 2966c5deaf04SGleb Smirnoff #endif 2967c5deaf04SGleb Smirnoff zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { 2968bb15d1c7SGleb Smirnoff zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); 29690095a784SJeff Roberson goto fail; 2970b23f72e9SBrian Feldman } 2971ef72505eSJeff Roberson #ifdef INVARIANTS 2972c5deaf04SGleb Smirnoff if (!skipdbg) 29730095a784SJeff Roberson uma_dbg_alloc(zone, NULL, item); 2974ef72505eSJeff Roberson #endif 29752cc35ff9SJeff Roberson if (flags & M_ZERO) 297648343a2fSGleb Smirnoff uma_zero_item(item, zone); 29778355f576SJeff Roberson 29782efcc8cbSGleb Smirnoff counter_u64_add(zone->uz_allocs, 1); 29791431a748SGleb Smirnoff CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item, 29801431a748SGleb Smirnoff zone->uz_name, zone); 29811431a748SGleb Smirnoff 29828355f576SJeff Roberson return (item); 29830095a784SJeff Roberson 29840095a784SJeff Roberson fail: 2985bb45b411SGleb Smirnoff if (zone->uz_max_items > 0) { 2986bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 2987bb15d1c7SGleb Smirnoff zone->uz_items--; 2988bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 2989bb45b411SGleb Smirnoff } 29902efcc8cbSGleb Smirnoff counter_u64_add(zone->uz_fails, 1); 29911431a748SGleb Smirnoff CTR2(KTR_UMA, "zone_alloc_item failed from %s(%p)", 29921431a748SGleb Smirnoff zone->uz_name, zone); 29930095a784SJeff Roberson return (NULL); 29948355f576SJeff Roberson } 29958355f576SJeff Roberson 29968355f576SJeff Roberson /* See uma.h */ 29978355f576SJeff Roberson void 29988355f576SJeff Roberson uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 29998355f576SJeff Roberson { 30008355f576SJeff Roberson uma_cache_t cache; 30018355f576SJeff Roberson uma_bucket_t bucket; 3002ab3185d1SJeff Roberson uma_zone_domain_t zdom; 3003bb15d1c7SGleb Smirnoff int cpu, domain; 3004bb15d1c7SGleb Smirnoff bool lockfail; 3005c5deaf04SGleb Smirnoff #ifdef INVARIANTS 3006c5deaf04SGleb Smirnoff bool skipdbg; 3007c5deaf04SGleb Smirnoff #endif 30088355f576SJeff Roberson 3009e866d8f0SMark Murray /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 301019fa89e9SMark Murray random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 301110cb2424SMark Murray 30123659f747SRobert Watson CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, 30133659f747SRobert Watson zone->uz_name); 30143659f747SRobert Watson 3015d9e2e68dSMark Johnston KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), 30161067a2baSJonathan T. Looney ("uma_zfree_arg: called with spinlock or critical section held")); 30171067a2baSJonathan T. Looney 301820ed0cb0SMatthew D Fleming /* uma_zfree(..., NULL) does nothing, to match free(9). */ 301920ed0cb0SMatthew D Fleming if (item == NULL) 302020ed0cb0SMatthew D Fleming return; 30218d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD 30228d689e04SGleb Smirnoff if (is_memguard_addr(item)) { 3023bc9d08e1SMark Johnston if (zone->uz_dtor != NULL) 30248d689e04SGleb Smirnoff zone->uz_dtor(item, zone->uz_size, udata); 3025bc9d08e1SMark Johnston if (zone->uz_fini != NULL) 30268d689e04SGleb Smirnoff zone->uz_fini(item, zone->uz_size); 30278d689e04SGleb Smirnoff memguard_free(item); 30288d689e04SGleb Smirnoff return; 30298d689e04SGleb Smirnoff } 30308d689e04SGleb Smirnoff #endif 30315d1ae027SRobert Watson #ifdef INVARIANTS 3032c5deaf04SGleb Smirnoff skipdbg = uma_dbg_zskip(zone, item); 3033c5deaf04SGleb Smirnoff if (skipdbg == false) { 3034e20a199fSJeff Roberson if (zone->uz_flags & UMA_ZONE_MALLOC) 30355d1ae027SRobert Watson uma_dbg_free(zone, udata, item); 30365d1ae027SRobert Watson else 30375d1ae027SRobert Watson uma_dbg_free(zone, NULL, item); 3038c5deaf04SGleb Smirnoff } 3039c5deaf04SGleb Smirnoff if (zone->uz_dtor != NULL && (!skipdbg || 3040c5deaf04SGleb Smirnoff zone->uz_dtor != trash_dtor || zone->uz_ctor != trash_ctor)) 3041c5deaf04SGleb Smirnoff #else 3042fc03d22bSJeff Roberson if (zone->uz_dtor != NULL) 3043c5deaf04SGleb Smirnoff #endif 3044ef72505eSJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 3045ef72505eSJeff Roberson 3046af7f9b97SJeff Roberson /* 3047af7f9b97SJeff Roberson * The race here is acceptable. If we miss it we'll just have to wait 3048af7f9b97SJeff Roberson * a little longer for the limits to be reset. 3049af7f9b97SJeff Roberson */ 3050bb15d1c7SGleb Smirnoff if (zone->uz_sleepers > 0) 3051fc03d22bSJeff Roberson goto zfree_item; 3052af7f9b97SJeff Roberson 30535d1ae027SRobert Watson /* 30545d1ae027SRobert Watson * If possible, free to the per-CPU cache. There are two 30555d1ae027SRobert Watson * requirements for safe access to the per-CPU cache: (1) the thread 30565d1ae027SRobert Watson * accessing the cache must not be preempted or yield during access, 30575d1ae027SRobert Watson * and (2) the thread must not migrate CPUs without switching which 30585d1ae027SRobert Watson * cache it accesses. We rely on a critical section to prevent 30595d1ae027SRobert Watson * preemption and migration. We release the critical section in 30605d1ae027SRobert Watson * order to acquire the zone mutex if we are unable to free to the 30615d1ae027SRobert Watson * current cache; when we re-acquire the critical section, we must 30625d1ae027SRobert Watson * detect and handle migration if it has occurred. 30635d1ae027SRobert Watson */ 3064a553d4b8SJeff Roberson zfree_restart: 30655d1ae027SRobert Watson critical_enter(); 30665d1ae027SRobert Watson cpu = curcpu; 30678355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 30688355f576SJeff Roberson 30698355f576SJeff Roberson zfree_start: 3070a553d4b8SJeff Roberson /* 3071fc03d22bSJeff Roberson * Try to free into the allocbucket first to give LIFO ordering 3072fc03d22bSJeff Roberson * for cache-hot datastructures. Spill over into the freebucket 3073fc03d22bSJeff Roberson * if necessary. Alloc will swap them if one runs dry. 3074a553d4b8SJeff Roberson */ 3075fc03d22bSJeff Roberson bucket = cache->uc_allocbucket; 3076fc03d22bSJeff Roberson if (bucket == NULL || bucket->ub_cnt >= bucket->ub_entries) 3077fc03d22bSJeff Roberson bucket = cache->uc_freebucket; 3078fc03d22bSJeff Roberson if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) { 3079cae33c14SJeff Roberson KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL, 30808355f576SJeff Roberson ("uma_zfree: Freeing to non free bucket index.")); 3081cae33c14SJeff Roberson bucket->ub_bucket[bucket->ub_cnt] = item; 3082cae33c14SJeff Roberson bucket->ub_cnt++; 3083773df9abSRobert Watson cache->uc_frees++; 30845d1ae027SRobert Watson critical_exit(); 30858355f576SJeff Roberson return; 3086fc03d22bSJeff Roberson } 3087fc03d22bSJeff Roberson 30888355f576SJeff Roberson /* 30895d1ae027SRobert Watson * We must go back the zone, which requires acquiring the zone lock, 30905d1ae027SRobert Watson * which in turn means we must release and re-acquire the critical 30915d1ae027SRobert Watson * section. Since the critical section is released, we may be 30925d1ae027SRobert Watson * preempted or migrate. As such, make sure not to maintain any 30935d1ae027SRobert Watson * thread-local state specific to the cache from prior to releasing 30945d1ae027SRobert Watson * the critical section. 30958355f576SJeff Roberson */ 30965d1ae027SRobert Watson critical_exit(); 3097fc03d22bSJeff Roberson if (zone->uz_count == 0 || bucketdisable) 3098fc03d22bSJeff Roberson goto zfree_item; 3099fc03d22bSJeff Roberson 3100bb15d1c7SGleb Smirnoff lockfail = false; 31014d104ba0SAlexander Motin if (ZONE_TRYLOCK(zone) == 0) { 31024d104ba0SAlexander Motin /* Record contention to size the buckets. */ 31038355f576SJeff Roberson ZONE_LOCK(zone); 3104bb15d1c7SGleb Smirnoff lockfail = true; 31054d104ba0SAlexander Motin } 31065d1ae027SRobert Watson critical_enter(); 31075d1ae027SRobert Watson cpu = curcpu; 31085d1ae027SRobert Watson cache = &zone->uz_cpu[cpu]; 31098355f576SJeff Roberson 31108355f576SJeff Roberson bucket = cache->uc_freebucket; 3111fc03d22bSJeff Roberson if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) { 3112fc03d22bSJeff Roberson ZONE_UNLOCK(zone); 3113fc03d22bSJeff Roberson goto zfree_start; 3114fc03d22bSJeff Roberson } 31158355f576SJeff Roberson cache->uc_freebucket = NULL; 3116afa5d703SMark Johnston /* We are no longer associated with this CPU. */ 3117afa5d703SMark Johnston critical_exit(); 31188355f576SJeff Roberson 311930c5525bSAndrew Gallatin if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) { 3120ab3185d1SJeff Roberson domain = PCPU_GET(domain); 312130c5525bSAndrew Gallatin if (VM_DOMAIN_EMPTY(domain)) 312230c5525bSAndrew Gallatin domain = UMA_ANYDOMAIN; 312330c5525bSAndrew Gallatin } else 3124ab3185d1SJeff Roberson domain = 0; 3125ab3185d1SJeff Roberson zdom = &zone->uz_domain[0]; 3126ab3185d1SJeff Roberson 31278355f576SJeff Roberson /* Can we throw this on the zone full list? */ 31288355f576SJeff Roberson if (bucket != NULL) { 31291431a748SGleb Smirnoff CTR3(KTR_UMA, 31301431a748SGleb Smirnoff "uma_zfree: zone %s(%p) putting bucket %p on free list", 31311431a748SGleb Smirnoff zone->uz_name, zone, bucket); 3132cae33c14SJeff Roberson /* ub_cnt is pointing to the last free item */ 3133bb15d1c7SGleb Smirnoff KASSERT(bucket->ub_cnt == bucket->ub_entries, 3134bb15d1c7SGleb Smirnoff ("uma_zfree: Attempting to insert not full bucket onto the full list.\n")); 3135bb15d1c7SGleb Smirnoff if (zone->uz_bkt_count >= zone->uz_bkt_max) { 3136e8bb2dc7SJeff Roberson ZONE_UNLOCK(zone); 3137e8bb2dc7SJeff Roberson bucket_drain(zone, bucket); 3138e8bb2dc7SJeff Roberson bucket_free(zone, bucket, udata); 3139e8bb2dc7SJeff Roberson goto zfree_restart; 3140e8bb2dc7SJeff Roberson } else 31410f9b7bf3SMark Johnston zone_put_bucket(zone, zdom, bucket, true); 31428355f576SJeff Roberson } 3143fc03d22bSJeff Roberson 31444d104ba0SAlexander Motin /* 31454d104ba0SAlexander Motin * We bump the uz count when the cache size is insufficient to 31464d104ba0SAlexander Motin * handle the working set. 31474d104ba0SAlexander Motin */ 3148bb15d1c7SGleb Smirnoff if (lockfail && zone->uz_count < zone->uz_count_max) 31494d104ba0SAlexander Motin zone->uz_count++; 3150a553d4b8SJeff Roberson ZONE_UNLOCK(zone); 3151a553d4b8SJeff Roberson 31526fd34d6fSJeff Roberson bucket = bucket_alloc(zone, udata, M_NOWAIT); 31531431a748SGleb Smirnoff CTR3(KTR_UMA, "uma_zfree: zone %s(%p) allocated bucket %p", 31541431a748SGleb Smirnoff zone->uz_name, zone, bucket); 31554741dcbfSJeff Roberson if (bucket) { 3156fc03d22bSJeff Roberson critical_enter(); 3157fc03d22bSJeff Roberson cpu = curcpu; 3158fc03d22bSJeff Roberson cache = &zone->uz_cpu[cpu]; 3159ab3185d1SJeff Roberson if (cache->uc_freebucket == NULL && 3160ab3185d1SJeff Roberson ((zone->uz_flags & UMA_ZONE_NUMA) == 0 || 3161ab3185d1SJeff Roberson domain == PCPU_GET(domain))) { 3162fc03d22bSJeff Roberson cache->uc_freebucket = bucket; 3163fc03d22bSJeff Roberson goto zfree_start; 3164fc03d22bSJeff Roberson } 3165fc03d22bSJeff Roberson /* 3166fc03d22bSJeff Roberson * We lost the race, start over. We have to drop our 3167fc03d22bSJeff Roberson * critical section to free the bucket. 3168fc03d22bSJeff Roberson */ 3169fc03d22bSJeff Roberson critical_exit(); 31706fd34d6fSJeff Roberson bucket_free(zone, bucket, udata); 3171a553d4b8SJeff Roberson goto zfree_restart; 31728355f576SJeff Roberson } 31738355f576SJeff Roberson 3174a553d4b8SJeff Roberson /* 3175a553d4b8SJeff Roberson * If nothing else caught this, we'll just do an internal free. 3176a553d4b8SJeff Roberson */ 3177fc03d22bSJeff Roberson zfree_item: 31780095a784SJeff Roberson zone_free_item(zone, item, udata, SKIP_DTOR); 31798355f576SJeff Roberson } 31808355f576SJeff Roberson 3181ab3185d1SJeff Roberson void 3182ab3185d1SJeff Roberson uma_zfree_domain(uma_zone_t zone, void *item, void *udata) 3183ab3185d1SJeff Roberson { 3184ab3185d1SJeff Roberson 3185ab3185d1SJeff Roberson /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 318619fa89e9SMark Murray random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 3187ab3185d1SJeff Roberson 3188ab3185d1SJeff Roberson CTR2(KTR_UMA, "uma_zfree_domain thread %x zone %s", curthread, 3189ab3185d1SJeff Roberson zone->uz_name); 3190ab3185d1SJeff Roberson 3191ab3185d1SJeff Roberson KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), 3192ab3185d1SJeff Roberson ("uma_zfree_domain: called with spinlock or critical section held")); 3193ab3185d1SJeff Roberson 3194ab3185d1SJeff Roberson /* uma_zfree(..., NULL) does nothing, to match free(9). */ 3195ab3185d1SJeff Roberson if (item == NULL) 3196ab3185d1SJeff Roberson return; 3197ab3185d1SJeff Roberson zone_free_item(zone, item, udata, SKIP_NONE); 3198ab3185d1SJeff Roberson } 3199ab3185d1SJeff Roberson 32008355f576SJeff Roberson static void 3201bb15d1c7SGleb Smirnoff slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item) 32028355f576SJeff Roberson { 3203bb15d1c7SGleb Smirnoff uma_keg_t keg; 3204ab3185d1SJeff Roberson uma_domain_t dom; 320585dcf349SGleb Smirnoff uint8_t freei; 3206099a0e58SBosko Milekic 3207bb15d1c7SGleb Smirnoff keg = zone->uz_keg; 3208bb15d1c7SGleb Smirnoff MPASS(zone->uz_lockptr == &keg->uk_lock); 3209bb15d1c7SGleb Smirnoff KEG_LOCK_ASSERT(keg); 3210e20a199fSJeff Roberson MPASS(keg == slab->us_keg); 32118355f576SJeff Roberson 3212ab3185d1SJeff Roberson dom = &keg->uk_domain[slab->us_domain]; 3213ab3185d1SJeff Roberson 32148355f576SJeff Roberson /* Do we need to remove from any lists? */ 3215099a0e58SBosko Milekic if (slab->us_freecount+1 == keg->uk_ipers) { 32168355f576SJeff Roberson LIST_REMOVE(slab, us_link); 3217ab3185d1SJeff Roberson LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); 32188355f576SJeff Roberson } else if (slab->us_freecount == 0) { 32198355f576SJeff Roberson LIST_REMOVE(slab, us_link); 3220ab3185d1SJeff Roberson LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); 32218355f576SJeff Roberson } 32228355f576SJeff Roberson 3223ef72505eSJeff Roberson /* Slab management. */ 3224ef72505eSJeff Roberson freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; 3225ef72505eSJeff Roberson BIT_SET(SLAB_SETSIZE, freei, &slab->us_free); 32268355f576SJeff Roberson slab->us_freecount++; 32278355f576SJeff Roberson 3228ef72505eSJeff Roberson /* Keg statistics. */ 3229099a0e58SBosko Milekic keg->uk_free++; 32300095a784SJeff Roberson } 32310095a784SJeff Roberson 32320095a784SJeff Roberson static void 32330095a784SJeff Roberson zone_release(uma_zone_t zone, void **bucket, int cnt) 32340095a784SJeff Roberson { 32350095a784SJeff Roberson void *item; 32360095a784SJeff Roberson uma_slab_t slab; 32370095a784SJeff Roberson uma_keg_t keg; 32380095a784SJeff Roberson uint8_t *mem; 32390095a784SJeff Roberson int i; 32408355f576SJeff Roberson 3241bb15d1c7SGleb Smirnoff keg = zone->uz_keg; 3242af526374SJeff Roberson KEG_LOCK(keg); 32430095a784SJeff Roberson for (i = 0; i < cnt; i++) { 32440095a784SJeff Roberson item = bucket[i]; 32450095a784SJeff Roberson if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) { 32460095a784SJeff Roberson mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); 32470095a784SJeff Roberson if (zone->uz_flags & UMA_ZONE_HASH) { 32480095a784SJeff Roberson slab = hash_sfind(&keg->uk_hash, mem); 32490095a784SJeff Roberson } else { 32500095a784SJeff Roberson mem += keg->uk_pgoff; 32510095a784SJeff Roberson slab = (uma_slab_t)mem; 32520095a784SJeff Roberson } 32530095a784SJeff Roberson } else { 32540095a784SJeff Roberson slab = vtoslab((vm_offset_t)item); 3255bb15d1c7SGleb Smirnoff MPASS(slab->us_keg == keg); 32560095a784SJeff Roberson } 3257bb15d1c7SGleb Smirnoff slab_free_item(zone, slab, item); 32580095a784SJeff Roberson } 3259af526374SJeff Roberson KEG_UNLOCK(keg); 32608355f576SJeff Roberson } 32618355f576SJeff Roberson 32620095a784SJeff Roberson /* 32630095a784SJeff Roberson * Frees a single item to any zone. 32640095a784SJeff Roberson * 32650095a784SJeff Roberson * Arguments: 32660095a784SJeff Roberson * zone The zone to free to 32670095a784SJeff Roberson * item The item we're freeing 32680095a784SJeff Roberson * udata User supplied data for the dtor 32690095a784SJeff Roberson * skip Skip dtors and finis 32700095a784SJeff Roberson */ 32710095a784SJeff Roberson static void 32720095a784SJeff Roberson zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) 32730095a784SJeff Roberson { 32740095a784SJeff Roberson #ifdef INVARIANTS 3275c5deaf04SGleb Smirnoff bool skipdbg; 3276c5deaf04SGleb Smirnoff 3277c5deaf04SGleb Smirnoff skipdbg = uma_dbg_zskip(zone, item); 3278c5deaf04SGleb Smirnoff if (skip == SKIP_NONE && !skipdbg) { 32790095a784SJeff Roberson if (zone->uz_flags & UMA_ZONE_MALLOC) 32800095a784SJeff Roberson uma_dbg_free(zone, udata, item); 32810095a784SJeff Roberson else 32820095a784SJeff Roberson uma_dbg_free(zone, NULL, item); 32830095a784SJeff Roberson } 3284c5deaf04SGleb Smirnoff 3285c5deaf04SGleb Smirnoff if (skip < SKIP_DTOR && zone->uz_dtor != NULL && 3286c5deaf04SGleb Smirnoff (!skipdbg || zone->uz_dtor != trash_dtor || 3287c5deaf04SGleb Smirnoff zone->uz_ctor != trash_ctor)) 3288c5deaf04SGleb Smirnoff #else 3289c5deaf04SGleb Smirnoff if (skip < SKIP_DTOR && zone->uz_dtor != NULL) 32900095a784SJeff Roberson #endif 32910095a784SJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 32920095a784SJeff Roberson 32930095a784SJeff Roberson if (skip < SKIP_FINI && zone->uz_fini) 32940095a784SJeff Roberson zone->uz_fini(item, zone->uz_size); 32950095a784SJeff Roberson 32960095a784SJeff Roberson zone->uz_release(zone->uz_arg, &item, 1); 3297bb15d1c7SGleb Smirnoff 3298bb15d1c7SGleb Smirnoff if (skip & SKIP_CNT) 3299bb15d1c7SGleb Smirnoff return; 3300bb15d1c7SGleb Smirnoff 33012efcc8cbSGleb Smirnoff counter_u64_add(zone->uz_frees, 1); 33022efcc8cbSGleb Smirnoff 3303bb45b411SGleb Smirnoff if (zone->uz_max_items > 0) { 3304bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 3305bb15d1c7SGleb Smirnoff zone->uz_items--; 3306bb45b411SGleb Smirnoff if (zone->uz_sleepers > 0 && 3307bb45b411SGleb Smirnoff zone->uz_items < zone->uz_max_items) 3308bb15d1c7SGleb Smirnoff wakeup_one(zone); 3309bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 33100095a784SJeff Roberson } 3311bb45b411SGleb Smirnoff } 33120095a784SJeff Roberson 33138355f576SJeff Roberson /* See uma.h */ 33141c6cae97SLawrence Stewart int 3315736ee590SJeff Roberson uma_zone_set_max(uma_zone_t zone, int nitems) 3316736ee590SJeff Roberson { 3317bb15d1c7SGleb Smirnoff struct uma_bucket_zone *ubz; 3318099a0e58SBosko Milekic 3319bb15d1c7SGleb Smirnoff /* 3320bb15d1c7SGleb Smirnoff * If limit is very low we may need to limit how 3321bb15d1c7SGleb Smirnoff * much items are allowed in CPU caches. 3322bb15d1c7SGleb Smirnoff */ 3323bb15d1c7SGleb Smirnoff ubz = &bucket_zones[0]; 3324bb15d1c7SGleb Smirnoff for (; ubz->ubz_entries != 0; ubz++) 3325bb15d1c7SGleb Smirnoff if (ubz->ubz_entries * 2 * mp_ncpus > nitems) 3326bb15d1c7SGleb Smirnoff break; 3327bb15d1c7SGleb Smirnoff if (ubz == &bucket_zones[0]) 3328bb15d1c7SGleb Smirnoff nitems = ubz->ubz_entries * 2 * mp_ncpus; 3329bb15d1c7SGleb Smirnoff else 3330bb15d1c7SGleb Smirnoff ubz--; 3331bb15d1c7SGleb Smirnoff 3332bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 3333bb15d1c7SGleb Smirnoff zone->uz_count_max = zone->uz_count = ubz->ubz_entries; 3334bb15d1c7SGleb Smirnoff if (zone->uz_count_min > zone->uz_count_max) 3335bb15d1c7SGleb Smirnoff zone->uz_count_min = zone->uz_count_max; 3336bb15d1c7SGleb Smirnoff zone->uz_max_items = nitems; 3337bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 3338bb15d1c7SGleb Smirnoff 3339bb15d1c7SGleb Smirnoff return (nitems); 3340bb15d1c7SGleb Smirnoff } 3341bb15d1c7SGleb Smirnoff 3342bb15d1c7SGleb Smirnoff /* See uma.h */ 3343bb15d1c7SGleb Smirnoff int 3344bb15d1c7SGleb Smirnoff uma_zone_set_maxcache(uma_zone_t zone, int nitems) 3345bb15d1c7SGleb Smirnoff { 3346bb15d1c7SGleb Smirnoff 3347bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 3348bb15d1c7SGleb Smirnoff zone->uz_bkt_max = nitems; 3349bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 33501c6cae97SLawrence Stewart 33511c6cae97SLawrence Stewart return (nitems); 3352736ee590SJeff Roberson } 3353736ee590SJeff Roberson 3354736ee590SJeff Roberson /* See uma.h */ 3355e49471b0SAndre Oppermann int 3356e49471b0SAndre Oppermann uma_zone_get_max(uma_zone_t zone) 3357e49471b0SAndre Oppermann { 3358e49471b0SAndre Oppermann int nitems; 3359e49471b0SAndre Oppermann 3360bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 3361bb15d1c7SGleb Smirnoff nitems = zone->uz_max_items; 3362bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 3363e49471b0SAndre Oppermann 3364e49471b0SAndre Oppermann return (nitems); 3365e49471b0SAndre Oppermann } 3366e49471b0SAndre Oppermann 3367e49471b0SAndre Oppermann /* See uma.h */ 33682f891cd5SPawel Jakub Dawidek void 33692f891cd5SPawel Jakub Dawidek uma_zone_set_warning(uma_zone_t zone, const char *warning) 33702f891cd5SPawel Jakub Dawidek { 33712f891cd5SPawel Jakub Dawidek 33722f891cd5SPawel Jakub Dawidek ZONE_LOCK(zone); 33732f891cd5SPawel Jakub Dawidek zone->uz_warning = warning; 33742f891cd5SPawel Jakub Dawidek ZONE_UNLOCK(zone); 33752f891cd5SPawel Jakub Dawidek } 33762f891cd5SPawel Jakub Dawidek 33772f891cd5SPawel Jakub Dawidek /* See uma.h */ 337854503a13SJonathan T. Looney void 337954503a13SJonathan T. Looney uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction) 338054503a13SJonathan T. Looney { 338154503a13SJonathan T. Looney 338254503a13SJonathan T. Looney ZONE_LOCK(zone); 3383e60b2fcbSGleb Smirnoff TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone); 338454503a13SJonathan T. Looney ZONE_UNLOCK(zone); 338554503a13SJonathan T. Looney } 338654503a13SJonathan T. Looney 338754503a13SJonathan T. Looney /* See uma.h */ 3388c4ae7908SLawrence Stewart int 3389c4ae7908SLawrence Stewart uma_zone_get_cur(uma_zone_t zone) 3390c4ae7908SLawrence Stewart { 3391c4ae7908SLawrence Stewart int64_t nitems; 3392c4ae7908SLawrence Stewart u_int i; 3393c4ae7908SLawrence Stewart 3394c4ae7908SLawrence Stewart ZONE_LOCK(zone); 33952efcc8cbSGleb Smirnoff nitems = counter_u64_fetch(zone->uz_allocs) - 33962efcc8cbSGleb Smirnoff counter_u64_fetch(zone->uz_frees); 3397c4ae7908SLawrence Stewart CPU_FOREACH(i) { 3398c4ae7908SLawrence Stewart /* 33994a9f6ba7SGleb Smirnoff * See the comment in uma_vm_zone_stats() regarding the 3400c4ae7908SLawrence Stewart * safety of accessing the per-cpu caches. With the zone lock 3401c4ae7908SLawrence Stewart * held, it is safe, but can potentially result in stale data. 3402c4ae7908SLawrence Stewart */ 3403c4ae7908SLawrence Stewart nitems += zone->uz_cpu[i].uc_allocs - 3404c4ae7908SLawrence Stewart zone->uz_cpu[i].uc_frees; 3405c4ae7908SLawrence Stewart } 3406c4ae7908SLawrence Stewart ZONE_UNLOCK(zone); 3407c4ae7908SLawrence Stewart 3408c4ae7908SLawrence Stewart return (nitems < 0 ? 0 : nitems); 3409c4ae7908SLawrence Stewart } 3410c4ae7908SLawrence Stewart 3411c4ae7908SLawrence Stewart /* See uma.h */ 3412736ee590SJeff Roberson void 3413099a0e58SBosko Milekic uma_zone_set_init(uma_zone_t zone, uma_init uminit) 3414099a0e58SBosko Milekic { 3415e20a199fSJeff Roberson uma_keg_t keg; 3416e20a199fSJeff Roberson 3417bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 3418af526374SJeff Roberson KEG_LOCK(keg); 3419e20a199fSJeff Roberson KASSERT(keg->uk_pages == 0, 3420099a0e58SBosko Milekic ("uma_zone_set_init on non-empty keg")); 3421e20a199fSJeff Roberson keg->uk_init = uminit; 3422af526374SJeff Roberson KEG_UNLOCK(keg); 3423099a0e58SBosko Milekic } 3424099a0e58SBosko Milekic 3425099a0e58SBosko Milekic /* See uma.h */ 3426099a0e58SBosko Milekic void 3427099a0e58SBosko Milekic uma_zone_set_fini(uma_zone_t zone, uma_fini fini) 3428099a0e58SBosko Milekic { 3429e20a199fSJeff Roberson uma_keg_t keg; 3430e20a199fSJeff Roberson 3431bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 3432af526374SJeff Roberson KEG_LOCK(keg); 3433e20a199fSJeff Roberson KASSERT(keg->uk_pages == 0, 3434099a0e58SBosko Milekic ("uma_zone_set_fini on non-empty keg")); 3435e20a199fSJeff Roberson keg->uk_fini = fini; 3436af526374SJeff Roberson KEG_UNLOCK(keg); 3437099a0e58SBosko Milekic } 3438099a0e58SBosko Milekic 3439099a0e58SBosko Milekic /* See uma.h */ 3440099a0e58SBosko Milekic void 3441099a0e58SBosko Milekic uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) 3442099a0e58SBosko Milekic { 3443af526374SJeff Roberson 3444099a0e58SBosko Milekic ZONE_LOCK(zone); 3445bb15d1c7SGleb Smirnoff KASSERT(zone->uz_keg->uk_pages == 0, 3446099a0e58SBosko Milekic ("uma_zone_set_zinit on non-empty keg")); 3447099a0e58SBosko Milekic zone->uz_init = zinit; 3448099a0e58SBosko Milekic ZONE_UNLOCK(zone); 3449099a0e58SBosko Milekic } 3450099a0e58SBosko Milekic 3451099a0e58SBosko Milekic /* See uma.h */ 3452099a0e58SBosko Milekic void 3453099a0e58SBosko Milekic uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) 3454099a0e58SBosko Milekic { 3455af526374SJeff Roberson 3456099a0e58SBosko Milekic ZONE_LOCK(zone); 3457bb15d1c7SGleb Smirnoff KASSERT(zone->uz_keg->uk_pages == 0, 3458099a0e58SBosko Milekic ("uma_zone_set_zfini on non-empty keg")); 3459099a0e58SBosko Milekic zone->uz_fini = zfini; 3460099a0e58SBosko Milekic ZONE_UNLOCK(zone); 3461099a0e58SBosko Milekic } 3462099a0e58SBosko Milekic 3463099a0e58SBosko Milekic /* See uma.h */ 3464b23f72e9SBrian Feldman /* XXX uk_freef is not actually used with the zone locked */ 3465099a0e58SBosko Milekic void 34668355f576SJeff Roberson uma_zone_set_freef(uma_zone_t zone, uma_free freef) 34678355f576SJeff Roberson { 34680095a784SJeff Roberson uma_keg_t keg; 3469e20a199fSJeff Roberson 3470bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 34711d2c0c46SDmitry Chagin KASSERT(keg != NULL, ("uma_zone_set_freef: Invalid zone type")); 3472af526374SJeff Roberson KEG_LOCK(keg); 34730095a784SJeff Roberson keg->uk_freef = freef; 3474af526374SJeff Roberson KEG_UNLOCK(keg); 34758355f576SJeff Roberson } 34768355f576SJeff Roberson 34778355f576SJeff Roberson /* See uma.h */ 3478b23f72e9SBrian Feldman /* XXX uk_allocf is not actually used with the zone locked */ 34798355f576SJeff Roberson void 34808355f576SJeff Roberson uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 34818355f576SJeff Roberson { 3482e20a199fSJeff Roberson uma_keg_t keg; 3483e20a199fSJeff Roberson 3484bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 3485af526374SJeff Roberson KEG_LOCK(keg); 3486e20a199fSJeff Roberson keg->uk_allocf = allocf; 3487af526374SJeff Roberson KEG_UNLOCK(keg); 34888355f576SJeff Roberson } 34898355f576SJeff Roberson 34908355f576SJeff Roberson /* See uma.h */ 34916fd34d6fSJeff Roberson void 34926fd34d6fSJeff Roberson uma_zone_reserve(uma_zone_t zone, int items) 34936fd34d6fSJeff Roberson { 34946fd34d6fSJeff Roberson uma_keg_t keg; 34956fd34d6fSJeff Roberson 3496bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 34976fd34d6fSJeff Roberson KEG_LOCK(keg); 34986fd34d6fSJeff Roberson keg->uk_reserve = items; 34996fd34d6fSJeff Roberson KEG_UNLOCK(keg); 35006fd34d6fSJeff Roberson } 35016fd34d6fSJeff Roberson 35026fd34d6fSJeff Roberson /* See uma.h */ 35038355f576SJeff Roberson int 3504a4915c21SAttilio Rao uma_zone_reserve_kva(uma_zone_t zone, int count) 35058355f576SJeff Roberson { 3506099a0e58SBosko Milekic uma_keg_t keg; 35078355f576SJeff Roberson vm_offset_t kva; 35089ba30bcbSZbigniew Bodek u_int pages; 35098355f576SJeff Roberson 3510bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 35118355f576SJeff Roberson 3512bb15d1c7SGleb Smirnoff pages = count / keg->uk_ipers; 3513099a0e58SBosko Milekic if (pages * keg->uk_ipers < count) 35148355f576SJeff Roberson pages++; 351557223e99SAndriy Gapon pages *= keg->uk_ppera; 3516a553d4b8SJeff Roberson 3517a4915c21SAttilio Rao #ifdef UMA_MD_SMALL_ALLOC 3518a4915c21SAttilio Rao if (keg->uk_ppera > 1) { 3519a4915c21SAttilio Rao #else 3520a4915c21SAttilio Rao if (1) { 3521a4915c21SAttilio Rao #endif 352257223e99SAndriy Gapon kva = kva_alloc((vm_size_t)pages * PAGE_SIZE); 3523d1f42ac2SAlan Cox if (kva == 0) 35248355f576SJeff Roberson return (0); 3525a4915c21SAttilio Rao } else 3526a4915c21SAttilio Rao kva = 0; 3527bb15d1c7SGleb Smirnoff 3528bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 3529bb15d1c7SGleb Smirnoff MPASS(keg->uk_kva == 0); 3530099a0e58SBosko Milekic keg->uk_kva = kva; 3531a4915c21SAttilio Rao keg->uk_offset = 0; 3532bb15d1c7SGleb Smirnoff zone->uz_max_items = pages * keg->uk_ipers; 3533a4915c21SAttilio Rao #ifdef UMA_MD_SMALL_ALLOC 3534a4915c21SAttilio Rao keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc; 3535a4915c21SAttilio Rao #else 3536a4915c21SAttilio Rao keg->uk_allocf = noobj_alloc; 3537a4915c21SAttilio Rao #endif 35386fd34d6fSJeff Roberson keg->uk_flags |= UMA_ZONE_NOFREE; 3539bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 3540af526374SJeff Roberson 35418355f576SJeff Roberson return (1); 35428355f576SJeff Roberson } 35438355f576SJeff Roberson 35448355f576SJeff Roberson /* See uma.h */ 35458355f576SJeff Roberson void 35468355f576SJeff Roberson uma_prealloc(uma_zone_t zone, int items) 35478355f576SJeff Roberson { 3548920239efSMark Johnston struct vm_domainset_iter di; 3549ab3185d1SJeff Roberson uma_domain_t dom; 35508355f576SJeff Roberson uma_slab_t slab; 3551099a0e58SBosko Milekic uma_keg_t keg; 355286220393SMark Johnston int aflags, domain, slabs; 35538355f576SJeff Roberson 3554bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 3555af526374SJeff Roberson KEG_LOCK(keg); 3556099a0e58SBosko Milekic slabs = items / keg->uk_ipers; 3557099a0e58SBosko Milekic if (slabs * keg->uk_ipers < items) 35588355f576SJeff Roberson slabs++; 3559194a979eSMark Johnston while (slabs-- > 0) { 356086220393SMark Johnston aflags = M_NOWAIT; 356186220393SMark Johnston vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, 356286220393SMark Johnston &aflags); 356386220393SMark Johnston for (;;) { 356486220393SMark Johnston slab = keg_alloc_slab(keg, zone, domain, M_WAITOK, 356586220393SMark Johnston aflags); 356686220393SMark Johnston if (slab != NULL) { 3567e20a199fSJeff Roberson MPASS(slab->us_keg == keg); 3568ab3185d1SJeff Roberson dom = &keg->uk_domain[slab->us_domain]; 356986220393SMark Johnston LIST_INSERT_HEAD(&dom->ud_free_slab, slab, 357086220393SMark Johnston us_link); 3571920239efSMark Johnston break; 35728355f576SJeff Roberson } 357386220393SMark Johnston KEG_LOCK(keg); 357486220393SMark Johnston if (vm_domainset_iter_policy(&di, &domain) != 0) { 357586220393SMark Johnston KEG_UNLOCK(keg); 357686220393SMark Johnston vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask); 357786220393SMark Johnston KEG_LOCK(keg); 357886220393SMark Johnston } 357986220393SMark Johnston } 358086220393SMark Johnston } 3581af526374SJeff Roberson KEG_UNLOCK(keg); 35828355f576SJeff Roberson } 35838355f576SJeff Roberson 35848355f576SJeff Roberson /* See uma.h */ 358544ec2b63SKonstantin Belousov static void 358644ec2b63SKonstantin Belousov uma_reclaim_locked(bool kmem_danger) 35878355f576SJeff Roberson { 358844ec2b63SKonstantin Belousov 35891431a748SGleb Smirnoff CTR0(KTR_UMA, "UMA: vm asked us to release pages!"); 359044ec2b63SKonstantin Belousov sx_assert(&uma_drain_lock, SA_XLOCKED); 359186bbae32SJeff Roberson bucket_enable(); 35928355f576SJeff Roberson zone_foreach(zone_drain); 359344ec2b63SKonstantin Belousov if (vm_page_count_min() || kmem_danger) { 3594a2de44abSAlexander Motin cache_drain_safe(NULL); 3595a2de44abSAlexander Motin zone_foreach(zone_drain); 3596a2de44abSAlexander Motin } 35970f9b7bf3SMark Johnston 35988355f576SJeff Roberson /* 35998355f576SJeff Roberson * Some slabs may have been freed but this zone will be visited early 36008355f576SJeff Roberson * we visit again so that we can free pages that are empty once other 36018355f576SJeff Roberson * zones are drained. We have to do the same for buckets. 36028355f576SJeff Roberson */ 36039643769aSJeff Roberson zone_drain(slabzone); 3604cae33c14SJeff Roberson bucket_zone_drain(); 360544ec2b63SKonstantin Belousov } 360644ec2b63SKonstantin Belousov 360744ec2b63SKonstantin Belousov void 360844ec2b63SKonstantin Belousov uma_reclaim(void) 360944ec2b63SKonstantin Belousov { 361044ec2b63SKonstantin Belousov 361144ec2b63SKonstantin Belousov sx_xlock(&uma_drain_lock); 361244ec2b63SKonstantin Belousov uma_reclaim_locked(false); 361395c4bf75SKonstantin Belousov sx_xunlock(&uma_drain_lock); 36148355f576SJeff Roberson } 36158355f576SJeff Roberson 36162e47807cSJeff Roberson static volatile int uma_reclaim_needed; 361744ec2b63SKonstantin Belousov 361844ec2b63SKonstantin Belousov void 361944ec2b63SKonstantin Belousov uma_reclaim_wakeup(void) 362044ec2b63SKonstantin Belousov { 362144ec2b63SKonstantin Belousov 36222e47807cSJeff Roberson if (atomic_fetchadd_int(&uma_reclaim_needed, 1) == 0) 36232e47807cSJeff Roberson wakeup(uma_reclaim); 362444ec2b63SKonstantin Belousov } 362544ec2b63SKonstantin Belousov 362644ec2b63SKonstantin Belousov void 362744ec2b63SKonstantin Belousov uma_reclaim_worker(void *arg __unused) 362844ec2b63SKonstantin Belousov { 362944ec2b63SKonstantin Belousov 363044ec2b63SKonstantin Belousov for (;;) { 36312e47807cSJeff Roberson sx_xlock(&uma_drain_lock); 3632200f8117SKonstantin Belousov while (atomic_load_int(&uma_reclaim_needed) == 0) 36332e47807cSJeff Roberson sx_sleep(uma_reclaim, &uma_drain_lock, PVM, "umarcl", 36342e47807cSJeff Roberson hz); 36359b43bc27SAndriy Gapon sx_xunlock(&uma_drain_lock); 36369b43bc27SAndriy Gapon EVENTHANDLER_INVOKE(vm_lowmem, VM_LOW_KMEM); 36379b43bc27SAndriy Gapon sx_xlock(&uma_drain_lock); 363844ec2b63SKonstantin Belousov uma_reclaim_locked(true); 3639200f8117SKonstantin Belousov atomic_store_int(&uma_reclaim_needed, 0); 36402e47807cSJeff Roberson sx_xunlock(&uma_drain_lock); 36412e47807cSJeff Roberson /* Don't fire more than once per-second. */ 36422e47807cSJeff Roberson pause("umarclslp", hz); 364344ec2b63SKonstantin Belousov } 364444ec2b63SKonstantin Belousov } 364544ec2b63SKonstantin Belousov 3646663b416fSJohn Baldwin /* See uma.h */ 3647663b416fSJohn Baldwin int 3648663b416fSJohn Baldwin uma_zone_exhausted(uma_zone_t zone) 3649663b416fSJohn Baldwin { 3650663b416fSJohn Baldwin int full; 3651663b416fSJohn Baldwin 3652663b416fSJohn Baldwin ZONE_LOCK(zone); 3653bb15d1c7SGleb Smirnoff full = zone->uz_sleepers > 0; 3654663b416fSJohn Baldwin ZONE_UNLOCK(zone); 3655663b416fSJohn Baldwin return (full); 3656663b416fSJohn Baldwin } 3657663b416fSJohn Baldwin 36586c125b8dSMohan Srinivasan int 36596c125b8dSMohan Srinivasan uma_zone_exhausted_nolock(uma_zone_t zone) 36606c125b8dSMohan Srinivasan { 3661bb15d1c7SGleb Smirnoff return (zone->uz_sleepers > 0); 36626c125b8dSMohan Srinivasan } 36636c125b8dSMohan Srinivasan 36648355f576SJeff Roberson void * 3665ab3185d1SJeff Roberson uma_large_malloc_domain(vm_size_t size, int domain, int wait) 36668355f576SJeff Roberson { 36679978bd99SMark Johnston struct domainset *policy; 3668ab3185d1SJeff Roberson vm_offset_t addr; 36698355f576SJeff Roberson uma_slab_t slab; 36708355f576SJeff Roberson 367130c5525bSAndrew Gallatin if (domain != UMA_ANYDOMAIN) { 367230c5525bSAndrew Gallatin /* avoid allocs targeting empty domains */ 367330c5525bSAndrew Gallatin if (VM_DOMAIN_EMPTY(domain)) 367430c5525bSAndrew Gallatin domain = UMA_ANYDOMAIN; 367530c5525bSAndrew Gallatin } 3676ab3185d1SJeff Roberson slab = zone_alloc_item(slabzone, NULL, domain, wait); 36778355f576SJeff Roberson if (slab == NULL) 36788355f576SJeff Roberson return (NULL); 36799978bd99SMark Johnston policy = (domain == UMA_ANYDOMAIN) ? DOMAINSET_RR() : 36809978bd99SMark Johnston DOMAINSET_FIXED(domain); 36819978bd99SMark Johnston addr = kmem_malloc_domainset(policy, size, wait); 3682ab3185d1SJeff Roberson if (addr != 0) { 3683ab3185d1SJeff Roberson vsetslab(addr, slab); 3684ab3185d1SJeff Roberson slab->us_data = (void *)addr; 3685ab3185d1SJeff Roberson slab->us_flags = UMA_SLAB_KERNEL | UMA_SLAB_MALLOC; 36868355f576SJeff Roberson slab->us_size = size; 3687e2068d0bSJeff Roberson slab->us_domain = vm_phys_domain(PHYS_TO_VM_PAGE( 3688ab3185d1SJeff Roberson pmap_kextract(addr))); 36892e47807cSJeff Roberson uma_total_inc(size); 36908355f576SJeff Roberson } else { 36910095a784SJeff Roberson zone_free_item(slabzone, slab, NULL, SKIP_NONE); 36928355f576SJeff Roberson } 36938355f576SJeff Roberson 3694ab3185d1SJeff Roberson return ((void *)addr); 3695ab3185d1SJeff Roberson } 3696ab3185d1SJeff Roberson 3697ab3185d1SJeff Roberson void * 3698ab3185d1SJeff Roberson uma_large_malloc(vm_size_t size, int wait) 3699ab3185d1SJeff Roberson { 3700ab3185d1SJeff Roberson 3701ab3185d1SJeff Roberson return uma_large_malloc_domain(size, UMA_ANYDOMAIN, wait); 37028355f576SJeff Roberson } 37038355f576SJeff Roberson 37048355f576SJeff Roberson void 37058355f576SJeff Roberson uma_large_free(uma_slab_t slab) 37068355f576SJeff Roberson { 3707c325e866SKonstantin Belousov 3708ab3185d1SJeff Roberson KASSERT((slab->us_flags & UMA_SLAB_KERNEL) != 0, 3709ab3185d1SJeff Roberson ("uma_large_free: Memory not allocated with uma_large_malloc.")); 371049bfa624SAlan Cox kmem_free((vm_offset_t)slab->us_data, slab->us_size); 37112e47807cSJeff Roberson uma_total_dec(slab->us_size); 37120095a784SJeff Roberson zone_free_item(slabzone, slab, NULL, SKIP_NONE); 37138355f576SJeff Roberson } 37148355f576SJeff Roberson 371548343a2fSGleb Smirnoff static void 371648343a2fSGleb Smirnoff uma_zero_item(void *item, uma_zone_t zone) 371748343a2fSGleb Smirnoff { 371848343a2fSGleb Smirnoff 371948343a2fSGleb Smirnoff bzero(item, zone->uz_size); 372048343a2fSGleb Smirnoff } 372148343a2fSGleb Smirnoff 37222e47807cSJeff Roberson unsigned long 37232e47807cSJeff Roberson uma_limit(void) 37242e47807cSJeff Roberson { 37252e47807cSJeff Roberson 37262e47807cSJeff Roberson return (uma_kmem_limit); 37272e47807cSJeff Roberson } 37282e47807cSJeff Roberson 37292e47807cSJeff Roberson void 37302e47807cSJeff Roberson uma_set_limit(unsigned long limit) 37312e47807cSJeff Roberson { 37322e47807cSJeff Roberson 37332e47807cSJeff Roberson uma_kmem_limit = limit; 37342e47807cSJeff Roberson } 37352e47807cSJeff Roberson 37362e47807cSJeff Roberson unsigned long 37372e47807cSJeff Roberson uma_size(void) 37382e47807cSJeff Roberson { 37392e47807cSJeff Roberson 3740*058f0f74SMark Johnston return (atomic_load_long(&uma_kmem_total)); 3741ad5b0f5bSJeff Roberson } 3742ad5b0f5bSJeff Roberson 3743ad5b0f5bSJeff Roberson long 3744ad5b0f5bSJeff Roberson uma_avail(void) 3745ad5b0f5bSJeff Roberson { 3746ad5b0f5bSJeff Roberson 3747*058f0f74SMark Johnston return (uma_kmem_limit - uma_size()); 37482e47807cSJeff Roberson } 37492e47807cSJeff Roberson 37508355f576SJeff Roberson void 37518355f576SJeff Roberson uma_print_stats(void) 37528355f576SJeff Roberson { 37538355f576SJeff Roberson zone_foreach(uma_print_zone); 37548355f576SJeff Roberson } 37558355f576SJeff Roberson 3756504d5de3SJeff Roberson static void 3757504d5de3SJeff Roberson slab_print(uma_slab_t slab) 3758504d5de3SJeff Roberson { 3759ef72505eSJeff Roberson printf("slab: keg %p, data %p, freecount %d\n", 3760ef72505eSJeff Roberson slab->us_keg, slab->us_data, slab->us_freecount); 3761504d5de3SJeff Roberson } 3762504d5de3SJeff Roberson 3763504d5de3SJeff Roberson static void 3764504d5de3SJeff Roberson cache_print(uma_cache_t cache) 3765504d5de3SJeff Roberson { 3766504d5de3SJeff Roberson printf("alloc: %p(%d), free: %p(%d)\n", 3767504d5de3SJeff Roberson cache->uc_allocbucket, 3768504d5de3SJeff Roberson cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0, 3769504d5de3SJeff Roberson cache->uc_freebucket, 3770504d5de3SJeff Roberson cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0); 3771504d5de3SJeff Roberson } 3772504d5de3SJeff Roberson 3773e20a199fSJeff Roberson static void 3774e20a199fSJeff Roberson uma_print_keg(uma_keg_t keg) 37758355f576SJeff Roberson { 3776ab3185d1SJeff Roberson uma_domain_t dom; 3777504d5de3SJeff Roberson uma_slab_t slab; 3778ab3185d1SJeff Roberson int i; 3779504d5de3SJeff Roberson 37800b80c1e4SEitan Adler printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d " 3781bb15d1c7SGleb Smirnoff "out %d free %d\n", 3782e20a199fSJeff Roberson keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags, 3783099a0e58SBosko Milekic keg->uk_ipers, keg->uk_ppera, 378457223e99SAndriy Gapon (keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free, 3785bb15d1c7SGleb Smirnoff keg->uk_free); 3786ab3185d1SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 3787ab3185d1SJeff Roberson dom = &keg->uk_domain[i]; 3788504d5de3SJeff Roberson printf("Part slabs:\n"); 3789ab3185d1SJeff Roberson LIST_FOREACH(slab, &dom->ud_part_slab, us_link) 3790504d5de3SJeff Roberson slab_print(slab); 3791504d5de3SJeff Roberson printf("Free slabs:\n"); 3792ab3185d1SJeff Roberson LIST_FOREACH(slab, &dom->ud_free_slab, us_link) 3793504d5de3SJeff Roberson slab_print(slab); 3794504d5de3SJeff Roberson printf("Full slabs:\n"); 3795ab3185d1SJeff Roberson LIST_FOREACH(slab, &dom->ud_full_slab, us_link) 3796504d5de3SJeff Roberson slab_print(slab); 3797e20a199fSJeff Roberson } 3798ab3185d1SJeff Roberson } 3799e20a199fSJeff Roberson 3800e20a199fSJeff Roberson void 3801e20a199fSJeff Roberson uma_print_zone(uma_zone_t zone) 3802e20a199fSJeff Roberson { 3803e20a199fSJeff Roberson uma_cache_t cache; 3804e20a199fSJeff Roberson int i; 3805e20a199fSJeff Roberson 38065a8eee2bSGleb Smirnoff printf("zone: %s(%p) size %d maxitems %ju flags %#x\n", 38075a8eee2bSGleb Smirnoff zone->uz_name, zone, zone->uz_size, (uintmax_t)zone->uz_max_items, 3808bb15d1c7SGleb Smirnoff zone->uz_flags); 3809bb15d1c7SGleb Smirnoff if (zone->uz_lockptr != &zone->uz_lock) 3810bb15d1c7SGleb Smirnoff uma_print_keg(zone->uz_keg); 38113aa6d94eSJohn Baldwin CPU_FOREACH(i) { 3812504d5de3SJeff Roberson cache = &zone->uz_cpu[i]; 3813504d5de3SJeff Roberson printf("CPU %d Cache:\n", i); 3814504d5de3SJeff Roberson cache_print(cache); 3815504d5de3SJeff Roberson } 38168355f576SJeff Roberson } 38178355f576SJeff Roberson 3818a0d4b0aeSRobert Watson #ifdef DDB 38198355f576SJeff Roberson /* 38207a52a97eSRobert Watson * Generate statistics across both the zone and its per-cpu cache's. Return 38217a52a97eSRobert Watson * desired statistics if the pointer is non-NULL for that statistic. 38227a52a97eSRobert Watson * 38237a52a97eSRobert Watson * Note: does not update the zone statistics, as it can't safely clear the 38247a52a97eSRobert Watson * per-CPU cache statistic. 38257a52a97eSRobert Watson * 38267a52a97eSRobert Watson * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't 38277a52a97eSRobert Watson * safe from off-CPU; we should modify the caches to track this information 38287a52a97eSRobert Watson * directly so that we don't have to. 38297a52a97eSRobert Watson */ 38307a52a97eSRobert Watson static void 38310f9b7bf3SMark Johnston uma_zone_sumstat(uma_zone_t z, long *cachefreep, uint64_t *allocsp, 383285dcf349SGleb Smirnoff uint64_t *freesp, uint64_t *sleepsp) 38337a52a97eSRobert Watson { 38347a52a97eSRobert Watson uma_cache_t cache; 383585dcf349SGleb Smirnoff uint64_t allocs, frees, sleeps; 38367a52a97eSRobert Watson int cachefree, cpu; 38377a52a97eSRobert Watson 3838bf965959SSean Bruno allocs = frees = sleeps = 0; 38397a52a97eSRobert Watson cachefree = 0; 38403aa6d94eSJohn Baldwin CPU_FOREACH(cpu) { 38417a52a97eSRobert Watson cache = &z->uz_cpu[cpu]; 38427a52a97eSRobert Watson if (cache->uc_allocbucket != NULL) 38437a52a97eSRobert Watson cachefree += cache->uc_allocbucket->ub_cnt; 38447a52a97eSRobert Watson if (cache->uc_freebucket != NULL) 38457a52a97eSRobert Watson cachefree += cache->uc_freebucket->ub_cnt; 38467a52a97eSRobert Watson allocs += cache->uc_allocs; 38477a52a97eSRobert Watson frees += cache->uc_frees; 38487a52a97eSRobert Watson } 38492efcc8cbSGleb Smirnoff allocs += counter_u64_fetch(z->uz_allocs); 38502efcc8cbSGleb Smirnoff frees += counter_u64_fetch(z->uz_frees); 3851bf965959SSean Bruno sleeps += z->uz_sleeps; 38527a52a97eSRobert Watson if (cachefreep != NULL) 38537a52a97eSRobert Watson *cachefreep = cachefree; 38547a52a97eSRobert Watson if (allocsp != NULL) 38557a52a97eSRobert Watson *allocsp = allocs; 38567a52a97eSRobert Watson if (freesp != NULL) 38577a52a97eSRobert Watson *freesp = frees; 3858bf965959SSean Bruno if (sleepsp != NULL) 3859bf965959SSean Bruno *sleepsp = sleeps; 38607a52a97eSRobert Watson } 3861a0d4b0aeSRobert Watson #endif /* DDB */ 38627a52a97eSRobert Watson 38637a52a97eSRobert Watson static int 38647a52a97eSRobert Watson sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS) 38657a52a97eSRobert Watson { 38667a52a97eSRobert Watson uma_keg_t kz; 38677a52a97eSRobert Watson uma_zone_t z; 38687a52a97eSRobert Watson int count; 38697a52a97eSRobert Watson 38707a52a97eSRobert Watson count = 0; 3871111fbcd5SBryan Venteicher rw_rlock(&uma_rwlock); 38727a52a97eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 38737a52a97eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) 38747a52a97eSRobert Watson count++; 38757a52a97eSRobert Watson } 3876b47acb0aSGleb Smirnoff LIST_FOREACH(z, &uma_cachezones, uz_link) 3877b47acb0aSGleb Smirnoff count++; 3878b47acb0aSGleb Smirnoff 3879111fbcd5SBryan Venteicher rw_runlock(&uma_rwlock); 38807a52a97eSRobert Watson return (sysctl_handle_int(oidp, &count, 0, req)); 38817a52a97eSRobert Watson } 38827a52a97eSRobert Watson 3883b47acb0aSGleb Smirnoff static void 3884b47acb0aSGleb Smirnoff uma_vm_zone_stats(struct uma_type_header *uth, uma_zone_t z, struct sbuf *sbuf, 3885b47acb0aSGleb Smirnoff struct uma_percpu_stat *ups, bool internal) 3886b47acb0aSGleb Smirnoff { 3887b47acb0aSGleb Smirnoff uma_zone_domain_t zdom; 3888b47acb0aSGleb Smirnoff uma_cache_t cache; 3889b47acb0aSGleb Smirnoff int i; 3890b47acb0aSGleb Smirnoff 3891b47acb0aSGleb Smirnoff 3892b47acb0aSGleb Smirnoff for (i = 0; i < vm_ndomains; i++) { 3893b47acb0aSGleb Smirnoff zdom = &z->uz_domain[i]; 3894b47acb0aSGleb Smirnoff uth->uth_zone_free += zdom->uzd_nitems; 3895b47acb0aSGleb Smirnoff } 3896b47acb0aSGleb Smirnoff uth->uth_allocs = counter_u64_fetch(z->uz_allocs); 3897b47acb0aSGleb Smirnoff uth->uth_frees = counter_u64_fetch(z->uz_frees); 3898b47acb0aSGleb Smirnoff uth->uth_fails = counter_u64_fetch(z->uz_fails); 3899b47acb0aSGleb Smirnoff uth->uth_sleeps = z->uz_sleeps; 3900b47acb0aSGleb Smirnoff /* 3901b47acb0aSGleb Smirnoff * While it is not normally safe to access the cache 3902b47acb0aSGleb Smirnoff * bucket pointers while not on the CPU that owns the 3903b47acb0aSGleb Smirnoff * cache, we only allow the pointers to be exchanged 3904b47acb0aSGleb Smirnoff * without the zone lock held, not invalidated, so 3905b47acb0aSGleb Smirnoff * accept the possible race associated with bucket 3906b47acb0aSGleb Smirnoff * exchange during monitoring. 3907b47acb0aSGleb Smirnoff */ 3908b47acb0aSGleb Smirnoff for (i = 0; i < mp_maxid + 1; i++) { 3909b47acb0aSGleb Smirnoff bzero(&ups[i], sizeof(*ups)); 3910b47acb0aSGleb Smirnoff if (internal || CPU_ABSENT(i)) 3911b47acb0aSGleb Smirnoff continue; 3912b47acb0aSGleb Smirnoff cache = &z->uz_cpu[i]; 3913b47acb0aSGleb Smirnoff if (cache->uc_allocbucket != NULL) 3914b47acb0aSGleb Smirnoff ups[i].ups_cache_free += 3915b47acb0aSGleb Smirnoff cache->uc_allocbucket->ub_cnt; 3916b47acb0aSGleb Smirnoff if (cache->uc_freebucket != NULL) 3917b47acb0aSGleb Smirnoff ups[i].ups_cache_free += 3918b47acb0aSGleb Smirnoff cache->uc_freebucket->ub_cnt; 3919b47acb0aSGleb Smirnoff ups[i].ups_allocs = cache->uc_allocs; 3920b47acb0aSGleb Smirnoff ups[i].ups_frees = cache->uc_frees; 3921b47acb0aSGleb Smirnoff } 3922b47acb0aSGleb Smirnoff } 3923b47acb0aSGleb Smirnoff 39247a52a97eSRobert Watson static int 39257a52a97eSRobert Watson sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) 39267a52a97eSRobert Watson { 39277a52a97eSRobert Watson struct uma_stream_header ush; 39287a52a97eSRobert Watson struct uma_type_header uth; 392963b5d112SKonstantin Belousov struct uma_percpu_stat *ups; 39307a52a97eSRobert Watson struct sbuf sbuf; 39317a52a97eSRobert Watson uma_keg_t kz; 39327a52a97eSRobert Watson uma_zone_t z; 39334e657159SMatthew D Fleming int count, error, i; 39347a52a97eSRobert Watson 393500f0e671SMatthew D Fleming error = sysctl_wire_old_buffer(req, 0); 393600f0e671SMatthew D Fleming if (error != 0) 393700f0e671SMatthew D Fleming return (error); 39384e657159SMatthew D Fleming sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 39391eafc078SIan Lepore sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL); 394063b5d112SKonstantin Belousov ups = malloc((mp_maxid + 1) * sizeof(*ups), M_TEMP, M_WAITOK); 39414e657159SMatthew D Fleming 3942404a593eSMatthew D Fleming count = 0; 3943111fbcd5SBryan Venteicher rw_rlock(&uma_rwlock); 39447a52a97eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 39457a52a97eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) 39467a52a97eSRobert Watson count++; 39477a52a97eSRobert Watson } 39487a52a97eSRobert Watson 3949b47acb0aSGleb Smirnoff LIST_FOREACH(z, &uma_cachezones, uz_link) 3950b47acb0aSGleb Smirnoff count++; 3951b47acb0aSGleb Smirnoff 39527a52a97eSRobert Watson /* 39537a52a97eSRobert Watson * Insert stream header. 39547a52a97eSRobert Watson */ 39557a52a97eSRobert Watson bzero(&ush, sizeof(ush)); 39567a52a97eSRobert Watson ush.ush_version = UMA_STREAM_VERSION; 3957ab3a57c0SRobert Watson ush.ush_maxcpus = (mp_maxid + 1); 39587a52a97eSRobert Watson ush.ush_count = count; 39594e657159SMatthew D Fleming (void)sbuf_bcat(&sbuf, &ush, sizeof(ush)); 39607a52a97eSRobert Watson 39617a52a97eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 39627a52a97eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) { 39637a52a97eSRobert Watson bzero(&uth, sizeof(uth)); 39647a52a97eSRobert Watson ZONE_LOCK(z); 3965cbbb4a00SRobert Watson strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); 39667a52a97eSRobert Watson uth.uth_align = kz->uk_align; 39677a52a97eSRobert Watson uth.uth_size = kz->uk_size; 39687a52a97eSRobert Watson uth.uth_rsize = kz->uk_rsize; 3969bb45b411SGleb Smirnoff if (z->uz_max_items > 0) 3970bb45b411SGleb Smirnoff uth.uth_pages = (z->uz_items / kz->uk_ipers) * 3971bb15d1c7SGleb Smirnoff kz->uk_ppera; 3972bb45b411SGleb Smirnoff else 3973bb45b411SGleb Smirnoff uth.uth_pages = kz->uk_pages; 3974f8c86a5fSGleb Smirnoff uth.uth_maxpages = (z->uz_max_items / kz->uk_ipers) * 3975bb15d1c7SGleb Smirnoff kz->uk_ppera; 3976bb15d1c7SGleb Smirnoff uth.uth_limit = z->uz_max_items; 3977f8c86a5fSGleb Smirnoff uth.uth_keg_free = z->uz_keg->uk_free; 3978cbbb4a00SRobert Watson 3979cbbb4a00SRobert Watson /* 3980cbbb4a00SRobert Watson * A zone is secondary is it is not the first entry 3981cbbb4a00SRobert Watson * on the keg's zone list. 3982cbbb4a00SRobert Watson */ 3983e20a199fSJeff Roberson if ((z->uz_flags & UMA_ZONE_SECONDARY) && 3984cbbb4a00SRobert Watson (LIST_FIRST(&kz->uk_zones) != z)) 3985cbbb4a00SRobert Watson uth.uth_zone_flags = UTH_ZONE_SECONDARY; 3986b47acb0aSGleb Smirnoff uma_vm_zone_stats(&uth, z, &sbuf, ups, 3987b47acb0aSGleb Smirnoff kz->uk_flags & UMA_ZFLAG_INTERNAL); 39882450bbb8SRobert Watson ZONE_UNLOCK(z); 398963b5d112SKonstantin Belousov (void)sbuf_bcat(&sbuf, &uth, sizeof(uth)); 399063b5d112SKonstantin Belousov for (i = 0; i < mp_maxid + 1; i++) 399163b5d112SKonstantin Belousov (void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i])); 39927a52a97eSRobert Watson } 39937a52a97eSRobert Watson } 3994b47acb0aSGleb Smirnoff LIST_FOREACH(z, &uma_cachezones, uz_link) { 3995b47acb0aSGleb Smirnoff bzero(&uth, sizeof(uth)); 3996b47acb0aSGleb Smirnoff ZONE_LOCK(z); 3997b47acb0aSGleb Smirnoff strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); 3998b47acb0aSGleb Smirnoff uth.uth_size = z->uz_size; 3999b47acb0aSGleb Smirnoff uma_vm_zone_stats(&uth, z, &sbuf, ups, false); 4000b47acb0aSGleb Smirnoff ZONE_UNLOCK(z); 4001b47acb0aSGleb Smirnoff (void)sbuf_bcat(&sbuf, &uth, sizeof(uth)); 4002b47acb0aSGleb Smirnoff for (i = 0; i < mp_maxid + 1; i++) 4003b47acb0aSGleb Smirnoff (void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i])); 4004b47acb0aSGleb Smirnoff } 4005b47acb0aSGleb Smirnoff 4006111fbcd5SBryan Venteicher rw_runlock(&uma_rwlock); 40074e657159SMatthew D Fleming error = sbuf_finish(&sbuf); 40084e657159SMatthew D Fleming sbuf_delete(&sbuf); 400963b5d112SKonstantin Belousov free(ups, M_TEMP); 40107a52a97eSRobert Watson return (error); 40117a52a97eSRobert Watson } 401248c5777eSRobert Watson 40130a5a3ccbSGleb Smirnoff int 40140a5a3ccbSGleb Smirnoff sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS) 40150a5a3ccbSGleb Smirnoff { 40160a5a3ccbSGleb Smirnoff uma_zone_t zone = *(uma_zone_t *)arg1; 401716be9f54SGleb Smirnoff int error, max; 40180a5a3ccbSGleb Smirnoff 401916be9f54SGleb Smirnoff max = uma_zone_get_max(zone); 40200a5a3ccbSGleb Smirnoff error = sysctl_handle_int(oidp, &max, 0, req); 40210a5a3ccbSGleb Smirnoff if (error || !req->newptr) 40220a5a3ccbSGleb Smirnoff return (error); 40230a5a3ccbSGleb Smirnoff 40240a5a3ccbSGleb Smirnoff uma_zone_set_max(zone, max); 40250a5a3ccbSGleb Smirnoff 40260a5a3ccbSGleb Smirnoff return (0); 40270a5a3ccbSGleb Smirnoff } 40280a5a3ccbSGleb Smirnoff 40290a5a3ccbSGleb Smirnoff int 40300a5a3ccbSGleb Smirnoff sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS) 40310a5a3ccbSGleb Smirnoff { 40320a5a3ccbSGleb Smirnoff uma_zone_t zone = *(uma_zone_t *)arg1; 40330a5a3ccbSGleb Smirnoff int cur; 40340a5a3ccbSGleb Smirnoff 40350a5a3ccbSGleb Smirnoff cur = uma_zone_get_cur(zone); 40360a5a3ccbSGleb Smirnoff return (sysctl_handle_int(oidp, &cur, 0, req)); 40370a5a3ccbSGleb Smirnoff } 40380a5a3ccbSGleb Smirnoff 40399542ea7bSGleb Smirnoff #ifdef INVARIANTS 40409542ea7bSGleb Smirnoff static uma_slab_t 40419542ea7bSGleb Smirnoff uma_dbg_getslab(uma_zone_t zone, void *item) 40429542ea7bSGleb Smirnoff { 40439542ea7bSGleb Smirnoff uma_slab_t slab; 40449542ea7bSGleb Smirnoff uma_keg_t keg; 40459542ea7bSGleb Smirnoff uint8_t *mem; 40469542ea7bSGleb Smirnoff 40479542ea7bSGleb Smirnoff mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); 40489542ea7bSGleb Smirnoff if (zone->uz_flags & UMA_ZONE_VTOSLAB) { 40499542ea7bSGleb Smirnoff slab = vtoslab((vm_offset_t)mem); 40509542ea7bSGleb Smirnoff } else { 40519542ea7bSGleb Smirnoff /* 40529542ea7bSGleb Smirnoff * It is safe to return the slab here even though the 40539542ea7bSGleb Smirnoff * zone is unlocked because the item's allocation state 40549542ea7bSGleb Smirnoff * essentially holds a reference. 40559542ea7bSGleb Smirnoff */ 4056bb15d1c7SGleb Smirnoff if (zone->uz_lockptr == &zone->uz_lock) 4057bb15d1c7SGleb Smirnoff return (NULL); 40589542ea7bSGleb Smirnoff ZONE_LOCK(zone); 4059bb15d1c7SGleb Smirnoff keg = zone->uz_keg; 40609542ea7bSGleb Smirnoff if (keg->uk_flags & UMA_ZONE_HASH) 40619542ea7bSGleb Smirnoff slab = hash_sfind(&keg->uk_hash, mem); 40629542ea7bSGleb Smirnoff else 40639542ea7bSGleb Smirnoff slab = (uma_slab_t)(mem + keg->uk_pgoff); 40649542ea7bSGleb Smirnoff ZONE_UNLOCK(zone); 40659542ea7bSGleb Smirnoff } 40669542ea7bSGleb Smirnoff 40679542ea7bSGleb Smirnoff return (slab); 40689542ea7bSGleb Smirnoff } 40699542ea7bSGleb Smirnoff 4070c5deaf04SGleb Smirnoff static bool 4071c5deaf04SGleb Smirnoff uma_dbg_zskip(uma_zone_t zone, void *mem) 4072c5deaf04SGleb Smirnoff { 4073c5deaf04SGleb Smirnoff 4074bb15d1c7SGleb Smirnoff if (zone->uz_lockptr == &zone->uz_lock) 4075c5deaf04SGleb Smirnoff return (true); 4076c5deaf04SGleb Smirnoff 4077bb15d1c7SGleb Smirnoff return (uma_dbg_kskip(zone->uz_keg, mem)); 4078c5deaf04SGleb Smirnoff } 4079c5deaf04SGleb Smirnoff 4080c5deaf04SGleb Smirnoff static bool 4081c5deaf04SGleb Smirnoff uma_dbg_kskip(uma_keg_t keg, void *mem) 4082c5deaf04SGleb Smirnoff { 4083c5deaf04SGleb Smirnoff uintptr_t idx; 4084c5deaf04SGleb Smirnoff 4085c5deaf04SGleb Smirnoff if (dbg_divisor == 0) 4086c5deaf04SGleb Smirnoff return (true); 4087c5deaf04SGleb Smirnoff 4088c5deaf04SGleb Smirnoff if (dbg_divisor == 1) 4089c5deaf04SGleb Smirnoff return (false); 4090c5deaf04SGleb Smirnoff 4091c5deaf04SGleb Smirnoff idx = (uintptr_t)mem >> PAGE_SHIFT; 4092c5deaf04SGleb Smirnoff if (keg->uk_ipers > 1) { 4093c5deaf04SGleb Smirnoff idx *= keg->uk_ipers; 4094c5deaf04SGleb Smirnoff idx += ((uintptr_t)mem & PAGE_MASK) / keg->uk_rsize; 4095c5deaf04SGleb Smirnoff } 4096c5deaf04SGleb Smirnoff 4097c5deaf04SGleb Smirnoff if ((idx / dbg_divisor) * dbg_divisor != idx) { 4098c5deaf04SGleb Smirnoff counter_u64_add(uma_skip_cnt, 1); 4099c5deaf04SGleb Smirnoff return (true); 4100c5deaf04SGleb Smirnoff } 4101c5deaf04SGleb Smirnoff counter_u64_add(uma_dbg_cnt, 1); 4102c5deaf04SGleb Smirnoff 4103c5deaf04SGleb Smirnoff return (false); 4104c5deaf04SGleb Smirnoff } 4105c5deaf04SGleb Smirnoff 41069542ea7bSGleb Smirnoff /* 41079542ea7bSGleb Smirnoff * Set up the slab's freei data such that uma_dbg_free can function. 41089542ea7bSGleb Smirnoff * 41099542ea7bSGleb Smirnoff */ 41109542ea7bSGleb Smirnoff static void 41119542ea7bSGleb Smirnoff uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item) 41129542ea7bSGleb Smirnoff { 41139542ea7bSGleb Smirnoff uma_keg_t keg; 41149542ea7bSGleb Smirnoff int freei; 41159542ea7bSGleb Smirnoff 41169542ea7bSGleb Smirnoff if (slab == NULL) { 41179542ea7bSGleb Smirnoff slab = uma_dbg_getslab(zone, item); 41189542ea7bSGleb Smirnoff if (slab == NULL) 41199542ea7bSGleb Smirnoff panic("uma: item %p did not belong to zone %s\n", 41209542ea7bSGleb Smirnoff item, zone->uz_name); 41219542ea7bSGleb Smirnoff } 41229542ea7bSGleb Smirnoff keg = slab->us_keg; 41239542ea7bSGleb Smirnoff freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; 41249542ea7bSGleb Smirnoff 41259542ea7bSGleb Smirnoff if (BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree)) 41269542ea7bSGleb Smirnoff panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)\n", 41279542ea7bSGleb Smirnoff item, zone, zone->uz_name, slab, freei); 41289542ea7bSGleb Smirnoff BIT_SET_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree); 41299542ea7bSGleb Smirnoff 41309542ea7bSGleb Smirnoff return; 41319542ea7bSGleb Smirnoff } 41329542ea7bSGleb Smirnoff 41339542ea7bSGleb Smirnoff /* 41349542ea7bSGleb Smirnoff * Verifies freed addresses. Checks for alignment, valid slab membership 41359542ea7bSGleb Smirnoff * and duplicate frees. 41369542ea7bSGleb Smirnoff * 41379542ea7bSGleb Smirnoff */ 41389542ea7bSGleb Smirnoff static void 41399542ea7bSGleb Smirnoff uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item) 41409542ea7bSGleb Smirnoff { 41419542ea7bSGleb Smirnoff uma_keg_t keg; 41429542ea7bSGleb Smirnoff int freei; 41439542ea7bSGleb Smirnoff 41449542ea7bSGleb Smirnoff if (slab == NULL) { 41459542ea7bSGleb Smirnoff slab = uma_dbg_getslab(zone, item); 41469542ea7bSGleb Smirnoff if (slab == NULL) 41479542ea7bSGleb Smirnoff panic("uma: Freed item %p did not belong to zone %s\n", 41489542ea7bSGleb Smirnoff item, zone->uz_name); 41499542ea7bSGleb Smirnoff } 41509542ea7bSGleb Smirnoff keg = slab->us_keg; 41519542ea7bSGleb Smirnoff freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; 41529542ea7bSGleb Smirnoff 41539542ea7bSGleb Smirnoff if (freei >= keg->uk_ipers) 41549542ea7bSGleb Smirnoff panic("Invalid free of %p from zone %p(%s) slab %p(%d)\n", 41559542ea7bSGleb Smirnoff item, zone, zone->uz_name, slab, freei); 41569542ea7bSGleb Smirnoff 41579542ea7bSGleb Smirnoff if (((freei * keg->uk_rsize) + slab->us_data) != item) 41589542ea7bSGleb Smirnoff panic("Unaligned free of %p from zone %p(%s) slab %p(%d)\n", 41599542ea7bSGleb Smirnoff item, zone, zone->uz_name, slab, freei); 41609542ea7bSGleb Smirnoff 41619542ea7bSGleb Smirnoff if (!BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree)) 41629542ea7bSGleb Smirnoff panic("Duplicate free of %p from zone %p(%s) slab %p(%d)\n", 41639542ea7bSGleb Smirnoff item, zone, zone->uz_name, slab, freei); 41649542ea7bSGleb Smirnoff 41659542ea7bSGleb Smirnoff BIT_CLR_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree); 41669542ea7bSGleb Smirnoff } 41679542ea7bSGleb Smirnoff #endif /* INVARIANTS */ 41689542ea7bSGleb Smirnoff 416948c5777eSRobert Watson #ifdef DDB 417048c5777eSRobert Watson DB_SHOW_COMMAND(uma, db_show_uma) 417148c5777eSRobert Watson { 417248c5777eSRobert Watson uma_keg_t kz; 417348c5777eSRobert Watson uma_zone_t z; 4174ab3185d1SJeff Roberson uint64_t allocs, frees, sleeps; 41750f9b7bf3SMark Johnston long cachefree; 41760f9b7bf3SMark Johnston int i; 417748c5777eSRobert Watson 417803175483SAlexander Motin db_printf("%18s %8s %8s %8s %12s %8s %8s\n", "Zone", "Size", "Used", 417903175483SAlexander Motin "Free", "Requests", "Sleeps", "Bucket"); 418048c5777eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 418148c5777eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) { 418248c5777eSRobert Watson if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { 41832efcc8cbSGleb Smirnoff allocs = counter_u64_fetch(z->uz_allocs); 41842efcc8cbSGleb Smirnoff frees = counter_u64_fetch(z->uz_frees); 4185bf965959SSean Bruno sleeps = z->uz_sleeps; 418648c5777eSRobert Watson cachefree = 0; 418748c5777eSRobert Watson } else 418848c5777eSRobert Watson uma_zone_sumstat(z, &cachefree, &allocs, 4189bf965959SSean Bruno &frees, &sleeps); 4190e20a199fSJeff Roberson if (!((z->uz_flags & UMA_ZONE_SECONDARY) && 419148c5777eSRobert Watson (LIST_FIRST(&kz->uk_zones) != z))) 419248c5777eSRobert Watson cachefree += kz->uk_free; 41930f9b7bf3SMark Johnston for (i = 0; i < vm_ndomains; i++) 41940f9b7bf3SMark Johnston cachefree += z->uz_domain[i].uzd_nitems; 41950f9b7bf3SMark Johnston 41960f9b7bf3SMark Johnston db_printf("%18s %8ju %8jd %8ld %12ju %8ju %8u\n", 419703175483SAlexander Motin z->uz_name, (uintmax_t)kz->uk_size, 4198ae4e9636SRobert Watson (intmax_t)(allocs - frees), cachefree, 419903175483SAlexander Motin (uintmax_t)allocs, sleeps, z->uz_count); 4200687c94aaSJohn Baldwin if (db_pager_quit) 4201687c94aaSJohn Baldwin return; 420248c5777eSRobert Watson } 420348c5777eSRobert Watson } 420448c5777eSRobert Watson } 420503175483SAlexander Motin 420603175483SAlexander Motin DB_SHOW_COMMAND(umacache, db_show_umacache) 420703175483SAlexander Motin { 420803175483SAlexander Motin uma_zone_t z; 4209ab3185d1SJeff Roberson uint64_t allocs, frees; 42100f9b7bf3SMark Johnston long cachefree; 42110f9b7bf3SMark Johnston int i; 421203175483SAlexander Motin 421303175483SAlexander Motin db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free", 421403175483SAlexander Motin "Requests", "Bucket"); 421503175483SAlexander Motin LIST_FOREACH(z, &uma_cachezones, uz_link) { 421603175483SAlexander Motin uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL); 42170f9b7bf3SMark Johnston for (i = 0; i < vm_ndomains; i++) 42180f9b7bf3SMark Johnston cachefree += z->uz_domain[i].uzd_nitems; 42190f9b7bf3SMark Johnston db_printf("%18s %8ju %8jd %8ld %12ju %8u\n", 422003175483SAlexander Motin z->uz_name, (uintmax_t)z->uz_size, 422103175483SAlexander Motin (intmax_t)(allocs - frees), cachefree, 422203175483SAlexander Motin (uintmax_t)allocs, z->uz_count); 422303175483SAlexander Motin if (db_pager_quit) 422403175483SAlexander Motin return; 422503175483SAlexander Motin } 422603175483SAlexander Motin } 42279542ea7bSGleb Smirnoff #endif /* DDB */ 4228