160727d8bSWarner Losh /*- 2fe267a55SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3fe267a55SPedro F. Giffuni * 4584061b4SJeff Roberson * Copyright (c) 2002-2019 Jeffrey Roberson <jeff@FreeBSD.org> 508ecce74SRobert Watson * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 6ae4e9636SRobert Watson * Copyright (c) 2004-2006 Robert N. M. Watson 708ecce74SRobert Watson * All rights reserved. 88355f576SJeff Roberson * 98355f576SJeff Roberson * Redistribution and use in source and binary forms, with or without 108355f576SJeff Roberson * modification, are permitted provided that the following conditions 118355f576SJeff Roberson * are met: 128355f576SJeff Roberson * 1. Redistributions of source code must retain the above copyright 138355f576SJeff Roberson * notice unmodified, this list of conditions, and the following 148355f576SJeff Roberson * disclaimer. 158355f576SJeff Roberson * 2. Redistributions in binary form must reproduce the above copyright 168355f576SJeff Roberson * notice, this list of conditions and the following disclaimer in the 178355f576SJeff Roberson * documentation and/or other materials provided with the distribution. 188355f576SJeff Roberson * 198355f576SJeff Roberson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 208355f576SJeff Roberson * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 218355f576SJeff Roberson * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 228355f576SJeff Roberson * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 238355f576SJeff Roberson * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 248355f576SJeff Roberson * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 258355f576SJeff Roberson * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 268355f576SJeff Roberson * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 278355f576SJeff Roberson * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 288355f576SJeff Roberson * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 298355f576SJeff Roberson */ 308355f576SJeff Roberson 318355f576SJeff Roberson /* 328355f576SJeff Roberson * uma_core.c Implementation of the Universal Memory allocator 338355f576SJeff Roberson * 348355f576SJeff Roberson * This allocator is intended to replace the multitude of similar object caches 358355f576SJeff Roberson * in the standard FreeBSD kernel. The intent is to be flexible as well as 36763df3ecSPedro F. Giffuni * efficient. A primary design goal is to return unused memory to the rest of 378355f576SJeff Roberson * the system. This will make the system as a whole more flexible due to the 388355f576SJeff Roberson * ability to move memory to subsystems which most need it instead of leaving 398355f576SJeff Roberson * pools of reserved memory unused. 408355f576SJeff Roberson * 418355f576SJeff Roberson * The basic ideas stem from similar slab/zone based allocators whose algorithms 428355f576SJeff Roberson * are well known. 438355f576SJeff Roberson * 448355f576SJeff Roberson */ 458355f576SJeff Roberson 468355f576SJeff Roberson /* 478355f576SJeff Roberson * TODO: 488355f576SJeff Roberson * - Improve memory usage for large allocations 498355f576SJeff Roberson * - Investigate cache size adjustments 508355f576SJeff Roberson */ 518355f576SJeff Roberson 52874651b1SDavid E. O'Brien #include <sys/cdefs.h> 53874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$"); 54874651b1SDavid E. O'Brien 5548c5777eSRobert Watson #include "opt_ddb.h" 568355f576SJeff Roberson #include "opt_param.h" 578d689e04SGleb Smirnoff #include "opt_vm.h" 5848c5777eSRobert Watson 598355f576SJeff Roberson #include <sys/param.h> 608355f576SJeff Roberson #include <sys/systm.h> 61ef72505eSJeff Roberson #include <sys/bitset.h> 62194a979eSMark Johnston #include <sys/domainset.h> 639b43bc27SAndriy Gapon #include <sys/eventhandler.h> 648355f576SJeff Roberson #include <sys/kernel.h> 658355f576SJeff Roberson #include <sys/types.h> 66ad5b0f5bSJeff Roberson #include <sys/limits.h> 678355f576SJeff Roberson #include <sys/queue.h> 688355f576SJeff Roberson #include <sys/malloc.h> 693659f747SRobert Watson #include <sys/ktr.h> 708355f576SJeff Roberson #include <sys/lock.h> 718355f576SJeff Roberson #include <sys/sysctl.h> 728355f576SJeff Roberson #include <sys/mutex.h> 734c1cc01cSJohn Baldwin #include <sys/proc.h> 7410cb2424SMark Murray #include <sys/random.h> 7589f6b863SAttilio Rao #include <sys/rwlock.h> 767a52a97eSRobert Watson #include <sys/sbuf.h> 77a2de44abSAlexander Motin #include <sys/sched.h> 784bd61e19SJeff Roberson #include <sys/sleepqueue.h> 798355f576SJeff Roberson #include <sys/smp.h> 80d4665eaaSJeff Roberson #include <sys/smr.h> 81e60b2fcbSGleb Smirnoff #include <sys/taskqueue.h> 8286bbae32SJeff Roberson #include <sys/vmmeter.h> 8386bbae32SJeff Roberson 848355f576SJeff Roberson #include <vm/vm.h> 85194a979eSMark Johnston #include <vm/vm_domainset.h> 868355f576SJeff Roberson #include <vm/vm_object.h> 878355f576SJeff Roberson #include <vm/vm_page.h> 88a4915c21SAttilio Rao #include <vm/vm_pageout.h> 898355f576SJeff Roberson #include <vm/vm_param.h> 90ab3185d1SJeff Roberson #include <vm/vm_phys.h> 9130c5525bSAndrew Gallatin #include <vm/vm_pagequeue.h> 928355f576SJeff Roberson #include <vm/vm_map.h> 938355f576SJeff Roberson #include <vm/vm_kern.h> 948355f576SJeff Roberson #include <vm/vm_extern.h> 958355f576SJeff Roberson #include <vm/uma.h> 968355f576SJeff Roberson #include <vm/uma_int.h> 97639c9550SJeff Roberson #include <vm/uma_dbg.h> 988355f576SJeff Roberson 9948c5777eSRobert Watson #include <ddb/ddb.h> 10048c5777eSRobert Watson 1018d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD 1028d689e04SGleb Smirnoff #include <vm/memguard.h> 1038d689e04SGleb Smirnoff #endif 1048d689e04SGleb Smirnoff 105a81c400eSJeff Roberson #include <machine/md_var.h> 106a81c400eSJeff Roberson 107d4665eaaSJeff Roberson #ifdef INVARIANTS 108d4665eaaSJeff Roberson #define UMA_ALWAYS_CTORDTOR 1 109d4665eaaSJeff Roberson #else 110d4665eaaSJeff Roberson #define UMA_ALWAYS_CTORDTOR 0 111d4665eaaSJeff Roberson #endif 112d4665eaaSJeff Roberson 1138355f576SJeff Roberson /* 114ab3185d1SJeff Roberson * This is the zone and keg from which all zones are spawned. 1158355f576SJeff Roberson */ 116ab3185d1SJeff Roberson static uma_zone_t kegs; 117ab3185d1SJeff Roberson static uma_zone_t zones; 1188355f576SJeff Roberson 1199b8db4d0SRyan Libby /* 1209b8db4d0SRyan Libby * These are the two zones from which all offpage uma_slab_ts are allocated. 1219b8db4d0SRyan Libby * 1229b8db4d0SRyan Libby * One zone is for slab headers that can represent a larger number of items, 1239b8db4d0SRyan Libby * making the slabs themselves more efficient, and the other zone is for 1249b8db4d0SRyan Libby * headers that are smaller and represent fewer items, making the headers more 1259b8db4d0SRyan Libby * efficient. 1269b8db4d0SRyan Libby */ 1279b8db4d0SRyan Libby #define SLABZONE_SIZE(setsize) \ 1289b8db4d0SRyan Libby (sizeof(struct uma_hash_slab) + BITSET_SIZE(setsize) * SLAB_BITSETS) 1299b8db4d0SRyan Libby #define SLABZONE0_SETSIZE (PAGE_SIZE / 16) 1309b8db4d0SRyan Libby #define SLABZONE1_SETSIZE SLAB_MAX_SETSIZE 1319b8db4d0SRyan Libby #define SLABZONE0_SIZE SLABZONE_SIZE(SLABZONE0_SETSIZE) 1329b8db4d0SRyan Libby #define SLABZONE1_SIZE SLABZONE_SIZE(SLABZONE1_SETSIZE) 1339b8db4d0SRyan Libby static uma_zone_t slabzones[2]; 1348355f576SJeff Roberson 1358355f576SJeff Roberson /* 1368355f576SJeff Roberson * The initial hash tables come out of this zone so they can be allocated 1378355f576SJeff Roberson * prior to malloc coming up. 1388355f576SJeff Roberson */ 1398355f576SJeff Roberson static uma_zone_t hashzone; 1408355f576SJeff Roberson 1411e319f6dSRobert Watson /* The boot-time adjusted value for cache line alignment. */ 142e4cd31ddSJeff Roberson int uma_align_cache = 64 - 1; 1431e319f6dSRobert Watson 144961647dfSJeff Roberson static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets"); 14520a4e154SJeff Roberson static MALLOC_DEFINE(M_UMA, "UMA", "UMA Misc"); 146961647dfSJeff Roberson 1478355f576SJeff Roberson /* 14886bbae32SJeff Roberson * Are we allowed to allocate buckets? 14986bbae32SJeff Roberson */ 15086bbae32SJeff Roberson static int bucketdisable = 1; 15186bbae32SJeff Roberson 152099a0e58SBosko Milekic /* Linked list of all kegs in the system */ 15313e403fdSAntoine Brodin static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs); 1548355f576SJeff Roberson 15503175483SAlexander Motin /* Linked list of all cache-only zones in the system */ 15603175483SAlexander Motin static LIST_HEAD(,uma_zone) uma_cachezones = 15703175483SAlexander Motin LIST_HEAD_INITIALIZER(uma_cachezones); 15803175483SAlexander Motin 159111fbcd5SBryan Venteicher /* This RW lock protects the keg list */ 160fe933c1dSMateusz Guzik static struct rwlock_padalign __exclusive_cache_line uma_rwlock; 1618355f576SJeff Roberson 162ac0a6fd0SGleb Smirnoff /* 163a81c400eSJeff Roberson * First available virual address for boot time allocations. 164ac0a6fd0SGleb Smirnoff */ 165a81c400eSJeff Roberson static vm_offset_t bootstart; 166a81c400eSJeff Roberson static vm_offset_t bootmem; 1678355f576SJeff Roberson 16808cfa56eSMark Johnston static struct sx uma_reclaim_lock; 16995c4bf75SKonstantin Belousov 170fbd95859SMark Johnston /* 171fbd95859SMark Johnston * kmem soft limit, initialized by uma_set_limit(). Ensure that early 172fbd95859SMark Johnston * allocations don't trigger a wakeup of the reclaim thread. 173fbd95859SMark Johnston */ 1746d6a03d7SJeff Roberson unsigned long uma_kmem_limit = LONG_MAX; 175fbd95859SMark Johnston SYSCTL_ULONG(_vm, OID_AUTO, uma_kmem_limit, CTLFLAG_RD, &uma_kmem_limit, 0, 176fbd95859SMark Johnston "UMA kernel memory soft limit"); 1776d6a03d7SJeff Roberson unsigned long uma_kmem_total; 178fbd95859SMark Johnston SYSCTL_ULONG(_vm, OID_AUTO, uma_kmem_total, CTLFLAG_RD, &uma_kmem_total, 0, 179fbd95859SMark Johnston "UMA kernel memory usage"); 1802e47807cSJeff Roberson 1818355f576SJeff Roberson /* Is the VM done starting up? */ 182860bb7a0SMark Johnston static enum { 183860bb7a0SMark Johnston BOOT_COLD, 184a81c400eSJeff Roberson BOOT_KVA, 185860bb7a0SMark Johnston BOOT_RUNNING, 186860bb7a0SMark Johnston BOOT_SHUTDOWN, 187860bb7a0SMark Johnston } booted = BOOT_COLD; 1888355f576SJeff Roberson 189ef72505eSJeff Roberson /* 1909643769aSJeff Roberson * This is the handle used to schedule events that need to happen 1919643769aSJeff Roberson * outside of the allocation fast path. 1929643769aSJeff Roberson */ 1938355f576SJeff Roberson static struct callout uma_callout; 1949643769aSJeff Roberson #define UMA_TIMEOUT 20 /* Seconds for callout interval. */ 1958355f576SJeff Roberson 1968355f576SJeff Roberson /* 1978355f576SJeff Roberson * This structure is passed as the zone ctor arg so that I don't have to create 1988355f576SJeff Roberson * a special allocation function just for zones. 1998355f576SJeff Roberson */ 2008355f576SJeff Roberson struct uma_zctor_args { 201bb196eb4SMatthew D Fleming const char *name; 202c3bdc05fSAndrew R. Reiter size_t size; 2038355f576SJeff Roberson uma_ctor ctor; 2048355f576SJeff Roberson uma_dtor dtor; 2058355f576SJeff Roberson uma_init uminit; 2068355f576SJeff Roberson uma_fini fini; 2070095a784SJeff Roberson uma_import import; 2080095a784SJeff Roberson uma_release release; 2090095a784SJeff Roberson void *arg; 210099a0e58SBosko Milekic uma_keg_t keg; 211099a0e58SBosko Milekic int align; 21285dcf349SGleb Smirnoff uint32_t flags; 213099a0e58SBosko Milekic }; 214099a0e58SBosko Milekic 215099a0e58SBosko Milekic struct uma_kctor_args { 216099a0e58SBosko Milekic uma_zone_t zone; 217099a0e58SBosko Milekic size_t size; 218099a0e58SBosko Milekic uma_init uminit; 219099a0e58SBosko Milekic uma_fini fini; 2208355f576SJeff Roberson int align; 22185dcf349SGleb Smirnoff uint32_t flags; 2228355f576SJeff Roberson }; 2238355f576SJeff Roberson 224cae33c14SJeff Roberson struct uma_bucket_zone { 225cae33c14SJeff Roberson uma_zone_t ubz_zone; 226*eaa17d42SRyan Libby const char *ubz_name; 227fc03d22bSJeff Roberson int ubz_entries; /* Number of items it can hold. */ 228fc03d22bSJeff Roberson int ubz_maxsize; /* Maximum allocation size per-item. */ 229cae33c14SJeff Roberson }; 230cae33c14SJeff Roberson 231f9d27e75SRobert Watson /* 232fc03d22bSJeff Roberson * Compute the actual number of bucket entries to pack them in power 233fc03d22bSJeff Roberson * of two sizes for more efficient space utilization. 234f9d27e75SRobert Watson */ 235fc03d22bSJeff Roberson #define BUCKET_SIZE(n) \ 236fc03d22bSJeff Roberson (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *)) 237fc03d22bSJeff Roberson 2381aa6c758SAlexander Motin #define BUCKET_MAX BUCKET_SIZE(256) 239e84130a0SJeff Roberson #define BUCKET_MIN 2 240fc03d22bSJeff Roberson 241fc03d22bSJeff Roberson struct uma_bucket_zone bucket_zones[] = { 242e84130a0SJeff Roberson /* Literal bucket sizes. */ 243e84130a0SJeff Roberson { NULL, "2 Bucket", 2, 4096 }, 244e84130a0SJeff Roberson { NULL, "4 Bucket", 4, 3072 }, 245e84130a0SJeff Roberson { NULL, "8 Bucket", 8, 2048 }, 246e84130a0SJeff Roberson { NULL, "16 Bucket", 16, 1024 }, 247e84130a0SJeff Roberson /* Rounded down power of 2 sizes for efficiency. */ 248fc03d22bSJeff Roberson { NULL, "32 Bucket", BUCKET_SIZE(32), 512 }, 249fc03d22bSJeff Roberson { NULL, "64 Bucket", BUCKET_SIZE(64), 256 }, 250fc03d22bSJeff Roberson { NULL, "128 Bucket", BUCKET_SIZE(128), 128 }, 2511aa6c758SAlexander Motin { NULL, "256 Bucket", BUCKET_SIZE(256), 64 }, 252fc03d22bSJeff Roberson { NULL, NULL, 0} 253fc03d22bSJeff Roberson }; 254cae33c14SJeff Roberson 2552019094aSRobert Watson /* 2562019094aSRobert Watson * Flags and enumerations to be passed to internal functions. 2572019094aSRobert Watson */ 258bb15d1c7SGleb Smirnoff enum zfreeskip { 259bb15d1c7SGleb Smirnoff SKIP_NONE = 0, 260bb15d1c7SGleb Smirnoff SKIP_CNT = 0x00000001, 261bb15d1c7SGleb Smirnoff SKIP_DTOR = 0x00010000, 262bb15d1c7SGleb Smirnoff SKIP_FINI = 0x00020000, 263bb15d1c7SGleb Smirnoff }; 264b23f72e9SBrian Feldman 2658355f576SJeff Roberson /* Prototypes.. */ 2668355f576SJeff Roberson 267a81c400eSJeff Roberson void uma_startup1(vm_offset_t); 268f4bef67cSGleb Smirnoff void uma_startup2(void); 269f4bef67cSGleb Smirnoff 270ab3185d1SJeff Roberson static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 271ab3185d1SJeff Roberson static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 272ab3059a8SMatt Macy static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 273ab3185d1SJeff Roberson static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 274ec0d8280SRyan Libby static void *contig_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 275f2c2231eSRyan Stone static void page_free(void *, vm_size_t, uint8_t); 276ab3059a8SMatt Macy static void pcpu_page_free(void *, vm_size_t, uint8_t); 27786220393SMark Johnston static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int); 2789643769aSJeff Roberson static void cache_drain(uma_zone_t); 2798355f576SJeff Roberson static void bucket_drain(uma_zone_t, uma_bucket_t); 28008cfa56eSMark Johnston static void bucket_cache_reclaim(uma_zone_t zone, bool); 281b23f72e9SBrian Feldman static int keg_ctor(void *, int, void *, int); 282099a0e58SBosko Milekic static void keg_dtor(void *, int, void *); 283b23f72e9SBrian Feldman static int zone_ctor(void *, int, void *, int); 2849c2cd7e5SJeff Roberson static void zone_dtor(void *, int, void *); 285d4665eaaSJeff Roberson static inline void item_dtor(uma_zone_t zone, void *item, int size, 286d4665eaaSJeff Roberson void *udata, enum zfreeskip skip); 287b23f72e9SBrian Feldman static int zero_init(void *, int, int); 288c6fd3e23SJeff Roberson static void zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, 289c6fd3e23SJeff Roberson int itemdomain, bool ws); 29020a4e154SJeff Roberson static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *); 291a81c400eSJeff Roberson static void zone_foreach_unlocked(void (*zfunc)(uma_zone_t, void *), void *); 29220a4e154SJeff Roberson static void zone_timeout(uma_zone_t zone, void *); 2933b2f2cb8SAlexander Motin static int hash_alloc(struct uma_hash *, u_int); 2940aef6126SJeff Roberson static int hash_expand(struct uma_hash *, struct uma_hash *); 2950aef6126SJeff Roberson static void hash_free(struct uma_hash *hash); 2968355f576SJeff Roberson static void uma_timeout(void *); 2978355f576SJeff Roberson static void uma_startup3(void); 298860bb7a0SMark Johnston static void uma_shutdown(void); 299ab3185d1SJeff Roberson static void *zone_alloc_item(uma_zone_t, void *, int, int); 3000095a784SJeff Roberson static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip); 3014bd61e19SJeff Roberson static int zone_alloc_limit(uma_zone_t zone, int count, int flags); 3024bd61e19SJeff Roberson static void zone_free_limit(uma_zone_t zone, int count); 30386bbae32SJeff Roberson static void bucket_enable(void); 304cae33c14SJeff Roberson static void bucket_init(void); 3056fd34d6fSJeff Roberson static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int); 3066fd34d6fSJeff Roberson static void bucket_free(uma_zone_t zone, uma_bucket_t, void *); 307cae33c14SJeff Roberson static void bucket_zone_drain(void); 308beb8beefSJeff Roberson static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int); 3090095a784SJeff Roberson static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); 310bb15d1c7SGleb Smirnoff static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item); 311e20a199fSJeff Roberson static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, 31285dcf349SGleb Smirnoff uma_fini fini, int align, uint32_t flags); 313b75c4efcSAndrew Turner static int zone_import(void *, void **, int, int, int); 314b75c4efcSAndrew Turner static void zone_release(void *, void **, int); 315beb8beefSJeff Roberson static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int); 3160a81b439SJeff Roberson static bool cache_free(uma_zone_t, uma_cache_t, void *, void *, int); 317bbee39c6SJeff Roberson 3187a52a97eSRobert Watson static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); 3197a52a97eSRobert Watson static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); 32020a4e154SJeff Roberson static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS); 32120a4e154SJeff Roberson static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS); 3226d204a6aSRyan Libby static int sysctl_handle_uma_zone_flags(SYSCTL_HANDLER_ARGS); 323f7af5015SRyan Libby static int sysctl_handle_uma_slab_efficiency(SYSCTL_HANDLER_ARGS); 3244bd61e19SJeff Roberson static int sysctl_handle_uma_zone_items(SYSCTL_HANDLER_ARGS); 3258355f576SJeff Roberson 32631c251a0SJeff Roberson static uint64_t uma_zone_get_allocs(uma_zone_t zone); 32731c251a0SJeff Roberson 32833e5a1eaSRyan Libby static SYSCTL_NODE(_vm, OID_AUTO, debug, CTLFLAG_RD, 0, 32933e5a1eaSRyan Libby "Memory allocation debugging"); 33033e5a1eaSRyan Libby 3319542ea7bSGleb Smirnoff #ifdef INVARIANTS 33231c251a0SJeff Roberson static uint64_t uma_keg_get_allocs(uma_keg_t zone); 333815db204SRyan Libby static inline struct noslabbits *slab_dbg_bits(uma_slab_t slab, uma_keg_t keg); 334815db204SRyan Libby 335c5deaf04SGleb Smirnoff static bool uma_dbg_kskip(uma_keg_t keg, void *mem); 336c5deaf04SGleb Smirnoff static bool uma_dbg_zskip(uma_zone_t zone, void *mem); 3379542ea7bSGleb Smirnoff static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item); 3389542ea7bSGleb Smirnoff static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item); 339c5deaf04SGleb Smirnoff 340c5deaf04SGleb Smirnoff static u_int dbg_divisor = 1; 341c5deaf04SGleb Smirnoff SYSCTL_UINT(_vm_debug, OID_AUTO, divisor, 342c5deaf04SGleb Smirnoff CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &dbg_divisor, 0, 343c5deaf04SGleb Smirnoff "Debug & thrash every this item in memory allocator"); 344c5deaf04SGleb Smirnoff 345c5deaf04SGleb Smirnoff static counter_u64_t uma_dbg_cnt = EARLY_COUNTER; 346c5deaf04SGleb Smirnoff static counter_u64_t uma_skip_cnt = EARLY_COUNTER; 347c5deaf04SGleb Smirnoff SYSCTL_COUNTER_U64(_vm_debug, OID_AUTO, trashed, CTLFLAG_RD, 348c5deaf04SGleb Smirnoff &uma_dbg_cnt, "memory items debugged"); 349c5deaf04SGleb Smirnoff SYSCTL_COUNTER_U64(_vm_debug, OID_AUTO, skipped, CTLFLAG_RD, 350c5deaf04SGleb Smirnoff &uma_skip_cnt, "memory items skipped, not debugged"); 3519542ea7bSGleb Smirnoff #endif 3529542ea7bSGleb Smirnoff 3538355f576SJeff Roberson SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 3548355f576SJeff Roberson 35535ec24f3SRyan Libby SYSCTL_NODE(_vm, OID_AUTO, uma, CTLFLAG_RW, 0, "Universal Memory Allocator"); 35635ec24f3SRyan Libby 357a314aba8SMateusz Guzik SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLFLAG_MPSAFE|CTLTYPE_INT, 3587a52a97eSRobert Watson 0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones"); 3597a52a97eSRobert Watson 360a314aba8SMateusz Guzik SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLFLAG_MPSAFE|CTLTYPE_STRUCT, 3617a52a97eSRobert Watson 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats"); 3627a52a97eSRobert Watson 3632f891cd5SPawel Jakub Dawidek static int zone_warnings = 1; 364af3b2549SHans Petter Selasky SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RWTUN, &zone_warnings, 0, 3652f891cd5SPawel Jakub Dawidek "Warn when UMA zones becomes full"); 3662f891cd5SPawel Jakub Dawidek 36733e5a1eaSRyan Libby static int multipage_slabs = 1; 36833e5a1eaSRyan Libby TUNABLE_INT("vm.debug.uma_multipage_slabs", &multipage_slabs); 36933e5a1eaSRyan Libby SYSCTL_INT(_vm_debug, OID_AUTO, uma_multipage_slabs, 37033e5a1eaSRyan Libby CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &multipage_slabs, 0, 37133e5a1eaSRyan Libby "UMA may choose larger slab sizes for better efficiency"); 37233e5a1eaSRyan Libby 37386bbae32SJeff Roberson /* 3749b8db4d0SRyan Libby * Select the slab zone for an offpage slab with the given maximum item count. 3759b8db4d0SRyan Libby */ 3769b8db4d0SRyan Libby static inline uma_zone_t 3779b8db4d0SRyan Libby slabzone(int ipers) 3789b8db4d0SRyan Libby { 3799b8db4d0SRyan Libby 3809b8db4d0SRyan Libby return (slabzones[ipers > SLABZONE0_SETSIZE]); 3819b8db4d0SRyan Libby } 3829b8db4d0SRyan Libby 3839b8db4d0SRyan Libby /* 38486bbae32SJeff Roberson * This routine checks to see whether or not it's safe to enable buckets. 38586bbae32SJeff Roberson */ 38686bbae32SJeff Roberson static void 38786bbae32SJeff Roberson bucket_enable(void) 38886bbae32SJeff Roberson { 3893182660aSRyan Libby 390a81c400eSJeff Roberson KASSERT(booted >= BOOT_KVA, ("Bucket enable before init")); 391251386b4SMaksim Yevmenkin bucketdisable = vm_page_count_min(); 39286bbae32SJeff Roberson } 39386bbae32SJeff Roberson 394dc2c7965SRobert Watson /* 395dc2c7965SRobert Watson * Initialize bucket_zones, the array of zones of buckets of various sizes. 396dc2c7965SRobert Watson * 397dc2c7965SRobert Watson * For each zone, calculate the memory required for each bucket, consisting 398fc03d22bSJeff Roberson * of the header and an array of pointers. 399dc2c7965SRobert Watson */ 400cae33c14SJeff Roberson static void 401cae33c14SJeff Roberson bucket_init(void) 402cae33c14SJeff Roberson { 403cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 404cae33c14SJeff Roberson int size; 405cae33c14SJeff Roberson 406d74e6a1dSAlan Cox for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) { 407cae33c14SJeff Roberson size = roundup(sizeof(struct uma_bucket), sizeof(void *)); 408cae33c14SJeff Roberson size += sizeof(void *) * ubz->ubz_entries; 409cae33c14SJeff Roberson ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size, 410e20a199fSJeff Roberson NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 411dfe13344SJeff Roberson UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET | 412dfe13344SJeff Roberson UMA_ZONE_FIRSTTOUCH); 413cae33c14SJeff Roberson } 414cae33c14SJeff Roberson } 415cae33c14SJeff Roberson 416dc2c7965SRobert Watson /* 417dc2c7965SRobert Watson * Given a desired number of entries for a bucket, return the zone from which 418dc2c7965SRobert Watson * to allocate the bucket. 419dc2c7965SRobert Watson */ 420dc2c7965SRobert Watson static struct uma_bucket_zone * 421dc2c7965SRobert Watson bucket_zone_lookup(int entries) 422dc2c7965SRobert Watson { 423fc03d22bSJeff Roberson struct uma_bucket_zone *ubz; 424dc2c7965SRobert Watson 425fc03d22bSJeff Roberson for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 426fc03d22bSJeff Roberson if (ubz->ubz_entries >= entries) 427fc03d22bSJeff Roberson return (ubz); 428fc03d22bSJeff Roberson ubz--; 429fc03d22bSJeff Roberson return (ubz); 430fc03d22bSJeff Roberson } 431fc03d22bSJeff Roberson 432003cf08bSMark Johnston static struct uma_bucket_zone * 433003cf08bSMark Johnston bucket_zone_max(uma_zone_t zone, int nitems) 434003cf08bSMark Johnston { 435003cf08bSMark Johnston struct uma_bucket_zone *ubz; 436003cf08bSMark Johnston int bpcpu; 437003cf08bSMark Johnston 438003cf08bSMark Johnston bpcpu = 2; 439dfe13344SJeff Roberson if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) 440003cf08bSMark Johnston /* Count the cross-domain bucket. */ 441003cf08bSMark Johnston bpcpu++; 442003cf08bSMark Johnston 443003cf08bSMark Johnston for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 444003cf08bSMark Johnston if (ubz->ubz_entries * bpcpu * mp_ncpus > nitems) 445003cf08bSMark Johnston break; 446003cf08bSMark Johnston if (ubz == &bucket_zones[0]) 447003cf08bSMark Johnston ubz = NULL; 448003cf08bSMark Johnston else 449003cf08bSMark Johnston ubz--; 450003cf08bSMark Johnston return (ubz); 451003cf08bSMark Johnston } 452003cf08bSMark Johnston 453fc03d22bSJeff Roberson static int 454fc03d22bSJeff Roberson bucket_select(int size) 455fc03d22bSJeff Roberson { 456fc03d22bSJeff Roberson struct uma_bucket_zone *ubz; 457fc03d22bSJeff Roberson 458fc03d22bSJeff Roberson ubz = &bucket_zones[0]; 459fc03d22bSJeff Roberson if (size > ubz->ubz_maxsize) 460fc03d22bSJeff Roberson return MAX((ubz->ubz_maxsize * ubz->ubz_entries) / size, 1); 461fc03d22bSJeff Roberson 462fc03d22bSJeff Roberson for (; ubz->ubz_entries != 0; ubz++) 463fc03d22bSJeff Roberson if (ubz->ubz_maxsize < size) 464fc03d22bSJeff Roberson break; 465fc03d22bSJeff Roberson ubz--; 466fc03d22bSJeff Roberson return (ubz->ubz_entries); 467dc2c7965SRobert Watson } 468dc2c7965SRobert Watson 469cae33c14SJeff Roberson static uma_bucket_t 4706fd34d6fSJeff Roberson bucket_alloc(uma_zone_t zone, void *udata, int flags) 471cae33c14SJeff Roberson { 472cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 473cae33c14SJeff Roberson uma_bucket_t bucket; 474cae33c14SJeff Roberson 475cae33c14SJeff Roberson /* 476d4665eaaSJeff Roberson * Don't allocate buckets early in boot. 477cae33c14SJeff Roberson */ 478d4665eaaSJeff Roberson if (__predict_false(booted < BOOT_KVA)) 479cae33c14SJeff Roberson return (NULL); 480a81c400eSJeff Roberson 4816fd34d6fSJeff Roberson /* 4826fd34d6fSJeff Roberson * To limit bucket recursion we store the original zone flags 4836fd34d6fSJeff Roberson * in a cookie passed via zalloc_arg/zfree_arg. This allows the 4846fd34d6fSJeff Roberson * NOVM flag to persist even through deep recursions. We also 4856fd34d6fSJeff Roberson * store ZFLAG_BUCKET once we have recursed attempting to allocate 4866fd34d6fSJeff Roberson * a bucket for a bucket zone so we do not allow infinite bucket 4876fd34d6fSJeff Roberson * recursion. This cookie will even persist to frees of unused 4886fd34d6fSJeff Roberson * buckets via the allocation path or bucket allocations in the 4896fd34d6fSJeff Roberson * free path. 4906fd34d6fSJeff Roberson */ 4916fd34d6fSJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) 4926fd34d6fSJeff Roberson udata = (void *)(uintptr_t)zone->uz_flags; 493e8a720feSAlexander Motin else { 494e8a720feSAlexander Motin if ((uintptr_t)udata & UMA_ZFLAG_BUCKET) 495e8a720feSAlexander Motin return (NULL); 4966fd34d6fSJeff Roberson udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET); 497e8a720feSAlexander Motin } 498bae55c4aSRyan Libby if (((uintptr_t)udata & UMA_ZONE_VM) != 0) 499af526374SJeff Roberson flags |= M_NOVM; 50020a4e154SJeff Roberson ubz = bucket_zone_lookup(zone->uz_bucket_size); 50120d3ab87SAlexander Motin if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) 50220d3ab87SAlexander Motin ubz++; 5036fd34d6fSJeff Roberson bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags); 504cae33c14SJeff Roberson if (bucket) { 505cae33c14SJeff Roberson #ifdef INVARIANTS 506cae33c14SJeff Roberson bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); 507cae33c14SJeff Roberson #endif 508cae33c14SJeff Roberson bucket->ub_cnt = 0; 509cae33c14SJeff Roberson bucket->ub_entries = ubz->ubz_entries; 510d4665eaaSJeff Roberson bucket->ub_seq = SMR_SEQ_INVALID; 511d4665eaaSJeff Roberson CTR3(KTR_UMA, "bucket_alloc: zone %s(%p) allocated bucket %p", 512d4665eaaSJeff Roberson zone->uz_name, zone, bucket); 513cae33c14SJeff Roberson } 514cae33c14SJeff Roberson 515cae33c14SJeff Roberson return (bucket); 516cae33c14SJeff Roberson } 517cae33c14SJeff Roberson 518cae33c14SJeff Roberson static void 5196fd34d6fSJeff Roberson bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata) 520cae33c14SJeff Roberson { 521cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 522cae33c14SJeff Roberson 523c6fd3e23SJeff Roberson if (bucket->ub_cnt != 0) 524c6fd3e23SJeff Roberson bucket_drain(zone, bucket); 525c6fd3e23SJeff Roberson 526fc03d22bSJeff Roberson KASSERT(bucket->ub_cnt == 0, 527fc03d22bSJeff Roberson ("bucket_free: Freeing a non free bucket.")); 528d4665eaaSJeff Roberson KASSERT(bucket->ub_seq == SMR_SEQ_INVALID, 529d4665eaaSJeff Roberson ("bucket_free: Freeing an SMR bucket.")); 5306fd34d6fSJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) 5316fd34d6fSJeff Roberson udata = (void *)(uintptr_t)zone->uz_flags; 532dc2c7965SRobert Watson ubz = bucket_zone_lookup(bucket->ub_entries); 5336fd34d6fSJeff Roberson uma_zfree_arg(ubz->ubz_zone, bucket, udata); 534cae33c14SJeff Roberson } 535cae33c14SJeff Roberson 536cae33c14SJeff Roberson static void 537cae33c14SJeff Roberson bucket_zone_drain(void) 538cae33c14SJeff Roberson { 539cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 540cae33c14SJeff Roberson 541cae33c14SJeff Roberson for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 54208cfa56eSMark Johnston uma_zone_reclaim(ubz->ubz_zone, UMA_RECLAIM_DRAIN); 543cae33c14SJeff Roberson } 544cae33c14SJeff Roberson 54508cfa56eSMark Johnston /* 546c6fd3e23SJeff Roberson * Acquire the domain lock and record contention. 547c6fd3e23SJeff Roberson */ 548c6fd3e23SJeff Roberson static uma_zone_domain_t 549c6fd3e23SJeff Roberson zone_domain_lock(uma_zone_t zone, int domain) 550c6fd3e23SJeff Roberson { 551c6fd3e23SJeff Roberson uma_zone_domain_t zdom; 552c6fd3e23SJeff Roberson bool lockfail; 553c6fd3e23SJeff Roberson 554c6fd3e23SJeff Roberson zdom = ZDOM_GET(zone, domain); 555c6fd3e23SJeff Roberson lockfail = false; 556c6fd3e23SJeff Roberson if (ZDOM_OWNED(zdom)) 557c6fd3e23SJeff Roberson lockfail = true; 558c6fd3e23SJeff Roberson ZDOM_LOCK(zdom); 559c6fd3e23SJeff Roberson /* This is unsynchronized. The counter does not need to be precise. */ 560c6fd3e23SJeff Roberson if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max) 561c6fd3e23SJeff Roberson zone->uz_bucket_size++; 562c6fd3e23SJeff Roberson return (zdom); 563c6fd3e23SJeff Roberson } 564c6fd3e23SJeff Roberson 565c6fd3e23SJeff Roberson /* 566c6fd3e23SJeff Roberson * Search for the domain with the least cached items and return it, breaking 567c6fd3e23SJeff Roberson * ties with a preferred domain by returning it. 568c6fd3e23SJeff Roberson */ 569c6fd3e23SJeff Roberson static __noinline int 570c6fd3e23SJeff Roberson zone_domain_lowest(uma_zone_t zone, int pref) 571c6fd3e23SJeff Roberson { 572c6fd3e23SJeff Roberson long least, nitems; 573c6fd3e23SJeff Roberson int domain; 574c6fd3e23SJeff Roberson int i; 575c6fd3e23SJeff Roberson 576c6fd3e23SJeff Roberson least = LONG_MAX; 577c6fd3e23SJeff Roberson domain = 0; 578c6fd3e23SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 579c6fd3e23SJeff Roberson nitems = ZDOM_GET(zone, i)->uzd_nitems; 580c6fd3e23SJeff Roberson if (nitems < least) { 581c6fd3e23SJeff Roberson domain = i; 582c6fd3e23SJeff Roberson least = nitems; 583c6fd3e23SJeff Roberson } else if (nitems == least && (i == pref || domain == pref)) 584c6fd3e23SJeff Roberson domain = pref; 585c6fd3e23SJeff Roberson } 586c6fd3e23SJeff Roberson 587c6fd3e23SJeff Roberson return (domain); 588c6fd3e23SJeff Roberson } 589c6fd3e23SJeff Roberson 590c6fd3e23SJeff Roberson /* 591c6fd3e23SJeff Roberson * Search for the domain with the most cached items and return it or the 592c6fd3e23SJeff Roberson * preferred domain if it has enough to proceed. 593c6fd3e23SJeff Roberson */ 594c6fd3e23SJeff Roberson static __noinline int 595c6fd3e23SJeff Roberson zone_domain_highest(uma_zone_t zone, int pref) 596c6fd3e23SJeff Roberson { 597c6fd3e23SJeff Roberson long most, nitems; 598c6fd3e23SJeff Roberson int domain; 599c6fd3e23SJeff Roberson int i; 600c6fd3e23SJeff Roberson 601c6fd3e23SJeff Roberson if (ZDOM_GET(zone, pref)->uzd_nitems > BUCKET_MAX) 602c6fd3e23SJeff Roberson return (pref); 603c6fd3e23SJeff Roberson 604c6fd3e23SJeff Roberson most = 0; 605c6fd3e23SJeff Roberson domain = 0; 606c6fd3e23SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 607c6fd3e23SJeff Roberson nitems = ZDOM_GET(zone, i)->uzd_nitems; 608c6fd3e23SJeff Roberson if (nitems > most) { 609c6fd3e23SJeff Roberson domain = i; 610c6fd3e23SJeff Roberson most = nitems; 611c6fd3e23SJeff Roberson } 612c6fd3e23SJeff Roberson } 613c6fd3e23SJeff Roberson 614c6fd3e23SJeff Roberson return (domain); 615c6fd3e23SJeff Roberson } 616c6fd3e23SJeff Roberson 617c6fd3e23SJeff Roberson /* 618c6fd3e23SJeff Roberson * Safely subtract cnt from imax. 619c6fd3e23SJeff Roberson */ 620c6fd3e23SJeff Roberson static void 621c6fd3e23SJeff Roberson zone_domain_imax_sub(uma_zone_domain_t zdom, int cnt) 622c6fd3e23SJeff Roberson { 623c6fd3e23SJeff Roberson long new; 624c6fd3e23SJeff Roberson long old; 625c6fd3e23SJeff Roberson 626c6fd3e23SJeff Roberson old = zdom->uzd_imax; 627c6fd3e23SJeff Roberson do { 628c6fd3e23SJeff Roberson if (old <= cnt) 629c6fd3e23SJeff Roberson new = 0; 630c6fd3e23SJeff Roberson else 631c6fd3e23SJeff Roberson new = old - cnt; 632c6fd3e23SJeff Roberson } while (atomic_fcmpset_long(&zdom->uzd_imax, &old, new) == 0); 633c6fd3e23SJeff Roberson } 634c6fd3e23SJeff Roberson 635c6fd3e23SJeff Roberson /* 636c6fd3e23SJeff Roberson * Set the maximum imax value. 637c6fd3e23SJeff Roberson */ 638c6fd3e23SJeff Roberson static void 639c6fd3e23SJeff Roberson zone_domain_imax_set(uma_zone_domain_t zdom, int nitems) 640c6fd3e23SJeff Roberson { 641c6fd3e23SJeff Roberson long old; 642c6fd3e23SJeff Roberson 643c6fd3e23SJeff Roberson old = zdom->uzd_imax; 644c6fd3e23SJeff Roberson do { 645c6fd3e23SJeff Roberson if (old >= nitems) 646c6fd3e23SJeff Roberson break; 647c6fd3e23SJeff Roberson } while (atomic_fcmpset_long(&zdom->uzd_imax, &old, nitems) == 0); 648c6fd3e23SJeff Roberson } 649c6fd3e23SJeff Roberson 650c6fd3e23SJeff Roberson /* 65108cfa56eSMark Johnston * Attempt to satisfy an allocation by retrieving a full bucket from one of the 652d4665eaaSJeff Roberson * zone's caches. If a bucket is found the zone is not locked on return. 65308cfa56eSMark Johnston */ 6540f9b7bf3SMark Johnston static uma_bucket_t 655c6fd3e23SJeff Roberson zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim) 6560f9b7bf3SMark Johnston { 6570f9b7bf3SMark Johnston uma_bucket_t bucket; 658d4665eaaSJeff Roberson int i; 659d4665eaaSJeff Roberson bool dtor = false; 6600f9b7bf3SMark Johnston 661c6fd3e23SJeff Roberson ZDOM_LOCK_ASSERT(zdom); 6620f9b7bf3SMark Johnston 663dc3915c8SJeff Roberson if ((bucket = STAILQ_FIRST(&zdom->uzd_buckets)) == NULL) 664d4665eaaSJeff Roberson return (NULL); 665d4665eaaSJeff Roberson 666543117beSJeff Roberson /* SMR Buckets can not be re-used until readers expire. */ 667d4665eaaSJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && 668d4665eaaSJeff Roberson bucket->ub_seq != SMR_SEQ_INVALID) { 669d4665eaaSJeff Roberson if (!smr_poll(zone->uz_smr, bucket->ub_seq, false)) 670d4665eaaSJeff Roberson return (NULL); 671d4665eaaSJeff Roberson bucket->ub_seq = SMR_SEQ_INVALID; 672543117beSJeff Roberson dtor = (zone->uz_dtor != NULL) || UMA_ALWAYS_CTORDTOR; 673c6fd3e23SJeff Roberson if (STAILQ_NEXT(bucket, ub_link) != NULL) 674c6fd3e23SJeff Roberson zdom->uzd_seq = STAILQ_NEXT(bucket, ub_link)->ub_seq; 675d4665eaaSJeff Roberson } 6760f9b7bf3SMark Johnston MPASS(zdom->uzd_nitems >= bucket->ub_cnt); 677dc3915c8SJeff Roberson STAILQ_REMOVE_HEAD(&zdom->uzd_buckets, ub_link); 6780f9b7bf3SMark Johnston zdom->uzd_nitems -= bucket->ub_cnt; 679c6fd3e23SJeff Roberson 680c6fd3e23SJeff Roberson /* 681c6fd3e23SJeff Roberson * Shift the bounds of the current WSS interval to avoid 682c6fd3e23SJeff Roberson * perturbing the estimate. 683c6fd3e23SJeff Roberson */ 684c6fd3e23SJeff Roberson if (reclaim) { 685c6fd3e23SJeff Roberson zdom->uzd_imin -= lmin(zdom->uzd_imin, bucket->ub_cnt); 686c6fd3e23SJeff Roberson zone_domain_imax_sub(zdom, bucket->ub_cnt); 687c6fd3e23SJeff Roberson } else if (zdom->uzd_imin > zdom->uzd_nitems) 6880f9b7bf3SMark Johnston zdom->uzd_imin = zdom->uzd_nitems; 689c6fd3e23SJeff Roberson 690c6fd3e23SJeff Roberson ZDOM_UNLOCK(zdom); 691d4665eaaSJeff Roberson if (dtor) 692d4665eaaSJeff Roberson for (i = 0; i < bucket->ub_cnt; i++) 693d4665eaaSJeff Roberson item_dtor(zone, bucket->ub_bucket[i], zone->uz_size, 694d4665eaaSJeff Roberson NULL, SKIP_NONE); 695d4665eaaSJeff Roberson 6960f9b7bf3SMark Johnston return (bucket); 6970f9b7bf3SMark Johnston } 6980f9b7bf3SMark Johnston 69908cfa56eSMark Johnston /* 70008cfa56eSMark Johnston * Insert a full bucket into the specified cache. The "ws" parameter indicates 70108cfa56eSMark Johnston * whether the bucket's contents should be counted as part of the zone's working 702c6fd3e23SJeff Roberson * set. The bucket may be freed if it exceeds the bucket limit. 70308cfa56eSMark Johnston */ 7040f9b7bf3SMark Johnston static void 705c6fd3e23SJeff Roberson zone_put_bucket(uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata, 7060f9b7bf3SMark Johnston const bool ws) 7070f9b7bf3SMark Johnston { 708c6fd3e23SJeff Roberson uma_zone_domain_t zdom; 7090f9b7bf3SMark Johnston 710c6fd3e23SJeff Roberson /* We don't cache empty buckets. This can happen after a reclaim. */ 711c6fd3e23SJeff Roberson if (bucket->ub_cnt == 0) 712c6fd3e23SJeff Roberson goto out; 713c6fd3e23SJeff Roberson zdom = zone_domain_lock(zone, domain); 714c6fd3e23SJeff Roberson 715c6fd3e23SJeff Roberson KASSERT(!ws || zdom->uzd_nitems < zone->uz_bucket_max, 71608034d10SKonstantin Belousov ("%s: zone %p overflow", __func__, zone)); 7170f9b7bf3SMark Johnston 718c6fd3e23SJeff Roberson /* 719c6fd3e23SJeff Roberson * Conditionally set the maximum number of items. 720c6fd3e23SJeff Roberson */ 7210f9b7bf3SMark Johnston zdom->uzd_nitems += bucket->ub_cnt; 722c6fd3e23SJeff Roberson if (__predict_true(zdom->uzd_nitems < zone->uz_bucket_max)) { 723c6fd3e23SJeff Roberson if (ws) 724c6fd3e23SJeff Roberson zone_domain_imax_set(zdom, zdom->uzd_nitems); 725c6fd3e23SJeff Roberson if (STAILQ_EMPTY(&zdom->uzd_buckets)) 726c6fd3e23SJeff Roberson zdom->uzd_seq = bucket->ub_seq; 727c6fd3e23SJeff Roberson STAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link); 728c6fd3e23SJeff Roberson ZDOM_UNLOCK(zdom); 729c6fd3e23SJeff Roberson return; 730c6fd3e23SJeff Roberson } 731c6fd3e23SJeff Roberson zdom->uzd_nitems -= bucket->ub_cnt; 732c6fd3e23SJeff Roberson ZDOM_UNLOCK(zdom); 733c6fd3e23SJeff Roberson out: 734c6fd3e23SJeff Roberson bucket_free(zone, bucket, udata); 7350f9b7bf3SMark Johnston } 7360f9b7bf3SMark Johnston 737376b1ba3SJeff Roberson /* Pops an item out of a per-cpu cache bucket. */ 738376b1ba3SJeff Roberson static inline void * 739376b1ba3SJeff Roberson cache_bucket_pop(uma_cache_t cache, uma_cache_bucket_t bucket) 740376b1ba3SJeff Roberson { 741376b1ba3SJeff Roberson void *item; 742376b1ba3SJeff Roberson 743376b1ba3SJeff Roberson CRITICAL_ASSERT(curthread); 744376b1ba3SJeff Roberson 745376b1ba3SJeff Roberson bucket->ucb_cnt--; 746376b1ba3SJeff Roberson item = bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt]; 747376b1ba3SJeff Roberson #ifdef INVARIANTS 748376b1ba3SJeff Roberson bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt] = NULL; 749376b1ba3SJeff Roberson KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled.")); 750376b1ba3SJeff Roberson #endif 751376b1ba3SJeff Roberson cache->uc_allocs++; 752376b1ba3SJeff Roberson 753376b1ba3SJeff Roberson return (item); 754376b1ba3SJeff Roberson } 755376b1ba3SJeff Roberson 756376b1ba3SJeff Roberson /* Pushes an item into a per-cpu cache bucket. */ 757376b1ba3SJeff Roberson static inline void 758376b1ba3SJeff Roberson cache_bucket_push(uma_cache_t cache, uma_cache_bucket_t bucket, void *item) 759376b1ba3SJeff Roberson { 760376b1ba3SJeff Roberson 761376b1ba3SJeff Roberson CRITICAL_ASSERT(curthread); 762376b1ba3SJeff Roberson KASSERT(bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt] == NULL, 763376b1ba3SJeff Roberson ("uma_zfree: Freeing to non free bucket index.")); 764376b1ba3SJeff Roberson 765376b1ba3SJeff Roberson bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt] = item; 766376b1ba3SJeff Roberson bucket->ucb_cnt++; 767376b1ba3SJeff Roberson cache->uc_frees++; 768376b1ba3SJeff Roberson } 769376b1ba3SJeff Roberson 770376b1ba3SJeff Roberson /* 771376b1ba3SJeff Roberson * Unload a UMA bucket from a per-cpu cache. 772376b1ba3SJeff Roberson */ 773376b1ba3SJeff Roberson static inline uma_bucket_t 774376b1ba3SJeff Roberson cache_bucket_unload(uma_cache_bucket_t bucket) 775376b1ba3SJeff Roberson { 776376b1ba3SJeff Roberson uma_bucket_t b; 777376b1ba3SJeff Roberson 778376b1ba3SJeff Roberson b = bucket->ucb_bucket; 779376b1ba3SJeff Roberson if (b != NULL) { 780376b1ba3SJeff Roberson MPASS(b->ub_entries == bucket->ucb_entries); 781376b1ba3SJeff Roberson b->ub_cnt = bucket->ucb_cnt; 782376b1ba3SJeff Roberson bucket->ucb_bucket = NULL; 783376b1ba3SJeff Roberson bucket->ucb_entries = bucket->ucb_cnt = 0; 784376b1ba3SJeff Roberson } 785376b1ba3SJeff Roberson 786376b1ba3SJeff Roberson return (b); 787376b1ba3SJeff Roberson } 788376b1ba3SJeff Roberson 789376b1ba3SJeff Roberson static inline uma_bucket_t 790376b1ba3SJeff Roberson cache_bucket_unload_alloc(uma_cache_t cache) 791376b1ba3SJeff Roberson { 792376b1ba3SJeff Roberson 793376b1ba3SJeff Roberson return (cache_bucket_unload(&cache->uc_allocbucket)); 794376b1ba3SJeff Roberson } 795376b1ba3SJeff Roberson 796376b1ba3SJeff Roberson static inline uma_bucket_t 797376b1ba3SJeff Roberson cache_bucket_unload_free(uma_cache_t cache) 798376b1ba3SJeff Roberson { 799376b1ba3SJeff Roberson 800376b1ba3SJeff Roberson return (cache_bucket_unload(&cache->uc_freebucket)); 801376b1ba3SJeff Roberson } 802376b1ba3SJeff Roberson 803376b1ba3SJeff Roberson static inline uma_bucket_t 804376b1ba3SJeff Roberson cache_bucket_unload_cross(uma_cache_t cache) 805376b1ba3SJeff Roberson { 806376b1ba3SJeff Roberson 807376b1ba3SJeff Roberson return (cache_bucket_unload(&cache->uc_crossbucket)); 808376b1ba3SJeff Roberson } 809376b1ba3SJeff Roberson 810376b1ba3SJeff Roberson /* 811376b1ba3SJeff Roberson * Load a bucket into a per-cpu cache bucket. 812376b1ba3SJeff Roberson */ 813376b1ba3SJeff Roberson static inline void 814376b1ba3SJeff Roberson cache_bucket_load(uma_cache_bucket_t bucket, uma_bucket_t b) 815376b1ba3SJeff Roberson { 816376b1ba3SJeff Roberson 817376b1ba3SJeff Roberson CRITICAL_ASSERT(curthread); 818376b1ba3SJeff Roberson MPASS(bucket->ucb_bucket == NULL); 819543117beSJeff Roberson MPASS(b->ub_seq == SMR_SEQ_INVALID); 820376b1ba3SJeff Roberson 821376b1ba3SJeff Roberson bucket->ucb_bucket = b; 822376b1ba3SJeff Roberson bucket->ucb_cnt = b->ub_cnt; 823376b1ba3SJeff Roberson bucket->ucb_entries = b->ub_entries; 824376b1ba3SJeff Roberson } 825376b1ba3SJeff Roberson 826376b1ba3SJeff Roberson static inline void 827376b1ba3SJeff Roberson cache_bucket_load_alloc(uma_cache_t cache, uma_bucket_t b) 828376b1ba3SJeff Roberson { 829376b1ba3SJeff Roberson 830376b1ba3SJeff Roberson cache_bucket_load(&cache->uc_allocbucket, b); 831376b1ba3SJeff Roberson } 832376b1ba3SJeff Roberson 833376b1ba3SJeff Roberson static inline void 834376b1ba3SJeff Roberson cache_bucket_load_free(uma_cache_t cache, uma_bucket_t b) 835376b1ba3SJeff Roberson { 836376b1ba3SJeff Roberson 837376b1ba3SJeff Roberson cache_bucket_load(&cache->uc_freebucket, b); 838376b1ba3SJeff Roberson } 839376b1ba3SJeff Roberson 840dfe13344SJeff Roberson #ifdef NUMA 841376b1ba3SJeff Roberson static inline void 842376b1ba3SJeff Roberson cache_bucket_load_cross(uma_cache_t cache, uma_bucket_t b) 843376b1ba3SJeff Roberson { 844376b1ba3SJeff Roberson 845376b1ba3SJeff Roberson cache_bucket_load(&cache->uc_crossbucket, b); 846376b1ba3SJeff Roberson } 847376b1ba3SJeff Roberson #endif 848376b1ba3SJeff Roberson 849376b1ba3SJeff Roberson /* 850376b1ba3SJeff Roberson * Copy and preserve ucb_spare. 851376b1ba3SJeff Roberson */ 852376b1ba3SJeff Roberson static inline void 853376b1ba3SJeff Roberson cache_bucket_copy(uma_cache_bucket_t b1, uma_cache_bucket_t b2) 854376b1ba3SJeff Roberson { 855376b1ba3SJeff Roberson 856376b1ba3SJeff Roberson b1->ucb_bucket = b2->ucb_bucket; 857376b1ba3SJeff Roberson b1->ucb_entries = b2->ucb_entries; 858376b1ba3SJeff Roberson b1->ucb_cnt = b2->ucb_cnt; 859376b1ba3SJeff Roberson } 860376b1ba3SJeff Roberson 861376b1ba3SJeff Roberson /* 862376b1ba3SJeff Roberson * Swap two cache buckets. 863376b1ba3SJeff Roberson */ 864376b1ba3SJeff Roberson static inline void 865376b1ba3SJeff Roberson cache_bucket_swap(uma_cache_bucket_t b1, uma_cache_bucket_t b2) 866376b1ba3SJeff Roberson { 867376b1ba3SJeff Roberson struct uma_cache_bucket b3; 868376b1ba3SJeff Roberson 869376b1ba3SJeff Roberson CRITICAL_ASSERT(curthread); 870376b1ba3SJeff Roberson 871376b1ba3SJeff Roberson cache_bucket_copy(&b3, b1); 872376b1ba3SJeff Roberson cache_bucket_copy(b1, b2); 873376b1ba3SJeff Roberson cache_bucket_copy(b2, &b3); 874376b1ba3SJeff Roberson } 875376b1ba3SJeff Roberson 876c6fd3e23SJeff Roberson /* 877c6fd3e23SJeff Roberson * Attempt to fetch a bucket from a zone on behalf of the current cpu cache. 878c6fd3e23SJeff Roberson */ 879c6fd3e23SJeff Roberson static uma_bucket_t 880c6fd3e23SJeff Roberson cache_fetch_bucket(uma_zone_t zone, uma_cache_t cache, int domain) 881c6fd3e23SJeff Roberson { 882c6fd3e23SJeff Roberson uma_zone_domain_t zdom; 883c6fd3e23SJeff Roberson uma_bucket_t bucket; 884c6fd3e23SJeff Roberson 885c6fd3e23SJeff Roberson /* 886c6fd3e23SJeff Roberson * Avoid the lock if possible. 887c6fd3e23SJeff Roberson */ 888c6fd3e23SJeff Roberson zdom = ZDOM_GET(zone, domain); 889c6fd3e23SJeff Roberson if (zdom->uzd_nitems == 0) 890c6fd3e23SJeff Roberson return (NULL); 891c6fd3e23SJeff Roberson 892c6fd3e23SJeff Roberson if ((cache_uz_flags(cache) & UMA_ZONE_SMR) != 0 && 893c6fd3e23SJeff Roberson !smr_poll(zone->uz_smr, zdom->uzd_seq, false)) 894c6fd3e23SJeff Roberson return (NULL); 895c6fd3e23SJeff Roberson 896c6fd3e23SJeff Roberson /* 897c6fd3e23SJeff Roberson * Check the zone's cache of buckets. 898c6fd3e23SJeff Roberson */ 899c6fd3e23SJeff Roberson zdom = zone_domain_lock(zone, domain); 900c6fd3e23SJeff Roberson if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) { 901c6fd3e23SJeff Roberson KASSERT(bucket->ub_cnt != 0, 902c6fd3e23SJeff Roberson ("cache_fetch_bucket: Returning an empty bucket.")); 903c6fd3e23SJeff Roberson return (bucket); 904c6fd3e23SJeff Roberson } 905c6fd3e23SJeff Roberson ZDOM_UNLOCK(zdom); 906c6fd3e23SJeff Roberson 907c6fd3e23SJeff Roberson return (NULL); 908c6fd3e23SJeff Roberson } 909c6fd3e23SJeff Roberson 9102f891cd5SPawel Jakub Dawidek static void 9112f891cd5SPawel Jakub Dawidek zone_log_warning(uma_zone_t zone) 9122f891cd5SPawel Jakub Dawidek { 9132f891cd5SPawel Jakub Dawidek static const struct timeval warninterval = { 300, 0 }; 9142f891cd5SPawel Jakub Dawidek 9152f891cd5SPawel Jakub Dawidek if (!zone_warnings || zone->uz_warning == NULL) 9162f891cd5SPawel Jakub Dawidek return; 9172f891cd5SPawel Jakub Dawidek 9182f891cd5SPawel Jakub Dawidek if (ratecheck(&zone->uz_ratecheck, &warninterval)) 9192f891cd5SPawel Jakub Dawidek printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning); 9202f891cd5SPawel Jakub Dawidek } 9212f891cd5SPawel Jakub Dawidek 92254503a13SJonathan T. Looney static inline void 92354503a13SJonathan T. Looney zone_maxaction(uma_zone_t zone) 92454503a13SJonathan T. Looney { 925e60b2fcbSGleb Smirnoff 926e60b2fcbSGleb Smirnoff if (zone->uz_maxaction.ta_func != NULL) 927e60b2fcbSGleb Smirnoff taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction); 92854503a13SJonathan T. Looney } 92954503a13SJonathan T. Looney 9308355f576SJeff Roberson /* 9318355f576SJeff Roberson * Routine called by timeout which is used to fire off some time interval 9329643769aSJeff Roberson * based calculations. (stats, hash size, etc.) 9338355f576SJeff Roberson * 9348355f576SJeff Roberson * Arguments: 9358355f576SJeff Roberson * arg Unused 9368355f576SJeff Roberson * 9378355f576SJeff Roberson * Returns: 9388355f576SJeff Roberson * Nothing 9398355f576SJeff Roberson */ 9408355f576SJeff Roberson static void 9418355f576SJeff Roberson uma_timeout(void *unused) 9428355f576SJeff Roberson { 94386bbae32SJeff Roberson bucket_enable(); 94420a4e154SJeff Roberson zone_foreach(zone_timeout, NULL); 9458355f576SJeff Roberson 9468355f576SJeff Roberson /* Reschedule this event */ 9479643769aSJeff Roberson callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 9488355f576SJeff Roberson } 9498355f576SJeff Roberson 9508355f576SJeff Roberson /* 9510f9b7bf3SMark Johnston * Update the working set size estimate for the zone's bucket cache. 9520f9b7bf3SMark Johnston * The constants chosen here are somewhat arbitrary. With an update period of 9530f9b7bf3SMark Johnston * 20s (UMA_TIMEOUT), this estimate is dominated by zone activity over the 9540f9b7bf3SMark Johnston * last 100s. 9550f9b7bf3SMark Johnston */ 9560f9b7bf3SMark Johnston static void 9570f9b7bf3SMark Johnston zone_domain_update_wss(uma_zone_domain_t zdom) 9580f9b7bf3SMark Johnston { 9590f9b7bf3SMark Johnston long wss; 9600f9b7bf3SMark Johnston 961c6fd3e23SJeff Roberson ZDOM_LOCK(zdom); 9620f9b7bf3SMark Johnston MPASS(zdom->uzd_imax >= zdom->uzd_imin); 9630f9b7bf3SMark Johnston wss = zdom->uzd_imax - zdom->uzd_imin; 9640f9b7bf3SMark Johnston zdom->uzd_imax = zdom->uzd_imin = zdom->uzd_nitems; 96508cfa56eSMark Johnston zdom->uzd_wss = (4 * wss + zdom->uzd_wss) / 5; 966c6fd3e23SJeff Roberson ZDOM_UNLOCK(zdom); 9670f9b7bf3SMark Johnston } 9680f9b7bf3SMark Johnston 9690f9b7bf3SMark Johnston /* 9709643769aSJeff Roberson * Routine to perform timeout driven calculations. This expands the 9719643769aSJeff Roberson * hashes and does per cpu statistics aggregation. 9728355f576SJeff Roberson * 973e20a199fSJeff Roberson * Returns nothing. 9748355f576SJeff Roberson */ 9758355f576SJeff Roberson static void 97620a4e154SJeff Roberson zone_timeout(uma_zone_t zone, void *unused) 9778355f576SJeff Roberson { 97808034d10SKonstantin Belousov uma_keg_t keg; 9798b987a77SJeff Roberson u_int slabs, pages; 9808355f576SJeff Roberson 98154c5ae80SRyan Libby if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) 98208034d10SKonstantin Belousov goto update_wss; 98308034d10SKonstantin Belousov 98408034d10SKonstantin Belousov keg = zone->uz_keg; 9858b987a77SJeff Roberson 9868b987a77SJeff Roberson /* 9878b987a77SJeff Roberson * Hash zones are non-numa by definition so the first domain 9888b987a77SJeff Roberson * is the only one present. 9898b987a77SJeff Roberson */ 9908b987a77SJeff Roberson KEG_LOCK(keg, 0); 9918b987a77SJeff Roberson pages = keg->uk_domain[0].ud_pages; 9928b987a77SJeff Roberson 9938355f576SJeff Roberson /* 994e20a199fSJeff Roberson * Expand the keg hash table. 9958355f576SJeff Roberson * 9968355f576SJeff Roberson * This is done if the number of slabs is larger than the hash size. 9978355f576SJeff Roberson * What I'm trying to do here is completely reduce collisions. This 9988355f576SJeff Roberson * may be a little aggressive. Should I allow for two collisions max? 9998355f576SJeff Roberson */ 10008b987a77SJeff Roberson if ((slabs = pages / keg->uk_ppera) > keg->uk_hash.uh_hashsize) { 10010aef6126SJeff Roberson struct uma_hash newhash; 10020aef6126SJeff Roberson struct uma_hash oldhash; 10030aef6126SJeff Roberson int ret; 10045300d9ddSJeff Roberson 10050aef6126SJeff Roberson /* 10060aef6126SJeff Roberson * This is so involved because allocating and freeing 1007e20a199fSJeff Roberson * while the keg lock is held will lead to deadlock. 10080aef6126SJeff Roberson * I have to do everything in stages and check for 10090aef6126SJeff Roberson * races. 10100aef6126SJeff Roberson */ 10118b987a77SJeff Roberson KEG_UNLOCK(keg, 0); 10123b2f2cb8SAlexander Motin ret = hash_alloc(&newhash, 1 << fls(slabs)); 10138b987a77SJeff Roberson KEG_LOCK(keg, 0); 10140aef6126SJeff Roberson if (ret) { 1015099a0e58SBosko Milekic if (hash_expand(&keg->uk_hash, &newhash)) { 1016099a0e58SBosko Milekic oldhash = keg->uk_hash; 1017099a0e58SBosko Milekic keg->uk_hash = newhash; 10180aef6126SJeff Roberson } else 10190aef6126SJeff Roberson oldhash = newhash; 10200aef6126SJeff Roberson 10218b987a77SJeff Roberson KEG_UNLOCK(keg, 0); 10220aef6126SJeff Roberson hash_free(&oldhash); 10238b987a77SJeff Roberson goto update_wss; 10240aef6126SJeff Roberson } 10255300d9ddSJeff Roberson } 10268b987a77SJeff Roberson KEG_UNLOCK(keg, 0); 1027e20a199fSJeff Roberson 102808034d10SKonstantin Belousov update_wss: 1029bb15d1c7SGleb Smirnoff for (int i = 0; i < vm_ndomains; i++) 1030c6fd3e23SJeff Roberson zone_domain_update_wss(ZDOM_GET(zone, i)); 10318355f576SJeff Roberson } 10328355f576SJeff Roberson 10338355f576SJeff Roberson /* 10345300d9ddSJeff Roberson * Allocate and zero fill the next sized hash table from the appropriate 10355300d9ddSJeff Roberson * backing store. 10365300d9ddSJeff Roberson * 10375300d9ddSJeff Roberson * Arguments: 10380aef6126SJeff Roberson * hash A new hash structure with the old hash size in uh_hashsize 10395300d9ddSJeff Roberson * 10405300d9ddSJeff Roberson * Returns: 1041763df3ecSPedro F. Giffuni * 1 on success and 0 on failure. 10425300d9ddSJeff Roberson */ 104337c84183SPoul-Henning Kamp static int 10443b2f2cb8SAlexander Motin hash_alloc(struct uma_hash *hash, u_int size) 10455300d9ddSJeff Roberson { 104659568a0eSAlexander Motin size_t alloc; 10475300d9ddSJeff Roberson 10483b2f2cb8SAlexander Motin KASSERT(powerof2(size), ("hash size must be power of 2")); 10493b2f2cb8SAlexander Motin if (size > UMA_HASH_SIZE_INIT) { 10503b2f2cb8SAlexander Motin hash->uh_hashsize = size; 10510aef6126SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; 10521e0701e1SJeff Roberson hash->uh_slab_hash = malloc(alloc, M_UMAHASH, M_NOWAIT); 10535300d9ddSJeff Roberson } else { 10540aef6126SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; 1055e20a199fSJeff Roberson hash->uh_slab_hash = zone_alloc_item(hashzone, NULL, 1056ab3185d1SJeff Roberson UMA_ANYDOMAIN, M_WAITOK); 10570aef6126SJeff Roberson hash->uh_hashsize = UMA_HASH_SIZE_INIT; 10585300d9ddSJeff Roberson } 10590aef6126SJeff Roberson if (hash->uh_slab_hash) { 10600aef6126SJeff Roberson bzero(hash->uh_slab_hash, alloc); 10610aef6126SJeff Roberson hash->uh_hashmask = hash->uh_hashsize - 1; 10620aef6126SJeff Roberson return (1); 10630aef6126SJeff Roberson } 10645300d9ddSJeff Roberson 10650aef6126SJeff Roberson return (0); 10665300d9ddSJeff Roberson } 10675300d9ddSJeff Roberson 10685300d9ddSJeff Roberson /* 106964f051e9SJeff Roberson * Expands the hash table for HASH zones. This is done from zone_timeout 107064f051e9SJeff Roberson * to reduce collisions. This must not be done in the regular allocation 107164f051e9SJeff Roberson * path, otherwise, we can recurse on the vm while allocating pages. 10728355f576SJeff Roberson * 10738355f576SJeff Roberson * Arguments: 10740aef6126SJeff Roberson * oldhash The hash you want to expand 10750aef6126SJeff Roberson * newhash The hash structure for the new table 10768355f576SJeff Roberson * 10778355f576SJeff Roberson * Returns: 10788355f576SJeff Roberson * Nothing 10798355f576SJeff Roberson * 10808355f576SJeff Roberson * Discussion: 10818355f576SJeff Roberson */ 10820aef6126SJeff Roberson static int 10830aef6126SJeff Roberson hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash) 10848355f576SJeff Roberson { 10851e0701e1SJeff Roberson uma_hash_slab_t slab; 10866929b7d1SPedro F. Giffuni u_int hval; 10876929b7d1SPedro F. Giffuni u_int idx; 10888355f576SJeff Roberson 10890aef6126SJeff Roberson if (!newhash->uh_slab_hash) 10900aef6126SJeff Roberson return (0); 10918355f576SJeff Roberson 10920aef6126SJeff Roberson if (oldhash->uh_hashsize >= newhash->uh_hashsize) 10930aef6126SJeff Roberson return (0); 10948355f576SJeff Roberson 10958355f576SJeff Roberson /* 10968355f576SJeff Roberson * I need to investigate hash algorithms for resizing without a 10978355f576SJeff Roberson * full rehash. 10988355f576SJeff Roberson */ 10998355f576SJeff Roberson 11006929b7d1SPedro F. Giffuni for (idx = 0; idx < oldhash->uh_hashsize; idx++) 11011e0701e1SJeff Roberson while (!LIST_EMPTY(&oldhash->uh_slab_hash[idx])) { 11021e0701e1SJeff Roberson slab = LIST_FIRST(&oldhash->uh_slab_hash[idx]); 11031e0701e1SJeff Roberson LIST_REMOVE(slab, uhs_hlink); 11041e0701e1SJeff Roberson hval = UMA_HASH(newhash, slab->uhs_data); 11051e0701e1SJeff Roberson LIST_INSERT_HEAD(&newhash->uh_slab_hash[hval], 11061e0701e1SJeff Roberson slab, uhs_hlink); 11078355f576SJeff Roberson } 11088355f576SJeff Roberson 11090aef6126SJeff Roberson return (1); 11109c2cd7e5SJeff Roberson } 11119c2cd7e5SJeff Roberson 11125300d9ddSJeff Roberson /* 11135300d9ddSJeff Roberson * Free the hash bucket to the appropriate backing store. 11145300d9ddSJeff Roberson * 11155300d9ddSJeff Roberson * Arguments: 11165300d9ddSJeff Roberson * slab_hash The hash bucket we're freeing 11175300d9ddSJeff Roberson * hashsize The number of entries in that hash bucket 11185300d9ddSJeff Roberson * 11195300d9ddSJeff Roberson * Returns: 11205300d9ddSJeff Roberson * Nothing 11215300d9ddSJeff Roberson */ 11229c2cd7e5SJeff Roberson static void 11230aef6126SJeff Roberson hash_free(struct uma_hash *hash) 11249c2cd7e5SJeff Roberson { 11250aef6126SJeff Roberson if (hash->uh_slab_hash == NULL) 11260aef6126SJeff Roberson return; 11270aef6126SJeff Roberson if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) 11280095a784SJeff Roberson zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE); 11298355f576SJeff Roberson else 1130961647dfSJeff Roberson free(hash->uh_slab_hash, M_UMAHASH); 11318355f576SJeff Roberson } 11328355f576SJeff Roberson 11338355f576SJeff Roberson /* 11348355f576SJeff Roberson * Frees all outstanding items in a bucket 11358355f576SJeff Roberson * 11368355f576SJeff Roberson * Arguments: 11378355f576SJeff Roberson * zone The zone to free to, must be unlocked. 11384bd61e19SJeff Roberson * bucket The free/alloc bucket with items. 11398355f576SJeff Roberson * 11408355f576SJeff Roberson * Returns: 11418355f576SJeff Roberson * Nothing 11428355f576SJeff Roberson */ 11438355f576SJeff Roberson static void 11448355f576SJeff Roberson bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 11458355f576SJeff Roberson { 11460095a784SJeff Roberson int i; 11478355f576SJeff Roberson 1148c6fd3e23SJeff Roberson if (bucket->ub_cnt == 0) 11498355f576SJeff Roberson return; 11508355f576SJeff Roberson 1151d4665eaaSJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && 1152d4665eaaSJeff Roberson bucket->ub_seq != SMR_SEQ_INVALID) { 1153d4665eaaSJeff Roberson smr_wait(zone->uz_smr, bucket->ub_seq); 1154543117beSJeff Roberson bucket->ub_seq = SMR_SEQ_INVALID; 1155d4665eaaSJeff Roberson for (i = 0; i < bucket->ub_cnt; i++) 1156d4665eaaSJeff Roberson item_dtor(zone, bucket->ub_bucket[i], 1157d4665eaaSJeff Roberson zone->uz_size, NULL, SKIP_NONE); 1158d4665eaaSJeff Roberson } 11590095a784SJeff Roberson if (zone->uz_fini) 11600095a784SJeff Roberson for (i = 0; i < bucket->ub_cnt; i++) 11610095a784SJeff Roberson zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); 11620095a784SJeff Roberson zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); 11634bd61e19SJeff Roberson if (zone->uz_max_items > 0) 11644bd61e19SJeff Roberson zone_free_limit(zone, bucket->ub_cnt); 1165d4665eaaSJeff Roberson #ifdef INVARIANTS 1166d4665eaaSJeff Roberson bzero(bucket->ub_bucket, sizeof(void *) * bucket->ub_cnt); 1167d4665eaaSJeff Roberson #endif 11680095a784SJeff Roberson bucket->ub_cnt = 0; 11698355f576SJeff Roberson } 11708355f576SJeff Roberson 11718355f576SJeff Roberson /* 11728355f576SJeff Roberson * Drains the per cpu caches for a zone. 11738355f576SJeff Roberson * 1174727c6918SJeff Roberson * NOTE: This may only be called while the zone is being torn down, and not 11755d1ae027SRobert Watson * during normal operation. This is necessary in order that we do not have 11765d1ae027SRobert Watson * to migrate CPUs to drain the per-CPU caches. 11775d1ae027SRobert Watson * 11788355f576SJeff Roberson * Arguments: 11798355f576SJeff Roberson * zone The zone to drain, must be unlocked. 11808355f576SJeff Roberson * 11818355f576SJeff Roberson * Returns: 11828355f576SJeff Roberson * Nothing 11838355f576SJeff Roberson */ 11848355f576SJeff Roberson static void 11859643769aSJeff Roberson cache_drain(uma_zone_t zone) 11868355f576SJeff Roberson { 11878355f576SJeff Roberson uma_cache_t cache; 1188376b1ba3SJeff Roberson uma_bucket_t bucket; 1189543117beSJeff Roberson smr_seq_t seq; 11908355f576SJeff Roberson int cpu; 11918355f576SJeff Roberson 11928355f576SJeff Roberson /* 11935d1ae027SRobert Watson * XXX: It is safe to not lock the per-CPU caches, because we're 11945d1ae027SRobert Watson * tearing down the zone anyway. I.e., there will be no further use 11955d1ae027SRobert Watson * of the caches at this point. 11965d1ae027SRobert Watson * 11975d1ae027SRobert Watson * XXX: It would good to be able to assert that the zone is being 11985d1ae027SRobert Watson * torn down to prevent improper use of cache_drain(). 11998355f576SJeff Roberson */ 1200543117beSJeff Roberson seq = SMR_SEQ_INVALID; 1201543117beSJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) != 0) 1202226dd6dbSJeff Roberson seq = smr_advance(zone->uz_smr); 12033aa6d94eSJohn Baldwin CPU_FOREACH(cpu) { 12048355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 1205376b1ba3SJeff Roberson bucket = cache_bucket_unload_alloc(cache); 1206c6fd3e23SJeff Roberson if (bucket != NULL) 1207376b1ba3SJeff Roberson bucket_free(zone, bucket, NULL); 1208376b1ba3SJeff Roberson bucket = cache_bucket_unload_free(cache); 1209376b1ba3SJeff Roberson if (bucket != NULL) { 1210543117beSJeff Roberson bucket->ub_seq = seq; 1211376b1ba3SJeff Roberson bucket_free(zone, bucket, NULL); 1212376b1ba3SJeff Roberson } 1213376b1ba3SJeff Roberson bucket = cache_bucket_unload_cross(cache); 1214376b1ba3SJeff Roberson if (bucket != NULL) { 1215543117beSJeff Roberson bucket->ub_seq = seq; 1216376b1ba3SJeff Roberson bucket_free(zone, bucket, NULL); 1217376b1ba3SJeff Roberson } 1218d56368d7SBosko Milekic } 121908cfa56eSMark Johnston bucket_cache_reclaim(zone, true); 1220aaa8bb16SJeff Roberson } 1221aaa8bb16SJeff Roberson 1222a2de44abSAlexander Motin static void 122320a4e154SJeff Roberson cache_shrink(uma_zone_t zone, void *unused) 1224a2de44abSAlexander Motin { 1225a2de44abSAlexander Motin 1226a2de44abSAlexander Motin if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 1227a2de44abSAlexander Motin return; 1228a2de44abSAlexander Motin 122920a4e154SJeff Roberson zone->uz_bucket_size = 123020a4e154SJeff Roberson (zone->uz_bucket_size_min + zone->uz_bucket_size) / 2; 1231a2de44abSAlexander Motin } 1232a2de44abSAlexander Motin 1233a2de44abSAlexander Motin static void 123420a4e154SJeff Roberson cache_drain_safe_cpu(uma_zone_t zone, void *unused) 1235a2de44abSAlexander Motin { 1236a2de44abSAlexander Motin uma_cache_t cache; 1237c1685086SJeff Roberson uma_bucket_t b1, b2, b3; 1238ab3185d1SJeff Roberson int domain; 1239a2de44abSAlexander Motin 1240a2de44abSAlexander Motin if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 1241a2de44abSAlexander Motin return; 1242a2de44abSAlexander Motin 1243c1685086SJeff Roberson b1 = b2 = b3 = NULL; 1244a2de44abSAlexander Motin critical_enter(); 1245a2de44abSAlexander Motin cache = &zone->uz_cpu[curcpu]; 1246c6fd3e23SJeff Roberson domain = PCPU_GET(domain); 1247376b1ba3SJeff Roberson b1 = cache_bucket_unload_alloc(cache); 1248d4665eaaSJeff Roberson 1249d4665eaaSJeff Roberson /* 1250d4665eaaSJeff Roberson * Don't flush SMR zone buckets. This leaves the zone without a 1251d4665eaaSJeff Roberson * bucket and forces every free to synchronize(). 1252d4665eaaSJeff Roberson */ 1253543117beSJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) == 0) { 1254376b1ba3SJeff Roberson b2 = cache_bucket_unload_free(cache); 1255543117beSJeff Roberson b3 = cache_bucket_unload_cross(cache); 1256543117beSJeff Roberson } 1257543117beSJeff Roberson critical_exit(); 1258543117beSJeff Roberson 1259543117beSJeff Roberson if (b1 != NULL) 1260c6fd3e23SJeff Roberson zone_free_bucket(zone, b1, NULL, domain, false); 1261543117beSJeff Roberson if (b2 != NULL) 1262c6fd3e23SJeff Roberson zone_free_bucket(zone, b2, NULL, domain, false); 1263543117beSJeff Roberson if (b3 != NULL) { 1264c6fd3e23SJeff Roberson /* Adjust the domain so it goes to zone_free_cross. */ 1265c6fd3e23SJeff Roberson domain = (domain + 1) % vm_ndomains; 1266c6fd3e23SJeff Roberson zone_free_bucket(zone, b3, NULL, domain, false); 1267c1685086SJeff Roberson } 1268a2de44abSAlexander Motin } 1269a2de44abSAlexander Motin 1270a2de44abSAlexander Motin /* 1271a2de44abSAlexander Motin * Safely drain per-CPU caches of a zone(s) to alloc bucket. 1272a2de44abSAlexander Motin * This is an expensive call because it needs to bind to all CPUs 1273a2de44abSAlexander Motin * one by one and enter a critical section on each of them in order 1274a2de44abSAlexander Motin * to safely access their cache buckets. 1275a2de44abSAlexander Motin * Zone lock must not be held on call this function. 1276a2de44abSAlexander Motin */ 1277a2de44abSAlexander Motin static void 127808cfa56eSMark Johnston pcpu_cache_drain_safe(uma_zone_t zone) 1279a2de44abSAlexander Motin { 1280a2de44abSAlexander Motin int cpu; 1281a2de44abSAlexander Motin 1282a2de44abSAlexander Motin /* 1283727c6918SJeff Roberson * Polite bucket sizes shrinking was not enough, shrink aggressively. 1284a2de44abSAlexander Motin */ 1285a2de44abSAlexander Motin if (zone) 128620a4e154SJeff Roberson cache_shrink(zone, NULL); 1287a2de44abSAlexander Motin else 128820a4e154SJeff Roberson zone_foreach(cache_shrink, NULL); 1289a2de44abSAlexander Motin 1290a2de44abSAlexander Motin CPU_FOREACH(cpu) { 1291a2de44abSAlexander Motin thread_lock(curthread); 1292a2de44abSAlexander Motin sched_bind(curthread, cpu); 1293a2de44abSAlexander Motin thread_unlock(curthread); 1294a2de44abSAlexander Motin 1295a2de44abSAlexander Motin if (zone) 129620a4e154SJeff Roberson cache_drain_safe_cpu(zone, NULL); 1297a2de44abSAlexander Motin else 129820a4e154SJeff Roberson zone_foreach(cache_drain_safe_cpu, NULL); 1299a2de44abSAlexander Motin } 1300a2de44abSAlexander Motin thread_lock(curthread); 1301a2de44abSAlexander Motin sched_unbind(curthread); 1302a2de44abSAlexander Motin thread_unlock(curthread); 1303a2de44abSAlexander Motin } 1304a2de44abSAlexander Motin 1305aaa8bb16SJeff Roberson /* 130608cfa56eSMark Johnston * Reclaim cached buckets from a zone. All buckets are reclaimed if the caller 130708cfa56eSMark Johnston * requested a drain, otherwise the per-domain caches are trimmed to either 130808cfa56eSMark Johnston * estimated working set size. 1309aaa8bb16SJeff Roberson */ 1310aaa8bb16SJeff Roberson static void 131108cfa56eSMark Johnston bucket_cache_reclaim(uma_zone_t zone, bool drain) 1312aaa8bb16SJeff Roberson { 1313ab3185d1SJeff Roberson uma_zone_domain_t zdom; 1314aaa8bb16SJeff Roberson uma_bucket_t bucket; 1315c6fd3e23SJeff Roberson long target; 1316ab3185d1SJeff Roberson int i; 13178355f576SJeff Roberson 1318c6fd3e23SJeff Roberson /* 1319c6fd3e23SJeff Roberson * Shrink the zone bucket size to ensure that the per-CPU caches 1320c6fd3e23SJeff Roberson * don't grow too large. 1321c6fd3e23SJeff Roberson */ 1322c6fd3e23SJeff Roberson if (zone->uz_bucket_size > zone->uz_bucket_size_min) 1323c6fd3e23SJeff Roberson zone->uz_bucket_size--; 1324c6fd3e23SJeff Roberson 1325ab3185d1SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 132691d947bfSJeff Roberson /* 132791d947bfSJeff Roberson * The cross bucket is partially filled and not part of 132891d947bfSJeff Roberson * the item count. Reclaim it individually here. 132991d947bfSJeff Roberson */ 1330c6fd3e23SJeff Roberson zdom = ZDOM_GET(zone, i); 1331226dd6dbSJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) == 0 || drain) { 133291d947bfSJeff Roberson ZONE_CROSS_LOCK(zone); 133391d947bfSJeff Roberson bucket = zdom->uzd_cross; 133491d947bfSJeff Roberson zdom->uzd_cross = NULL; 133591d947bfSJeff Roberson ZONE_CROSS_UNLOCK(zone); 1336c6fd3e23SJeff Roberson if (bucket != NULL) 133791d947bfSJeff Roberson bucket_free(zone, bucket, NULL); 133891d947bfSJeff Roberson } 133991d947bfSJeff Roberson 134091d947bfSJeff Roberson /* 134108cfa56eSMark Johnston * If we were asked to drain the zone, we are done only once 134208cfa56eSMark Johnston * this bucket cache is empty. Otherwise, we reclaim items in 134308cfa56eSMark Johnston * excess of the zone's estimated working set size. If the 134408cfa56eSMark Johnston * difference nitems - imin is larger than the WSS estimate, 134508cfa56eSMark Johnston * then the estimate will grow at the end of this interval and 134608cfa56eSMark Johnston * we ignore the historical average. 134708cfa56eSMark Johnston */ 1348c6fd3e23SJeff Roberson ZDOM_LOCK(zdom); 134908cfa56eSMark Johnston target = drain ? 0 : lmax(zdom->uzd_wss, zdom->uzd_nitems - 135008cfa56eSMark Johnston zdom->uzd_imin); 135108cfa56eSMark Johnston while (zdom->uzd_nitems > target) { 1352c6fd3e23SJeff Roberson bucket = zone_fetch_bucket(zone, zdom, true); 135308cfa56eSMark Johnston if (bucket == NULL) 135408cfa56eSMark Johnston break; 13556fd34d6fSJeff Roberson bucket_free(zone, bucket, NULL); 1356c6fd3e23SJeff Roberson ZDOM_LOCK(zdom); 13578355f576SJeff Roberson } 1358c6fd3e23SJeff Roberson ZDOM_UNLOCK(zdom); 1359ab3185d1SJeff Roberson } 13608355f576SJeff Roberson } 1361fc03d22bSJeff Roberson 1362fc03d22bSJeff Roberson static void 1363fc03d22bSJeff Roberson keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) 1364fc03d22bSJeff Roberson { 1365fc03d22bSJeff Roberson uint8_t *mem; 1366fc03d22bSJeff Roberson int i; 1367fc03d22bSJeff Roberson uint8_t flags; 1368fc03d22bSJeff Roberson 13691431a748SGleb Smirnoff CTR4(KTR_UMA, "keg_free_slab keg %s(%p) slab %p, returning %d bytes", 13701431a748SGleb Smirnoff keg->uk_name, keg, slab, PAGE_SIZE * keg->uk_ppera); 13711431a748SGleb Smirnoff 13721e0701e1SJeff Roberson mem = slab_data(slab, keg); 1373fc03d22bSJeff Roberson flags = slab->us_flags; 1374fc03d22bSJeff Roberson i = start; 1375fc03d22bSJeff Roberson if (keg->uk_fini != NULL) { 1376fc03d22bSJeff Roberson for (i--; i > -1; i--) 1377c5deaf04SGleb Smirnoff #ifdef INVARIANTS 1378c5deaf04SGleb Smirnoff /* 1379c5deaf04SGleb Smirnoff * trash_fini implies that dtor was trash_dtor. trash_fini 1380c5deaf04SGleb Smirnoff * would check that memory hasn't been modified since free, 1381c5deaf04SGleb Smirnoff * which executed trash_dtor. 1382c5deaf04SGleb Smirnoff * That's why we need to run uma_dbg_kskip() check here, 1383c5deaf04SGleb Smirnoff * albeit we don't make skip check for other init/fini 1384c5deaf04SGleb Smirnoff * invocations. 1385c5deaf04SGleb Smirnoff */ 13861e0701e1SJeff Roberson if (!uma_dbg_kskip(keg, slab_item(slab, keg, i)) || 1387c5deaf04SGleb Smirnoff keg->uk_fini != trash_fini) 1388c5deaf04SGleb Smirnoff #endif 13891e0701e1SJeff Roberson keg->uk_fini(slab_item(slab, keg, i), keg->uk_size); 1390fc03d22bSJeff Roberson } 139154c5ae80SRyan Libby if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) 13929b8db4d0SRyan Libby zone_free_item(slabzone(keg->uk_ipers), slab_tohashslab(slab), 13939b8db4d0SRyan Libby NULL, SKIP_NONE); 1394fc03d22bSJeff Roberson keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags); 13952e47807cSJeff Roberson uma_total_dec(PAGE_SIZE * keg->uk_ppera); 13968355f576SJeff Roberson } 13978355f576SJeff Roberson 13988355f576SJeff Roberson /* 1399e20a199fSJeff Roberson * Frees pages from a keg back to the system. This is done on demand from 14008355f576SJeff Roberson * the pageout daemon. 14018355f576SJeff Roberson * 1402e20a199fSJeff Roberson * Returns nothing. 14038355f576SJeff Roberson */ 1404e20a199fSJeff Roberson static void 1405e20a199fSJeff Roberson keg_drain(uma_keg_t keg) 14068355f576SJeff Roberson { 14074ab3aee8SMark Johnston struct slabhead freeslabs; 1408ab3185d1SJeff Roberson uma_domain_t dom; 1409829be516SMark Johnston uma_slab_t slab, tmp; 14108b987a77SJeff Roberson int i, n; 14118355f576SJeff Roberson 1412099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) 14138355f576SJeff Roberson return; 14148355f576SJeff Roberson 1415ab3185d1SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 14168b987a77SJeff Roberson CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u", 14174ab3aee8SMark Johnston keg->uk_name, keg, i, dom->ud_free_items); 1418ab3185d1SJeff Roberson dom = &keg->uk_domain[i]; 14194ab3aee8SMark Johnston LIST_INIT(&freeslabs); 1420ab3185d1SJeff Roberson 14214ab3aee8SMark Johnston KEG_LOCK(keg, i); 14224ab3aee8SMark Johnston if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) { 14234ab3aee8SMark Johnston LIST_FOREACH(slab, &dom->ud_free_slab, us_link) 14244ab3aee8SMark Johnston UMA_HASH_REMOVE(&keg->uk_hash, slab); 14254ab3aee8SMark Johnston } 14264ab3aee8SMark Johnston n = dom->ud_free_slabs; 14274ab3aee8SMark Johnston LIST_SWAP(&freeslabs, &dom->ud_free_slab, uma_slab, us_link); 14284ab3aee8SMark Johnston dom->ud_free_slabs = 0; 14294ab3aee8SMark Johnston dom->ud_free_items -= n * keg->uk_ipers; 14304ab3aee8SMark Johnston dom->ud_pages -= n * keg->uk_ppera; 14314ab3aee8SMark Johnston KEG_UNLOCK(keg, i); 14324ab3aee8SMark Johnston 14334ab3aee8SMark Johnston LIST_FOREACH_SAFE(slab, &freeslabs, us_link, tmp) 14341645995bSKirk McKusick keg_free_slab(keg, slab, keg->uk_ipers); 14358355f576SJeff Roberson } 14368355f576SJeff Roberson } 14378355f576SJeff Roberson 1438e20a199fSJeff Roberson static void 143908cfa56eSMark Johnston zone_reclaim(uma_zone_t zone, int waitok, bool drain) 1440e20a199fSJeff Roberson { 1441e20a199fSJeff Roberson 14428355f576SJeff Roberson /* 1443e20a199fSJeff Roberson * Set draining to interlock with zone_dtor() so we can release our 1444e20a199fSJeff Roberson * locks as we go. Only dtor() should do a WAITOK call since it 1445e20a199fSJeff Roberson * is the only call that knows the structure will still be available 1446e20a199fSJeff Roberson * when it wakes up. 1447e20a199fSJeff Roberson */ 1448e20a199fSJeff Roberson ZONE_LOCK(zone); 144908cfa56eSMark Johnston while (zone->uz_flags & UMA_ZFLAG_RECLAIMING) { 1450e20a199fSJeff Roberson if (waitok == M_NOWAIT) 1451e20a199fSJeff Roberson goto out; 1452c6fd3e23SJeff Roberson msleep(zone, &ZDOM_GET(zone, 0)->uzd_lock, PVM, "zonedrain", 1453c6fd3e23SJeff Roberson 1); 1454e20a199fSJeff Roberson } 145508cfa56eSMark Johnston zone->uz_flags |= UMA_ZFLAG_RECLAIMING; 1456e20a199fSJeff Roberson ZONE_UNLOCK(zone); 145791d947bfSJeff Roberson bucket_cache_reclaim(zone, drain); 145808cfa56eSMark Johnston 1459e20a199fSJeff Roberson /* 1460e20a199fSJeff Roberson * The DRAINING flag protects us from being freed while 1461111fbcd5SBryan Venteicher * we're running. Normally the uma_rwlock would protect us but we 1462e20a199fSJeff Roberson * must be able to release and acquire the right lock for each keg. 1463e20a199fSJeff Roberson */ 146408034d10SKonstantin Belousov if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) 1465bb15d1c7SGleb Smirnoff keg_drain(zone->uz_keg); 1466e20a199fSJeff Roberson ZONE_LOCK(zone); 146708cfa56eSMark Johnston zone->uz_flags &= ~UMA_ZFLAG_RECLAIMING; 1468e20a199fSJeff Roberson wakeup(zone); 1469e20a199fSJeff Roberson out: 1470e20a199fSJeff Roberson ZONE_UNLOCK(zone); 1471e20a199fSJeff Roberson } 1472e20a199fSJeff Roberson 147308cfa56eSMark Johnston static void 147420a4e154SJeff Roberson zone_drain(uma_zone_t zone, void *unused) 1475e20a199fSJeff Roberson { 1476e20a199fSJeff Roberson 147708cfa56eSMark Johnston zone_reclaim(zone, M_NOWAIT, true); 147808cfa56eSMark Johnston } 147908cfa56eSMark Johnston 148008cfa56eSMark Johnston static void 148120a4e154SJeff Roberson zone_trim(uma_zone_t zone, void *unused) 148208cfa56eSMark Johnston { 148308cfa56eSMark Johnston 148408cfa56eSMark Johnston zone_reclaim(zone, M_NOWAIT, false); 1485e20a199fSJeff Roberson } 1486e20a199fSJeff Roberson 1487e20a199fSJeff Roberson /* 14888b987a77SJeff Roberson * Allocate a new slab for a keg and inserts it into the partial slab list. 14898b987a77SJeff Roberson * The keg should be unlocked on entry. If the allocation succeeds it will 14908b987a77SJeff Roberson * be locked on return. 14918355f576SJeff Roberson * 14928355f576SJeff Roberson * Arguments: 149386220393SMark Johnston * flags Wait flags for the item initialization routine 149486220393SMark Johnston * aflags Wait flags for the slab allocation 14958355f576SJeff Roberson * 14968355f576SJeff Roberson * Returns: 14978355f576SJeff Roberson * The slab that was allocated or NULL if there is no memory and the 14988355f576SJeff Roberson * caller specified M_NOWAIT. 14998355f576SJeff Roberson */ 15008355f576SJeff Roberson static uma_slab_t 150186220393SMark Johnston keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, 150286220393SMark Johnston int aflags) 15038355f576SJeff Roberson { 15048b987a77SJeff Roberson uma_domain_t dom; 1505e20a199fSJeff Roberson uma_alloc allocf; 1506099a0e58SBosko Milekic uma_slab_t slab; 15072e47807cSJeff Roberson unsigned long size; 150885dcf349SGleb Smirnoff uint8_t *mem; 150986220393SMark Johnston uint8_t sflags; 15108355f576SJeff Roberson int i; 15118355f576SJeff Roberson 1512ab3185d1SJeff Roberson KASSERT(domain >= 0 && domain < vm_ndomains, 1513ab3185d1SJeff Roberson ("keg_alloc_slab: domain %d out of range", domain)); 1514a553d4b8SJeff Roberson 15158b987a77SJeff Roberson allocf = keg->uk_allocf; 1516194a979eSMark Johnston slab = NULL; 1517194a979eSMark Johnston mem = NULL; 151854c5ae80SRyan Libby if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) { 15199b8db4d0SRyan Libby uma_hash_slab_t hslab; 15209b8db4d0SRyan Libby hslab = zone_alloc_item(slabzone(keg->uk_ipers), NULL, 15219b8db4d0SRyan Libby domain, aflags); 15229b8db4d0SRyan Libby if (hslab == NULL) 1523727c6918SJeff Roberson goto fail; 15249b8db4d0SRyan Libby slab = &hslab->uhs_slab; 1525a553d4b8SJeff Roberson } 1526a553d4b8SJeff Roberson 15273370c5bfSJeff Roberson /* 15283370c5bfSJeff Roberson * This reproduces the old vm_zone behavior of zero filling pages the 15293370c5bfSJeff Roberson * first time they are added to a zone. 15303370c5bfSJeff Roberson * 15313370c5bfSJeff Roberson * Malloced items are zeroed in uma_zalloc. 15323370c5bfSJeff Roberson */ 15333370c5bfSJeff Roberson 1534099a0e58SBosko Milekic if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) 153586220393SMark Johnston aflags |= M_ZERO; 15363370c5bfSJeff Roberson else 153786220393SMark Johnston aflags &= ~M_ZERO; 15383370c5bfSJeff Roberson 1539263811f7SKip Macy if (keg->uk_flags & UMA_ZONE_NODUMP) 154086220393SMark Johnston aflags |= M_NODUMP; 1541263811f7SKip Macy 1542e20a199fSJeff Roberson /* zone is passed for legacy reasons. */ 1543194a979eSMark Johnston size = keg->uk_ppera * PAGE_SIZE; 154486220393SMark Johnston mem = allocf(zone, size, domain, &sflags, aflags); 1545a553d4b8SJeff Roberson if (mem == NULL) { 154654c5ae80SRyan Libby if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) 15479b8db4d0SRyan Libby zone_free_item(slabzone(keg->uk_ipers), 15489b8db4d0SRyan Libby slab_tohashslab(slab), NULL, SKIP_NONE); 1549727c6918SJeff Roberson goto fail; 1550a553d4b8SJeff Roberson } 15512e47807cSJeff Roberson uma_total_inc(size); 15528355f576SJeff Roberson 15538b987a77SJeff Roberson /* For HASH zones all pages go to the same uma_domain. */ 155454c5ae80SRyan Libby if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) 15558b987a77SJeff Roberson domain = 0; 15568b987a77SJeff Roberson 15575c0e403bSJeff Roberson /* Point the slab into the allocated memory */ 155854c5ae80SRyan Libby if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) 1559099a0e58SBosko Milekic slab = (uma_slab_t )(mem + keg->uk_pgoff); 15601e0701e1SJeff Roberson else 15619b8db4d0SRyan Libby slab_tohashslab(slab)->uhs_data = mem; 15625c0e403bSJeff Roberson 156354c5ae80SRyan Libby if (keg->uk_flags & UMA_ZFLAG_VTOSLAB) 1564099a0e58SBosko Milekic for (i = 0; i < keg->uk_ppera; i++) 1565584061b4SJeff Roberson vsetzoneslab((vm_offset_t)mem + (i * PAGE_SIZE), 1566584061b4SJeff Roberson zone, slab); 15678355f576SJeff Roberson 1568099a0e58SBosko Milekic slab->us_freecount = keg->uk_ipers; 156986220393SMark Johnston slab->us_flags = sflags; 1570ab3185d1SJeff Roberson slab->us_domain = domain; 15718b987a77SJeff Roberson 15729b78b1f4SJeff Roberson BIT_FILL(keg->uk_ipers, &slab->us_free); 1573ef72505eSJeff Roberson #ifdef INVARIANTS 1574815db204SRyan Libby BIT_ZERO(keg->uk_ipers, slab_dbg_bits(slab, keg)); 1575ef72505eSJeff Roberson #endif 1576099a0e58SBosko Milekic 1577b23f72e9SBrian Feldman if (keg->uk_init != NULL) { 1578099a0e58SBosko Milekic for (i = 0; i < keg->uk_ipers; i++) 15791e0701e1SJeff Roberson if (keg->uk_init(slab_item(slab, keg, i), 158086220393SMark Johnston keg->uk_size, flags) != 0) 1581b23f72e9SBrian Feldman break; 1582b23f72e9SBrian Feldman if (i != keg->uk_ipers) { 1583fc03d22bSJeff Roberson keg_free_slab(keg, slab, i); 1584727c6918SJeff Roberson goto fail; 1585b23f72e9SBrian Feldman } 1586b23f72e9SBrian Feldman } 15878b987a77SJeff Roberson KEG_LOCK(keg, domain); 15885c0e403bSJeff Roberson 15891431a748SGleb Smirnoff CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)", 15901431a748SGleb Smirnoff slab, keg->uk_name, keg); 15911431a748SGleb Smirnoff 159254c5ae80SRyan Libby if (keg->uk_flags & UMA_ZFLAG_HASH) 1593099a0e58SBosko Milekic UMA_HASH_INSERT(&keg->uk_hash, slab, mem); 15948355f576SJeff Roberson 15958b987a77SJeff Roberson /* 15968b987a77SJeff Roberson * If we got a slab here it's safe to mark it partially used 15978b987a77SJeff Roberson * and return. We assume that the caller is going to remove 15988b987a77SJeff Roberson * at least one item. 15998b987a77SJeff Roberson */ 16008b987a77SJeff Roberson dom = &keg->uk_domain[domain]; 16018b987a77SJeff Roberson LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); 16028b987a77SJeff Roberson dom->ud_pages += keg->uk_ppera; 16034ab3aee8SMark Johnston dom->ud_free_items += keg->uk_ipers; 16048355f576SJeff Roberson 16058355f576SJeff Roberson return (slab); 1606727c6918SJeff Roberson 1607727c6918SJeff Roberson fail: 1608727c6918SJeff Roberson return (NULL); 16098355f576SJeff Roberson } 16108355f576SJeff Roberson 16118355f576SJeff Roberson /* 1612009b6fcbSJeff Roberson * This function is intended to be used early on in place of page_alloc() so 1613009b6fcbSJeff Roberson * that we may use the boot time page cache to satisfy allocations before 1614009b6fcbSJeff Roberson * the VM is ready. 1615009b6fcbSJeff Roberson */ 1616009b6fcbSJeff Roberson static void * 1617ab3185d1SJeff Roberson startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 1618ab3185d1SJeff Roberson int wait) 1619009b6fcbSJeff Roberson { 1620a81c400eSJeff Roberson vm_paddr_t pa; 1621a81c400eSJeff Roberson vm_page_t m; 1622ac0a6fd0SGleb Smirnoff void *mem; 1623ac0a6fd0SGleb Smirnoff int pages; 1624a81c400eSJeff Roberson int i; 1625099a0e58SBosko Milekic 1626f7d35785SGleb Smirnoff pages = howmany(bytes, PAGE_SIZE); 1627f7d35785SGleb Smirnoff KASSERT(pages > 0, ("%s can't reserve 0 pages", __func__)); 1628a81c400eSJeff Roberson 1629f7d35785SGleb Smirnoff *pflag = UMA_SLAB_BOOT; 1630a81c400eSJeff Roberson m = vm_page_alloc_contig_domain(NULL, 0, domain, 1631a81c400eSJeff Roberson malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED, pages, 1632a81c400eSJeff Roberson (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT); 1633a81c400eSJeff Roberson if (m == NULL) 1634a81c400eSJeff Roberson return (NULL); 1635a81c400eSJeff Roberson 1636a81c400eSJeff Roberson pa = VM_PAGE_TO_PHYS(m); 1637a81c400eSJeff Roberson for (i = 0; i < pages; i++, pa += PAGE_SIZE) { 1638a81c400eSJeff Roberson #if defined(__aarch64__) || defined(__amd64__) || defined(__mips__) || \ 1639a81c400eSJeff Roberson defined(__riscv) || defined(__powerpc64__) 1640a81c400eSJeff Roberson if ((wait & M_NODUMP) == 0) 1641a81c400eSJeff Roberson dump_add_page(pa); 1642a81c400eSJeff Roberson #endif 1643a81c400eSJeff Roberson } 1644a81c400eSJeff Roberson /* Allocate KVA and indirectly advance bootmem. */ 1645a81c400eSJeff Roberson mem = (void *)pmap_map(&bootmem, m->phys_addr, 1646a81c400eSJeff Roberson m->phys_addr + (pages * PAGE_SIZE), VM_PROT_READ | VM_PROT_WRITE); 1647a81c400eSJeff Roberson if ((wait & M_ZERO) != 0) 1648a81c400eSJeff Roberson bzero(mem, pages * PAGE_SIZE); 1649f7d35785SGleb Smirnoff 1650f7d35785SGleb Smirnoff return (mem); 1651f7d35785SGleb Smirnoff } 1652f7d35785SGleb Smirnoff 1653a81c400eSJeff Roberson static void 1654a81c400eSJeff Roberson startup_free(void *mem, vm_size_t bytes) 1655a81c400eSJeff Roberson { 1656a81c400eSJeff Roberson vm_offset_t va; 1657a81c400eSJeff Roberson vm_page_t m; 1658a81c400eSJeff Roberson 1659a81c400eSJeff Roberson va = (vm_offset_t)mem; 1660a81c400eSJeff Roberson m = PHYS_TO_VM_PAGE(pmap_kextract(va)); 1661a81c400eSJeff Roberson pmap_remove(kernel_pmap, va, va + bytes); 1662a81c400eSJeff Roberson for (; bytes != 0; bytes -= PAGE_SIZE, m++) { 1663a81c400eSJeff Roberson #if defined(__aarch64__) || defined(__amd64__) || defined(__mips__) || \ 1664a81c400eSJeff Roberson defined(__riscv) || defined(__powerpc64__) 1665a81c400eSJeff Roberson dump_drop_page(VM_PAGE_TO_PHYS(m)); 1666a81c400eSJeff Roberson #endif 1667a81c400eSJeff Roberson vm_page_unwire_noq(m); 1668a81c400eSJeff Roberson vm_page_free(m); 1669a81c400eSJeff Roberson } 1670a81c400eSJeff Roberson } 1671a81c400eSJeff Roberson 1672f7d35785SGleb Smirnoff /* 16738355f576SJeff Roberson * Allocates a number of pages from the system 16748355f576SJeff Roberson * 16758355f576SJeff Roberson * Arguments: 16768355f576SJeff Roberson * bytes The number of bytes requested 16778355f576SJeff Roberson * wait Shall we wait? 16788355f576SJeff Roberson * 16798355f576SJeff Roberson * Returns: 16808355f576SJeff Roberson * A pointer to the alloced memory or possibly 16818355f576SJeff Roberson * NULL if M_NOWAIT is set. 16828355f576SJeff Roberson */ 16838355f576SJeff Roberson static void * 1684ab3185d1SJeff Roberson page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 1685ab3185d1SJeff Roberson int wait) 16868355f576SJeff Roberson { 16878355f576SJeff Roberson void *p; /* Returned page */ 16888355f576SJeff Roberson 16892e47807cSJeff Roberson *pflag = UMA_SLAB_KERNEL; 16909978bd99SMark Johnston p = (void *)kmem_malloc_domainset(DOMAINSET_FIXED(domain), bytes, wait); 16918355f576SJeff Roberson 16928355f576SJeff Roberson return (p); 16938355f576SJeff Roberson } 16948355f576SJeff Roberson 1695ab3059a8SMatt Macy static void * 1696ab3059a8SMatt Macy pcpu_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 1697ab3059a8SMatt Macy int wait) 1698ab3059a8SMatt Macy { 1699ab3059a8SMatt Macy struct pglist alloctail; 1700ab3059a8SMatt Macy vm_offset_t addr, zkva; 1701ab3059a8SMatt Macy int cpu, flags; 1702ab3059a8SMatt Macy vm_page_t p, p_next; 1703ab3059a8SMatt Macy #ifdef NUMA 1704ab3059a8SMatt Macy struct pcpu *pc; 1705ab3059a8SMatt Macy #endif 1706ab3059a8SMatt Macy 1707ab3059a8SMatt Macy MPASS(bytes == (mp_maxid + 1) * PAGE_SIZE); 1708ab3059a8SMatt Macy 1709013072f0SMark Johnston TAILQ_INIT(&alloctail); 1710ab3059a8SMatt Macy flags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ | 1711013072f0SMark Johnston malloc2vm_flags(wait); 1712013072f0SMark Johnston *pflag = UMA_SLAB_KERNEL; 1713ab3059a8SMatt Macy for (cpu = 0; cpu <= mp_maxid; cpu++) { 1714ab3059a8SMatt Macy if (CPU_ABSENT(cpu)) { 1715ab3059a8SMatt Macy p = vm_page_alloc(NULL, 0, flags); 1716ab3059a8SMatt Macy } else { 1717ab3059a8SMatt Macy #ifndef NUMA 1718ab3059a8SMatt Macy p = vm_page_alloc(NULL, 0, flags); 1719ab3059a8SMatt Macy #else 1720ab3059a8SMatt Macy pc = pcpu_find(cpu); 172120526802SAndrew Gallatin if (__predict_false(VM_DOMAIN_EMPTY(pc->pc_domain))) 172220526802SAndrew Gallatin p = NULL; 172320526802SAndrew Gallatin else 172420526802SAndrew Gallatin p = vm_page_alloc_domain(NULL, 0, 172520526802SAndrew Gallatin pc->pc_domain, flags); 1726ab3059a8SMatt Macy if (__predict_false(p == NULL)) 1727ab3059a8SMatt Macy p = vm_page_alloc(NULL, 0, flags); 1728ab3059a8SMatt Macy #endif 1729ab3059a8SMatt Macy } 1730ab3059a8SMatt Macy if (__predict_false(p == NULL)) 1731ab3059a8SMatt Macy goto fail; 1732ab3059a8SMatt Macy TAILQ_INSERT_TAIL(&alloctail, p, listq); 1733ab3059a8SMatt Macy } 1734ab3059a8SMatt Macy if ((addr = kva_alloc(bytes)) == 0) 1735ab3059a8SMatt Macy goto fail; 1736ab3059a8SMatt Macy zkva = addr; 1737ab3059a8SMatt Macy TAILQ_FOREACH(p, &alloctail, listq) { 1738ab3059a8SMatt Macy pmap_qenter(zkva, &p, 1); 1739ab3059a8SMatt Macy zkva += PAGE_SIZE; 1740ab3059a8SMatt Macy } 1741ab3059a8SMatt Macy return ((void*)addr); 1742ab3059a8SMatt Macy fail: 1743ab3059a8SMatt Macy TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) { 174488ea538aSMark Johnston vm_page_unwire_noq(p); 1745ab3059a8SMatt Macy vm_page_free(p); 1746ab3059a8SMatt Macy } 1747ab3059a8SMatt Macy return (NULL); 1748ab3059a8SMatt Macy } 1749ab3059a8SMatt Macy 17508355f576SJeff Roberson /* 17518355f576SJeff Roberson * Allocates a number of pages from within an object 17528355f576SJeff Roberson * 17538355f576SJeff Roberson * Arguments: 17548355f576SJeff Roberson * bytes The number of bytes requested 17558355f576SJeff Roberson * wait Shall we wait? 17568355f576SJeff Roberson * 17578355f576SJeff Roberson * Returns: 17588355f576SJeff Roberson * A pointer to the alloced memory or possibly 17598355f576SJeff Roberson * NULL if M_NOWAIT is set. 17608355f576SJeff Roberson */ 17618355f576SJeff Roberson static void * 1762ab3185d1SJeff Roberson noobj_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, 1763ab3185d1SJeff Roberson int wait) 17648355f576SJeff Roberson { 1765a4915c21SAttilio Rao TAILQ_HEAD(, vm_page) alloctail; 1766a4915c21SAttilio Rao u_long npages; 1767b245ac95SAlan Cox vm_offset_t retkva, zkva; 1768a4915c21SAttilio Rao vm_page_t p, p_next; 1769e20a199fSJeff Roberson uma_keg_t keg; 17708355f576SJeff Roberson 1771a4915c21SAttilio Rao TAILQ_INIT(&alloctail); 1772bb15d1c7SGleb Smirnoff keg = zone->uz_keg; 1773a4915c21SAttilio Rao 1774a4915c21SAttilio Rao npages = howmany(bytes, PAGE_SIZE); 1775a4915c21SAttilio Rao while (npages > 0) { 1776ab3185d1SJeff Roberson p = vm_page_alloc_domain(NULL, 0, domain, VM_ALLOC_INTERRUPT | 17778d6fbbb8SJeff Roberson VM_ALLOC_WIRED | VM_ALLOC_NOOBJ | 1778772c8b67SKonstantin Belousov ((wait & M_WAITOK) != 0 ? VM_ALLOC_WAITOK : 1779772c8b67SKonstantin Belousov VM_ALLOC_NOWAIT)); 1780a4915c21SAttilio Rao if (p != NULL) { 1781a4915c21SAttilio Rao /* 1782a4915c21SAttilio Rao * Since the page does not belong to an object, its 1783a4915c21SAttilio Rao * listq is unused. 1784a4915c21SAttilio Rao */ 1785a4915c21SAttilio Rao TAILQ_INSERT_TAIL(&alloctail, p, listq); 1786a4915c21SAttilio Rao npages--; 1787a4915c21SAttilio Rao continue; 1788a4915c21SAttilio Rao } 17898355f576SJeff Roberson /* 1790a4915c21SAttilio Rao * Page allocation failed, free intermediate pages and 1791a4915c21SAttilio Rao * exit. 17928355f576SJeff Roberson */ 1793a4915c21SAttilio Rao TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) { 179488ea538aSMark Johnston vm_page_unwire_noq(p); 1795b245ac95SAlan Cox vm_page_free(p); 1796b245ac95SAlan Cox } 1797a4915c21SAttilio Rao return (NULL); 1798b245ac95SAlan Cox } 17998355f576SJeff Roberson *flags = UMA_SLAB_PRIV; 1800a4915c21SAttilio Rao zkva = keg->uk_kva + 1801a4915c21SAttilio Rao atomic_fetchadd_long(&keg->uk_offset, round_page(bytes)); 1802a4915c21SAttilio Rao retkva = zkva; 1803a4915c21SAttilio Rao TAILQ_FOREACH(p, &alloctail, listq) { 1804a4915c21SAttilio Rao pmap_qenter(zkva, &p, 1); 1805a4915c21SAttilio Rao zkva += PAGE_SIZE; 1806a4915c21SAttilio Rao } 18078355f576SJeff Roberson 18088355f576SJeff Roberson return ((void *)retkva); 18098355f576SJeff Roberson } 18108355f576SJeff Roberson 18118355f576SJeff Roberson /* 1812ec0d8280SRyan Libby * Allocate physically contiguous pages. 1813ec0d8280SRyan Libby */ 1814ec0d8280SRyan Libby static void * 1815ec0d8280SRyan Libby contig_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 1816ec0d8280SRyan Libby int wait) 1817ec0d8280SRyan Libby { 1818ec0d8280SRyan Libby 1819ec0d8280SRyan Libby *pflag = UMA_SLAB_KERNEL; 1820ec0d8280SRyan Libby return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain), 1821ec0d8280SRyan Libby bytes, wait, 0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT)); 1822ec0d8280SRyan Libby } 1823ec0d8280SRyan Libby 1824ec0d8280SRyan Libby /* 18258355f576SJeff Roberson * Frees a number of pages to the system 18268355f576SJeff Roberson * 18278355f576SJeff Roberson * Arguments: 18288355f576SJeff Roberson * mem A pointer to the memory to be freed 18298355f576SJeff Roberson * size The size of the memory being freed 18308355f576SJeff Roberson * flags The original p->us_flags field 18318355f576SJeff Roberson * 18328355f576SJeff Roberson * Returns: 18338355f576SJeff Roberson * Nothing 18348355f576SJeff Roberson */ 18358355f576SJeff Roberson static void 1836f2c2231eSRyan Stone page_free(void *mem, vm_size_t size, uint8_t flags) 18378355f576SJeff Roberson { 18383370c5bfSJeff Roberson 1839a81c400eSJeff Roberson if ((flags & UMA_SLAB_BOOT) != 0) { 1840a81c400eSJeff Roberson startup_free(mem, size); 1841a81c400eSJeff Roberson return; 1842a81c400eSJeff Roberson } 1843a81c400eSJeff Roberson 1844ec0d8280SRyan Libby KASSERT((flags & UMA_SLAB_KERNEL) != 0, 1845ec0d8280SRyan Libby ("UMA: page_free used with invalid flags %x", flags)); 18468355f576SJeff Roberson 184749bfa624SAlan Cox kmem_free((vm_offset_t)mem, size); 18488355f576SJeff Roberson } 18498355f576SJeff Roberson 18508355f576SJeff Roberson /* 1851ab3059a8SMatt Macy * Frees pcpu zone allocations 1852ab3059a8SMatt Macy * 1853ab3059a8SMatt Macy * Arguments: 1854ab3059a8SMatt Macy * mem A pointer to the memory to be freed 1855ab3059a8SMatt Macy * size The size of the memory being freed 1856ab3059a8SMatt Macy * flags The original p->us_flags field 1857ab3059a8SMatt Macy * 1858ab3059a8SMatt Macy * Returns: 1859ab3059a8SMatt Macy * Nothing 1860ab3059a8SMatt Macy */ 1861ab3059a8SMatt Macy static void 1862ab3059a8SMatt Macy pcpu_page_free(void *mem, vm_size_t size, uint8_t flags) 1863ab3059a8SMatt Macy { 1864ab3059a8SMatt Macy vm_offset_t sva, curva; 1865ab3059a8SMatt Macy vm_paddr_t paddr; 1866ab3059a8SMatt Macy vm_page_t m; 1867ab3059a8SMatt Macy 1868ab3059a8SMatt Macy MPASS(size == (mp_maxid+1)*PAGE_SIZE); 18695ba16cf3SRyan Libby 18705ba16cf3SRyan Libby if ((flags & UMA_SLAB_BOOT) != 0) { 18715ba16cf3SRyan Libby startup_free(mem, size); 18725ba16cf3SRyan Libby return; 18735ba16cf3SRyan Libby } 18745ba16cf3SRyan Libby 1875ab3059a8SMatt Macy sva = (vm_offset_t)mem; 1876ab3059a8SMatt Macy for (curva = sva; curva < sva + size; curva += PAGE_SIZE) { 1877ab3059a8SMatt Macy paddr = pmap_kextract(curva); 1878ab3059a8SMatt Macy m = PHYS_TO_VM_PAGE(paddr); 187988ea538aSMark Johnston vm_page_unwire_noq(m); 1880ab3059a8SMatt Macy vm_page_free(m); 1881ab3059a8SMatt Macy } 1882ab3059a8SMatt Macy pmap_qremove(sva, size >> PAGE_SHIFT); 1883ab3059a8SMatt Macy kva_free(sva, size); 1884ab3059a8SMatt Macy } 1885ab3059a8SMatt Macy 1886ab3059a8SMatt Macy 1887ab3059a8SMatt Macy /* 18888355f576SJeff Roberson * Zero fill initializer 18898355f576SJeff Roberson * 18908355f576SJeff Roberson * Arguments/Returns follow uma_init specifications 18918355f576SJeff Roberson */ 1892b23f72e9SBrian Feldman static int 1893b23f72e9SBrian Feldman zero_init(void *mem, int size, int flags) 18948355f576SJeff Roberson { 18958355f576SJeff Roberson bzero(mem, size); 1896b23f72e9SBrian Feldman return (0); 18978355f576SJeff Roberson } 18988355f576SJeff Roberson 1899815db204SRyan Libby #ifdef INVARIANTS 1900815db204SRyan Libby struct noslabbits * 1901815db204SRyan Libby slab_dbg_bits(uma_slab_t slab, uma_keg_t keg) 1902815db204SRyan Libby { 1903815db204SRyan Libby 1904815db204SRyan Libby return ((void *)((char *)&slab->us_free + BITSET_SIZE(keg->uk_ipers))); 1905815db204SRyan Libby } 1906815db204SRyan Libby #endif 1907815db204SRyan Libby 19088355f576SJeff Roberson /* 19099b78b1f4SJeff Roberson * Actual size of embedded struct slab (!OFFPAGE). 19109b78b1f4SJeff Roberson */ 19119b78b1f4SJeff Roberson size_t 19129b78b1f4SJeff Roberson slab_sizeof(int nitems) 19139b78b1f4SJeff Roberson { 19149b78b1f4SJeff Roberson size_t s; 19159b78b1f4SJeff Roberson 1916815db204SRyan Libby s = sizeof(struct uma_slab) + BITSET_SIZE(nitems) * SLAB_BITSETS; 19179b78b1f4SJeff Roberson return (roundup(s, UMA_ALIGN_PTR + 1)); 19189b78b1f4SJeff Roberson } 19199b78b1f4SJeff Roberson 19209b78b1f4SJeff Roberson /* 19219b78b1f4SJeff Roberson * Size of memory for embedded slabs (!OFFPAGE). 19229b78b1f4SJeff Roberson */ 19239b78b1f4SJeff Roberson size_t 19249b78b1f4SJeff Roberson slab_space(int nitems) 19259b78b1f4SJeff Roberson { 19269b78b1f4SJeff Roberson return (UMA_SLAB_SIZE - slab_sizeof(nitems)); 19279b78b1f4SJeff Roberson } 19289b78b1f4SJeff Roberson 19294a8b575cSRyan Libby #define UMA_FIXPT_SHIFT 31 19304a8b575cSRyan Libby #define UMA_FRAC_FIXPT(n, d) \ 19314a8b575cSRyan Libby ((uint32_t)(((uint64_t)(n) << UMA_FIXPT_SHIFT) / (d))) 19324a8b575cSRyan Libby #define UMA_FIXPT_PCT(f) \ 19334a8b575cSRyan Libby ((u_int)(((uint64_t)100 * (f)) >> UMA_FIXPT_SHIFT)) 19344a8b575cSRyan Libby #define UMA_PCT_FIXPT(pct) UMA_FRAC_FIXPT((pct), 100) 19354a8b575cSRyan Libby #define UMA_MIN_EFF UMA_PCT_FIXPT(100 - UMA_MAX_WASTE) 19364a8b575cSRyan Libby 19379b78b1f4SJeff Roberson /* 19384a8b575cSRyan Libby * Compute the number of items that will fit in a slab. If hdr is true, the 19394a8b575cSRyan Libby * item count may be limited to provide space in the slab for an inline slab 19404a8b575cSRyan Libby * header. Otherwise, all slab space will be provided for item storage. 19414a8b575cSRyan Libby */ 19424a8b575cSRyan Libby static u_int 19434a8b575cSRyan Libby slab_ipers_hdr(u_int size, u_int rsize, u_int slabsize, bool hdr) 19444a8b575cSRyan Libby { 19454a8b575cSRyan Libby u_int ipers; 19464a8b575cSRyan Libby u_int padpi; 19474a8b575cSRyan Libby 19484a8b575cSRyan Libby /* The padding between items is not needed after the last item. */ 19494a8b575cSRyan Libby padpi = rsize - size; 19504a8b575cSRyan Libby 19514a8b575cSRyan Libby if (hdr) { 19524a8b575cSRyan Libby /* 19534a8b575cSRyan Libby * Start with the maximum item count and remove items until 19544a8b575cSRyan Libby * the slab header first alongside the allocatable memory. 19554a8b575cSRyan Libby */ 19564a8b575cSRyan Libby for (ipers = MIN(SLAB_MAX_SETSIZE, 19574a8b575cSRyan Libby (slabsize + padpi - slab_sizeof(1)) / rsize); 19584a8b575cSRyan Libby ipers > 0 && 19594a8b575cSRyan Libby ipers * rsize - padpi + slab_sizeof(ipers) > slabsize; 19604a8b575cSRyan Libby ipers--) 19614a8b575cSRyan Libby continue; 19624a8b575cSRyan Libby } else { 19634a8b575cSRyan Libby ipers = MIN((slabsize + padpi) / rsize, SLAB_MAX_SETSIZE); 19644a8b575cSRyan Libby } 19654a8b575cSRyan Libby 19664a8b575cSRyan Libby return (ipers); 19674a8b575cSRyan Libby } 19684a8b575cSRyan Libby 19694a8b575cSRyan Libby /* 19704a8b575cSRyan Libby * Compute the number of items that will fit in a slab for a startup zone. 19719b78b1f4SJeff Roberson */ 19729b78b1f4SJeff Roberson int 19739b78b1f4SJeff Roberson slab_ipers(size_t size, int align) 19749b78b1f4SJeff Roberson { 19759b78b1f4SJeff Roberson int rsize; 19769b78b1f4SJeff Roberson 19774a8b575cSRyan Libby rsize = roundup(size, align + 1); /* Assume no CACHESPREAD */ 19784a8b575cSRyan Libby return (slab_ipers_hdr(size, rsize, UMA_SLAB_SIZE, true)); 19799b78b1f4SJeff Roberson } 19809b78b1f4SJeff Roberson 198127ca37acSRyan Libby struct keg_layout_result { 198227ca37acSRyan Libby u_int format; 198327ca37acSRyan Libby u_int slabsize; 198427ca37acSRyan Libby u_int ipers; 198527ca37acSRyan Libby u_int eff; 198627ca37acSRyan Libby }; 198727ca37acSRyan Libby 198827ca37acSRyan Libby static void 198927ca37acSRyan Libby keg_layout_one(uma_keg_t keg, u_int rsize, u_int slabsize, u_int fmt, 199027ca37acSRyan Libby struct keg_layout_result *kl) 199127ca37acSRyan Libby { 199227ca37acSRyan Libby u_int total; 199327ca37acSRyan Libby 199427ca37acSRyan Libby kl->format = fmt; 199527ca37acSRyan Libby kl->slabsize = slabsize; 199627ca37acSRyan Libby 199727ca37acSRyan Libby /* Handle INTERNAL as inline with an extra page. */ 199827ca37acSRyan Libby if ((fmt & UMA_ZFLAG_INTERNAL) != 0) { 199927ca37acSRyan Libby kl->format &= ~UMA_ZFLAG_INTERNAL; 200027ca37acSRyan Libby kl->slabsize += PAGE_SIZE; 200127ca37acSRyan Libby } 200227ca37acSRyan Libby 200327ca37acSRyan Libby kl->ipers = slab_ipers_hdr(keg->uk_size, rsize, kl->slabsize, 200427ca37acSRyan Libby (fmt & UMA_ZFLAG_OFFPAGE) == 0); 200527ca37acSRyan Libby 200627ca37acSRyan Libby /* Account for memory used by an offpage slab header. */ 200727ca37acSRyan Libby total = kl->slabsize; 200827ca37acSRyan Libby if ((fmt & UMA_ZFLAG_OFFPAGE) != 0) 200927ca37acSRyan Libby total += slabzone(kl->ipers)->uz_keg->uk_rsize; 201027ca37acSRyan Libby 201127ca37acSRyan Libby kl->eff = UMA_FRAC_FIXPT(kl->ipers * rsize, total); 201227ca37acSRyan Libby } 201327ca37acSRyan Libby 20149b78b1f4SJeff Roberson /* 20154a8b575cSRyan Libby * Determine the format of a uma keg. This determines where the slab header 20164a8b575cSRyan Libby * will be placed (inline or offpage) and calculates ipers, rsize, and ppera. 20178355f576SJeff Roberson * 20188355f576SJeff Roberson * Arguments 2019e20a199fSJeff Roberson * keg The zone we should initialize 20208355f576SJeff Roberson * 20218355f576SJeff Roberson * Returns 20228355f576SJeff Roberson * Nothing 20238355f576SJeff Roberson */ 20248355f576SJeff Roberson static void 20254a8b575cSRyan Libby keg_layout(uma_keg_t keg) 20268355f576SJeff Roberson { 202727ca37acSRyan Libby struct keg_layout_result kl = {}, kl_tmp; 202827ca37acSRyan Libby u_int fmts[2]; 20294a8b575cSRyan Libby u_int alignsize; 203027ca37acSRyan Libby u_int nfmt; 20314a8b575cSRyan Libby u_int pages; 2032244f4554SBosko Milekic u_int rsize; 2033a55ebb7cSAndriy Gapon u_int slabsize; 203427ca37acSRyan Libby u_int i, j; 20358355f576SJeff Roberson 20364a8b575cSRyan Libby KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || 20374a8b575cSRyan Libby (keg->uk_size <= UMA_PCPU_ALLOC_SIZE && 20384a8b575cSRyan Libby (keg->uk_flags & UMA_ZONE_CACHESPREAD) == 0), 20394a8b575cSRyan Libby ("%s: cannot configure for PCPU: keg=%s, size=%u, flags=0x%b", 20404a8b575cSRyan Libby __func__, keg->uk_name, keg->uk_size, keg->uk_flags, 20414a8b575cSRyan Libby PRINT_UMA_ZFLAGS)); 2042bae55c4aSRyan Libby KASSERT((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZONE_VM)) == 0 || 20434a8b575cSRyan Libby (keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0, 20444a8b575cSRyan Libby ("%s: incompatible flags 0x%b", __func__, keg->uk_flags, 20454a8b575cSRyan Libby PRINT_UMA_ZFLAGS)); 2046e28a647dSGleb Smirnoff 20474a8b575cSRyan Libby alignsize = keg->uk_align + 1; 2048ad97af7eSGleb Smirnoff 2049ef72505eSJeff Roberson /* 2050ef72505eSJeff Roberson * Calculate the size of each allocation (rsize) according to 2051ef72505eSJeff Roberson * alignment. If the requested size is smaller than we have 2052ef72505eSJeff Roberson * allocation bits for we round it up. 2053ef72505eSJeff Roberson */ 20549b8db4d0SRyan Libby rsize = MAX(keg->uk_size, UMA_SMALLEST_UNIT); 20554a8b575cSRyan Libby rsize = roundup2(rsize, alignsize); 2056ad97af7eSGleb Smirnoff 205727ca37acSRyan Libby if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) { 20589b78b1f4SJeff Roberson /* 20594a8b575cSRyan Libby * We want one item to start on every align boundary in a page. 20604a8b575cSRyan Libby * To do this we will span pages. We will also extend the item 20614a8b575cSRyan Libby * by the size of align if it is an even multiple of align. 20624a8b575cSRyan Libby * Otherwise, it would fall on the same boundary every time. 20639b78b1f4SJeff Roberson */ 20644a8b575cSRyan Libby if ((rsize & alignsize) == 0) 20654a8b575cSRyan Libby rsize += alignsize; 20664a8b575cSRyan Libby slabsize = rsize * (PAGE_SIZE / alignsize); 20674a8b575cSRyan Libby slabsize = MIN(slabsize, rsize * SLAB_MAX_SETSIZE); 20684a8b575cSRyan Libby slabsize = MIN(slabsize, UMA_CACHESPREAD_MAX_SIZE); 206927ca37acSRyan Libby slabsize = round_page(slabsize); 20704a8b575cSRyan Libby } else { 20714a8b575cSRyan Libby /* 207227ca37acSRyan Libby * Start with a slab size of as many pages as it takes to 207327ca37acSRyan Libby * represent a single item. We will try to fit as many 207427ca37acSRyan Libby * additional items into the slab as possible. 20754a8b575cSRyan Libby */ 207627ca37acSRyan Libby slabsize = round_page(keg->uk_size); 20771ca6ed45SGleb Smirnoff } 2078ad97af7eSGleb Smirnoff 207927ca37acSRyan Libby /* Build a list of all of the available formats for this keg. */ 208027ca37acSRyan Libby nfmt = 0; 208127ca37acSRyan Libby 20824a8b575cSRyan Libby /* Evaluate an inline slab layout. */ 20834a8b575cSRyan Libby if ((keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0) 208427ca37acSRyan Libby fmts[nfmt++] = 0; 20854a8b575cSRyan Libby 20864a8b575cSRyan Libby /* TODO: vm_page-embedded slab. */ 2087244f4554SBosko Milekic 208820e8e865SBosko Milekic /* 2089244f4554SBosko Milekic * We can't do OFFPAGE if we're internal or if we've been 209020e8e865SBosko Milekic * asked to not go to the VM for buckets. If we do this we 2091bae55c4aSRyan Libby * may end up going to the VM for slabs which we do not want 2092bae55c4aSRyan Libby * to do if we're UMA_ZONE_VM, which clearly forbids it. 2093bae55c4aSRyan Libby * In those cases, evaluate a pseudo-format called INTERNAL 2094bae55c4aSRyan Libby * which has an inline slab header and one extra page to 2095bae55c4aSRyan Libby * guarantee that it fits. 209627ca37acSRyan Libby * 209727ca37acSRyan Libby * Otherwise, see if using an OFFPAGE slab will improve our 209827ca37acSRyan Libby * efficiency. 209920e8e865SBosko Milekic */ 2100bae55c4aSRyan Libby if ((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZONE_VM)) != 0) 210127ca37acSRyan Libby fmts[nfmt++] = UMA_ZFLAG_INTERNAL; 210227ca37acSRyan Libby else 210327ca37acSRyan Libby fmts[nfmt++] = UMA_ZFLAG_OFFPAGE; 2104244f4554SBosko Milekic 2105ef72505eSJeff Roberson /* 210627ca37acSRyan Libby * Choose a slab size and format which satisfy the minimum efficiency. 210727ca37acSRyan Libby * Prefer the smallest slab size that meets the constraints. 2108ef72505eSJeff Roberson * 210927ca37acSRyan Libby * Start with a minimum slab size, to accommodate CACHESPREAD. Then, 211027ca37acSRyan Libby * for small items (up to PAGE_SIZE), the iteration increment is one 211127ca37acSRyan Libby * page; and for large items, the increment is one item. 2112ef72505eSJeff Roberson */ 211327ca37acSRyan Libby i = (slabsize + rsize - keg->uk_size) / MAX(PAGE_SIZE, rsize); 211427ca37acSRyan Libby KASSERT(i >= 1, ("keg %s(%p) flags=0x%b slabsize=%u, rsize=%u, i=%u", 211527ca37acSRyan Libby keg->uk_name, keg, keg->uk_flags, PRINT_UMA_ZFLAGS, slabsize, 211627ca37acSRyan Libby rsize, i)); 211727ca37acSRyan Libby for ( ; ; i++) { 211827ca37acSRyan Libby slabsize = (rsize <= PAGE_SIZE) ? ptoa(i) : 211927ca37acSRyan Libby round_page(rsize * (i - 1) + keg->uk_size); 212027ca37acSRyan Libby 212127ca37acSRyan Libby for (j = 0; j < nfmt; j++) { 212227ca37acSRyan Libby /* Only if we have no viable format yet. */ 212327ca37acSRyan Libby if ((fmts[j] & UMA_ZFLAG_INTERNAL) != 0 && 212427ca37acSRyan Libby kl.ipers > 0) 212527ca37acSRyan Libby continue; 212627ca37acSRyan Libby 212727ca37acSRyan Libby keg_layout_one(keg, rsize, slabsize, fmts[j], &kl_tmp); 212827ca37acSRyan Libby if (kl_tmp.eff <= kl.eff) 212927ca37acSRyan Libby continue; 213027ca37acSRyan Libby 213127ca37acSRyan Libby kl = kl_tmp; 213227ca37acSRyan Libby 213327ca37acSRyan Libby CTR6(KTR_UMA, "keg %s layout: format %#x " 213427ca37acSRyan Libby "(ipers %u * rsize %u) / slabsize %#x = %u%% eff", 213527ca37acSRyan Libby keg->uk_name, kl.format, kl.ipers, rsize, 213627ca37acSRyan Libby kl.slabsize, UMA_FIXPT_PCT(kl.eff)); 213727ca37acSRyan Libby 213827ca37acSRyan Libby /* Stop when we reach the minimum efficiency. */ 213927ca37acSRyan Libby if (kl.eff >= UMA_MIN_EFF) 214027ca37acSRyan Libby break; 21418355f576SJeff Roberson } 2142ad97af7eSGleb Smirnoff 214333e5a1eaSRyan Libby if (kl.eff >= UMA_MIN_EFF || !multipage_slabs || 214427ca37acSRyan Libby slabsize >= SLAB_MAX_SETSIZE * rsize || 214527ca37acSRyan Libby (keg->uk_flags & (UMA_ZONE_PCPU | UMA_ZONE_CONTIG)) != 0) 214627ca37acSRyan Libby break; 214727ca37acSRyan Libby } 214827ca37acSRyan Libby 214927ca37acSRyan Libby pages = atop(kl.slabsize); 215027ca37acSRyan Libby if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) 215127ca37acSRyan Libby pages *= mp_maxid + 1; 215227ca37acSRyan Libby 215327ca37acSRyan Libby keg->uk_rsize = rsize; 215427ca37acSRyan Libby keg->uk_ipers = kl.ipers; 215527ca37acSRyan Libby keg->uk_ppera = pages; 215627ca37acSRyan Libby keg->uk_flags |= kl.format; 215727ca37acSRyan Libby 21584a8b575cSRyan Libby /* 21594a8b575cSRyan Libby * How do we find the slab header if it is offpage or if not all item 21604a8b575cSRyan Libby * start addresses are in the same page? We could solve the latter 21614a8b575cSRyan Libby * case with vaddr alignment, but we don't. 21624a8b575cSRyan Libby */ 216327ca37acSRyan Libby if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0 || 216427ca37acSRyan Libby (keg->uk_ipers - 1) * rsize >= PAGE_SIZE) { 216554c5ae80SRyan Libby if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0) 216627ca37acSRyan Libby keg->uk_flags |= UMA_ZFLAG_HASH; 216754c5ae80SRyan Libby else 216827ca37acSRyan Libby keg->uk_flags |= UMA_ZFLAG_VTOSLAB; 216954c5ae80SRyan Libby } 217027ca37acSRyan Libby 2171e63a1c2fSRyan Libby CTR6(KTR_UMA, "%s: keg=%s, flags=%#x, rsize=%u, ipers=%u, ppera=%u", 217227ca37acSRyan Libby __func__, keg->uk_name, keg->uk_flags, rsize, keg->uk_ipers, 217327ca37acSRyan Libby pages); 21744a8b575cSRyan Libby KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_MAX_SETSIZE, 21754a8b575cSRyan Libby ("%s: keg=%s, flags=0x%b, rsize=%u, ipers=%u, ppera=%u", __func__, 217627ca37acSRyan Libby keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize, 217727ca37acSRyan Libby keg->uk_ipers, pages)); 2178e20a199fSJeff Roberson } 2179e20a199fSJeff Roberson 21808355f576SJeff Roberson /* 2181099a0e58SBosko Milekic * Keg header ctor. This initializes all fields, locks, etc. And inserts 2182099a0e58SBosko Milekic * the keg onto the global keg list. 21838355f576SJeff Roberson * 21848355f576SJeff Roberson * Arguments/Returns follow uma_ctor specifications 2185099a0e58SBosko Milekic * udata Actually uma_kctor_args 2186099a0e58SBosko Milekic */ 2187b23f72e9SBrian Feldman static int 2188b23f72e9SBrian Feldman keg_ctor(void *mem, int size, void *udata, int flags) 2189099a0e58SBosko Milekic { 2190099a0e58SBosko Milekic struct uma_kctor_args *arg = udata; 2191099a0e58SBosko Milekic uma_keg_t keg = mem; 2192099a0e58SBosko Milekic uma_zone_t zone; 21938b987a77SJeff Roberson int i; 2194099a0e58SBosko Milekic 2195099a0e58SBosko Milekic bzero(keg, size); 2196099a0e58SBosko Milekic keg->uk_size = arg->size; 2197099a0e58SBosko Milekic keg->uk_init = arg->uminit; 2198099a0e58SBosko Milekic keg->uk_fini = arg->fini; 2199099a0e58SBosko Milekic keg->uk_align = arg->align; 22006fd34d6fSJeff Roberson keg->uk_reserve = 0; 2201099a0e58SBosko Milekic keg->uk_flags = arg->flags; 2202099a0e58SBosko Milekic 2203099a0e58SBosko Milekic /* 2204194a979eSMark Johnston * We use a global round-robin policy by default. Zones with 2205dfe13344SJeff Roberson * UMA_ZONE_FIRSTTOUCH set will use first-touch instead, in which 2206dfe13344SJeff Roberson * case the iterator is never run. 2207194a979eSMark Johnston */ 2208194a979eSMark Johnston keg->uk_dr.dr_policy = DOMAINSET_RR(); 2209194a979eSMark Johnston keg->uk_dr.dr_iter = 0; 2210194a979eSMark Johnston 2211194a979eSMark Johnston /* 2212099a0e58SBosko Milekic * The master zone is passed to us at keg-creation time. 2213099a0e58SBosko Milekic */ 2214099a0e58SBosko Milekic zone = arg->zone; 2215e20a199fSJeff Roberson keg->uk_name = zone->uz_name; 2216099a0e58SBosko Milekic 2217099a0e58SBosko Milekic if (arg->flags & UMA_ZONE_ZINIT) 2218099a0e58SBosko Milekic keg->uk_init = zero_init; 2219099a0e58SBosko Milekic 2220cfcae3f8SGleb Smirnoff if (arg->flags & UMA_ZONE_MALLOC) 222154c5ae80SRyan Libby keg->uk_flags |= UMA_ZFLAG_VTOSLAB; 2222e20a199fSJeff Roberson 222354c5ae80SRyan Libby #ifndef SMP 2224ad97af7eSGleb Smirnoff keg->uk_flags &= ~UMA_ZONE_PCPU; 2225ad97af7eSGleb Smirnoff #endif 2226ad97af7eSGleb Smirnoff 22274a8b575cSRyan Libby keg_layout(keg); 2228099a0e58SBosko Milekic 22298b987a77SJeff Roberson /* 2230c6fd3e23SJeff Roberson * Use a first-touch NUMA policy for kegs that pmap_extract() will 2231c6fd3e23SJeff Roberson * work on. Use round-robin for everything else. 2232dfe13344SJeff Roberson * 2233dfe13344SJeff Roberson * Zones may override the default by specifying either. 22348b987a77SJeff Roberson */ 2235dfe13344SJeff Roberson #ifdef NUMA 2236dfe13344SJeff Roberson if ((keg->uk_flags & 2237c6fd3e23SJeff Roberson (UMA_ZONE_ROUNDROBIN | UMA_ZFLAG_CACHE | UMA_ZONE_NOTPAGE)) == 0) 2238dfe13344SJeff Roberson keg->uk_flags |= UMA_ZONE_FIRSTTOUCH; 2239dfe13344SJeff Roberson else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) 2240dfe13344SJeff Roberson keg->uk_flags |= UMA_ZONE_ROUNDROBIN; 22418b987a77SJeff Roberson #endif 22428b987a77SJeff Roberson 2243099a0e58SBosko Milekic /* 2244099a0e58SBosko Milekic * If we haven't booted yet we need allocations to go through the 2245099a0e58SBosko Milekic * startup cache until the vm is ready. 2246099a0e58SBosko Milekic */ 224777e19437SGleb Smirnoff #ifdef UMA_MD_SMALL_ALLOC 2248a81c400eSJeff Roberson if (keg->uk_ppera == 1) 224977e19437SGleb Smirnoff keg->uk_allocf = uma_small_alloc; 2250a81c400eSJeff Roberson else 22518cd02d00SAlan Cox #endif 2252a81c400eSJeff Roberson if (booted < BOOT_KVA) 2253a81c400eSJeff Roberson keg->uk_allocf = startup_alloc; 2254ab3059a8SMatt Macy else if (keg->uk_flags & UMA_ZONE_PCPU) 2255ab3059a8SMatt Macy keg->uk_allocf = pcpu_page_alloc; 2256ec0d8280SRyan Libby else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && keg->uk_ppera > 1) 2257ec0d8280SRyan Libby keg->uk_allocf = contig_alloc; 225877e19437SGleb Smirnoff else 225977e19437SGleb Smirnoff keg->uk_allocf = page_alloc; 226077e19437SGleb Smirnoff #ifdef UMA_MD_SMALL_ALLOC 226177e19437SGleb Smirnoff if (keg->uk_ppera == 1) 226277e19437SGleb Smirnoff keg->uk_freef = uma_small_free; 226377e19437SGleb Smirnoff else 226477e19437SGleb Smirnoff #endif 2265ab3059a8SMatt Macy if (keg->uk_flags & UMA_ZONE_PCPU) 2266ab3059a8SMatt Macy keg->uk_freef = pcpu_page_free; 2267ab3059a8SMatt Macy else 226877e19437SGleb Smirnoff keg->uk_freef = page_free; 2269099a0e58SBosko Milekic 2270099a0e58SBosko Milekic /* 22718b987a77SJeff Roberson * Initialize keg's locks. 2272099a0e58SBosko Milekic */ 22738b987a77SJeff Roberson for (i = 0; i < vm_ndomains; i++) 22748b987a77SJeff Roberson KEG_LOCK_INIT(keg, i, (arg->flags & UMA_ZONE_MTXCLASS)); 2275099a0e58SBosko Milekic 2276099a0e58SBosko Milekic /* 2277099a0e58SBosko Milekic * If we're putting the slab header in the actual page we need to 22789b78b1f4SJeff Roberson * figure out where in each page it goes. See slab_sizeof 22799b78b1f4SJeff Roberson * definition. 2280099a0e58SBosko Milekic */ 228154c5ae80SRyan Libby if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) { 22829b78b1f4SJeff Roberson size_t shsize; 22839b78b1f4SJeff Roberson 22849b78b1f4SJeff Roberson shsize = slab_sizeof(keg->uk_ipers); 22859b78b1f4SJeff Roberson keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - shsize; 2286244f4554SBosko Milekic /* 2287244f4554SBosko Milekic * The only way the following is possible is if with our 2288244f4554SBosko Milekic * UMA_ALIGN_PTR adjustments we are now bigger than 2289244f4554SBosko Milekic * UMA_SLAB_SIZE. I haven't checked whether this is 2290244f4554SBosko Milekic * mathematically possible for all cases, so we make 2291244f4554SBosko Milekic * sure here anyway. 2292244f4554SBosko Milekic */ 22939b78b1f4SJeff Roberson KASSERT(keg->uk_pgoff + shsize <= PAGE_SIZE * keg->uk_ppera, 22943d5e3df7SGleb Smirnoff ("zone %s ipers %d rsize %d size %d slab won't fit", 22953d5e3df7SGleb Smirnoff zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); 2296099a0e58SBosko Milekic } 2297099a0e58SBosko Milekic 229854c5ae80SRyan Libby if (keg->uk_flags & UMA_ZFLAG_HASH) 22993b2f2cb8SAlexander Motin hash_alloc(&keg->uk_hash, 0); 2300099a0e58SBosko Milekic 2301e63a1c2fSRyan Libby CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)", keg, zone->uz_name, zone); 2302099a0e58SBosko Milekic 2303099a0e58SBosko Milekic LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); 2304099a0e58SBosko Milekic 2305111fbcd5SBryan Venteicher rw_wlock(&uma_rwlock); 2306099a0e58SBosko Milekic LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); 2307111fbcd5SBryan Venteicher rw_wunlock(&uma_rwlock); 2308b23f72e9SBrian Feldman return (0); 2309099a0e58SBosko Milekic } 2310099a0e58SBosko Milekic 23112efcc8cbSGleb Smirnoff static void 2312a81c400eSJeff Roberson zone_kva_available(uma_zone_t zone, void *unused) 2313a81c400eSJeff Roberson { 2314a81c400eSJeff Roberson uma_keg_t keg; 2315a81c400eSJeff Roberson 2316a81c400eSJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) 2317a81c400eSJeff Roberson return; 2318a81c400eSJeff Roberson KEG_GET(zone, keg); 2319ec0d8280SRyan Libby 2320ec0d8280SRyan Libby if (keg->uk_allocf == startup_alloc) { 2321ec0d8280SRyan Libby /* Switch to the real allocator. */ 2322f96d4157SJeff Roberson if (keg->uk_flags & UMA_ZONE_PCPU) 2323f96d4157SJeff Roberson keg->uk_allocf = pcpu_page_alloc; 2324ec0d8280SRyan Libby else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && 2325ec0d8280SRyan Libby keg->uk_ppera > 1) 2326ec0d8280SRyan Libby keg->uk_allocf = contig_alloc; 2327ec0d8280SRyan Libby else 2328a81c400eSJeff Roberson keg->uk_allocf = page_alloc; 2329a81c400eSJeff Roberson } 2330ec0d8280SRyan Libby } 2331a81c400eSJeff Roberson 2332a81c400eSJeff Roberson static void 233320a4e154SJeff Roberson zone_alloc_counters(uma_zone_t zone, void *unused) 23342efcc8cbSGleb Smirnoff { 23352efcc8cbSGleb Smirnoff 23362efcc8cbSGleb Smirnoff zone->uz_allocs = counter_u64_alloc(M_WAITOK); 23372efcc8cbSGleb Smirnoff zone->uz_frees = counter_u64_alloc(M_WAITOK); 23382efcc8cbSGleb Smirnoff zone->uz_fails = counter_u64_alloc(M_WAITOK); 2339c6fd3e23SJeff Roberson zone->uz_xdomain = counter_u64_alloc(M_WAITOK); 23402efcc8cbSGleb Smirnoff } 23412efcc8cbSGleb Smirnoff 234220a4e154SJeff Roberson static void 234320a4e154SJeff Roberson zone_alloc_sysctl(uma_zone_t zone, void *unused) 234420a4e154SJeff Roberson { 234520a4e154SJeff Roberson uma_zone_domain_t zdom; 23468b987a77SJeff Roberson uma_domain_t dom; 234720a4e154SJeff Roberson uma_keg_t keg; 234820a4e154SJeff Roberson struct sysctl_oid *oid, *domainoid; 23493b490537SJeff Roberson int domains, i, cnt; 235020a4e154SJeff Roberson static const char *nokeg = "cache zone"; 235120a4e154SJeff Roberson char *c; 235220a4e154SJeff Roberson 235320a4e154SJeff Roberson /* 235420a4e154SJeff Roberson * Make a sysctl safe copy of the zone name by removing 235520a4e154SJeff Roberson * any special characters and handling dups by appending 235620a4e154SJeff Roberson * an index. 235720a4e154SJeff Roberson */ 235820a4e154SJeff Roberson if (zone->uz_namecnt != 0) { 23593b490537SJeff Roberson /* Count the number of decimal digits and '_' separator. */ 23603b490537SJeff Roberson for (i = 1, cnt = zone->uz_namecnt; cnt != 0; i++) 23613b490537SJeff Roberson cnt /= 10; 23623b490537SJeff Roberson zone->uz_ctlname = malloc(strlen(zone->uz_name) + i + 1, 23633b490537SJeff Roberson M_UMA, M_WAITOK); 236420a4e154SJeff Roberson sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name, 236520a4e154SJeff Roberson zone->uz_namecnt); 236620a4e154SJeff Roberson } else 236720a4e154SJeff Roberson zone->uz_ctlname = strdup(zone->uz_name, M_UMA); 236820a4e154SJeff Roberson for (c = zone->uz_ctlname; *c != '\0'; c++) 236920a4e154SJeff Roberson if (strchr("./\\ -", *c) != NULL) 237020a4e154SJeff Roberson *c = '_'; 237120a4e154SJeff Roberson 237220a4e154SJeff Roberson /* 237320a4e154SJeff Roberson * Basic parameters at the root. 237420a4e154SJeff Roberson */ 237520a4e154SJeff Roberson zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma), 237620a4e154SJeff Roberson OID_AUTO, zone->uz_ctlname, CTLFLAG_RD, NULL, ""); 237720a4e154SJeff Roberson oid = zone->uz_oid; 237820a4e154SJeff Roberson SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 237920a4e154SJeff Roberson "size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size"); 23806d204a6aSRyan Libby SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 23816d204a6aSRyan Libby "flags", CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_MPSAFE, 23826d204a6aSRyan Libby zone, 0, sysctl_handle_uma_zone_flags, "A", 238320a4e154SJeff Roberson "Allocator configuration flags"); 238420a4e154SJeff Roberson SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 238520a4e154SJeff Roberson "bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0, 238620a4e154SJeff Roberson "Desired per-cpu cache size"); 238720a4e154SJeff Roberson SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 238820a4e154SJeff Roberson "bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0, 238920a4e154SJeff Roberson "Maximum allowed per-cpu cache size"); 239020a4e154SJeff Roberson 239120a4e154SJeff Roberson /* 239220a4e154SJeff Roberson * keg if present. 239320a4e154SJeff Roberson */ 239454c5ae80SRyan Libby if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) 23958b987a77SJeff Roberson domains = vm_ndomains; 23968b987a77SJeff Roberson else 23978b987a77SJeff Roberson domains = 1; 239820a4e154SJeff Roberson oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, 239920a4e154SJeff Roberson "keg", CTLFLAG_RD, NULL, ""); 240020a4e154SJeff Roberson keg = zone->uz_keg; 24013b490537SJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) { 240220a4e154SJeff Roberson SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 240320a4e154SJeff Roberson "name", CTLFLAG_RD, keg->uk_name, "Keg name"); 240420a4e154SJeff Roberson SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 240520a4e154SJeff Roberson "rsize", CTLFLAG_RD, &keg->uk_rsize, 0, 240620a4e154SJeff Roberson "Real object size with alignment"); 240720a4e154SJeff Roberson SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 240820a4e154SJeff Roberson "ppera", CTLFLAG_RD, &keg->uk_ppera, 0, 240920a4e154SJeff Roberson "pages per-slab allocation"); 241020a4e154SJeff Roberson SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 241120a4e154SJeff Roberson "ipers", CTLFLAG_RD, &keg->uk_ipers, 0, 241220a4e154SJeff Roberson "items available per-slab"); 241320a4e154SJeff Roberson SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 241420a4e154SJeff Roberson "align", CTLFLAG_RD, &keg->uk_align, 0, 241520a4e154SJeff Roberson "item alignment mask"); 2416f7af5015SRyan Libby SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2417f7af5015SRyan Libby "efficiency", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE, 2418f7af5015SRyan Libby keg, 0, sysctl_handle_uma_slab_efficiency, "I", 2419f7af5015SRyan Libby "Slab utilization (100 - internal fragmentation %)"); 24208b987a77SJeff Roberson domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(oid), 24218b987a77SJeff Roberson OID_AUTO, "domain", CTLFLAG_RD, NULL, ""); 24228b987a77SJeff Roberson for (i = 0; i < domains; i++) { 24238b987a77SJeff Roberson dom = &keg->uk_domain[i]; 24248b987a77SJeff Roberson oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid), 24258b987a77SJeff Roberson OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD, 24268b987a77SJeff Roberson NULL, ""); 24278b987a77SJeff Roberson SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 24288b987a77SJeff Roberson "pages", CTLFLAG_RD, &dom->ud_pages, 0, 24298b987a77SJeff Roberson "Total pages currently allocated from VM"); 24308b987a77SJeff Roberson SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 24314ab3aee8SMark Johnston "free_items", CTLFLAG_RD, &dom->ud_free_items, 0, 24328b987a77SJeff Roberson "items free in the slab layer"); 24338b987a77SJeff Roberson } 243420a4e154SJeff Roberson } else 243520a4e154SJeff Roberson SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 243620a4e154SJeff Roberson "name", CTLFLAG_RD, nokeg, "Keg name"); 243720a4e154SJeff Roberson 243820a4e154SJeff Roberson /* 243920a4e154SJeff Roberson * Information about zone limits. 244020a4e154SJeff Roberson */ 244120a4e154SJeff Roberson oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, 244220a4e154SJeff Roberson "limit", CTLFLAG_RD, NULL, ""); 24434bd61e19SJeff Roberson SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 24444bd61e19SJeff Roberson "items", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE, 24454bd61e19SJeff Roberson zone, 0, sysctl_handle_uma_zone_items, "QU", 24464bd61e19SJeff Roberson "current number of allocated items if limit is set"); 244720a4e154SJeff Roberson SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 244820a4e154SJeff Roberson "max_items", CTLFLAG_RD, &zone->uz_max_items, 0, 244920a4e154SJeff Roberson "Maximum number of cached items"); 245020a4e154SJeff Roberson SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 245120a4e154SJeff Roberson "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0, 245220a4e154SJeff Roberson "Number of threads sleeping at limit"); 245320a4e154SJeff Roberson SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 245420a4e154SJeff Roberson "sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0, 245520a4e154SJeff Roberson "Total zone limit sleeps"); 24564bd61e19SJeff Roberson SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2457c6fd3e23SJeff Roberson "bucket_max", CTLFLAG_RD, &zone->uz_bucket_max, 0, 2458c6fd3e23SJeff Roberson "Maximum number of items in each domain's bucket cache"); 245920a4e154SJeff Roberson 246020a4e154SJeff Roberson /* 24618b987a77SJeff Roberson * Per-domain zone information. 246220a4e154SJeff Roberson */ 246320a4e154SJeff Roberson domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), 246420a4e154SJeff Roberson OID_AUTO, "domain", CTLFLAG_RD, NULL, ""); 246520a4e154SJeff Roberson for (i = 0; i < domains; i++) { 2466c6fd3e23SJeff Roberson zdom = ZDOM_GET(zone, i); 246720a4e154SJeff Roberson oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid), 246820a4e154SJeff Roberson OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD, NULL, ""); 246920a4e154SJeff Roberson SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 247020a4e154SJeff Roberson "nitems", CTLFLAG_RD, &zdom->uzd_nitems, 247120a4e154SJeff Roberson "number of items in this domain"); 247220a4e154SJeff Roberson SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 247320a4e154SJeff Roberson "imax", CTLFLAG_RD, &zdom->uzd_imax, 247420a4e154SJeff Roberson "maximum item count in this period"); 247520a4e154SJeff Roberson SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 247620a4e154SJeff Roberson "imin", CTLFLAG_RD, &zdom->uzd_imin, 247720a4e154SJeff Roberson "minimum item count in this period"); 247820a4e154SJeff Roberson SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 247920a4e154SJeff Roberson "wss", CTLFLAG_RD, &zdom->uzd_wss, 248020a4e154SJeff Roberson "Working set size"); 248120a4e154SJeff Roberson } 248220a4e154SJeff Roberson 248320a4e154SJeff Roberson /* 248420a4e154SJeff Roberson * General statistics. 248520a4e154SJeff Roberson */ 248620a4e154SJeff Roberson oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, 248720a4e154SJeff Roberson "stats", CTLFLAG_RD, NULL, ""); 248820a4e154SJeff Roberson SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 248920a4e154SJeff Roberson "current", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE, 249020a4e154SJeff Roberson zone, 1, sysctl_handle_uma_zone_cur, "I", 249120a4e154SJeff Roberson "Current number of allocated items"); 249220a4e154SJeff Roberson SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 249320a4e154SJeff Roberson "allocs", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE, 249420a4e154SJeff Roberson zone, 0, sysctl_handle_uma_zone_allocs, "QU", 249520a4e154SJeff Roberson "Total allocation calls"); 249620a4e154SJeff Roberson SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 249720a4e154SJeff Roberson "frees", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE, 249820a4e154SJeff Roberson zone, 0, sysctl_handle_uma_zone_frees, "QU", 249920a4e154SJeff Roberson "Total free calls"); 250020a4e154SJeff Roberson SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 250120a4e154SJeff Roberson "fails", CTLFLAG_RD, &zone->uz_fails, 250220a4e154SJeff Roberson "Number of allocation failures"); 2503c6fd3e23SJeff Roberson SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2504c6fd3e23SJeff Roberson "xdomain", CTLFLAG_RD, &zone->uz_xdomain, 250520a4e154SJeff Roberson "Free calls from the wrong domain"); 250620a4e154SJeff Roberson } 250720a4e154SJeff Roberson 250820a4e154SJeff Roberson struct uma_zone_count { 250920a4e154SJeff Roberson const char *name; 251020a4e154SJeff Roberson int count; 251120a4e154SJeff Roberson }; 251220a4e154SJeff Roberson 251320a4e154SJeff Roberson static void 251420a4e154SJeff Roberson zone_count(uma_zone_t zone, void *arg) 251520a4e154SJeff Roberson { 251620a4e154SJeff Roberson struct uma_zone_count *cnt; 251720a4e154SJeff Roberson 251820a4e154SJeff Roberson cnt = arg; 25193b490537SJeff Roberson /* 25203b490537SJeff Roberson * Some zones are rapidly created with identical names and 25213b490537SJeff Roberson * destroyed out of order. This can lead to gaps in the count. 25223b490537SJeff Roberson * Use one greater than the maximum observed for this name. 25233b490537SJeff Roberson */ 252420a4e154SJeff Roberson if (strcmp(zone->uz_name, cnt->name) == 0) 25253b490537SJeff Roberson cnt->count = MAX(cnt->count, 25263b490537SJeff Roberson zone->uz_namecnt + 1); 252720a4e154SJeff Roberson } 252820a4e154SJeff Roberson 2529cc7ce83aSJeff Roberson static void 2530cc7ce83aSJeff Roberson zone_update_caches(uma_zone_t zone) 2531cc7ce83aSJeff Roberson { 2532cc7ce83aSJeff Roberson int i; 2533cc7ce83aSJeff Roberson 2534cc7ce83aSJeff Roberson for (i = 0; i <= mp_maxid; i++) { 2535cc7ce83aSJeff Roberson cache_set_uz_size(&zone->uz_cpu[i], zone->uz_size); 2536cc7ce83aSJeff Roberson cache_set_uz_flags(&zone->uz_cpu[i], zone->uz_flags); 2537cc7ce83aSJeff Roberson } 2538cc7ce83aSJeff Roberson } 2539cc7ce83aSJeff Roberson 2540099a0e58SBosko Milekic /* 2541099a0e58SBosko Milekic * Zone header ctor. This initializes all fields, locks, etc. 2542099a0e58SBosko Milekic * 2543099a0e58SBosko Milekic * Arguments/Returns follow uma_ctor specifications 2544099a0e58SBosko Milekic * udata Actually uma_zctor_args 25458355f576SJeff Roberson */ 2546b23f72e9SBrian Feldman static int 2547b23f72e9SBrian Feldman zone_ctor(void *mem, int size, void *udata, int flags) 25488355f576SJeff Roberson { 254920a4e154SJeff Roberson struct uma_zone_count cnt; 25508355f576SJeff Roberson struct uma_zctor_args *arg = udata; 2551c6fd3e23SJeff Roberson uma_zone_domain_t zdom; 25528355f576SJeff Roberson uma_zone_t zone = mem; 2553099a0e58SBosko Milekic uma_zone_t z; 2554099a0e58SBosko Milekic uma_keg_t keg; 255508cfa56eSMark Johnston int i; 25568355f576SJeff Roberson 25578355f576SJeff Roberson bzero(zone, size); 25588355f576SJeff Roberson zone->uz_name = arg->name; 25598355f576SJeff Roberson zone->uz_ctor = arg->ctor; 25608355f576SJeff Roberson zone->uz_dtor = arg->dtor; 2561099a0e58SBosko Milekic zone->uz_init = NULL; 2562099a0e58SBosko Milekic zone->uz_fini = NULL; 2563bf965959SSean Bruno zone->uz_sleeps = 0; 256420a4e154SJeff Roberson zone->uz_bucket_size = 0; 256520a4e154SJeff Roberson zone->uz_bucket_size_min = 0; 256620a4e154SJeff Roberson zone->uz_bucket_size_max = BUCKET_MAX; 2567d4665eaaSJeff Roberson zone->uz_flags = (arg->flags & UMA_ZONE_SMR); 25682f891cd5SPawel Jakub Dawidek zone->uz_warning = NULL; 2569ab3185d1SJeff Roberson /* The domain structures follow the cpu structures. */ 2570c6fd3e23SJeff Roberson zone->uz_bucket_max = ULONG_MAX; 25712f891cd5SPawel Jakub Dawidek timevalclear(&zone->uz_ratecheck); 2572af526374SJeff Roberson 257320a4e154SJeff Roberson /* Count the number of duplicate names. */ 257420a4e154SJeff Roberson cnt.name = arg->name; 257520a4e154SJeff Roberson cnt.count = 0; 257620a4e154SJeff Roberson zone_foreach(zone_count, &cnt); 257720a4e154SJeff Roberson zone->uz_namecnt = cnt.count; 257891d947bfSJeff Roberson ZONE_CROSS_LOCK_INIT(zone); 25792efcc8cbSGleb Smirnoff 2580c6fd3e23SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 2581c6fd3e23SJeff Roberson zdom = ZDOM_GET(zone, i); 2582c6fd3e23SJeff Roberson ZDOM_LOCK_INIT(zone, zdom, (arg->flags & UMA_ZONE_MTXCLASS)); 2583c6fd3e23SJeff Roberson STAILQ_INIT(&zdom->uzd_buckets); 2584c6fd3e23SJeff Roberson } 258508cfa56eSMark Johnston 2586ca293436SRyan Libby #ifdef INVARIANTS 2587ca293436SRyan Libby if (arg->uminit == trash_init && arg->fini == trash_fini) 2588cc7ce83aSJeff Roberson zone->uz_flags |= UMA_ZFLAG_TRASH | UMA_ZFLAG_CTORDTOR; 2589ca293436SRyan Libby #endif 2590ca293436SRyan Libby 25910095a784SJeff Roberson /* 25920095a784SJeff Roberson * This is a pure cache zone, no kegs. 25930095a784SJeff Roberson */ 25940095a784SJeff Roberson if (arg->import) { 2595727c6918SJeff Roberson KASSERT((arg->flags & UMA_ZFLAG_CACHE) != 0, 2596727c6918SJeff Roberson ("zone_ctor: Import specified for non-cache zone.")); 25976fd34d6fSJeff Roberson zone->uz_flags = arg->flags; 2598af526374SJeff Roberson zone->uz_size = arg->size; 25990095a784SJeff Roberson zone->uz_import = arg->import; 26000095a784SJeff Roberson zone->uz_release = arg->release; 26010095a784SJeff Roberson zone->uz_arg = arg->arg; 2602c6fd3e23SJeff Roberson #ifdef NUMA 2603c6fd3e23SJeff Roberson /* 2604c6fd3e23SJeff Roberson * Cache zones are round-robin unless a policy is 2605c6fd3e23SJeff Roberson * specified because they may have incompatible 2606c6fd3e23SJeff Roberson * constraints. 2607c6fd3e23SJeff Roberson */ 2608c6fd3e23SJeff Roberson if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) == 0) 2609c6fd3e23SJeff Roberson zone->uz_flags |= UMA_ZONE_ROUNDROBIN; 2610c6fd3e23SJeff Roberson #endif 2611111fbcd5SBryan Venteicher rw_wlock(&uma_rwlock); 261203175483SAlexander Motin LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link); 2613111fbcd5SBryan Venteicher rw_wunlock(&uma_rwlock); 2614af526374SJeff Roberson goto out; 26150095a784SJeff Roberson } 26160095a784SJeff Roberson 26170095a784SJeff Roberson /* 26180095a784SJeff Roberson * Use the regular zone/keg/slab allocator. 26190095a784SJeff Roberson */ 2620b75c4efcSAndrew Turner zone->uz_import = zone_import; 2621b75c4efcSAndrew Turner zone->uz_release = zone_release; 26220095a784SJeff Roberson zone->uz_arg = zone; 2623bb15d1c7SGleb Smirnoff keg = arg->keg; 26240095a784SJeff Roberson 2625099a0e58SBosko Milekic if (arg->flags & UMA_ZONE_SECONDARY) { 262620a4e154SJeff Roberson KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0, 262720a4e154SJeff Roberson ("Secondary zone requested UMA_ZFLAG_INTERNAL")); 2628099a0e58SBosko Milekic KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); 26298355f576SJeff Roberson zone->uz_init = arg->uminit; 2630e221e841SJeff Roberson zone->uz_fini = arg->fini; 2631e20a199fSJeff Roberson zone->uz_flags |= UMA_ZONE_SECONDARY; 2632111fbcd5SBryan Venteicher rw_wlock(&uma_rwlock); 2633099a0e58SBosko Milekic ZONE_LOCK(zone); 2634099a0e58SBosko Milekic LIST_FOREACH(z, &keg->uk_zones, uz_link) { 2635099a0e58SBosko Milekic if (LIST_NEXT(z, uz_link) == NULL) { 2636099a0e58SBosko Milekic LIST_INSERT_AFTER(z, zone, uz_link); 2637099a0e58SBosko Milekic break; 2638099a0e58SBosko Milekic } 2639099a0e58SBosko Milekic } 2640099a0e58SBosko Milekic ZONE_UNLOCK(zone); 2641111fbcd5SBryan Venteicher rw_wunlock(&uma_rwlock); 2642e20a199fSJeff Roberson } else if (keg == NULL) { 2643e20a199fSJeff Roberson if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini, 2644e20a199fSJeff Roberson arg->align, arg->flags)) == NULL) 2645b23f72e9SBrian Feldman return (ENOMEM); 2646099a0e58SBosko Milekic } else { 2647099a0e58SBosko Milekic struct uma_kctor_args karg; 2648b23f72e9SBrian Feldman int error; 2649099a0e58SBosko Milekic 2650099a0e58SBosko Milekic /* We should only be here from uma_startup() */ 2651099a0e58SBosko Milekic karg.size = arg->size; 2652099a0e58SBosko Milekic karg.uminit = arg->uminit; 2653099a0e58SBosko Milekic karg.fini = arg->fini; 2654099a0e58SBosko Milekic karg.align = arg->align; 2655d4665eaaSJeff Roberson karg.flags = (arg->flags & ~UMA_ZONE_SMR); 2656099a0e58SBosko Milekic karg.zone = zone; 2657b23f72e9SBrian Feldman error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, 2658b23f72e9SBrian Feldman flags); 2659b23f72e9SBrian Feldman if (error) 2660b23f72e9SBrian Feldman return (error); 2661099a0e58SBosko Milekic } 26620095a784SJeff Roberson 266320a4e154SJeff Roberson /* Inherit properties from the keg. */ 2664bb15d1c7SGleb Smirnoff zone->uz_keg = keg; 2665e20a199fSJeff Roberson zone->uz_size = keg->uk_size; 2666e20a199fSJeff Roberson zone->uz_flags |= (keg->uk_flags & 2667e20a199fSJeff Roberson (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT)); 26688355f576SJeff Roberson 266920a4e154SJeff Roberson out: 2670860bb7a0SMark Johnston if (__predict_true(booted >= BOOT_RUNNING)) { 267120a4e154SJeff Roberson zone_alloc_counters(zone, NULL); 267220a4e154SJeff Roberson zone_alloc_sysctl(zone, NULL); 267320a4e154SJeff Roberson } else { 267420a4e154SJeff Roberson zone->uz_allocs = EARLY_COUNTER; 267520a4e154SJeff Roberson zone->uz_frees = EARLY_COUNTER; 267620a4e154SJeff Roberson zone->uz_fails = EARLY_COUNTER; 2677099a0e58SBosko Milekic } 26788355f576SJeff Roberson 2679d4665eaaSJeff Roberson /* Caller requests a private SMR context. */ 2680d4665eaaSJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) != 0) 2681226dd6dbSJeff Roberson zone->uz_smr = smr_create(zone->uz_name, 0, 0); 2682d4665eaaSJeff Roberson 26837e28037aSMark Johnston KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) != 26847e28037aSMark Johnston (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET), 26857e28037aSMark Johnston ("Invalid zone flag combination")); 268620a4e154SJeff Roberson if (arg->flags & UMA_ZFLAG_INTERNAL) 268720a4e154SJeff Roberson zone->uz_bucket_size_max = zone->uz_bucket_size = 0; 268820a4e154SJeff Roberson if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) 268920a4e154SJeff Roberson zone->uz_bucket_size = BUCKET_MAX; 269020a4e154SJeff Roberson else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) 269120a4e154SJeff Roberson zone->uz_bucket_size_max = zone->uz_bucket_size = BUCKET_MIN; 269220a4e154SJeff Roberson else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0) 269320a4e154SJeff Roberson zone->uz_bucket_size = 0; 26947e28037aSMark Johnston else 269520a4e154SJeff Roberson zone->uz_bucket_size = bucket_select(zone->uz_size); 269620a4e154SJeff Roberson zone->uz_bucket_size_min = zone->uz_bucket_size; 2697cc7ce83aSJeff Roberson if (zone->uz_dtor != NULL || zone->uz_ctor != NULL) 2698cc7ce83aSJeff Roberson zone->uz_flags |= UMA_ZFLAG_CTORDTOR; 2699cc7ce83aSJeff Roberson zone_update_caches(zone); 2700fc03d22bSJeff Roberson 2701b23f72e9SBrian Feldman return (0); 27028355f576SJeff Roberson } 27038355f576SJeff Roberson 27048355f576SJeff Roberson /* 2705099a0e58SBosko Milekic * Keg header dtor. This frees all data, destroys locks, frees the hash 2706099a0e58SBosko Milekic * table and removes the keg from the global list. 27079c2cd7e5SJeff Roberson * 27089c2cd7e5SJeff Roberson * Arguments/Returns follow uma_dtor specifications 27099c2cd7e5SJeff Roberson * udata unused 27109c2cd7e5SJeff Roberson */ 2711099a0e58SBosko Milekic static void 2712099a0e58SBosko Milekic keg_dtor(void *arg, int size, void *udata) 2713099a0e58SBosko Milekic { 2714099a0e58SBosko Milekic uma_keg_t keg; 27158b987a77SJeff Roberson uint32_t free, pages; 27168b987a77SJeff Roberson int i; 27179c2cd7e5SJeff Roberson 2718099a0e58SBosko Milekic keg = (uma_keg_t)arg; 27198b987a77SJeff Roberson free = pages = 0; 27208b987a77SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 27214ab3aee8SMark Johnston free += keg->uk_domain[i].ud_free_items; 27228b987a77SJeff Roberson pages += keg->uk_domain[i].ud_pages; 27238b987a77SJeff Roberson KEG_LOCK_FINI(keg, i); 2724099a0e58SBosko Milekic } 27257e240677SRyan Libby if (pages != 0) 27268b987a77SJeff Roberson printf("Freed UMA keg (%s) was not empty (%u items). " 27278b987a77SJeff Roberson " Lost %u pages of memory.\n", 27288b987a77SJeff Roberson keg->uk_name ? keg->uk_name : "", 27297e240677SRyan Libby pages / keg->uk_ppera * keg->uk_ipers - free, pages); 2730099a0e58SBosko Milekic 2731099a0e58SBosko Milekic hash_free(&keg->uk_hash); 2732099a0e58SBosko Milekic } 2733099a0e58SBosko Milekic 2734099a0e58SBosko Milekic /* 2735099a0e58SBosko Milekic * Zone header dtor. 2736099a0e58SBosko Milekic * 2737099a0e58SBosko Milekic * Arguments/Returns follow uma_dtor specifications 2738099a0e58SBosko Milekic * udata unused 2739099a0e58SBosko Milekic */ 27409c2cd7e5SJeff Roberson static void 27419c2cd7e5SJeff Roberson zone_dtor(void *arg, int size, void *udata) 27429c2cd7e5SJeff Roberson { 27439c2cd7e5SJeff Roberson uma_zone_t zone; 2744099a0e58SBosko Milekic uma_keg_t keg; 2745c6fd3e23SJeff Roberson int i; 27469c2cd7e5SJeff Roberson 27479c2cd7e5SJeff Roberson zone = (uma_zone_t)arg; 27489643769aSJeff Roberson 274920a4e154SJeff Roberson sysctl_remove_oid(zone->uz_oid, 1, 1); 275020a4e154SJeff Roberson 2751e20a199fSJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) 27529643769aSJeff Roberson cache_drain(zone); 2753099a0e58SBosko Milekic 2754111fbcd5SBryan Venteicher rw_wlock(&uma_rwlock); 2755099a0e58SBosko Milekic LIST_REMOVE(zone, uz_link); 2756111fbcd5SBryan Venteicher rw_wunlock(&uma_rwlock); 275708cfa56eSMark Johnston zone_reclaim(zone, M_WAITOK, true); 2758c6fd3e23SJeff Roberson 2759e20a199fSJeff Roberson /* 2760323ad386STycho Nightingale * We only destroy kegs from non secondary/non cache zones. 2761e20a199fSJeff Roberson */ 2762323ad386STycho Nightingale if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) { 2763323ad386STycho Nightingale keg = zone->uz_keg; 2764111fbcd5SBryan Venteicher rw_wlock(&uma_rwlock); 2765099a0e58SBosko Milekic LIST_REMOVE(keg, uk_link); 2766111fbcd5SBryan Venteicher rw_wunlock(&uma_rwlock); 27670095a784SJeff Roberson zone_free_item(kegs, keg, NULL, SKIP_NONE); 27689c2cd7e5SJeff Roberson } 27692efcc8cbSGleb Smirnoff counter_u64_free(zone->uz_allocs); 27702efcc8cbSGleb Smirnoff counter_u64_free(zone->uz_frees); 27712efcc8cbSGleb Smirnoff counter_u64_free(zone->uz_fails); 2772c6fd3e23SJeff Roberson counter_u64_free(zone->uz_xdomain); 277320a4e154SJeff Roberson free(zone->uz_ctlname, M_UMA); 2774c6fd3e23SJeff Roberson for (i = 0; i < vm_ndomains; i++) 2775c6fd3e23SJeff Roberson ZDOM_LOCK_FINI(ZDOM_GET(zone, i)); 277691d947bfSJeff Roberson ZONE_CROSS_LOCK_FINI(zone); 2777099a0e58SBosko Milekic } 2778099a0e58SBosko Milekic 2779a81c400eSJeff Roberson static void 2780a81c400eSJeff Roberson zone_foreach_unlocked(void (*zfunc)(uma_zone_t, void *arg), void *arg) 2781a81c400eSJeff Roberson { 2782a81c400eSJeff Roberson uma_keg_t keg; 2783a81c400eSJeff Roberson uma_zone_t zone; 2784a81c400eSJeff Roberson 2785a81c400eSJeff Roberson LIST_FOREACH(keg, &uma_kegs, uk_link) { 2786a81c400eSJeff Roberson LIST_FOREACH(zone, &keg->uk_zones, uz_link) 2787a81c400eSJeff Roberson zfunc(zone, arg); 2788a81c400eSJeff Roberson } 2789a81c400eSJeff Roberson LIST_FOREACH(zone, &uma_cachezones, uz_link) 2790a81c400eSJeff Roberson zfunc(zone, arg); 2791a81c400eSJeff Roberson } 2792a81c400eSJeff Roberson 27939c2cd7e5SJeff Roberson /* 27948355f576SJeff Roberson * Traverses every zone in the system and calls a callback 27958355f576SJeff Roberson * 27968355f576SJeff Roberson * Arguments: 27978355f576SJeff Roberson * zfunc A pointer to a function which accepts a zone 27988355f576SJeff Roberson * as an argument. 27998355f576SJeff Roberson * 28008355f576SJeff Roberson * Returns: 28018355f576SJeff Roberson * Nothing 28028355f576SJeff Roberson */ 28038355f576SJeff Roberson static void 280420a4e154SJeff Roberson zone_foreach(void (*zfunc)(uma_zone_t, void *arg), void *arg) 28058355f576SJeff Roberson { 28068355f576SJeff Roberson 2807111fbcd5SBryan Venteicher rw_rlock(&uma_rwlock); 2808a81c400eSJeff Roberson zone_foreach_unlocked(zfunc, arg); 2809111fbcd5SBryan Venteicher rw_runlock(&uma_rwlock); 28108355f576SJeff Roberson } 28118355f576SJeff Roberson 2812f4bef67cSGleb Smirnoff /* 2813a81c400eSJeff Roberson * Initialize the kernel memory allocator. This is done after pages can be 2814a81c400eSJeff Roberson * allocated but before general KVA is available. 2815f4bef67cSGleb Smirnoff */ 2816a81c400eSJeff Roberson void 2817a81c400eSJeff Roberson uma_startup1(vm_offset_t virtual_avail) 2818f4bef67cSGleb Smirnoff { 2819a81c400eSJeff Roberson struct uma_zctor_args args; 2820a81c400eSJeff Roberson size_t ksize, zsize, size; 2821a81c400eSJeff Roberson uma_keg_t masterkeg; 2822a81c400eSJeff Roberson uintptr_t m; 2823a81c400eSJeff Roberson uint8_t pflag; 2824a81c400eSJeff Roberson 2825a81c400eSJeff Roberson bootstart = bootmem = virtual_avail; 2826a81c400eSJeff Roberson 2827a81c400eSJeff Roberson rw_init(&uma_rwlock, "UMA lock"); 2828a81c400eSJeff Roberson sx_init(&uma_reclaim_lock, "umareclaim"); 2829f4bef67cSGleb Smirnoff 2830f4bef67cSGleb Smirnoff ksize = sizeof(struct uma_keg) + 2831f4bef67cSGleb Smirnoff (sizeof(struct uma_domain) * vm_ndomains); 283279c9f942SJeff Roberson ksize = roundup(ksize, UMA_SUPER_ALIGN); 2833f4bef67cSGleb Smirnoff zsize = sizeof(struct uma_zone) + 2834f4bef67cSGleb Smirnoff (sizeof(struct uma_cache) * (mp_maxid + 1)) + 2835f4bef67cSGleb Smirnoff (sizeof(struct uma_zone_domain) * vm_ndomains); 283679c9f942SJeff Roberson zsize = roundup(zsize, UMA_SUPER_ALIGN); 2837f4bef67cSGleb Smirnoff 2838a81c400eSJeff Roberson /* Allocate the zone of zones, zone of kegs, and zone of zones keg. */ 2839a81c400eSJeff Roberson size = (zsize * 2) + ksize; 2840a81c400eSJeff Roberson m = (uintptr_t)startup_alloc(NULL, size, 0, &pflag, M_NOWAIT | M_ZERO); 2841ab3185d1SJeff Roberson zones = (uma_zone_t)m; 284279c9f942SJeff Roberson m += zsize; 2843ab3185d1SJeff Roberson kegs = (uma_zone_t)m; 284479c9f942SJeff Roberson m += zsize; 2845ab3185d1SJeff Roberson masterkeg = (uma_keg_t)m; 2846ab3185d1SJeff Roberson 2847099a0e58SBosko Milekic /* "manually" create the initial zone */ 28480095a784SJeff Roberson memset(&args, 0, sizeof(args)); 2849099a0e58SBosko Milekic args.name = "UMA Kegs"; 2850ab3185d1SJeff Roberson args.size = ksize; 2851099a0e58SBosko Milekic args.ctor = keg_ctor; 2852099a0e58SBosko Milekic args.dtor = keg_dtor; 28538355f576SJeff Roberson args.uminit = zero_init; 28548355f576SJeff Roberson args.fini = NULL; 2855ab3185d1SJeff Roberson args.keg = masterkeg; 285679c9f942SJeff Roberson args.align = UMA_SUPER_ALIGN - 1; 2857b60f5b79SJeff Roberson args.flags = UMA_ZFLAG_INTERNAL; 2858ab3185d1SJeff Roberson zone_ctor(kegs, zsize, &args, M_WAITOK); 28598355f576SJeff Roberson 2860099a0e58SBosko Milekic args.name = "UMA Zones"; 2861f4bef67cSGleb Smirnoff args.size = zsize; 2862099a0e58SBosko Milekic args.ctor = zone_ctor; 2863099a0e58SBosko Milekic args.dtor = zone_dtor; 2864099a0e58SBosko Milekic args.uminit = zero_init; 2865099a0e58SBosko Milekic args.fini = NULL; 2866099a0e58SBosko Milekic args.keg = NULL; 286779c9f942SJeff Roberson args.align = UMA_SUPER_ALIGN - 1; 2868099a0e58SBosko Milekic args.flags = UMA_ZFLAG_INTERNAL; 2869ab3185d1SJeff Roberson zone_ctor(zones, zsize, &args, M_WAITOK); 2870099a0e58SBosko Milekic 28719b8db4d0SRyan Libby /* Now make zones for slab headers */ 28729b8db4d0SRyan Libby slabzones[0] = uma_zcreate("UMA Slabs 0", SLABZONE0_SIZE, 28739b8db4d0SRyan Libby NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 28749b8db4d0SRyan Libby slabzones[1] = uma_zcreate("UMA Slabs 1", SLABZONE1_SIZE, 28751e0701e1SJeff Roberson NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 28768355f576SJeff Roberson 28778355f576SJeff Roberson hashzone = uma_zcreate("UMA Hash", 28788355f576SJeff Roberson sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 28791e0701e1SJeff Roberson NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 28808355f576SJeff Roberson 2881a81c400eSJeff Roberson bucket_init(); 2882d4665eaaSJeff Roberson smr_init(); 28838355f576SJeff Roberson } 28848355f576SJeff Roberson 2885a81c400eSJeff Roberson #ifndef UMA_MD_SMALL_ALLOC 2886a81c400eSJeff Roberson extern void vm_radix_reserve_kva(void); 2887f4bef67cSGleb Smirnoff #endif 2888f4bef67cSGleb Smirnoff 2889a81c400eSJeff Roberson /* 2890a81c400eSJeff Roberson * Advertise the availability of normal kva allocations and switch to 2891a81c400eSJeff Roberson * the default back-end allocator. Marks the KVA we consumed on startup 2892a81c400eSJeff Roberson * as used in the map. 2893a81c400eSJeff Roberson */ 28948355f576SJeff Roberson void 289599571dc3SJeff Roberson uma_startup2(void) 28968355f576SJeff Roberson { 2897f4bef67cSGleb Smirnoff 2898530cc6a2SJeff Roberson if (bootstart != bootmem) { 2899a81c400eSJeff Roberson vm_map_lock(kernel_map); 2900a81c400eSJeff Roberson (void)vm_map_insert(kernel_map, NULL, 0, bootstart, bootmem, 2901a81c400eSJeff Roberson VM_PROT_RW, VM_PROT_RW, MAP_NOFAULT); 2902a81c400eSJeff Roberson vm_map_unlock(kernel_map); 2903a81c400eSJeff Roberson } 2904a81c400eSJeff Roberson 2905a81c400eSJeff Roberson #ifndef UMA_MD_SMALL_ALLOC 2906a81c400eSJeff Roberson /* Set up radix zone to use noobj_alloc. */ 2907a81c400eSJeff Roberson vm_radix_reserve_kva(); 2908f7d35785SGleb Smirnoff #endif 2909a81c400eSJeff Roberson 2910a81c400eSJeff Roberson booted = BOOT_KVA; 2911a81c400eSJeff Roberson zone_foreach_unlocked(zone_kva_available, NULL); 2912f4bef67cSGleb Smirnoff bucket_enable(); 29138355f576SJeff Roberson } 29148355f576SJeff Roberson 2915a81c400eSJeff Roberson /* 2916a81c400eSJeff Roberson * Finish our initialization steps. 2917a81c400eSJeff Roberson */ 29188355f576SJeff Roberson static void 29198355f576SJeff Roberson uma_startup3(void) 29208355f576SJeff Roberson { 29211431a748SGleb Smirnoff 2922c5deaf04SGleb Smirnoff #ifdef INVARIANTS 2923c5deaf04SGleb Smirnoff TUNABLE_INT_FETCH("vm.debug.divisor", &dbg_divisor); 2924c5deaf04SGleb Smirnoff uma_dbg_cnt = counter_u64_alloc(M_WAITOK); 2925c5deaf04SGleb Smirnoff uma_skip_cnt = counter_u64_alloc(M_WAITOK); 2926c5deaf04SGleb Smirnoff #endif 2927a81c400eSJeff Roberson zone_foreach_unlocked(zone_alloc_counters, NULL); 2928a81c400eSJeff Roberson zone_foreach_unlocked(zone_alloc_sysctl, NULL); 2929fd90e2edSJung-uk Kim callout_init(&uma_callout, 1); 29309643769aSJeff Roberson callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 2931c5deaf04SGleb Smirnoff booted = BOOT_RUNNING; 2932860bb7a0SMark Johnston 2933860bb7a0SMark Johnston EVENTHANDLER_REGISTER(shutdown_post_sync, uma_shutdown, NULL, 2934860bb7a0SMark Johnston EVENTHANDLER_PRI_FIRST); 2935860bb7a0SMark Johnston } 2936860bb7a0SMark Johnston 2937860bb7a0SMark Johnston static void 2938860bb7a0SMark Johnston uma_shutdown(void) 2939860bb7a0SMark Johnston { 2940860bb7a0SMark Johnston 2941860bb7a0SMark Johnston booted = BOOT_SHUTDOWN; 29428355f576SJeff Roberson } 29438355f576SJeff Roberson 2944e20a199fSJeff Roberson static uma_keg_t 2945099a0e58SBosko Milekic uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, 294685dcf349SGleb Smirnoff int align, uint32_t flags) 2947099a0e58SBosko Milekic { 2948099a0e58SBosko Milekic struct uma_kctor_args args; 2949099a0e58SBosko Milekic 2950099a0e58SBosko Milekic args.size = size; 2951099a0e58SBosko Milekic args.uminit = uminit; 2952099a0e58SBosko Milekic args.fini = fini; 29531e319f6dSRobert Watson args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align; 2954099a0e58SBosko Milekic args.flags = flags; 2955099a0e58SBosko Milekic args.zone = zone; 2956ab3185d1SJeff Roberson return (zone_alloc_item(kegs, &args, UMA_ANYDOMAIN, M_WAITOK)); 2957099a0e58SBosko Milekic } 2958099a0e58SBosko Milekic 2959f4bef67cSGleb Smirnoff /* Public functions */ 29608355f576SJeff Roberson /* See uma.h */ 29611e319f6dSRobert Watson void 29621e319f6dSRobert Watson uma_set_align(int align) 29631e319f6dSRobert Watson { 29641e319f6dSRobert Watson 29651e319f6dSRobert Watson if (align != UMA_ALIGN_CACHE) 29661e319f6dSRobert Watson uma_align_cache = align; 29671e319f6dSRobert Watson } 29681e319f6dSRobert Watson 29691e319f6dSRobert Watson /* See uma.h */ 29708355f576SJeff Roberson uma_zone_t 2971bb196eb4SMatthew D Fleming uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, 297285dcf349SGleb Smirnoff uma_init uminit, uma_fini fini, int align, uint32_t flags) 29738355f576SJeff Roberson 29748355f576SJeff Roberson { 29758355f576SJeff Roberson struct uma_zctor_args args; 297695c4bf75SKonstantin Belousov uma_zone_t res; 29778355f576SJeff Roberson 2978a5a35578SJohn Baldwin KASSERT(powerof2(align + 1), ("invalid zone alignment %d for \"%s\"", 2979a5a35578SJohn Baldwin align, name)); 2980a5a35578SJohn Baldwin 29818355f576SJeff Roberson /* This stuff is essential for the zone ctor */ 29820095a784SJeff Roberson memset(&args, 0, sizeof(args)); 29838355f576SJeff Roberson args.name = name; 29848355f576SJeff Roberson args.size = size; 29858355f576SJeff Roberson args.ctor = ctor; 29868355f576SJeff Roberson args.dtor = dtor; 29878355f576SJeff Roberson args.uminit = uminit; 29888355f576SJeff Roberson args.fini = fini; 2989afc6dc36SJohn-Mark Gurney #ifdef INVARIANTS 2990afc6dc36SJohn-Mark Gurney /* 2991ca293436SRyan Libby * Inject procedures which check for memory use after free if we are 2992ca293436SRyan Libby * allowed to scramble the memory while it is not allocated. This 2993ca293436SRyan Libby * requires that: UMA is actually able to access the memory, no init 2994ca293436SRyan Libby * or fini procedures, no dependency on the initial value of the 2995ca293436SRyan Libby * memory, and no (legitimate) use of the memory after free. Note, 2996ca293436SRyan Libby * the ctor and dtor do not need to be empty. 2997afc6dc36SJohn-Mark Gurney */ 299854c5ae80SRyan Libby if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOTOUCH | 299954c5ae80SRyan Libby UMA_ZONE_NOFREE))) && uminit == NULL && fini == NULL) { 3000afc6dc36SJohn-Mark Gurney args.uminit = trash_init; 3001afc6dc36SJohn-Mark Gurney args.fini = trash_fini; 3002afc6dc36SJohn-Mark Gurney } 3003afc6dc36SJohn-Mark Gurney #endif 30048355f576SJeff Roberson args.align = align; 30058355f576SJeff Roberson args.flags = flags; 3006099a0e58SBosko Milekic args.keg = NULL; 3007099a0e58SBosko Milekic 300808cfa56eSMark Johnston sx_slock(&uma_reclaim_lock); 3009ab3185d1SJeff Roberson res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); 301008cfa56eSMark Johnston sx_sunlock(&uma_reclaim_lock); 3011a81c400eSJeff Roberson 301295c4bf75SKonstantin Belousov return (res); 3013099a0e58SBosko Milekic } 3014099a0e58SBosko Milekic 3015099a0e58SBosko Milekic /* See uma.h */ 3016099a0e58SBosko Milekic uma_zone_t 30170464f16eSMark Johnston uma_zsecond_create(const char *name, uma_ctor ctor, uma_dtor dtor, 3018099a0e58SBosko Milekic uma_init zinit, uma_fini zfini, uma_zone_t master) 3019099a0e58SBosko Milekic { 3020099a0e58SBosko Milekic struct uma_zctor_args args; 3021e20a199fSJeff Roberson uma_keg_t keg; 302295c4bf75SKonstantin Belousov uma_zone_t res; 3023099a0e58SBosko Milekic 3024bb15d1c7SGleb Smirnoff keg = master->uz_keg; 30250095a784SJeff Roberson memset(&args, 0, sizeof(args)); 3026099a0e58SBosko Milekic args.name = name; 3027e20a199fSJeff Roberson args.size = keg->uk_size; 3028099a0e58SBosko Milekic args.ctor = ctor; 3029099a0e58SBosko Milekic args.dtor = dtor; 3030099a0e58SBosko Milekic args.uminit = zinit; 3031099a0e58SBosko Milekic args.fini = zfini; 3032e20a199fSJeff Roberson args.align = keg->uk_align; 3033e20a199fSJeff Roberson args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; 3034e20a199fSJeff Roberson args.keg = keg; 30358355f576SJeff Roberson 303608cfa56eSMark Johnston sx_slock(&uma_reclaim_lock); 3037ab3185d1SJeff Roberson res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); 303808cfa56eSMark Johnston sx_sunlock(&uma_reclaim_lock); 3039a81c400eSJeff Roberson 304095c4bf75SKonstantin Belousov return (res); 30418355f576SJeff Roberson } 30428355f576SJeff Roberson 30430095a784SJeff Roberson /* See uma.h */ 30440095a784SJeff Roberson uma_zone_t 30450464f16eSMark Johnston uma_zcache_create(const char *name, int size, uma_ctor ctor, uma_dtor dtor, 30460464f16eSMark Johnston uma_init zinit, uma_fini zfini, uma_import zimport, uma_release zrelease, 30470464f16eSMark Johnston void *arg, int flags) 30480095a784SJeff Roberson { 30490095a784SJeff Roberson struct uma_zctor_args args; 30500095a784SJeff Roberson 30510095a784SJeff Roberson memset(&args, 0, sizeof(args)); 30520095a784SJeff Roberson args.name = name; 3053af526374SJeff Roberson args.size = size; 30540095a784SJeff Roberson args.ctor = ctor; 30550095a784SJeff Roberson args.dtor = dtor; 30560095a784SJeff Roberson args.uminit = zinit; 30570095a784SJeff Roberson args.fini = zfini; 30580095a784SJeff Roberson args.import = zimport; 30590095a784SJeff Roberson args.release = zrelease; 30600095a784SJeff Roberson args.arg = arg; 30610095a784SJeff Roberson args.align = 0; 3062bb15d1c7SGleb Smirnoff args.flags = flags | UMA_ZFLAG_CACHE; 30630095a784SJeff Roberson 3064ab3185d1SJeff Roberson return (zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK)); 30650095a784SJeff Roberson } 30660095a784SJeff Roberson 30678355f576SJeff Roberson /* See uma.h */ 30689c2cd7e5SJeff Roberson void 30699c2cd7e5SJeff Roberson uma_zdestroy(uma_zone_t zone) 30709c2cd7e5SJeff Roberson { 3071f4ff923bSRobert Watson 3072860bb7a0SMark Johnston /* 3073860bb7a0SMark Johnston * Large slabs are expensive to reclaim, so don't bother doing 3074860bb7a0SMark Johnston * unnecessary work if we're shutting down. 3075860bb7a0SMark Johnston */ 3076860bb7a0SMark Johnston if (booted == BOOT_SHUTDOWN && 3077860bb7a0SMark Johnston zone->uz_fini == NULL && zone->uz_release == zone_release) 3078860bb7a0SMark Johnston return; 307908cfa56eSMark Johnston sx_slock(&uma_reclaim_lock); 30800095a784SJeff Roberson zone_free_item(zones, zone, NULL, SKIP_NONE); 308108cfa56eSMark Johnston sx_sunlock(&uma_reclaim_lock); 30829c2cd7e5SJeff Roberson } 30839c2cd7e5SJeff Roberson 30848d6fbbb8SJeff Roberson void 30858d6fbbb8SJeff Roberson uma_zwait(uma_zone_t zone) 30868d6fbbb8SJeff Roberson { 30878d6fbbb8SJeff Roberson 308870260874SJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) != 0) 308970260874SJeff Roberson uma_zfree_smr(zone, uma_zalloc_smr(zone, M_WAITOK)); 309070260874SJeff Roberson else if ((zone->uz_flags & UMA_ZONE_PCPU) != 0) 309170260874SJeff Roberson uma_zfree_pcpu(zone, uma_zalloc_pcpu(zone, M_WAITOK)); 309270260874SJeff Roberson else 309370260874SJeff Roberson uma_zfree(zone, uma_zalloc(zone, M_WAITOK)); 30948d6fbbb8SJeff Roberson } 30958d6fbbb8SJeff Roberson 30964e180881SMateusz Guzik void * 30974e180881SMateusz Guzik uma_zalloc_pcpu_arg(uma_zone_t zone, void *udata, int flags) 30984e180881SMateusz Guzik { 30993acb6572SMateusz Guzik void *item, *pcpu_item; 3100b4799947SRuslan Bukin #ifdef SMP 31014e180881SMateusz Guzik int i; 31024e180881SMateusz Guzik 31034e180881SMateusz Guzik MPASS(zone->uz_flags & UMA_ZONE_PCPU); 3104b4799947SRuslan Bukin #endif 31054e180881SMateusz Guzik item = uma_zalloc_arg(zone, udata, flags & ~M_ZERO); 31063acb6572SMateusz Guzik if (item == NULL) 31073acb6572SMateusz Guzik return (NULL); 31083acb6572SMateusz Guzik pcpu_item = zpcpu_base_to_offset(item); 31093acb6572SMateusz Guzik if (flags & M_ZERO) { 3110b4799947SRuslan Bukin #ifdef SMP 3111013072f0SMark Johnston for (i = 0; i <= mp_maxid; i++) 31123acb6572SMateusz Guzik bzero(zpcpu_get_cpu(pcpu_item, i), zone->uz_size); 3113b4799947SRuslan Bukin #else 3114b4799947SRuslan Bukin bzero(item, zone->uz_size); 3115b4799947SRuslan Bukin #endif 31164e180881SMateusz Guzik } 31173acb6572SMateusz Guzik return (pcpu_item); 31184e180881SMateusz Guzik } 31194e180881SMateusz Guzik 31204e180881SMateusz Guzik /* 31214e180881SMateusz Guzik * A stub while both regular and pcpu cases are identical. 31224e180881SMateusz Guzik */ 31234e180881SMateusz Guzik void 31243acb6572SMateusz Guzik uma_zfree_pcpu_arg(uma_zone_t zone, void *pcpu_item, void *udata) 31254e180881SMateusz Guzik { 31263acb6572SMateusz Guzik void *item; 31274e180881SMateusz Guzik 3128c5b7751fSIan Lepore #ifdef SMP 31294e180881SMateusz Guzik MPASS(zone->uz_flags & UMA_ZONE_PCPU); 3130c5b7751fSIan Lepore #endif 31313acb6572SMateusz Guzik item = zpcpu_offset_to_base(pcpu_item); 31324e180881SMateusz Guzik uma_zfree_arg(zone, item, udata); 31334e180881SMateusz Guzik } 31344e180881SMateusz Guzik 3135d4665eaaSJeff Roberson static inline void * 3136d4665eaaSJeff Roberson item_ctor(uma_zone_t zone, int uz_flags, int size, void *udata, int flags, 3137d4665eaaSJeff Roberson void *item) 3138beb8beefSJeff Roberson { 3139beb8beefSJeff Roberson #ifdef INVARIANTS 3140ca293436SRyan Libby bool skipdbg; 3141beb8beefSJeff Roberson 3142beb8beefSJeff Roberson skipdbg = uma_dbg_zskip(zone, item); 3143ca293436SRyan Libby if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && 3144ca293436SRyan Libby zone->uz_ctor != trash_ctor) 3145cc7ce83aSJeff Roberson trash_ctor(item, size, udata, flags); 3146beb8beefSJeff Roberson #endif 3147d4665eaaSJeff Roberson /* Check flags before loading ctor pointer. */ 3148d4665eaaSJeff Roberson if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0) && 3149d4665eaaSJeff Roberson __predict_false(zone->uz_ctor != NULL) && 3150cc7ce83aSJeff Roberson zone->uz_ctor(item, size, udata, flags) != 0) { 3151beb8beefSJeff Roberson counter_u64_add(zone->uz_fails, 1); 3152beb8beefSJeff Roberson zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); 3153beb8beefSJeff Roberson return (NULL); 3154beb8beefSJeff Roberson } 3155beb8beefSJeff Roberson #ifdef INVARIANTS 3156beb8beefSJeff Roberson if (!skipdbg) 3157beb8beefSJeff Roberson uma_dbg_alloc(zone, NULL, item); 3158beb8beefSJeff Roberson #endif 31596d88d784SJeff Roberson if (__predict_false(flags & M_ZERO)) 31606d88d784SJeff Roberson return (memset(item, 0, size)); 3161beb8beefSJeff Roberson 3162beb8beefSJeff Roberson return (item); 3163beb8beefSJeff Roberson } 3164beb8beefSJeff Roberson 3165ca293436SRyan Libby static inline void 3166cc7ce83aSJeff Roberson item_dtor(uma_zone_t zone, void *item, int size, void *udata, 3167cc7ce83aSJeff Roberson enum zfreeskip skip) 3168ca293436SRyan Libby { 3169ca293436SRyan Libby #ifdef INVARIANTS 3170ca293436SRyan Libby bool skipdbg; 3171ca293436SRyan Libby 3172ca293436SRyan Libby skipdbg = uma_dbg_zskip(zone, item); 3173ca293436SRyan Libby if (skip == SKIP_NONE && !skipdbg) { 3174ca293436SRyan Libby if ((zone->uz_flags & UMA_ZONE_MALLOC) != 0) 3175ca293436SRyan Libby uma_dbg_free(zone, udata, item); 3176ca293436SRyan Libby else 3177ca293436SRyan Libby uma_dbg_free(zone, NULL, item); 3178ca293436SRyan Libby } 3179ca293436SRyan Libby #endif 3180cc7ce83aSJeff Roberson if (__predict_true(skip < SKIP_DTOR)) { 3181ca293436SRyan Libby if (zone->uz_dtor != NULL) 3182cc7ce83aSJeff Roberson zone->uz_dtor(item, size, udata); 3183ca293436SRyan Libby #ifdef INVARIANTS 3184ca293436SRyan Libby if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && 3185ca293436SRyan Libby zone->uz_dtor != trash_dtor) 3186cc7ce83aSJeff Roberson trash_dtor(item, size, udata); 3187ca293436SRyan Libby #endif 3188ca293436SRyan Libby } 3189ca293436SRyan Libby } 3190ca293436SRyan Libby 3191d4665eaaSJeff Roberson #if defined(INVARIANTS) || defined(DEBUG_MEMGUARD) || defined(WITNESS) 3192d4665eaaSJeff Roberson #define UMA_ZALLOC_DEBUG 3193d4665eaaSJeff Roberson static int 3194d4665eaaSJeff Roberson uma_zalloc_debug(uma_zone_t zone, void **itemp, void *udata, int flags) 3195d4665eaaSJeff Roberson { 3196d4665eaaSJeff Roberson int error; 3197d4665eaaSJeff Roberson 3198d4665eaaSJeff Roberson error = 0; 3199d4665eaaSJeff Roberson #ifdef WITNESS 3200d4665eaaSJeff Roberson if (flags & M_WAITOK) { 3201d4665eaaSJeff Roberson WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 3202d4665eaaSJeff Roberson "uma_zalloc_debug: zone \"%s\"", zone->uz_name); 3203d4665eaaSJeff Roberson } 3204d4665eaaSJeff Roberson #endif 3205d4665eaaSJeff Roberson 3206d4665eaaSJeff Roberson #ifdef INVARIANTS 3207d4665eaaSJeff Roberson KASSERT((flags & M_EXEC) == 0, 3208d4665eaaSJeff Roberson ("uma_zalloc_debug: called with M_EXEC")); 3209d4665eaaSJeff Roberson KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), 3210d4665eaaSJeff Roberson ("uma_zalloc_debug: called within spinlock or critical section")); 3211d4665eaaSJeff Roberson KASSERT((zone->uz_flags & UMA_ZONE_PCPU) == 0 || (flags & M_ZERO) == 0, 3212d4665eaaSJeff Roberson ("uma_zalloc_debug: allocating from a pcpu zone with M_ZERO")); 3213d4665eaaSJeff Roberson #endif 3214d4665eaaSJeff Roberson 3215d4665eaaSJeff Roberson #ifdef DEBUG_MEMGUARD 32169e47b341SJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) == 0 && memguard_cmp_zone(zone)) { 3217d4665eaaSJeff Roberson void *item; 3218d4665eaaSJeff Roberson item = memguard_alloc(zone->uz_size, flags); 3219d4665eaaSJeff Roberson if (item != NULL) { 3220d4665eaaSJeff Roberson error = EJUSTRETURN; 3221d4665eaaSJeff Roberson if (zone->uz_init != NULL && 3222d4665eaaSJeff Roberson zone->uz_init(item, zone->uz_size, flags) != 0) { 3223d4665eaaSJeff Roberson *itemp = NULL; 3224d4665eaaSJeff Roberson return (error); 3225d4665eaaSJeff Roberson } 3226d4665eaaSJeff Roberson if (zone->uz_ctor != NULL && 3227d4665eaaSJeff Roberson zone->uz_ctor(item, zone->uz_size, udata, 3228d4665eaaSJeff Roberson flags) != 0) { 3229d4665eaaSJeff Roberson counter_u64_add(zone->uz_fails, 1); 3230d4665eaaSJeff Roberson zone->uz_fini(item, zone->uz_size); 3231d4665eaaSJeff Roberson *itemp = NULL; 3232d4665eaaSJeff Roberson return (error); 3233d4665eaaSJeff Roberson } 3234d4665eaaSJeff Roberson *itemp = item; 3235d4665eaaSJeff Roberson return (error); 3236d4665eaaSJeff Roberson } 3237d4665eaaSJeff Roberson /* This is unfortunate but should not be fatal. */ 3238d4665eaaSJeff Roberson } 3239d4665eaaSJeff Roberson #endif 3240d4665eaaSJeff Roberson return (error); 3241d4665eaaSJeff Roberson } 3242d4665eaaSJeff Roberson 3243d4665eaaSJeff Roberson static int 3244d4665eaaSJeff Roberson uma_zfree_debug(uma_zone_t zone, void *item, void *udata) 3245d4665eaaSJeff Roberson { 3246d4665eaaSJeff Roberson KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), 3247d4665eaaSJeff Roberson ("uma_zfree_debug: called with spinlock or critical section held")); 3248d4665eaaSJeff Roberson 3249d4665eaaSJeff Roberson #ifdef DEBUG_MEMGUARD 32509e47b341SJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) == 0 && is_memguard_addr(item)) { 3251d4665eaaSJeff Roberson if (zone->uz_dtor != NULL) 3252d4665eaaSJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 3253d4665eaaSJeff Roberson if (zone->uz_fini != NULL) 3254d4665eaaSJeff Roberson zone->uz_fini(item, zone->uz_size); 3255d4665eaaSJeff Roberson memguard_free(item); 3256d4665eaaSJeff Roberson return (EJUSTRETURN); 3257d4665eaaSJeff Roberson } 3258d4665eaaSJeff Roberson #endif 3259d4665eaaSJeff Roberson return (0); 3260d4665eaaSJeff Roberson } 3261d4665eaaSJeff Roberson #endif 3262d4665eaaSJeff Roberson 32636d88d784SJeff Roberson static inline void * 32646d88d784SJeff Roberson cache_alloc_item(uma_zone_t zone, uma_cache_t cache, uma_cache_bucket_t bucket, 32656d88d784SJeff Roberson void *udata, int flags) 3266d4665eaaSJeff Roberson { 32676d88d784SJeff Roberson void *item; 32686d88d784SJeff Roberson int size, uz_flags; 32696d88d784SJeff Roberson 32706d88d784SJeff Roberson item = cache_bucket_pop(cache, bucket); 32716d88d784SJeff Roberson size = cache_uz_size(cache); 32726d88d784SJeff Roberson uz_flags = cache_uz_flags(cache); 32736d88d784SJeff Roberson critical_exit(); 32746d88d784SJeff Roberson return (item_ctor(zone, uz_flags, size, udata, flags, item)); 32756d88d784SJeff Roberson } 32766d88d784SJeff Roberson 32776d88d784SJeff Roberson static __noinline void * 32786d88d784SJeff Roberson cache_alloc_retry(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) 32796d88d784SJeff Roberson { 32806d88d784SJeff Roberson uma_cache_bucket_t bucket; 3281d4665eaaSJeff Roberson int domain; 3282d4665eaaSJeff Roberson 32836d88d784SJeff Roberson while (cache_alloc(zone, cache, udata, flags)) { 32846d88d784SJeff Roberson cache = &zone->uz_cpu[curcpu]; 32856d88d784SJeff Roberson bucket = &cache->uc_allocbucket; 32866d88d784SJeff Roberson if (__predict_false(bucket->ucb_cnt == 0)) 32876d88d784SJeff Roberson continue; 32886d88d784SJeff Roberson return (cache_alloc_item(zone, cache, bucket, udata, flags)); 32896d88d784SJeff Roberson } 32906d88d784SJeff Roberson critical_exit(); 32916d88d784SJeff Roberson 3292d4665eaaSJeff Roberson /* 3293d4665eaaSJeff Roberson * We can not get a bucket so try to return a single item. 3294d4665eaaSJeff Roberson */ 3295d4665eaaSJeff Roberson if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) 3296d4665eaaSJeff Roberson domain = PCPU_GET(domain); 3297d4665eaaSJeff Roberson else 3298d4665eaaSJeff Roberson domain = UMA_ANYDOMAIN; 3299d4665eaaSJeff Roberson return (zone_alloc_item(zone, udata, domain, flags)); 3300d4665eaaSJeff Roberson } 3301d4665eaaSJeff Roberson 3302d4665eaaSJeff Roberson /* See uma.h */ 3303d4665eaaSJeff Roberson void * 3304d4665eaaSJeff Roberson uma_zalloc_smr(uma_zone_t zone, int flags) 3305d4665eaaSJeff Roberson { 3306d4665eaaSJeff Roberson uma_cache_bucket_t bucket; 3307d4665eaaSJeff Roberson uma_cache_t cache; 3308d4665eaaSJeff Roberson 3309d4665eaaSJeff Roberson #ifdef UMA_ZALLOC_DEBUG 33106d88d784SJeff Roberson void *item; 33116d88d784SJeff Roberson 3312d4665eaaSJeff Roberson KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, 3313d4665eaaSJeff Roberson ("uma_zalloc_arg: called with non-SMR zone.\n")); 3314d4665eaaSJeff Roberson if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) 3315d4665eaaSJeff Roberson return (item); 3316d4665eaaSJeff Roberson #endif 3317d4665eaaSJeff Roberson 3318d4665eaaSJeff Roberson critical_enter(); 3319d4665eaaSJeff Roberson cache = &zone->uz_cpu[curcpu]; 3320d4665eaaSJeff Roberson bucket = &cache->uc_allocbucket; 33216d88d784SJeff Roberson if (__predict_false(bucket->ucb_cnt == 0)) 33226d88d784SJeff Roberson return (cache_alloc_retry(zone, cache, NULL, flags)); 33236d88d784SJeff Roberson return (cache_alloc_item(zone, cache, bucket, NULL, flags)); 3324d4665eaaSJeff Roberson } 3325d4665eaaSJeff Roberson 33269c2cd7e5SJeff Roberson /* See uma.h */ 33278355f576SJeff Roberson void * 33282cc35ff9SJeff Roberson uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 33298355f576SJeff Roberson { 3330376b1ba3SJeff Roberson uma_cache_bucket_t bucket; 3331ab3185d1SJeff Roberson uma_cache_t cache; 33328355f576SJeff Roberson 3333e866d8f0SMark Murray /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 333419fa89e9SMark Murray random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 333510cb2424SMark Murray 33368355f576SJeff Roberson /* This is the fast path allocation */ 3337e63a1c2fSRyan Libby CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name, 3338e63a1c2fSRyan Libby zone, flags); 3339a553d4b8SJeff Roberson 3340d4665eaaSJeff Roberson #ifdef UMA_ZALLOC_DEBUG 33416d88d784SJeff Roberson void *item; 33426d88d784SJeff Roberson 3343d4665eaaSJeff Roberson KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, 3344d4665eaaSJeff Roberson ("uma_zalloc_arg: called with SMR zone.\n")); 3345d4665eaaSJeff Roberson if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) 33468d689e04SGleb Smirnoff return (item); 33478d689e04SGleb Smirnoff #endif 3348d4665eaaSJeff Roberson 33495d1ae027SRobert Watson /* 33505d1ae027SRobert Watson * If possible, allocate from the per-CPU cache. There are two 33515d1ae027SRobert Watson * requirements for safe access to the per-CPU cache: (1) the thread 33525d1ae027SRobert Watson * accessing the cache must not be preempted or yield during access, 33535d1ae027SRobert Watson * and (2) the thread must not migrate CPUs without switching which 33545d1ae027SRobert Watson * cache it accesses. We rely on a critical section to prevent 33555d1ae027SRobert Watson * preemption and migration. We release the critical section in 33565d1ae027SRobert Watson * order to acquire the zone mutex if we are unable to allocate from 33575d1ae027SRobert Watson * the current cache; when we re-acquire the critical section, we 33585d1ae027SRobert Watson * must detect and handle migration if it has occurred. 33595d1ae027SRobert Watson */ 33605d1ae027SRobert Watson critical_enter(); 3361cc7ce83aSJeff Roberson cache = &zone->uz_cpu[curcpu]; 3362376b1ba3SJeff Roberson bucket = &cache->uc_allocbucket; 33636d88d784SJeff Roberson if (__predict_false(bucket->ucb_cnt == 0)) 33646d88d784SJeff Roberson return (cache_alloc_retry(zone, cache, udata, flags)); 33656d88d784SJeff Roberson return (cache_alloc_item(zone, cache, bucket, udata, flags)); 3366fc03d22bSJeff Roberson } 3367fc03d22bSJeff Roberson 33688355f576SJeff Roberson /* 3369beb8beefSJeff Roberson * Replenish an alloc bucket and possibly restore an old one. Called in 3370beb8beefSJeff Roberson * a critical section. Returns in a critical section. 3371beb8beefSJeff Roberson * 33724bd61e19SJeff Roberson * A false return value indicates an allocation failure. 33734bd61e19SJeff Roberson * A true return value indicates success and the caller should retry. 3374beb8beefSJeff Roberson */ 3375beb8beefSJeff Roberson static __noinline bool 3376beb8beefSJeff Roberson cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) 3377beb8beefSJeff Roberson { 3378beb8beefSJeff Roberson uma_bucket_t bucket; 3379cc7ce83aSJeff Roberson int domain; 3380c6fd3e23SJeff Roberson bool new; 3381beb8beefSJeff Roberson 3382beb8beefSJeff Roberson CRITICAL_ASSERT(curthread); 3383beb8beefSJeff Roberson 3384beb8beefSJeff Roberson /* 3385beb8beefSJeff Roberson * If we have run out of items in our alloc bucket see 3386beb8beefSJeff Roberson * if we can switch with the free bucket. 3387d4665eaaSJeff Roberson * 3388d4665eaaSJeff Roberson * SMR Zones can't re-use the free bucket until the sequence has 3389d4665eaaSJeff Roberson * expired. 33908355f576SJeff Roberson */ 3391c6fd3e23SJeff Roberson if ((cache_uz_flags(cache) & UMA_ZONE_SMR) == 0 && 3392d4665eaaSJeff Roberson cache->uc_freebucket.ucb_cnt != 0) { 3393d4665eaaSJeff Roberson cache_bucket_swap(&cache->uc_freebucket, 3394d4665eaaSJeff Roberson &cache->uc_allocbucket); 3395beb8beefSJeff Roberson return (true); 33968355f576SJeff Roberson } 3397fc03d22bSJeff Roberson 3398fc03d22bSJeff Roberson /* 3399fc03d22bSJeff Roberson * Discard any empty allocation bucket while we hold no locks. 3400fc03d22bSJeff Roberson */ 3401376b1ba3SJeff Roberson bucket = cache_bucket_unload_alloc(cache); 3402fc03d22bSJeff Roberson critical_exit(); 3403c6fd3e23SJeff Roberson 3404c6fd3e23SJeff Roberson if (bucket != NULL) { 3405c6fd3e23SJeff Roberson KASSERT(bucket->ub_cnt == 0, 3406c6fd3e23SJeff Roberson ("cache_alloc: Entered with non-empty alloc bucket.")); 34076fd34d6fSJeff Roberson bucket_free(zone, bucket, udata); 3408c6fd3e23SJeff Roberson } 3409fc03d22bSJeff Roberson 34104bd61e19SJeff Roberson /* Short-circuit for zones without buckets and low memory. */ 34114bd61e19SJeff Roberson if (zone->uz_bucket_size == 0 || bucketdisable) { 34124bd61e19SJeff Roberson critical_enter(); 34134bd61e19SJeff Roberson return (false); 34144bd61e19SJeff Roberson } 34154bd61e19SJeff Roberson 34165d1ae027SRobert Watson /* 34175d1ae027SRobert Watson * Attempt to retrieve the item from the per-CPU cache has failed, so 3418c6fd3e23SJeff Roberson * we must go back to the zone. This requires the zdom lock, so we 34195d1ae027SRobert Watson * must drop the critical section, then re-acquire it when we go back 34205d1ae027SRobert Watson * to the cache. Since the critical section is released, we may be 34215d1ae027SRobert Watson * preempted or migrate. As such, make sure not to maintain any 34225d1ae027SRobert Watson * thread-local state specific to the cache from prior to releasing 34235d1ae027SRobert Watson * the critical section. 34245d1ae027SRobert Watson */ 3425c1685086SJeff Roberson domain = PCPU_GET(domain); 3426c6fd3e23SJeff Roberson if ((cache_uz_flags(cache) & UMA_ZONE_ROUNDROBIN) != 0) 3427c6fd3e23SJeff Roberson domain = zone_domain_highest(zone, domain); 3428c6fd3e23SJeff Roberson bucket = cache_fetch_bucket(zone, cache, domain); 3429c6fd3e23SJeff Roberson if (bucket == NULL) { 3430beb8beefSJeff Roberson bucket = zone_alloc_bucket(zone, udata, domain, flags); 3431c6fd3e23SJeff Roberson new = true; 3432c6fd3e23SJeff Roberson } else 3433c6fd3e23SJeff Roberson new = false; 3434c6fd3e23SJeff Roberson 34351431a748SGleb Smirnoff CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p", 34361431a748SGleb Smirnoff zone->uz_name, zone, bucket); 34374bd61e19SJeff Roberson if (bucket == NULL) { 3438fc03d22bSJeff Roberson critical_enter(); 3439beb8beefSJeff Roberson return (false); 34404bd61e19SJeff Roberson } 34410f9b7bf3SMark Johnston 3442fc03d22bSJeff Roberson /* 3443fc03d22bSJeff Roberson * See if we lost the race or were migrated. Cache the 3444fc03d22bSJeff Roberson * initialized bucket to make this less likely or claim 3445fc03d22bSJeff Roberson * the memory directly. 3446fc03d22bSJeff Roberson */ 34474bd61e19SJeff Roberson critical_enter(); 3448cc7ce83aSJeff Roberson cache = &zone->uz_cpu[curcpu]; 3449376b1ba3SJeff Roberson if (cache->uc_allocbucket.ucb_bucket == NULL && 3450c6fd3e23SJeff Roberson ((cache_uz_flags(cache) & UMA_ZONE_FIRSTTOUCH) == 0 || 345181c0d72cSGleb Smirnoff domain == PCPU_GET(domain))) { 3452c6fd3e23SJeff Roberson if (new) 3453c6fd3e23SJeff Roberson atomic_add_long(&ZDOM_GET(zone, domain)->uzd_imax, 3454c6fd3e23SJeff Roberson bucket->ub_cnt); 3455376b1ba3SJeff Roberson cache_bucket_load_alloc(cache, bucket); 3456beb8beefSJeff Roberson return (true); 3457c6fd3e23SJeff Roberson } 3458c6fd3e23SJeff Roberson 3459c6fd3e23SJeff Roberson /* 3460c6fd3e23SJeff Roberson * We lost the race, release this bucket and start over. 3461c6fd3e23SJeff Roberson */ 3462c6fd3e23SJeff Roberson critical_exit(); 3463c6fd3e23SJeff Roberson zone_put_bucket(zone, domain, bucket, udata, false); 3464c6fd3e23SJeff Roberson critical_enter(); 3465c6fd3e23SJeff Roberson 3466beb8beefSJeff Roberson return (true); 3467bbee39c6SJeff Roberson } 3468bbee39c6SJeff Roberson 3469ab3185d1SJeff Roberson void * 3470ab3185d1SJeff Roberson uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) 3471bbee39c6SJeff Roberson { 3472ab3185d1SJeff Roberson 3473ab3185d1SJeff Roberson /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 347419fa89e9SMark Murray random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 3475ab3185d1SJeff Roberson 3476ab3185d1SJeff Roberson /* This is the fast path allocation */ 3477e63a1c2fSRyan Libby CTR4(KTR_UMA, "uma_zalloc_domain zone %s(%p) domain %d flags %d", 3478e63a1c2fSRyan Libby zone->uz_name, zone, domain, flags); 3479ab3185d1SJeff Roberson 3480ab3185d1SJeff Roberson if (flags & M_WAITOK) { 3481ab3185d1SJeff Roberson WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 3482ab3185d1SJeff Roberson "uma_zalloc_domain: zone \"%s\"", zone->uz_name); 3483ab3185d1SJeff Roberson } 3484ab3185d1SJeff Roberson KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), 3485ab3185d1SJeff Roberson ("uma_zalloc_domain: called with spinlock or critical section held")); 3486ab3185d1SJeff Roberson 3487ab3185d1SJeff Roberson return (zone_alloc_item(zone, udata, domain, flags)); 3488ab3185d1SJeff Roberson } 3489ab3185d1SJeff Roberson 3490ab3185d1SJeff Roberson /* 3491ab3185d1SJeff Roberson * Find a slab with some space. Prefer slabs that are partially used over those 3492ab3185d1SJeff Roberson * that are totally full. This helps to reduce fragmentation. 3493ab3185d1SJeff Roberson * 3494ab3185d1SJeff Roberson * If 'rr' is 1, search all domains starting from 'domain'. Otherwise check 3495ab3185d1SJeff Roberson * only 'domain'. 3496ab3185d1SJeff Roberson */ 3497ab3185d1SJeff Roberson static uma_slab_t 3498194a979eSMark Johnston keg_first_slab(uma_keg_t keg, int domain, bool rr) 3499ab3185d1SJeff Roberson { 3500ab3185d1SJeff Roberson uma_domain_t dom; 3501bbee39c6SJeff Roberson uma_slab_t slab; 3502ab3185d1SJeff Roberson int start; 3503ab3185d1SJeff Roberson 3504ab3185d1SJeff Roberson KASSERT(domain >= 0 && domain < vm_ndomains, 3505ab3185d1SJeff Roberson ("keg_first_slab: domain %d out of range", domain)); 35068b987a77SJeff Roberson KEG_LOCK_ASSERT(keg, domain); 3507ab3185d1SJeff Roberson 3508ab3185d1SJeff Roberson slab = NULL; 3509ab3185d1SJeff Roberson start = domain; 3510ab3185d1SJeff Roberson do { 3511ab3185d1SJeff Roberson dom = &keg->uk_domain[domain]; 35124ab3aee8SMark Johnston if ((slab = LIST_FIRST(&dom->ud_part_slab)) != NULL) 35134ab3aee8SMark Johnston return (slab); 35144ab3aee8SMark Johnston if ((slab = LIST_FIRST(&dom->ud_free_slab)) != NULL) { 3515ab3185d1SJeff Roberson LIST_REMOVE(slab, us_link); 35164ab3aee8SMark Johnston dom->ud_free_slabs--; 3517ab3185d1SJeff Roberson LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); 3518ab3185d1SJeff Roberson return (slab); 3519ab3185d1SJeff Roberson } 3520ab3185d1SJeff Roberson if (rr) 3521ab3185d1SJeff Roberson domain = (domain + 1) % vm_ndomains; 3522ab3185d1SJeff Roberson } while (domain != start); 3523ab3185d1SJeff Roberson 3524ab3185d1SJeff Roberson return (NULL); 3525ab3185d1SJeff Roberson } 3526ab3185d1SJeff Roberson 35278b987a77SJeff Roberson /* 35288b987a77SJeff Roberson * Fetch an existing slab from a free or partial list. Returns with the 35298b987a77SJeff Roberson * keg domain lock held if a slab was found or unlocked if not. 35308b987a77SJeff Roberson */ 3531ab3185d1SJeff Roberson static uma_slab_t 3532194a979eSMark Johnston keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags) 3533ab3185d1SJeff Roberson { 35348b987a77SJeff Roberson uma_slab_t slab; 3535194a979eSMark Johnston uint32_t reserve; 3536099a0e58SBosko Milekic 35378b987a77SJeff Roberson /* HASH has a single free list. */ 353854c5ae80SRyan Libby if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) 35398b987a77SJeff Roberson domain = 0; 3540194a979eSMark Johnston 35418b987a77SJeff Roberson KEG_LOCK(keg, domain); 3542194a979eSMark Johnston reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; 35434ab3aee8SMark Johnston if (keg->uk_domain[domain].ud_free_items <= reserve || 35448b987a77SJeff Roberson (slab = keg_first_slab(keg, domain, rr)) == NULL) { 35458b987a77SJeff Roberson KEG_UNLOCK(keg, domain); 3546194a979eSMark Johnston return (NULL); 35478b987a77SJeff Roberson } 35488b987a77SJeff Roberson return (slab); 3549194a979eSMark Johnston } 3550194a979eSMark Johnston 3551194a979eSMark Johnston static uma_slab_t 3552194a979eSMark Johnston keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) 3553194a979eSMark Johnston { 3554194a979eSMark Johnston struct vm_domainset_iter di; 3555194a979eSMark Johnston uma_slab_t slab; 3556194a979eSMark Johnston int aflags, domain; 3557194a979eSMark Johnston bool rr; 3558194a979eSMark Johnston 3559194a979eSMark Johnston restart: 3560bbee39c6SJeff Roberson /* 3561194a979eSMark Johnston * Use the keg's policy if upper layers haven't already specified a 3562194a979eSMark Johnston * domain (as happens with first-touch zones). 3563194a979eSMark Johnston * 3564194a979eSMark Johnston * To avoid races we run the iterator with the keg lock held, but that 3565194a979eSMark Johnston * means that we cannot allow the vm_domainset layer to sleep. Thus, 3566194a979eSMark Johnston * clear M_WAITOK and handle low memory conditions locally. 3567bbee39c6SJeff Roberson */ 3568ab3185d1SJeff Roberson rr = rdomain == UMA_ANYDOMAIN; 3569ab3185d1SJeff Roberson if (rr) { 3570194a979eSMark Johnston aflags = (flags & ~M_WAITOK) | M_NOWAIT; 3571194a979eSMark Johnston vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, 3572194a979eSMark Johnston &aflags); 3573194a979eSMark Johnston } else { 3574194a979eSMark Johnston aflags = flags; 3575194a979eSMark Johnston domain = rdomain; 3576194a979eSMark Johnston } 3577ab3185d1SJeff Roberson 3578194a979eSMark Johnston for (;;) { 3579194a979eSMark Johnston slab = keg_fetch_free_slab(keg, domain, rr, flags); 3580584061b4SJeff Roberson if (slab != NULL) 3581bbee39c6SJeff Roberson return (slab); 3582bbee39c6SJeff Roberson 3583bbee39c6SJeff Roberson /* 3584bbee39c6SJeff Roberson * M_NOVM means don't ask at all! 3585bbee39c6SJeff Roberson */ 3586bbee39c6SJeff Roberson if (flags & M_NOVM) 3587bbee39c6SJeff Roberson break; 3588bbee39c6SJeff Roberson 358986220393SMark Johnston slab = keg_alloc_slab(keg, zone, domain, flags, aflags); 35908b987a77SJeff Roberson if (slab != NULL) 3591bbee39c6SJeff Roberson return (slab); 35923639ac42SJeff Roberson if (!rr && (flags & M_WAITOK) == 0) 35933639ac42SJeff Roberson break; 3594194a979eSMark Johnston if (rr && vm_domainset_iter_policy(&di, &domain) != 0) { 3595194a979eSMark Johnston if ((flags & M_WAITOK) != 0) { 3596194a979eSMark Johnston vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask); 3597194a979eSMark Johnston goto restart; 359830c5525bSAndrew Gallatin } 3599194a979eSMark Johnston break; 3600194a979eSMark Johnston } 3601ab3185d1SJeff Roberson } 3602ab3185d1SJeff Roberson 3603bbee39c6SJeff Roberson /* 3604bbee39c6SJeff Roberson * We might not have been able to get a slab but another cpu 3605bbee39c6SJeff Roberson * could have while we were unlocked. Check again before we 3606bbee39c6SJeff Roberson * fail. 3607bbee39c6SJeff Roberson */ 36088b987a77SJeff Roberson if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) 3609bbee39c6SJeff Roberson return (slab); 36108b987a77SJeff Roberson 3611ab3185d1SJeff Roberson return (NULL); 3612ab3185d1SJeff Roberson } 3613bbee39c6SJeff Roberson 3614d56368d7SBosko Milekic static void * 36150095a784SJeff Roberson slab_alloc_item(uma_keg_t keg, uma_slab_t slab) 3616bbee39c6SJeff Roberson { 3617ab3185d1SJeff Roberson uma_domain_t dom; 3618bbee39c6SJeff Roberson void *item; 36199b8db4d0SRyan Libby int freei; 3620bbee39c6SJeff Roberson 36218b987a77SJeff Roberson KEG_LOCK_ASSERT(keg, slab->us_domain); 3622099a0e58SBosko Milekic 36238b987a77SJeff Roberson dom = &keg->uk_domain[slab->us_domain]; 36249b78b1f4SJeff Roberson freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1; 36259b78b1f4SJeff Roberson BIT_CLR(keg->uk_ipers, freei, &slab->us_free); 36261e0701e1SJeff Roberson item = slab_item(slab, keg, freei); 3627bbee39c6SJeff Roberson slab->us_freecount--; 36284ab3aee8SMark Johnston dom->ud_free_items--; 3629ef72505eSJeff Roberson 36304ab3aee8SMark Johnston /* 36314ab3aee8SMark Johnston * Move this slab to the full list. It must be on the partial list, so 36324ab3aee8SMark Johnston * we do not need to update the free slab count. In particular, 36334ab3aee8SMark Johnston * keg_fetch_slab() always returns slabs on the partial list. 36344ab3aee8SMark Johnston */ 3635bbee39c6SJeff Roberson if (slab->us_freecount == 0) { 3636bbee39c6SJeff Roberson LIST_REMOVE(slab, us_link); 3637ab3185d1SJeff Roberson LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link); 3638bbee39c6SJeff Roberson } 3639bbee39c6SJeff Roberson 3640bbee39c6SJeff Roberson return (item); 3641bbee39c6SJeff Roberson } 3642bbee39c6SJeff Roberson 3643bbee39c6SJeff Roberson static int 3644b75c4efcSAndrew Turner zone_import(void *arg, void **bucket, int max, int domain, int flags) 36450095a784SJeff Roberson { 36468b987a77SJeff Roberson uma_domain_t dom; 3647b75c4efcSAndrew Turner uma_zone_t zone; 36480095a784SJeff Roberson uma_slab_t slab; 36490095a784SJeff Roberson uma_keg_t keg; 3650a03af342SSean Bruno #ifdef NUMA 3651ab3185d1SJeff Roberson int stripe; 3652a03af342SSean Bruno #endif 36530095a784SJeff Roberson int i; 36540095a784SJeff Roberson 3655b75c4efcSAndrew Turner zone = arg; 36560095a784SJeff Roberson slab = NULL; 3657584061b4SJeff Roberson keg = zone->uz_keg; 3658af526374SJeff Roberson /* Try to keep the buckets totally full */ 36590095a784SJeff Roberson for (i = 0; i < max; ) { 3660584061b4SJeff Roberson if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) 36610095a784SJeff Roberson break; 3662a03af342SSean Bruno #ifdef NUMA 3663ab3185d1SJeff Roberson stripe = howmany(max, vm_ndomains); 3664a03af342SSean Bruno #endif 36658b987a77SJeff Roberson dom = &keg->uk_domain[slab->us_domain]; 36666fd34d6fSJeff Roberson while (slab->us_freecount && i < max) { 36670095a784SJeff Roberson bucket[i++] = slab_alloc_item(keg, slab); 36684ab3aee8SMark Johnston if (dom->ud_free_items <= keg->uk_reserve) 36696fd34d6fSJeff Roberson break; 3670b6715dabSJeff Roberson #ifdef NUMA 3671ab3185d1SJeff Roberson /* 3672ab3185d1SJeff Roberson * If the zone is striped we pick a new slab for every 3673ab3185d1SJeff Roberson * N allocations. Eliminating this conditional will 3674ab3185d1SJeff Roberson * instead pick a new domain for each bucket rather 3675ab3185d1SJeff Roberson * than stripe within each bucket. The current option 3676ab3185d1SJeff Roberson * produces more fragmentation and requires more cpu 3677ab3185d1SJeff Roberson * time but yields better distribution. 3678ab3185d1SJeff Roberson */ 3679dfe13344SJeff Roberson if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0 && 3680ab3185d1SJeff Roberson vm_ndomains > 1 && --stripe == 0) 3681ab3185d1SJeff Roberson break; 3682ab3185d1SJeff Roberson #endif 36836fd34d6fSJeff Roberson } 36848b987a77SJeff Roberson KEG_UNLOCK(keg, slab->us_domain); 3685ab3185d1SJeff Roberson /* Don't block if we allocated any successfully. */ 36860095a784SJeff Roberson flags &= ~M_WAITOK; 36870095a784SJeff Roberson flags |= M_NOWAIT; 36880095a784SJeff Roberson } 36890095a784SJeff Roberson 36900095a784SJeff Roberson return i; 36910095a784SJeff Roberson } 36920095a784SJeff Roberson 36934bd61e19SJeff Roberson static int 36944bd61e19SJeff Roberson zone_alloc_limit_hard(uma_zone_t zone, int count, int flags) 36954bd61e19SJeff Roberson { 36964bd61e19SJeff Roberson uint64_t old, new, total, max; 36974bd61e19SJeff Roberson 36984bd61e19SJeff Roberson /* 36994bd61e19SJeff Roberson * The hard case. We're going to sleep because there were existing 37004bd61e19SJeff Roberson * sleepers or because we ran out of items. This routine enforces 37014bd61e19SJeff Roberson * fairness by keeping fifo order. 37024bd61e19SJeff Roberson * 37034bd61e19SJeff Roberson * First release our ill gotten gains and make some noise. 37044bd61e19SJeff Roberson */ 37054bd61e19SJeff Roberson for (;;) { 37064bd61e19SJeff Roberson zone_free_limit(zone, count); 37074bd61e19SJeff Roberson zone_log_warning(zone); 37084bd61e19SJeff Roberson zone_maxaction(zone); 37094bd61e19SJeff Roberson if (flags & M_NOWAIT) 37104bd61e19SJeff Roberson return (0); 37114bd61e19SJeff Roberson 37124bd61e19SJeff Roberson /* 37134bd61e19SJeff Roberson * We need to allocate an item or set ourself as a sleeper 37144bd61e19SJeff Roberson * while the sleepq lock is held to avoid wakeup races. This 37154bd61e19SJeff Roberson * is essentially a home rolled semaphore. 37164bd61e19SJeff Roberson */ 37174bd61e19SJeff Roberson sleepq_lock(&zone->uz_max_items); 37184bd61e19SJeff Roberson old = zone->uz_items; 37194bd61e19SJeff Roberson do { 37204bd61e19SJeff Roberson MPASS(UZ_ITEMS_SLEEPERS(old) < UZ_ITEMS_SLEEPERS_MAX); 37214bd61e19SJeff Roberson /* Cache the max since we will evaluate twice. */ 37224bd61e19SJeff Roberson max = zone->uz_max_items; 37234bd61e19SJeff Roberson if (UZ_ITEMS_SLEEPERS(old) != 0 || 37244bd61e19SJeff Roberson UZ_ITEMS_COUNT(old) >= max) 37254bd61e19SJeff Roberson new = old + UZ_ITEMS_SLEEPER; 37264bd61e19SJeff Roberson else 37274bd61e19SJeff Roberson new = old + MIN(count, max - old); 37284bd61e19SJeff Roberson } while (atomic_fcmpset_64(&zone->uz_items, &old, new) == 0); 37294bd61e19SJeff Roberson 37304bd61e19SJeff Roberson /* We may have successfully allocated under the sleepq lock. */ 37314bd61e19SJeff Roberson if (UZ_ITEMS_SLEEPERS(new) == 0) { 37324bd61e19SJeff Roberson sleepq_release(&zone->uz_max_items); 37334bd61e19SJeff Roberson return (new - old); 37344bd61e19SJeff Roberson } 37354bd61e19SJeff Roberson 37364bd61e19SJeff Roberson /* 37374bd61e19SJeff Roberson * This is in a different cacheline from uz_items so that we 37384bd61e19SJeff Roberson * don't constantly invalidate the fastpath cacheline when we 37394bd61e19SJeff Roberson * adjust item counts. This could be limited to toggling on 37404bd61e19SJeff Roberson * transitions. 37414bd61e19SJeff Roberson */ 37424bd61e19SJeff Roberson atomic_add_32(&zone->uz_sleepers, 1); 37434bd61e19SJeff Roberson atomic_add_64(&zone->uz_sleeps, 1); 37444bd61e19SJeff Roberson 37454bd61e19SJeff Roberson /* 37464bd61e19SJeff Roberson * We have added ourselves as a sleeper. The sleepq lock 37474bd61e19SJeff Roberson * protects us from wakeup races. Sleep now and then retry. 37484bd61e19SJeff Roberson */ 37494bd61e19SJeff Roberson sleepq_add(&zone->uz_max_items, NULL, "zonelimit", 0, 0); 37504bd61e19SJeff Roberson sleepq_wait(&zone->uz_max_items, PVM); 37514bd61e19SJeff Roberson 37524bd61e19SJeff Roberson /* 37534bd61e19SJeff Roberson * After wakeup, remove ourselves as a sleeper and try 37544bd61e19SJeff Roberson * again. We no longer have the sleepq lock for protection. 37554bd61e19SJeff Roberson * 37564bd61e19SJeff Roberson * Subract ourselves as a sleeper while attempting to add 37574bd61e19SJeff Roberson * our count. 37584bd61e19SJeff Roberson */ 37594bd61e19SJeff Roberson atomic_subtract_32(&zone->uz_sleepers, 1); 37604bd61e19SJeff Roberson old = atomic_fetchadd_64(&zone->uz_items, 37614bd61e19SJeff Roberson -(UZ_ITEMS_SLEEPER - count)); 37624bd61e19SJeff Roberson /* We're no longer a sleeper. */ 37634bd61e19SJeff Roberson old -= UZ_ITEMS_SLEEPER; 37644bd61e19SJeff Roberson 37654bd61e19SJeff Roberson /* 37664bd61e19SJeff Roberson * If we're still at the limit, restart. Notably do not 37674bd61e19SJeff Roberson * block on other sleepers. Cache the max value to protect 37684bd61e19SJeff Roberson * against changes via sysctl. 37694bd61e19SJeff Roberson */ 37704bd61e19SJeff Roberson total = UZ_ITEMS_COUNT(old); 37714bd61e19SJeff Roberson max = zone->uz_max_items; 37724bd61e19SJeff Roberson if (total >= max) 37734bd61e19SJeff Roberson continue; 37744bd61e19SJeff Roberson /* Truncate if necessary, otherwise wake other sleepers. */ 37754bd61e19SJeff Roberson if (total + count > max) { 37764bd61e19SJeff Roberson zone_free_limit(zone, total + count - max); 37774bd61e19SJeff Roberson count = max - total; 37784bd61e19SJeff Roberson } else if (total + count < max && UZ_ITEMS_SLEEPERS(old) != 0) 37794bd61e19SJeff Roberson wakeup_one(&zone->uz_max_items); 37804bd61e19SJeff Roberson 37814bd61e19SJeff Roberson return (count); 37824bd61e19SJeff Roberson } 37834bd61e19SJeff Roberson } 37844bd61e19SJeff Roberson 37854bd61e19SJeff Roberson /* 37864bd61e19SJeff Roberson * Allocate 'count' items from our max_items limit. Returns the number 37874bd61e19SJeff Roberson * available. If M_NOWAIT is not specified it will sleep until at least 37884bd61e19SJeff Roberson * one item can be allocated. 37894bd61e19SJeff Roberson */ 37904bd61e19SJeff Roberson static int 37914bd61e19SJeff Roberson zone_alloc_limit(uma_zone_t zone, int count, int flags) 37924bd61e19SJeff Roberson { 37934bd61e19SJeff Roberson uint64_t old; 37944bd61e19SJeff Roberson uint64_t max; 37954bd61e19SJeff Roberson 37964bd61e19SJeff Roberson max = zone->uz_max_items; 37974bd61e19SJeff Roberson MPASS(max > 0); 37984bd61e19SJeff Roberson 37994bd61e19SJeff Roberson /* 38004bd61e19SJeff Roberson * We expect normal allocations to succeed with a simple 38014bd61e19SJeff Roberson * fetchadd. 38024bd61e19SJeff Roberson */ 38034bd61e19SJeff Roberson old = atomic_fetchadd_64(&zone->uz_items, count); 38044bd61e19SJeff Roberson if (__predict_true(old + count <= max)) 38054bd61e19SJeff Roberson return (count); 38064bd61e19SJeff Roberson 38074bd61e19SJeff Roberson /* 38084bd61e19SJeff Roberson * If we had some items and no sleepers just return the 38094bd61e19SJeff Roberson * truncated value. We have to release the excess space 38104bd61e19SJeff Roberson * though because that may wake sleepers who weren't woken 38114bd61e19SJeff Roberson * because we were temporarily over the limit. 38124bd61e19SJeff Roberson */ 38134bd61e19SJeff Roberson if (old < max) { 38144bd61e19SJeff Roberson zone_free_limit(zone, (old + count) - max); 38154bd61e19SJeff Roberson return (max - old); 38164bd61e19SJeff Roberson } 38174bd61e19SJeff Roberson return (zone_alloc_limit_hard(zone, count, flags)); 38184bd61e19SJeff Roberson } 38194bd61e19SJeff Roberson 38204bd61e19SJeff Roberson /* 38214bd61e19SJeff Roberson * Free a number of items back to the limit. 38224bd61e19SJeff Roberson */ 38234bd61e19SJeff Roberson static void 38244bd61e19SJeff Roberson zone_free_limit(uma_zone_t zone, int count) 38254bd61e19SJeff Roberson { 38264bd61e19SJeff Roberson uint64_t old; 38274bd61e19SJeff Roberson 38284bd61e19SJeff Roberson MPASS(count > 0); 38294bd61e19SJeff Roberson 38304bd61e19SJeff Roberson /* 38314bd61e19SJeff Roberson * In the common case we either have no sleepers or 38324bd61e19SJeff Roberson * are still over the limit and can just return. 38334bd61e19SJeff Roberson */ 38344bd61e19SJeff Roberson old = atomic_fetchadd_64(&zone->uz_items, -count); 38354bd61e19SJeff Roberson if (__predict_true(UZ_ITEMS_SLEEPERS(old) == 0 || 38364bd61e19SJeff Roberson UZ_ITEMS_COUNT(old) - count >= zone->uz_max_items)) 38374bd61e19SJeff Roberson return; 38384bd61e19SJeff Roberson 38394bd61e19SJeff Roberson /* 38404bd61e19SJeff Roberson * Moderate the rate of wakeups. Sleepers will continue 38414bd61e19SJeff Roberson * to generate wakeups if necessary. 38424bd61e19SJeff Roberson */ 38434bd61e19SJeff Roberson wakeup_one(&zone->uz_max_items); 38444bd61e19SJeff Roberson } 38454bd61e19SJeff Roberson 3846fc03d22bSJeff Roberson static uma_bucket_t 3847beb8beefSJeff Roberson zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) 3848bbee39c6SJeff Roberson { 3849bbee39c6SJeff Roberson uma_bucket_t bucket; 3850beb8beefSJeff Roberson int maxbucket, cnt; 3851bbee39c6SJeff Roberson 3852e63a1c2fSRyan Libby CTR3(KTR_UMA, "zone_alloc_bucket zone %s(%p) domain %d", zone->uz_name, 3853e63a1c2fSRyan Libby zone, domain); 385430c5525bSAndrew Gallatin 3855c1685086SJeff Roberson /* Avoid allocs targeting empty domains. */ 3856c1685086SJeff Roberson if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) 3857c1685086SJeff Roberson domain = UMA_ANYDOMAIN; 3858c6fd3e23SJeff Roberson if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0) 3859c6fd3e23SJeff Roberson domain = UMA_ANYDOMAIN; 3860c1685086SJeff Roberson 38614bd61e19SJeff Roberson if (zone->uz_max_items > 0) 38624bd61e19SJeff Roberson maxbucket = zone_alloc_limit(zone, zone->uz_bucket_size, 38634bd61e19SJeff Roberson M_NOWAIT); 38644bd61e19SJeff Roberson else 386520a4e154SJeff Roberson maxbucket = zone->uz_bucket_size; 38664bd61e19SJeff Roberson if (maxbucket == 0) 38674bd61e19SJeff Roberson return (false); 3868beb8beefSJeff Roberson 38696fd34d6fSJeff Roberson /* Don't wait for buckets, preserve caller's NOVM setting. */ 38706fd34d6fSJeff Roberson bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); 3871beb8beefSJeff Roberson if (bucket == NULL) { 3872beb8beefSJeff Roberson cnt = 0; 3873beb8beefSJeff Roberson goto out; 3874beb8beefSJeff Roberson } 38750095a784SJeff Roberson 38760095a784SJeff Roberson bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, 3877beb8beefSJeff Roberson MIN(maxbucket, bucket->ub_entries), domain, flags); 38780095a784SJeff Roberson 38790095a784SJeff Roberson /* 38800095a784SJeff Roberson * Initialize the memory if necessary. 38810095a784SJeff Roberson */ 38820095a784SJeff Roberson if (bucket->ub_cnt != 0 && zone->uz_init != NULL) { 3883099a0e58SBosko Milekic int i; 3884bbee39c6SJeff Roberson 38850095a784SJeff Roberson for (i = 0; i < bucket->ub_cnt; i++) 3886e20a199fSJeff Roberson if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size, 38870095a784SJeff Roberson flags) != 0) 3888b23f72e9SBrian Feldman break; 3889b23f72e9SBrian Feldman /* 3890b23f72e9SBrian Feldman * If we couldn't initialize the whole bucket, put the 3891b23f72e9SBrian Feldman * rest back onto the freelist. 3892b23f72e9SBrian Feldman */ 3893b23f72e9SBrian Feldman if (i != bucket->ub_cnt) { 3894af526374SJeff Roberson zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i], 38950095a784SJeff Roberson bucket->ub_cnt - i); 3896a5a262c6SBosko Milekic #ifdef INVARIANTS 38970095a784SJeff Roberson bzero(&bucket->ub_bucket[i], 38980095a784SJeff Roberson sizeof(void *) * (bucket->ub_cnt - i)); 3899a5a262c6SBosko Milekic #endif 3900b23f72e9SBrian Feldman bucket->ub_cnt = i; 3901b23f72e9SBrian Feldman } 3902099a0e58SBosko Milekic } 3903099a0e58SBosko Milekic 3904beb8beefSJeff Roberson cnt = bucket->ub_cnt; 3905f7104ccdSAlexander Motin if (bucket->ub_cnt == 0) { 39066fd34d6fSJeff Roberson bucket_free(zone, bucket, udata); 39072efcc8cbSGleb Smirnoff counter_u64_add(zone->uz_fails, 1); 3908beb8beefSJeff Roberson bucket = NULL; 3909beb8beefSJeff Roberson } 3910beb8beefSJeff Roberson out: 39114bd61e19SJeff Roberson if (zone->uz_max_items > 0 && cnt < maxbucket) 39124bd61e19SJeff Roberson zone_free_limit(zone, maxbucket - cnt); 3913fc03d22bSJeff Roberson 3914fc03d22bSJeff Roberson return (bucket); 3915fc03d22bSJeff Roberson } 3916fc03d22bSJeff Roberson 39178355f576SJeff Roberson /* 39180095a784SJeff Roberson * Allocates a single item from a zone. 39198355f576SJeff Roberson * 39208355f576SJeff Roberson * Arguments 39218355f576SJeff Roberson * zone The zone to alloc for. 39228355f576SJeff Roberson * udata The data to be passed to the constructor. 3923ab3185d1SJeff Roberson * domain The domain to allocate from or UMA_ANYDOMAIN. 3924a163d034SWarner Losh * flags M_WAITOK, M_NOWAIT, M_ZERO. 39258355f576SJeff Roberson * 39268355f576SJeff Roberson * Returns 39278355f576SJeff Roberson * NULL if there is no memory and M_NOWAIT is set 3928bbee39c6SJeff Roberson * An item if successful 39298355f576SJeff Roberson */ 39308355f576SJeff Roberson 39318355f576SJeff Roberson static void * 3932ab3185d1SJeff Roberson zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags) 39338355f576SJeff Roberson { 39348355f576SJeff Roberson void *item; 39358355f576SJeff Roberson 39364bd61e19SJeff Roberson if (zone->uz_max_items > 0 && zone_alloc_limit(zone, 1, flags) == 0) 3937bb15d1c7SGleb Smirnoff return (NULL); 39388355f576SJeff Roberson 3939c1685086SJeff Roberson /* Avoid allocs targeting empty domains. */ 3940c1685086SJeff Roberson if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) 394130c5525bSAndrew Gallatin domain = UMA_ANYDOMAIN; 3942c1685086SJeff Roberson 3943ab3185d1SJeff Roberson if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) 3944beb8beefSJeff Roberson goto fail_cnt; 39458355f576SJeff Roberson 3946099a0e58SBosko Milekic /* 3947099a0e58SBosko Milekic * We have to call both the zone's init (not the keg's init) 3948099a0e58SBosko Milekic * and the zone's ctor. This is because the item is going from 3949099a0e58SBosko Milekic * a keg slab directly to the user, and the user is expecting it 3950099a0e58SBosko Milekic * to be both zone-init'd as well as zone-ctor'd. 3951099a0e58SBosko Milekic */ 3952b23f72e9SBrian Feldman if (zone->uz_init != NULL) { 3953e20a199fSJeff Roberson if (zone->uz_init(item, zone->uz_size, flags) != 0) { 3954bb15d1c7SGleb Smirnoff zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT); 3955beb8beefSJeff Roberson goto fail_cnt; 3956beb8beefSJeff Roberson } 3957beb8beefSJeff Roberson } 3958d4665eaaSJeff Roberson item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, flags, 3959d4665eaaSJeff Roberson item); 3960beb8beefSJeff Roberson if (item == NULL) 39610095a784SJeff Roberson goto fail; 39628355f576SJeff Roberson 39632efcc8cbSGleb Smirnoff counter_u64_add(zone->uz_allocs, 1); 39641431a748SGleb Smirnoff CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item, 39651431a748SGleb Smirnoff zone->uz_name, zone); 39661431a748SGleb Smirnoff 39678355f576SJeff Roberson return (item); 39680095a784SJeff Roberson 3969beb8beefSJeff Roberson fail_cnt: 3970beb8beefSJeff Roberson counter_u64_add(zone->uz_fails, 1); 39710095a784SJeff Roberson fail: 39724bd61e19SJeff Roberson if (zone->uz_max_items > 0) 39734bd61e19SJeff Roberson zone_free_limit(zone, 1); 39741431a748SGleb Smirnoff CTR2(KTR_UMA, "zone_alloc_item failed from %s(%p)", 39751431a748SGleb Smirnoff zone->uz_name, zone); 39764bd61e19SJeff Roberson 39770095a784SJeff Roberson return (NULL); 39788355f576SJeff Roberson } 39798355f576SJeff Roberson 39808355f576SJeff Roberson /* See uma.h */ 39818355f576SJeff Roberson void 3982d4665eaaSJeff Roberson uma_zfree_smr(uma_zone_t zone, void *item) 3983d4665eaaSJeff Roberson { 3984d4665eaaSJeff Roberson uma_cache_t cache; 3985d4665eaaSJeff Roberson uma_cache_bucket_t bucket; 3986c6fd3e23SJeff Roberson int itemdomain, uz_flags; 3987d4665eaaSJeff Roberson 3988d4665eaaSJeff Roberson #ifdef UMA_ZALLOC_DEBUG 3989d4665eaaSJeff Roberson KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, 3990d4665eaaSJeff Roberson ("uma_zfree_smr: called with non-SMR zone.\n")); 3991d4665eaaSJeff Roberson KASSERT(item != NULL, ("uma_zfree_smr: Called with NULL pointer.")); 3992c6fd3e23SJeff Roberson SMR_ASSERT_NOT_ENTERED(zone->uz_smr); 3993d4665eaaSJeff Roberson if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN) 3994d4665eaaSJeff Roberson return; 3995d4665eaaSJeff Roberson #endif 3996d4665eaaSJeff Roberson cache = &zone->uz_cpu[curcpu]; 3997d4665eaaSJeff Roberson uz_flags = cache_uz_flags(cache); 3998c6fd3e23SJeff Roberson itemdomain = 0; 3999d4665eaaSJeff Roberson #ifdef NUMA 4000d4665eaaSJeff Roberson if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) 4001d4665eaaSJeff Roberson itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item)); 4002d4665eaaSJeff Roberson #endif 4003d4665eaaSJeff Roberson critical_enter(); 4004d4665eaaSJeff Roberson do { 4005d4665eaaSJeff Roberson cache = &zone->uz_cpu[curcpu]; 4006d4665eaaSJeff Roberson /* SMR Zones must free to the free bucket. */ 4007d4665eaaSJeff Roberson bucket = &cache->uc_freebucket; 4008d4665eaaSJeff Roberson #ifdef NUMA 4009d4665eaaSJeff Roberson if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && 4010c6fd3e23SJeff Roberson PCPU_GET(domain) != itemdomain) { 4011d4665eaaSJeff Roberson bucket = &cache->uc_crossbucket; 4012d4665eaaSJeff Roberson } 4013d4665eaaSJeff Roberson #endif 4014d4665eaaSJeff Roberson if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { 4015d4665eaaSJeff Roberson cache_bucket_push(cache, bucket, item); 4016d4665eaaSJeff Roberson critical_exit(); 4017d4665eaaSJeff Roberson return; 4018d4665eaaSJeff Roberson } 4019d4665eaaSJeff Roberson } while (cache_free(zone, cache, NULL, item, itemdomain)); 4020d4665eaaSJeff Roberson critical_exit(); 4021d4665eaaSJeff Roberson 4022d4665eaaSJeff Roberson /* 4023d4665eaaSJeff Roberson * If nothing else caught this, we'll just do an internal free. 4024d4665eaaSJeff Roberson */ 4025d4665eaaSJeff Roberson zone_free_item(zone, item, NULL, SKIP_NONE); 4026d4665eaaSJeff Roberson } 4027d4665eaaSJeff Roberson 4028d4665eaaSJeff Roberson /* See uma.h */ 4029d4665eaaSJeff Roberson void 40308355f576SJeff Roberson uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 40318355f576SJeff Roberson { 40328355f576SJeff Roberson uma_cache_t cache; 4033376b1ba3SJeff Roberson uma_cache_bucket_t bucket; 4034c6fd3e23SJeff Roberson int itemdomain, uz_flags; 40358355f576SJeff Roberson 4036e866d8f0SMark Murray /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 403719fa89e9SMark Murray random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 403810cb2424SMark Murray 4039e63a1c2fSRyan Libby CTR2(KTR_UMA, "uma_zfree_arg zone %s(%p)", zone->uz_name, zone); 40403659f747SRobert Watson 4041d4665eaaSJeff Roberson #ifdef UMA_ZALLOC_DEBUG 4042d4665eaaSJeff Roberson KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, 4043d4665eaaSJeff Roberson ("uma_zfree_arg: called with SMR zone.\n")); 4044d4665eaaSJeff Roberson if (uma_zfree_debug(zone, item, udata) == EJUSTRETURN) 4045d4665eaaSJeff Roberson return; 4046d4665eaaSJeff Roberson #endif 404720ed0cb0SMatthew D Fleming /* uma_zfree(..., NULL) does nothing, to match free(9). */ 404820ed0cb0SMatthew D Fleming if (item == NULL) 404920ed0cb0SMatthew D Fleming return; 4050cc7ce83aSJeff Roberson 4051cc7ce83aSJeff Roberson /* 4052cc7ce83aSJeff Roberson * We are accessing the per-cpu cache without a critical section to 4053cc7ce83aSJeff Roberson * fetch size and flags. This is acceptable, if we are preempted we 4054cc7ce83aSJeff Roberson * will simply read another cpu's line. 4055cc7ce83aSJeff Roberson */ 4056cc7ce83aSJeff Roberson cache = &zone->uz_cpu[curcpu]; 4057cc7ce83aSJeff Roberson uz_flags = cache_uz_flags(cache); 4058d4665eaaSJeff Roberson if (UMA_ALWAYS_CTORDTOR || 4059d4665eaaSJeff Roberson __predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0)) 4060cc7ce83aSJeff Roberson item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE); 4061ef72505eSJeff Roberson 4062af7f9b97SJeff Roberson /* 4063af7f9b97SJeff Roberson * The race here is acceptable. If we miss it we'll just have to wait 4064af7f9b97SJeff Roberson * a little longer for the limits to be reset. 4065af7f9b97SJeff Roberson */ 4066cc7ce83aSJeff Roberson if (__predict_false(uz_flags & UMA_ZFLAG_LIMIT)) { 4067bb15d1c7SGleb Smirnoff if (zone->uz_sleepers > 0) 4068fc03d22bSJeff Roberson goto zfree_item; 4069cc7ce83aSJeff Roberson } 4070af7f9b97SJeff Roberson 40715d1ae027SRobert Watson /* 40725d1ae027SRobert Watson * If possible, free to the per-CPU cache. There are two 40735d1ae027SRobert Watson * requirements for safe access to the per-CPU cache: (1) the thread 40745d1ae027SRobert Watson * accessing the cache must not be preempted or yield during access, 40755d1ae027SRobert Watson * and (2) the thread must not migrate CPUs without switching which 40765d1ae027SRobert Watson * cache it accesses. We rely on a critical section to prevent 40775d1ae027SRobert Watson * preemption and migration. We release the critical section in 40785d1ae027SRobert Watson * order to acquire the zone mutex if we are unable to free to the 40795d1ae027SRobert Watson * current cache; when we re-acquire the critical section, we must 40805d1ae027SRobert Watson * detect and handle migration if it has occurred. 40815d1ae027SRobert Watson */ 4082c6fd3e23SJeff Roberson itemdomain = 0; 4083dfe13344SJeff Roberson #ifdef NUMA 4084dfe13344SJeff Roberson if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) 4085dfe13344SJeff Roberson itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item)); 4086dfe13344SJeff Roberson #endif 40875d1ae027SRobert Watson critical_enter(); 40880a81b439SJeff Roberson do { 4089cc7ce83aSJeff Roberson cache = &zone->uz_cpu[curcpu]; 4090a553d4b8SJeff Roberson /* 4091dfe13344SJeff Roberson * Try to free into the allocbucket first to give LIFO 4092dfe13344SJeff Roberson * ordering for cache-hot datastructures. Spill over 4093dfe13344SJeff Roberson * into the freebucket if necessary. Alloc will swap 4094dfe13344SJeff Roberson * them if one runs dry. 4095a553d4b8SJeff Roberson */ 4096dfe13344SJeff Roberson bucket = &cache->uc_allocbucket; 4097d4665eaaSJeff Roberson #ifdef NUMA 4098d4665eaaSJeff Roberson if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && 4099c6fd3e23SJeff Roberson PCPU_GET(domain) != itemdomain) { 4100d4665eaaSJeff Roberson bucket = &cache->uc_crossbucket; 4101d4665eaaSJeff Roberson } else 4102d4665eaaSJeff Roberson #endif 4103d4665eaaSJeff Roberson if (bucket->ucb_cnt >= bucket->ucb_entries) 4104376b1ba3SJeff Roberson bucket = &cache->uc_freebucket; 4105376b1ba3SJeff Roberson if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { 4106376b1ba3SJeff Roberson cache_bucket_push(cache, bucket, item); 41075d1ae027SRobert Watson critical_exit(); 41088355f576SJeff Roberson return; 4109fc03d22bSJeff Roberson } 41100a81b439SJeff Roberson } while (cache_free(zone, cache, udata, item, itemdomain)); 41110a81b439SJeff Roberson critical_exit(); 4112fc03d22bSJeff Roberson 41138355f576SJeff Roberson /* 41140a81b439SJeff Roberson * If nothing else caught this, we'll just do an internal free. 41158355f576SJeff Roberson */ 41160a81b439SJeff Roberson zfree_item: 41170a81b439SJeff Roberson zone_free_item(zone, item, udata, SKIP_DTOR); 41180a81b439SJeff Roberson } 4119fc03d22bSJeff Roberson 4120dfe13344SJeff Roberson #ifdef NUMA 412191d947bfSJeff Roberson /* 412291d947bfSJeff Roberson * sort crossdomain free buckets to domain correct buckets and cache 412391d947bfSJeff Roberson * them. 412491d947bfSJeff Roberson */ 412591d947bfSJeff Roberson static void 412691d947bfSJeff Roberson zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata) 412791d947bfSJeff Roberson { 412891d947bfSJeff Roberson struct uma_bucketlist fullbuckets; 412991d947bfSJeff Roberson uma_zone_domain_t zdom; 413091d947bfSJeff Roberson uma_bucket_t b; 4131543117beSJeff Roberson smr_seq_t seq; 413291d947bfSJeff Roberson void *item; 413391d947bfSJeff Roberson int domain; 413491d947bfSJeff Roberson 413591d947bfSJeff Roberson CTR3(KTR_UMA, 413691d947bfSJeff Roberson "uma_zfree: zone %s(%p) draining cross bucket %p", 413791d947bfSJeff Roberson zone->uz_name, zone, bucket); 413891d947bfSJeff Roberson 4139543117beSJeff Roberson /* 4140543117beSJeff Roberson * It is possible for buckets to arrive here out of order so we fetch 4141543117beSJeff Roberson * the current smr seq rather than accepting the bucket's. 4142543117beSJeff Roberson */ 4143543117beSJeff Roberson seq = SMR_SEQ_INVALID; 4144543117beSJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) != 0) 4145226dd6dbSJeff Roberson seq = smr_advance(zone->uz_smr); 4146226dd6dbSJeff Roberson 4147226dd6dbSJeff Roberson /* 4148226dd6dbSJeff Roberson * To avoid having ndomain * ndomain buckets for sorting we have a 4149226dd6dbSJeff Roberson * lock on the current crossfree bucket. A full matrix with 4150226dd6dbSJeff Roberson * per-domain locking could be used if necessary. 4151226dd6dbSJeff Roberson */ 4152226dd6dbSJeff Roberson STAILQ_INIT(&fullbuckets); 4153226dd6dbSJeff Roberson ZONE_CROSS_LOCK(zone); 415491d947bfSJeff Roberson while (bucket->ub_cnt > 0) { 415591d947bfSJeff Roberson item = bucket->ub_bucket[bucket->ub_cnt - 1]; 415691d947bfSJeff Roberson domain = _vm_phys_domain(pmap_kextract((vm_offset_t)item)); 4157c6fd3e23SJeff Roberson zdom = ZDOM_GET(zone, domain); 415891d947bfSJeff Roberson if (zdom->uzd_cross == NULL) { 415991d947bfSJeff Roberson zdom->uzd_cross = bucket_alloc(zone, udata, M_NOWAIT); 416091d947bfSJeff Roberson if (zdom->uzd_cross == NULL) 416191d947bfSJeff Roberson break; 416291d947bfSJeff Roberson } 4163543117beSJeff Roberson b = zdom->uzd_cross; 4164543117beSJeff Roberson b->ub_bucket[b->ub_cnt++] = item; 4165543117beSJeff Roberson b->ub_seq = seq; 4166543117beSJeff Roberson if (b->ub_cnt == b->ub_entries) { 4167543117beSJeff Roberson STAILQ_INSERT_HEAD(&fullbuckets, b, ub_link); 416891d947bfSJeff Roberson zdom->uzd_cross = NULL; 416991d947bfSJeff Roberson } 417091d947bfSJeff Roberson bucket->ub_cnt--; 417191d947bfSJeff Roberson } 417291d947bfSJeff Roberson ZONE_CROSS_UNLOCK(zone); 4173c6fd3e23SJeff Roberson if (bucket->ub_cnt == 0) 4174d4665eaaSJeff Roberson bucket->ub_seq = SMR_SEQ_INVALID; 417591d947bfSJeff Roberson bucket_free(zone, bucket, udata); 4176c6fd3e23SJeff Roberson 4177c6fd3e23SJeff Roberson while ((b = STAILQ_FIRST(&fullbuckets)) != NULL) { 4178c6fd3e23SJeff Roberson STAILQ_REMOVE_HEAD(&fullbuckets, ub_link); 4179c6fd3e23SJeff Roberson domain = _vm_phys_domain(pmap_kextract( 4180c6fd3e23SJeff Roberson (vm_offset_t)b->ub_bucket[0])); 4181c6fd3e23SJeff Roberson zone_put_bucket(zone, domain, b, udata, true); 4182c6fd3e23SJeff Roberson } 418391d947bfSJeff Roberson } 418491d947bfSJeff Roberson #endif 418591d947bfSJeff Roberson 41860a81b439SJeff Roberson static void 41870a81b439SJeff Roberson zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, 4188c6fd3e23SJeff Roberson int itemdomain, bool ws) 41890a81b439SJeff Roberson { 41900a81b439SJeff Roberson 4191dfe13344SJeff Roberson #ifdef NUMA 41920a81b439SJeff Roberson /* 41930a81b439SJeff Roberson * Buckets coming from the wrong domain will be entirely for the 41940a81b439SJeff Roberson * only other domain on two domain systems. In this case we can 41950a81b439SJeff Roberson * simply cache them. Otherwise we need to sort them back to 419691d947bfSJeff Roberson * correct domains. 41970a81b439SJeff Roberson */ 4198c6fd3e23SJeff Roberson if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && 4199c6fd3e23SJeff Roberson vm_ndomains > 2 && PCPU_GET(domain) != itemdomain) { 420091d947bfSJeff Roberson zone_free_cross(zone, bucket, udata); 42010a81b439SJeff Roberson return; 42020a81b439SJeff Roberson } 42030a81b439SJeff Roberson #endif 420491d947bfSJeff Roberson 42050a81b439SJeff Roberson /* 42060a81b439SJeff Roberson * Attempt to save the bucket in the zone's domain bucket cache. 42070a81b439SJeff Roberson */ 42080a81b439SJeff Roberson CTR3(KTR_UMA, 42090a81b439SJeff Roberson "uma_zfree: zone %s(%p) putting bucket %p on free list", 42100a81b439SJeff Roberson zone->uz_name, zone, bucket); 42110a81b439SJeff Roberson /* ub_cnt is pointing to the last free item */ 4212c6fd3e23SJeff Roberson if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0) 4213c6fd3e23SJeff Roberson itemdomain = zone_domain_lowest(zone, itemdomain); 4214c6fd3e23SJeff Roberson zone_put_bucket(zone, itemdomain, bucket, udata, ws); 42158355f576SJeff Roberson } 4216fc03d22bSJeff Roberson 42174d104ba0SAlexander Motin /* 42180a81b439SJeff Roberson * Populate a free or cross bucket for the current cpu cache. Free any 42190a81b439SJeff Roberson * existing full bucket either to the zone cache or back to the slab layer. 42200a81b439SJeff Roberson * 42210a81b439SJeff Roberson * Enters and returns in a critical section. false return indicates that 42220a81b439SJeff Roberson * we can not satisfy this free in the cache layer. true indicates that 42230a81b439SJeff Roberson * the caller should retry. 42244d104ba0SAlexander Motin */ 42250a81b439SJeff Roberson static __noinline bool 42260a81b439SJeff Roberson cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, void *item, 42270a81b439SJeff Roberson int itemdomain) 42280a81b439SJeff Roberson { 4229dfe13344SJeff Roberson uma_cache_bucket_t cbucket; 4230d4665eaaSJeff Roberson uma_bucket_t newbucket, bucket; 42310a81b439SJeff Roberson 42320a81b439SJeff Roberson CRITICAL_ASSERT(curthread); 42330a81b439SJeff Roberson 4234d4665eaaSJeff Roberson if (zone->uz_bucket_size == 0) 42350a81b439SJeff Roberson return false; 42360a81b439SJeff Roberson 4237cc7ce83aSJeff Roberson cache = &zone->uz_cpu[curcpu]; 4238d4665eaaSJeff Roberson newbucket = NULL; 42390a81b439SJeff Roberson 42400a81b439SJeff Roberson /* 4241dfe13344SJeff Roberson * FIRSTTOUCH domains need to free to the correct zdom. When 4242dfe13344SJeff Roberson * enabled this is the zdom of the item. The bucket is the 4243dfe13344SJeff Roberson * cross bucket if the current domain and itemdomain do not match. 42440a81b439SJeff Roberson */ 4245dfe13344SJeff Roberson cbucket = &cache->uc_freebucket; 4246dfe13344SJeff Roberson #ifdef NUMA 4247c6fd3e23SJeff Roberson if ((cache_uz_flags(cache) & UMA_ZONE_FIRSTTOUCH) != 0) { 4248c6fd3e23SJeff Roberson if (PCPU_GET(domain) != itemdomain) { 4249dfe13344SJeff Roberson cbucket = &cache->uc_crossbucket; 4250dfe13344SJeff Roberson if (cbucket->ucb_cnt != 0) 4251c6fd3e23SJeff Roberson counter_u64_add(zone->uz_xdomain, 4252dfe13344SJeff Roberson cbucket->ucb_cnt); 4253dfe13344SJeff Roberson } 4254c6fd3e23SJeff Roberson } 42550a81b439SJeff Roberson #endif 4256dfe13344SJeff Roberson bucket = cache_bucket_unload(cbucket); 4257c6fd3e23SJeff Roberson KASSERT(bucket == NULL || bucket->ub_cnt == bucket->ub_entries, 4258c6fd3e23SJeff Roberson ("cache_free: Entered with non-full free bucket.")); 42590a81b439SJeff Roberson 42600a81b439SJeff Roberson /* We are no longer associated with this CPU. */ 42610a81b439SJeff Roberson critical_exit(); 42620a81b439SJeff Roberson 4263d4665eaaSJeff Roberson /* 4264d4665eaaSJeff Roberson * Don't let SMR zones operate without a free bucket. Force 4265d4665eaaSJeff Roberson * a synchronize and re-use this one. We will only degrade 4266d4665eaaSJeff Roberson * to a synchronize every bucket_size items rather than every 4267d4665eaaSJeff Roberson * item if we fail to allocate a bucket. 4268d4665eaaSJeff Roberson */ 4269d4665eaaSJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) != 0) { 4270d4665eaaSJeff Roberson if (bucket != NULL) 4271d4665eaaSJeff Roberson bucket->ub_seq = smr_advance(zone->uz_smr); 4272d4665eaaSJeff Roberson newbucket = bucket_alloc(zone, udata, M_NOWAIT); 4273d4665eaaSJeff Roberson if (newbucket == NULL && bucket != NULL) { 4274d4665eaaSJeff Roberson bucket_drain(zone, bucket); 4275d4665eaaSJeff Roberson newbucket = bucket; 4276d4665eaaSJeff Roberson bucket = NULL; 4277d4665eaaSJeff Roberson } 4278d4665eaaSJeff Roberson } else if (!bucketdisable) 4279d4665eaaSJeff Roberson newbucket = bucket_alloc(zone, udata, M_NOWAIT); 4280d4665eaaSJeff Roberson 42810a81b439SJeff Roberson if (bucket != NULL) 4282c6fd3e23SJeff Roberson zone_free_bucket(zone, bucket, udata, itemdomain, true); 4283a553d4b8SJeff Roberson 4284fc03d22bSJeff Roberson critical_enter(); 4285d4665eaaSJeff Roberson if ((bucket = newbucket) == NULL) 42860a81b439SJeff Roberson return (false); 4287cc7ce83aSJeff Roberson cache = &zone->uz_cpu[curcpu]; 4288dfe13344SJeff Roberson #ifdef NUMA 4289fc03d22bSJeff Roberson /* 42900a81b439SJeff Roberson * Check to see if we should be populating the cross bucket. If it 42910a81b439SJeff Roberson * is already populated we will fall through and attempt to populate 42920a81b439SJeff Roberson * the free bucket. 4293fc03d22bSJeff Roberson */ 4294c6fd3e23SJeff Roberson if ((cache_uz_flags(cache) & UMA_ZONE_FIRSTTOUCH) != 0) { 4295c6fd3e23SJeff Roberson if (PCPU_GET(domain) != itemdomain && 4296376b1ba3SJeff Roberson cache->uc_crossbucket.ucb_bucket == NULL) { 4297376b1ba3SJeff Roberson cache_bucket_load_cross(cache, bucket); 42980a81b439SJeff Roberson return (true); 42990a81b439SJeff Roberson } 43000a81b439SJeff Roberson } 43010a81b439SJeff Roberson #endif 43020a81b439SJeff Roberson /* 43030a81b439SJeff Roberson * We may have lost the race to fill the bucket or switched CPUs. 43040a81b439SJeff Roberson */ 4305376b1ba3SJeff Roberson if (cache->uc_freebucket.ucb_bucket != NULL) { 4306fc03d22bSJeff Roberson critical_exit(); 43076fd34d6fSJeff Roberson bucket_free(zone, bucket, udata); 43080a81b439SJeff Roberson critical_enter(); 43090a81b439SJeff Roberson } else 4310376b1ba3SJeff Roberson cache_bucket_load_free(cache, bucket); 43118355f576SJeff Roberson 43120a81b439SJeff Roberson return (true); 43138355f576SJeff Roberson } 43148355f576SJeff Roberson 4315ab3185d1SJeff Roberson void 4316ab3185d1SJeff Roberson uma_zfree_domain(uma_zone_t zone, void *item, void *udata) 4317ab3185d1SJeff Roberson { 4318ab3185d1SJeff Roberson 4319ab3185d1SJeff Roberson /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 432019fa89e9SMark Murray random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 4321ab3185d1SJeff Roberson 4322e63a1c2fSRyan Libby CTR2(KTR_UMA, "uma_zfree_domain zone %s(%p)", zone->uz_name, zone); 4323ab3185d1SJeff Roberson 4324ab3185d1SJeff Roberson KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), 4325ab3185d1SJeff Roberson ("uma_zfree_domain: called with spinlock or critical section held")); 4326ab3185d1SJeff Roberson 4327ab3185d1SJeff Roberson /* uma_zfree(..., NULL) does nothing, to match free(9). */ 4328ab3185d1SJeff Roberson if (item == NULL) 4329ab3185d1SJeff Roberson return; 4330ab3185d1SJeff Roberson zone_free_item(zone, item, udata, SKIP_NONE); 4331ab3185d1SJeff Roberson } 4332ab3185d1SJeff Roberson 43338355f576SJeff Roberson static void 4334bb15d1c7SGleb Smirnoff slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item) 43358355f576SJeff Roberson { 4336bb15d1c7SGleb Smirnoff uma_keg_t keg; 4337ab3185d1SJeff Roberson uma_domain_t dom; 43389b8db4d0SRyan Libby int freei; 4339099a0e58SBosko Milekic 4340bb15d1c7SGleb Smirnoff keg = zone->uz_keg; 43418b987a77SJeff Roberson KEG_LOCK_ASSERT(keg, slab->us_domain); 4342ab3185d1SJeff Roberson 43438355f576SJeff Roberson /* Do we need to remove from any lists? */ 43448b987a77SJeff Roberson dom = &keg->uk_domain[slab->us_domain]; 4345099a0e58SBosko Milekic if (slab->us_freecount + 1 == keg->uk_ipers) { 43468355f576SJeff Roberson LIST_REMOVE(slab, us_link); 4347ab3185d1SJeff Roberson LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); 43484ab3aee8SMark Johnston dom->ud_free_slabs++; 43498355f576SJeff Roberson } else if (slab->us_freecount == 0) { 43508355f576SJeff Roberson LIST_REMOVE(slab, us_link); 4351ab3185d1SJeff Roberson LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); 43528355f576SJeff Roberson } 43538355f576SJeff Roberson 4354ef72505eSJeff Roberson /* Slab management. */ 43551e0701e1SJeff Roberson freei = slab_item_index(slab, keg, item); 43569b78b1f4SJeff Roberson BIT_SET(keg->uk_ipers, freei, &slab->us_free); 43578355f576SJeff Roberson slab->us_freecount++; 43588355f576SJeff Roberson 4359ef72505eSJeff Roberson /* Keg statistics. */ 43604ab3aee8SMark Johnston dom->ud_free_items++; 43610095a784SJeff Roberson } 43620095a784SJeff Roberson 43630095a784SJeff Roberson static void 4364b75c4efcSAndrew Turner zone_release(void *arg, void **bucket, int cnt) 43650095a784SJeff Roberson { 43668b987a77SJeff Roberson struct mtx *lock; 4367b75c4efcSAndrew Turner uma_zone_t zone; 43680095a784SJeff Roberson uma_slab_t slab; 43690095a784SJeff Roberson uma_keg_t keg; 43700095a784SJeff Roberson uint8_t *mem; 43718b987a77SJeff Roberson void *item; 43720095a784SJeff Roberson int i; 43738355f576SJeff Roberson 4374b75c4efcSAndrew Turner zone = arg; 4375bb15d1c7SGleb Smirnoff keg = zone->uz_keg; 43768b987a77SJeff Roberson lock = NULL; 437754c5ae80SRyan Libby if (__predict_false((zone->uz_flags & UMA_ZFLAG_HASH) != 0)) 43788b987a77SJeff Roberson lock = KEG_LOCK(keg, 0); 43790095a784SJeff Roberson for (i = 0; i < cnt; i++) { 43800095a784SJeff Roberson item = bucket[i]; 438154c5ae80SRyan Libby if (__predict_true((zone->uz_flags & UMA_ZFLAG_VTOSLAB) != 0)) { 43820095a784SJeff Roberson slab = vtoslab((vm_offset_t)item); 43838b987a77SJeff Roberson } else { 43848b987a77SJeff Roberson mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); 438554c5ae80SRyan Libby if ((zone->uz_flags & UMA_ZFLAG_HASH) != 0) 43868b987a77SJeff Roberson slab = hash_sfind(&keg->uk_hash, mem); 43878b987a77SJeff Roberson else 43888b987a77SJeff Roberson slab = (uma_slab_t)(mem + keg->uk_pgoff); 43898b987a77SJeff Roberson } 43908b987a77SJeff Roberson if (lock != KEG_LOCKPTR(keg, slab->us_domain)) { 43918b987a77SJeff Roberson if (lock != NULL) 43928b987a77SJeff Roberson mtx_unlock(lock); 43938b987a77SJeff Roberson lock = KEG_LOCK(keg, slab->us_domain); 43948b987a77SJeff Roberson } 4395bb15d1c7SGleb Smirnoff slab_free_item(zone, slab, item); 43960095a784SJeff Roberson } 43978b987a77SJeff Roberson if (lock != NULL) 43988b987a77SJeff Roberson mtx_unlock(lock); 43998355f576SJeff Roberson } 44008355f576SJeff Roberson 44010095a784SJeff Roberson /* 44020095a784SJeff Roberson * Frees a single item to any zone. 44030095a784SJeff Roberson * 44040095a784SJeff Roberson * Arguments: 44050095a784SJeff Roberson * zone The zone to free to 44060095a784SJeff Roberson * item The item we're freeing 44070095a784SJeff Roberson * udata User supplied data for the dtor 44080095a784SJeff Roberson * skip Skip dtors and finis 44090095a784SJeff Roberson */ 44106d88d784SJeff Roberson static __noinline void 44110095a784SJeff Roberson zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) 44120095a784SJeff Roberson { 4413c5deaf04SGleb Smirnoff 4414d4665eaaSJeff Roberson /* 4415d4665eaaSJeff Roberson * If a free is sent directly to an SMR zone we have to 4416d4665eaaSJeff Roberson * synchronize immediately because the item can instantly 4417d4665eaaSJeff Roberson * be reallocated. This should only happen in degenerate 4418d4665eaaSJeff Roberson * cases when no memory is available for per-cpu caches. 4419d4665eaaSJeff Roberson */ 4420d4665eaaSJeff Roberson if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && skip == SKIP_NONE) 4421d4665eaaSJeff Roberson smr_synchronize(zone->uz_smr); 4422d4665eaaSJeff Roberson 4423cc7ce83aSJeff Roberson item_dtor(zone, item, zone->uz_size, udata, skip); 44240095a784SJeff Roberson 44250095a784SJeff Roberson if (skip < SKIP_FINI && zone->uz_fini) 44260095a784SJeff Roberson zone->uz_fini(item, zone->uz_size); 44270095a784SJeff Roberson 44280095a784SJeff Roberson zone->uz_release(zone->uz_arg, &item, 1); 4429bb15d1c7SGleb Smirnoff 4430bb15d1c7SGleb Smirnoff if (skip & SKIP_CNT) 4431bb15d1c7SGleb Smirnoff return; 4432bb15d1c7SGleb Smirnoff 44332efcc8cbSGleb Smirnoff counter_u64_add(zone->uz_frees, 1); 44342efcc8cbSGleb Smirnoff 44354bd61e19SJeff Roberson if (zone->uz_max_items > 0) 44364bd61e19SJeff Roberson zone_free_limit(zone, 1); 4437bb45b411SGleb Smirnoff } 44380095a784SJeff Roberson 44398355f576SJeff Roberson /* See uma.h */ 44401c6cae97SLawrence Stewart int 4441736ee590SJeff Roberson uma_zone_set_max(uma_zone_t zone, int nitems) 4442736ee590SJeff Roberson { 4443bb15d1c7SGleb Smirnoff struct uma_bucket_zone *ubz; 4444003cf08bSMark Johnston int count; 4445bb15d1c7SGleb Smirnoff 44464bd61e19SJeff Roberson /* 44474bd61e19SJeff Roberson * XXX This can misbehave if the zone has any allocations with 44484bd61e19SJeff Roberson * no limit and a limit is imposed. There is currently no 44494bd61e19SJeff Roberson * way to clear a limit. 44504bd61e19SJeff Roberson */ 4451bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 4452003cf08bSMark Johnston ubz = bucket_zone_max(zone, nitems); 4453003cf08bSMark Johnston count = ubz != NULL ? ubz->ubz_entries : 0; 445420a4e154SJeff Roberson zone->uz_bucket_size_max = zone->uz_bucket_size = count; 445520a4e154SJeff Roberson if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) 445620a4e154SJeff Roberson zone->uz_bucket_size_min = zone->uz_bucket_size_max; 4457bb15d1c7SGleb Smirnoff zone->uz_max_items = nitems; 4458cc7ce83aSJeff Roberson zone->uz_flags |= UMA_ZFLAG_LIMIT; 4459cc7ce83aSJeff Roberson zone_update_caches(zone); 44604bd61e19SJeff Roberson /* We may need to wake waiters. */ 44614bd61e19SJeff Roberson wakeup(&zone->uz_max_items); 4462bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 4463bb15d1c7SGleb Smirnoff 4464bb15d1c7SGleb Smirnoff return (nitems); 4465bb15d1c7SGleb Smirnoff } 4466bb15d1c7SGleb Smirnoff 4467bb15d1c7SGleb Smirnoff /* See uma.h */ 4468003cf08bSMark Johnston void 4469bb15d1c7SGleb Smirnoff uma_zone_set_maxcache(uma_zone_t zone, int nitems) 4470bb15d1c7SGleb Smirnoff { 4471003cf08bSMark Johnston struct uma_bucket_zone *ubz; 4472003cf08bSMark Johnston int bpcpu; 4473bb15d1c7SGleb Smirnoff 4474bb15d1c7SGleb Smirnoff ZONE_LOCK(zone); 4475003cf08bSMark Johnston ubz = bucket_zone_max(zone, nitems); 4476003cf08bSMark Johnston if (ubz != NULL) { 4477003cf08bSMark Johnston bpcpu = 2; 4478dfe13344SJeff Roberson if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) 4479003cf08bSMark Johnston /* Count the cross-domain bucket. */ 4480003cf08bSMark Johnston bpcpu++; 4481003cf08bSMark Johnston nitems -= ubz->ubz_entries * bpcpu * mp_ncpus; 448220a4e154SJeff Roberson zone->uz_bucket_size_max = ubz->ubz_entries; 4483003cf08bSMark Johnston } else { 448420a4e154SJeff Roberson zone->uz_bucket_size_max = zone->uz_bucket_size = 0; 4485003cf08bSMark Johnston } 448620a4e154SJeff Roberson if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) 448720a4e154SJeff Roberson zone->uz_bucket_size_min = zone->uz_bucket_size_max; 4488c6fd3e23SJeff Roberson zone->uz_bucket_max = nitems / vm_ndomains; 4489bb15d1c7SGleb Smirnoff ZONE_UNLOCK(zone); 4490736ee590SJeff Roberson } 4491736ee590SJeff Roberson 4492736ee590SJeff Roberson /* See uma.h */ 4493e49471b0SAndre Oppermann int 4494e49471b0SAndre Oppermann uma_zone_get_max(uma_zone_t zone) 4495e49471b0SAndre Oppermann { 4496e49471b0SAndre Oppermann int nitems; 4497e49471b0SAndre Oppermann 4498727c6918SJeff Roberson nitems = atomic_load_64(&zone->uz_max_items); 4499e49471b0SAndre Oppermann 4500e49471b0SAndre Oppermann return (nitems); 4501e49471b0SAndre Oppermann } 4502e49471b0SAndre Oppermann 4503e49471b0SAndre Oppermann /* See uma.h */ 45042f891cd5SPawel Jakub Dawidek void 45052f891cd5SPawel Jakub Dawidek uma_zone_set_warning(uma_zone_t zone, const char *warning) 45062f891cd5SPawel Jakub Dawidek { 45072f891cd5SPawel Jakub Dawidek 4508727c6918SJeff Roberson ZONE_ASSERT_COLD(zone); 45092f891cd5SPawel Jakub Dawidek zone->uz_warning = warning; 45102f891cd5SPawel Jakub Dawidek } 45112f891cd5SPawel Jakub Dawidek 45122f891cd5SPawel Jakub Dawidek /* See uma.h */ 451354503a13SJonathan T. Looney void 451454503a13SJonathan T. Looney uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction) 451554503a13SJonathan T. Looney { 451654503a13SJonathan T. Looney 4517727c6918SJeff Roberson ZONE_ASSERT_COLD(zone); 4518e60b2fcbSGleb Smirnoff TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone); 451954503a13SJonathan T. Looney } 452054503a13SJonathan T. Looney 452154503a13SJonathan T. Looney /* See uma.h */ 4522c4ae7908SLawrence Stewart int 4523c4ae7908SLawrence Stewart uma_zone_get_cur(uma_zone_t zone) 4524c4ae7908SLawrence Stewart { 4525c4ae7908SLawrence Stewart int64_t nitems; 4526c4ae7908SLawrence Stewart u_int i; 4527c4ae7908SLawrence Stewart 4528bfb6b7a1SJeff Roberson nitems = 0; 4529bfb6b7a1SJeff Roberson if (zone->uz_allocs != EARLY_COUNTER && zone->uz_frees != EARLY_COUNTER) 45302efcc8cbSGleb Smirnoff nitems = counter_u64_fetch(zone->uz_allocs) - 45312efcc8cbSGleb Smirnoff counter_u64_fetch(zone->uz_frees); 4532727c6918SJeff Roberson CPU_FOREACH(i) 4533727c6918SJeff Roberson nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs) - 4534727c6918SJeff Roberson atomic_load_64(&zone->uz_cpu[i].uc_frees); 4535c4ae7908SLawrence Stewart 4536c4ae7908SLawrence Stewart return (nitems < 0 ? 0 : nitems); 4537c4ae7908SLawrence Stewart } 4538c4ae7908SLawrence Stewart 453920a4e154SJeff Roberson static uint64_t 454020a4e154SJeff Roberson uma_zone_get_allocs(uma_zone_t zone) 454120a4e154SJeff Roberson { 454220a4e154SJeff Roberson uint64_t nitems; 454320a4e154SJeff Roberson u_int i; 454420a4e154SJeff Roberson 4545bfb6b7a1SJeff Roberson nitems = 0; 4546bfb6b7a1SJeff Roberson if (zone->uz_allocs != EARLY_COUNTER) 454720a4e154SJeff Roberson nitems = counter_u64_fetch(zone->uz_allocs); 4548727c6918SJeff Roberson CPU_FOREACH(i) 4549727c6918SJeff Roberson nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs); 455020a4e154SJeff Roberson 455120a4e154SJeff Roberson return (nitems); 455220a4e154SJeff Roberson } 455320a4e154SJeff Roberson 455420a4e154SJeff Roberson static uint64_t 455520a4e154SJeff Roberson uma_zone_get_frees(uma_zone_t zone) 455620a4e154SJeff Roberson { 455720a4e154SJeff Roberson uint64_t nitems; 455820a4e154SJeff Roberson u_int i; 455920a4e154SJeff Roberson 4560bfb6b7a1SJeff Roberson nitems = 0; 4561bfb6b7a1SJeff Roberson if (zone->uz_frees != EARLY_COUNTER) 456220a4e154SJeff Roberson nitems = counter_u64_fetch(zone->uz_frees); 4563727c6918SJeff Roberson CPU_FOREACH(i) 4564727c6918SJeff Roberson nitems += atomic_load_64(&zone->uz_cpu[i].uc_frees); 456520a4e154SJeff Roberson 456620a4e154SJeff Roberson return (nitems); 456720a4e154SJeff Roberson } 456820a4e154SJeff Roberson 456931c251a0SJeff Roberson #ifdef INVARIANTS 457031c251a0SJeff Roberson /* Used only for KEG_ASSERT_COLD(). */ 457131c251a0SJeff Roberson static uint64_t 457231c251a0SJeff Roberson uma_keg_get_allocs(uma_keg_t keg) 457331c251a0SJeff Roberson { 457431c251a0SJeff Roberson uma_zone_t z; 457531c251a0SJeff Roberson uint64_t nitems; 457631c251a0SJeff Roberson 457731c251a0SJeff Roberson nitems = 0; 457831c251a0SJeff Roberson LIST_FOREACH(z, &keg->uk_zones, uz_link) 457931c251a0SJeff Roberson nitems += uma_zone_get_allocs(z); 458031c251a0SJeff Roberson 458131c251a0SJeff Roberson return (nitems); 458231c251a0SJeff Roberson } 458331c251a0SJeff Roberson #endif 458431c251a0SJeff Roberson 4585c4ae7908SLawrence Stewart /* See uma.h */ 4586736ee590SJeff Roberson void 4587099a0e58SBosko Milekic uma_zone_set_init(uma_zone_t zone, uma_init uminit) 4588099a0e58SBosko Milekic { 4589e20a199fSJeff Roberson uma_keg_t keg; 4590e20a199fSJeff Roberson 4591bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 4592727c6918SJeff Roberson KEG_ASSERT_COLD(keg); 4593e20a199fSJeff Roberson keg->uk_init = uminit; 4594099a0e58SBosko Milekic } 4595099a0e58SBosko Milekic 4596099a0e58SBosko Milekic /* See uma.h */ 4597099a0e58SBosko Milekic void 4598099a0e58SBosko Milekic uma_zone_set_fini(uma_zone_t zone, uma_fini fini) 4599099a0e58SBosko Milekic { 4600e20a199fSJeff Roberson uma_keg_t keg; 4601e20a199fSJeff Roberson 4602bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 4603727c6918SJeff Roberson KEG_ASSERT_COLD(keg); 4604e20a199fSJeff Roberson keg->uk_fini = fini; 4605099a0e58SBosko Milekic } 4606099a0e58SBosko Milekic 4607099a0e58SBosko Milekic /* See uma.h */ 4608099a0e58SBosko Milekic void 4609099a0e58SBosko Milekic uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) 4610099a0e58SBosko Milekic { 4611af526374SJeff Roberson 4612727c6918SJeff Roberson ZONE_ASSERT_COLD(zone); 4613099a0e58SBosko Milekic zone->uz_init = zinit; 4614099a0e58SBosko Milekic } 4615099a0e58SBosko Milekic 4616099a0e58SBosko Milekic /* See uma.h */ 4617099a0e58SBosko Milekic void 4618099a0e58SBosko Milekic uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) 4619099a0e58SBosko Milekic { 4620af526374SJeff Roberson 4621727c6918SJeff Roberson ZONE_ASSERT_COLD(zone); 4622099a0e58SBosko Milekic zone->uz_fini = zfini; 4623099a0e58SBosko Milekic } 4624099a0e58SBosko Milekic 4625099a0e58SBosko Milekic /* See uma.h */ 4626099a0e58SBosko Milekic void 46278355f576SJeff Roberson uma_zone_set_freef(uma_zone_t zone, uma_free freef) 46288355f576SJeff Roberson { 46290095a784SJeff Roberson uma_keg_t keg; 4630e20a199fSJeff Roberson 4631bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 4632727c6918SJeff Roberson KEG_ASSERT_COLD(keg); 46330095a784SJeff Roberson keg->uk_freef = freef; 46348355f576SJeff Roberson } 46358355f576SJeff Roberson 46368355f576SJeff Roberson /* See uma.h */ 46378355f576SJeff Roberson void 46388355f576SJeff Roberson uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 46398355f576SJeff Roberson { 4640e20a199fSJeff Roberson uma_keg_t keg; 4641e20a199fSJeff Roberson 4642bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 4643727c6918SJeff Roberson KEG_ASSERT_COLD(keg); 4644e20a199fSJeff Roberson keg->uk_allocf = allocf; 46458355f576SJeff Roberson } 46468355f576SJeff Roberson 46478355f576SJeff Roberson /* See uma.h */ 46486fd34d6fSJeff Roberson void 4649d4665eaaSJeff Roberson uma_zone_set_smr(uma_zone_t zone, smr_t smr) 4650d4665eaaSJeff Roberson { 4651d4665eaaSJeff Roberson 4652d4665eaaSJeff Roberson ZONE_ASSERT_COLD(zone); 4653d4665eaaSJeff Roberson 4654d4665eaaSJeff Roberson zone->uz_flags |= UMA_ZONE_SMR; 4655d4665eaaSJeff Roberson zone->uz_smr = smr; 4656d4665eaaSJeff Roberson zone_update_caches(zone); 4657d4665eaaSJeff Roberson } 4658d4665eaaSJeff Roberson 4659d4665eaaSJeff Roberson smr_t 4660d4665eaaSJeff Roberson uma_zone_get_smr(uma_zone_t zone) 4661d4665eaaSJeff Roberson { 4662d4665eaaSJeff Roberson 4663d4665eaaSJeff Roberson return (zone->uz_smr); 4664d4665eaaSJeff Roberson } 4665d4665eaaSJeff Roberson 4666d4665eaaSJeff Roberson /* See uma.h */ 4667d4665eaaSJeff Roberson void 46686fd34d6fSJeff Roberson uma_zone_reserve(uma_zone_t zone, int items) 46696fd34d6fSJeff Roberson { 46706fd34d6fSJeff Roberson uma_keg_t keg; 46716fd34d6fSJeff Roberson 4672bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 4673727c6918SJeff Roberson KEG_ASSERT_COLD(keg); 46746fd34d6fSJeff Roberson keg->uk_reserve = items; 46756fd34d6fSJeff Roberson } 46766fd34d6fSJeff Roberson 46776fd34d6fSJeff Roberson /* See uma.h */ 46788355f576SJeff Roberson int 4679a4915c21SAttilio Rao uma_zone_reserve_kva(uma_zone_t zone, int count) 46808355f576SJeff Roberson { 4681099a0e58SBosko Milekic uma_keg_t keg; 46828355f576SJeff Roberson vm_offset_t kva; 46839ba30bcbSZbigniew Bodek u_int pages; 46848355f576SJeff Roberson 4685bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 4686727c6918SJeff Roberson KEG_ASSERT_COLD(keg); 4687727c6918SJeff Roberson ZONE_ASSERT_COLD(zone); 46888355f576SJeff Roberson 468979c9f942SJeff Roberson pages = howmany(count, keg->uk_ipers) * keg->uk_ppera; 4690a553d4b8SJeff Roberson 4691a4915c21SAttilio Rao #ifdef UMA_MD_SMALL_ALLOC 4692a4915c21SAttilio Rao if (keg->uk_ppera > 1) { 4693a4915c21SAttilio Rao #else 4694a4915c21SAttilio Rao if (1) { 4695a4915c21SAttilio Rao #endif 469657223e99SAndriy Gapon kva = kva_alloc((vm_size_t)pages * PAGE_SIZE); 4697d1f42ac2SAlan Cox if (kva == 0) 46988355f576SJeff Roberson return (0); 4699a4915c21SAttilio Rao } else 4700a4915c21SAttilio Rao kva = 0; 4701bb15d1c7SGleb Smirnoff 4702bb15d1c7SGleb Smirnoff MPASS(keg->uk_kva == 0); 4703099a0e58SBosko Milekic keg->uk_kva = kva; 4704a4915c21SAttilio Rao keg->uk_offset = 0; 4705bb15d1c7SGleb Smirnoff zone->uz_max_items = pages * keg->uk_ipers; 4706a4915c21SAttilio Rao #ifdef UMA_MD_SMALL_ALLOC 4707a4915c21SAttilio Rao keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc; 4708a4915c21SAttilio Rao #else 4709a4915c21SAttilio Rao keg->uk_allocf = noobj_alloc; 4710a4915c21SAttilio Rao #endif 4711cc7ce83aSJeff Roberson keg->uk_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE; 4712cc7ce83aSJeff Roberson zone->uz_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE; 4713cc7ce83aSJeff Roberson zone_update_caches(zone); 4714af526374SJeff Roberson 47158355f576SJeff Roberson return (1); 47168355f576SJeff Roberson } 47178355f576SJeff Roberson 47188355f576SJeff Roberson /* See uma.h */ 47198355f576SJeff Roberson void 47208355f576SJeff Roberson uma_prealloc(uma_zone_t zone, int items) 47218355f576SJeff Roberson { 4722920239efSMark Johnston struct vm_domainset_iter di; 4723ab3185d1SJeff Roberson uma_domain_t dom; 47248355f576SJeff Roberson uma_slab_t slab; 4725099a0e58SBosko Milekic uma_keg_t keg; 472686220393SMark Johnston int aflags, domain, slabs; 47278355f576SJeff Roberson 4728bb15d1c7SGleb Smirnoff KEG_GET(zone, keg); 472979c9f942SJeff Roberson slabs = howmany(items, keg->uk_ipers); 4730194a979eSMark Johnston while (slabs-- > 0) { 473186220393SMark Johnston aflags = M_NOWAIT; 473286220393SMark Johnston vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, 473386220393SMark Johnston &aflags); 473486220393SMark Johnston for (;;) { 473586220393SMark Johnston slab = keg_alloc_slab(keg, zone, domain, M_WAITOK, 473686220393SMark Johnston aflags); 473786220393SMark Johnston if (slab != NULL) { 4738ab3185d1SJeff Roberson dom = &keg->uk_domain[slab->us_domain]; 47394ab3aee8SMark Johnston /* 47404ab3aee8SMark Johnston * keg_alloc_slab() always returns a slab on the 47414ab3aee8SMark Johnston * partial list. 47424ab3aee8SMark Johnston */ 47438b987a77SJeff Roberson LIST_REMOVE(slab, us_link); 474486220393SMark Johnston LIST_INSERT_HEAD(&dom->ud_free_slab, slab, 474586220393SMark Johnston us_link); 47464ab3aee8SMark Johnston dom->ud_free_slabs++; 47478b987a77SJeff Roberson KEG_UNLOCK(keg, slab->us_domain); 4748920239efSMark Johnston break; 47498355f576SJeff Roberson } 47508b987a77SJeff Roberson if (vm_domainset_iter_policy(&di, &domain) != 0) 475186220393SMark Johnston vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask); 475286220393SMark Johnston } 475386220393SMark Johnston } 475486220393SMark Johnston } 47558355f576SJeff Roberson 4756ed581bf6SJeff Roberson /* 4757ed581bf6SJeff Roberson * Returns a snapshot of memory consumption in bytes. 4758ed581bf6SJeff Roberson */ 4759ed581bf6SJeff Roberson size_t 4760ed581bf6SJeff Roberson uma_zone_memory(uma_zone_t zone) 4761ed581bf6SJeff Roberson { 4762ed581bf6SJeff Roberson size_t sz; 4763ed581bf6SJeff Roberson int i; 4764ed581bf6SJeff Roberson 4765ed581bf6SJeff Roberson sz = 0; 4766ed581bf6SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_CACHE) { 4767ed581bf6SJeff Roberson for (i = 0; i < vm_ndomains; i++) 4768c6fd3e23SJeff Roberson sz += ZDOM_GET(zone, i)->uzd_nitems; 4769ed581bf6SJeff Roberson return (sz * zone->uz_size); 4770ed581bf6SJeff Roberson } 4771ed581bf6SJeff Roberson for (i = 0; i < vm_ndomains; i++) 4772ed581bf6SJeff Roberson sz += zone->uz_keg->uk_domain[i].ud_pages; 4773ed581bf6SJeff Roberson 4774ed581bf6SJeff Roberson return (sz * PAGE_SIZE); 4775ed581bf6SJeff Roberson } 4776ed581bf6SJeff Roberson 47778355f576SJeff Roberson /* See uma.h */ 477808cfa56eSMark Johnston void 477908cfa56eSMark Johnston uma_reclaim(int req) 47808355f576SJeff Roberson { 478144ec2b63SKonstantin Belousov 47821431a748SGleb Smirnoff CTR0(KTR_UMA, "UMA: vm asked us to release pages!"); 478308cfa56eSMark Johnston sx_xlock(&uma_reclaim_lock); 478486bbae32SJeff Roberson bucket_enable(); 478508cfa56eSMark Johnston 478608cfa56eSMark Johnston switch (req) { 478708cfa56eSMark Johnston case UMA_RECLAIM_TRIM: 478820a4e154SJeff Roberson zone_foreach(zone_trim, NULL); 478908cfa56eSMark Johnston break; 479008cfa56eSMark Johnston case UMA_RECLAIM_DRAIN: 479108cfa56eSMark Johnston case UMA_RECLAIM_DRAIN_CPU: 479220a4e154SJeff Roberson zone_foreach(zone_drain, NULL); 479308cfa56eSMark Johnston if (req == UMA_RECLAIM_DRAIN_CPU) { 479408cfa56eSMark Johnston pcpu_cache_drain_safe(NULL); 479520a4e154SJeff Roberson zone_foreach(zone_drain, NULL); 4796a2de44abSAlexander Motin } 479708cfa56eSMark Johnston break; 479808cfa56eSMark Johnston default: 479908cfa56eSMark Johnston panic("unhandled reclamation request %d", req); 480008cfa56eSMark Johnston } 48010f9b7bf3SMark Johnston 48028355f576SJeff Roberson /* 48038355f576SJeff Roberson * Some slabs may have been freed but this zone will be visited early 48048355f576SJeff Roberson * we visit again so that we can free pages that are empty once other 48058355f576SJeff Roberson * zones are drained. We have to do the same for buckets. 48068355f576SJeff Roberson */ 48079b8db4d0SRyan Libby zone_drain(slabzones[0], NULL); 48089b8db4d0SRyan Libby zone_drain(slabzones[1], NULL); 4809cae33c14SJeff Roberson bucket_zone_drain(); 481008cfa56eSMark Johnston sx_xunlock(&uma_reclaim_lock); 48118355f576SJeff Roberson } 48128355f576SJeff Roberson 48132e47807cSJeff Roberson static volatile int uma_reclaim_needed; 481444ec2b63SKonstantin Belousov 481544ec2b63SKonstantin Belousov void 481644ec2b63SKonstantin Belousov uma_reclaim_wakeup(void) 481744ec2b63SKonstantin Belousov { 481844ec2b63SKonstantin Belousov 48192e47807cSJeff Roberson if (atomic_fetchadd_int(&uma_reclaim_needed, 1) == 0) 48202e47807cSJeff Roberson wakeup(uma_reclaim); 482144ec2b63SKonstantin Belousov } 482244ec2b63SKonstantin Belousov 482344ec2b63SKonstantin Belousov void 482444ec2b63SKonstantin Belousov uma_reclaim_worker(void *arg __unused) 482544ec2b63SKonstantin Belousov { 482644ec2b63SKonstantin Belousov 482744ec2b63SKonstantin Belousov for (;;) { 482808cfa56eSMark Johnston sx_xlock(&uma_reclaim_lock); 4829200f8117SKonstantin Belousov while (atomic_load_int(&uma_reclaim_needed) == 0) 483008cfa56eSMark Johnston sx_sleep(uma_reclaim, &uma_reclaim_lock, PVM, "umarcl", 48312e47807cSJeff Roberson hz); 483208cfa56eSMark Johnston sx_xunlock(&uma_reclaim_lock); 48339b43bc27SAndriy Gapon EVENTHANDLER_INVOKE(vm_lowmem, VM_LOW_KMEM); 483408cfa56eSMark Johnston uma_reclaim(UMA_RECLAIM_DRAIN_CPU); 4835200f8117SKonstantin Belousov atomic_store_int(&uma_reclaim_needed, 0); 48362e47807cSJeff Roberson /* Don't fire more than once per-second. */ 48372e47807cSJeff Roberson pause("umarclslp", hz); 483844ec2b63SKonstantin Belousov } 483944ec2b63SKonstantin Belousov } 484044ec2b63SKonstantin Belousov 4841663b416fSJohn Baldwin /* See uma.h */ 484208cfa56eSMark Johnston void 484308cfa56eSMark Johnston uma_zone_reclaim(uma_zone_t zone, int req) 484408cfa56eSMark Johnston { 484508cfa56eSMark Johnston 484608cfa56eSMark Johnston switch (req) { 484708cfa56eSMark Johnston case UMA_RECLAIM_TRIM: 484820a4e154SJeff Roberson zone_trim(zone, NULL); 484908cfa56eSMark Johnston break; 485008cfa56eSMark Johnston case UMA_RECLAIM_DRAIN: 485120a4e154SJeff Roberson zone_drain(zone, NULL); 485208cfa56eSMark Johnston break; 485308cfa56eSMark Johnston case UMA_RECLAIM_DRAIN_CPU: 485408cfa56eSMark Johnston pcpu_cache_drain_safe(zone); 485520a4e154SJeff Roberson zone_drain(zone, NULL); 485608cfa56eSMark Johnston break; 485708cfa56eSMark Johnston default: 485808cfa56eSMark Johnston panic("unhandled reclamation request %d", req); 485908cfa56eSMark Johnston } 486008cfa56eSMark Johnston } 486108cfa56eSMark Johnston 486208cfa56eSMark Johnston /* See uma.h */ 4863663b416fSJohn Baldwin int 4864663b416fSJohn Baldwin uma_zone_exhausted(uma_zone_t zone) 4865663b416fSJohn Baldwin { 4866663b416fSJohn Baldwin 4867727c6918SJeff Roberson return (atomic_load_32(&zone->uz_sleepers) > 0); 48686c125b8dSMohan Srinivasan } 48696c125b8dSMohan Srinivasan 48702e47807cSJeff Roberson unsigned long 48712e47807cSJeff Roberson uma_limit(void) 48722e47807cSJeff Roberson { 48732e47807cSJeff Roberson 48742e47807cSJeff Roberson return (uma_kmem_limit); 48752e47807cSJeff Roberson } 48762e47807cSJeff Roberson 48772e47807cSJeff Roberson void 48782e47807cSJeff Roberson uma_set_limit(unsigned long limit) 48792e47807cSJeff Roberson { 48802e47807cSJeff Roberson 48812e47807cSJeff Roberson uma_kmem_limit = limit; 48822e47807cSJeff Roberson } 48832e47807cSJeff Roberson 48842e47807cSJeff Roberson unsigned long 48852e47807cSJeff Roberson uma_size(void) 48862e47807cSJeff Roberson { 48872e47807cSJeff Roberson 4888058f0f74SMark Johnston return (atomic_load_long(&uma_kmem_total)); 4889ad5b0f5bSJeff Roberson } 4890ad5b0f5bSJeff Roberson 4891ad5b0f5bSJeff Roberson long 4892ad5b0f5bSJeff Roberson uma_avail(void) 4893ad5b0f5bSJeff Roberson { 4894ad5b0f5bSJeff Roberson 4895058f0f74SMark Johnston return (uma_kmem_limit - uma_size()); 48962e47807cSJeff Roberson } 48972e47807cSJeff Roberson 4898a0d4b0aeSRobert Watson #ifdef DDB 48998355f576SJeff Roberson /* 49007a52a97eSRobert Watson * Generate statistics across both the zone and its per-cpu cache's. Return 49017a52a97eSRobert Watson * desired statistics if the pointer is non-NULL for that statistic. 49027a52a97eSRobert Watson * 49037a52a97eSRobert Watson * Note: does not update the zone statistics, as it can't safely clear the 49047a52a97eSRobert Watson * per-CPU cache statistic. 49057a52a97eSRobert Watson * 49067a52a97eSRobert Watson */ 49077a52a97eSRobert Watson static void 49080f9b7bf3SMark Johnston uma_zone_sumstat(uma_zone_t z, long *cachefreep, uint64_t *allocsp, 4909c1685086SJeff Roberson uint64_t *freesp, uint64_t *sleepsp, uint64_t *xdomainp) 49107a52a97eSRobert Watson { 49117a52a97eSRobert Watson uma_cache_t cache; 4912c1685086SJeff Roberson uint64_t allocs, frees, sleeps, xdomain; 49137a52a97eSRobert Watson int cachefree, cpu; 49147a52a97eSRobert Watson 4915c1685086SJeff Roberson allocs = frees = sleeps = xdomain = 0; 49167a52a97eSRobert Watson cachefree = 0; 49173aa6d94eSJohn Baldwin CPU_FOREACH(cpu) { 49187a52a97eSRobert Watson cache = &z->uz_cpu[cpu]; 4919376b1ba3SJeff Roberson cachefree += cache->uc_allocbucket.ucb_cnt; 4920376b1ba3SJeff Roberson cachefree += cache->uc_freebucket.ucb_cnt; 4921376b1ba3SJeff Roberson xdomain += cache->uc_crossbucket.ucb_cnt; 4922376b1ba3SJeff Roberson cachefree += cache->uc_crossbucket.ucb_cnt; 49237a52a97eSRobert Watson allocs += cache->uc_allocs; 49247a52a97eSRobert Watson frees += cache->uc_frees; 49257a52a97eSRobert Watson } 49262efcc8cbSGleb Smirnoff allocs += counter_u64_fetch(z->uz_allocs); 49272efcc8cbSGleb Smirnoff frees += counter_u64_fetch(z->uz_frees); 4928c6fd3e23SJeff Roberson xdomain += counter_u64_fetch(z->uz_xdomain); 4929bf965959SSean Bruno sleeps += z->uz_sleeps; 49307a52a97eSRobert Watson if (cachefreep != NULL) 49317a52a97eSRobert Watson *cachefreep = cachefree; 49327a52a97eSRobert Watson if (allocsp != NULL) 49337a52a97eSRobert Watson *allocsp = allocs; 49347a52a97eSRobert Watson if (freesp != NULL) 49357a52a97eSRobert Watson *freesp = frees; 4936bf965959SSean Bruno if (sleepsp != NULL) 4937bf965959SSean Bruno *sleepsp = sleeps; 4938c1685086SJeff Roberson if (xdomainp != NULL) 4939c1685086SJeff Roberson *xdomainp = xdomain; 49407a52a97eSRobert Watson } 4941a0d4b0aeSRobert Watson #endif /* DDB */ 49427a52a97eSRobert Watson 49437a52a97eSRobert Watson static int 49447a52a97eSRobert Watson sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS) 49457a52a97eSRobert Watson { 49467a52a97eSRobert Watson uma_keg_t kz; 49477a52a97eSRobert Watson uma_zone_t z; 49487a52a97eSRobert Watson int count; 49497a52a97eSRobert Watson 49507a52a97eSRobert Watson count = 0; 4951111fbcd5SBryan Venteicher rw_rlock(&uma_rwlock); 49527a52a97eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 49537a52a97eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) 49547a52a97eSRobert Watson count++; 49557a52a97eSRobert Watson } 4956b47acb0aSGleb Smirnoff LIST_FOREACH(z, &uma_cachezones, uz_link) 4957b47acb0aSGleb Smirnoff count++; 4958b47acb0aSGleb Smirnoff 4959111fbcd5SBryan Venteicher rw_runlock(&uma_rwlock); 49607a52a97eSRobert Watson return (sysctl_handle_int(oidp, &count, 0, req)); 49617a52a97eSRobert Watson } 49627a52a97eSRobert Watson 4963b47acb0aSGleb Smirnoff static void 4964b47acb0aSGleb Smirnoff uma_vm_zone_stats(struct uma_type_header *uth, uma_zone_t z, struct sbuf *sbuf, 4965b47acb0aSGleb Smirnoff struct uma_percpu_stat *ups, bool internal) 4966b47acb0aSGleb Smirnoff { 4967b47acb0aSGleb Smirnoff uma_zone_domain_t zdom; 4968b47acb0aSGleb Smirnoff uma_cache_t cache; 4969b47acb0aSGleb Smirnoff int i; 4970b47acb0aSGleb Smirnoff 4971b47acb0aSGleb Smirnoff 4972b47acb0aSGleb Smirnoff for (i = 0; i < vm_ndomains; i++) { 4973c6fd3e23SJeff Roberson zdom = ZDOM_GET(z, i); 4974b47acb0aSGleb Smirnoff uth->uth_zone_free += zdom->uzd_nitems; 4975b47acb0aSGleb Smirnoff } 4976b47acb0aSGleb Smirnoff uth->uth_allocs = counter_u64_fetch(z->uz_allocs); 4977b47acb0aSGleb Smirnoff uth->uth_frees = counter_u64_fetch(z->uz_frees); 4978b47acb0aSGleb Smirnoff uth->uth_fails = counter_u64_fetch(z->uz_fails); 4979c6fd3e23SJeff Roberson uth->uth_xdomain = counter_u64_fetch(z->uz_xdomain); 4980b47acb0aSGleb Smirnoff uth->uth_sleeps = z->uz_sleeps; 49811de9724eSMark Johnston 4982b47acb0aSGleb Smirnoff for (i = 0; i < mp_maxid + 1; i++) { 4983b47acb0aSGleb Smirnoff bzero(&ups[i], sizeof(*ups)); 4984b47acb0aSGleb Smirnoff if (internal || CPU_ABSENT(i)) 4985b47acb0aSGleb Smirnoff continue; 4986b47acb0aSGleb Smirnoff cache = &z->uz_cpu[i]; 4987376b1ba3SJeff Roberson ups[i].ups_cache_free += cache->uc_allocbucket.ucb_cnt; 4988376b1ba3SJeff Roberson ups[i].ups_cache_free += cache->uc_freebucket.ucb_cnt; 4989376b1ba3SJeff Roberson ups[i].ups_cache_free += cache->uc_crossbucket.ucb_cnt; 4990b47acb0aSGleb Smirnoff ups[i].ups_allocs = cache->uc_allocs; 4991b47acb0aSGleb Smirnoff ups[i].ups_frees = cache->uc_frees; 4992b47acb0aSGleb Smirnoff } 4993b47acb0aSGleb Smirnoff } 4994b47acb0aSGleb Smirnoff 49957a52a97eSRobert Watson static int 49967a52a97eSRobert Watson sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) 49977a52a97eSRobert Watson { 49987a52a97eSRobert Watson struct uma_stream_header ush; 49997a52a97eSRobert Watson struct uma_type_header uth; 500063b5d112SKonstantin Belousov struct uma_percpu_stat *ups; 50017a52a97eSRobert Watson struct sbuf sbuf; 50027a52a97eSRobert Watson uma_keg_t kz; 50037a52a97eSRobert Watson uma_zone_t z; 50044bd61e19SJeff Roberson uint64_t items; 50058b987a77SJeff Roberson uint32_t kfree, pages; 50064e657159SMatthew D Fleming int count, error, i; 50077a52a97eSRobert Watson 500800f0e671SMatthew D Fleming error = sysctl_wire_old_buffer(req, 0); 500900f0e671SMatthew D Fleming if (error != 0) 501000f0e671SMatthew D Fleming return (error); 50114e657159SMatthew D Fleming sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 50121eafc078SIan Lepore sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL); 501363b5d112SKonstantin Belousov ups = malloc((mp_maxid + 1) * sizeof(*ups), M_TEMP, M_WAITOK); 50144e657159SMatthew D Fleming 5015404a593eSMatthew D Fleming count = 0; 5016111fbcd5SBryan Venteicher rw_rlock(&uma_rwlock); 50177a52a97eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 50187a52a97eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) 50197a52a97eSRobert Watson count++; 50207a52a97eSRobert Watson } 50217a52a97eSRobert Watson 5022b47acb0aSGleb Smirnoff LIST_FOREACH(z, &uma_cachezones, uz_link) 5023b47acb0aSGleb Smirnoff count++; 5024b47acb0aSGleb Smirnoff 50257a52a97eSRobert Watson /* 50267a52a97eSRobert Watson * Insert stream header. 50277a52a97eSRobert Watson */ 50287a52a97eSRobert Watson bzero(&ush, sizeof(ush)); 50297a52a97eSRobert Watson ush.ush_version = UMA_STREAM_VERSION; 5030ab3a57c0SRobert Watson ush.ush_maxcpus = (mp_maxid + 1); 50317a52a97eSRobert Watson ush.ush_count = count; 50324e657159SMatthew D Fleming (void)sbuf_bcat(&sbuf, &ush, sizeof(ush)); 50337a52a97eSRobert Watson 50347a52a97eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 50358b987a77SJeff Roberson kfree = pages = 0; 50368b987a77SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 50374ab3aee8SMark Johnston kfree += kz->uk_domain[i].ud_free_items; 50388b987a77SJeff Roberson pages += kz->uk_domain[i].ud_pages; 50398b987a77SJeff Roberson } 50407a52a97eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) { 50417a52a97eSRobert Watson bzero(&uth, sizeof(uth)); 5042cbbb4a00SRobert Watson strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); 50437a52a97eSRobert Watson uth.uth_align = kz->uk_align; 50447a52a97eSRobert Watson uth.uth_size = kz->uk_size; 50457a52a97eSRobert Watson uth.uth_rsize = kz->uk_rsize; 50464bd61e19SJeff Roberson if (z->uz_max_items > 0) { 50474bd61e19SJeff Roberson items = UZ_ITEMS_COUNT(z->uz_items); 50484bd61e19SJeff Roberson uth.uth_pages = (items / kz->uk_ipers) * 5049bb15d1c7SGleb Smirnoff kz->uk_ppera; 50504bd61e19SJeff Roberson } else 50518b987a77SJeff Roberson uth.uth_pages = pages; 5052f8c86a5fSGleb Smirnoff uth.uth_maxpages = (z->uz_max_items / kz->uk_ipers) * 5053bb15d1c7SGleb Smirnoff kz->uk_ppera; 5054bb15d1c7SGleb Smirnoff uth.uth_limit = z->uz_max_items; 50558b987a77SJeff Roberson uth.uth_keg_free = kfree; 5056cbbb4a00SRobert Watson 5057cbbb4a00SRobert Watson /* 5058cbbb4a00SRobert Watson * A zone is secondary is it is not the first entry 5059cbbb4a00SRobert Watson * on the keg's zone list. 5060cbbb4a00SRobert Watson */ 5061e20a199fSJeff Roberson if ((z->uz_flags & UMA_ZONE_SECONDARY) && 5062cbbb4a00SRobert Watson (LIST_FIRST(&kz->uk_zones) != z)) 5063cbbb4a00SRobert Watson uth.uth_zone_flags = UTH_ZONE_SECONDARY; 5064b47acb0aSGleb Smirnoff uma_vm_zone_stats(&uth, z, &sbuf, ups, 5065b47acb0aSGleb Smirnoff kz->uk_flags & UMA_ZFLAG_INTERNAL); 506663b5d112SKonstantin Belousov (void)sbuf_bcat(&sbuf, &uth, sizeof(uth)); 506763b5d112SKonstantin Belousov for (i = 0; i < mp_maxid + 1; i++) 506863b5d112SKonstantin Belousov (void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i])); 50697a52a97eSRobert Watson } 50707a52a97eSRobert Watson } 5071b47acb0aSGleb Smirnoff LIST_FOREACH(z, &uma_cachezones, uz_link) { 5072b47acb0aSGleb Smirnoff bzero(&uth, sizeof(uth)); 5073b47acb0aSGleb Smirnoff strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); 5074b47acb0aSGleb Smirnoff uth.uth_size = z->uz_size; 5075b47acb0aSGleb Smirnoff uma_vm_zone_stats(&uth, z, &sbuf, ups, false); 5076b47acb0aSGleb Smirnoff (void)sbuf_bcat(&sbuf, &uth, sizeof(uth)); 5077b47acb0aSGleb Smirnoff for (i = 0; i < mp_maxid + 1; i++) 5078b47acb0aSGleb Smirnoff (void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i])); 5079b47acb0aSGleb Smirnoff } 5080b47acb0aSGleb Smirnoff 5081111fbcd5SBryan Venteicher rw_runlock(&uma_rwlock); 50824e657159SMatthew D Fleming error = sbuf_finish(&sbuf); 50834e657159SMatthew D Fleming sbuf_delete(&sbuf); 508463b5d112SKonstantin Belousov free(ups, M_TEMP); 50857a52a97eSRobert Watson return (error); 50867a52a97eSRobert Watson } 508748c5777eSRobert Watson 50880a5a3ccbSGleb Smirnoff int 50890a5a3ccbSGleb Smirnoff sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS) 50900a5a3ccbSGleb Smirnoff { 50910a5a3ccbSGleb Smirnoff uma_zone_t zone = *(uma_zone_t *)arg1; 509216be9f54SGleb Smirnoff int error, max; 50930a5a3ccbSGleb Smirnoff 509416be9f54SGleb Smirnoff max = uma_zone_get_max(zone); 50950a5a3ccbSGleb Smirnoff error = sysctl_handle_int(oidp, &max, 0, req); 50960a5a3ccbSGleb Smirnoff if (error || !req->newptr) 50970a5a3ccbSGleb Smirnoff return (error); 50980a5a3ccbSGleb Smirnoff 50990a5a3ccbSGleb Smirnoff uma_zone_set_max(zone, max); 51000a5a3ccbSGleb Smirnoff 51010a5a3ccbSGleb Smirnoff return (0); 51020a5a3ccbSGleb Smirnoff } 51030a5a3ccbSGleb Smirnoff 51040a5a3ccbSGleb Smirnoff int 51050a5a3ccbSGleb Smirnoff sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS) 51060a5a3ccbSGleb Smirnoff { 510720a4e154SJeff Roberson uma_zone_t zone; 51080a5a3ccbSGleb Smirnoff int cur; 51090a5a3ccbSGleb Smirnoff 511020a4e154SJeff Roberson /* 511120a4e154SJeff Roberson * Some callers want to add sysctls for global zones that 511220a4e154SJeff Roberson * may not yet exist so they pass a pointer to a pointer. 511320a4e154SJeff Roberson */ 511420a4e154SJeff Roberson if (arg2 == 0) 511520a4e154SJeff Roberson zone = *(uma_zone_t *)arg1; 511620a4e154SJeff Roberson else 511720a4e154SJeff Roberson zone = arg1; 51180a5a3ccbSGleb Smirnoff cur = uma_zone_get_cur(zone); 51190a5a3ccbSGleb Smirnoff return (sysctl_handle_int(oidp, &cur, 0, req)); 51200a5a3ccbSGleb Smirnoff } 51210a5a3ccbSGleb Smirnoff 512220a4e154SJeff Roberson static int 512320a4e154SJeff Roberson sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS) 512420a4e154SJeff Roberson { 512520a4e154SJeff Roberson uma_zone_t zone = arg1; 512620a4e154SJeff Roberson uint64_t cur; 512720a4e154SJeff Roberson 512820a4e154SJeff Roberson cur = uma_zone_get_allocs(zone); 512920a4e154SJeff Roberson return (sysctl_handle_64(oidp, &cur, 0, req)); 513020a4e154SJeff Roberson } 513120a4e154SJeff Roberson 513220a4e154SJeff Roberson static int 513320a4e154SJeff Roberson sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS) 513420a4e154SJeff Roberson { 513520a4e154SJeff Roberson uma_zone_t zone = arg1; 513620a4e154SJeff Roberson uint64_t cur; 513720a4e154SJeff Roberson 513820a4e154SJeff Roberson cur = uma_zone_get_frees(zone); 513920a4e154SJeff Roberson return (sysctl_handle_64(oidp, &cur, 0, req)); 514020a4e154SJeff Roberson } 514120a4e154SJeff Roberson 51426d204a6aSRyan Libby static int 51436d204a6aSRyan Libby sysctl_handle_uma_zone_flags(SYSCTL_HANDLER_ARGS) 51446d204a6aSRyan Libby { 51456d204a6aSRyan Libby struct sbuf sbuf; 51466d204a6aSRyan Libby uma_zone_t zone = arg1; 51476d204a6aSRyan Libby int error; 51486d204a6aSRyan Libby 51496d204a6aSRyan Libby sbuf_new_for_sysctl(&sbuf, NULL, 0, req); 51506d204a6aSRyan Libby if (zone->uz_flags != 0) 51516d204a6aSRyan Libby sbuf_printf(&sbuf, "0x%b", zone->uz_flags, PRINT_UMA_ZFLAGS); 51526d204a6aSRyan Libby else 51536d204a6aSRyan Libby sbuf_printf(&sbuf, "0"); 51546d204a6aSRyan Libby error = sbuf_finish(&sbuf); 51556d204a6aSRyan Libby sbuf_delete(&sbuf); 51566d204a6aSRyan Libby 51576d204a6aSRyan Libby return (error); 51586d204a6aSRyan Libby } 51596d204a6aSRyan Libby 5160f7af5015SRyan Libby static int 5161f7af5015SRyan Libby sysctl_handle_uma_slab_efficiency(SYSCTL_HANDLER_ARGS) 5162f7af5015SRyan Libby { 5163f7af5015SRyan Libby uma_keg_t keg = arg1; 5164f7af5015SRyan Libby int avail, effpct, total; 5165f7af5015SRyan Libby 5166f7af5015SRyan Libby total = keg->uk_ppera * PAGE_SIZE; 516754c5ae80SRyan Libby if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0) 51689b8db4d0SRyan Libby total += slabzone(keg->uk_ipers)->uz_keg->uk_rsize; 5169f7af5015SRyan Libby /* 5170f7af5015SRyan Libby * We consider the client's requested size and alignment here, not the 5171f7af5015SRyan Libby * real size determination uk_rsize, because we also adjust the real 5172f7af5015SRyan Libby * size for internal implementation reasons (max bitset size). 5173f7af5015SRyan Libby */ 5174f7af5015SRyan Libby avail = keg->uk_ipers * roundup2(keg->uk_size, keg->uk_align + 1); 5175f7af5015SRyan Libby if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) 5176f7af5015SRyan Libby avail *= mp_maxid + 1; 5177f7af5015SRyan Libby effpct = 100 * avail / total; 5178f7af5015SRyan Libby return (sysctl_handle_int(oidp, &effpct, 0, req)); 5179f7af5015SRyan Libby } 5180f7af5015SRyan Libby 51814bd61e19SJeff Roberson static int 51824bd61e19SJeff Roberson sysctl_handle_uma_zone_items(SYSCTL_HANDLER_ARGS) 51834bd61e19SJeff Roberson { 51844bd61e19SJeff Roberson uma_zone_t zone = arg1; 51854bd61e19SJeff Roberson uint64_t cur; 51864bd61e19SJeff Roberson 51874bd61e19SJeff Roberson cur = UZ_ITEMS_COUNT(atomic_load_64(&zone->uz_items)); 51884bd61e19SJeff Roberson return (sysctl_handle_64(oidp, &cur, 0, req)); 51894bd61e19SJeff Roberson } 51904bd61e19SJeff Roberson 51919542ea7bSGleb Smirnoff #ifdef INVARIANTS 51929542ea7bSGleb Smirnoff static uma_slab_t 51939542ea7bSGleb Smirnoff uma_dbg_getslab(uma_zone_t zone, void *item) 51949542ea7bSGleb Smirnoff { 51959542ea7bSGleb Smirnoff uma_slab_t slab; 51969542ea7bSGleb Smirnoff uma_keg_t keg; 51979542ea7bSGleb Smirnoff uint8_t *mem; 51989542ea7bSGleb Smirnoff 51999542ea7bSGleb Smirnoff /* 52009542ea7bSGleb Smirnoff * It is safe to return the slab here even though the 52019542ea7bSGleb Smirnoff * zone is unlocked because the item's allocation state 52029542ea7bSGleb Smirnoff * essentially holds a reference. 52039542ea7bSGleb Smirnoff */ 5204727c6918SJeff Roberson mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); 5205727c6918SJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) 5206bb15d1c7SGleb Smirnoff return (NULL); 520754c5ae80SRyan Libby if (zone->uz_flags & UMA_ZFLAG_VTOSLAB) 5208727c6918SJeff Roberson return (vtoslab((vm_offset_t)mem)); 5209bb15d1c7SGleb Smirnoff keg = zone->uz_keg; 521054c5ae80SRyan Libby if ((keg->uk_flags & UMA_ZFLAG_HASH) == 0) 5211727c6918SJeff Roberson return ((uma_slab_t)(mem + keg->uk_pgoff)); 52128b987a77SJeff Roberson KEG_LOCK(keg, 0); 52139542ea7bSGleb Smirnoff slab = hash_sfind(&keg->uk_hash, mem); 52148b987a77SJeff Roberson KEG_UNLOCK(keg, 0); 52159542ea7bSGleb Smirnoff 52169542ea7bSGleb Smirnoff return (slab); 52179542ea7bSGleb Smirnoff } 52189542ea7bSGleb Smirnoff 5219c5deaf04SGleb Smirnoff static bool 5220c5deaf04SGleb Smirnoff uma_dbg_zskip(uma_zone_t zone, void *mem) 5221c5deaf04SGleb Smirnoff { 5222c5deaf04SGleb Smirnoff 5223727c6918SJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) 5224c5deaf04SGleb Smirnoff return (true); 5225c5deaf04SGleb Smirnoff 5226bb15d1c7SGleb Smirnoff return (uma_dbg_kskip(zone->uz_keg, mem)); 5227c5deaf04SGleb Smirnoff } 5228c5deaf04SGleb Smirnoff 5229c5deaf04SGleb Smirnoff static bool 5230c5deaf04SGleb Smirnoff uma_dbg_kskip(uma_keg_t keg, void *mem) 5231c5deaf04SGleb Smirnoff { 5232c5deaf04SGleb Smirnoff uintptr_t idx; 5233c5deaf04SGleb Smirnoff 5234c5deaf04SGleb Smirnoff if (dbg_divisor == 0) 5235c5deaf04SGleb Smirnoff return (true); 5236c5deaf04SGleb Smirnoff 5237c5deaf04SGleb Smirnoff if (dbg_divisor == 1) 5238c5deaf04SGleb Smirnoff return (false); 5239c5deaf04SGleb Smirnoff 5240c5deaf04SGleb Smirnoff idx = (uintptr_t)mem >> PAGE_SHIFT; 5241c5deaf04SGleb Smirnoff if (keg->uk_ipers > 1) { 5242c5deaf04SGleb Smirnoff idx *= keg->uk_ipers; 5243c5deaf04SGleb Smirnoff idx += ((uintptr_t)mem & PAGE_MASK) / keg->uk_rsize; 5244c5deaf04SGleb Smirnoff } 5245c5deaf04SGleb Smirnoff 5246c5deaf04SGleb Smirnoff if ((idx / dbg_divisor) * dbg_divisor != idx) { 5247c5deaf04SGleb Smirnoff counter_u64_add(uma_skip_cnt, 1); 5248c5deaf04SGleb Smirnoff return (true); 5249c5deaf04SGleb Smirnoff } 5250c5deaf04SGleb Smirnoff counter_u64_add(uma_dbg_cnt, 1); 5251c5deaf04SGleb Smirnoff 5252c5deaf04SGleb Smirnoff return (false); 5253c5deaf04SGleb Smirnoff } 5254c5deaf04SGleb Smirnoff 52559542ea7bSGleb Smirnoff /* 52569542ea7bSGleb Smirnoff * Set up the slab's freei data such that uma_dbg_free can function. 52579542ea7bSGleb Smirnoff * 52589542ea7bSGleb Smirnoff */ 52599542ea7bSGleb Smirnoff static void 52609542ea7bSGleb Smirnoff uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item) 52619542ea7bSGleb Smirnoff { 52629542ea7bSGleb Smirnoff uma_keg_t keg; 52639542ea7bSGleb Smirnoff int freei; 52649542ea7bSGleb Smirnoff 52659542ea7bSGleb Smirnoff if (slab == NULL) { 52669542ea7bSGleb Smirnoff slab = uma_dbg_getslab(zone, item); 52679542ea7bSGleb Smirnoff if (slab == NULL) 52689542ea7bSGleb Smirnoff panic("uma: item %p did not belong to zone %s\n", 52699542ea7bSGleb Smirnoff item, zone->uz_name); 52709542ea7bSGleb Smirnoff } 5271584061b4SJeff Roberson keg = zone->uz_keg; 52721e0701e1SJeff Roberson freei = slab_item_index(slab, keg, item); 52739542ea7bSGleb Smirnoff 5274815db204SRyan Libby if (BIT_ISSET(keg->uk_ipers, freei, slab_dbg_bits(slab, keg))) 52759542ea7bSGleb Smirnoff panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)\n", 52769542ea7bSGleb Smirnoff item, zone, zone->uz_name, slab, freei); 5277815db204SRyan Libby BIT_SET_ATOMIC(keg->uk_ipers, freei, slab_dbg_bits(slab, keg)); 52789542ea7bSGleb Smirnoff } 52799542ea7bSGleb Smirnoff 52809542ea7bSGleb Smirnoff /* 52819542ea7bSGleb Smirnoff * Verifies freed addresses. Checks for alignment, valid slab membership 52829542ea7bSGleb Smirnoff * and duplicate frees. 52839542ea7bSGleb Smirnoff * 52849542ea7bSGleb Smirnoff */ 52859542ea7bSGleb Smirnoff static void 52869542ea7bSGleb Smirnoff uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item) 52879542ea7bSGleb Smirnoff { 52889542ea7bSGleb Smirnoff uma_keg_t keg; 52899542ea7bSGleb Smirnoff int freei; 52909542ea7bSGleb Smirnoff 52919542ea7bSGleb Smirnoff if (slab == NULL) { 52929542ea7bSGleb Smirnoff slab = uma_dbg_getslab(zone, item); 52939542ea7bSGleb Smirnoff if (slab == NULL) 52949542ea7bSGleb Smirnoff panic("uma: Freed item %p did not belong to zone %s\n", 52959542ea7bSGleb Smirnoff item, zone->uz_name); 52969542ea7bSGleb Smirnoff } 5297584061b4SJeff Roberson keg = zone->uz_keg; 52981e0701e1SJeff Roberson freei = slab_item_index(slab, keg, item); 52999542ea7bSGleb Smirnoff 53009542ea7bSGleb Smirnoff if (freei >= keg->uk_ipers) 53019542ea7bSGleb Smirnoff panic("Invalid free of %p from zone %p(%s) slab %p(%d)\n", 53029542ea7bSGleb Smirnoff item, zone, zone->uz_name, slab, freei); 53039542ea7bSGleb Smirnoff 53041e0701e1SJeff Roberson if (slab_item(slab, keg, freei) != item) 53059542ea7bSGleb Smirnoff panic("Unaligned free of %p from zone %p(%s) slab %p(%d)\n", 53069542ea7bSGleb Smirnoff item, zone, zone->uz_name, slab, freei); 53079542ea7bSGleb Smirnoff 5308815db204SRyan Libby if (!BIT_ISSET(keg->uk_ipers, freei, slab_dbg_bits(slab, keg))) 53099542ea7bSGleb Smirnoff panic("Duplicate free of %p from zone %p(%s) slab %p(%d)\n", 53109542ea7bSGleb Smirnoff item, zone, zone->uz_name, slab, freei); 53119542ea7bSGleb Smirnoff 5312815db204SRyan Libby BIT_CLR_ATOMIC(keg->uk_ipers, freei, slab_dbg_bits(slab, keg)); 53139542ea7bSGleb Smirnoff } 53149542ea7bSGleb Smirnoff #endif /* INVARIANTS */ 53159542ea7bSGleb Smirnoff 531648c5777eSRobert Watson #ifdef DDB 531746d70077SConrad Meyer static int64_t 531846d70077SConrad Meyer get_uma_stats(uma_keg_t kz, uma_zone_t z, uint64_t *allocs, uint64_t *used, 53190223790fSConrad Meyer uint64_t *sleeps, long *cachefree, uint64_t *xdomain) 532048c5777eSRobert Watson { 532146d70077SConrad Meyer uint64_t frees; 53220f9b7bf3SMark Johnston int i; 532348c5777eSRobert Watson 532448c5777eSRobert Watson if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { 532546d70077SConrad Meyer *allocs = counter_u64_fetch(z->uz_allocs); 53262efcc8cbSGleb Smirnoff frees = counter_u64_fetch(z->uz_frees); 532746d70077SConrad Meyer *sleeps = z->uz_sleeps; 532846d70077SConrad Meyer *cachefree = 0; 532946d70077SConrad Meyer *xdomain = 0; 533048c5777eSRobert Watson } else 533146d70077SConrad Meyer uma_zone_sumstat(z, cachefree, allocs, &frees, sleeps, 533246d70077SConrad Meyer xdomain); 53338b987a77SJeff Roberson for (i = 0; i < vm_ndomains; i++) { 5334c6fd3e23SJeff Roberson *cachefree += ZDOM_GET(z, i)->uzd_nitems; 5335e20a199fSJeff Roberson if (!((z->uz_flags & UMA_ZONE_SECONDARY) && 533648c5777eSRobert Watson (LIST_FIRST(&kz->uk_zones) != z))) 53374ab3aee8SMark Johnston *cachefree += kz->uk_domain[i].ud_free_items; 53388b987a77SJeff Roberson } 533946d70077SConrad Meyer *used = *allocs - frees; 534046d70077SConrad Meyer return (((int64_t)*used + *cachefree) * kz->uk_size); 534146d70077SConrad Meyer } 53420f9b7bf3SMark Johnston 534346d70077SConrad Meyer DB_SHOW_COMMAND(uma, db_show_uma) 534446d70077SConrad Meyer { 534546d70077SConrad Meyer const char *fmt_hdr, *fmt_entry; 534646d70077SConrad Meyer uma_keg_t kz; 534746d70077SConrad Meyer uma_zone_t z; 534846d70077SConrad Meyer uint64_t allocs, used, sleeps, xdomain; 534946d70077SConrad Meyer long cachefree; 535046d70077SConrad Meyer /* variables for sorting */ 535146d70077SConrad Meyer uma_keg_t cur_keg; 535246d70077SConrad Meyer uma_zone_t cur_zone, last_zone; 535346d70077SConrad Meyer int64_t cur_size, last_size, size; 535446d70077SConrad Meyer int ties; 535546d70077SConrad Meyer 535646d70077SConrad Meyer /* /i option produces machine-parseable CSV output */ 535746d70077SConrad Meyer if (modif[0] == 'i') { 535846d70077SConrad Meyer fmt_hdr = "%s,%s,%s,%s,%s,%s,%s,%s,%s\n"; 535946d70077SConrad Meyer fmt_entry = "\"%s\",%ju,%jd,%ld,%ju,%ju,%u,%jd,%ju\n"; 536046d70077SConrad Meyer } else { 536146d70077SConrad Meyer fmt_hdr = "%18s %6s %7s %7s %11s %7s %7s %10s %8s\n"; 536246d70077SConrad Meyer fmt_entry = "%18s %6ju %7jd %7ld %11ju %7ju %7u %10jd %8ju\n"; 536346d70077SConrad Meyer } 536446d70077SConrad Meyer 536546d70077SConrad Meyer db_printf(fmt_hdr, "Zone", "Size", "Used", "Free", "Requests", 536646d70077SConrad Meyer "Sleeps", "Bucket", "Total Mem", "XFree"); 536746d70077SConrad Meyer 536846d70077SConrad Meyer /* Sort the zones with largest size first. */ 536946d70077SConrad Meyer last_zone = NULL; 537046d70077SConrad Meyer last_size = INT64_MAX; 537146d70077SConrad Meyer for (;;) { 537246d70077SConrad Meyer cur_zone = NULL; 537346d70077SConrad Meyer cur_size = -1; 537446d70077SConrad Meyer ties = 0; 537546d70077SConrad Meyer LIST_FOREACH(kz, &uma_kegs, uk_link) { 537646d70077SConrad Meyer LIST_FOREACH(z, &kz->uk_zones, uz_link) { 537746d70077SConrad Meyer /* 537846d70077SConrad Meyer * In the case of size ties, print out zones 537946d70077SConrad Meyer * in the order they are encountered. That is, 538046d70077SConrad Meyer * when we encounter the most recently output 538146d70077SConrad Meyer * zone, we have already printed all preceding 538246d70077SConrad Meyer * ties, and we must print all following ties. 538346d70077SConrad Meyer */ 538446d70077SConrad Meyer if (z == last_zone) { 538546d70077SConrad Meyer ties = 1; 538646d70077SConrad Meyer continue; 538746d70077SConrad Meyer } 538846d70077SConrad Meyer size = get_uma_stats(kz, z, &allocs, &used, 538946d70077SConrad Meyer &sleeps, &cachefree, &xdomain); 539046d70077SConrad Meyer if (size > cur_size && size < last_size + ties) 539146d70077SConrad Meyer { 539246d70077SConrad Meyer cur_size = size; 539346d70077SConrad Meyer cur_zone = z; 539446d70077SConrad Meyer cur_keg = kz; 539546d70077SConrad Meyer } 539646d70077SConrad Meyer } 539746d70077SConrad Meyer } 539846d70077SConrad Meyer if (cur_zone == NULL) 539946d70077SConrad Meyer break; 540046d70077SConrad Meyer 540146d70077SConrad Meyer size = get_uma_stats(cur_keg, cur_zone, &allocs, &used, 540246d70077SConrad Meyer &sleeps, &cachefree, &xdomain); 540346d70077SConrad Meyer db_printf(fmt_entry, cur_zone->uz_name, 540446d70077SConrad Meyer (uintmax_t)cur_keg->uk_size, (intmax_t)used, cachefree, 540546d70077SConrad Meyer (uintmax_t)allocs, (uintmax_t)sleeps, 540620a4e154SJeff Roberson (unsigned)cur_zone->uz_bucket_size, (intmax_t)size, 540720a4e154SJeff Roberson xdomain); 540846d70077SConrad Meyer 5409687c94aaSJohn Baldwin if (db_pager_quit) 5410687c94aaSJohn Baldwin return; 541146d70077SConrad Meyer last_zone = cur_zone; 541246d70077SConrad Meyer last_size = cur_size; 541348c5777eSRobert Watson } 541448c5777eSRobert Watson } 541503175483SAlexander Motin 541603175483SAlexander Motin DB_SHOW_COMMAND(umacache, db_show_umacache) 541703175483SAlexander Motin { 541803175483SAlexander Motin uma_zone_t z; 5419ab3185d1SJeff Roberson uint64_t allocs, frees; 54200f9b7bf3SMark Johnston long cachefree; 54210f9b7bf3SMark Johnston int i; 542203175483SAlexander Motin 542303175483SAlexander Motin db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free", 542403175483SAlexander Motin "Requests", "Bucket"); 542503175483SAlexander Motin LIST_FOREACH(z, &uma_cachezones, uz_link) { 5426c1685086SJeff Roberson uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL, NULL); 54270f9b7bf3SMark Johnston for (i = 0; i < vm_ndomains; i++) 5428c6fd3e23SJeff Roberson cachefree += ZDOM_GET(z, i)->uzd_nitems; 54290f9b7bf3SMark Johnston db_printf("%18s %8ju %8jd %8ld %12ju %8u\n", 543003175483SAlexander Motin z->uz_name, (uintmax_t)z->uz_size, 543103175483SAlexander Motin (intmax_t)(allocs - frees), cachefree, 543220a4e154SJeff Roberson (uintmax_t)allocs, z->uz_bucket_size); 543303175483SAlexander Motin if (db_pager_quit) 543403175483SAlexander Motin return; 543503175483SAlexander Motin } 543603175483SAlexander Motin } 54379542ea7bSGleb Smirnoff #endif /* DDB */ 5438