160727d8bSWarner Losh /*- 2ef72505eSJeff Roberson * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org> 308ecce74SRobert Watson * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 4ae4e9636SRobert Watson * Copyright (c) 2004-2006 Robert N. M. Watson 508ecce74SRobert Watson * All rights reserved. 68355f576SJeff Roberson * 78355f576SJeff Roberson * Redistribution and use in source and binary forms, with or without 88355f576SJeff Roberson * modification, are permitted provided that the following conditions 98355f576SJeff Roberson * are met: 108355f576SJeff Roberson * 1. Redistributions of source code must retain the above copyright 118355f576SJeff Roberson * notice unmodified, this list of conditions, and the following 128355f576SJeff Roberson * disclaimer. 138355f576SJeff Roberson * 2. Redistributions in binary form must reproduce the above copyright 148355f576SJeff Roberson * notice, this list of conditions and the following disclaimer in the 158355f576SJeff Roberson * documentation and/or other materials provided with the distribution. 168355f576SJeff Roberson * 178355f576SJeff Roberson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 188355f576SJeff Roberson * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 198355f576SJeff Roberson * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 208355f576SJeff Roberson * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 218355f576SJeff Roberson * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 228355f576SJeff Roberson * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 238355f576SJeff Roberson * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 248355f576SJeff Roberson * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 258355f576SJeff Roberson * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 268355f576SJeff Roberson * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 278355f576SJeff Roberson */ 288355f576SJeff Roberson 298355f576SJeff Roberson /* 308355f576SJeff Roberson * uma_core.c Implementation of the Universal Memory allocator 318355f576SJeff Roberson * 328355f576SJeff Roberson * This allocator is intended to replace the multitude of similar object caches 338355f576SJeff Roberson * in the standard FreeBSD kernel. The intent is to be flexible as well as 348355f576SJeff Roberson * effecient. A primary design goal is to return unused memory to the rest of 358355f576SJeff Roberson * the system. This will make the system as a whole more flexible due to the 368355f576SJeff Roberson * ability to move memory to subsystems which most need it instead of leaving 378355f576SJeff Roberson * pools of reserved memory unused. 388355f576SJeff Roberson * 398355f576SJeff Roberson * The basic ideas stem from similar slab/zone based allocators whose algorithms 408355f576SJeff Roberson * are well known. 418355f576SJeff Roberson * 428355f576SJeff Roberson */ 438355f576SJeff Roberson 448355f576SJeff Roberson /* 458355f576SJeff Roberson * TODO: 468355f576SJeff Roberson * - Improve memory usage for large allocations 478355f576SJeff Roberson * - Investigate cache size adjustments 488355f576SJeff Roberson */ 498355f576SJeff Roberson 50874651b1SDavid E. O'Brien #include <sys/cdefs.h> 51874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$"); 52874651b1SDavid E. O'Brien 538355f576SJeff Roberson /* I should really use ktr.. */ 548355f576SJeff Roberson /* 558355f576SJeff Roberson #define UMA_DEBUG 1 568355f576SJeff Roberson #define UMA_DEBUG_ALLOC 1 578355f576SJeff Roberson #define UMA_DEBUG_ALLOC_1 1 588355f576SJeff Roberson */ 598355f576SJeff Roberson 6048c5777eSRobert Watson #include "opt_ddb.h" 618355f576SJeff Roberson #include "opt_param.h" 628d689e04SGleb Smirnoff #include "opt_vm.h" 6348c5777eSRobert Watson 648355f576SJeff Roberson #include <sys/param.h> 658355f576SJeff Roberson #include <sys/systm.h> 66ef72505eSJeff Roberson #include <sys/bitset.h> 678355f576SJeff Roberson #include <sys/kernel.h> 688355f576SJeff Roberson #include <sys/types.h> 698355f576SJeff Roberson #include <sys/queue.h> 708355f576SJeff Roberson #include <sys/malloc.h> 713659f747SRobert Watson #include <sys/ktr.h> 728355f576SJeff Roberson #include <sys/lock.h> 738355f576SJeff Roberson #include <sys/sysctl.h> 748355f576SJeff Roberson #include <sys/mutex.h> 754c1cc01cSJohn Baldwin #include <sys/proc.h> 7689f6b863SAttilio Rao #include <sys/rwlock.h> 777a52a97eSRobert Watson #include <sys/sbuf.h> 78a2de44abSAlexander Motin #include <sys/sched.h> 798355f576SJeff Roberson #include <sys/smp.h> 8086bbae32SJeff Roberson #include <sys/vmmeter.h> 8186bbae32SJeff Roberson 828355f576SJeff Roberson #include <vm/vm.h> 838355f576SJeff Roberson #include <vm/vm_object.h> 848355f576SJeff Roberson #include <vm/vm_page.h> 85a4915c21SAttilio Rao #include <vm/vm_pageout.h> 868355f576SJeff Roberson #include <vm/vm_param.h> 878355f576SJeff Roberson #include <vm/vm_map.h> 888355f576SJeff Roberson #include <vm/vm_kern.h> 898355f576SJeff Roberson #include <vm/vm_extern.h> 908355f576SJeff Roberson #include <vm/uma.h> 918355f576SJeff Roberson #include <vm/uma_int.h> 92639c9550SJeff Roberson #include <vm/uma_dbg.h> 938355f576SJeff Roberson 9448c5777eSRobert Watson #include <ddb/ddb.h> 9548c5777eSRobert Watson 968d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD 978d689e04SGleb Smirnoff #include <vm/memguard.h> 988d689e04SGleb Smirnoff #endif 998d689e04SGleb Smirnoff 1008355f576SJeff Roberson /* 101099a0e58SBosko Milekic * This is the zone and keg from which all zones are spawned. The idea is that 102099a0e58SBosko Milekic * even the zone & keg heads are allocated from the allocator, so we use the 103099a0e58SBosko Milekic * bss section to bootstrap us. 1048355f576SJeff Roberson */ 105099a0e58SBosko Milekic static struct uma_keg masterkeg; 106099a0e58SBosko Milekic static struct uma_zone masterzone_k; 107099a0e58SBosko Milekic static struct uma_zone masterzone_z; 108099a0e58SBosko Milekic static uma_zone_t kegs = &masterzone_k; 109099a0e58SBosko Milekic static uma_zone_t zones = &masterzone_z; 1108355f576SJeff Roberson 1118355f576SJeff Roberson /* This is the zone from which all of uma_slab_t's are allocated. */ 1128355f576SJeff Roberson static uma_zone_t slabzone; 113099a0e58SBosko Milekic static uma_zone_t slabrefzone; /* With refcounters (for UMA_ZONE_REFCNT) */ 1148355f576SJeff Roberson 1158355f576SJeff Roberson /* 1168355f576SJeff Roberson * The initial hash tables come out of this zone so they can be allocated 1178355f576SJeff Roberson * prior to malloc coming up. 1188355f576SJeff Roberson */ 1198355f576SJeff Roberson static uma_zone_t hashzone; 1208355f576SJeff Roberson 1211e319f6dSRobert Watson /* The boot-time adjusted value for cache line alignment. */ 122e4cd31ddSJeff Roberson int uma_align_cache = 64 - 1; 1231e319f6dSRobert Watson 124961647dfSJeff Roberson static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets"); 125961647dfSJeff Roberson 1268355f576SJeff Roberson /* 12786bbae32SJeff Roberson * Are we allowed to allocate buckets? 12886bbae32SJeff Roberson */ 12986bbae32SJeff Roberson static int bucketdisable = 1; 13086bbae32SJeff Roberson 131099a0e58SBosko Milekic /* Linked list of all kegs in the system */ 13213e403fdSAntoine Brodin static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs); 1338355f576SJeff Roberson 13403175483SAlexander Motin /* Linked list of all cache-only zones in the system */ 13503175483SAlexander Motin static LIST_HEAD(,uma_zone) uma_cachezones = 13603175483SAlexander Motin LIST_HEAD_INITIALIZER(uma_cachezones); 13703175483SAlexander Motin 138099a0e58SBosko Milekic /* This mutex protects the keg list */ 1390095a784SJeff Roberson static struct mtx_padalign uma_mtx; 1408355f576SJeff Roberson 1418355f576SJeff Roberson /* Linked list of boot time pages */ 1428355f576SJeff Roberson static LIST_HEAD(,uma_slab) uma_boot_pages = 14313e403fdSAntoine Brodin LIST_HEAD_INITIALIZER(uma_boot_pages); 1448355f576SJeff Roberson 145f353d338SAlan Cox /* This mutex protects the boot time pages list */ 1460095a784SJeff Roberson static struct mtx_padalign uma_boot_pages_mtx; 1478355f576SJeff Roberson 1488355f576SJeff Roberson /* Is the VM done starting up? */ 1498355f576SJeff Roberson static int booted = 0; 150342f1793SAlan Cox #define UMA_STARTUP 1 151342f1793SAlan Cox #define UMA_STARTUP2 2 1528355f576SJeff Roberson 153ef72505eSJeff Roberson /* 154ef72505eSJeff Roberson * Only mbuf clusters use ref zones. Just provide enough references 155ef72505eSJeff Roberson * to support the one user. New code should not use the ref facility. 156ef72505eSJeff Roberson */ 157ef72505eSJeff Roberson static const u_int uma_max_ipers_ref = PAGE_SIZE / MCLBYTES; 158244f4554SBosko Milekic 1599643769aSJeff Roberson /* 1609643769aSJeff Roberson * This is the handle used to schedule events that need to happen 1619643769aSJeff Roberson * outside of the allocation fast path. 1629643769aSJeff Roberson */ 1638355f576SJeff Roberson static struct callout uma_callout; 1649643769aSJeff Roberson #define UMA_TIMEOUT 20 /* Seconds for callout interval. */ 1658355f576SJeff Roberson 1668355f576SJeff Roberson /* 1678355f576SJeff Roberson * This structure is passed as the zone ctor arg so that I don't have to create 1688355f576SJeff Roberson * a special allocation function just for zones. 1698355f576SJeff Roberson */ 1708355f576SJeff Roberson struct uma_zctor_args { 171bb196eb4SMatthew D Fleming const char *name; 172c3bdc05fSAndrew R. Reiter size_t size; 1738355f576SJeff Roberson uma_ctor ctor; 1748355f576SJeff Roberson uma_dtor dtor; 1758355f576SJeff Roberson uma_init uminit; 1768355f576SJeff Roberson uma_fini fini; 1770095a784SJeff Roberson uma_import import; 1780095a784SJeff Roberson uma_release release; 1790095a784SJeff Roberson void *arg; 180099a0e58SBosko Milekic uma_keg_t keg; 181099a0e58SBosko Milekic int align; 18285dcf349SGleb Smirnoff uint32_t flags; 183099a0e58SBosko Milekic }; 184099a0e58SBosko Milekic 185099a0e58SBosko Milekic struct uma_kctor_args { 186099a0e58SBosko Milekic uma_zone_t zone; 187099a0e58SBosko Milekic size_t size; 188099a0e58SBosko Milekic uma_init uminit; 189099a0e58SBosko Milekic uma_fini fini; 1908355f576SJeff Roberson int align; 19185dcf349SGleb Smirnoff uint32_t flags; 1928355f576SJeff Roberson }; 1938355f576SJeff Roberson 194cae33c14SJeff Roberson struct uma_bucket_zone { 195cae33c14SJeff Roberson uma_zone_t ubz_zone; 196cae33c14SJeff Roberson char *ubz_name; 197fc03d22bSJeff Roberson int ubz_entries; /* Number of items it can hold. */ 198fc03d22bSJeff Roberson int ubz_maxsize; /* Maximum allocation size per-item. */ 199cae33c14SJeff Roberson }; 200cae33c14SJeff Roberson 201f9d27e75SRobert Watson /* 202fc03d22bSJeff Roberson * Compute the actual number of bucket entries to pack them in power 203fc03d22bSJeff Roberson * of two sizes for more efficient space utilization. 204f9d27e75SRobert Watson */ 205fc03d22bSJeff Roberson #define BUCKET_SIZE(n) \ 206fc03d22bSJeff Roberson (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *)) 207fc03d22bSJeff Roberson 208fc03d22bSJeff Roberson #define BUCKET_MAX BUCKET_SIZE(128) 209fc03d22bSJeff Roberson 210fc03d22bSJeff Roberson struct uma_bucket_zone bucket_zones[] = { 2116fd34d6fSJeff Roberson { NULL, "4 Bucket", BUCKET_SIZE(4), 4096 }, 212f3932e90SAlexander Motin { NULL, "6 Bucket", BUCKET_SIZE(6), 3072 }, 2136fd34d6fSJeff Roberson { NULL, "8 Bucket", BUCKET_SIZE(8), 2048 }, 214f3932e90SAlexander Motin { NULL, "12 Bucket", BUCKET_SIZE(12), 1536 }, 2156fd34d6fSJeff Roberson { NULL, "16 Bucket", BUCKET_SIZE(16), 1024 }, 216fc03d22bSJeff Roberson { NULL, "32 Bucket", BUCKET_SIZE(32), 512 }, 217fc03d22bSJeff Roberson { NULL, "64 Bucket", BUCKET_SIZE(64), 256 }, 218fc03d22bSJeff Roberson { NULL, "128 Bucket", BUCKET_SIZE(128), 128 }, 219fc03d22bSJeff Roberson { NULL, NULL, 0} 220fc03d22bSJeff Roberson }; 221cae33c14SJeff Roberson 2222019094aSRobert Watson /* 2232019094aSRobert Watson * Flags and enumerations to be passed to internal functions. 2242019094aSRobert Watson */ 225ef72505eSJeff Roberson enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI }; 226b23f72e9SBrian Feldman 2278355f576SJeff Roberson /* Prototypes.. */ 2288355f576SJeff Roberson 22985dcf349SGleb Smirnoff static void *noobj_alloc(uma_zone_t, int, uint8_t *, int); 23085dcf349SGleb Smirnoff static void *page_alloc(uma_zone_t, int, uint8_t *, int); 23185dcf349SGleb Smirnoff static void *startup_alloc(uma_zone_t, int, uint8_t *, int); 23285dcf349SGleb Smirnoff static void page_free(void *, int, uint8_t); 233e20a199fSJeff Roberson static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int); 2349643769aSJeff Roberson static void cache_drain(uma_zone_t); 2358355f576SJeff Roberson static void bucket_drain(uma_zone_t, uma_bucket_t); 236aaa8bb16SJeff Roberson static void bucket_cache_drain(uma_zone_t zone); 237b23f72e9SBrian Feldman static int keg_ctor(void *, int, void *, int); 238099a0e58SBosko Milekic static void keg_dtor(void *, int, void *); 239b23f72e9SBrian Feldman static int zone_ctor(void *, int, void *, int); 2409c2cd7e5SJeff Roberson static void zone_dtor(void *, int, void *); 241b23f72e9SBrian Feldman static int zero_init(void *, int, int); 242e20a199fSJeff Roberson static void keg_small_init(uma_keg_t keg); 243e20a199fSJeff Roberson static void keg_large_init(uma_keg_t keg); 2448355f576SJeff Roberson static void zone_foreach(void (*zfunc)(uma_zone_t)); 2458355f576SJeff Roberson static void zone_timeout(uma_zone_t zone); 2460aef6126SJeff Roberson static int hash_alloc(struct uma_hash *); 2470aef6126SJeff Roberson static int hash_expand(struct uma_hash *, struct uma_hash *); 2480aef6126SJeff Roberson static void hash_free(struct uma_hash *hash); 2498355f576SJeff Roberson static void uma_timeout(void *); 2508355f576SJeff Roberson static void uma_startup3(void); 251e20a199fSJeff Roberson static void *zone_alloc_item(uma_zone_t, void *, int); 2520095a784SJeff Roberson static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip); 25386bbae32SJeff Roberson static void bucket_enable(void); 254cae33c14SJeff Roberson static void bucket_init(void); 2556fd34d6fSJeff Roberson static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int); 2566fd34d6fSJeff Roberson static void bucket_free(uma_zone_t zone, uma_bucket_t, void *); 257cae33c14SJeff Roberson static void bucket_zone_drain(void); 2586fd34d6fSJeff Roberson static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, void *, int flags); 259e20a199fSJeff Roberson static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags); 260e20a199fSJeff Roberson static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags); 2610095a784SJeff Roberson static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); 2620095a784SJeff Roberson static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item); 263e20a199fSJeff Roberson static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, 26485dcf349SGleb Smirnoff uma_fini fini, int align, uint32_t flags); 2650095a784SJeff Roberson static int zone_import(uma_zone_t zone, void **bucket, int max, int flags); 2660095a784SJeff Roberson static void zone_release(uma_zone_t zone, void **bucket, int cnt); 26748343a2fSGleb Smirnoff static void uma_zero_item(void *item, uma_zone_t zone); 268bbee39c6SJeff Roberson 2698355f576SJeff Roberson void uma_print_zone(uma_zone_t); 2708355f576SJeff Roberson void uma_print_stats(void); 2717a52a97eSRobert Watson static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); 2727a52a97eSRobert Watson static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); 2738355f576SJeff Roberson 2748355f576SJeff Roberson SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 2758355f576SJeff Roberson 2767a52a97eSRobert Watson SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT, 2777a52a97eSRobert Watson 0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones"); 2787a52a97eSRobert Watson 2797a52a97eSRobert Watson SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT, 2807a52a97eSRobert Watson 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats"); 2817a52a97eSRobert Watson 2822f891cd5SPawel Jakub Dawidek static int zone_warnings = 1; 2832f891cd5SPawel Jakub Dawidek TUNABLE_INT("vm.zone_warnings", &zone_warnings); 2842f891cd5SPawel Jakub Dawidek SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RW, &zone_warnings, 0, 2852f891cd5SPawel Jakub Dawidek "Warn when UMA zones becomes full"); 2862f891cd5SPawel Jakub Dawidek 28786bbae32SJeff Roberson /* 28886bbae32SJeff Roberson * This routine checks to see whether or not it's safe to enable buckets. 28986bbae32SJeff Roberson */ 29086bbae32SJeff Roberson static void 29186bbae32SJeff Roberson bucket_enable(void) 29286bbae32SJeff Roberson { 293251386b4SMaksim Yevmenkin bucketdisable = vm_page_count_min(); 29486bbae32SJeff Roberson } 29586bbae32SJeff Roberson 296dc2c7965SRobert Watson /* 297dc2c7965SRobert Watson * Initialize bucket_zones, the array of zones of buckets of various sizes. 298dc2c7965SRobert Watson * 299dc2c7965SRobert Watson * For each zone, calculate the memory required for each bucket, consisting 300fc03d22bSJeff Roberson * of the header and an array of pointers. 301dc2c7965SRobert Watson */ 302cae33c14SJeff Roberson static void 303cae33c14SJeff Roberson bucket_init(void) 304cae33c14SJeff Roberson { 305cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 306cae33c14SJeff Roberson int size; 307fc03d22bSJeff Roberson int i; 308cae33c14SJeff Roberson 309fc03d22bSJeff Roberson for (i = 0, ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) { 310cae33c14SJeff Roberson size = roundup(sizeof(struct uma_bucket), sizeof(void *)); 311cae33c14SJeff Roberson size += sizeof(void *) * ubz->ubz_entries; 312cae33c14SJeff Roberson ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size, 313e20a199fSJeff Roberson NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 3146fd34d6fSJeff Roberson UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET); 315cae33c14SJeff Roberson } 316cae33c14SJeff Roberson } 317cae33c14SJeff Roberson 318dc2c7965SRobert Watson /* 319dc2c7965SRobert Watson * Given a desired number of entries for a bucket, return the zone from which 320dc2c7965SRobert Watson * to allocate the bucket. 321dc2c7965SRobert Watson */ 322dc2c7965SRobert Watson static struct uma_bucket_zone * 323dc2c7965SRobert Watson bucket_zone_lookup(int entries) 324dc2c7965SRobert Watson { 325fc03d22bSJeff Roberson struct uma_bucket_zone *ubz; 326dc2c7965SRobert Watson 327fc03d22bSJeff Roberson for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 328fc03d22bSJeff Roberson if (ubz->ubz_entries >= entries) 329fc03d22bSJeff Roberson return (ubz); 330fc03d22bSJeff Roberson ubz--; 331fc03d22bSJeff Roberson return (ubz); 332fc03d22bSJeff Roberson } 333fc03d22bSJeff Roberson 334fc03d22bSJeff Roberson static int 335fc03d22bSJeff Roberson bucket_select(int size) 336fc03d22bSJeff Roberson { 337fc03d22bSJeff Roberson struct uma_bucket_zone *ubz; 338fc03d22bSJeff Roberson 339fc03d22bSJeff Roberson ubz = &bucket_zones[0]; 340fc03d22bSJeff Roberson if (size > ubz->ubz_maxsize) 341fc03d22bSJeff Roberson return MAX((ubz->ubz_maxsize * ubz->ubz_entries) / size, 1); 342fc03d22bSJeff Roberson 343fc03d22bSJeff Roberson for (; ubz->ubz_entries != 0; ubz++) 344fc03d22bSJeff Roberson if (ubz->ubz_maxsize < size) 345fc03d22bSJeff Roberson break; 346fc03d22bSJeff Roberson ubz--; 347fc03d22bSJeff Roberson return (ubz->ubz_entries); 348dc2c7965SRobert Watson } 349dc2c7965SRobert Watson 350cae33c14SJeff Roberson static uma_bucket_t 3516fd34d6fSJeff Roberson bucket_alloc(uma_zone_t zone, void *udata, int flags) 352cae33c14SJeff Roberson { 353cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 354cae33c14SJeff Roberson uma_bucket_t bucket; 355cae33c14SJeff Roberson 356cae33c14SJeff Roberson /* 357cae33c14SJeff Roberson * This is to stop us from allocating per cpu buckets while we're 3583803b26bSDag-Erling Smørgrav * running out of vm.boot_pages. Otherwise, we would exhaust the 359cae33c14SJeff Roberson * boot pages. This also prevents us from allocating buckets in 360cae33c14SJeff Roberson * low memory situations. 361cae33c14SJeff Roberson */ 362cae33c14SJeff Roberson if (bucketdisable) 363cae33c14SJeff Roberson return (NULL); 3646fd34d6fSJeff Roberson /* 3656fd34d6fSJeff Roberson * To limit bucket recursion we store the original zone flags 3666fd34d6fSJeff Roberson * in a cookie passed via zalloc_arg/zfree_arg. This allows the 3676fd34d6fSJeff Roberson * NOVM flag to persist even through deep recursions. We also 3686fd34d6fSJeff Roberson * store ZFLAG_BUCKET once we have recursed attempting to allocate 3696fd34d6fSJeff Roberson * a bucket for a bucket zone so we do not allow infinite bucket 3706fd34d6fSJeff Roberson * recursion. This cookie will even persist to frees of unused 3716fd34d6fSJeff Roberson * buckets via the allocation path or bucket allocations in the 3726fd34d6fSJeff Roberson * free path. 3736fd34d6fSJeff Roberson */ 3746fd34d6fSJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) 3756fd34d6fSJeff Roberson udata = (void *)(uintptr_t)zone->uz_flags; 376e8a720feSAlexander Motin else { 377e8a720feSAlexander Motin if ((uintptr_t)udata & UMA_ZFLAG_BUCKET) 378e8a720feSAlexander Motin return (NULL); 3796fd34d6fSJeff Roberson udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET); 380e8a720feSAlexander Motin } 3816fd34d6fSJeff Roberson if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY) 382af526374SJeff Roberson flags |= M_NOVM; 383af526374SJeff Roberson ubz = bucket_zone_lookup(zone->uz_count); 384*20d3ab87SAlexander Motin if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) 385*20d3ab87SAlexander Motin ubz++; 3866fd34d6fSJeff Roberson bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags); 387cae33c14SJeff Roberson if (bucket) { 388cae33c14SJeff Roberson #ifdef INVARIANTS 389cae33c14SJeff Roberson bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); 390cae33c14SJeff Roberson #endif 391cae33c14SJeff Roberson bucket->ub_cnt = 0; 392cae33c14SJeff Roberson bucket->ub_entries = ubz->ubz_entries; 393cae33c14SJeff Roberson } 394cae33c14SJeff Roberson 395cae33c14SJeff Roberson return (bucket); 396cae33c14SJeff Roberson } 397cae33c14SJeff Roberson 398cae33c14SJeff Roberson static void 3996fd34d6fSJeff Roberson bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata) 400cae33c14SJeff Roberson { 401cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 402cae33c14SJeff Roberson 403fc03d22bSJeff Roberson KASSERT(bucket->ub_cnt == 0, 404fc03d22bSJeff Roberson ("bucket_free: Freeing a non free bucket.")); 4056fd34d6fSJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) 4066fd34d6fSJeff Roberson udata = (void *)(uintptr_t)zone->uz_flags; 407dc2c7965SRobert Watson ubz = bucket_zone_lookup(bucket->ub_entries); 4086fd34d6fSJeff Roberson uma_zfree_arg(ubz->ubz_zone, bucket, udata); 409cae33c14SJeff Roberson } 410cae33c14SJeff Roberson 411cae33c14SJeff Roberson static void 412cae33c14SJeff Roberson bucket_zone_drain(void) 413cae33c14SJeff Roberson { 414cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 415cae33c14SJeff Roberson 416cae33c14SJeff Roberson for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 417cae33c14SJeff Roberson zone_drain(ubz->ubz_zone); 418cae33c14SJeff Roberson } 419cae33c14SJeff Roberson 4202f891cd5SPawel Jakub Dawidek static void 4212f891cd5SPawel Jakub Dawidek zone_log_warning(uma_zone_t zone) 4222f891cd5SPawel Jakub Dawidek { 4232f891cd5SPawel Jakub Dawidek static const struct timeval warninterval = { 300, 0 }; 4242f891cd5SPawel Jakub Dawidek 4252f891cd5SPawel Jakub Dawidek if (!zone_warnings || zone->uz_warning == NULL) 4262f891cd5SPawel Jakub Dawidek return; 4272f891cd5SPawel Jakub Dawidek 4282f891cd5SPawel Jakub Dawidek if (ratecheck(&zone->uz_ratecheck, &warninterval)) 4292f891cd5SPawel Jakub Dawidek printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning); 4302f891cd5SPawel Jakub Dawidek } 4312f891cd5SPawel Jakub Dawidek 432e20a199fSJeff Roberson static void 433e20a199fSJeff Roberson zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t)) 434e20a199fSJeff Roberson { 435e20a199fSJeff Roberson uma_klink_t klink; 436e20a199fSJeff Roberson 437e20a199fSJeff Roberson LIST_FOREACH(klink, &zone->uz_kegs, kl_link) 438e20a199fSJeff Roberson kegfn(klink->kl_keg); 439e20a199fSJeff Roberson } 4408355f576SJeff Roberson 4418355f576SJeff Roberson /* 4428355f576SJeff Roberson * Routine called by timeout which is used to fire off some time interval 4439643769aSJeff Roberson * based calculations. (stats, hash size, etc.) 4448355f576SJeff Roberson * 4458355f576SJeff Roberson * Arguments: 4468355f576SJeff Roberson * arg Unused 4478355f576SJeff Roberson * 4488355f576SJeff Roberson * Returns: 4498355f576SJeff Roberson * Nothing 4508355f576SJeff Roberson */ 4518355f576SJeff Roberson static void 4528355f576SJeff Roberson uma_timeout(void *unused) 4538355f576SJeff Roberson { 45486bbae32SJeff Roberson bucket_enable(); 4558355f576SJeff Roberson zone_foreach(zone_timeout); 4568355f576SJeff Roberson 4578355f576SJeff Roberson /* Reschedule this event */ 4589643769aSJeff Roberson callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 4598355f576SJeff Roberson } 4608355f576SJeff Roberson 4618355f576SJeff Roberson /* 4629643769aSJeff Roberson * Routine to perform timeout driven calculations. This expands the 4639643769aSJeff Roberson * hashes and does per cpu statistics aggregation. 4648355f576SJeff Roberson * 465e20a199fSJeff Roberson * Returns nothing. 4668355f576SJeff Roberson */ 4678355f576SJeff Roberson static void 468e20a199fSJeff Roberson keg_timeout(uma_keg_t keg) 4698355f576SJeff Roberson { 4708355f576SJeff Roberson 471e20a199fSJeff Roberson KEG_LOCK(keg); 4728355f576SJeff Roberson /* 473e20a199fSJeff Roberson * Expand the keg hash table. 4748355f576SJeff Roberson * 4758355f576SJeff Roberson * This is done if the number of slabs is larger than the hash size. 4768355f576SJeff Roberson * What I'm trying to do here is completely reduce collisions. This 4778355f576SJeff Roberson * may be a little aggressive. Should I allow for two collisions max? 4788355f576SJeff Roberson */ 479099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_HASH && 480099a0e58SBosko Milekic keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) { 4810aef6126SJeff Roberson struct uma_hash newhash; 4820aef6126SJeff Roberson struct uma_hash oldhash; 4830aef6126SJeff Roberson int ret; 4845300d9ddSJeff Roberson 4850aef6126SJeff Roberson /* 4860aef6126SJeff Roberson * This is so involved because allocating and freeing 487e20a199fSJeff Roberson * while the keg lock is held will lead to deadlock. 4880aef6126SJeff Roberson * I have to do everything in stages and check for 4890aef6126SJeff Roberson * races. 4900aef6126SJeff Roberson */ 491099a0e58SBosko Milekic newhash = keg->uk_hash; 492e20a199fSJeff Roberson KEG_UNLOCK(keg); 4930aef6126SJeff Roberson ret = hash_alloc(&newhash); 494e20a199fSJeff Roberson KEG_LOCK(keg); 4950aef6126SJeff Roberson if (ret) { 496099a0e58SBosko Milekic if (hash_expand(&keg->uk_hash, &newhash)) { 497099a0e58SBosko Milekic oldhash = keg->uk_hash; 498099a0e58SBosko Milekic keg->uk_hash = newhash; 4990aef6126SJeff Roberson } else 5000aef6126SJeff Roberson oldhash = newhash; 5010aef6126SJeff Roberson 502e20a199fSJeff Roberson KEG_UNLOCK(keg); 5030aef6126SJeff Roberson hash_free(&oldhash); 504a1dff920SDavide Italiano return; 5050aef6126SJeff Roberson } 5065300d9ddSJeff Roberson } 507e20a199fSJeff Roberson KEG_UNLOCK(keg); 508e20a199fSJeff Roberson } 509e20a199fSJeff Roberson 510e20a199fSJeff Roberson static void 511e20a199fSJeff Roberson zone_timeout(uma_zone_t zone) 512e20a199fSJeff Roberson { 513e20a199fSJeff Roberson 514e20a199fSJeff Roberson zone_foreach_keg(zone, &keg_timeout); 5158355f576SJeff Roberson } 5168355f576SJeff Roberson 5178355f576SJeff Roberson /* 5185300d9ddSJeff Roberson * Allocate and zero fill the next sized hash table from the appropriate 5195300d9ddSJeff Roberson * backing store. 5205300d9ddSJeff Roberson * 5215300d9ddSJeff Roberson * Arguments: 5220aef6126SJeff Roberson * hash A new hash structure with the old hash size in uh_hashsize 5235300d9ddSJeff Roberson * 5245300d9ddSJeff Roberson * Returns: 5250aef6126SJeff Roberson * 1 on sucess and 0 on failure. 5265300d9ddSJeff Roberson */ 52737c84183SPoul-Henning Kamp static int 5280aef6126SJeff Roberson hash_alloc(struct uma_hash *hash) 5295300d9ddSJeff Roberson { 5300aef6126SJeff Roberson int oldsize; 5315300d9ddSJeff Roberson int alloc; 5325300d9ddSJeff Roberson 5330aef6126SJeff Roberson oldsize = hash->uh_hashsize; 5340aef6126SJeff Roberson 5355300d9ddSJeff Roberson /* We're just going to go to a power of two greater */ 5360aef6126SJeff Roberson if (oldsize) { 5370aef6126SJeff Roberson hash->uh_hashsize = oldsize * 2; 5380aef6126SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; 5390aef6126SJeff Roberson hash->uh_slab_hash = (struct slabhead *)malloc(alloc, 540961647dfSJeff Roberson M_UMAHASH, M_NOWAIT); 5415300d9ddSJeff Roberson } else { 5420aef6126SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; 543e20a199fSJeff Roberson hash->uh_slab_hash = zone_alloc_item(hashzone, NULL, 544a163d034SWarner Losh M_WAITOK); 5450aef6126SJeff Roberson hash->uh_hashsize = UMA_HASH_SIZE_INIT; 5465300d9ddSJeff Roberson } 5470aef6126SJeff Roberson if (hash->uh_slab_hash) { 5480aef6126SJeff Roberson bzero(hash->uh_slab_hash, alloc); 5490aef6126SJeff Roberson hash->uh_hashmask = hash->uh_hashsize - 1; 5500aef6126SJeff Roberson return (1); 5510aef6126SJeff Roberson } 5525300d9ddSJeff Roberson 5530aef6126SJeff Roberson return (0); 5545300d9ddSJeff Roberson } 5555300d9ddSJeff Roberson 5565300d9ddSJeff Roberson /* 55764f051e9SJeff Roberson * Expands the hash table for HASH zones. This is done from zone_timeout 55864f051e9SJeff Roberson * to reduce collisions. This must not be done in the regular allocation 55964f051e9SJeff Roberson * path, otherwise, we can recurse on the vm while allocating pages. 5608355f576SJeff Roberson * 5618355f576SJeff Roberson * Arguments: 5620aef6126SJeff Roberson * oldhash The hash you want to expand 5630aef6126SJeff Roberson * newhash The hash structure for the new table 5648355f576SJeff Roberson * 5658355f576SJeff Roberson * Returns: 5668355f576SJeff Roberson * Nothing 5678355f576SJeff Roberson * 5688355f576SJeff Roberson * Discussion: 5698355f576SJeff Roberson */ 5700aef6126SJeff Roberson static int 5710aef6126SJeff Roberson hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash) 5728355f576SJeff Roberson { 5738355f576SJeff Roberson uma_slab_t slab; 5748355f576SJeff Roberson int hval; 5758355f576SJeff Roberson int i; 5768355f576SJeff Roberson 5770aef6126SJeff Roberson if (!newhash->uh_slab_hash) 5780aef6126SJeff Roberson return (0); 5798355f576SJeff Roberson 5800aef6126SJeff Roberson if (oldhash->uh_hashsize >= newhash->uh_hashsize) 5810aef6126SJeff Roberson return (0); 5828355f576SJeff Roberson 5838355f576SJeff Roberson /* 5848355f576SJeff Roberson * I need to investigate hash algorithms for resizing without a 5858355f576SJeff Roberson * full rehash. 5868355f576SJeff Roberson */ 5878355f576SJeff Roberson 5880aef6126SJeff Roberson for (i = 0; i < oldhash->uh_hashsize; i++) 5890aef6126SJeff Roberson while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) { 5900aef6126SJeff Roberson slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]); 5910aef6126SJeff Roberson SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink); 5920aef6126SJeff Roberson hval = UMA_HASH(newhash, slab->us_data); 5930aef6126SJeff Roberson SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval], 5940aef6126SJeff Roberson slab, us_hlink); 5958355f576SJeff Roberson } 5968355f576SJeff Roberson 5970aef6126SJeff Roberson return (1); 5989c2cd7e5SJeff Roberson } 5999c2cd7e5SJeff Roberson 6005300d9ddSJeff Roberson /* 6015300d9ddSJeff Roberson * Free the hash bucket to the appropriate backing store. 6025300d9ddSJeff Roberson * 6035300d9ddSJeff Roberson * Arguments: 6045300d9ddSJeff Roberson * slab_hash The hash bucket we're freeing 6055300d9ddSJeff Roberson * hashsize The number of entries in that hash bucket 6065300d9ddSJeff Roberson * 6075300d9ddSJeff Roberson * Returns: 6085300d9ddSJeff Roberson * Nothing 6095300d9ddSJeff Roberson */ 6109c2cd7e5SJeff Roberson static void 6110aef6126SJeff Roberson hash_free(struct uma_hash *hash) 6129c2cd7e5SJeff Roberson { 6130aef6126SJeff Roberson if (hash->uh_slab_hash == NULL) 6140aef6126SJeff Roberson return; 6150aef6126SJeff Roberson if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) 6160095a784SJeff Roberson zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE); 6178355f576SJeff Roberson else 618961647dfSJeff Roberson free(hash->uh_slab_hash, M_UMAHASH); 6198355f576SJeff Roberson } 6208355f576SJeff Roberson 6218355f576SJeff Roberson /* 6228355f576SJeff Roberson * Frees all outstanding items in a bucket 6238355f576SJeff Roberson * 6248355f576SJeff Roberson * Arguments: 6258355f576SJeff Roberson * zone The zone to free to, must be unlocked. 6268355f576SJeff Roberson * bucket The free/alloc bucket with items, cpu queue must be locked. 6278355f576SJeff Roberson * 6288355f576SJeff Roberson * Returns: 6298355f576SJeff Roberson * Nothing 6308355f576SJeff Roberson */ 6318355f576SJeff Roberson 6328355f576SJeff Roberson static void 6338355f576SJeff Roberson bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 6348355f576SJeff Roberson { 6350095a784SJeff Roberson int i; 6368355f576SJeff Roberson 6378355f576SJeff Roberson if (bucket == NULL) 6388355f576SJeff Roberson return; 6398355f576SJeff Roberson 6400095a784SJeff Roberson if (zone->uz_fini) 6410095a784SJeff Roberson for (i = 0; i < bucket->ub_cnt; i++) 6420095a784SJeff Roberson zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); 6430095a784SJeff Roberson zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); 6440095a784SJeff Roberson bucket->ub_cnt = 0; 6458355f576SJeff Roberson } 6468355f576SJeff Roberson 6478355f576SJeff Roberson /* 6488355f576SJeff Roberson * Drains the per cpu caches for a zone. 6498355f576SJeff Roberson * 6505d1ae027SRobert Watson * NOTE: This may only be called while the zone is being turn down, and not 6515d1ae027SRobert Watson * during normal operation. This is necessary in order that we do not have 6525d1ae027SRobert Watson * to migrate CPUs to drain the per-CPU caches. 6535d1ae027SRobert Watson * 6548355f576SJeff Roberson * Arguments: 6558355f576SJeff Roberson * zone The zone to drain, must be unlocked. 6568355f576SJeff Roberson * 6578355f576SJeff Roberson * Returns: 6588355f576SJeff Roberson * Nothing 6598355f576SJeff Roberson */ 6608355f576SJeff Roberson static void 6619643769aSJeff Roberson cache_drain(uma_zone_t zone) 6628355f576SJeff Roberson { 6638355f576SJeff Roberson uma_cache_t cache; 6648355f576SJeff Roberson int cpu; 6658355f576SJeff Roberson 6668355f576SJeff Roberson /* 6675d1ae027SRobert Watson * XXX: It is safe to not lock the per-CPU caches, because we're 6685d1ae027SRobert Watson * tearing down the zone anyway. I.e., there will be no further use 6695d1ae027SRobert Watson * of the caches at this point. 6705d1ae027SRobert Watson * 6715d1ae027SRobert Watson * XXX: It would good to be able to assert that the zone is being 6725d1ae027SRobert Watson * torn down to prevent improper use of cache_drain(). 6735d1ae027SRobert Watson * 6745d1ae027SRobert Watson * XXX: We lock the zone before passing into bucket_cache_drain() as 6755d1ae027SRobert Watson * it is used elsewhere. Should the tear-down path be made special 6765d1ae027SRobert Watson * there in some form? 6778355f576SJeff Roberson */ 6783aa6d94eSJohn Baldwin CPU_FOREACH(cpu) { 6798355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 6808355f576SJeff Roberson bucket_drain(zone, cache->uc_allocbucket); 6818355f576SJeff Roberson bucket_drain(zone, cache->uc_freebucket); 682174ab450SBosko Milekic if (cache->uc_allocbucket != NULL) 6836fd34d6fSJeff Roberson bucket_free(zone, cache->uc_allocbucket, NULL); 684174ab450SBosko Milekic if (cache->uc_freebucket != NULL) 6856fd34d6fSJeff Roberson bucket_free(zone, cache->uc_freebucket, NULL); 686d56368d7SBosko Milekic cache->uc_allocbucket = cache->uc_freebucket = NULL; 687d56368d7SBosko Milekic } 688aaa8bb16SJeff Roberson ZONE_LOCK(zone); 689aaa8bb16SJeff Roberson bucket_cache_drain(zone); 690aaa8bb16SJeff Roberson ZONE_UNLOCK(zone); 691aaa8bb16SJeff Roberson } 692aaa8bb16SJeff Roberson 693a2de44abSAlexander Motin static void 694a2de44abSAlexander Motin cache_shrink(uma_zone_t zone) 695a2de44abSAlexander Motin { 696a2de44abSAlexander Motin 697a2de44abSAlexander Motin if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 698a2de44abSAlexander Motin return; 699a2de44abSAlexander Motin 700a2de44abSAlexander Motin ZONE_LOCK(zone); 701a2de44abSAlexander Motin zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2; 702a2de44abSAlexander Motin ZONE_UNLOCK(zone); 703a2de44abSAlexander Motin } 704a2de44abSAlexander Motin 705a2de44abSAlexander Motin static void 706a2de44abSAlexander Motin cache_drain_safe_cpu(uma_zone_t zone) 707a2de44abSAlexander Motin { 708a2de44abSAlexander Motin uma_cache_t cache; 7098a8d9d14SAlexander Motin uma_bucket_t b1, b2; 710a2de44abSAlexander Motin 711a2de44abSAlexander Motin if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 712a2de44abSAlexander Motin return; 713a2de44abSAlexander Motin 7148a8d9d14SAlexander Motin b1 = b2 = NULL; 715a2de44abSAlexander Motin ZONE_LOCK(zone); 716a2de44abSAlexander Motin critical_enter(); 717a2de44abSAlexander Motin cache = &zone->uz_cpu[curcpu]; 718a2de44abSAlexander Motin if (cache->uc_allocbucket) { 7198a8d9d14SAlexander Motin if (cache->uc_allocbucket->ub_cnt != 0) 7208a8d9d14SAlexander Motin LIST_INSERT_HEAD(&zone->uz_buckets, 7218a8d9d14SAlexander Motin cache->uc_allocbucket, ub_link); 7228a8d9d14SAlexander Motin else 7238a8d9d14SAlexander Motin b1 = cache->uc_allocbucket; 724a2de44abSAlexander Motin cache->uc_allocbucket = NULL; 725a2de44abSAlexander Motin } 726a2de44abSAlexander Motin if (cache->uc_freebucket) { 7278a8d9d14SAlexander Motin if (cache->uc_freebucket->ub_cnt != 0) 7288a8d9d14SAlexander Motin LIST_INSERT_HEAD(&zone->uz_buckets, 7298a8d9d14SAlexander Motin cache->uc_freebucket, ub_link); 7308a8d9d14SAlexander Motin else 7318a8d9d14SAlexander Motin b2 = cache->uc_freebucket; 732a2de44abSAlexander Motin cache->uc_freebucket = NULL; 733a2de44abSAlexander Motin } 734a2de44abSAlexander Motin critical_exit(); 735a2de44abSAlexander Motin ZONE_UNLOCK(zone); 7368a8d9d14SAlexander Motin if (b1) 7378a8d9d14SAlexander Motin bucket_free(zone, b1, NULL); 7388a8d9d14SAlexander Motin if (b2) 7398a8d9d14SAlexander Motin bucket_free(zone, b2, NULL); 740a2de44abSAlexander Motin } 741a2de44abSAlexander Motin 742a2de44abSAlexander Motin /* 743a2de44abSAlexander Motin * Safely drain per-CPU caches of a zone(s) to alloc bucket. 744a2de44abSAlexander Motin * This is an expensive call because it needs to bind to all CPUs 745a2de44abSAlexander Motin * one by one and enter a critical section on each of them in order 746a2de44abSAlexander Motin * to safely access their cache buckets. 747a2de44abSAlexander Motin * Zone lock must not be held on call this function. 748a2de44abSAlexander Motin */ 749a2de44abSAlexander Motin static void 750a2de44abSAlexander Motin cache_drain_safe(uma_zone_t zone) 751a2de44abSAlexander Motin { 752a2de44abSAlexander Motin int cpu; 753a2de44abSAlexander Motin 754a2de44abSAlexander Motin /* 755a2de44abSAlexander Motin * Polite bucket sizes shrinking was not enouth, shrink aggressively. 756a2de44abSAlexander Motin */ 757a2de44abSAlexander Motin if (zone) 758a2de44abSAlexander Motin cache_shrink(zone); 759a2de44abSAlexander Motin else 760a2de44abSAlexander Motin zone_foreach(cache_shrink); 761a2de44abSAlexander Motin 762a2de44abSAlexander Motin CPU_FOREACH(cpu) { 763a2de44abSAlexander Motin thread_lock(curthread); 764a2de44abSAlexander Motin sched_bind(curthread, cpu); 765a2de44abSAlexander Motin thread_unlock(curthread); 766a2de44abSAlexander Motin 767a2de44abSAlexander Motin if (zone) 768a2de44abSAlexander Motin cache_drain_safe_cpu(zone); 769a2de44abSAlexander Motin else 770a2de44abSAlexander Motin zone_foreach(cache_drain_safe_cpu); 771a2de44abSAlexander Motin } 772a2de44abSAlexander Motin thread_lock(curthread); 773a2de44abSAlexander Motin sched_unbind(curthread); 774a2de44abSAlexander Motin thread_unlock(curthread); 775a2de44abSAlexander Motin } 776a2de44abSAlexander Motin 777aaa8bb16SJeff Roberson /* 778aaa8bb16SJeff Roberson * Drain the cached buckets from a zone. Expects a locked zone on entry. 779aaa8bb16SJeff Roberson */ 780aaa8bb16SJeff Roberson static void 781aaa8bb16SJeff Roberson bucket_cache_drain(uma_zone_t zone) 782aaa8bb16SJeff Roberson { 783aaa8bb16SJeff Roberson uma_bucket_t bucket; 7848355f576SJeff Roberson 7858355f576SJeff Roberson /* 7868355f576SJeff Roberson * Drain the bucket queues and free the buckets, we just keep two per 7878355f576SJeff Roberson * cpu (alloc/free). 7888355f576SJeff Roberson */ 789fc03d22bSJeff Roberson while ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) { 7908355f576SJeff Roberson LIST_REMOVE(bucket, ub_link); 7918355f576SJeff Roberson ZONE_UNLOCK(zone); 7928355f576SJeff Roberson bucket_drain(zone, bucket); 7936fd34d6fSJeff Roberson bucket_free(zone, bucket, NULL); 7948355f576SJeff Roberson ZONE_LOCK(zone); 7958355f576SJeff Roberson } 796ace66b56SAlexander Motin 797ace66b56SAlexander Motin /* 798ace66b56SAlexander Motin * Shrink further bucket sizes. Price of single zone lock collision 799ace66b56SAlexander Motin * is probably lower then price of global cache drain. 800ace66b56SAlexander Motin */ 801ace66b56SAlexander Motin if (zone->uz_count > zone->uz_count_min) 802ace66b56SAlexander Motin zone->uz_count--; 8038355f576SJeff Roberson } 804fc03d22bSJeff Roberson 805fc03d22bSJeff Roberson static void 806fc03d22bSJeff Roberson keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) 807fc03d22bSJeff Roberson { 808fc03d22bSJeff Roberson uint8_t *mem; 809fc03d22bSJeff Roberson int i; 810fc03d22bSJeff Roberson uint8_t flags; 811fc03d22bSJeff Roberson 812fc03d22bSJeff Roberson mem = slab->us_data; 813fc03d22bSJeff Roberson flags = slab->us_flags; 814fc03d22bSJeff Roberson i = start; 815fc03d22bSJeff Roberson if (keg->uk_fini != NULL) { 816fc03d22bSJeff Roberson for (i--; i > -1; i--) 817fc03d22bSJeff Roberson keg->uk_fini(slab->us_data + (keg->uk_rsize * i), 818fc03d22bSJeff Roberson keg->uk_size); 819fc03d22bSJeff Roberson } 820fc03d22bSJeff Roberson if (keg->uk_flags & UMA_ZONE_OFFPAGE) 821fc03d22bSJeff Roberson zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); 822fc03d22bSJeff Roberson #ifdef UMA_DEBUG 823fc03d22bSJeff Roberson printf("%s: Returning %d bytes.\n", keg->uk_name, 824fc03d22bSJeff Roberson PAGE_SIZE * keg->uk_ppera); 825fc03d22bSJeff Roberson #endif 826fc03d22bSJeff Roberson keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags); 8278355f576SJeff Roberson } 8288355f576SJeff Roberson 8298355f576SJeff Roberson /* 830e20a199fSJeff Roberson * Frees pages from a keg back to the system. This is done on demand from 8318355f576SJeff Roberson * the pageout daemon. 8328355f576SJeff Roberson * 833e20a199fSJeff Roberson * Returns nothing. 8348355f576SJeff Roberson */ 835e20a199fSJeff Roberson static void 836e20a199fSJeff Roberson keg_drain(uma_keg_t keg) 8378355f576SJeff Roberson { 8381e183df2SStefan Farfeleder struct slabhead freeslabs = { 0 }; 8398355f576SJeff Roberson uma_slab_t slab; 8408355f576SJeff Roberson uma_slab_t n; 8418355f576SJeff Roberson 8428355f576SJeff Roberson /* 843e20a199fSJeff Roberson * We don't want to take pages from statically allocated kegs at this 8448355f576SJeff Roberson * time 8458355f576SJeff Roberson */ 846099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) 8478355f576SJeff Roberson return; 8488355f576SJeff Roberson 8498355f576SJeff Roberson #ifdef UMA_DEBUG 850e20a199fSJeff Roberson printf("%s free items: %u\n", keg->uk_name, keg->uk_free); 8518355f576SJeff Roberson #endif 852e20a199fSJeff Roberson KEG_LOCK(keg); 853099a0e58SBosko Milekic if (keg->uk_free == 0) 8548355f576SJeff Roberson goto finished; 8558355f576SJeff Roberson 856099a0e58SBosko Milekic slab = LIST_FIRST(&keg->uk_free_slab); 8579643769aSJeff Roberson while (slab) { 8588355f576SJeff Roberson n = LIST_NEXT(slab, us_link); 8598355f576SJeff Roberson 8608355f576SJeff Roberson /* We have no where to free these to */ 8618355f576SJeff Roberson if (slab->us_flags & UMA_SLAB_BOOT) { 8628355f576SJeff Roberson slab = n; 8638355f576SJeff Roberson continue; 8648355f576SJeff Roberson } 8658355f576SJeff Roberson 8668355f576SJeff Roberson LIST_REMOVE(slab, us_link); 867099a0e58SBosko Milekic keg->uk_pages -= keg->uk_ppera; 868099a0e58SBosko Milekic keg->uk_free -= keg->uk_ipers; 869713deb36SJeff Roberson 870099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_HASH) 871099a0e58SBosko Milekic UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data); 872713deb36SJeff Roberson 873713deb36SJeff Roberson SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink); 874713deb36SJeff Roberson 875713deb36SJeff Roberson slab = n; 876713deb36SJeff Roberson } 877713deb36SJeff Roberson finished: 878e20a199fSJeff Roberson KEG_UNLOCK(keg); 879713deb36SJeff Roberson 880713deb36SJeff Roberson while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { 881713deb36SJeff Roberson SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); 8821645995bSKirk McKusick keg_free_slab(keg, slab, keg->uk_ipers); 8838355f576SJeff Roberson } 8848355f576SJeff Roberson } 8858355f576SJeff Roberson 886e20a199fSJeff Roberson static void 887e20a199fSJeff Roberson zone_drain_wait(uma_zone_t zone, int waitok) 888e20a199fSJeff Roberson { 889e20a199fSJeff Roberson 8908355f576SJeff Roberson /* 891e20a199fSJeff Roberson * Set draining to interlock with zone_dtor() so we can release our 892e20a199fSJeff Roberson * locks as we go. Only dtor() should do a WAITOK call since it 893e20a199fSJeff Roberson * is the only call that knows the structure will still be available 894e20a199fSJeff Roberson * when it wakes up. 895e20a199fSJeff Roberson */ 896e20a199fSJeff Roberson ZONE_LOCK(zone); 897e20a199fSJeff Roberson while (zone->uz_flags & UMA_ZFLAG_DRAINING) { 898e20a199fSJeff Roberson if (waitok == M_NOWAIT) 899e20a199fSJeff Roberson goto out; 900e20a199fSJeff Roberson mtx_unlock(&uma_mtx); 901af526374SJeff Roberson msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1); 902e20a199fSJeff Roberson mtx_lock(&uma_mtx); 903e20a199fSJeff Roberson } 904e20a199fSJeff Roberson zone->uz_flags |= UMA_ZFLAG_DRAINING; 905e20a199fSJeff Roberson bucket_cache_drain(zone); 906e20a199fSJeff Roberson ZONE_UNLOCK(zone); 907e20a199fSJeff Roberson /* 908e20a199fSJeff Roberson * The DRAINING flag protects us from being freed while 909e20a199fSJeff Roberson * we're running. Normally the uma_mtx would protect us but we 910e20a199fSJeff Roberson * must be able to release and acquire the right lock for each keg. 911e20a199fSJeff Roberson */ 912e20a199fSJeff Roberson zone_foreach_keg(zone, &keg_drain); 913e20a199fSJeff Roberson ZONE_LOCK(zone); 914e20a199fSJeff Roberson zone->uz_flags &= ~UMA_ZFLAG_DRAINING; 915e20a199fSJeff Roberson wakeup(zone); 916e20a199fSJeff Roberson out: 917e20a199fSJeff Roberson ZONE_UNLOCK(zone); 918e20a199fSJeff Roberson } 919e20a199fSJeff Roberson 920e20a199fSJeff Roberson void 921e20a199fSJeff Roberson zone_drain(uma_zone_t zone) 922e20a199fSJeff Roberson { 923e20a199fSJeff Roberson 924e20a199fSJeff Roberson zone_drain_wait(zone, M_NOWAIT); 925e20a199fSJeff Roberson } 926e20a199fSJeff Roberson 927e20a199fSJeff Roberson /* 928e20a199fSJeff Roberson * Allocate a new slab for a keg. This does not insert the slab onto a list. 9298355f576SJeff Roberson * 9308355f576SJeff Roberson * Arguments: 9318355f576SJeff Roberson * wait Shall we wait? 9328355f576SJeff Roberson * 9338355f576SJeff Roberson * Returns: 9348355f576SJeff Roberson * The slab that was allocated or NULL if there is no memory and the 9358355f576SJeff Roberson * caller specified M_NOWAIT. 9368355f576SJeff Roberson */ 9378355f576SJeff Roberson static uma_slab_t 938e20a199fSJeff Roberson keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait) 9398355f576SJeff Roberson { 940099a0e58SBosko Milekic uma_slabrefcnt_t slabref; 941e20a199fSJeff Roberson uma_alloc allocf; 942099a0e58SBosko Milekic uma_slab_t slab; 94385dcf349SGleb Smirnoff uint8_t *mem; 94485dcf349SGleb Smirnoff uint8_t flags; 9458355f576SJeff Roberson int i; 9468355f576SJeff Roberson 947e20a199fSJeff Roberson mtx_assert(&keg->uk_lock, MA_OWNED); 948a553d4b8SJeff Roberson slab = NULL; 949fc03d22bSJeff Roberson mem = NULL; 950a553d4b8SJeff Roberson 9518355f576SJeff Roberson #ifdef UMA_DEBUG 9520095a784SJeff Roberson printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name); 9538355f576SJeff Roberson #endif 954e20a199fSJeff Roberson allocf = keg->uk_allocf; 955e20a199fSJeff Roberson KEG_UNLOCK(keg); 956a553d4b8SJeff Roberson 957099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_OFFPAGE) { 958e20a199fSJeff Roberson slab = zone_alloc_item(keg->uk_slabzone, NULL, wait); 959fc03d22bSJeff Roberson if (slab == NULL) 960fc03d22bSJeff Roberson goto out; 961a553d4b8SJeff Roberson } 962a553d4b8SJeff Roberson 9633370c5bfSJeff Roberson /* 9643370c5bfSJeff Roberson * This reproduces the old vm_zone behavior of zero filling pages the 9653370c5bfSJeff Roberson * first time they are added to a zone. 9663370c5bfSJeff Roberson * 9673370c5bfSJeff Roberson * Malloced items are zeroed in uma_zalloc. 9683370c5bfSJeff Roberson */ 9693370c5bfSJeff Roberson 970099a0e58SBosko Milekic if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) 9713370c5bfSJeff Roberson wait |= M_ZERO; 9723370c5bfSJeff Roberson else 9733370c5bfSJeff Roberson wait &= ~M_ZERO; 9743370c5bfSJeff Roberson 975263811f7SKip Macy if (keg->uk_flags & UMA_ZONE_NODUMP) 976263811f7SKip Macy wait |= M_NODUMP; 977263811f7SKip Macy 978e20a199fSJeff Roberson /* zone is passed for legacy reasons. */ 979ad97af7eSGleb Smirnoff mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait); 980a553d4b8SJeff Roberson if (mem == NULL) { 981b23f72e9SBrian Feldman if (keg->uk_flags & UMA_ZONE_OFFPAGE) 9820095a784SJeff Roberson zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); 983fc03d22bSJeff Roberson slab = NULL; 984fc03d22bSJeff Roberson goto out; 985a553d4b8SJeff Roberson } 9868355f576SJeff Roberson 9875c0e403bSJeff Roberson /* Point the slab into the allocated memory */ 988099a0e58SBosko Milekic if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) 989099a0e58SBosko Milekic slab = (uma_slab_t )(mem + keg->uk_pgoff); 9905c0e403bSJeff Roberson 991e20a199fSJeff Roberson if (keg->uk_flags & UMA_ZONE_VTOSLAB) 992099a0e58SBosko Milekic for (i = 0; i < keg->uk_ppera; i++) 99399571dc3SJeff Roberson vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab); 9948355f576SJeff Roberson 995099a0e58SBosko Milekic slab->us_keg = keg; 9968355f576SJeff Roberson slab->us_data = mem; 997099a0e58SBosko Milekic slab->us_freecount = keg->uk_ipers; 9988355f576SJeff Roberson slab->us_flags = flags; 999ef72505eSJeff Roberson BIT_FILL(SLAB_SETSIZE, &slab->us_free); 1000ef72505eSJeff Roberson #ifdef INVARIANTS 1001ef72505eSJeff Roberson BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree); 1002ef72505eSJeff Roberson #endif 1003099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_REFCNT) { 1004099a0e58SBosko Milekic slabref = (uma_slabrefcnt_t)slab; 1005ab14a3f7SBrian Feldman for (i = 0; i < keg->uk_ipers; i++) 1006ef72505eSJeff Roberson slabref->us_refcnt[i] = 0; 1007099a0e58SBosko Milekic } 1008099a0e58SBosko Milekic 1009b23f72e9SBrian Feldman if (keg->uk_init != NULL) { 1010099a0e58SBosko Milekic for (i = 0; i < keg->uk_ipers; i++) 1011b23f72e9SBrian Feldman if (keg->uk_init(slab->us_data + (keg->uk_rsize * i), 1012b23f72e9SBrian Feldman keg->uk_size, wait) != 0) 1013b23f72e9SBrian Feldman break; 1014b23f72e9SBrian Feldman if (i != keg->uk_ipers) { 1015fc03d22bSJeff Roberson keg_free_slab(keg, slab, i); 1016fc03d22bSJeff Roberson slab = NULL; 1017fc03d22bSJeff Roberson goto out; 1018b23f72e9SBrian Feldman } 1019b23f72e9SBrian Feldman } 1020fc03d22bSJeff Roberson out: 1021e20a199fSJeff Roberson KEG_LOCK(keg); 10225c0e403bSJeff Roberson 1023fc03d22bSJeff Roberson if (slab != NULL) { 1024099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_HASH) 1025099a0e58SBosko Milekic UMA_HASH_INSERT(&keg->uk_hash, slab, mem); 10268355f576SJeff Roberson 1027099a0e58SBosko Milekic keg->uk_pages += keg->uk_ppera; 1028099a0e58SBosko Milekic keg->uk_free += keg->uk_ipers; 1029fc03d22bSJeff Roberson } 10308355f576SJeff Roberson 10318355f576SJeff Roberson return (slab); 10328355f576SJeff Roberson } 10338355f576SJeff Roberson 10348355f576SJeff Roberson /* 1035009b6fcbSJeff Roberson * This function is intended to be used early on in place of page_alloc() so 1036009b6fcbSJeff Roberson * that we may use the boot time page cache to satisfy allocations before 1037009b6fcbSJeff Roberson * the VM is ready. 1038009b6fcbSJeff Roberson */ 1039009b6fcbSJeff Roberson static void * 104085dcf349SGleb Smirnoff startup_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait) 1041009b6fcbSJeff Roberson { 1042099a0e58SBosko Milekic uma_keg_t keg; 1043f353d338SAlan Cox uma_slab_t tmps; 1044e9a069d8SJohn Baldwin int pages, check_pages; 1045099a0e58SBosko Milekic 1046e20a199fSJeff Roberson keg = zone_first_keg(zone); 1047e9a069d8SJohn Baldwin pages = howmany(bytes, PAGE_SIZE); 1048e9a069d8SJohn Baldwin check_pages = pages - 1; 1049e9a069d8SJohn Baldwin KASSERT(pages > 0, ("startup_alloc can't reserve 0 pages\n")); 1050099a0e58SBosko Milekic 1051009b6fcbSJeff Roberson /* 1052009b6fcbSJeff Roberson * Check our small startup cache to see if it has pages remaining. 1053009b6fcbSJeff Roberson */ 1054f353d338SAlan Cox mtx_lock(&uma_boot_pages_mtx); 1055e9a069d8SJohn Baldwin 1056e9a069d8SJohn Baldwin /* First check if we have enough room. */ 1057e9a069d8SJohn Baldwin tmps = LIST_FIRST(&uma_boot_pages); 1058e9a069d8SJohn Baldwin while (tmps != NULL && check_pages-- > 0) 1059e9a069d8SJohn Baldwin tmps = LIST_NEXT(tmps, us_link); 1060e9a069d8SJohn Baldwin if (tmps != NULL) { 1061e9a069d8SJohn Baldwin /* 1062e9a069d8SJohn Baldwin * It's ok to lose tmps references. The last one will 1063e9a069d8SJohn Baldwin * have tmps->us_data pointing to the start address of 1064e9a069d8SJohn Baldwin * "pages" contiguous pages of memory. 1065e9a069d8SJohn Baldwin */ 1066e9a069d8SJohn Baldwin while (pages-- > 0) { 1067e9a069d8SJohn Baldwin tmps = LIST_FIRST(&uma_boot_pages); 1068009b6fcbSJeff Roberson LIST_REMOVE(tmps, us_link); 1069e9a069d8SJohn Baldwin } 1070f353d338SAlan Cox mtx_unlock(&uma_boot_pages_mtx); 1071009b6fcbSJeff Roberson *pflag = tmps->us_flags; 1072009b6fcbSJeff Roberson return (tmps->us_data); 1073009b6fcbSJeff Roberson } 1074f353d338SAlan Cox mtx_unlock(&uma_boot_pages_mtx); 1075342f1793SAlan Cox if (booted < UMA_STARTUP2) 10763803b26bSDag-Erling Smørgrav panic("UMA: Increase vm.boot_pages"); 1077009b6fcbSJeff Roberson /* 1078009b6fcbSJeff Roberson * Now that we've booted reset these users to their real allocator. 1079009b6fcbSJeff Roberson */ 1080009b6fcbSJeff Roberson #ifdef UMA_MD_SMALL_ALLOC 1081e9a069d8SJohn Baldwin keg->uk_allocf = (keg->uk_ppera > 1) ? page_alloc : uma_small_alloc; 1082009b6fcbSJeff Roberson #else 1083099a0e58SBosko Milekic keg->uk_allocf = page_alloc; 1084009b6fcbSJeff Roberson #endif 1085099a0e58SBosko Milekic return keg->uk_allocf(zone, bytes, pflag, wait); 1086009b6fcbSJeff Roberson } 1087009b6fcbSJeff Roberson 1088009b6fcbSJeff Roberson /* 10898355f576SJeff Roberson * Allocates a number of pages from the system 10908355f576SJeff Roberson * 10918355f576SJeff Roberson * Arguments: 10928355f576SJeff Roberson * bytes The number of bytes requested 10938355f576SJeff Roberson * wait Shall we wait? 10948355f576SJeff Roberson * 10958355f576SJeff Roberson * Returns: 10968355f576SJeff Roberson * A pointer to the alloced memory or possibly 10978355f576SJeff Roberson * NULL if M_NOWAIT is set. 10988355f576SJeff Roberson */ 10998355f576SJeff Roberson static void * 110085dcf349SGleb Smirnoff page_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait) 11018355f576SJeff Roberson { 11028355f576SJeff Roberson void *p; /* Returned page */ 11038355f576SJeff Roberson 11048355f576SJeff Roberson *pflag = UMA_SLAB_KMEM; 11055df87b21SJeff Roberson p = (void *) kmem_malloc(kmem_arena, bytes, wait); 11068355f576SJeff Roberson 11078355f576SJeff Roberson return (p); 11088355f576SJeff Roberson } 11098355f576SJeff Roberson 11108355f576SJeff Roberson /* 11118355f576SJeff Roberson * Allocates a number of pages from within an object 11128355f576SJeff Roberson * 11138355f576SJeff Roberson * Arguments: 11148355f576SJeff Roberson * bytes The number of bytes requested 11158355f576SJeff Roberson * wait Shall we wait? 11168355f576SJeff Roberson * 11178355f576SJeff Roberson * Returns: 11188355f576SJeff Roberson * A pointer to the alloced memory or possibly 11198355f576SJeff Roberson * NULL if M_NOWAIT is set. 11208355f576SJeff Roberson */ 11218355f576SJeff Roberson static void * 112285dcf349SGleb Smirnoff noobj_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait) 11238355f576SJeff Roberson { 1124a4915c21SAttilio Rao TAILQ_HEAD(, vm_page) alloctail; 1125a4915c21SAttilio Rao u_long npages; 1126b245ac95SAlan Cox vm_offset_t retkva, zkva; 1127a4915c21SAttilio Rao vm_page_t p, p_next; 1128e20a199fSJeff Roberson uma_keg_t keg; 11298355f576SJeff Roberson 1130a4915c21SAttilio Rao TAILQ_INIT(&alloctail); 1131e20a199fSJeff Roberson keg = zone_first_keg(zone); 1132a4915c21SAttilio Rao 1133a4915c21SAttilio Rao npages = howmany(bytes, PAGE_SIZE); 1134a4915c21SAttilio Rao while (npages > 0) { 1135a4915c21SAttilio Rao p = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT | 1136a4915c21SAttilio Rao VM_ALLOC_WIRED | VM_ALLOC_NOOBJ); 1137a4915c21SAttilio Rao if (p != NULL) { 1138a4915c21SAttilio Rao /* 1139a4915c21SAttilio Rao * Since the page does not belong to an object, its 1140a4915c21SAttilio Rao * listq is unused. 1141a4915c21SAttilio Rao */ 1142a4915c21SAttilio Rao TAILQ_INSERT_TAIL(&alloctail, p, listq); 1143a4915c21SAttilio Rao npages--; 1144a4915c21SAttilio Rao continue; 1145a4915c21SAttilio Rao } 1146a4915c21SAttilio Rao if (wait & M_WAITOK) { 1147a4915c21SAttilio Rao VM_WAIT; 1148a4915c21SAttilio Rao continue; 1149a4915c21SAttilio Rao } 11508355f576SJeff Roberson 11518355f576SJeff Roberson /* 1152a4915c21SAttilio Rao * Page allocation failed, free intermediate pages and 1153a4915c21SAttilio Rao * exit. 11548355f576SJeff Roberson */ 1155a4915c21SAttilio Rao TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) { 1156b245ac95SAlan Cox vm_page_unwire(p, 0); 1157b245ac95SAlan Cox vm_page_free(p); 1158b245ac95SAlan Cox } 1159a4915c21SAttilio Rao return (NULL); 1160b245ac95SAlan Cox } 11618355f576SJeff Roberson *flags = UMA_SLAB_PRIV; 1162a4915c21SAttilio Rao zkva = keg->uk_kva + 1163a4915c21SAttilio Rao atomic_fetchadd_long(&keg->uk_offset, round_page(bytes)); 1164a4915c21SAttilio Rao retkva = zkva; 1165a4915c21SAttilio Rao TAILQ_FOREACH(p, &alloctail, listq) { 1166a4915c21SAttilio Rao pmap_qenter(zkva, &p, 1); 1167a4915c21SAttilio Rao zkva += PAGE_SIZE; 1168a4915c21SAttilio Rao } 11698355f576SJeff Roberson 11708355f576SJeff Roberson return ((void *)retkva); 11718355f576SJeff Roberson } 11728355f576SJeff Roberson 11738355f576SJeff Roberson /* 11748355f576SJeff Roberson * Frees a number of pages to the system 11758355f576SJeff Roberson * 11768355f576SJeff Roberson * Arguments: 11778355f576SJeff Roberson * mem A pointer to the memory to be freed 11788355f576SJeff Roberson * size The size of the memory being freed 11798355f576SJeff Roberson * flags The original p->us_flags field 11808355f576SJeff Roberson * 11818355f576SJeff Roberson * Returns: 11828355f576SJeff Roberson * Nothing 11838355f576SJeff Roberson */ 11848355f576SJeff Roberson static void 118585dcf349SGleb Smirnoff page_free(void *mem, int size, uint8_t flags) 11868355f576SJeff Roberson { 11875df87b21SJeff Roberson struct vmem *vmem; 11883370c5bfSJeff Roberson 11898355f576SJeff Roberson if (flags & UMA_SLAB_KMEM) 11905df87b21SJeff Roberson vmem = kmem_arena; 1191aea6e893SAlan Cox else if (flags & UMA_SLAB_KERNEL) 11925df87b21SJeff Roberson vmem = kernel_arena; 11938355f576SJeff Roberson else 1194aea6e893SAlan Cox panic("UMA: page_free used with invalid flags %d", flags); 11958355f576SJeff Roberson 11965df87b21SJeff Roberson kmem_free(vmem, (vm_offset_t)mem, size); 11978355f576SJeff Roberson } 11988355f576SJeff Roberson 11998355f576SJeff Roberson /* 12008355f576SJeff Roberson * Zero fill initializer 12018355f576SJeff Roberson * 12028355f576SJeff Roberson * Arguments/Returns follow uma_init specifications 12038355f576SJeff Roberson */ 1204b23f72e9SBrian Feldman static int 1205b23f72e9SBrian Feldman zero_init(void *mem, int size, int flags) 12068355f576SJeff Roberson { 12078355f576SJeff Roberson bzero(mem, size); 1208b23f72e9SBrian Feldman return (0); 12098355f576SJeff Roberson } 12108355f576SJeff Roberson 12118355f576SJeff Roberson /* 1212e20a199fSJeff Roberson * Finish creating a small uma keg. This calculates ipers, and the keg size. 12138355f576SJeff Roberson * 12148355f576SJeff Roberson * Arguments 1215e20a199fSJeff Roberson * keg The zone we should initialize 12168355f576SJeff Roberson * 12178355f576SJeff Roberson * Returns 12188355f576SJeff Roberson * Nothing 12198355f576SJeff Roberson */ 12208355f576SJeff Roberson static void 1221e20a199fSJeff Roberson keg_small_init(uma_keg_t keg) 12228355f576SJeff Roberson { 1223244f4554SBosko Milekic u_int rsize; 1224244f4554SBosko Milekic u_int memused; 1225244f4554SBosko Milekic u_int wastedspace; 1226244f4554SBosko Milekic u_int shsize; 12278355f576SJeff Roberson 1228ad97af7eSGleb Smirnoff if (keg->uk_flags & UMA_ZONE_PCPU) { 1229e28a647dSGleb Smirnoff u_int ncpus = mp_ncpus ? mp_ncpus : MAXCPU; 1230e28a647dSGleb Smirnoff 1231ad97af7eSGleb Smirnoff keg->uk_slabsize = sizeof(struct pcpu); 1232e28a647dSGleb Smirnoff keg->uk_ppera = howmany(ncpus * sizeof(struct pcpu), 1233ad97af7eSGleb Smirnoff PAGE_SIZE); 1234ad97af7eSGleb Smirnoff } else { 1235ad97af7eSGleb Smirnoff keg->uk_slabsize = UMA_SLAB_SIZE; 1236ad97af7eSGleb Smirnoff keg->uk_ppera = 1; 1237ad97af7eSGleb Smirnoff } 1238ad97af7eSGleb Smirnoff 1239ef72505eSJeff Roberson /* 1240ef72505eSJeff Roberson * Calculate the size of each allocation (rsize) according to 1241ef72505eSJeff Roberson * alignment. If the requested size is smaller than we have 1242ef72505eSJeff Roberson * allocation bits for we round it up. 1243ef72505eSJeff Roberson */ 1244099a0e58SBosko Milekic rsize = keg->uk_size; 1245ef72505eSJeff Roberson if (rsize < keg->uk_slabsize / SLAB_SETSIZE) 1246ef72505eSJeff Roberson rsize = keg->uk_slabsize / SLAB_SETSIZE; 1247099a0e58SBosko Milekic if (rsize & keg->uk_align) 1248099a0e58SBosko Milekic rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1); 1249099a0e58SBosko Milekic keg->uk_rsize = rsize; 1250ad97af7eSGleb Smirnoff 1251ad97af7eSGleb Smirnoff KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || 1252ad97af7eSGleb Smirnoff keg->uk_rsize < sizeof(struct pcpu), 1253ad97af7eSGleb Smirnoff ("%s: size %u too large", __func__, keg->uk_rsize)); 12548355f576SJeff Roberson 1255ef72505eSJeff Roberson if (keg->uk_flags & UMA_ZONE_REFCNT) 1256ef72505eSJeff Roberson rsize += sizeof(uint32_t); 1257ef72505eSJeff Roberson 1258ef72505eSJeff Roberson if (keg->uk_flags & UMA_ZONE_OFFPAGE) 12592864dbbfSGleb Smirnoff shsize = 0; 1260ef72505eSJeff Roberson else 1261244f4554SBosko Milekic shsize = sizeof(struct uma_slab); 12628355f576SJeff Roberson 1263ad97af7eSGleb Smirnoff keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize; 1264ef72505eSJeff Roberson KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE, 1265ad97af7eSGleb Smirnoff ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); 1266ad97af7eSGleb Smirnoff 1267244f4554SBosko Milekic memused = keg->uk_ipers * rsize + shsize; 1268ad97af7eSGleb Smirnoff wastedspace = keg->uk_slabsize - memused; 1269244f4554SBosko Milekic 127020e8e865SBosko Milekic /* 1271244f4554SBosko Milekic * We can't do OFFPAGE if we're internal or if we've been 127220e8e865SBosko Milekic * asked to not go to the VM for buckets. If we do this we 12736fd34d6fSJeff Roberson * may end up going to the VM for slabs which we do not 12746fd34d6fSJeff Roberson * want to do if we're UMA_ZFLAG_CACHEONLY as a result 12756fd34d6fSJeff Roberson * of UMA_ZONE_VM, which clearly forbids it. 127620e8e865SBosko Milekic */ 1277099a0e58SBosko Milekic if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) || 1278099a0e58SBosko Milekic (keg->uk_flags & UMA_ZFLAG_CACHEONLY)) 12798355f576SJeff Roberson return; 1280244f4554SBosko Milekic 1281ef72505eSJeff Roberson /* 1282ef72505eSJeff Roberson * See if using an OFFPAGE slab will limit our waste. Only do 1283ef72505eSJeff Roberson * this if it permits more items per-slab. 1284ef72505eSJeff Roberson * 1285ef72505eSJeff Roberson * XXX We could try growing slabsize to limit max waste as well. 1286ef72505eSJeff Roberson * Historically this was not done because the VM could not 1287ef72505eSJeff Roberson * efficiently handle contiguous allocations. 1288ef72505eSJeff Roberson */ 1289ad97af7eSGleb Smirnoff if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) && 1290ad97af7eSGleb Smirnoff (keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) { 1291ad97af7eSGleb Smirnoff keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize; 1292ef72505eSJeff Roberson KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE, 1293ad97af7eSGleb Smirnoff ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); 1294244f4554SBosko Milekic #ifdef UMA_DEBUG 1295244f4554SBosko Milekic printf("UMA decided we need offpage slab headers for " 1296e20a199fSJeff Roberson "keg: %s, calculated wastedspace = %d, " 1297244f4554SBosko Milekic "maximum wasted space allowed = %d, " 1298244f4554SBosko Milekic "calculated ipers = %d, " 1299e20a199fSJeff Roberson "new wasted space = %d\n", keg->uk_name, wastedspace, 1300ad97af7eSGleb Smirnoff keg->uk_slabsize / UMA_MAX_WASTE, keg->uk_ipers, 1301ad97af7eSGleb Smirnoff keg->uk_slabsize - keg->uk_ipers * keg->uk_rsize); 1302244f4554SBosko Milekic #endif 1303099a0e58SBosko Milekic keg->uk_flags |= UMA_ZONE_OFFPAGE; 13048355f576SJeff Roberson } 1305ad97af7eSGleb Smirnoff 1306ad97af7eSGleb Smirnoff if ((keg->uk_flags & UMA_ZONE_OFFPAGE) && 1307ad97af7eSGleb Smirnoff (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0) 1308ad97af7eSGleb Smirnoff keg->uk_flags |= UMA_ZONE_HASH; 13098355f576SJeff Roberson } 13108355f576SJeff Roberson 13118355f576SJeff Roberson /* 1312e20a199fSJeff Roberson * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do 13138355f576SJeff Roberson * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be 13148355f576SJeff Roberson * more complicated. 13158355f576SJeff Roberson * 13168355f576SJeff Roberson * Arguments 1317e20a199fSJeff Roberson * keg The keg we should initialize 13188355f576SJeff Roberson * 13198355f576SJeff Roberson * Returns 13208355f576SJeff Roberson * Nothing 13218355f576SJeff Roberson */ 13228355f576SJeff Roberson static void 1323e20a199fSJeff Roberson keg_large_init(uma_keg_t keg) 13248355f576SJeff Roberson { 1325cec48e00SAlexander Motin u_int shsize; 13268355f576SJeff Roberson 1327e20a199fSJeff Roberson KASSERT(keg != NULL, ("Keg is null in keg_large_init")); 1328099a0e58SBosko Milekic KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0, 1329e20a199fSJeff Roberson ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg")); 1330ad97af7eSGleb Smirnoff KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0, 1331ad97af7eSGleb Smirnoff ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__)); 133220e8e865SBosko Milekic 1333ad97af7eSGleb Smirnoff keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE); 1334ad97af7eSGleb Smirnoff keg->uk_slabsize = keg->uk_ppera * PAGE_SIZE; 1335099a0e58SBosko Milekic keg->uk_ipers = 1; 1336e9a069d8SJohn Baldwin keg->uk_rsize = keg->uk_size; 1337e9a069d8SJohn Baldwin 1338e9a069d8SJohn Baldwin /* We can't do OFFPAGE if we're internal, bail out here. */ 1339e9a069d8SJohn Baldwin if (keg->uk_flags & UMA_ZFLAG_INTERNAL) 1340e9a069d8SJohn Baldwin return; 13418355f576SJeff Roberson 1342cec48e00SAlexander Motin /* Check whether we have enough space to not do OFFPAGE. */ 1343cec48e00SAlexander Motin if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0) { 1344cec48e00SAlexander Motin shsize = sizeof(struct uma_slab); 1345cec48e00SAlexander Motin if (keg->uk_flags & UMA_ZONE_REFCNT) 1346cec48e00SAlexander Motin shsize += keg->uk_ipers * sizeof(uint32_t); 1347cec48e00SAlexander Motin if (shsize & UMA_ALIGN_PTR) 1348cec48e00SAlexander Motin shsize = (shsize & ~UMA_ALIGN_PTR) + 1349cec48e00SAlexander Motin (UMA_ALIGN_PTR + 1); 1350cec48e00SAlexander Motin 1351cec48e00SAlexander Motin if ((PAGE_SIZE * keg->uk_ppera) - keg->uk_rsize < shsize) 1352099a0e58SBosko Milekic keg->uk_flags |= UMA_ZONE_OFFPAGE; 1353cec48e00SAlexander Motin } 1354cec48e00SAlexander Motin 1355cec48e00SAlexander Motin if ((keg->uk_flags & UMA_ZONE_OFFPAGE) && 1356cec48e00SAlexander Motin (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0) 1357099a0e58SBosko Milekic keg->uk_flags |= UMA_ZONE_HASH; 13588355f576SJeff Roberson } 13598355f576SJeff Roberson 1360e20a199fSJeff Roberson static void 1361e20a199fSJeff Roberson keg_cachespread_init(uma_keg_t keg) 1362e20a199fSJeff Roberson { 1363e20a199fSJeff Roberson int alignsize; 1364e20a199fSJeff Roberson int trailer; 1365e20a199fSJeff Roberson int pages; 1366e20a199fSJeff Roberson int rsize; 1367e20a199fSJeff Roberson 1368ad97af7eSGleb Smirnoff KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0, 1369ad97af7eSGleb Smirnoff ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__)); 1370ad97af7eSGleb Smirnoff 1371e20a199fSJeff Roberson alignsize = keg->uk_align + 1; 1372e20a199fSJeff Roberson rsize = keg->uk_size; 1373e20a199fSJeff Roberson /* 1374e20a199fSJeff Roberson * We want one item to start on every align boundary in a page. To 1375e20a199fSJeff Roberson * do this we will span pages. We will also extend the item by the 1376e20a199fSJeff Roberson * size of align if it is an even multiple of align. Otherwise, it 1377e20a199fSJeff Roberson * would fall on the same boundary every time. 1378e20a199fSJeff Roberson */ 1379e20a199fSJeff Roberson if (rsize & keg->uk_align) 1380e20a199fSJeff Roberson rsize = (rsize & ~keg->uk_align) + alignsize; 1381e20a199fSJeff Roberson if ((rsize & alignsize) == 0) 1382e20a199fSJeff Roberson rsize += alignsize; 1383e20a199fSJeff Roberson trailer = rsize - keg->uk_size; 1384e20a199fSJeff Roberson pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE; 1385e20a199fSJeff Roberson pages = MIN(pages, (128 * 1024) / PAGE_SIZE); 1386e20a199fSJeff Roberson keg->uk_rsize = rsize; 1387e20a199fSJeff Roberson keg->uk_ppera = pages; 1388ad97af7eSGleb Smirnoff keg->uk_slabsize = UMA_SLAB_SIZE; 1389e20a199fSJeff Roberson keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize; 1390e20a199fSJeff Roberson keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB; 13912367b4ddSDimitry Andric KASSERT(keg->uk_ipers <= SLAB_SETSIZE, 139242321809SGleb Smirnoff ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__, 1393e20a199fSJeff Roberson keg->uk_ipers)); 1394e20a199fSJeff Roberson } 1395e20a199fSJeff Roberson 13968355f576SJeff Roberson /* 1397099a0e58SBosko Milekic * Keg header ctor. This initializes all fields, locks, etc. And inserts 1398099a0e58SBosko Milekic * the keg onto the global keg list. 13998355f576SJeff Roberson * 14008355f576SJeff Roberson * Arguments/Returns follow uma_ctor specifications 1401099a0e58SBosko Milekic * udata Actually uma_kctor_args 1402099a0e58SBosko Milekic */ 1403b23f72e9SBrian Feldman static int 1404b23f72e9SBrian Feldman keg_ctor(void *mem, int size, void *udata, int flags) 1405099a0e58SBosko Milekic { 1406099a0e58SBosko Milekic struct uma_kctor_args *arg = udata; 1407099a0e58SBosko Milekic uma_keg_t keg = mem; 1408099a0e58SBosko Milekic uma_zone_t zone; 1409099a0e58SBosko Milekic 1410099a0e58SBosko Milekic bzero(keg, size); 1411099a0e58SBosko Milekic keg->uk_size = arg->size; 1412099a0e58SBosko Milekic keg->uk_init = arg->uminit; 1413099a0e58SBosko Milekic keg->uk_fini = arg->fini; 1414099a0e58SBosko Milekic keg->uk_align = arg->align; 1415099a0e58SBosko Milekic keg->uk_free = 0; 14166fd34d6fSJeff Roberson keg->uk_reserve = 0; 1417099a0e58SBosko Milekic keg->uk_pages = 0; 1418099a0e58SBosko Milekic keg->uk_flags = arg->flags; 1419099a0e58SBosko Milekic keg->uk_allocf = page_alloc; 1420099a0e58SBosko Milekic keg->uk_freef = page_free; 1421099a0e58SBosko Milekic keg->uk_slabzone = NULL; 1422099a0e58SBosko Milekic 1423099a0e58SBosko Milekic /* 1424099a0e58SBosko Milekic * The master zone is passed to us at keg-creation time. 1425099a0e58SBosko Milekic */ 1426099a0e58SBosko Milekic zone = arg->zone; 1427e20a199fSJeff Roberson keg->uk_name = zone->uz_name; 1428099a0e58SBosko Milekic 1429099a0e58SBosko Milekic if (arg->flags & UMA_ZONE_VM) 1430099a0e58SBosko Milekic keg->uk_flags |= UMA_ZFLAG_CACHEONLY; 1431099a0e58SBosko Milekic 1432099a0e58SBosko Milekic if (arg->flags & UMA_ZONE_ZINIT) 1433099a0e58SBosko Milekic keg->uk_init = zero_init; 1434099a0e58SBosko Milekic 1435e20a199fSJeff Roberson if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC) 1436e20a199fSJeff Roberson keg->uk_flags |= UMA_ZONE_VTOSLAB; 1437e20a199fSJeff Roberson 1438ad97af7eSGleb Smirnoff if (arg->flags & UMA_ZONE_PCPU) 1439ad97af7eSGleb Smirnoff #ifdef SMP 1440ad97af7eSGleb Smirnoff keg->uk_flags |= UMA_ZONE_OFFPAGE; 1441ad97af7eSGleb Smirnoff #else 1442ad97af7eSGleb Smirnoff keg->uk_flags &= ~UMA_ZONE_PCPU; 1443ad97af7eSGleb Smirnoff #endif 1444ad97af7eSGleb Smirnoff 1445ef72505eSJeff Roberson if (keg->uk_flags & UMA_ZONE_CACHESPREAD) { 1446e20a199fSJeff Roberson keg_cachespread_init(keg); 1447ef72505eSJeff Roberson } else if (keg->uk_flags & UMA_ZONE_REFCNT) { 1448ef72505eSJeff Roberson if (keg->uk_size > 1449ef72505eSJeff Roberson (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) - 1450ef72505eSJeff Roberson sizeof(uint32_t))) 1451e20a199fSJeff Roberson keg_large_init(keg); 1452099a0e58SBosko Milekic else 1453e20a199fSJeff Roberson keg_small_init(keg); 1454244f4554SBosko Milekic } else { 1455ef72505eSJeff Roberson if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab))) 1456e20a199fSJeff Roberson keg_large_init(keg); 1457244f4554SBosko Milekic else 1458e20a199fSJeff Roberson keg_small_init(keg); 1459244f4554SBosko Milekic } 1460099a0e58SBosko Milekic 1461244f4554SBosko Milekic if (keg->uk_flags & UMA_ZONE_OFFPAGE) { 1462ef72505eSJeff Roberson if (keg->uk_flags & UMA_ZONE_REFCNT) { 1463ef72505eSJeff Roberson if (keg->uk_ipers > uma_max_ipers_ref) 1464ef72505eSJeff Roberson panic("Too many ref items per zone: %d > %d\n", 1465ef72505eSJeff Roberson keg->uk_ipers, uma_max_ipers_ref); 1466099a0e58SBosko Milekic keg->uk_slabzone = slabrefzone; 1467ef72505eSJeff Roberson } else 1468099a0e58SBosko Milekic keg->uk_slabzone = slabzone; 1469244f4554SBosko Milekic } 1470099a0e58SBosko Milekic 1471099a0e58SBosko Milekic /* 1472099a0e58SBosko Milekic * If we haven't booted yet we need allocations to go through the 1473099a0e58SBosko Milekic * startup cache until the vm is ready. 1474099a0e58SBosko Milekic */ 1475099a0e58SBosko Milekic if (keg->uk_ppera == 1) { 1476099a0e58SBosko Milekic #ifdef UMA_MD_SMALL_ALLOC 1477099a0e58SBosko Milekic keg->uk_allocf = uma_small_alloc; 1478099a0e58SBosko Milekic keg->uk_freef = uma_small_free; 14798cd02d00SAlan Cox 1480342f1793SAlan Cox if (booted < UMA_STARTUP) 1481099a0e58SBosko Milekic keg->uk_allocf = startup_alloc; 14828cd02d00SAlan Cox #else 14838cd02d00SAlan Cox if (booted < UMA_STARTUP2) 14848cd02d00SAlan Cox keg->uk_allocf = startup_alloc; 14858cd02d00SAlan Cox #endif 1486342f1793SAlan Cox } else if (booted < UMA_STARTUP2 && 1487342f1793SAlan Cox (keg->uk_flags & UMA_ZFLAG_INTERNAL)) 1488e9a069d8SJohn Baldwin keg->uk_allocf = startup_alloc; 1489099a0e58SBosko Milekic 1490099a0e58SBosko Milekic /* 1491af526374SJeff Roberson * Initialize keg's lock 1492099a0e58SBosko Milekic */ 1493af526374SJeff Roberson KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS)); 1494099a0e58SBosko Milekic 1495099a0e58SBosko Milekic /* 1496099a0e58SBosko Milekic * If we're putting the slab header in the actual page we need to 1497099a0e58SBosko Milekic * figure out where in each page it goes. This calculates a right 1498099a0e58SBosko Milekic * justified offset into the memory on an ALIGN_PTR boundary. 1499099a0e58SBosko Milekic */ 1500099a0e58SBosko Milekic if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) { 1501244f4554SBosko Milekic u_int totsize; 1502099a0e58SBosko Milekic 1503099a0e58SBosko Milekic /* Size of the slab struct and free list */ 1504ef72505eSJeff Roberson totsize = sizeof(struct uma_slab); 1505ef72505eSJeff Roberson 1506ef72505eSJeff Roberson /* Size of the reference counts. */ 1507244f4554SBosko Milekic if (keg->uk_flags & UMA_ZONE_REFCNT) 1508ef72505eSJeff Roberson totsize += keg->uk_ipers * sizeof(uint32_t); 1509244f4554SBosko Milekic 1510099a0e58SBosko Milekic if (totsize & UMA_ALIGN_PTR) 1511099a0e58SBosko Milekic totsize = (totsize & ~UMA_ALIGN_PTR) + 1512099a0e58SBosko Milekic (UMA_ALIGN_PTR + 1); 1513ad97af7eSGleb Smirnoff keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize; 1514244f4554SBosko Milekic 1515244f4554SBosko Milekic /* 1516244f4554SBosko Milekic * The only way the following is possible is if with our 1517244f4554SBosko Milekic * UMA_ALIGN_PTR adjustments we are now bigger than 1518244f4554SBosko Milekic * UMA_SLAB_SIZE. I haven't checked whether this is 1519244f4554SBosko Milekic * mathematically possible for all cases, so we make 1520244f4554SBosko Milekic * sure here anyway. 1521244f4554SBosko Milekic */ 1522ef72505eSJeff Roberson totsize = keg->uk_pgoff + sizeof(struct uma_slab); 1523ef72505eSJeff Roberson if (keg->uk_flags & UMA_ZONE_REFCNT) 1524ef72505eSJeff Roberson totsize += keg->uk_ipers * sizeof(uint32_t); 1525ad97af7eSGleb Smirnoff if (totsize > PAGE_SIZE * keg->uk_ppera) { 1526099a0e58SBosko Milekic printf("zone %s ipers %d rsize %d size %d\n", 1527099a0e58SBosko Milekic zone->uz_name, keg->uk_ipers, keg->uk_rsize, 1528099a0e58SBosko Milekic keg->uk_size); 1529aea6e893SAlan Cox panic("UMA slab won't fit."); 1530099a0e58SBosko Milekic } 1531099a0e58SBosko Milekic } 1532099a0e58SBosko Milekic 1533099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZONE_HASH) 1534099a0e58SBosko Milekic hash_alloc(&keg->uk_hash); 1535099a0e58SBosko Milekic 1536099a0e58SBosko Milekic #ifdef UMA_DEBUG 15370b80c1e4SEitan Adler printf("UMA: %s(%p) size %d(%d) flags %#x ipers %d ppera %d out %d free %d\n", 1538e20a199fSJeff Roberson zone->uz_name, zone, keg->uk_size, keg->uk_rsize, keg->uk_flags, 1539e20a199fSJeff Roberson keg->uk_ipers, keg->uk_ppera, 1540e20a199fSJeff Roberson (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free); 1541099a0e58SBosko Milekic #endif 1542099a0e58SBosko Milekic 1543099a0e58SBosko Milekic LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); 1544099a0e58SBosko Milekic 1545099a0e58SBosko Milekic mtx_lock(&uma_mtx); 1546099a0e58SBosko Milekic LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); 1547099a0e58SBosko Milekic mtx_unlock(&uma_mtx); 1548b23f72e9SBrian Feldman return (0); 1549099a0e58SBosko Milekic } 1550099a0e58SBosko Milekic 1551099a0e58SBosko Milekic /* 1552099a0e58SBosko Milekic * Zone header ctor. This initializes all fields, locks, etc. 1553099a0e58SBosko Milekic * 1554099a0e58SBosko Milekic * Arguments/Returns follow uma_ctor specifications 1555099a0e58SBosko Milekic * udata Actually uma_zctor_args 15568355f576SJeff Roberson */ 1557b23f72e9SBrian Feldman static int 1558b23f72e9SBrian Feldman zone_ctor(void *mem, int size, void *udata, int flags) 15598355f576SJeff Roberson { 15608355f576SJeff Roberson struct uma_zctor_args *arg = udata; 15618355f576SJeff Roberson uma_zone_t zone = mem; 1562099a0e58SBosko Milekic uma_zone_t z; 1563099a0e58SBosko Milekic uma_keg_t keg; 15648355f576SJeff Roberson 15658355f576SJeff Roberson bzero(zone, size); 15668355f576SJeff Roberson zone->uz_name = arg->name; 15678355f576SJeff Roberson zone->uz_ctor = arg->ctor; 15688355f576SJeff Roberson zone->uz_dtor = arg->dtor; 1569e20a199fSJeff Roberson zone->uz_slab = zone_fetch_slab; 1570099a0e58SBosko Milekic zone->uz_init = NULL; 1571099a0e58SBosko Milekic zone->uz_fini = NULL; 1572099a0e58SBosko Milekic zone->uz_allocs = 0; 1573773df9abSRobert Watson zone->uz_frees = 0; 15742019094aSRobert Watson zone->uz_fails = 0; 1575bf965959SSean Bruno zone->uz_sleeps = 0; 1576fc03d22bSJeff Roberson zone->uz_count = 0; 1577ace66b56SAlexander Motin zone->uz_count_min = 0; 1578e20a199fSJeff Roberson zone->uz_flags = 0; 15792f891cd5SPawel Jakub Dawidek zone->uz_warning = NULL; 15802f891cd5SPawel Jakub Dawidek timevalclear(&zone->uz_ratecheck); 1581e20a199fSJeff Roberson keg = arg->keg; 1582099a0e58SBosko Milekic 1583af526374SJeff Roberson ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS)); 1584af526374SJeff Roberson 15850095a784SJeff Roberson /* 15860095a784SJeff Roberson * This is a pure cache zone, no kegs. 15870095a784SJeff Roberson */ 15880095a784SJeff Roberson if (arg->import) { 15896fd34d6fSJeff Roberson if (arg->flags & UMA_ZONE_VM) 15906fd34d6fSJeff Roberson arg->flags |= UMA_ZFLAG_CACHEONLY; 15916fd34d6fSJeff Roberson zone->uz_flags = arg->flags; 1592af526374SJeff Roberson zone->uz_size = arg->size; 15930095a784SJeff Roberson zone->uz_import = arg->import; 15940095a784SJeff Roberson zone->uz_release = arg->release; 15950095a784SJeff Roberson zone->uz_arg = arg->arg; 1596af526374SJeff Roberson zone->uz_lockptr = &zone->uz_lock; 159703175483SAlexander Motin mtx_lock(&uma_mtx); 159803175483SAlexander Motin LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link); 159903175483SAlexander Motin mtx_unlock(&uma_mtx); 1600af526374SJeff Roberson goto out; 16010095a784SJeff Roberson } 16020095a784SJeff Roberson 16030095a784SJeff Roberson /* 16040095a784SJeff Roberson * Use the regular zone/keg/slab allocator. 16050095a784SJeff Roberson */ 16060095a784SJeff Roberson zone->uz_import = (uma_import)zone_import; 16070095a784SJeff Roberson zone->uz_release = (uma_release)zone_release; 16080095a784SJeff Roberson zone->uz_arg = zone; 16090095a784SJeff Roberson 1610099a0e58SBosko Milekic if (arg->flags & UMA_ZONE_SECONDARY) { 1611099a0e58SBosko Milekic KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); 16128355f576SJeff Roberson zone->uz_init = arg->uminit; 1613e221e841SJeff Roberson zone->uz_fini = arg->fini; 1614af526374SJeff Roberson zone->uz_lockptr = &keg->uk_lock; 1615e20a199fSJeff Roberson zone->uz_flags |= UMA_ZONE_SECONDARY; 16168355f576SJeff Roberson mtx_lock(&uma_mtx); 1617099a0e58SBosko Milekic ZONE_LOCK(zone); 1618099a0e58SBosko Milekic LIST_FOREACH(z, &keg->uk_zones, uz_link) { 1619099a0e58SBosko Milekic if (LIST_NEXT(z, uz_link) == NULL) { 1620099a0e58SBosko Milekic LIST_INSERT_AFTER(z, zone, uz_link); 1621099a0e58SBosko Milekic break; 1622099a0e58SBosko Milekic } 1623099a0e58SBosko Milekic } 1624099a0e58SBosko Milekic ZONE_UNLOCK(zone); 16258355f576SJeff Roberson mtx_unlock(&uma_mtx); 1626e20a199fSJeff Roberson } else if (keg == NULL) { 1627e20a199fSJeff Roberson if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini, 1628e20a199fSJeff Roberson arg->align, arg->flags)) == NULL) 1629b23f72e9SBrian Feldman return (ENOMEM); 1630099a0e58SBosko Milekic } else { 1631099a0e58SBosko Milekic struct uma_kctor_args karg; 1632b23f72e9SBrian Feldman int error; 1633099a0e58SBosko Milekic 1634099a0e58SBosko Milekic /* We should only be here from uma_startup() */ 1635099a0e58SBosko Milekic karg.size = arg->size; 1636099a0e58SBosko Milekic karg.uminit = arg->uminit; 1637099a0e58SBosko Milekic karg.fini = arg->fini; 1638099a0e58SBosko Milekic karg.align = arg->align; 1639099a0e58SBosko Milekic karg.flags = arg->flags; 1640099a0e58SBosko Milekic karg.zone = zone; 1641b23f72e9SBrian Feldman error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, 1642b23f72e9SBrian Feldman flags); 1643b23f72e9SBrian Feldman if (error) 1644b23f72e9SBrian Feldman return (error); 1645099a0e58SBosko Milekic } 16460095a784SJeff Roberson 1647e20a199fSJeff Roberson /* 1648e20a199fSJeff Roberson * Link in the first keg. 1649e20a199fSJeff Roberson */ 1650e20a199fSJeff Roberson zone->uz_klink.kl_keg = keg; 1651e20a199fSJeff Roberson LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link); 1652af526374SJeff Roberson zone->uz_lockptr = &keg->uk_lock; 1653e20a199fSJeff Roberson zone->uz_size = keg->uk_size; 1654e20a199fSJeff Roberson zone->uz_flags |= (keg->uk_flags & 1655e20a199fSJeff Roberson (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT)); 16568355f576SJeff Roberson 16578355f576SJeff Roberson /* 16588355f576SJeff Roberson * Some internal zones don't have room allocated for the per cpu 16598355f576SJeff Roberson * caches. If we're internal, bail out here. 16608355f576SJeff Roberson */ 1661099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZFLAG_INTERNAL) { 1662e20a199fSJeff Roberson KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0, 1663099a0e58SBosko Milekic ("Secondary zone requested UMA_ZFLAG_INTERNAL")); 1664b23f72e9SBrian Feldman return (0); 1665099a0e58SBosko Milekic } 16668355f576SJeff Roberson 1667af526374SJeff Roberson out: 1668af526374SJeff Roberson if ((arg->flags & UMA_ZONE_MAXBUCKET) == 0) 1669af526374SJeff Roberson zone->uz_count = bucket_select(zone->uz_size); 16708355f576SJeff Roberson else 1671cae33c14SJeff Roberson zone->uz_count = BUCKET_MAX; 1672ace66b56SAlexander Motin zone->uz_count_min = zone->uz_count; 1673fc03d22bSJeff Roberson 1674b23f72e9SBrian Feldman return (0); 16758355f576SJeff Roberson } 16768355f576SJeff Roberson 16778355f576SJeff Roberson /* 1678099a0e58SBosko Milekic * Keg header dtor. This frees all data, destroys locks, frees the hash 1679099a0e58SBosko Milekic * table and removes the keg from the global list. 16809c2cd7e5SJeff Roberson * 16819c2cd7e5SJeff Roberson * Arguments/Returns follow uma_dtor specifications 16829c2cd7e5SJeff Roberson * udata unused 16839c2cd7e5SJeff Roberson */ 1684099a0e58SBosko Milekic static void 1685099a0e58SBosko Milekic keg_dtor(void *arg, int size, void *udata) 1686099a0e58SBosko Milekic { 1687099a0e58SBosko Milekic uma_keg_t keg; 16889c2cd7e5SJeff Roberson 1689099a0e58SBosko Milekic keg = (uma_keg_t)arg; 1690e20a199fSJeff Roberson KEG_LOCK(keg); 1691099a0e58SBosko Milekic if (keg->uk_free != 0) { 1692a3845534SCraig Rodrigues printf("Freed UMA keg (%s) was not empty (%d items). " 1693099a0e58SBosko Milekic " Lost %d pages of memory.\n", 1694a3845534SCraig Rodrigues keg->uk_name ? keg->uk_name : "", 1695099a0e58SBosko Milekic keg->uk_free, keg->uk_pages); 1696099a0e58SBosko Milekic } 1697e20a199fSJeff Roberson KEG_UNLOCK(keg); 1698099a0e58SBosko Milekic 1699099a0e58SBosko Milekic hash_free(&keg->uk_hash); 1700099a0e58SBosko Milekic 1701e20a199fSJeff Roberson KEG_LOCK_FINI(keg); 1702099a0e58SBosko Milekic } 1703099a0e58SBosko Milekic 1704099a0e58SBosko Milekic /* 1705099a0e58SBosko Milekic * Zone header dtor. 1706099a0e58SBosko Milekic * 1707099a0e58SBosko Milekic * Arguments/Returns follow uma_dtor specifications 1708099a0e58SBosko Milekic * udata unused 1709099a0e58SBosko Milekic */ 17109c2cd7e5SJeff Roberson static void 17119c2cd7e5SJeff Roberson zone_dtor(void *arg, int size, void *udata) 17129c2cd7e5SJeff Roberson { 1713e20a199fSJeff Roberson uma_klink_t klink; 17149c2cd7e5SJeff Roberson uma_zone_t zone; 1715099a0e58SBosko Milekic uma_keg_t keg; 17169c2cd7e5SJeff Roberson 17179c2cd7e5SJeff Roberson zone = (uma_zone_t)arg; 1718e20a199fSJeff Roberson keg = zone_first_keg(zone); 17199643769aSJeff Roberson 1720e20a199fSJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) 17219643769aSJeff Roberson cache_drain(zone); 1722099a0e58SBosko Milekic 172317b9cc49SJeff Roberson mtx_lock(&uma_mtx); 1724099a0e58SBosko Milekic LIST_REMOVE(zone, uz_link); 1725e20a199fSJeff Roberson mtx_unlock(&uma_mtx); 1726099a0e58SBosko Milekic /* 1727099a0e58SBosko Milekic * XXX there are some races here where 1728099a0e58SBosko Milekic * the zone can be drained but zone lock 1729099a0e58SBosko Milekic * released and then refilled before we 1730099a0e58SBosko Milekic * remove it... we dont care for now 1731099a0e58SBosko Milekic */ 1732e20a199fSJeff Roberson zone_drain_wait(zone, M_WAITOK); 1733e20a199fSJeff Roberson /* 1734e20a199fSJeff Roberson * Unlink all of our kegs. 1735e20a199fSJeff Roberson */ 1736e20a199fSJeff Roberson while ((klink = LIST_FIRST(&zone->uz_kegs)) != NULL) { 1737e20a199fSJeff Roberson klink->kl_keg = NULL; 1738e20a199fSJeff Roberson LIST_REMOVE(klink, kl_link); 1739e20a199fSJeff Roberson if (klink == &zone->uz_klink) 1740e20a199fSJeff Roberson continue; 1741e20a199fSJeff Roberson free(klink, M_TEMP); 1742e20a199fSJeff Roberson } 1743e20a199fSJeff Roberson /* 1744e20a199fSJeff Roberson * We only destroy kegs from non secondary zones. 1745e20a199fSJeff Roberson */ 17460095a784SJeff Roberson if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) { 1747e20a199fSJeff Roberson mtx_lock(&uma_mtx); 1748099a0e58SBosko Milekic LIST_REMOVE(keg, uk_link); 1749099a0e58SBosko Milekic mtx_unlock(&uma_mtx); 17500095a784SJeff Roberson zone_free_item(kegs, keg, NULL, SKIP_NONE); 17519c2cd7e5SJeff Roberson } 1752af526374SJeff Roberson ZONE_LOCK_FINI(zone); 1753099a0e58SBosko Milekic } 1754099a0e58SBosko Milekic 17559c2cd7e5SJeff Roberson /* 17568355f576SJeff Roberson * Traverses every zone in the system and calls a callback 17578355f576SJeff Roberson * 17588355f576SJeff Roberson * Arguments: 17598355f576SJeff Roberson * zfunc A pointer to a function which accepts a zone 17608355f576SJeff Roberson * as an argument. 17618355f576SJeff Roberson * 17628355f576SJeff Roberson * Returns: 17638355f576SJeff Roberson * Nothing 17648355f576SJeff Roberson */ 17658355f576SJeff Roberson static void 17668355f576SJeff Roberson zone_foreach(void (*zfunc)(uma_zone_t)) 17678355f576SJeff Roberson { 1768099a0e58SBosko Milekic uma_keg_t keg; 17698355f576SJeff Roberson uma_zone_t zone; 17708355f576SJeff Roberson 17718355f576SJeff Roberson mtx_lock(&uma_mtx); 1772099a0e58SBosko Milekic LIST_FOREACH(keg, &uma_kegs, uk_link) { 1773099a0e58SBosko Milekic LIST_FOREACH(zone, &keg->uk_zones, uz_link) 17748355f576SJeff Roberson zfunc(zone); 1775099a0e58SBosko Milekic } 17768355f576SJeff Roberson mtx_unlock(&uma_mtx); 17778355f576SJeff Roberson } 17788355f576SJeff Roberson 17798355f576SJeff Roberson /* Public functions */ 17808355f576SJeff Roberson /* See uma.h */ 17818355f576SJeff Roberson void 17823803b26bSDag-Erling Smørgrav uma_startup(void *bootmem, int boot_pages) 17838355f576SJeff Roberson { 17848355f576SJeff Roberson struct uma_zctor_args args; 17858355f576SJeff Roberson uma_slab_t slab; 1786244f4554SBosko Milekic u_int slabsize; 17878355f576SJeff Roberson int i; 17888355f576SJeff Roberson 17898355f576SJeff Roberson #ifdef UMA_DEBUG 1790099a0e58SBosko Milekic printf("Creating uma keg headers zone and keg.\n"); 17918355f576SJeff Roberson #endif 1792f353d338SAlan Cox mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF); 1793099a0e58SBosko Milekic 1794099a0e58SBosko Milekic /* "manually" create the initial zone */ 17950095a784SJeff Roberson memset(&args, 0, sizeof(args)); 1796099a0e58SBosko Milekic args.name = "UMA Kegs"; 1797099a0e58SBosko Milekic args.size = sizeof(struct uma_keg); 1798099a0e58SBosko Milekic args.ctor = keg_ctor; 1799099a0e58SBosko Milekic args.dtor = keg_dtor; 18008355f576SJeff Roberson args.uminit = zero_init; 18018355f576SJeff Roberson args.fini = NULL; 1802099a0e58SBosko Milekic args.keg = &masterkeg; 18038355f576SJeff Roberson args.align = 32 - 1; 1804b60f5b79SJeff Roberson args.flags = UMA_ZFLAG_INTERNAL; 18058355f576SJeff Roberson /* The initial zone has no Per cpu queues so it's smaller */ 1806b23f72e9SBrian Feldman zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK); 18078355f576SJeff Roberson 18088355f576SJeff Roberson #ifdef UMA_DEBUG 18098355f576SJeff Roberson printf("Filling boot free list.\n"); 18108355f576SJeff Roberson #endif 18113803b26bSDag-Erling Smørgrav for (i = 0; i < boot_pages; i++) { 181285dcf349SGleb Smirnoff slab = (uma_slab_t)((uint8_t *)bootmem + (i * UMA_SLAB_SIZE)); 181385dcf349SGleb Smirnoff slab->us_data = (uint8_t *)slab; 18148355f576SJeff Roberson slab->us_flags = UMA_SLAB_BOOT; 18158355f576SJeff Roberson LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link); 18168355f576SJeff Roberson } 1817f353d338SAlan Cox mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF); 18188355f576SJeff Roberson 18198355f576SJeff Roberson #ifdef UMA_DEBUG 1820099a0e58SBosko Milekic printf("Creating uma zone headers zone and keg.\n"); 1821099a0e58SBosko Milekic #endif 1822099a0e58SBosko Milekic args.name = "UMA Zones"; 1823099a0e58SBosko Milekic args.size = sizeof(struct uma_zone) + 1824099a0e58SBosko Milekic (sizeof(struct uma_cache) * (mp_maxid + 1)); 1825099a0e58SBosko Milekic args.ctor = zone_ctor; 1826099a0e58SBosko Milekic args.dtor = zone_dtor; 1827099a0e58SBosko Milekic args.uminit = zero_init; 1828099a0e58SBosko Milekic args.fini = NULL; 1829099a0e58SBosko Milekic args.keg = NULL; 1830099a0e58SBosko Milekic args.align = 32 - 1; 1831099a0e58SBosko Milekic args.flags = UMA_ZFLAG_INTERNAL; 1832099a0e58SBosko Milekic /* The initial zone has no Per cpu queues so it's smaller */ 1833b23f72e9SBrian Feldman zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK); 1834099a0e58SBosko Milekic 1835099a0e58SBosko Milekic #ifdef UMA_DEBUG 1836099a0e58SBosko Milekic printf("Initializing pcpu cache locks.\n"); 1837099a0e58SBosko Milekic #endif 1838099a0e58SBosko Milekic #ifdef UMA_DEBUG 1839099a0e58SBosko Milekic printf("Creating slab and hash zones.\n"); 18408355f576SJeff Roberson #endif 18418355f576SJeff Roberson 18428355f576SJeff Roberson /* Now make a zone for slab headers */ 18438355f576SJeff Roberson slabzone = uma_zcreate("UMA Slabs", 1844ef72505eSJeff Roberson sizeof(struct uma_slab), 18458355f576SJeff Roberson NULL, NULL, NULL, NULL, 1846b60f5b79SJeff Roberson UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 18478355f576SJeff Roberson 1848099a0e58SBosko Milekic /* 1849099a0e58SBosko Milekic * We also create a zone for the bigger slabs with reference 1850099a0e58SBosko Milekic * counts in them, to accomodate UMA_ZONE_REFCNT zones. 1851099a0e58SBosko Milekic */ 1852ef72505eSJeff Roberson slabsize = sizeof(struct uma_slab_refcnt); 1853ef72505eSJeff Roberson slabsize += uma_max_ipers_ref * sizeof(uint32_t); 1854099a0e58SBosko Milekic slabrefzone = uma_zcreate("UMA RCntSlabs", 1855099a0e58SBosko Milekic slabsize, 1856099a0e58SBosko Milekic NULL, NULL, NULL, NULL, 1857e66468eaSBosko Milekic UMA_ALIGN_PTR, 18587fd87882SBosko Milekic UMA_ZFLAG_INTERNAL); 1859099a0e58SBosko Milekic 18608355f576SJeff Roberson hashzone = uma_zcreate("UMA Hash", 18618355f576SJeff Roberson sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 18628355f576SJeff Roberson NULL, NULL, NULL, NULL, 1863b60f5b79SJeff Roberson UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 18648355f576SJeff Roberson 1865cae33c14SJeff Roberson bucket_init(); 18668355f576SJeff Roberson 1867342f1793SAlan Cox booted = UMA_STARTUP; 18688355f576SJeff Roberson 18698355f576SJeff Roberson #ifdef UMA_DEBUG 18708355f576SJeff Roberson printf("UMA startup complete.\n"); 18718355f576SJeff Roberson #endif 18728355f576SJeff Roberson } 18738355f576SJeff Roberson 18748355f576SJeff Roberson /* see uma.h */ 18758355f576SJeff Roberson void 187699571dc3SJeff Roberson uma_startup2(void) 18778355f576SJeff Roberson { 1878342f1793SAlan Cox booted = UMA_STARTUP2; 187986bbae32SJeff Roberson bucket_enable(); 18808355f576SJeff Roberson #ifdef UMA_DEBUG 18818355f576SJeff Roberson printf("UMA startup2 complete.\n"); 18828355f576SJeff Roberson #endif 18838355f576SJeff Roberson } 18848355f576SJeff Roberson 18858355f576SJeff Roberson /* 18868355f576SJeff Roberson * Initialize our callout handle 18878355f576SJeff Roberson * 18888355f576SJeff Roberson */ 18898355f576SJeff Roberson 18908355f576SJeff Roberson static void 18918355f576SJeff Roberson uma_startup3(void) 18928355f576SJeff Roberson { 18938355f576SJeff Roberson #ifdef UMA_DEBUG 18948355f576SJeff Roberson printf("Starting callout.\n"); 18958355f576SJeff Roberson #endif 1896a3c07611SRobert Watson callout_init(&uma_callout, CALLOUT_MPSAFE); 18979643769aSJeff Roberson callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 18988355f576SJeff Roberson #ifdef UMA_DEBUG 18998355f576SJeff Roberson printf("UMA startup3 complete.\n"); 19008355f576SJeff Roberson #endif 19018355f576SJeff Roberson } 19028355f576SJeff Roberson 1903e20a199fSJeff Roberson static uma_keg_t 1904099a0e58SBosko Milekic uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, 190585dcf349SGleb Smirnoff int align, uint32_t flags) 1906099a0e58SBosko Milekic { 1907099a0e58SBosko Milekic struct uma_kctor_args args; 1908099a0e58SBosko Milekic 1909099a0e58SBosko Milekic args.size = size; 1910099a0e58SBosko Milekic args.uminit = uminit; 1911099a0e58SBosko Milekic args.fini = fini; 19121e319f6dSRobert Watson args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align; 1913099a0e58SBosko Milekic args.flags = flags; 1914099a0e58SBosko Milekic args.zone = zone; 1915e20a199fSJeff Roberson return (zone_alloc_item(kegs, &args, M_WAITOK)); 1916099a0e58SBosko Milekic } 1917099a0e58SBosko Milekic 19188355f576SJeff Roberson /* See uma.h */ 19191e319f6dSRobert Watson void 19201e319f6dSRobert Watson uma_set_align(int align) 19211e319f6dSRobert Watson { 19221e319f6dSRobert Watson 19231e319f6dSRobert Watson if (align != UMA_ALIGN_CACHE) 19241e319f6dSRobert Watson uma_align_cache = align; 19251e319f6dSRobert Watson } 19261e319f6dSRobert Watson 19271e319f6dSRobert Watson /* See uma.h */ 19288355f576SJeff Roberson uma_zone_t 1929bb196eb4SMatthew D Fleming uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, 193085dcf349SGleb Smirnoff uma_init uminit, uma_fini fini, int align, uint32_t flags) 19318355f576SJeff Roberson 19328355f576SJeff Roberson { 19338355f576SJeff Roberson struct uma_zctor_args args; 19348355f576SJeff Roberson 19358355f576SJeff Roberson /* This stuff is essential for the zone ctor */ 19360095a784SJeff Roberson memset(&args, 0, sizeof(args)); 19378355f576SJeff Roberson args.name = name; 19388355f576SJeff Roberson args.size = size; 19398355f576SJeff Roberson args.ctor = ctor; 19408355f576SJeff Roberson args.dtor = dtor; 19418355f576SJeff Roberson args.uminit = uminit; 19428355f576SJeff Roberson args.fini = fini; 19438355f576SJeff Roberson args.align = align; 19448355f576SJeff Roberson args.flags = flags; 1945099a0e58SBosko Milekic args.keg = NULL; 1946099a0e58SBosko Milekic 1947e20a199fSJeff Roberson return (zone_alloc_item(zones, &args, M_WAITOK)); 1948099a0e58SBosko Milekic } 1949099a0e58SBosko Milekic 1950099a0e58SBosko Milekic /* See uma.h */ 1951099a0e58SBosko Milekic uma_zone_t 1952099a0e58SBosko Milekic uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, 1953099a0e58SBosko Milekic uma_init zinit, uma_fini zfini, uma_zone_t master) 1954099a0e58SBosko Milekic { 1955099a0e58SBosko Milekic struct uma_zctor_args args; 1956e20a199fSJeff Roberson uma_keg_t keg; 1957099a0e58SBosko Milekic 1958e20a199fSJeff Roberson keg = zone_first_keg(master); 19590095a784SJeff Roberson memset(&args, 0, sizeof(args)); 1960099a0e58SBosko Milekic args.name = name; 1961e20a199fSJeff Roberson args.size = keg->uk_size; 1962099a0e58SBosko Milekic args.ctor = ctor; 1963099a0e58SBosko Milekic args.dtor = dtor; 1964099a0e58SBosko Milekic args.uminit = zinit; 1965099a0e58SBosko Milekic args.fini = zfini; 1966e20a199fSJeff Roberson args.align = keg->uk_align; 1967e20a199fSJeff Roberson args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; 1968e20a199fSJeff Roberson args.keg = keg; 19698355f576SJeff Roberson 1970e20a199fSJeff Roberson /* XXX Attaches only one keg of potentially many. */ 1971e20a199fSJeff Roberson return (zone_alloc_item(zones, &args, M_WAITOK)); 19728355f576SJeff Roberson } 19738355f576SJeff Roberson 19740095a784SJeff Roberson /* See uma.h */ 19750095a784SJeff Roberson uma_zone_t 1976af526374SJeff Roberson uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor, 1977af526374SJeff Roberson uma_init zinit, uma_fini zfini, uma_import zimport, 1978af526374SJeff Roberson uma_release zrelease, void *arg, int flags) 19790095a784SJeff Roberson { 19800095a784SJeff Roberson struct uma_zctor_args args; 19810095a784SJeff Roberson 19820095a784SJeff Roberson memset(&args, 0, sizeof(args)); 19830095a784SJeff Roberson args.name = name; 1984af526374SJeff Roberson args.size = size; 19850095a784SJeff Roberson args.ctor = ctor; 19860095a784SJeff Roberson args.dtor = dtor; 19870095a784SJeff Roberson args.uminit = zinit; 19880095a784SJeff Roberson args.fini = zfini; 19890095a784SJeff Roberson args.import = zimport; 19900095a784SJeff Roberson args.release = zrelease; 19910095a784SJeff Roberson args.arg = arg; 19920095a784SJeff Roberson args.align = 0; 19930095a784SJeff Roberson args.flags = flags; 19940095a784SJeff Roberson 19950095a784SJeff Roberson return (zone_alloc_item(zones, &args, M_WAITOK)); 19960095a784SJeff Roberson } 19970095a784SJeff Roberson 1998e20a199fSJeff Roberson static void 1999e20a199fSJeff Roberson zone_lock_pair(uma_zone_t a, uma_zone_t b) 2000e20a199fSJeff Roberson { 2001e20a199fSJeff Roberson if (a < b) { 2002e20a199fSJeff Roberson ZONE_LOCK(a); 2003af526374SJeff Roberson mtx_lock_flags(b->uz_lockptr, MTX_DUPOK); 2004e20a199fSJeff Roberson } else { 2005e20a199fSJeff Roberson ZONE_LOCK(b); 2006af526374SJeff Roberson mtx_lock_flags(a->uz_lockptr, MTX_DUPOK); 2007e20a199fSJeff Roberson } 2008e20a199fSJeff Roberson } 2009e20a199fSJeff Roberson 2010e20a199fSJeff Roberson static void 2011e20a199fSJeff Roberson zone_unlock_pair(uma_zone_t a, uma_zone_t b) 2012e20a199fSJeff Roberson { 2013e20a199fSJeff Roberson 2014e20a199fSJeff Roberson ZONE_UNLOCK(a); 2015e20a199fSJeff Roberson ZONE_UNLOCK(b); 2016e20a199fSJeff Roberson } 2017e20a199fSJeff Roberson 2018e20a199fSJeff Roberson int 2019e20a199fSJeff Roberson uma_zsecond_add(uma_zone_t zone, uma_zone_t master) 2020e20a199fSJeff Roberson { 2021e20a199fSJeff Roberson uma_klink_t klink; 2022e20a199fSJeff Roberson uma_klink_t kl; 2023e20a199fSJeff Roberson int error; 2024e20a199fSJeff Roberson 2025e20a199fSJeff Roberson error = 0; 2026e20a199fSJeff Roberson klink = malloc(sizeof(*klink), M_TEMP, M_WAITOK | M_ZERO); 2027e20a199fSJeff Roberson 2028e20a199fSJeff Roberson zone_lock_pair(zone, master); 2029e20a199fSJeff Roberson /* 2030e20a199fSJeff Roberson * zone must use vtoslab() to resolve objects and must already be 2031e20a199fSJeff Roberson * a secondary. 2032e20a199fSJeff Roberson */ 2033e20a199fSJeff Roberson if ((zone->uz_flags & (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY)) 2034e20a199fSJeff Roberson != (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY)) { 2035e20a199fSJeff Roberson error = EINVAL; 2036e20a199fSJeff Roberson goto out; 2037e20a199fSJeff Roberson } 2038e20a199fSJeff Roberson /* 2039e20a199fSJeff Roberson * The new master must also use vtoslab(). 2040e20a199fSJeff Roberson */ 2041e20a199fSJeff Roberson if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) { 2042e20a199fSJeff Roberson error = EINVAL; 2043e20a199fSJeff Roberson goto out; 2044e20a199fSJeff Roberson } 2045e20a199fSJeff Roberson /* 2046e20a199fSJeff Roberson * Both must either be refcnt, or not be refcnt. 2047e20a199fSJeff Roberson */ 2048e20a199fSJeff Roberson if ((zone->uz_flags & UMA_ZONE_REFCNT) != 2049e20a199fSJeff Roberson (master->uz_flags & UMA_ZONE_REFCNT)) { 2050e20a199fSJeff Roberson error = EINVAL; 2051e20a199fSJeff Roberson goto out; 2052e20a199fSJeff Roberson } 2053e20a199fSJeff Roberson /* 2054e20a199fSJeff Roberson * The underlying object must be the same size. rsize 2055e20a199fSJeff Roberson * may be different. 2056e20a199fSJeff Roberson */ 2057e20a199fSJeff Roberson if (master->uz_size != zone->uz_size) { 2058e20a199fSJeff Roberson error = E2BIG; 2059e20a199fSJeff Roberson goto out; 2060e20a199fSJeff Roberson } 2061e20a199fSJeff Roberson /* 2062e20a199fSJeff Roberson * Put it at the end of the list. 2063e20a199fSJeff Roberson */ 2064e20a199fSJeff Roberson klink->kl_keg = zone_first_keg(master); 2065e20a199fSJeff Roberson LIST_FOREACH(kl, &zone->uz_kegs, kl_link) { 2066e20a199fSJeff Roberson if (LIST_NEXT(kl, kl_link) == NULL) { 2067e20a199fSJeff Roberson LIST_INSERT_AFTER(kl, klink, kl_link); 2068e20a199fSJeff Roberson break; 2069e20a199fSJeff Roberson } 2070e20a199fSJeff Roberson } 2071e20a199fSJeff Roberson klink = NULL; 2072e20a199fSJeff Roberson zone->uz_flags |= UMA_ZFLAG_MULTI; 2073e20a199fSJeff Roberson zone->uz_slab = zone_fetch_slab_multi; 2074e20a199fSJeff Roberson 2075e20a199fSJeff Roberson out: 2076e20a199fSJeff Roberson zone_unlock_pair(zone, master); 2077e20a199fSJeff Roberson if (klink != NULL) 2078e20a199fSJeff Roberson free(klink, M_TEMP); 2079e20a199fSJeff Roberson 2080e20a199fSJeff Roberson return (error); 2081e20a199fSJeff Roberson } 2082e20a199fSJeff Roberson 2083e20a199fSJeff Roberson 20848355f576SJeff Roberson /* See uma.h */ 20859c2cd7e5SJeff Roberson void 20869c2cd7e5SJeff Roberson uma_zdestroy(uma_zone_t zone) 20879c2cd7e5SJeff Roberson { 2088f4ff923bSRobert Watson 20890095a784SJeff Roberson zone_free_item(zones, zone, NULL, SKIP_NONE); 20909c2cd7e5SJeff Roberson } 20919c2cd7e5SJeff Roberson 20929c2cd7e5SJeff Roberson /* See uma.h */ 20938355f576SJeff Roberson void * 20942cc35ff9SJeff Roberson uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 20958355f576SJeff Roberson { 20968355f576SJeff Roberson void *item; 20978355f576SJeff Roberson uma_cache_t cache; 20988355f576SJeff Roberson uma_bucket_t bucket; 2099fc03d22bSJeff Roberson int lockfail; 21008355f576SJeff Roberson int cpu; 21018355f576SJeff Roberson 21028355f576SJeff Roberson /* This is the fast path allocation */ 21038355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC_1 21048355f576SJeff Roberson printf("Allocating one item from %s(%p)\n", zone->uz_name, zone); 21058355f576SJeff Roberson #endif 21063659f747SRobert Watson CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread, 21073659f747SRobert Watson zone->uz_name, flags); 2108a553d4b8SJeff Roberson 2109635fd505SRobert Watson if (flags & M_WAITOK) { 2110b23f72e9SBrian Feldman WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 2111635fd505SRobert Watson "uma_zalloc_arg: zone \"%s\"", zone->uz_name); 21124c1cc01cSJohn Baldwin } 21138d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD 21148d689e04SGleb Smirnoff if (memguard_cmp_zone(zone)) { 21158d689e04SGleb Smirnoff item = memguard_alloc(zone->uz_size, flags); 21168d689e04SGleb Smirnoff if (item != NULL) { 21178d689e04SGleb Smirnoff /* 21188d689e04SGleb Smirnoff * Avoid conflict with the use-after-free 21198d689e04SGleb Smirnoff * protecting infrastructure from INVARIANTS. 21208d689e04SGleb Smirnoff */ 21218d689e04SGleb Smirnoff if (zone->uz_init != NULL && 21228d689e04SGleb Smirnoff zone->uz_init != mtrash_init && 21238d689e04SGleb Smirnoff zone->uz_init(item, zone->uz_size, flags) != 0) 21248d689e04SGleb Smirnoff return (NULL); 21258d689e04SGleb Smirnoff if (zone->uz_ctor != NULL && 21268d689e04SGleb Smirnoff zone->uz_ctor != mtrash_ctor && 2127fc03d22bSJeff Roberson zone->uz_ctor(item, zone->uz_size, udata, 2128fc03d22bSJeff Roberson flags) != 0) { 21298d689e04SGleb Smirnoff zone->uz_fini(item, zone->uz_size); 21308d689e04SGleb Smirnoff return (NULL); 21318d689e04SGleb Smirnoff } 21328d689e04SGleb Smirnoff return (item); 21338d689e04SGleb Smirnoff } 21348d689e04SGleb Smirnoff /* This is unfortunate but should not be fatal. */ 21358d689e04SGleb Smirnoff } 21368d689e04SGleb Smirnoff #endif 21375d1ae027SRobert Watson /* 21385d1ae027SRobert Watson * If possible, allocate from the per-CPU cache. There are two 21395d1ae027SRobert Watson * requirements for safe access to the per-CPU cache: (1) the thread 21405d1ae027SRobert Watson * accessing the cache must not be preempted or yield during access, 21415d1ae027SRobert Watson * and (2) the thread must not migrate CPUs without switching which 21425d1ae027SRobert Watson * cache it accesses. We rely on a critical section to prevent 21435d1ae027SRobert Watson * preemption and migration. We release the critical section in 21445d1ae027SRobert Watson * order to acquire the zone mutex if we are unable to allocate from 21455d1ae027SRobert Watson * the current cache; when we re-acquire the critical section, we 21465d1ae027SRobert Watson * must detect and handle migration if it has occurred. 21475d1ae027SRobert Watson */ 21485d1ae027SRobert Watson critical_enter(); 21495d1ae027SRobert Watson cpu = curcpu; 21508355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 21518355f576SJeff Roberson 21528355f576SJeff Roberson zalloc_start: 21538355f576SJeff Roberson bucket = cache->uc_allocbucket; 2154fc03d22bSJeff Roberson if (bucket != NULL && bucket->ub_cnt > 0) { 2155cae33c14SJeff Roberson bucket->ub_cnt--; 2156cae33c14SJeff Roberson item = bucket->ub_bucket[bucket->ub_cnt]; 21578355f576SJeff Roberson #ifdef INVARIANTS 2158cae33c14SJeff Roberson bucket->ub_bucket[bucket->ub_cnt] = NULL; 21598355f576SJeff Roberson #endif 2160fc03d22bSJeff Roberson KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled.")); 21618355f576SJeff Roberson cache->uc_allocs++; 21625d1ae027SRobert Watson critical_exit(); 2163fc03d22bSJeff Roberson if (zone->uz_ctor != NULL && 2164fc03d22bSJeff Roberson zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { 21650095a784SJeff Roberson atomic_add_long(&zone->uz_fails, 1); 2166fc03d22bSJeff Roberson zone_free_item(zone, item, udata, SKIP_DTOR); 2167b23f72e9SBrian Feldman return (NULL); 2168b23f72e9SBrian Feldman } 2169ef72505eSJeff Roberson #ifdef INVARIANTS 2170ef72505eSJeff Roberson uma_dbg_alloc(zone, NULL, item); 2171ef72505eSJeff Roberson #endif 21722cc35ff9SJeff Roberson if (flags & M_ZERO) 217348343a2fSGleb Smirnoff uma_zero_item(item, zone); 21748355f576SJeff Roberson return (item); 2175fc03d22bSJeff Roberson } 2176fc03d22bSJeff Roberson 21778355f576SJeff Roberson /* 21788355f576SJeff Roberson * We have run out of items in our alloc bucket. 21798355f576SJeff Roberson * See if we can switch with our free bucket. 21808355f576SJeff Roberson */ 2181b983089aSJeff Roberson bucket = cache->uc_freebucket; 2182fc03d22bSJeff Roberson if (bucket != NULL && bucket->ub_cnt > 0) { 2183fc03d22bSJeff Roberson #ifdef UMA_DEBUG_ALLOC 2184fc03d22bSJeff Roberson printf("uma_zalloc: Swapping empty with alloc.\n"); 2185fc03d22bSJeff Roberson #endif 21868355f576SJeff Roberson cache->uc_freebucket = cache->uc_allocbucket; 2187b983089aSJeff Roberson cache->uc_allocbucket = bucket; 21888355f576SJeff Roberson goto zalloc_start; 21898355f576SJeff Roberson } 2190fc03d22bSJeff Roberson 2191fc03d22bSJeff Roberson /* 2192fc03d22bSJeff Roberson * Discard any empty allocation bucket while we hold no locks. 2193fc03d22bSJeff Roberson */ 2194fc03d22bSJeff Roberson bucket = cache->uc_allocbucket; 2195fc03d22bSJeff Roberson cache->uc_allocbucket = NULL; 2196fc03d22bSJeff Roberson critical_exit(); 2197fc03d22bSJeff Roberson if (bucket != NULL) 21986fd34d6fSJeff Roberson bucket_free(zone, bucket, udata); 2199fc03d22bSJeff Roberson 2200fc03d22bSJeff Roberson /* Short-circuit for zones without buckets and low memory. */ 2201fc03d22bSJeff Roberson if (zone->uz_count == 0 || bucketdisable) 2202fc03d22bSJeff Roberson goto zalloc_item; 2203fc03d22bSJeff Roberson 22045d1ae027SRobert Watson /* 22055d1ae027SRobert Watson * Attempt to retrieve the item from the per-CPU cache has failed, so 22065d1ae027SRobert Watson * we must go back to the zone. This requires the zone lock, so we 22075d1ae027SRobert Watson * must drop the critical section, then re-acquire it when we go back 22085d1ae027SRobert Watson * to the cache. Since the critical section is released, we may be 22095d1ae027SRobert Watson * preempted or migrate. As such, make sure not to maintain any 22105d1ae027SRobert Watson * thread-local state specific to the cache from prior to releasing 22115d1ae027SRobert Watson * the critical section. 22125d1ae027SRobert Watson */ 2213fc03d22bSJeff Roberson lockfail = 0; 2214fc03d22bSJeff Roberson if (ZONE_TRYLOCK(zone) == 0) { 2215fc03d22bSJeff Roberson /* Record contention to size the buckets. */ 2216a553d4b8SJeff Roberson ZONE_LOCK(zone); 2217fc03d22bSJeff Roberson lockfail = 1; 2218fc03d22bSJeff Roberson } 22195d1ae027SRobert Watson critical_enter(); 22205d1ae027SRobert Watson cpu = curcpu; 22215d1ae027SRobert Watson cache = &zone->uz_cpu[cpu]; 22225d1ae027SRobert Watson 2223fc03d22bSJeff Roberson /* 2224fc03d22bSJeff Roberson * Since we have locked the zone we may as well send back our stats. 2225fc03d22bSJeff Roberson */ 22260095a784SJeff Roberson atomic_add_long(&zone->uz_allocs, cache->uc_allocs); 22270095a784SJeff Roberson atomic_add_long(&zone->uz_frees, cache->uc_frees); 2228a553d4b8SJeff Roberson cache->uc_allocs = 0; 2229773df9abSRobert Watson cache->uc_frees = 0; 22308355f576SJeff Roberson 2231fc03d22bSJeff Roberson /* See if we lost the race to fill the cache. */ 2232fc03d22bSJeff Roberson if (cache->uc_allocbucket != NULL) { 2233fc03d22bSJeff Roberson ZONE_UNLOCK(zone); 2234fc03d22bSJeff Roberson goto zalloc_start; 2235a553d4b8SJeff Roberson } 22368355f576SJeff Roberson 2237fc03d22bSJeff Roberson /* 2238fc03d22bSJeff Roberson * Check the zone's cache of buckets. 2239fc03d22bSJeff Roberson */ 2240fc03d22bSJeff Roberson if ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) { 2241cae33c14SJeff Roberson KASSERT(bucket->ub_cnt != 0, 2242a553d4b8SJeff Roberson ("uma_zalloc_arg: Returning an empty bucket.")); 22438355f576SJeff Roberson 2244a553d4b8SJeff Roberson LIST_REMOVE(bucket, ub_link); 2245a553d4b8SJeff Roberson cache->uc_allocbucket = bucket; 2246a553d4b8SJeff Roberson ZONE_UNLOCK(zone); 22478355f576SJeff Roberson goto zalloc_start; 2248a553d4b8SJeff Roberson } 22495d1ae027SRobert Watson /* We are no longer associated with this CPU. */ 22505d1ae027SRobert Watson critical_exit(); 2251bbee39c6SJeff Roberson 2252fc03d22bSJeff Roberson /* 2253fc03d22bSJeff Roberson * We bump the uz count when the cache size is insufficient to 2254fc03d22bSJeff Roberson * handle the working set. 2255fc03d22bSJeff Roberson */ 22566fd34d6fSJeff Roberson if (lockfail && zone->uz_count < BUCKET_MAX) 2257a553d4b8SJeff Roberson zone->uz_count++; 2258fc03d22bSJeff Roberson ZONE_UNLOCK(zone); 2259099a0e58SBosko Milekic 22608355f576SJeff Roberson /* 2261a553d4b8SJeff Roberson * Now lets just fill a bucket and put it on the free list. If that 2262fc03d22bSJeff Roberson * works we'll restart the allocation from the begining and it 2263fc03d22bSJeff Roberson * will use the just filled bucket. 2264bbee39c6SJeff Roberson */ 22656fd34d6fSJeff Roberson bucket = zone_alloc_bucket(zone, udata, flags); 2266fc03d22bSJeff Roberson if (bucket != NULL) { 2267fc03d22bSJeff Roberson ZONE_LOCK(zone); 2268fc03d22bSJeff Roberson critical_enter(); 2269fc03d22bSJeff Roberson cpu = curcpu; 2270fc03d22bSJeff Roberson cache = &zone->uz_cpu[cpu]; 2271fc03d22bSJeff Roberson /* 2272fc03d22bSJeff Roberson * See if we lost the race or were migrated. Cache the 2273fc03d22bSJeff Roberson * initialized bucket to make this less likely or claim 2274fc03d22bSJeff Roberson * the memory directly. 2275fc03d22bSJeff Roberson */ 2276fc03d22bSJeff Roberson if (cache->uc_allocbucket == NULL) 2277fc03d22bSJeff Roberson cache->uc_allocbucket = bucket; 2278fc03d22bSJeff Roberson else 2279fc03d22bSJeff Roberson LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link); 2280bbee39c6SJeff Roberson ZONE_UNLOCK(zone); 2281fc03d22bSJeff Roberson goto zalloc_start; 2282bbee39c6SJeff Roberson } 2283fc03d22bSJeff Roberson 2284bbee39c6SJeff Roberson /* 2285bbee39c6SJeff Roberson * We may not be able to get a bucket so return an actual item. 2286bbee39c6SJeff Roberson */ 2287bbee39c6SJeff Roberson #ifdef UMA_DEBUG 2288bbee39c6SJeff Roberson printf("uma_zalloc_arg: Bucketzone returned NULL\n"); 2289bbee39c6SJeff Roberson #endif 2290bbee39c6SJeff Roberson 2291fc03d22bSJeff Roberson zalloc_item: 2292e20a199fSJeff Roberson item = zone_alloc_item(zone, udata, flags); 2293fc03d22bSJeff Roberson 2294e20a199fSJeff Roberson return (item); 2295bbee39c6SJeff Roberson } 2296bbee39c6SJeff Roberson 2297bbee39c6SJeff Roberson static uma_slab_t 2298e20a199fSJeff Roberson keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int flags) 2299bbee39c6SJeff Roberson { 2300bbee39c6SJeff Roberson uma_slab_t slab; 23016fd34d6fSJeff Roberson int reserve; 2302099a0e58SBosko Milekic 2303e20a199fSJeff Roberson mtx_assert(&keg->uk_lock, MA_OWNED); 2304bbee39c6SJeff Roberson slab = NULL; 23056fd34d6fSJeff Roberson reserve = 0; 23066fd34d6fSJeff Roberson if ((flags & M_USE_RESERVE) == 0) 23076fd34d6fSJeff Roberson reserve = keg->uk_reserve; 2308bbee39c6SJeff Roberson 2309bbee39c6SJeff Roberson for (;;) { 2310bbee39c6SJeff Roberson /* 2311bbee39c6SJeff Roberson * Find a slab with some space. Prefer slabs that are partially 2312bbee39c6SJeff Roberson * used over those that are totally full. This helps to reduce 2313bbee39c6SJeff Roberson * fragmentation. 2314bbee39c6SJeff Roberson */ 23156fd34d6fSJeff Roberson if (keg->uk_free > reserve) { 2316099a0e58SBosko Milekic if (!LIST_EMPTY(&keg->uk_part_slab)) { 2317099a0e58SBosko Milekic slab = LIST_FIRST(&keg->uk_part_slab); 2318bbee39c6SJeff Roberson } else { 2319099a0e58SBosko Milekic slab = LIST_FIRST(&keg->uk_free_slab); 2320bbee39c6SJeff Roberson LIST_REMOVE(slab, us_link); 2321099a0e58SBosko Milekic LIST_INSERT_HEAD(&keg->uk_part_slab, slab, 2322bbee39c6SJeff Roberson us_link); 2323bbee39c6SJeff Roberson } 2324e20a199fSJeff Roberson MPASS(slab->us_keg == keg); 2325bbee39c6SJeff Roberson return (slab); 2326bbee39c6SJeff Roberson } 2327bbee39c6SJeff Roberson 2328bbee39c6SJeff Roberson /* 2329bbee39c6SJeff Roberson * M_NOVM means don't ask at all! 2330bbee39c6SJeff Roberson */ 2331bbee39c6SJeff Roberson if (flags & M_NOVM) 2332bbee39c6SJeff Roberson break; 2333bbee39c6SJeff Roberson 2334e20a199fSJeff Roberson if (keg->uk_maxpages && keg->uk_pages >= keg->uk_maxpages) { 2335099a0e58SBosko Milekic keg->uk_flags |= UMA_ZFLAG_FULL; 2336e20a199fSJeff Roberson /* 2337e20a199fSJeff Roberson * If this is not a multi-zone, set the FULL bit. 2338e20a199fSJeff Roberson * Otherwise slab_multi() takes care of it. 2339e20a199fSJeff Roberson */ 23402f891cd5SPawel Jakub Dawidek if ((zone->uz_flags & UMA_ZFLAG_MULTI) == 0) { 2341e20a199fSJeff Roberson zone->uz_flags |= UMA_ZFLAG_FULL; 23422f891cd5SPawel Jakub Dawidek zone_log_warning(zone); 23432f891cd5SPawel Jakub Dawidek } 2344ebc85edfSJeff Roberson if (flags & M_NOWAIT) 2345bbee39c6SJeff Roberson break; 2346c288b548SEitan Adler zone->uz_sleeps++; 2347e20a199fSJeff Roberson msleep(keg, &keg->uk_lock, PVM, "keglimit", 0); 2348bbee39c6SJeff Roberson continue; 2349bbee39c6SJeff Roberson } 2350e20a199fSJeff Roberson slab = keg_alloc_slab(keg, zone, flags); 2351bbee39c6SJeff Roberson /* 2352bbee39c6SJeff Roberson * If we got a slab here it's safe to mark it partially used 2353bbee39c6SJeff Roberson * and return. We assume that the caller is going to remove 2354bbee39c6SJeff Roberson * at least one item. 2355bbee39c6SJeff Roberson */ 2356bbee39c6SJeff Roberson if (slab) { 2357e20a199fSJeff Roberson MPASS(slab->us_keg == keg); 2358099a0e58SBosko Milekic LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); 2359bbee39c6SJeff Roberson return (slab); 2360bbee39c6SJeff Roberson } 2361bbee39c6SJeff Roberson /* 2362bbee39c6SJeff Roberson * We might not have been able to get a slab but another cpu 2363bbee39c6SJeff Roberson * could have while we were unlocked. Check again before we 2364bbee39c6SJeff Roberson * fail. 2365bbee39c6SJeff Roberson */ 2366bbee39c6SJeff Roberson flags |= M_NOVM; 2367bbee39c6SJeff Roberson } 2368bbee39c6SJeff Roberson return (slab); 2369bbee39c6SJeff Roberson } 2370bbee39c6SJeff Roberson 2371e20a199fSJeff Roberson static uma_slab_t 2372e20a199fSJeff Roberson zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int flags) 2373e20a199fSJeff Roberson { 2374e20a199fSJeff Roberson uma_slab_t slab; 2375e20a199fSJeff Roberson 2376af526374SJeff Roberson if (keg == NULL) { 2377e20a199fSJeff Roberson keg = zone_first_keg(zone); 2378af526374SJeff Roberson KEG_LOCK(keg); 2379af526374SJeff Roberson } 2380e20a199fSJeff Roberson 2381e20a199fSJeff Roberson for (;;) { 2382e20a199fSJeff Roberson slab = keg_fetch_slab(keg, zone, flags); 2383e20a199fSJeff Roberson if (slab) 2384e20a199fSJeff Roberson return (slab); 2385e20a199fSJeff Roberson if (flags & (M_NOWAIT | M_NOVM)) 2386e20a199fSJeff Roberson break; 2387e20a199fSJeff Roberson } 2388af526374SJeff Roberson KEG_UNLOCK(keg); 2389e20a199fSJeff Roberson return (NULL); 2390e20a199fSJeff Roberson } 2391e20a199fSJeff Roberson 2392e20a199fSJeff Roberson /* 2393e20a199fSJeff Roberson * uma_zone_fetch_slab_multi: Fetches a slab from one available keg. Returns 2394af526374SJeff Roberson * with the keg locked. On NULL no lock is held. 2395e20a199fSJeff Roberson * 2396e20a199fSJeff Roberson * The last pointer is used to seed the search. It is not required. 2397e20a199fSJeff Roberson */ 2398e20a199fSJeff Roberson static uma_slab_t 2399e20a199fSJeff Roberson zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags) 2400e20a199fSJeff Roberson { 2401e20a199fSJeff Roberson uma_klink_t klink; 2402e20a199fSJeff Roberson uma_slab_t slab; 2403e20a199fSJeff Roberson uma_keg_t keg; 2404e20a199fSJeff Roberson int flags; 2405e20a199fSJeff Roberson int empty; 2406e20a199fSJeff Roberson int full; 2407e20a199fSJeff Roberson 2408e20a199fSJeff Roberson /* 2409e20a199fSJeff Roberson * Don't wait on the first pass. This will skip limit tests 2410e20a199fSJeff Roberson * as well. We don't want to block if we can find a provider 2411e20a199fSJeff Roberson * without blocking. 2412e20a199fSJeff Roberson */ 2413e20a199fSJeff Roberson flags = (rflags & ~M_WAITOK) | M_NOWAIT; 2414e20a199fSJeff Roberson /* 2415e20a199fSJeff Roberson * Use the last slab allocated as a hint for where to start 2416e20a199fSJeff Roberson * the search. 2417e20a199fSJeff Roberson */ 2418af526374SJeff Roberson if (last != NULL) { 2419e20a199fSJeff Roberson slab = keg_fetch_slab(last, zone, flags); 2420e20a199fSJeff Roberson if (slab) 2421e20a199fSJeff Roberson return (slab); 2422af526374SJeff Roberson KEG_UNLOCK(last); 2423e20a199fSJeff Roberson } 2424e20a199fSJeff Roberson /* 2425e20a199fSJeff Roberson * Loop until we have a slab incase of transient failures 2426e20a199fSJeff Roberson * while M_WAITOK is specified. I'm not sure this is 100% 2427e20a199fSJeff Roberson * required but we've done it for so long now. 2428e20a199fSJeff Roberson */ 2429e20a199fSJeff Roberson for (;;) { 2430e20a199fSJeff Roberson empty = 0; 2431e20a199fSJeff Roberson full = 0; 2432e20a199fSJeff Roberson /* 2433e20a199fSJeff Roberson * Search the available kegs for slabs. Be careful to hold the 2434e20a199fSJeff Roberson * correct lock while calling into the keg layer. 2435e20a199fSJeff Roberson */ 2436e20a199fSJeff Roberson LIST_FOREACH(klink, &zone->uz_kegs, kl_link) { 2437e20a199fSJeff Roberson keg = klink->kl_keg; 2438af526374SJeff Roberson KEG_LOCK(keg); 2439e20a199fSJeff Roberson if ((keg->uk_flags & UMA_ZFLAG_FULL) == 0) { 2440e20a199fSJeff Roberson slab = keg_fetch_slab(keg, zone, flags); 2441e20a199fSJeff Roberson if (slab) 2442e20a199fSJeff Roberson return (slab); 2443e20a199fSJeff Roberson } 2444e20a199fSJeff Roberson if (keg->uk_flags & UMA_ZFLAG_FULL) 2445e20a199fSJeff Roberson full++; 2446e20a199fSJeff Roberson else 2447e20a199fSJeff Roberson empty++; 2448af526374SJeff Roberson KEG_UNLOCK(keg); 2449e20a199fSJeff Roberson } 2450e20a199fSJeff Roberson if (rflags & (M_NOWAIT | M_NOVM)) 2451e20a199fSJeff Roberson break; 2452e20a199fSJeff Roberson flags = rflags; 2453e20a199fSJeff Roberson /* 2454e20a199fSJeff Roberson * All kegs are full. XXX We can't atomically check all kegs 2455e20a199fSJeff Roberson * and sleep so just sleep for a short period and retry. 2456e20a199fSJeff Roberson */ 2457e20a199fSJeff Roberson if (full && !empty) { 2458af526374SJeff Roberson ZONE_LOCK(zone); 2459e20a199fSJeff Roberson zone->uz_flags |= UMA_ZFLAG_FULL; 2460bf965959SSean Bruno zone->uz_sleeps++; 24612f891cd5SPawel Jakub Dawidek zone_log_warning(zone); 2462af526374SJeff Roberson msleep(zone, zone->uz_lockptr, PVM, 2463af526374SJeff Roberson "zonelimit", hz/100); 2464e20a199fSJeff Roberson zone->uz_flags &= ~UMA_ZFLAG_FULL; 2465af526374SJeff Roberson ZONE_UNLOCK(zone); 2466e20a199fSJeff Roberson continue; 2467e20a199fSJeff Roberson } 2468e20a199fSJeff Roberson } 2469e20a199fSJeff Roberson return (NULL); 2470e20a199fSJeff Roberson } 2471e20a199fSJeff Roberson 2472d56368d7SBosko Milekic static void * 24730095a784SJeff Roberson slab_alloc_item(uma_keg_t keg, uma_slab_t slab) 2474bbee39c6SJeff Roberson { 2475bbee39c6SJeff Roberson void *item; 247685dcf349SGleb Smirnoff uint8_t freei; 2477bbee39c6SJeff Roberson 24780095a784SJeff Roberson MPASS(keg == slab->us_keg); 2479e20a199fSJeff Roberson mtx_assert(&keg->uk_lock, MA_OWNED); 2480099a0e58SBosko Milekic 2481ef72505eSJeff Roberson freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1; 2482ef72505eSJeff Roberson BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free); 2483099a0e58SBosko Milekic item = slab->us_data + (keg->uk_rsize * freei); 2484bbee39c6SJeff Roberson slab->us_freecount--; 2485099a0e58SBosko Milekic keg->uk_free--; 2486ef72505eSJeff Roberson 2487bbee39c6SJeff Roberson /* Move this slab to the full list */ 2488bbee39c6SJeff Roberson if (slab->us_freecount == 0) { 2489bbee39c6SJeff Roberson LIST_REMOVE(slab, us_link); 2490099a0e58SBosko Milekic LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link); 2491bbee39c6SJeff Roberson } 2492bbee39c6SJeff Roberson 2493bbee39c6SJeff Roberson return (item); 2494bbee39c6SJeff Roberson } 2495bbee39c6SJeff Roberson 2496bbee39c6SJeff Roberson static int 24970095a784SJeff Roberson zone_import(uma_zone_t zone, void **bucket, int max, int flags) 24980095a784SJeff Roberson { 24990095a784SJeff Roberson uma_slab_t slab; 25000095a784SJeff Roberson uma_keg_t keg; 25010095a784SJeff Roberson int i; 25020095a784SJeff Roberson 25030095a784SJeff Roberson slab = NULL; 25040095a784SJeff Roberson keg = NULL; 2505af526374SJeff Roberson /* Try to keep the buckets totally full */ 25060095a784SJeff Roberson for (i = 0; i < max; ) { 25070095a784SJeff Roberson if ((slab = zone->uz_slab(zone, keg, flags)) == NULL) 25080095a784SJeff Roberson break; 25090095a784SJeff Roberson keg = slab->us_keg; 25106fd34d6fSJeff Roberson while (slab->us_freecount && i < max) { 25110095a784SJeff Roberson bucket[i++] = slab_alloc_item(keg, slab); 25126fd34d6fSJeff Roberson if (keg->uk_free <= keg->uk_reserve) 25136fd34d6fSJeff Roberson break; 25146fd34d6fSJeff Roberson } 25156fd34d6fSJeff Roberson /* Don't grab more than one slab at a time. */ 25160095a784SJeff Roberson flags &= ~M_WAITOK; 25170095a784SJeff Roberson flags |= M_NOWAIT; 25180095a784SJeff Roberson } 25190095a784SJeff Roberson if (slab != NULL) 25200095a784SJeff Roberson KEG_UNLOCK(keg); 25210095a784SJeff Roberson 25220095a784SJeff Roberson return i; 25230095a784SJeff Roberson } 25240095a784SJeff Roberson 2525fc03d22bSJeff Roberson static uma_bucket_t 25266fd34d6fSJeff Roberson zone_alloc_bucket(uma_zone_t zone, void *udata, int flags) 2527bbee39c6SJeff Roberson { 2528bbee39c6SJeff Roberson uma_bucket_t bucket; 25290095a784SJeff Roberson int max; 2530bbee39c6SJeff Roberson 25316fd34d6fSJeff Roberson /* Don't wait for buckets, preserve caller's NOVM setting. */ 25326fd34d6fSJeff Roberson bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); 25330095a784SJeff Roberson if (bucket == NULL) 2534f7104ccdSAlexander Motin return (NULL); 25350095a784SJeff Roberson 2536af526374SJeff Roberson max = MIN(bucket->ub_entries, zone->uz_count); 25370095a784SJeff Roberson bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, 25380095a784SJeff Roberson max, flags); 25390095a784SJeff Roberson 25400095a784SJeff Roberson /* 25410095a784SJeff Roberson * Initialize the memory if necessary. 25420095a784SJeff Roberson */ 25430095a784SJeff Roberson if (bucket->ub_cnt != 0 && zone->uz_init != NULL) { 2544099a0e58SBosko Milekic int i; 2545bbee39c6SJeff Roberson 25460095a784SJeff Roberson for (i = 0; i < bucket->ub_cnt; i++) 2547e20a199fSJeff Roberson if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size, 25480095a784SJeff Roberson flags) != 0) 2549b23f72e9SBrian Feldman break; 2550b23f72e9SBrian Feldman /* 2551b23f72e9SBrian Feldman * If we couldn't initialize the whole bucket, put the 2552b23f72e9SBrian Feldman * rest back onto the freelist. 2553b23f72e9SBrian Feldman */ 2554b23f72e9SBrian Feldman if (i != bucket->ub_cnt) { 2555af526374SJeff Roberson zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i], 25560095a784SJeff Roberson bucket->ub_cnt - i); 2557a5a262c6SBosko Milekic #ifdef INVARIANTS 25580095a784SJeff Roberson bzero(&bucket->ub_bucket[i], 25590095a784SJeff Roberson sizeof(void *) * (bucket->ub_cnt - i)); 2560a5a262c6SBosko Milekic #endif 2561b23f72e9SBrian Feldman bucket->ub_cnt = i; 2562b23f72e9SBrian Feldman } 2563099a0e58SBosko Milekic } 2564099a0e58SBosko Milekic 2565f7104ccdSAlexander Motin if (bucket->ub_cnt == 0) { 25666fd34d6fSJeff Roberson bucket_free(zone, bucket, udata); 2567fc03d22bSJeff Roberson atomic_add_long(&zone->uz_fails, 1); 2568fc03d22bSJeff Roberson return (NULL); 2569bbee39c6SJeff Roberson } 2570fc03d22bSJeff Roberson 2571fc03d22bSJeff Roberson return (bucket); 2572fc03d22bSJeff Roberson } 2573fc03d22bSJeff Roberson 25748355f576SJeff Roberson /* 25750095a784SJeff Roberson * Allocates a single item from a zone. 25768355f576SJeff Roberson * 25778355f576SJeff Roberson * Arguments 25788355f576SJeff Roberson * zone The zone to alloc for. 25798355f576SJeff Roberson * udata The data to be passed to the constructor. 2580a163d034SWarner Losh * flags M_WAITOK, M_NOWAIT, M_ZERO. 25818355f576SJeff Roberson * 25828355f576SJeff Roberson * Returns 25838355f576SJeff Roberson * NULL if there is no memory and M_NOWAIT is set 2584bbee39c6SJeff Roberson * An item if successful 25858355f576SJeff Roberson */ 25868355f576SJeff Roberson 25878355f576SJeff Roberson static void * 2588e20a199fSJeff Roberson zone_alloc_item(uma_zone_t zone, void *udata, int flags) 25898355f576SJeff Roberson { 25908355f576SJeff Roberson void *item; 25918355f576SJeff Roberson 25928355f576SJeff Roberson item = NULL; 25938355f576SJeff Roberson 25948355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 25958355f576SJeff Roberson printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); 25968355f576SJeff Roberson #endif 25970095a784SJeff Roberson if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1) 25980095a784SJeff Roberson goto fail; 25990095a784SJeff Roberson atomic_add_long(&zone->uz_allocs, 1); 26008355f576SJeff Roberson 2601099a0e58SBosko Milekic /* 2602099a0e58SBosko Milekic * We have to call both the zone's init (not the keg's init) 2603099a0e58SBosko Milekic * and the zone's ctor. This is because the item is going from 2604099a0e58SBosko Milekic * a keg slab directly to the user, and the user is expecting it 2605099a0e58SBosko Milekic * to be both zone-init'd as well as zone-ctor'd. 2606099a0e58SBosko Milekic */ 2607b23f72e9SBrian Feldman if (zone->uz_init != NULL) { 2608e20a199fSJeff Roberson if (zone->uz_init(item, zone->uz_size, flags) != 0) { 26090095a784SJeff Roberson zone_free_item(zone, item, udata, SKIP_FINI); 26100095a784SJeff Roberson goto fail; 2611b23f72e9SBrian Feldman } 2612b23f72e9SBrian Feldman } 2613b23f72e9SBrian Feldman if (zone->uz_ctor != NULL) { 2614e20a199fSJeff Roberson if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { 26150095a784SJeff Roberson zone_free_item(zone, item, udata, SKIP_DTOR); 26160095a784SJeff Roberson goto fail; 2617b23f72e9SBrian Feldman } 2618b23f72e9SBrian Feldman } 2619ef72505eSJeff Roberson #ifdef INVARIANTS 26200095a784SJeff Roberson uma_dbg_alloc(zone, NULL, item); 2621ef72505eSJeff Roberson #endif 26222cc35ff9SJeff Roberson if (flags & M_ZERO) 262348343a2fSGleb Smirnoff uma_zero_item(item, zone); 26248355f576SJeff Roberson 26258355f576SJeff Roberson return (item); 26260095a784SJeff Roberson 26270095a784SJeff Roberson fail: 26280095a784SJeff Roberson atomic_add_long(&zone->uz_fails, 1); 26290095a784SJeff Roberson return (NULL); 26308355f576SJeff Roberson } 26318355f576SJeff Roberson 26328355f576SJeff Roberson /* See uma.h */ 26338355f576SJeff Roberson void 26348355f576SJeff Roberson uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 26358355f576SJeff Roberson { 26368355f576SJeff Roberson uma_cache_t cache; 26378355f576SJeff Roberson uma_bucket_t bucket; 26384d104ba0SAlexander Motin int lockfail; 26398355f576SJeff Roberson int cpu; 26408355f576SJeff Roberson 26418355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC_1 26428355f576SJeff Roberson printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone); 26438355f576SJeff Roberson #endif 26443659f747SRobert Watson CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, 26453659f747SRobert Watson zone->uz_name); 26463659f747SRobert Watson 264720ed0cb0SMatthew D Fleming /* uma_zfree(..., NULL) does nothing, to match free(9). */ 264820ed0cb0SMatthew D Fleming if (item == NULL) 264920ed0cb0SMatthew D Fleming return; 26508d689e04SGleb Smirnoff #ifdef DEBUG_MEMGUARD 26518d689e04SGleb Smirnoff if (is_memguard_addr(item)) { 26528d689e04SGleb Smirnoff if (zone->uz_dtor != NULL && zone->uz_dtor != mtrash_dtor) 26538d689e04SGleb Smirnoff zone->uz_dtor(item, zone->uz_size, udata); 26548d689e04SGleb Smirnoff if (zone->uz_fini != NULL && zone->uz_fini != mtrash_fini) 26558d689e04SGleb Smirnoff zone->uz_fini(item, zone->uz_size); 26568d689e04SGleb Smirnoff memguard_free(item); 26578d689e04SGleb Smirnoff return; 26588d689e04SGleb Smirnoff } 26598d689e04SGleb Smirnoff #endif 26605d1ae027SRobert Watson #ifdef INVARIANTS 2661e20a199fSJeff Roberson if (zone->uz_flags & UMA_ZONE_MALLOC) 26625d1ae027SRobert Watson uma_dbg_free(zone, udata, item); 26635d1ae027SRobert Watson else 26645d1ae027SRobert Watson uma_dbg_free(zone, NULL, item); 26655d1ae027SRobert Watson #endif 2666fc03d22bSJeff Roberson if (zone->uz_dtor != NULL) 2667ef72505eSJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 2668ef72505eSJeff Roberson 2669af7f9b97SJeff Roberson /* 2670af7f9b97SJeff Roberson * The race here is acceptable. If we miss it we'll just have to wait 2671af7f9b97SJeff Roberson * a little longer for the limits to be reset. 2672af7f9b97SJeff Roberson */ 2673e20a199fSJeff Roberson if (zone->uz_flags & UMA_ZFLAG_FULL) 2674fc03d22bSJeff Roberson goto zfree_item; 2675af7f9b97SJeff Roberson 26765d1ae027SRobert Watson /* 26775d1ae027SRobert Watson * If possible, free to the per-CPU cache. There are two 26785d1ae027SRobert Watson * requirements for safe access to the per-CPU cache: (1) the thread 26795d1ae027SRobert Watson * accessing the cache must not be preempted or yield during access, 26805d1ae027SRobert Watson * and (2) the thread must not migrate CPUs without switching which 26815d1ae027SRobert Watson * cache it accesses. We rely on a critical section to prevent 26825d1ae027SRobert Watson * preemption and migration. We release the critical section in 26835d1ae027SRobert Watson * order to acquire the zone mutex if we are unable to free to the 26845d1ae027SRobert Watson * current cache; when we re-acquire the critical section, we must 26855d1ae027SRobert Watson * detect and handle migration if it has occurred. 26865d1ae027SRobert Watson */ 2687a553d4b8SJeff Roberson zfree_restart: 26885d1ae027SRobert Watson critical_enter(); 26895d1ae027SRobert Watson cpu = curcpu; 26908355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 26918355f576SJeff Roberson 26928355f576SJeff Roberson zfree_start: 2693a553d4b8SJeff Roberson /* 2694fc03d22bSJeff Roberson * Try to free into the allocbucket first to give LIFO ordering 2695fc03d22bSJeff Roberson * for cache-hot datastructures. Spill over into the freebucket 2696fc03d22bSJeff Roberson * if necessary. Alloc will swap them if one runs dry. 2697a553d4b8SJeff Roberson */ 2698fc03d22bSJeff Roberson bucket = cache->uc_allocbucket; 2699fc03d22bSJeff Roberson if (bucket == NULL || bucket->ub_cnt >= bucket->ub_entries) 2700fc03d22bSJeff Roberson bucket = cache->uc_freebucket; 2701fc03d22bSJeff Roberson if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) { 2702cae33c14SJeff Roberson KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL, 27038355f576SJeff Roberson ("uma_zfree: Freeing to non free bucket index.")); 2704cae33c14SJeff Roberson bucket->ub_bucket[bucket->ub_cnt] = item; 2705cae33c14SJeff Roberson bucket->ub_cnt++; 2706773df9abSRobert Watson cache->uc_frees++; 27075d1ae027SRobert Watson critical_exit(); 27088355f576SJeff Roberson return; 2709fc03d22bSJeff Roberson } 2710fc03d22bSJeff Roberson 27118355f576SJeff Roberson /* 27125d1ae027SRobert Watson * We must go back the zone, which requires acquiring the zone lock, 27135d1ae027SRobert Watson * which in turn means we must release and re-acquire the critical 27145d1ae027SRobert Watson * section. Since the critical section is released, we may be 27155d1ae027SRobert Watson * preempted or migrate. As such, make sure not to maintain any 27165d1ae027SRobert Watson * thread-local state specific to the cache from prior to releasing 27175d1ae027SRobert Watson * the critical section. 27188355f576SJeff Roberson */ 27195d1ae027SRobert Watson critical_exit(); 2720fc03d22bSJeff Roberson if (zone->uz_count == 0 || bucketdisable) 2721fc03d22bSJeff Roberson goto zfree_item; 2722fc03d22bSJeff Roberson 27234d104ba0SAlexander Motin lockfail = 0; 27244d104ba0SAlexander Motin if (ZONE_TRYLOCK(zone) == 0) { 27254d104ba0SAlexander Motin /* Record contention to size the buckets. */ 27268355f576SJeff Roberson ZONE_LOCK(zone); 27274d104ba0SAlexander Motin lockfail = 1; 27284d104ba0SAlexander Motin } 27295d1ae027SRobert Watson critical_enter(); 27305d1ae027SRobert Watson cpu = curcpu; 27315d1ae027SRobert Watson cache = &zone->uz_cpu[cpu]; 27328355f576SJeff Roberson 2733fc03d22bSJeff Roberson /* 2734fc03d22bSJeff Roberson * Since we have locked the zone we may as well send back our stats. 2735fc03d22bSJeff Roberson */ 27360095a784SJeff Roberson atomic_add_long(&zone->uz_allocs, cache->uc_allocs); 27370095a784SJeff Roberson atomic_add_long(&zone->uz_frees, cache->uc_frees); 2738f4ff923bSRobert Watson cache->uc_allocs = 0; 2739f4ff923bSRobert Watson cache->uc_frees = 0; 2740f4ff923bSRobert Watson 27418355f576SJeff Roberson bucket = cache->uc_freebucket; 2742fc03d22bSJeff Roberson if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) { 2743fc03d22bSJeff Roberson ZONE_UNLOCK(zone); 2744fc03d22bSJeff Roberson goto zfree_start; 2745fc03d22bSJeff Roberson } 27468355f576SJeff Roberson cache->uc_freebucket = NULL; 27478355f576SJeff Roberson 27488355f576SJeff Roberson /* Can we throw this on the zone full list? */ 27498355f576SJeff Roberson if (bucket != NULL) { 27508355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 27518355f576SJeff Roberson printf("uma_zfree: Putting old bucket on the free list.\n"); 27528355f576SJeff Roberson #endif 2753cae33c14SJeff Roberson /* ub_cnt is pointing to the last free item */ 2754cae33c14SJeff Roberson KASSERT(bucket->ub_cnt != 0, 27558355f576SJeff Roberson ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n")); 2756fc03d22bSJeff Roberson LIST_INSERT_HEAD(&zone->uz_buckets, bucket, ub_link); 27578355f576SJeff Roberson } 2758fc03d22bSJeff Roberson 27595d1ae027SRobert Watson /* We are no longer associated with this CPU. */ 27605d1ae027SRobert Watson critical_exit(); 2761a553d4b8SJeff Roberson 27624d104ba0SAlexander Motin /* 27634d104ba0SAlexander Motin * We bump the uz count when the cache size is insufficient to 27644d104ba0SAlexander Motin * handle the working set. 27654d104ba0SAlexander Motin */ 27664d104ba0SAlexander Motin if (lockfail && zone->uz_count < BUCKET_MAX) 27674d104ba0SAlexander Motin zone->uz_count++; 2768a553d4b8SJeff Roberson ZONE_UNLOCK(zone); 2769a553d4b8SJeff Roberson 27708355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 27718355f576SJeff Roberson printf("uma_zfree: Allocating new free bucket.\n"); 27728355f576SJeff Roberson #endif 27736fd34d6fSJeff Roberson bucket = bucket_alloc(zone, udata, M_NOWAIT); 27744741dcbfSJeff Roberson if (bucket) { 2775fc03d22bSJeff Roberson critical_enter(); 2776fc03d22bSJeff Roberson cpu = curcpu; 2777fc03d22bSJeff Roberson cache = &zone->uz_cpu[cpu]; 2778fc03d22bSJeff Roberson if (cache->uc_freebucket == NULL) { 2779fc03d22bSJeff Roberson cache->uc_freebucket = bucket; 2780fc03d22bSJeff Roberson goto zfree_start; 2781fc03d22bSJeff Roberson } 2782fc03d22bSJeff Roberson /* 2783fc03d22bSJeff Roberson * We lost the race, start over. We have to drop our 2784fc03d22bSJeff Roberson * critical section to free the bucket. 2785fc03d22bSJeff Roberson */ 2786fc03d22bSJeff Roberson critical_exit(); 27876fd34d6fSJeff Roberson bucket_free(zone, bucket, udata); 2788a553d4b8SJeff Roberson goto zfree_restart; 27898355f576SJeff Roberson } 27908355f576SJeff Roberson 2791a553d4b8SJeff Roberson /* 2792a553d4b8SJeff Roberson * If nothing else caught this, we'll just do an internal free. 2793a553d4b8SJeff Roberson */ 2794fc03d22bSJeff Roberson zfree_item: 27950095a784SJeff Roberson zone_free_item(zone, item, udata, SKIP_DTOR); 27968355f576SJeff Roberson 27978355f576SJeff Roberson return; 27988355f576SJeff Roberson } 27998355f576SJeff Roberson 28008355f576SJeff Roberson static void 28010095a784SJeff Roberson slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item) 28028355f576SJeff Roberson { 280385dcf349SGleb Smirnoff uint8_t freei; 2804099a0e58SBosko Milekic 28050095a784SJeff Roberson mtx_assert(&keg->uk_lock, MA_OWNED); 2806e20a199fSJeff Roberson MPASS(keg == slab->us_keg); 28078355f576SJeff Roberson 28088355f576SJeff Roberson /* Do we need to remove from any lists? */ 2809099a0e58SBosko Milekic if (slab->us_freecount+1 == keg->uk_ipers) { 28108355f576SJeff Roberson LIST_REMOVE(slab, us_link); 2811099a0e58SBosko Milekic LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 28128355f576SJeff Roberson } else if (slab->us_freecount == 0) { 28138355f576SJeff Roberson LIST_REMOVE(slab, us_link); 2814099a0e58SBosko Milekic LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); 28158355f576SJeff Roberson } 28168355f576SJeff Roberson 2817ef72505eSJeff Roberson /* Slab management. */ 2818ef72505eSJeff Roberson freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; 2819ef72505eSJeff Roberson BIT_SET(SLAB_SETSIZE, freei, &slab->us_free); 28208355f576SJeff Roberson slab->us_freecount++; 28218355f576SJeff Roberson 2822ef72505eSJeff Roberson /* Keg statistics. */ 2823099a0e58SBosko Milekic keg->uk_free++; 28240095a784SJeff Roberson } 28250095a784SJeff Roberson 28260095a784SJeff Roberson static void 28270095a784SJeff Roberson zone_release(uma_zone_t zone, void **bucket, int cnt) 28280095a784SJeff Roberson { 28290095a784SJeff Roberson void *item; 28300095a784SJeff Roberson uma_slab_t slab; 28310095a784SJeff Roberson uma_keg_t keg; 28320095a784SJeff Roberson uint8_t *mem; 28330095a784SJeff Roberson int clearfull; 28340095a784SJeff Roberson int i; 28358355f576SJeff Roberson 2836e20a199fSJeff Roberson clearfull = 0; 28370095a784SJeff Roberson keg = zone_first_keg(zone); 2838af526374SJeff Roberson KEG_LOCK(keg); 28390095a784SJeff Roberson for (i = 0; i < cnt; i++) { 28400095a784SJeff Roberson item = bucket[i]; 28410095a784SJeff Roberson if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) { 28420095a784SJeff Roberson mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); 28430095a784SJeff Roberson if (zone->uz_flags & UMA_ZONE_HASH) { 28440095a784SJeff Roberson slab = hash_sfind(&keg->uk_hash, mem); 28450095a784SJeff Roberson } else { 28460095a784SJeff Roberson mem += keg->uk_pgoff; 28470095a784SJeff Roberson slab = (uma_slab_t)mem; 28480095a784SJeff Roberson } 28490095a784SJeff Roberson } else { 28500095a784SJeff Roberson slab = vtoslab((vm_offset_t)item); 28510095a784SJeff Roberson if (slab->us_keg != keg) { 28520095a784SJeff Roberson KEG_UNLOCK(keg); 28530095a784SJeff Roberson keg = slab->us_keg; 28540095a784SJeff Roberson KEG_LOCK(keg); 28550095a784SJeff Roberson } 28560095a784SJeff Roberson } 28570095a784SJeff Roberson slab_free_item(keg, slab, item); 2858099a0e58SBosko Milekic if (keg->uk_flags & UMA_ZFLAG_FULL) { 2859e20a199fSJeff Roberson if (keg->uk_pages < keg->uk_maxpages) { 2860099a0e58SBosko Milekic keg->uk_flags &= ~UMA_ZFLAG_FULL; 2861e20a199fSJeff Roberson clearfull = 1; 2862e20a199fSJeff Roberson } 2863af7f9b97SJeff Roberson 286477380291SMohan Srinivasan /* 2865ef72505eSJeff Roberson * We can handle one more allocation. Since we're 2866ef72505eSJeff Roberson * clearing ZFLAG_FULL, wake up all procs blocked 2867ef72505eSJeff Roberson * on pages. This should be uncommon, so keeping this 2868ef72505eSJeff Roberson * simple for now (rather than adding count of blocked 286977380291SMohan Srinivasan * threads etc). 287077380291SMohan Srinivasan */ 287177380291SMohan Srinivasan wakeup(keg); 2872af7f9b97SJeff Roberson } 28730095a784SJeff Roberson } 2874af526374SJeff Roberson KEG_UNLOCK(keg); 28750095a784SJeff Roberson if (clearfull) { 2876af526374SJeff Roberson ZONE_LOCK(zone); 2877e20a199fSJeff Roberson zone->uz_flags &= ~UMA_ZFLAG_FULL; 2878e20a199fSJeff Roberson wakeup(zone); 2879605cbd6aSJeff Roberson ZONE_UNLOCK(zone); 2880af526374SJeff Roberson } 2881ef72505eSJeff Roberson 28828355f576SJeff Roberson } 28838355f576SJeff Roberson 28840095a784SJeff Roberson /* 28850095a784SJeff Roberson * Frees a single item to any zone. 28860095a784SJeff Roberson * 28870095a784SJeff Roberson * Arguments: 28880095a784SJeff Roberson * zone The zone to free to 28890095a784SJeff Roberson * item The item we're freeing 28900095a784SJeff Roberson * udata User supplied data for the dtor 28910095a784SJeff Roberson * skip Skip dtors and finis 28920095a784SJeff Roberson */ 28930095a784SJeff Roberson static void 28940095a784SJeff Roberson zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) 28950095a784SJeff Roberson { 28960095a784SJeff Roberson 28970095a784SJeff Roberson #ifdef INVARIANTS 28980095a784SJeff Roberson if (skip == SKIP_NONE) { 28990095a784SJeff Roberson if (zone->uz_flags & UMA_ZONE_MALLOC) 29000095a784SJeff Roberson uma_dbg_free(zone, udata, item); 29010095a784SJeff Roberson else 29020095a784SJeff Roberson uma_dbg_free(zone, NULL, item); 29030095a784SJeff Roberson } 29040095a784SJeff Roberson #endif 29050095a784SJeff Roberson if (skip < SKIP_DTOR && zone->uz_dtor) 29060095a784SJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 29070095a784SJeff Roberson 29080095a784SJeff Roberson if (skip < SKIP_FINI && zone->uz_fini) 29090095a784SJeff Roberson zone->uz_fini(item, zone->uz_size); 29100095a784SJeff Roberson 29110095a784SJeff Roberson atomic_add_long(&zone->uz_frees, 1); 29120095a784SJeff Roberson zone->uz_release(zone->uz_arg, &item, 1); 29130095a784SJeff Roberson } 29140095a784SJeff Roberson 29158355f576SJeff Roberson /* See uma.h */ 29161c6cae97SLawrence Stewart int 2917736ee590SJeff Roberson uma_zone_set_max(uma_zone_t zone, int nitems) 2918736ee590SJeff Roberson { 2919099a0e58SBosko Milekic uma_keg_t keg; 2920099a0e58SBosko Milekic 2921e20a199fSJeff Roberson keg = zone_first_keg(zone); 29220095a784SJeff Roberson if (keg == NULL) 29230095a784SJeff Roberson return (0); 2924af526374SJeff Roberson KEG_LOCK(keg); 2925e20a199fSJeff Roberson keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera; 2926099a0e58SBosko Milekic if (keg->uk_maxpages * keg->uk_ipers < nitems) 2927e20a199fSJeff Roberson keg->uk_maxpages += keg->uk_ppera; 29281c6cae97SLawrence Stewart nitems = keg->uk_maxpages * keg->uk_ipers; 2929af526374SJeff Roberson KEG_UNLOCK(keg); 29301c6cae97SLawrence Stewart 29311c6cae97SLawrence Stewart return (nitems); 2932736ee590SJeff Roberson } 2933736ee590SJeff Roberson 2934736ee590SJeff Roberson /* See uma.h */ 2935e49471b0SAndre Oppermann int 2936e49471b0SAndre Oppermann uma_zone_get_max(uma_zone_t zone) 2937e49471b0SAndre Oppermann { 2938e49471b0SAndre Oppermann int nitems; 2939e49471b0SAndre Oppermann uma_keg_t keg; 2940e49471b0SAndre Oppermann 2941e49471b0SAndre Oppermann keg = zone_first_keg(zone); 29420095a784SJeff Roberson if (keg == NULL) 29430095a784SJeff Roberson return (0); 2944af526374SJeff Roberson KEG_LOCK(keg); 2945e49471b0SAndre Oppermann nitems = keg->uk_maxpages * keg->uk_ipers; 2946af526374SJeff Roberson KEG_UNLOCK(keg); 2947e49471b0SAndre Oppermann 2948e49471b0SAndre Oppermann return (nitems); 2949e49471b0SAndre Oppermann } 2950e49471b0SAndre Oppermann 2951e49471b0SAndre Oppermann /* See uma.h */ 29522f891cd5SPawel Jakub Dawidek void 29532f891cd5SPawel Jakub Dawidek uma_zone_set_warning(uma_zone_t zone, const char *warning) 29542f891cd5SPawel Jakub Dawidek { 29552f891cd5SPawel Jakub Dawidek 29562f891cd5SPawel Jakub Dawidek ZONE_LOCK(zone); 29572f891cd5SPawel Jakub Dawidek zone->uz_warning = warning; 29582f891cd5SPawel Jakub Dawidek ZONE_UNLOCK(zone); 29592f891cd5SPawel Jakub Dawidek } 29602f891cd5SPawel Jakub Dawidek 29612f891cd5SPawel Jakub Dawidek /* See uma.h */ 2962c4ae7908SLawrence Stewart int 2963c4ae7908SLawrence Stewart uma_zone_get_cur(uma_zone_t zone) 2964c4ae7908SLawrence Stewart { 2965c4ae7908SLawrence Stewart int64_t nitems; 2966c4ae7908SLawrence Stewart u_int i; 2967c4ae7908SLawrence Stewart 2968c4ae7908SLawrence Stewart ZONE_LOCK(zone); 2969c4ae7908SLawrence Stewart nitems = zone->uz_allocs - zone->uz_frees; 2970c4ae7908SLawrence Stewart CPU_FOREACH(i) { 2971c4ae7908SLawrence Stewart /* 2972c4ae7908SLawrence Stewart * See the comment in sysctl_vm_zone_stats() regarding the 2973c4ae7908SLawrence Stewart * safety of accessing the per-cpu caches. With the zone lock 2974c4ae7908SLawrence Stewart * held, it is safe, but can potentially result in stale data. 2975c4ae7908SLawrence Stewart */ 2976c4ae7908SLawrence Stewart nitems += zone->uz_cpu[i].uc_allocs - 2977c4ae7908SLawrence Stewart zone->uz_cpu[i].uc_frees; 2978c4ae7908SLawrence Stewart } 2979c4ae7908SLawrence Stewart ZONE_UNLOCK(zone); 2980c4ae7908SLawrence Stewart 2981c4ae7908SLawrence Stewart return (nitems < 0 ? 0 : nitems); 2982c4ae7908SLawrence Stewart } 2983c4ae7908SLawrence Stewart 2984c4ae7908SLawrence Stewart /* See uma.h */ 2985736ee590SJeff Roberson void 2986099a0e58SBosko Milekic uma_zone_set_init(uma_zone_t zone, uma_init uminit) 2987099a0e58SBosko Milekic { 2988e20a199fSJeff Roberson uma_keg_t keg; 2989e20a199fSJeff Roberson 2990e20a199fSJeff Roberson keg = zone_first_keg(zone); 29910095a784SJeff Roberson KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type")); 2992af526374SJeff Roberson KEG_LOCK(keg); 2993e20a199fSJeff Roberson KASSERT(keg->uk_pages == 0, 2994099a0e58SBosko Milekic ("uma_zone_set_init on non-empty keg")); 2995e20a199fSJeff Roberson keg->uk_init = uminit; 2996af526374SJeff Roberson KEG_UNLOCK(keg); 2997099a0e58SBosko Milekic } 2998099a0e58SBosko Milekic 2999099a0e58SBosko Milekic /* See uma.h */ 3000099a0e58SBosko Milekic void 3001099a0e58SBosko Milekic uma_zone_set_fini(uma_zone_t zone, uma_fini fini) 3002099a0e58SBosko Milekic { 3003e20a199fSJeff Roberson uma_keg_t keg; 3004e20a199fSJeff Roberson 3005e20a199fSJeff Roberson keg = zone_first_keg(zone); 30060095a784SJeff Roberson KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type")); 3007af526374SJeff Roberson KEG_LOCK(keg); 3008e20a199fSJeff Roberson KASSERT(keg->uk_pages == 0, 3009099a0e58SBosko Milekic ("uma_zone_set_fini on non-empty keg")); 3010e20a199fSJeff Roberson keg->uk_fini = fini; 3011af526374SJeff Roberson KEG_UNLOCK(keg); 3012099a0e58SBosko Milekic } 3013099a0e58SBosko Milekic 3014099a0e58SBosko Milekic /* See uma.h */ 3015099a0e58SBosko Milekic void 3016099a0e58SBosko Milekic uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) 3017099a0e58SBosko Milekic { 3018af526374SJeff Roberson 3019099a0e58SBosko Milekic ZONE_LOCK(zone); 3020e20a199fSJeff Roberson KASSERT(zone_first_keg(zone)->uk_pages == 0, 3021099a0e58SBosko Milekic ("uma_zone_set_zinit on non-empty keg")); 3022099a0e58SBosko Milekic zone->uz_init = zinit; 3023099a0e58SBosko Milekic ZONE_UNLOCK(zone); 3024099a0e58SBosko Milekic } 3025099a0e58SBosko Milekic 3026099a0e58SBosko Milekic /* See uma.h */ 3027099a0e58SBosko Milekic void 3028099a0e58SBosko Milekic uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) 3029099a0e58SBosko Milekic { 3030af526374SJeff Roberson 3031099a0e58SBosko Milekic ZONE_LOCK(zone); 3032e20a199fSJeff Roberson KASSERT(zone_first_keg(zone)->uk_pages == 0, 3033099a0e58SBosko Milekic ("uma_zone_set_zfini on non-empty keg")); 3034099a0e58SBosko Milekic zone->uz_fini = zfini; 3035099a0e58SBosko Milekic ZONE_UNLOCK(zone); 3036099a0e58SBosko Milekic } 3037099a0e58SBosko Milekic 3038099a0e58SBosko Milekic /* See uma.h */ 3039b23f72e9SBrian Feldman /* XXX uk_freef is not actually used with the zone locked */ 3040099a0e58SBosko Milekic void 30418355f576SJeff Roberson uma_zone_set_freef(uma_zone_t zone, uma_free freef) 30428355f576SJeff Roberson { 30430095a784SJeff Roberson uma_keg_t keg; 3044e20a199fSJeff Roberson 30450095a784SJeff Roberson keg = zone_first_keg(zone); 30460095a784SJeff Roberson KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type")); 3047af526374SJeff Roberson KEG_LOCK(keg); 30480095a784SJeff Roberson keg->uk_freef = freef; 3049af526374SJeff Roberson KEG_UNLOCK(keg); 30508355f576SJeff Roberson } 30518355f576SJeff Roberson 30528355f576SJeff Roberson /* See uma.h */ 3053b23f72e9SBrian Feldman /* XXX uk_allocf is not actually used with the zone locked */ 30548355f576SJeff Roberson void 30558355f576SJeff Roberson uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 30568355f576SJeff Roberson { 3057e20a199fSJeff Roberson uma_keg_t keg; 3058e20a199fSJeff Roberson 3059e20a199fSJeff Roberson keg = zone_first_keg(zone); 3060af526374SJeff Roberson KEG_LOCK(keg); 3061e20a199fSJeff Roberson keg->uk_allocf = allocf; 3062af526374SJeff Roberson KEG_UNLOCK(keg); 30638355f576SJeff Roberson } 30648355f576SJeff Roberson 30658355f576SJeff Roberson /* See uma.h */ 30666fd34d6fSJeff Roberson void 30676fd34d6fSJeff Roberson uma_zone_reserve(uma_zone_t zone, int items) 30686fd34d6fSJeff Roberson { 30696fd34d6fSJeff Roberson uma_keg_t keg; 30706fd34d6fSJeff Roberson 30716fd34d6fSJeff Roberson keg = zone_first_keg(zone); 30726fd34d6fSJeff Roberson if (keg == NULL) 30736fd34d6fSJeff Roberson return; 30746fd34d6fSJeff Roberson KEG_LOCK(keg); 30756fd34d6fSJeff Roberson keg->uk_reserve = items; 30766fd34d6fSJeff Roberson KEG_UNLOCK(keg); 30776fd34d6fSJeff Roberson 30786fd34d6fSJeff Roberson return; 30796fd34d6fSJeff Roberson } 30806fd34d6fSJeff Roberson 30816fd34d6fSJeff Roberson /* See uma.h */ 30828355f576SJeff Roberson int 3083a4915c21SAttilio Rao uma_zone_reserve_kva(uma_zone_t zone, int count) 30848355f576SJeff Roberson { 3085099a0e58SBosko Milekic uma_keg_t keg; 30868355f576SJeff Roberson vm_offset_t kva; 3087099a0e58SBosko Milekic int pages; 30888355f576SJeff Roberson 3089e20a199fSJeff Roberson keg = zone_first_keg(zone); 30900095a784SJeff Roberson if (keg == NULL) 30910095a784SJeff Roberson return (0); 3092099a0e58SBosko Milekic pages = count / keg->uk_ipers; 30938355f576SJeff Roberson 3094099a0e58SBosko Milekic if (pages * keg->uk_ipers < count) 30958355f576SJeff Roberson pages++; 3096a553d4b8SJeff Roberson 3097a4915c21SAttilio Rao #ifdef UMA_MD_SMALL_ALLOC 3098a4915c21SAttilio Rao if (keg->uk_ppera > 1) { 3099a4915c21SAttilio Rao #else 3100a4915c21SAttilio Rao if (1) { 3101a4915c21SAttilio Rao #endif 31025df87b21SJeff Roberson kva = kva_alloc(pages * UMA_SLAB_SIZE); 3103d1f42ac2SAlan Cox if (kva == 0) 31048355f576SJeff Roberson return (0); 3105a4915c21SAttilio Rao } else 3106a4915c21SAttilio Rao kva = 0; 3107af526374SJeff Roberson KEG_LOCK(keg); 3108099a0e58SBosko Milekic keg->uk_kva = kva; 3109a4915c21SAttilio Rao keg->uk_offset = 0; 3110099a0e58SBosko Milekic keg->uk_maxpages = pages; 3111a4915c21SAttilio Rao #ifdef UMA_MD_SMALL_ALLOC 3112a4915c21SAttilio Rao keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc; 3113a4915c21SAttilio Rao #else 3114a4915c21SAttilio Rao keg->uk_allocf = noobj_alloc; 3115a4915c21SAttilio Rao #endif 31166fd34d6fSJeff Roberson keg->uk_flags |= UMA_ZONE_NOFREE; 3117af526374SJeff Roberson KEG_UNLOCK(keg); 3118af526374SJeff Roberson 31198355f576SJeff Roberson return (1); 31208355f576SJeff Roberson } 31218355f576SJeff Roberson 31228355f576SJeff Roberson /* See uma.h */ 31238355f576SJeff Roberson void 31248355f576SJeff Roberson uma_prealloc(uma_zone_t zone, int items) 31258355f576SJeff Roberson { 31268355f576SJeff Roberson int slabs; 31278355f576SJeff Roberson uma_slab_t slab; 3128099a0e58SBosko Milekic uma_keg_t keg; 31298355f576SJeff Roberson 3130e20a199fSJeff Roberson keg = zone_first_keg(zone); 31310095a784SJeff Roberson if (keg == NULL) 31320095a784SJeff Roberson return; 3133af526374SJeff Roberson KEG_LOCK(keg); 3134099a0e58SBosko Milekic slabs = items / keg->uk_ipers; 3135099a0e58SBosko Milekic if (slabs * keg->uk_ipers < items) 31368355f576SJeff Roberson slabs++; 31378355f576SJeff Roberson while (slabs > 0) { 3138e20a199fSJeff Roberson slab = keg_alloc_slab(keg, zone, M_WAITOK); 3139e20a199fSJeff Roberson if (slab == NULL) 3140e20a199fSJeff Roberson break; 3141e20a199fSJeff Roberson MPASS(slab->us_keg == keg); 3142099a0e58SBosko Milekic LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 31438355f576SJeff Roberson slabs--; 31448355f576SJeff Roberson } 3145af526374SJeff Roberson KEG_UNLOCK(keg); 31468355f576SJeff Roberson } 31478355f576SJeff Roberson 31488355f576SJeff Roberson /* See uma.h */ 314985dcf349SGleb Smirnoff uint32_t * 3150099a0e58SBosko Milekic uma_find_refcnt(uma_zone_t zone, void *item) 3151099a0e58SBosko Milekic { 3152ab14a3f7SBrian Feldman uma_slabrefcnt_t slabref; 3153ef72505eSJeff Roberson uma_slab_t slab; 3154099a0e58SBosko Milekic uma_keg_t keg; 315585dcf349SGleb Smirnoff uint32_t *refcnt; 3156099a0e58SBosko Milekic int idx; 3157099a0e58SBosko Milekic 3158ef72505eSJeff Roberson slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK)); 3159ef72505eSJeff Roberson slabref = (uma_slabrefcnt_t)slab; 3160ef72505eSJeff Roberson keg = slab->us_keg; 3161ef72505eSJeff Roberson KASSERT(keg->uk_flags & UMA_ZONE_REFCNT, 3162099a0e58SBosko Milekic ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT")); 3163ef72505eSJeff Roberson idx = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; 3164ef72505eSJeff Roberson refcnt = &slabref->us_refcnt[idx]; 3165099a0e58SBosko Milekic return refcnt; 3166099a0e58SBosko Milekic } 3167099a0e58SBosko Milekic 3168099a0e58SBosko Milekic /* See uma.h */ 31698355f576SJeff Roberson void 31708355f576SJeff Roberson uma_reclaim(void) 31718355f576SJeff Roberson { 31728355f576SJeff Roberson #ifdef UMA_DEBUG 31738355f576SJeff Roberson printf("UMA: vm asked us to release pages!\n"); 31748355f576SJeff Roberson #endif 317586bbae32SJeff Roberson bucket_enable(); 31768355f576SJeff Roberson zone_foreach(zone_drain); 3177a2de44abSAlexander Motin if (vm_page_count_min()) { 3178a2de44abSAlexander Motin cache_drain_safe(NULL); 3179a2de44abSAlexander Motin zone_foreach(zone_drain); 3180a2de44abSAlexander Motin } 31818355f576SJeff Roberson /* 31828355f576SJeff Roberson * Some slabs may have been freed but this zone will be visited early 31838355f576SJeff Roberson * we visit again so that we can free pages that are empty once other 31848355f576SJeff Roberson * zones are drained. We have to do the same for buckets. 31858355f576SJeff Roberson */ 31869643769aSJeff Roberson zone_drain(slabzone); 3187099a0e58SBosko Milekic zone_drain(slabrefzone); 3188cae33c14SJeff Roberson bucket_zone_drain(); 31898355f576SJeff Roberson } 31908355f576SJeff Roberson 3191663b416fSJohn Baldwin /* See uma.h */ 3192663b416fSJohn Baldwin int 3193663b416fSJohn Baldwin uma_zone_exhausted(uma_zone_t zone) 3194663b416fSJohn Baldwin { 3195663b416fSJohn Baldwin int full; 3196663b416fSJohn Baldwin 3197663b416fSJohn Baldwin ZONE_LOCK(zone); 3198e20a199fSJeff Roberson full = (zone->uz_flags & UMA_ZFLAG_FULL); 3199663b416fSJohn Baldwin ZONE_UNLOCK(zone); 3200663b416fSJohn Baldwin return (full); 3201663b416fSJohn Baldwin } 3202663b416fSJohn Baldwin 32036c125b8dSMohan Srinivasan int 32046c125b8dSMohan Srinivasan uma_zone_exhausted_nolock(uma_zone_t zone) 32056c125b8dSMohan Srinivasan { 3206e20a199fSJeff Roberson return (zone->uz_flags & UMA_ZFLAG_FULL); 32076c125b8dSMohan Srinivasan } 32086c125b8dSMohan Srinivasan 32098355f576SJeff Roberson void * 32108355f576SJeff Roberson uma_large_malloc(int size, int wait) 32118355f576SJeff Roberson { 32128355f576SJeff Roberson void *mem; 32138355f576SJeff Roberson uma_slab_t slab; 321485dcf349SGleb Smirnoff uint8_t flags; 32158355f576SJeff Roberson 3216e20a199fSJeff Roberson slab = zone_alloc_item(slabzone, NULL, wait); 32178355f576SJeff Roberson if (slab == NULL) 32188355f576SJeff Roberson return (NULL); 32198355f576SJeff Roberson mem = page_alloc(NULL, size, &flags, wait); 32208355f576SJeff Roberson if (mem) { 322199571dc3SJeff Roberson vsetslab((vm_offset_t)mem, slab); 32228355f576SJeff Roberson slab->us_data = mem; 32238355f576SJeff Roberson slab->us_flags = flags | UMA_SLAB_MALLOC; 32248355f576SJeff Roberson slab->us_size = size; 32258355f576SJeff Roberson } else { 32260095a784SJeff Roberson zone_free_item(slabzone, slab, NULL, SKIP_NONE); 32278355f576SJeff Roberson } 32288355f576SJeff Roberson 32298355f576SJeff Roberson return (mem); 32308355f576SJeff Roberson } 32318355f576SJeff Roberson 32328355f576SJeff Roberson void 32338355f576SJeff Roberson uma_large_free(uma_slab_t slab) 32348355f576SJeff Roberson { 3235c325e866SKonstantin Belousov 32368355f576SJeff Roberson page_free(slab->us_data, slab->us_size, slab->us_flags); 32370095a784SJeff Roberson zone_free_item(slabzone, slab, NULL, SKIP_NONE); 32388355f576SJeff Roberson } 32398355f576SJeff Roberson 324048343a2fSGleb Smirnoff static void 324148343a2fSGleb Smirnoff uma_zero_item(void *item, uma_zone_t zone) 324248343a2fSGleb Smirnoff { 324348343a2fSGleb Smirnoff 324448343a2fSGleb Smirnoff if (zone->uz_flags & UMA_ZONE_PCPU) { 324548343a2fSGleb Smirnoff for (int i = 0; i < mp_ncpus; i++) 324648343a2fSGleb Smirnoff bzero(zpcpu_get_cpu(item, i), zone->uz_size); 324748343a2fSGleb Smirnoff } else 324848343a2fSGleb Smirnoff bzero(item, zone->uz_size); 324948343a2fSGleb Smirnoff } 325048343a2fSGleb Smirnoff 32518355f576SJeff Roberson void 32528355f576SJeff Roberson uma_print_stats(void) 32538355f576SJeff Roberson { 32548355f576SJeff Roberson zone_foreach(uma_print_zone); 32558355f576SJeff Roberson } 32568355f576SJeff Roberson 3257504d5de3SJeff Roberson static void 3258504d5de3SJeff Roberson slab_print(uma_slab_t slab) 3259504d5de3SJeff Roberson { 3260ef72505eSJeff Roberson printf("slab: keg %p, data %p, freecount %d\n", 3261ef72505eSJeff Roberson slab->us_keg, slab->us_data, slab->us_freecount); 3262504d5de3SJeff Roberson } 3263504d5de3SJeff Roberson 3264504d5de3SJeff Roberson static void 3265504d5de3SJeff Roberson cache_print(uma_cache_t cache) 3266504d5de3SJeff Roberson { 3267504d5de3SJeff Roberson printf("alloc: %p(%d), free: %p(%d)\n", 3268504d5de3SJeff Roberson cache->uc_allocbucket, 3269504d5de3SJeff Roberson cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0, 3270504d5de3SJeff Roberson cache->uc_freebucket, 3271504d5de3SJeff Roberson cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0); 3272504d5de3SJeff Roberson } 3273504d5de3SJeff Roberson 3274e20a199fSJeff Roberson static void 3275e20a199fSJeff Roberson uma_print_keg(uma_keg_t keg) 32768355f576SJeff Roberson { 3277504d5de3SJeff Roberson uma_slab_t slab; 3278504d5de3SJeff Roberson 32790b80c1e4SEitan Adler printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d " 3280e20a199fSJeff Roberson "out %d free %d limit %d\n", 3281e20a199fSJeff Roberson keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags, 3282099a0e58SBosko Milekic keg->uk_ipers, keg->uk_ppera, 3283e20a199fSJeff Roberson (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free, 3284e20a199fSJeff Roberson (keg->uk_maxpages / keg->uk_ppera) * keg->uk_ipers); 3285504d5de3SJeff Roberson printf("Part slabs:\n"); 3286099a0e58SBosko Milekic LIST_FOREACH(slab, &keg->uk_part_slab, us_link) 3287504d5de3SJeff Roberson slab_print(slab); 3288504d5de3SJeff Roberson printf("Free slabs:\n"); 3289099a0e58SBosko Milekic LIST_FOREACH(slab, &keg->uk_free_slab, us_link) 3290504d5de3SJeff Roberson slab_print(slab); 3291504d5de3SJeff Roberson printf("Full slabs:\n"); 3292099a0e58SBosko Milekic LIST_FOREACH(slab, &keg->uk_full_slab, us_link) 3293504d5de3SJeff Roberson slab_print(slab); 3294e20a199fSJeff Roberson } 3295e20a199fSJeff Roberson 3296e20a199fSJeff Roberson void 3297e20a199fSJeff Roberson uma_print_zone(uma_zone_t zone) 3298e20a199fSJeff Roberson { 3299e20a199fSJeff Roberson uma_cache_t cache; 3300e20a199fSJeff Roberson uma_klink_t kl; 3301e20a199fSJeff Roberson int i; 3302e20a199fSJeff Roberson 33030b80c1e4SEitan Adler printf("zone: %s(%p) size %d flags %#x\n", 3304e20a199fSJeff Roberson zone->uz_name, zone, zone->uz_size, zone->uz_flags); 3305e20a199fSJeff Roberson LIST_FOREACH(kl, &zone->uz_kegs, kl_link) 3306e20a199fSJeff Roberson uma_print_keg(kl->kl_keg); 33073aa6d94eSJohn Baldwin CPU_FOREACH(i) { 3308504d5de3SJeff Roberson cache = &zone->uz_cpu[i]; 3309504d5de3SJeff Roberson printf("CPU %d Cache:\n", i); 3310504d5de3SJeff Roberson cache_print(cache); 3311504d5de3SJeff Roberson } 33128355f576SJeff Roberson } 33138355f576SJeff Roberson 3314a0d4b0aeSRobert Watson #ifdef DDB 33158355f576SJeff Roberson /* 33167a52a97eSRobert Watson * Generate statistics across both the zone and its per-cpu cache's. Return 33177a52a97eSRobert Watson * desired statistics if the pointer is non-NULL for that statistic. 33187a52a97eSRobert Watson * 33197a52a97eSRobert Watson * Note: does not update the zone statistics, as it can't safely clear the 33207a52a97eSRobert Watson * per-CPU cache statistic. 33217a52a97eSRobert Watson * 33227a52a97eSRobert Watson * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't 33237a52a97eSRobert Watson * safe from off-CPU; we should modify the caches to track this information 33247a52a97eSRobert Watson * directly so that we don't have to. 33257a52a97eSRobert Watson */ 33267a52a97eSRobert Watson static void 332785dcf349SGleb Smirnoff uma_zone_sumstat(uma_zone_t z, int *cachefreep, uint64_t *allocsp, 332885dcf349SGleb Smirnoff uint64_t *freesp, uint64_t *sleepsp) 33297a52a97eSRobert Watson { 33307a52a97eSRobert Watson uma_cache_t cache; 333185dcf349SGleb Smirnoff uint64_t allocs, frees, sleeps; 33327a52a97eSRobert Watson int cachefree, cpu; 33337a52a97eSRobert Watson 3334bf965959SSean Bruno allocs = frees = sleeps = 0; 33357a52a97eSRobert Watson cachefree = 0; 33363aa6d94eSJohn Baldwin CPU_FOREACH(cpu) { 33377a52a97eSRobert Watson cache = &z->uz_cpu[cpu]; 33387a52a97eSRobert Watson if (cache->uc_allocbucket != NULL) 33397a52a97eSRobert Watson cachefree += cache->uc_allocbucket->ub_cnt; 33407a52a97eSRobert Watson if (cache->uc_freebucket != NULL) 33417a52a97eSRobert Watson cachefree += cache->uc_freebucket->ub_cnt; 33427a52a97eSRobert Watson allocs += cache->uc_allocs; 33437a52a97eSRobert Watson frees += cache->uc_frees; 33447a52a97eSRobert Watson } 33457a52a97eSRobert Watson allocs += z->uz_allocs; 33467a52a97eSRobert Watson frees += z->uz_frees; 3347bf965959SSean Bruno sleeps += z->uz_sleeps; 33487a52a97eSRobert Watson if (cachefreep != NULL) 33497a52a97eSRobert Watson *cachefreep = cachefree; 33507a52a97eSRobert Watson if (allocsp != NULL) 33517a52a97eSRobert Watson *allocsp = allocs; 33527a52a97eSRobert Watson if (freesp != NULL) 33537a52a97eSRobert Watson *freesp = frees; 3354bf965959SSean Bruno if (sleepsp != NULL) 3355bf965959SSean Bruno *sleepsp = sleeps; 33567a52a97eSRobert Watson } 3357a0d4b0aeSRobert Watson #endif /* DDB */ 33587a52a97eSRobert Watson 33597a52a97eSRobert Watson static int 33607a52a97eSRobert Watson sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS) 33617a52a97eSRobert Watson { 33627a52a97eSRobert Watson uma_keg_t kz; 33637a52a97eSRobert Watson uma_zone_t z; 33647a52a97eSRobert Watson int count; 33657a52a97eSRobert Watson 33667a52a97eSRobert Watson count = 0; 33677a52a97eSRobert Watson mtx_lock(&uma_mtx); 33687a52a97eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 33697a52a97eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) 33707a52a97eSRobert Watson count++; 33717a52a97eSRobert Watson } 33727a52a97eSRobert Watson mtx_unlock(&uma_mtx); 33737a52a97eSRobert Watson return (sysctl_handle_int(oidp, &count, 0, req)); 33747a52a97eSRobert Watson } 33757a52a97eSRobert Watson 33767a52a97eSRobert Watson static int 33777a52a97eSRobert Watson sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) 33787a52a97eSRobert Watson { 33797a52a97eSRobert Watson struct uma_stream_header ush; 33807a52a97eSRobert Watson struct uma_type_header uth; 33817a52a97eSRobert Watson struct uma_percpu_stat ups; 33827a52a97eSRobert Watson uma_bucket_t bucket; 33837a52a97eSRobert Watson struct sbuf sbuf; 33847a52a97eSRobert Watson uma_cache_t cache; 3385e20a199fSJeff Roberson uma_klink_t kl; 33867a52a97eSRobert Watson uma_keg_t kz; 33877a52a97eSRobert Watson uma_zone_t z; 3388e20a199fSJeff Roberson uma_keg_t k; 33894e657159SMatthew D Fleming int count, error, i; 33907a52a97eSRobert Watson 339100f0e671SMatthew D Fleming error = sysctl_wire_old_buffer(req, 0); 339200f0e671SMatthew D Fleming if (error != 0) 339300f0e671SMatthew D Fleming return (error); 33944e657159SMatthew D Fleming sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 33954e657159SMatthew D Fleming 3396404a593eSMatthew D Fleming count = 0; 33974e657159SMatthew D Fleming mtx_lock(&uma_mtx); 33987a52a97eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 33997a52a97eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) 34007a52a97eSRobert Watson count++; 34017a52a97eSRobert Watson } 34027a52a97eSRobert Watson 34037a52a97eSRobert Watson /* 34047a52a97eSRobert Watson * Insert stream header. 34057a52a97eSRobert Watson */ 34067a52a97eSRobert Watson bzero(&ush, sizeof(ush)); 34077a52a97eSRobert Watson ush.ush_version = UMA_STREAM_VERSION; 3408ab3a57c0SRobert Watson ush.ush_maxcpus = (mp_maxid + 1); 34097a52a97eSRobert Watson ush.ush_count = count; 34104e657159SMatthew D Fleming (void)sbuf_bcat(&sbuf, &ush, sizeof(ush)); 34117a52a97eSRobert Watson 34127a52a97eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 34137a52a97eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) { 34147a52a97eSRobert Watson bzero(&uth, sizeof(uth)); 34157a52a97eSRobert Watson ZONE_LOCK(z); 3416cbbb4a00SRobert Watson strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); 34177a52a97eSRobert Watson uth.uth_align = kz->uk_align; 34187a52a97eSRobert Watson uth.uth_size = kz->uk_size; 34197a52a97eSRobert Watson uth.uth_rsize = kz->uk_rsize; 3420e20a199fSJeff Roberson LIST_FOREACH(kl, &z->uz_kegs, kl_link) { 3421e20a199fSJeff Roberson k = kl->kl_keg; 3422e20a199fSJeff Roberson uth.uth_maxpages += k->uk_maxpages; 3423e20a199fSJeff Roberson uth.uth_pages += k->uk_pages; 3424e20a199fSJeff Roberson uth.uth_keg_free += k->uk_free; 3425e20a199fSJeff Roberson uth.uth_limit = (k->uk_maxpages / k->uk_ppera) 3426e20a199fSJeff Roberson * k->uk_ipers; 3427e20a199fSJeff Roberson } 3428cbbb4a00SRobert Watson 3429cbbb4a00SRobert Watson /* 3430cbbb4a00SRobert Watson * A zone is secondary is it is not the first entry 3431cbbb4a00SRobert Watson * on the keg's zone list. 3432cbbb4a00SRobert Watson */ 3433e20a199fSJeff Roberson if ((z->uz_flags & UMA_ZONE_SECONDARY) && 3434cbbb4a00SRobert Watson (LIST_FIRST(&kz->uk_zones) != z)) 3435cbbb4a00SRobert Watson uth.uth_zone_flags = UTH_ZONE_SECONDARY; 3436cbbb4a00SRobert Watson 3437fc03d22bSJeff Roberson LIST_FOREACH(bucket, &z->uz_buckets, ub_link) 34387a52a97eSRobert Watson uth.uth_zone_free += bucket->ub_cnt; 34397a52a97eSRobert Watson uth.uth_allocs = z->uz_allocs; 34407a52a97eSRobert Watson uth.uth_frees = z->uz_frees; 34412019094aSRobert Watson uth.uth_fails = z->uz_fails; 3442bf965959SSean Bruno uth.uth_sleeps = z->uz_sleeps; 34434e657159SMatthew D Fleming (void)sbuf_bcat(&sbuf, &uth, sizeof(uth)); 34447a52a97eSRobert Watson /* 34452450bbb8SRobert Watson * While it is not normally safe to access the cache 34462450bbb8SRobert Watson * bucket pointers while not on the CPU that owns the 34472450bbb8SRobert Watson * cache, we only allow the pointers to be exchanged 34482450bbb8SRobert Watson * without the zone lock held, not invalidated, so 34492450bbb8SRobert Watson * accept the possible race associated with bucket 34502450bbb8SRobert Watson * exchange during monitoring. 34517a52a97eSRobert Watson */ 3452ab3a57c0SRobert Watson for (i = 0; i < (mp_maxid + 1); i++) { 34537a52a97eSRobert Watson bzero(&ups, sizeof(ups)); 34547a52a97eSRobert Watson if (kz->uk_flags & UMA_ZFLAG_INTERNAL) 34557a52a97eSRobert Watson goto skip; 3456082dc776SRobert Watson if (CPU_ABSENT(i)) 3457082dc776SRobert Watson goto skip; 34587a52a97eSRobert Watson cache = &z->uz_cpu[i]; 34597a52a97eSRobert Watson if (cache->uc_allocbucket != NULL) 34607a52a97eSRobert Watson ups.ups_cache_free += 34617a52a97eSRobert Watson cache->uc_allocbucket->ub_cnt; 34627a52a97eSRobert Watson if (cache->uc_freebucket != NULL) 34637a52a97eSRobert Watson ups.ups_cache_free += 34647a52a97eSRobert Watson cache->uc_freebucket->ub_cnt; 34657a52a97eSRobert Watson ups.ups_allocs = cache->uc_allocs; 34667a52a97eSRobert Watson ups.ups_frees = cache->uc_frees; 34677a52a97eSRobert Watson skip: 34684e657159SMatthew D Fleming (void)sbuf_bcat(&sbuf, &ups, sizeof(ups)); 34697a52a97eSRobert Watson } 34702450bbb8SRobert Watson ZONE_UNLOCK(z); 34717a52a97eSRobert Watson } 34727a52a97eSRobert Watson } 34737a52a97eSRobert Watson mtx_unlock(&uma_mtx); 34744e657159SMatthew D Fleming error = sbuf_finish(&sbuf); 34754e657159SMatthew D Fleming sbuf_delete(&sbuf); 34767a52a97eSRobert Watson return (error); 34777a52a97eSRobert Watson } 347848c5777eSRobert Watson 34790a5a3ccbSGleb Smirnoff int 34800a5a3ccbSGleb Smirnoff sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS) 34810a5a3ccbSGleb Smirnoff { 34820a5a3ccbSGleb Smirnoff uma_zone_t zone = *(uma_zone_t *)arg1; 34830a5a3ccbSGleb Smirnoff int error, max, old; 34840a5a3ccbSGleb Smirnoff 34850a5a3ccbSGleb Smirnoff old = max = uma_zone_get_max(zone); 34860a5a3ccbSGleb Smirnoff error = sysctl_handle_int(oidp, &max, 0, req); 34870a5a3ccbSGleb Smirnoff if (error || !req->newptr) 34880a5a3ccbSGleb Smirnoff return (error); 34890a5a3ccbSGleb Smirnoff 34900a5a3ccbSGleb Smirnoff if (max < old) 34910a5a3ccbSGleb Smirnoff return (EINVAL); 34920a5a3ccbSGleb Smirnoff 34930a5a3ccbSGleb Smirnoff uma_zone_set_max(zone, max); 34940a5a3ccbSGleb Smirnoff 34950a5a3ccbSGleb Smirnoff return (0); 34960a5a3ccbSGleb Smirnoff } 34970a5a3ccbSGleb Smirnoff 34980a5a3ccbSGleb Smirnoff int 34990a5a3ccbSGleb Smirnoff sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS) 35000a5a3ccbSGleb Smirnoff { 35010a5a3ccbSGleb Smirnoff uma_zone_t zone = *(uma_zone_t *)arg1; 35020a5a3ccbSGleb Smirnoff int cur; 35030a5a3ccbSGleb Smirnoff 35040a5a3ccbSGleb Smirnoff cur = uma_zone_get_cur(zone); 35050a5a3ccbSGleb Smirnoff return (sysctl_handle_int(oidp, &cur, 0, req)); 35060a5a3ccbSGleb Smirnoff } 35070a5a3ccbSGleb Smirnoff 350848c5777eSRobert Watson #ifdef DDB 350948c5777eSRobert Watson DB_SHOW_COMMAND(uma, db_show_uma) 351048c5777eSRobert Watson { 351185dcf349SGleb Smirnoff uint64_t allocs, frees, sleeps; 351248c5777eSRobert Watson uma_bucket_t bucket; 351348c5777eSRobert Watson uma_keg_t kz; 351448c5777eSRobert Watson uma_zone_t z; 351548c5777eSRobert Watson int cachefree; 351648c5777eSRobert Watson 351703175483SAlexander Motin db_printf("%18s %8s %8s %8s %12s %8s %8s\n", "Zone", "Size", "Used", 351803175483SAlexander Motin "Free", "Requests", "Sleeps", "Bucket"); 351948c5777eSRobert Watson LIST_FOREACH(kz, &uma_kegs, uk_link) { 352048c5777eSRobert Watson LIST_FOREACH(z, &kz->uk_zones, uz_link) { 352148c5777eSRobert Watson if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { 352248c5777eSRobert Watson allocs = z->uz_allocs; 352348c5777eSRobert Watson frees = z->uz_frees; 3524bf965959SSean Bruno sleeps = z->uz_sleeps; 352548c5777eSRobert Watson cachefree = 0; 352648c5777eSRobert Watson } else 352748c5777eSRobert Watson uma_zone_sumstat(z, &cachefree, &allocs, 3528bf965959SSean Bruno &frees, &sleeps); 3529e20a199fSJeff Roberson if (!((z->uz_flags & UMA_ZONE_SECONDARY) && 353048c5777eSRobert Watson (LIST_FIRST(&kz->uk_zones) != z))) 353148c5777eSRobert Watson cachefree += kz->uk_free; 3532fc03d22bSJeff Roberson LIST_FOREACH(bucket, &z->uz_buckets, ub_link) 353348c5777eSRobert Watson cachefree += bucket->ub_cnt; 353403175483SAlexander Motin db_printf("%18s %8ju %8jd %8d %12ju %8ju %8u\n", 353503175483SAlexander Motin z->uz_name, (uintmax_t)kz->uk_size, 3536ae4e9636SRobert Watson (intmax_t)(allocs - frees), cachefree, 353703175483SAlexander Motin (uintmax_t)allocs, sleeps, z->uz_count); 3538687c94aaSJohn Baldwin if (db_pager_quit) 3539687c94aaSJohn Baldwin return; 354048c5777eSRobert Watson } 354148c5777eSRobert Watson } 354248c5777eSRobert Watson } 354303175483SAlexander Motin 354403175483SAlexander Motin DB_SHOW_COMMAND(umacache, db_show_umacache) 354503175483SAlexander Motin { 354603175483SAlexander Motin uint64_t allocs, frees; 354703175483SAlexander Motin uma_bucket_t bucket; 354803175483SAlexander Motin uma_zone_t z; 354903175483SAlexander Motin int cachefree; 355003175483SAlexander Motin 355103175483SAlexander Motin db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free", 355203175483SAlexander Motin "Requests", "Bucket"); 355303175483SAlexander Motin LIST_FOREACH(z, &uma_cachezones, uz_link) { 355403175483SAlexander Motin uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL); 355503175483SAlexander Motin LIST_FOREACH(bucket, &z->uz_buckets, ub_link) 355603175483SAlexander Motin cachefree += bucket->ub_cnt; 355703175483SAlexander Motin db_printf("%18s %8ju %8jd %8d %12ju %8u\n", 355803175483SAlexander Motin z->uz_name, (uintmax_t)z->uz_size, 355903175483SAlexander Motin (intmax_t)(allocs - frees), cachefree, 356003175483SAlexander Motin (uintmax_t)allocs, z->uz_count); 356103175483SAlexander Motin if (db_pager_quit) 356203175483SAlexander Motin return; 356303175483SAlexander Motin } 356403175483SAlexander Motin } 356548c5777eSRobert Watson #endif 3566