18355f576SJeff Roberson /* 2f461cf22SJeff Roberson * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 38355f576SJeff Roberson * All rights reserved. 48355f576SJeff Roberson * 58355f576SJeff Roberson * Redistribution and use in source and binary forms, with or without 68355f576SJeff Roberson * modification, are permitted provided that the following conditions 78355f576SJeff Roberson * are met: 88355f576SJeff Roberson * 1. Redistributions of source code must retain the above copyright 98355f576SJeff Roberson * notice unmodified, this list of conditions, and the following 108355f576SJeff Roberson * disclaimer. 118355f576SJeff Roberson * 2. Redistributions in binary form must reproduce the above copyright 128355f576SJeff Roberson * notice, this list of conditions and the following disclaimer in the 138355f576SJeff Roberson * documentation and/or other materials provided with the distribution. 148355f576SJeff Roberson * 158355f576SJeff Roberson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 168355f576SJeff Roberson * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 178355f576SJeff Roberson * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 188355f576SJeff Roberson * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 198355f576SJeff Roberson * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 208355f576SJeff Roberson * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 218355f576SJeff Roberson * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 228355f576SJeff Roberson * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 238355f576SJeff Roberson * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 248355f576SJeff Roberson * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 258355f576SJeff Roberson */ 268355f576SJeff Roberson 278355f576SJeff Roberson /* 288355f576SJeff Roberson * uma_core.c Implementation of the Universal Memory allocator 298355f576SJeff Roberson * 308355f576SJeff Roberson * This allocator is intended to replace the multitude of similar object caches 318355f576SJeff Roberson * in the standard FreeBSD kernel. The intent is to be flexible as well as 328355f576SJeff Roberson * effecient. A primary design goal is to return unused memory to the rest of 338355f576SJeff Roberson * the system. This will make the system as a whole more flexible due to the 348355f576SJeff Roberson * ability to move memory to subsystems which most need it instead of leaving 358355f576SJeff Roberson * pools of reserved memory unused. 368355f576SJeff Roberson * 378355f576SJeff Roberson * The basic ideas stem from similar slab/zone based allocators whose algorithms 388355f576SJeff Roberson * are well known. 398355f576SJeff Roberson * 408355f576SJeff Roberson */ 418355f576SJeff Roberson 428355f576SJeff Roberson /* 438355f576SJeff Roberson * TODO: 448355f576SJeff Roberson * - Improve memory usage for large allocations 458355f576SJeff Roberson * - Investigate cache size adjustments 468355f576SJeff Roberson */ 478355f576SJeff Roberson 48874651b1SDavid E. O'Brien #include <sys/cdefs.h> 49874651b1SDavid E. O'Brien __FBSDID("$FreeBSD$"); 50874651b1SDavid E. O'Brien 518355f576SJeff Roberson /* I should really use ktr.. */ 528355f576SJeff Roberson /* 538355f576SJeff Roberson #define UMA_DEBUG 1 548355f576SJeff Roberson #define UMA_DEBUG_ALLOC 1 558355f576SJeff Roberson #define UMA_DEBUG_ALLOC_1 1 568355f576SJeff Roberson */ 578355f576SJeff Roberson 588355f576SJeff Roberson #include "opt_param.h" 598355f576SJeff Roberson #include <sys/param.h> 608355f576SJeff Roberson #include <sys/systm.h> 618355f576SJeff Roberson #include <sys/kernel.h> 628355f576SJeff Roberson #include <sys/types.h> 638355f576SJeff Roberson #include <sys/queue.h> 648355f576SJeff Roberson #include <sys/malloc.h> 658355f576SJeff Roberson #include <sys/lock.h> 668355f576SJeff Roberson #include <sys/sysctl.h> 678355f576SJeff Roberson #include <sys/mutex.h> 684c1cc01cSJohn Baldwin #include <sys/proc.h> 698355f576SJeff Roberson #include <sys/smp.h> 7086bbae32SJeff Roberson #include <sys/vmmeter.h> 718522511bSHartmut Brandt #include <sys/mbuf.h> 7286bbae32SJeff Roberson 738355f576SJeff Roberson #include <vm/vm.h> 748355f576SJeff Roberson #include <vm/vm_object.h> 758355f576SJeff Roberson #include <vm/vm_page.h> 768355f576SJeff Roberson #include <vm/vm_param.h> 778355f576SJeff Roberson #include <vm/vm_map.h> 788355f576SJeff Roberson #include <vm/vm_kern.h> 798355f576SJeff Roberson #include <vm/vm_extern.h> 808355f576SJeff Roberson #include <vm/uma.h> 818355f576SJeff Roberson #include <vm/uma_int.h> 82639c9550SJeff Roberson #include <vm/uma_dbg.h> 838355f576SJeff Roberson 8448eea375SJeff Roberson #include <machine/vmparam.h> 8548eea375SJeff Roberson 868355f576SJeff Roberson /* 878355f576SJeff Roberson * This is the zone from which all zones are spawned. The idea is that even 888355f576SJeff Roberson * the zone heads are allocated from the allocator, so we use the bss section 898355f576SJeff Roberson * to bootstrap us. 908355f576SJeff Roberson */ 9186bbae32SJeff Roberson static struct uma_zone masterzone; 9286bbae32SJeff Roberson static uma_zone_t zones = &masterzone; 938355f576SJeff Roberson 948355f576SJeff Roberson /* This is the zone from which all of uma_slab_t's are allocated. */ 958355f576SJeff Roberson static uma_zone_t slabzone; 968355f576SJeff Roberson 978355f576SJeff Roberson /* 988355f576SJeff Roberson * The initial hash tables come out of this zone so they can be allocated 998355f576SJeff Roberson * prior to malloc coming up. 1008355f576SJeff Roberson */ 1018355f576SJeff Roberson static uma_zone_t hashzone; 1028355f576SJeff Roberson 103961647dfSJeff Roberson static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets"); 104961647dfSJeff Roberson 1058355f576SJeff Roberson /* 10686bbae32SJeff Roberson * Are we allowed to allocate buckets? 10786bbae32SJeff Roberson */ 10886bbae32SJeff Roberson static int bucketdisable = 1; 10986bbae32SJeff Roberson 1108355f576SJeff Roberson /* Linked list of all zones in the system */ 1118355f576SJeff Roberson static LIST_HEAD(,uma_zone) uma_zones = LIST_HEAD_INITIALIZER(&uma_zones); 1128355f576SJeff Roberson 1138355f576SJeff Roberson /* This mutex protects the zone list */ 1148355f576SJeff Roberson static struct mtx uma_mtx; 1158355f576SJeff Roberson 116d88797c2SBosko Milekic /* These are the pcpu cache locks */ 117d88797c2SBosko Milekic static struct mtx uma_pcpu_mtx[MAXCPU]; 118d88797c2SBosko Milekic 1198355f576SJeff Roberson /* Linked list of boot time pages */ 1208355f576SJeff Roberson static LIST_HEAD(,uma_slab) uma_boot_pages = 1218355f576SJeff Roberson LIST_HEAD_INITIALIZER(&uma_boot_pages); 1228355f576SJeff Roberson 1238355f576SJeff Roberson /* Count of free boottime pages */ 1248355f576SJeff Roberson static int uma_boot_free = 0; 1258355f576SJeff Roberson 1268355f576SJeff Roberson /* Is the VM done starting up? */ 1278355f576SJeff Roberson static int booted = 0; 1288355f576SJeff Roberson 1299643769aSJeff Roberson /* 1309643769aSJeff Roberson * This is the handle used to schedule events that need to happen 1319643769aSJeff Roberson * outside of the allocation fast path. 1329643769aSJeff Roberson */ 1338355f576SJeff Roberson static struct callout uma_callout; 1349643769aSJeff Roberson #define UMA_TIMEOUT 20 /* Seconds for callout interval. */ 1358355f576SJeff Roberson 1368355f576SJeff Roberson /* 1378355f576SJeff Roberson * This structure is passed as the zone ctor arg so that I don't have to create 1388355f576SJeff Roberson * a special allocation function just for zones. 1398355f576SJeff Roberson */ 1408355f576SJeff Roberson struct uma_zctor_args { 1418355f576SJeff Roberson char *name; 142c3bdc05fSAndrew R. Reiter size_t size; 1438355f576SJeff Roberson uma_ctor ctor; 1448355f576SJeff Roberson uma_dtor dtor; 1458355f576SJeff Roberson uma_init uminit; 1468355f576SJeff Roberson uma_fini fini; 1478355f576SJeff Roberson int align; 1488355f576SJeff Roberson u_int16_t flags; 1498355f576SJeff Roberson }; 1508355f576SJeff Roberson 151cae33c14SJeff Roberson struct uma_bucket_zone { 152cae33c14SJeff Roberson uma_zone_t ubz_zone; 153cae33c14SJeff Roberson char *ubz_name; 154cae33c14SJeff Roberson int ubz_entries; 155cae33c14SJeff Roberson }; 156cae33c14SJeff Roberson 157cae33c14SJeff Roberson #define BUCKET_MAX 128 158cae33c14SJeff Roberson 159cae33c14SJeff Roberson struct uma_bucket_zone bucket_zones[] = { 160cae33c14SJeff Roberson { NULL, "16 Bucket", 16 }, 161cae33c14SJeff Roberson { NULL, "32 Bucket", 32 }, 162cae33c14SJeff Roberson { NULL, "64 Bucket", 64 }, 163cae33c14SJeff Roberson { NULL, "128 Bucket", 128 }, 164cae33c14SJeff Roberson { NULL, NULL, 0} 165cae33c14SJeff Roberson }; 166cae33c14SJeff Roberson 167cae33c14SJeff Roberson #define BUCKET_SHIFT 4 168cae33c14SJeff Roberson #define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1) 169cae33c14SJeff Roberson 170cae33c14SJeff Roberson uint8_t bucket_size[BUCKET_ZONES]; 171cae33c14SJeff Roberson 1728355f576SJeff Roberson /* Prototypes.. */ 1738355f576SJeff Roberson 1748355f576SJeff Roberson static void *obj_alloc(uma_zone_t, int, u_int8_t *, int); 1758355f576SJeff Roberson static void *page_alloc(uma_zone_t, int, u_int8_t *, int); 176009b6fcbSJeff Roberson static void *startup_alloc(uma_zone_t, int, u_int8_t *, int); 1778355f576SJeff Roberson static void page_free(void *, int, u_int8_t); 1788355f576SJeff Roberson static uma_slab_t slab_zalloc(uma_zone_t, int); 1799643769aSJeff Roberson static void cache_drain(uma_zone_t); 1808355f576SJeff Roberson static void bucket_drain(uma_zone_t, uma_bucket_t); 181aaa8bb16SJeff Roberson static void bucket_cache_drain(uma_zone_t zone); 1828355f576SJeff Roberson static void zone_ctor(void *, int, void *); 1839c2cd7e5SJeff Roberson static void zone_dtor(void *, int, void *); 1848355f576SJeff Roberson static void zero_init(void *, int); 1858355f576SJeff Roberson static void zone_small_init(uma_zone_t zone); 1868355f576SJeff Roberson static void zone_large_init(uma_zone_t zone); 1878355f576SJeff Roberson static void zone_foreach(void (*zfunc)(uma_zone_t)); 1888355f576SJeff Roberson static void zone_timeout(uma_zone_t zone); 1890aef6126SJeff Roberson static int hash_alloc(struct uma_hash *); 1900aef6126SJeff Roberson static int hash_expand(struct uma_hash *, struct uma_hash *); 1910aef6126SJeff Roberson static void hash_free(struct uma_hash *hash); 1928355f576SJeff Roberson static void uma_timeout(void *); 1938355f576SJeff Roberson static void uma_startup3(void); 194bbee39c6SJeff Roberson static void *uma_zalloc_internal(uma_zone_t, void *, int); 19586bbae32SJeff Roberson static void uma_zfree_internal(uma_zone_t, void *, void *, int); 19686bbae32SJeff Roberson static void bucket_enable(void); 197cae33c14SJeff Roberson static void bucket_init(void); 198cae33c14SJeff Roberson static uma_bucket_t bucket_alloc(int, int); 199cae33c14SJeff Roberson static void bucket_free(uma_bucket_t); 200cae33c14SJeff Roberson static void bucket_zone_drain(void); 201bbee39c6SJeff Roberson static int uma_zalloc_bucket(uma_zone_t zone, int flags); 202bbee39c6SJeff Roberson static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags); 203bbee39c6SJeff Roberson static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab); 2049643769aSJeff Roberson static void zone_drain(uma_zone_t); 205bbee39c6SJeff Roberson 2068355f576SJeff Roberson void uma_print_zone(uma_zone_t); 2078355f576SJeff Roberson void uma_print_stats(void); 2088355f576SJeff Roberson static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS); 2098355f576SJeff Roberson 2108355f576SJeff Roberson SYSCTL_OID(_vm, OID_AUTO, zone, CTLTYPE_STRING|CTLFLAG_RD, 2118355f576SJeff Roberson NULL, 0, sysctl_vm_zone, "A", "Zone Info"); 2128355f576SJeff Roberson SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 2138355f576SJeff Roberson 21486bbae32SJeff Roberson /* 21586bbae32SJeff Roberson * This routine checks to see whether or not it's safe to enable buckets. 21686bbae32SJeff Roberson */ 21786bbae32SJeff Roberson 21886bbae32SJeff Roberson static void 21986bbae32SJeff Roberson bucket_enable(void) 22086bbae32SJeff Roberson { 22186bbae32SJeff Roberson if (cnt.v_free_count < cnt.v_free_min) 22286bbae32SJeff Roberson bucketdisable = 1; 22386bbae32SJeff Roberson else 22486bbae32SJeff Roberson bucketdisable = 0; 22586bbae32SJeff Roberson } 22686bbae32SJeff Roberson 227cae33c14SJeff Roberson static void 228cae33c14SJeff Roberson bucket_init(void) 229cae33c14SJeff Roberson { 230cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 231cae33c14SJeff Roberson int i; 232cae33c14SJeff Roberson int j; 233cae33c14SJeff Roberson 234cae33c14SJeff Roberson for (i = 0, j = 0; bucket_zones[j].ubz_entries != 0; j++) { 235cae33c14SJeff Roberson int size; 236cae33c14SJeff Roberson 237cae33c14SJeff Roberson ubz = &bucket_zones[j]; 238cae33c14SJeff Roberson size = roundup(sizeof(struct uma_bucket), sizeof(void *)); 239cae33c14SJeff Roberson size += sizeof(void *) * ubz->ubz_entries; 240cae33c14SJeff Roberson ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size, 241b60f5b79SJeff Roberson NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 242cae33c14SJeff Roberson for (; i <= ubz->ubz_entries; i += (1 << BUCKET_SHIFT)) 243cae33c14SJeff Roberson bucket_size[i >> BUCKET_SHIFT] = j; 244cae33c14SJeff Roberson } 245cae33c14SJeff Roberson } 246cae33c14SJeff Roberson 247cae33c14SJeff Roberson static uma_bucket_t 248cae33c14SJeff Roberson bucket_alloc(int entries, int bflags) 249cae33c14SJeff Roberson { 250cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 251cae33c14SJeff Roberson uma_bucket_t bucket; 252cae33c14SJeff Roberson int idx; 253cae33c14SJeff Roberson 254cae33c14SJeff Roberson /* 255cae33c14SJeff Roberson * This is to stop us from allocating per cpu buckets while we're 256cae33c14SJeff Roberson * running out of UMA_BOOT_PAGES. Otherwise, we would exhaust the 257cae33c14SJeff Roberson * boot pages. This also prevents us from allocating buckets in 258cae33c14SJeff Roberson * low memory situations. 259cae33c14SJeff Roberson */ 260cae33c14SJeff Roberson 261cae33c14SJeff Roberson if (bucketdisable) 262cae33c14SJeff Roberson return (NULL); 263cae33c14SJeff Roberson idx = howmany(entries, 1 << BUCKET_SHIFT); 264cae33c14SJeff Roberson ubz = &bucket_zones[bucket_size[idx]]; 265cae33c14SJeff Roberson bucket = uma_zalloc_internal(ubz->ubz_zone, NULL, bflags); 266cae33c14SJeff Roberson if (bucket) { 267cae33c14SJeff Roberson #ifdef INVARIANTS 268cae33c14SJeff Roberson bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); 269cae33c14SJeff Roberson #endif 270cae33c14SJeff Roberson bucket->ub_cnt = 0; 271cae33c14SJeff Roberson bucket->ub_entries = ubz->ubz_entries; 272cae33c14SJeff Roberson } 273cae33c14SJeff Roberson 274cae33c14SJeff Roberson return (bucket); 275cae33c14SJeff Roberson } 276cae33c14SJeff Roberson 277cae33c14SJeff Roberson static void 278cae33c14SJeff Roberson bucket_free(uma_bucket_t bucket) 279cae33c14SJeff Roberson { 280cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 281cae33c14SJeff Roberson int idx; 282cae33c14SJeff Roberson 283cae33c14SJeff Roberson idx = howmany(bucket->ub_entries, 1 << BUCKET_SHIFT); 284cae33c14SJeff Roberson ubz = &bucket_zones[bucket_size[idx]]; 285cae33c14SJeff Roberson uma_zfree_internal(ubz->ubz_zone, bucket, NULL, 0); 286cae33c14SJeff Roberson } 287cae33c14SJeff Roberson 288cae33c14SJeff Roberson static void 289cae33c14SJeff Roberson bucket_zone_drain(void) 290cae33c14SJeff Roberson { 291cae33c14SJeff Roberson struct uma_bucket_zone *ubz; 292cae33c14SJeff Roberson 293cae33c14SJeff Roberson for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 294cae33c14SJeff Roberson zone_drain(ubz->ubz_zone); 295cae33c14SJeff Roberson } 296cae33c14SJeff Roberson 2978355f576SJeff Roberson 2988355f576SJeff Roberson /* 2998355f576SJeff Roberson * Routine called by timeout which is used to fire off some time interval 3009643769aSJeff Roberson * based calculations. (stats, hash size, etc.) 3018355f576SJeff Roberson * 3028355f576SJeff Roberson * Arguments: 3038355f576SJeff Roberson * arg Unused 3048355f576SJeff Roberson * 3058355f576SJeff Roberson * Returns: 3068355f576SJeff Roberson * Nothing 3078355f576SJeff Roberson */ 3088355f576SJeff Roberson static void 3098355f576SJeff Roberson uma_timeout(void *unused) 3108355f576SJeff Roberson { 31186bbae32SJeff Roberson bucket_enable(); 3128355f576SJeff Roberson zone_foreach(zone_timeout); 3138355f576SJeff Roberson 3148355f576SJeff Roberson /* Reschedule this event */ 3159643769aSJeff Roberson callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 3168355f576SJeff Roberson } 3178355f576SJeff Roberson 3188355f576SJeff Roberson /* 3199643769aSJeff Roberson * Routine to perform timeout driven calculations. This expands the 3209643769aSJeff Roberson * hashes and does per cpu statistics aggregation. 3218355f576SJeff Roberson * 3228355f576SJeff Roberson * Arguments: 3238355f576SJeff Roberson * zone The zone to operate on 3248355f576SJeff Roberson * 3258355f576SJeff Roberson * Returns: 3268355f576SJeff Roberson * Nothing 3278355f576SJeff Roberson */ 3288355f576SJeff Roberson static void 3298355f576SJeff Roberson zone_timeout(uma_zone_t zone) 3308355f576SJeff Roberson { 3318355f576SJeff Roberson uma_cache_t cache; 3328355f576SJeff Roberson u_int64_t alloc; 3338355f576SJeff Roberson int cpu; 3348355f576SJeff Roberson 3358355f576SJeff Roberson alloc = 0; 3368355f576SJeff Roberson 3378355f576SJeff Roberson /* 3388355f576SJeff Roberson * Aggregate per cpu cache statistics back to the zone. 3398355f576SJeff Roberson * 340aaa8bb16SJeff Roberson * XXX This should be done in the sysctl handler. 341aaa8bb16SJeff Roberson * 3428355f576SJeff Roberson * I may rewrite this to set a flag in the per cpu cache instead of 3438355f576SJeff Roberson * locking. If the flag is not cleared on the next round I will have 3448355f576SJeff Roberson * to lock and do it here instead so that the statistics don't get too 3458355f576SJeff Roberson * far out of sync. 3468355f576SJeff Roberson */ 3478355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) { 348b6c71225SJohn Baldwin for (cpu = 0; cpu <= mp_maxid; cpu++) { 3498355f576SJeff Roberson if (CPU_ABSENT(cpu)) 3508355f576SJeff Roberson continue; 351d88797c2SBosko Milekic CPU_LOCK(cpu); 3528355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 3538355f576SJeff Roberson /* Add them up, and reset */ 3548355f576SJeff Roberson alloc += cache->uc_allocs; 3558355f576SJeff Roberson cache->uc_allocs = 0; 356d88797c2SBosko Milekic CPU_UNLOCK(cpu); 3578355f576SJeff Roberson } 3588355f576SJeff Roberson } 3598355f576SJeff Roberson 3608355f576SJeff Roberson /* Now push these stats back into the zone.. */ 3618355f576SJeff Roberson ZONE_LOCK(zone); 3628355f576SJeff Roberson zone->uz_allocs += alloc; 3638355f576SJeff Roberson 3648355f576SJeff Roberson /* 3658355f576SJeff Roberson * Expand the zone hash table. 3668355f576SJeff Roberson * 3678355f576SJeff Roberson * This is done if the number of slabs is larger than the hash size. 3688355f576SJeff Roberson * What I'm trying to do here is completely reduce collisions. This 3698355f576SJeff Roberson * may be a little aggressive. Should I allow for two collisions max? 3708355f576SJeff Roberson */ 3718355f576SJeff Roberson 372b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_HASH && 37399571dc3SJeff Roberson zone->uz_pages / zone->uz_ppera >= zone->uz_hash.uh_hashsize) { 3740aef6126SJeff Roberson struct uma_hash newhash; 3750aef6126SJeff Roberson struct uma_hash oldhash; 3760aef6126SJeff Roberson int ret; 3775300d9ddSJeff Roberson 3780aef6126SJeff Roberson /* 3790aef6126SJeff Roberson * This is so involved because allocating and freeing 3800aef6126SJeff Roberson * while the zone lock is held will lead to deadlock. 3810aef6126SJeff Roberson * I have to do everything in stages and check for 3820aef6126SJeff Roberson * races. 3830aef6126SJeff Roberson */ 3840aef6126SJeff Roberson newhash = zone->uz_hash; 3855300d9ddSJeff Roberson ZONE_UNLOCK(zone); 3860aef6126SJeff Roberson ret = hash_alloc(&newhash); 3875300d9ddSJeff Roberson ZONE_LOCK(zone); 3880aef6126SJeff Roberson if (ret) { 3890aef6126SJeff Roberson if (hash_expand(&zone->uz_hash, &newhash)) { 3900aef6126SJeff Roberson oldhash = zone->uz_hash; 3910aef6126SJeff Roberson zone->uz_hash = newhash; 3920aef6126SJeff Roberson } else 3930aef6126SJeff Roberson oldhash = newhash; 3940aef6126SJeff Roberson 3950aef6126SJeff Roberson ZONE_UNLOCK(zone); 3960aef6126SJeff Roberson hash_free(&oldhash); 3970aef6126SJeff Roberson ZONE_LOCK(zone); 3980aef6126SJeff Roberson } 3995300d9ddSJeff Roberson } 4008355f576SJeff Roberson ZONE_UNLOCK(zone); 4018355f576SJeff Roberson } 4028355f576SJeff Roberson 4038355f576SJeff Roberson /* 4045300d9ddSJeff Roberson * Allocate and zero fill the next sized hash table from the appropriate 4055300d9ddSJeff Roberson * backing store. 4065300d9ddSJeff Roberson * 4075300d9ddSJeff Roberson * Arguments: 4080aef6126SJeff Roberson * hash A new hash structure with the old hash size in uh_hashsize 4095300d9ddSJeff Roberson * 4105300d9ddSJeff Roberson * Returns: 4110aef6126SJeff Roberson * 1 on sucess and 0 on failure. 4125300d9ddSJeff Roberson */ 41337c84183SPoul-Henning Kamp static int 4140aef6126SJeff Roberson hash_alloc(struct uma_hash *hash) 4155300d9ddSJeff Roberson { 4160aef6126SJeff Roberson int oldsize; 4175300d9ddSJeff Roberson int alloc; 4185300d9ddSJeff Roberson 4190aef6126SJeff Roberson oldsize = hash->uh_hashsize; 4200aef6126SJeff Roberson 4215300d9ddSJeff Roberson /* We're just going to go to a power of two greater */ 4220aef6126SJeff Roberson if (oldsize) { 4230aef6126SJeff Roberson hash->uh_hashsize = oldsize * 2; 4240aef6126SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; 4250aef6126SJeff Roberson hash->uh_slab_hash = (struct slabhead *)malloc(alloc, 426961647dfSJeff Roberson M_UMAHASH, M_NOWAIT); 4275300d9ddSJeff Roberson } else { 4280aef6126SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; 4290aef6126SJeff Roberson hash->uh_slab_hash = uma_zalloc_internal(hashzone, NULL, 430a163d034SWarner Losh M_WAITOK); 4310aef6126SJeff Roberson hash->uh_hashsize = UMA_HASH_SIZE_INIT; 4325300d9ddSJeff Roberson } 4330aef6126SJeff Roberson if (hash->uh_slab_hash) { 4340aef6126SJeff Roberson bzero(hash->uh_slab_hash, alloc); 4350aef6126SJeff Roberson hash->uh_hashmask = hash->uh_hashsize - 1; 4360aef6126SJeff Roberson return (1); 4370aef6126SJeff Roberson } 4385300d9ddSJeff Roberson 4390aef6126SJeff Roberson return (0); 4405300d9ddSJeff Roberson } 4415300d9ddSJeff Roberson 4425300d9ddSJeff Roberson /* 44364f051e9SJeff Roberson * Expands the hash table for HASH zones. This is done from zone_timeout 44464f051e9SJeff Roberson * to reduce collisions. This must not be done in the regular allocation 44564f051e9SJeff Roberson * path, otherwise, we can recurse on the vm while allocating pages. 4468355f576SJeff Roberson * 4478355f576SJeff Roberson * Arguments: 4480aef6126SJeff Roberson * oldhash The hash you want to expand 4490aef6126SJeff Roberson * newhash The hash structure for the new table 4508355f576SJeff Roberson * 4518355f576SJeff Roberson * Returns: 4528355f576SJeff Roberson * Nothing 4538355f576SJeff Roberson * 4548355f576SJeff Roberson * Discussion: 4558355f576SJeff Roberson */ 4560aef6126SJeff Roberson static int 4570aef6126SJeff Roberson hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash) 4588355f576SJeff Roberson { 4598355f576SJeff Roberson uma_slab_t slab; 4608355f576SJeff Roberson int hval; 4618355f576SJeff Roberson int i; 4628355f576SJeff Roberson 4630aef6126SJeff Roberson if (!newhash->uh_slab_hash) 4640aef6126SJeff Roberson return (0); 4658355f576SJeff Roberson 4660aef6126SJeff Roberson if (oldhash->uh_hashsize >= newhash->uh_hashsize) 4670aef6126SJeff Roberson return (0); 4688355f576SJeff Roberson 4698355f576SJeff Roberson /* 4708355f576SJeff Roberson * I need to investigate hash algorithms for resizing without a 4718355f576SJeff Roberson * full rehash. 4728355f576SJeff Roberson */ 4738355f576SJeff Roberson 4740aef6126SJeff Roberson for (i = 0; i < oldhash->uh_hashsize; i++) 4750aef6126SJeff Roberson while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) { 4760aef6126SJeff Roberson slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]); 4770aef6126SJeff Roberson SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink); 4780aef6126SJeff Roberson hval = UMA_HASH(newhash, slab->us_data); 4790aef6126SJeff Roberson SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval], 4800aef6126SJeff Roberson slab, us_hlink); 4818355f576SJeff Roberson } 4828355f576SJeff Roberson 4830aef6126SJeff Roberson return (1); 4849c2cd7e5SJeff Roberson } 4859c2cd7e5SJeff Roberson 4865300d9ddSJeff Roberson /* 4875300d9ddSJeff Roberson * Free the hash bucket to the appropriate backing store. 4885300d9ddSJeff Roberson * 4895300d9ddSJeff Roberson * Arguments: 4905300d9ddSJeff Roberson * slab_hash The hash bucket we're freeing 4915300d9ddSJeff Roberson * hashsize The number of entries in that hash bucket 4925300d9ddSJeff Roberson * 4935300d9ddSJeff Roberson * Returns: 4945300d9ddSJeff Roberson * Nothing 4955300d9ddSJeff Roberson */ 4969c2cd7e5SJeff Roberson static void 4970aef6126SJeff Roberson hash_free(struct uma_hash *hash) 4989c2cd7e5SJeff Roberson { 4990aef6126SJeff Roberson if (hash->uh_slab_hash == NULL) 5000aef6126SJeff Roberson return; 5010aef6126SJeff Roberson if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) 5028355f576SJeff Roberson uma_zfree_internal(hashzone, 5030aef6126SJeff Roberson hash->uh_slab_hash, NULL, 0); 5048355f576SJeff Roberson else 505961647dfSJeff Roberson free(hash->uh_slab_hash, M_UMAHASH); 5068355f576SJeff Roberson } 5078355f576SJeff Roberson 5088355f576SJeff Roberson /* 5098355f576SJeff Roberson * Frees all outstanding items in a bucket 5108355f576SJeff Roberson * 5118355f576SJeff Roberson * Arguments: 5128355f576SJeff Roberson * zone The zone to free to, must be unlocked. 5138355f576SJeff Roberson * bucket The free/alloc bucket with items, cpu queue must be locked. 5148355f576SJeff Roberson * 5158355f576SJeff Roberson * Returns: 5168355f576SJeff Roberson * Nothing 5178355f576SJeff Roberson */ 5188355f576SJeff Roberson 5198355f576SJeff Roberson static void 5208355f576SJeff Roberson bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 5218355f576SJeff Roberson { 5228355f576SJeff Roberson uma_slab_t slab; 5238355f576SJeff Roberson int mzone; 5248355f576SJeff Roberson void *item; 5258355f576SJeff Roberson 5268355f576SJeff Roberson if (bucket == NULL) 5278355f576SJeff Roberson return; 5288355f576SJeff Roberson 5298355f576SJeff Roberson slab = NULL; 5308355f576SJeff Roberson mzone = 0; 5318355f576SJeff Roberson 5328355f576SJeff Roberson /* We have to lookup the slab again for malloc.. */ 533b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_MALLOC) 5348355f576SJeff Roberson mzone = 1; 5358355f576SJeff Roberson 536cae33c14SJeff Roberson while (bucket->ub_cnt > 0) { 537cae33c14SJeff Roberson bucket->ub_cnt--; 538cae33c14SJeff Roberson item = bucket->ub_bucket[bucket->ub_cnt]; 5398355f576SJeff Roberson #ifdef INVARIANTS 540cae33c14SJeff Roberson bucket->ub_bucket[bucket->ub_cnt] = NULL; 5418355f576SJeff Roberson KASSERT(item != NULL, 5428355f576SJeff Roberson ("bucket_drain: botched ptr, item is NULL")); 5438355f576SJeff Roberson #endif 5448355f576SJeff Roberson /* 5458355f576SJeff Roberson * This is extremely inefficient. The slab pointer was passed 5468355f576SJeff Roberson * to uma_zfree_arg, but we lost it because the buckets don't 5478355f576SJeff Roberson * hold them. This will go away when free() gets a size passed 5488355f576SJeff Roberson * to it. 5498355f576SJeff Roberson */ 55099571dc3SJeff Roberson if (mzone) 55199571dc3SJeff Roberson slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK)); 5528355f576SJeff Roberson uma_zfree_internal(zone, item, slab, 1); 5538355f576SJeff Roberson } 5548355f576SJeff Roberson } 5558355f576SJeff Roberson 5568355f576SJeff Roberson /* 5578355f576SJeff Roberson * Drains the per cpu caches for a zone. 5588355f576SJeff Roberson * 5598355f576SJeff Roberson * Arguments: 5608355f576SJeff Roberson * zone The zone to drain, must be unlocked. 5618355f576SJeff Roberson * 5628355f576SJeff Roberson * Returns: 5638355f576SJeff Roberson * Nothing 5648355f576SJeff Roberson */ 5658355f576SJeff Roberson static void 5669643769aSJeff Roberson cache_drain(uma_zone_t zone) 5678355f576SJeff Roberson { 5688355f576SJeff Roberson uma_cache_t cache; 5698355f576SJeff Roberson int cpu; 5708355f576SJeff Roberson 5718355f576SJeff Roberson /* 5728355f576SJeff Roberson * We have to lock each cpu cache before locking the zone 5738355f576SJeff Roberson */ 574b6c71225SJohn Baldwin for (cpu = 0; cpu <= mp_maxid; cpu++) { 5758355f576SJeff Roberson if (CPU_ABSENT(cpu)) 5768355f576SJeff Roberson continue; 577d88797c2SBosko Milekic CPU_LOCK(cpu); 5788355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 5798355f576SJeff Roberson bucket_drain(zone, cache->uc_allocbucket); 5808355f576SJeff Roberson bucket_drain(zone, cache->uc_freebucket); 581174ab450SBosko Milekic if (cache->uc_allocbucket != NULL) 582cae33c14SJeff Roberson bucket_free(cache->uc_allocbucket); 583174ab450SBosko Milekic if (cache->uc_freebucket != NULL) 584cae33c14SJeff Roberson bucket_free(cache->uc_freebucket); 585d56368d7SBosko Milekic cache->uc_allocbucket = cache->uc_freebucket = NULL; 586d56368d7SBosko Milekic } 587aaa8bb16SJeff Roberson ZONE_LOCK(zone); 588aaa8bb16SJeff Roberson bucket_cache_drain(zone); 589aaa8bb16SJeff Roberson ZONE_UNLOCK(zone); 590aaa8bb16SJeff Roberson for (cpu = 0; cpu <= mp_maxid; cpu++) { 591aaa8bb16SJeff Roberson if (CPU_ABSENT(cpu)) 592aaa8bb16SJeff Roberson continue; 593aaa8bb16SJeff Roberson CPU_UNLOCK(cpu); 594aaa8bb16SJeff Roberson } 595aaa8bb16SJeff Roberson } 596aaa8bb16SJeff Roberson 597aaa8bb16SJeff Roberson /* 598aaa8bb16SJeff Roberson * Drain the cached buckets from a zone. Expects a locked zone on entry. 599aaa8bb16SJeff Roberson */ 600aaa8bb16SJeff Roberson static void 601aaa8bb16SJeff Roberson bucket_cache_drain(uma_zone_t zone) 602aaa8bb16SJeff Roberson { 603aaa8bb16SJeff Roberson uma_bucket_t bucket; 6048355f576SJeff Roberson 6058355f576SJeff Roberson /* 6068355f576SJeff Roberson * Drain the bucket queues and free the buckets, we just keep two per 6078355f576SJeff Roberson * cpu (alloc/free). 6088355f576SJeff Roberson */ 6098355f576SJeff Roberson while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 6108355f576SJeff Roberson LIST_REMOVE(bucket, ub_link); 6118355f576SJeff Roberson ZONE_UNLOCK(zone); 6128355f576SJeff Roberson bucket_drain(zone, bucket); 613cae33c14SJeff Roberson bucket_free(bucket); 6148355f576SJeff Roberson ZONE_LOCK(zone); 6158355f576SJeff Roberson } 6168355f576SJeff Roberson 6178355f576SJeff Roberson /* Now we do the free queue.. */ 6188355f576SJeff Roberson while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 6198355f576SJeff Roberson LIST_REMOVE(bucket, ub_link); 620cae33c14SJeff Roberson bucket_free(bucket); 6218355f576SJeff Roberson } 6228355f576SJeff Roberson } 6238355f576SJeff Roberson 6248355f576SJeff Roberson /* 6258355f576SJeff Roberson * Frees pages from a zone back to the system. This is done on demand from 6268355f576SJeff Roberson * the pageout daemon. 6278355f576SJeff Roberson * 6288355f576SJeff Roberson * Arguments: 6298355f576SJeff Roberson * zone The zone to free pages from 6309c2cd7e5SJeff Roberson * all Should we drain all items? 6318355f576SJeff Roberson * 6328355f576SJeff Roberson * Returns: 6338355f576SJeff Roberson * Nothing. 6348355f576SJeff Roberson */ 6358355f576SJeff Roberson static void 6369643769aSJeff Roberson zone_drain(uma_zone_t zone) 6378355f576SJeff Roberson { 638713deb36SJeff Roberson struct slabhead freeslabs = {}; 6398355f576SJeff Roberson uma_slab_t slab; 6408355f576SJeff Roberson uma_slab_t n; 6418355f576SJeff Roberson u_int8_t flags; 6428355f576SJeff Roberson u_int8_t *mem; 6438355f576SJeff Roberson int i; 6448355f576SJeff Roberson 6458355f576SJeff Roberson /* 6468355f576SJeff Roberson * We don't want to take pages from staticly allocated zones at this 6478355f576SJeff Roberson * time 6488355f576SJeff Roberson */ 649b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_NOFREE || zone->uz_freef == NULL) 6508355f576SJeff Roberson return; 6518355f576SJeff Roberson 6528355f576SJeff Roberson ZONE_LOCK(zone); 6538355f576SJeff Roberson 6548355f576SJeff Roberson #ifdef UMA_DEBUG 6559643769aSJeff Roberson printf("%s free items: %u\n", zone->uz_name, zone->uz_free); 6568355f576SJeff Roberson #endif 657aaa8bb16SJeff Roberson bucket_cache_drain(zone); 6589643769aSJeff Roberson if (zone->uz_free == 0) 6598355f576SJeff Roberson goto finished; 6608355f576SJeff Roberson 6618355f576SJeff Roberson slab = LIST_FIRST(&zone->uz_free_slab); 6629643769aSJeff Roberson while (slab) { 6638355f576SJeff Roberson n = LIST_NEXT(slab, us_link); 6648355f576SJeff Roberson 6658355f576SJeff Roberson /* We have no where to free these to */ 6668355f576SJeff Roberson if (slab->us_flags & UMA_SLAB_BOOT) { 6678355f576SJeff Roberson slab = n; 6688355f576SJeff Roberson continue; 6698355f576SJeff Roberson } 6708355f576SJeff Roberson 6718355f576SJeff Roberson LIST_REMOVE(slab, us_link); 6728355f576SJeff Roberson zone->uz_pages -= zone->uz_ppera; 6738355f576SJeff Roberson zone->uz_free -= zone->uz_ipers; 674713deb36SJeff Roberson 675b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_HASH) 676713deb36SJeff Roberson UMA_HASH_REMOVE(&zone->uz_hash, slab, slab->us_data); 677713deb36SJeff Roberson 678713deb36SJeff Roberson SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink); 679713deb36SJeff Roberson 680713deb36SJeff Roberson slab = n; 681713deb36SJeff Roberson } 682713deb36SJeff Roberson finished: 683713deb36SJeff Roberson ZONE_UNLOCK(zone); 684713deb36SJeff Roberson 685713deb36SJeff Roberson while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { 686713deb36SJeff Roberson SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); 6878355f576SJeff Roberson if (zone->uz_fini) 6888355f576SJeff Roberson for (i = 0; i < zone->uz_ipers; i++) 6898355f576SJeff Roberson zone->uz_fini( 6908355f576SJeff Roberson slab->us_data + (zone->uz_rsize * i), 6918355f576SJeff Roberson zone->uz_size); 6928355f576SJeff Roberson flags = slab->us_flags; 6938355f576SJeff Roberson mem = slab->us_data; 69499571dc3SJeff Roberson 695b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_OFFPAGE) 6968355f576SJeff Roberson uma_zfree_internal(slabzone, slab, NULL, 0); 697b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_MALLOC) { 69848eea375SJeff Roberson vm_object_t obj; 69948eea375SJeff Roberson 70048eea375SJeff Roberson if (flags & UMA_SLAB_KMEM) 70148eea375SJeff Roberson obj = kmem_object; 70248eea375SJeff Roberson else 70348eea375SJeff Roberson obj = NULL; 70499571dc3SJeff Roberson for (i = 0; i < zone->uz_ppera; i++) 70599571dc3SJeff Roberson vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), 70648eea375SJeff Roberson obj); 70748eea375SJeff Roberson } 7088355f576SJeff Roberson #ifdef UMA_DEBUG 7098355f576SJeff Roberson printf("%s: Returning %d bytes.\n", 7108355f576SJeff Roberson zone->uz_name, UMA_SLAB_SIZE * zone->uz_ppera); 7118355f576SJeff Roberson #endif 7128355f576SJeff Roberson zone->uz_freef(mem, UMA_SLAB_SIZE * zone->uz_ppera, flags); 7138355f576SJeff Roberson } 7148355f576SJeff Roberson 7158355f576SJeff Roberson } 7168355f576SJeff Roberson 7178355f576SJeff Roberson /* 7188355f576SJeff Roberson * Allocate a new slab for a zone. This does not insert the slab onto a list. 7198355f576SJeff Roberson * 7208355f576SJeff Roberson * Arguments: 7218355f576SJeff Roberson * zone The zone to allocate slabs for 7228355f576SJeff Roberson * wait Shall we wait? 7238355f576SJeff Roberson * 7248355f576SJeff Roberson * Returns: 7258355f576SJeff Roberson * The slab that was allocated or NULL if there is no memory and the 7268355f576SJeff Roberson * caller specified M_NOWAIT. 7278355f576SJeff Roberson */ 7288355f576SJeff Roberson static uma_slab_t 7298355f576SJeff Roberson slab_zalloc(uma_zone_t zone, int wait) 7308355f576SJeff Roberson { 7318355f576SJeff Roberson uma_slab_t slab; /* Starting slab */ 7328355f576SJeff Roberson u_int8_t *mem; 7338355f576SJeff Roberson u_int8_t flags; 7348355f576SJeff Roberson int i; 7358355f576SJeff Roberson 736a553d4b8SJeff Roberson slab = NULL; 737a553d4b8SJeff Roberson 7388355f576SJeff Roberson #ifdef UMA_DEBUG 7398355f576SJeff Roberson printf("slab_zalloc: Allocating a new slab for %s\n", zone->uz_name); 7408355f576SJeff Roberson #endif 7418355f576SJeff Roberson ZONE_UNLOCK(zone); 742a553d4b8SJeff Roberson 743b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_OFFPAGE) { 744bbee39c6SJeff Roberson slab = uma_zalloc_internal(slabzone, NULL, wait); 745a553d4b8SJeff Roberson if (slab == NULL) { 746a553d4b8SJeff Roberson ZONE_LOCK(zone); 747a553d4b8SJeff Roberson return NULL; 748a553d4b8SJeff Roberson } 749a553d4b8SJeff Roberson } 750a553d4b8SJeff Roberson 7513370c5bfSJeff Roberson /* 7523370c5bfSJeff Roberson * This reproduces the old vm_zone behavior of zero filling pages the 7533370c5bfSJeff Roberson * first time they are added to a zone. 7543370c5bfSJeff Roberson * 7553370c5bfSJeff Roberson * Malloced items are zeroed in uma_zalloc. 7563370c5bfSJeff Roberson */ 7573370c5bfSJeff Roberson 758b60f5b79SJeff Roberson if ((zone->uz_flags & UMA_ZONE_MALLOC) == 0) 7593370c5bfSJeff Roberson wait |= M_ZERO; 7603370c5bfSJeff Roberson else 7613370c5bfSJeff Roberson wait &= ~M_ZERO; 7623370c5bfSJeff Roberson 763234c7726SAlan Cox mem = zone->uz_allocf(zone, zone->uz_ppera * UMA_SLAB_SIZE, 764234c7726SAlan Cox &flags, wait); 765a553d4b8SJeff Roberson if (mem == NULL) { 7668355f576SJeff Roberson ZONE_LOCK(zone); 7678355f576SJeff Roberson return (NULL); 768a553d4b8SJeff Roberson } 7698355f576SJeff Roberson 7705c0e403bSJeff Roberson /* Point the slab into the allocated memory */ 771b60f5b79SJeff Roberson if (!(zone->uz_flags & UMA_ZONE_OFFPAGE)) 7728355f576SJeff Roberson slab = (uma_slab_t )(mem + zone->uz_pgoff); 7735c0e403bSJeff Roberson 774b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_MALLOC) 77599571dc3SJeff Roberson for (i = 0; i < zone->uz_ppera; i++) 77699571dc3SJeff Roberson vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab); 7778355f576SJeff Roberson 7788355f576SJeff Roberson slab->us_zone = zone; 7798355f576SJeff Roberson slab->us_data = mem; 7808355f576SJeff Roberson slab->us_freecount = zone->uz_ipers; 7818355f576SJeff Roberson slab->us_firstfree = 0; 7828355f576SJeff Roberson slab->us_flags = flags; 7838355f576SJeff Roberson for (i = 0; i < zone->uz_ipers; i++) 7848355f576SJeff Roberson slab->us_freelist[i] = i+1; 7858355f576SJeff Roberson 7868355f576SJeff Roberson if (zone->uz_init) 7878355f576SJeff Roberson for (i = 0; i < zone->uz_ipers; i++) 7888355f576SJeff Roberson zone->uz_init(slab->us_data + (zone->uz_rsize * i), 7898355f576SJeff Roberson zone->uz_size); 7905c0e403bSJeff Roberson ZONE_LOCK(zone); 7915c0e403bSJeff Roberson 792b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_HASH) 7935c0e403bSJeff Roberson UMA_HASH_INSERT(&zone->uz_hash, slab, mem); 7948355f576SJeff Roberson 7958355f576SJeff Roberson zone->uz_pages += zone->uz_ppera; 7968355f576SJeff Roberson zone->uz_free += zone->uz_ipers; 7978355f576SJeff Roberson 7988355f576SJeff Roberson return (slab); 7998355f576SJeff Roberson } 8008355f576SJeff Roberson 8018355f576SJeff Roberson /* 802009b6fcbSJeff Roberson * This function is intended to be used early on in place of page_alloc() so 803009b6fcbSJeff Roberson * that we may use the boot time page cache to satisfy allocations before 804009b6fcbSJeff Roberson * the VM is ready. 805009b6fcbSJeff Roberson */ 806009b6fcbSJeff Roberson static void * 807009b6fcbSJeff Roberson startup_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 808009b6fcbSJeff Roberson { 809009b6fcbSJeff Roberson /* 810009b6fcbSJeff Roberson * Check our small startup cache to see if it has pages remaining. 811009b6fcbSJeff Roberson */ 812009b6fcbSJeff Roberson mtx_lock(&uma_mtx); 813009b6fcbSJeff Roberson if (uma_boot_free != 0) { 814009b6fcbSJeff Roberson uma_slab_t tmps; 815009b6fcbSJeff Roberson 816009b6fcbSJeff Roberson tmps = LIST_FIRST(&uma_boot_pages); 817009b6fcbSJeff Roberson LIST_REMOVE(tmps, us_link); 818009b6fcbSJeff Roberson uma_boot_free--; 819009b6fcbSJeff Roberson mtx_unlock(&uma_mtx); 820009b6fcbSJeff Roberson *pflag = tmps->us_flags; 821009b6fcbSJeff Roberson return (tmps->us_data); 822009b6fcbSJeff Roberson } 823009b6fcbSJeff Roberson mtx_unlock(&uma_mtx); 824009b6fcbSJeff Roberson if (booted == 0) 825009b6fcbSJeff Roberson panic("UMA: Increase UMA_BOOT_PAGES"); 826009b6fcbSJeff Roberson /* 827009b6fcbSJeff Roberson * Now that we've booted reset these users to their real allocator. 828009b6fcbSJeff Roberson */ 829009b6fcbSJeff Roberson #ifdef UMA_MD_SMALL_ALLOC 830009b6fcbSJeff Roberson zone->uz_allocf = uma_small_alloc; 831009b6fcbSJeff Roberson #else 832009b6fcbSJeff Roberson zone->uz_allocf = page_alloc; 833009b6fcbSJeff Roberson #endif 834009b6fcbSJeff Roberson return zone->uz_allocf(zone, bytes, pflag, wait); 835009b6fcbSJeff Roberson } 836009b6fcbSJeff Roberson 837009b6fcbSJeff Roberson /* 8388355f576SJeff Roberson * Allocates a number of pages from the system 8398355f576SJeff Roberson * 8408355f576SJeff Roberson * Arguments: 8418355f576SJeff Roberson * zone Unused 8428355f576SJeff Roberson * bytes The number of bytes requested 8438355f576SJeff Roberson * wait Shall we wait? 8448355f576SJeff Roberson * 8458355f576SJeff Roberson * Returns: 8468355f576SJeff Roberson * A pointer to the alloced memory or possibly 8478355f576SJeff Roberson * NULL if M_NOWAIT is set. 8488355f576SJeff Roberson */ 8498355f576SJeff Roberson static void * 8508355f576SJeff Roberson page_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 8518355f576SJeff Roberson { 8528355f576SJeff Roberson void *p; /* Returned page */ 8538355f576SJeff Roberson 8548355f576SJeff Roberson *pflag = UMA_SLAB_KMEM; 8558355f576SJeff Roberson p = (void *) kmem_malloc(kmem_map, bytes, wait); 8568355f576SJeff Roberson 8578355f576SJeff Roberson return (p); 8588355f576SJeff Roberson } 8598355f576SJeff Roberson 8608355f576SJeff Roberson /* 8618355f576SJeff Roberson * Allocates a number of pages from within an object 8628355f576SJeff Roberson * 8638355f576SJeff Roberson * Arguments: 8648355f576SJeff Roberson * zone Unused 8658355f576SJeff Roberson * bytes The number of bytes requested 8668355f576SJeff Roberson * wait Shall we wait? 8678355f576SJeff Roberson * 8688355f576SJeff Roberson * Returns: 8698355f576SJeff Roberson * A pointer to the alloced memory or possibly 8708355f576SJeff Roberson * NULL if M_NOWAIT is set. 8718355f576SJeff Roberson */ 8728355f576SJeff Roberson static void * 8738355f576SJeff Roberson obj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 8748355f576SJeff Roberson { 875b245ac95SAlan Cox vm_object_t object; 876b245ac95SAlan Cox vm_offset_t retkva, zkva; 8778355f576SJeff Roberson vm_page_t p; 878b245ac95SAlan Cox int pages, startpages; 8798355f576SJeff Roberson 880b245ac95SAlan Cox object = zone->uz_obj; 88155f7c614SArchie Cobbs retkva = 0; 8828355f576SJeff Roberson 8838355f576SJeff Roberson /* 88464f051e9SJeff Roberson * This looks a little weird since we're getting one page at a time. 8858355f576SJeff Roberson */ 886b245ac95SAlan Cox VM_OBJECT_LOCK(object); 887b245ac95SAlan Cox p = TAILQ_LAST(&object->memq, pglist); 888b245ac95SAlan Cox pages = p != NULL ? p->pindex + 1 : 0; 889b245ac95SAlan Cox startpages = pages; 8908355f576SJeff Roberson zkva = zone->uz_kva + pages * PAGE_SIZE; 891b245ac95SAlan Cox for (; bytes > 0; bytes -= PAGE_SIZE) { 892b245ac95SAlan Cox p = vm_page_alloc(object, pages, 893b245ac95SAlan Cox VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); 894b245ac95SAlan Cox if (p == NULL) { 895b245ac95SAlan Cox if (pages != startpages) 896b245ac95SAlan Cox pmap_qremove(retkva, pages - startpages); 897b245ac95SAlan Cox while (pages != startpages) { 898b245ac95SAlan Cox pages--; 899b245ac95SAlan Cox p = TAILQ_LAST(&object->memq, pglist); 900b245ac95SAlan Cox vm_page_lock_queues(); 901b245ac95SAlan Cox vm_page_unwire(p, 0); 902b245ac95SAlan Cox vm_page_free(p); 903b245ac95SAlan Cox vm_page_unlock_queues(); 904b245ac95SAlan Cox } 905b245ac95SAlan Cox retkva = 0; 906b245ac95SAlan Cox goto done; 907b245ac95SAlan Cox } 908b245ac95SAlan Cox pmap_qenter(zkva, &p, 1); 90955f7c614SArchie Cobbs if (retkva == 0) 9108355f576SJeff Roberson retkva = zkva; 911b245ac95SAlan Cox zkva += PAGE_SIZE; 9128355f576SJeff Roberson pages += 1; 9138355f576SJeff Roberson } 914b245ac95SAlan Cox done: 915b245ac95SAlan Cox VM_OBJECT_UNLOCK(object); 9168355f576SJeff Roberson *flags = UMA_SLAB_PRIV; 9178355f576SJeff Roberson 9188355f576SJeff Roberson return ((void *)retkva); 9198355f576SJeff Roberson } 9208355f576SJeff Roberson 9218355f576SJeff Roberson /* 9228355f576SJeff Roberson * Frees a number of pages to the system 9238355f576SJeff Roberson * 9248355f576SJeff Roberson * Arguments: 9258355f576SJeff Roberson * mem A pointer to the memory to be freed 9268355f576SJeff Roberson * size The size of the memory being freed 9278355f576SJeff Roberson * flags The original p->us_flags field 9288355f576SJeff Roberson * 9298355f576SJeff Roberson * Returns: 9308355f576SJeff Roberson * Nothing 9318355f576SJeff Roberson */ 9328355f576SJeff Roberson static void 9338355f576SJeff Roberson page_free(void *mem, int size, u_int8_t flags) 9348355f576SJeff Roberson { 9358355f576SJeff Roberson vm_map_t map; 9363370c5bfSJeff Roberson 9378355f576SJeff Roberson if (flags & UMA_SLAB_KMEM) 9388355f576SJeff Roberson map = kmem_map; 9398355f576SJeff Roberson else 9408355f576SJeff Roberson panic("UMA: page_free used with invalid flags %d\n", flags); 9418355f576SJeff Roberson 9428355f576SJeff Roberson kmem_free(map, (vm_offset_t)mem, size); 9438355f576SJeff Roberson } 9448355f576SJeff Roberson 9458355f576SJeff Roberson /* 9468355f576SJeff Roberson * Zero fill initializer 9478355f576SJeff Roberson * 9488355f576SJeff Roberson * Arguments/Returns follow uma_init specifications 9498355f576SJeff Roberson */ 9508355f576SJeff Roberson static void 9518355f576SJeff Roberson zero_init(void *mem, int size) 9528355f576SJeff Roberson { 9538355f576SJeff Roberson bzero(mem, size); 9548355f576SJeff Roberson } 9558355f576SJeff Roberson 9568355f576SJeff Roberson /* 9578355f576SJeff Roberson * Finish creating a small uma zone. This calculates ipers, and the zone size. 9588355f576SJeff Roberson * 9598355f576SJeff Roberson * Arguments 9608355f576SJeff Roberson * zone The zone we should initialize 9618355f576SJeff Roberson * 9628355f576SJeff Roberson * Returns 9638355f576SJeff Roberson * Nothing 9648355f576SJeff Roberson */ 9658355f576SJeff Roberson static void 9668355f576SJeff Roberson zone_small_init(uma_zone_t zone) 9678355f576SJeff Roberson { 9688355f576SJeff Roberson int rsize; 9698355f576SJeff Roberson int memused; 9708355f576SJeff Roberson int ipers; 9718355f576SJeff Roberson 9728355f576SJeff Roberson rsize = zone->uz_size; 9738355f576SJeff Roberson 9748355f576SJeff Roberson if (rsize < UMA_SMALLEST_UNIT) 9758355f576SJeff Roberson rsize = UMA_SMALLEST_UNIT; 9768355f576SJeff Roberson 9778355f576SJeff Roberson if (rsize & zone->uz_align) 9788355f576SJeff Roberson rsize = (rsize & ~zone->uz_align) + (zone->uz_align + 1); 9798355f576SJeff Roberson 9808355f576SJeff Roberson zone->uz_rsize = rsize; 9818355f576SJeff Roberson 9828355f576SJeff Roberson rsize += 1; /* Account for the byte of linkage */ 9838355f576SJeff Roberson zone->uz_ipers = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) / rsize; 9848355f576SJeff Roberson zone->uz_ppera = 1; 9858355f576SJeff Roberson 98620e8e865SBosko Milekic KASSERT(zone->uz_ipers != 0, ("zone_small_init: ipers is 0, uh-oh!")); 9878355f576SJeff Roberson memused = zone->uz_ipers * zone->uz_rsize; 9888355f576SJeff Roberson 9898355f576SJeff Roberson /* Can we do any better? */ 9908355f576SJeff Roberson if ((UMA_SLAB_SIZE - memused) >= UMA_MAX_WASTE) { 99120e8e865SBosko Milekic /* 99220e8e865SBosko Milekic * We can't do this if we're internal or if we've been 99320e8e865SBosko Milekic * asked to not go to the VM for buckets. If we do this we 99420e8e865SBosko Milekic * may end up going to the VM (kmem_map) for slabs which we 99520e8e865SBosko Milekic * do not want to do if we're UMA_ZFLAG_CACHEONLY as a 99620e8e865SBosko Milekic * result of UMA_ZONE_VM, which clearly forbids it. 99720e8e865SBosko Milekic */ 99820e8e865SBosko Milekic if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) || 99920e8e865SBosko Milekic (zone->uz_flags & UMA_ZFLAG_CACHEONLY)) 10008355f576SJeff Roberson return; 10018355f576SJeff Roberson ipers = UMA_SLAB_SIZE / zone->uz_rsize; 10028355f576SJeff Roberson if (ipers > zone->uz_ipers) { 1003b60f5b79SJeff Roberson zone->uz_flags |= UMA_ZONE_OFFPAGE; 1004b60f5b79SJeff Roberson if ((zone->uz_flags & UMA_ZONE_MALLOC) == 0) 1005b60f5b79SJeff Roberson zone->uz_flags |= UMA_ZONE_HASH; 10068355f576SJeff Roberson zone->uz_ipers = ipers; 10078355f576SJeff Roberson } 10088355f576SJeff Roberson } 10098355f576SJeff Roberson } 10108355f576SJeff Roberson 10118355f576SJeff Roberson /* 10128355f576SJeff Roberson * Finish creating a large (> UMA_SLAB_SIZE) uma zone. Just give in and do 10138355f576SJeff Roberson * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be 10148355f576SJeff Roberson * more complicated. 10158355f576SJeff Roberson * 10168355f576SJeff Roberson * Arguments 10178355f576SJeff Roberson * zone The zone we should initialize 10188355f576SJeff Roberson * 10198355f576SJeff Roberson * Returns 10208355f576SJeff Roberson * Nothing 10218355f576SJeff Roberson */ 10228355f576SJeff Roberson static void 10238355f576SJeff Roberson zone_large_init(uma_zone_t zone) 10248355f576SJeff Roberson { 10258355f576SJeff Roberson int pages; 10268355f576SJeff Roberson 102720e8e865SBosko Milekic KASSERT((zone->uz_flags & UMA_ZFLAG_CACHEONLY) == 0, 102820e8e865SBosko Milekic ("zone_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY zone")); 102920e8e865SBosko Milekic 10308355f576SJeff Roberson pages = zone->uz_size / UMA_SLAB_SIZE; 10318355f576SJeff Roberson 10328355f576SJeff Roberson /* Account for remainder */ 10338355f576SJeff Roberson if ((pages * UMA_SLAB_SIZE) < zone->uz_size) 10348355f576SJeff Roberson pages++; 10358355f576SJeff Roberson 10368355f576SJeff Roberson zone->uz_ppera = pages; 10378355f576SJeff Roberson zone->uz_ipers = 1; 10388355f576SJeff Roberson 1039b60f5b79SJeff Roberson zone->uz_flags |= UMA_ZONE_OFFPAGE; 1040b60f5b79SJeff Roberson if ((zone->uz_flags & UMA_ZONE_MALLOC) == 0) 1041b60f5b79SJeff Roberson zone->uz_flags |= UMA_ZONE_HASH; 104299571dc3SJeff Roberson 10438355f576SJeff Roberson zone->uz_rsize = zone->uz_size; 10448355f576SJeff Roberson } 10458355f576SJeff Roberson 10468355f576SJeff Roberson /* 10478355f576SJeff Roberson * Zone header ctor. This initializes all fields, locks, etc. And inserts 10488355f576SJeff Roberson * the zone onto the global zone list. 10498355f576SJeff Roberson * 10508355f576SJeff Roberson * Arguments/Returns follow uma_ctor specifications 10518355f576SJeff Roberson * udata Actually uma_zcreat_args 10528355f576SJeff Roberson */ 10538355f576SJeff Roberson 10548355f576SJeff Roberson static void 10558355f576SJeff Roberson zone_ctor(void *mem, int size, void *udata) 10568355f576SJeff Roberson { 10578355f576SJeff Roberson struct uma_zctor_args *arg = udata; 10588355f576SJeff Roberson uma_zone_t zone = mem; 105928bc4419SJeff Roberson int privlc; 10608355f576SJeff Roberson 10618355f576SJeff Roberson bzero(zone, size); 10628355f576SJeff Roberson zone->uz_name = arg->name; 10638355f576SJeff Roberson zone->uz_size = arg->size; 10648355f576SJeff Roberson zone->uz_ctor = arg->ctor; 10658355f576SJeff Roberson zone->uz_dtor = arg->dtor; 10668355f576SJeff Roberson zone->uz_init = arg->uminit; 1067e221e841SJeff Roberson zone->uz_fini = arg->fini; 10688355f576SJeff Roberson zone->uz_align = arg->align; 10698355f576SJeff Roberson zone->uz_free = 0; 10708355f576SJeff Roberson zone->uz_pages = 0; 1071b60f5b79SJeff Roberson zone->uz_flags = arg->flags; 10728355f576SJeff Roberson zone->uz_allocf = page_alloc; 10738355f576SJeff Roberson zone->uz_freef = page_free; 10748355f576SJeff Roberson 10758355f576SJeff Roberson if (arg->flags & UMA_ZONE_ZINIT) 10768355f576SJeff Roberson zone->uz_init = zero_init; 10778355f576SJeff Roberson 107818aa2de5SJeff Roberson if (arg->flags & UMA_ZONE_VM) 107920e8e865SBosko Milekic zone->uz_flags |= UMA_ZFLAG_CACHEONLY; 108018aa2de5SJeff Roberson 108120e8e865SBosko Milekic /* 108220e8e865SBosko Milekic * XXX: 108320e8e865SBosko Milekic * The +1 byte added to uz_size is to account for the byte of 108420e8e865SBosko Milekic * linkage that is added to the size in zone_small_init(). If 108520e8e865SBosko Milekic * we don't account for this here then we may end up in 108620e8e865SBosko Milekic * zone_small_init() with a calculated 'ipers' of 0. 108720e8e865SBosko Milekic */ 108820e8e865SBosko Milekic if ((zone->uz_size+1) > (UMA_SLAB_SIZE - sizeof(struct uma_slab))) 10898355f576SJeff Roberson zone_large_init(zone); 10908355f576SJeff Roberson else 10918355f576SJeff Roberson zone_small_init(zone); 1092009b6fcbSJeff Roberson /* 1093009b6fcbSJeff Roberson * If we haven't booted yet we need allocations to go through the 1094009b6fcbSJeff Roberson * startup cache until the vm is ready. 1095009b6fcbSJeff Roberson */ 109648eea375SJeff Roberson if (zone->uz_ppera == 1) { 1097009b6fcbSJeff Roberson #ifdef UMA_MD_SMALL_ALLOC 109848eea375SJeff Roberson zone->uz_allocf = uma_small_alloc; 109948eea375SJeff Roberson zone->uz_freef = uma_small_free; 1100009b6fcbSJeff Roberson #endif 1101009b6fcbSJeff Roberson if (booted == 0) 1102009b6fcbSJeff Roberson zone->uz_allocf = startup_alloc; 110348eea375SJeff Roberson } 110428bc4419SJeff Roberson if (arg->flags & UMA_ZONE_MTXCLASS) 110528bc4419SJeff Roberson privlc = 1; 110628bc4419SJeff Roberson else 110728bc4419SJeff Roberson privlc = 0; 110828bc4419SJeff Roberson 11098355f576SJeff Roberson /* 11108355f576SJeff Roberson * If we're putting the slab header in the actual page we need to 11118355f576SJeff Roberson * figure out where in each page it goes. This calculates a right 11129d5abbddSJens Schweikhardt * justified offset into the memory on an ALIGN_PTR boundary. 11138355f576SJeff Roberson */ 1114b60f5b79SJeff Roberson if (!(zone->uz_flags & UMA_ZONE_OFFPAGE)) { 11158355f576SJeff Roberson int totsize; 11168355f576SJeff Roberson 11178355f576SJeff Roberson /* Size of the slab struct and free list */ 11188355f576SJeff Roberson totsize = sizeof(struct uma_slab) + zone->uz_ipers; 11198355f576SJeff Roberson if (totsize & UMA_ALIGN_PTR) 11208355f576SJeff Roberson totsize = (totsize & ~UMA_ALIGN_PTR) + 11218355f576SJeff Roberson (UMA_ALIGN_PTR + 1); 11228355f576SJeff Roberson zone->uz_pgoff = UMA_SLAB_SIZE - totsize; 11238355f576SJeff Roberson totsize = zone->uz_pgoff + sizeof(struct uma_slab) 11248355f576SJeff Roberson + zone->uz_ipers; 11258355f576SJeff Roberson /* I don't think it's possible, but I'll make sure anyway */ 11268355f576SJeff Roberson if (totsize > UMA_SLAB_SIZE) { 11278355f576SJeff Roberson printf("zone %s ipers %d rsize %d size %d\n", 11288355f576SJeff Roberson zone->uz_name, zone->uz_ipers, zone->uz_rsize, 11298355f576SJeff Roberson zone->uz_size); 11308355f576SJeff Roberson panic("UMA slab won't fit.\n"); 11318355f576SJeff Roberson } 11328355f576SJeff Roberson } 11338355f576SJeff Roberson 1134b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_HASH) 113599571dc3SJeff Roberson hash_alloc(&zone->uz_hash); 113699571dc3SJeff Roberson 11378355f576SJeff Roberson #ifdef UMA_DEBUG 11388355f576SJeff Roberson printf("%s(%p) size = %d ipers = %d ppera = %d pgoff = %d\n", 11398355f576SJeff Roberson zone->uz_name, zone, 11408355f576SJeff Roberson zone->uz_size, zone->uz_ipers, 11418355f576SJeff Roberson zone->uz_ppera, zone->uz_pgoff); 11428355f576SJeff Roberson #endif 114328bc4419SJeff Roberson ZONE_LOCK_INIT(zone, privlc); 11448355f576SJeff Roberson 11458355f576SJeff Roberson mtx_lock(&uma_mtx); 11468355f576SJeff Roberson LIST_INSERT_HEAD(&uma_zones, zone, uz_link); 11478355f576SJeff Roberson mtx_unlock(&uma_mtx); 11488355f576SJeff Roberson 11498355f576SJeff Roberson /* 11508355f576SJeff Roberson * Some internal zones don't have room allocated for the per cpu 11518355f576SJeff Roberson * caches. If we're internal, bail out here. 11528355f576SJeff Roberson */ 11538355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 11548355f576SJeff Roberson return; 11558355f576SJeff Roberson 1156cae33c14SJeff Roberson if (zone->uz_ipers <= BUCKET_MAX) 1157cae33c14SJeff Roberson zone->uz_count = zone->uz_ipers; 11588355f576SJeff Roberson else 1159cae33c14SJeff Roberson zone->uz_count = BUCKET_MAX; 11608355f576SJeff Roberson } 11618355f576SJeff Roberson 11628355f576SJeff Roberson /* 11639c2cd7e5SJeff Roberson * Zone header dtor. This frees all data, destroys locks, frees the hash table 11649c2cd7e5SJeff Roberson * and removes the zone from the global list. 11659c2cd7e5SJeff Roberson * 11669c2cd7e5SJeff Roberson * Arguments/Returns follow uma_dtor specifications 11679c2cd7e5SJeff Roberson * udata unused 11689c2cd7e5SJeff Roberson */ 11699c2cd7e5SJeff Roberson 11709c2cd7e5SJeff Roberson static void 11719c2cd7e5SJeff Roberson zone_dtor(void *arg, int size, void *udata) 11729c2cd7e5SJeff Roberson { 11739c2cd7e5SJeff Roberson uma_zone_t zone; 11749c2cd7e5SJeff Roberson 11759c2cd7e5SJeff Roberson zone = (uma_zone_t)arg; 11769643769aSJeff Roberson 11779643769aSJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) 11789643769aSJeff Roberson cache_drain(zone); 117917b9cc49SJeff Roberson mtx_lock(&uma_mtx); 118017b9cc49SJeff Roberson LIST_REMOVE(zone, uz_link); 11819643769aSJeff Roberson zone_drain(zone); 118217b9cc49SJeff Roberson mtx_unlock(&uma_mtx); 118317b9cc49SJeff Roberson 11849c2cd7e5SJeff Roberson ZONE_LOCK(zone); 1185504d5de3SJeff Roberson if (zone->uz_free != 0) { 118664f051e9SJeff Roberson printf("Zone %s was not empty (%d items). " 118764f051e9SJeff Roberson " Lost %d pages of memory.\n", 1188886eaaacSPoul-Henning Kamp zone->uz_name, zone->uz_free, zone->uz_pages); 1189504d5de3SJeff Roberson uma_print_zone(zone); 1190504d5de3SJeff Roberson } 11919c2cd7e5SJeff Roberson 11929c2cd7e5SJeff Roberson ZONE_UNLOCK(zone); 1193b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_HASH) 11940aef6126SJeff Roberson hash_free(&zone->uz_hash); 11950aef6126SJeff Roberson 11969c2cd7e5SJeff Roberson ZONE_LOCK_FINI(zone); 11979c2cd7e5SJeff Roberson } 11989c2cd7e5SJeff Roberson /* 11998355f576SJeff Roberson * Traverses every zone in the system and calls a callback 12008355f576SJeff Roberson * 12018355f576SJeff Roberson * Arguments: 12028355f576SJeff Roberson * zfunc A pointer to a function which accepts a zone 12038355f576SJeff Roberson * as an argument. 12048355f576SJeff Roberson * 12058355f576SJeff Roberson * Returns: 12068355f576SJeff Roberson * Nothing 12078355f576SJeff Roberson */ 12088355f576SJeff Roberson static void 12098355f576SJeff Roberson zone_foreach(void (*zfunc)(uma_zone_t)) 12108355f576SJeff Roberson { 12118355f576SJeff Roberson uma_zone_t zone; 12128355f576SJeff Roberson 12138355f576SJeff Roberson mtx_lock(&uma_mtx); 121464f051e9SJeff Roberson LIST_FOREACH(zone, &uma_zones, uz_link) 12158355f576SJeff Roberson zfunc(zone); 12168355f576SJeff Roberson mtx_unlock(&uma_mtx); 12178355f576SJeff Roberson } 12188355f576SJeff Roberson 12198355f576SJeff Roberson /* Public functions */ 12208355f576SJeff Roberson /* See uma.h */ 12218355f576SJeff Roberson void 12228355f576SJeff Roberson uma_startup(void *bootmem) 12238355f576SJeff Roberson { 12248355f576SJeff Roberson struct uma_zctor_args args; 12258355f576SJeff Roberson uma_slab_t slab; 12268355f576SJeff Roberson int slabsize; 12278355f576SJeff Roberson int i; 12288355f576SJeff Roberson 12298355f576SJeff Roberson #ifdef UMA_DEBUG 12308355f576SJeff Roberson printf("Creating uma zone headers zone.\n"); 12318355f576SJeff Roberson #endif 12326008862bSJohn Baldwin mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF); 12338355f576SJeff Roberson /* "manually" Create the initial zone */ 12348355f576SJeff Roberson args.name = "UMA Zones"; 12358355f576SJeff Roberson args.size = sizeof(struct uma_zone) + 1236b6c71225SJohn Baldwin (sizeof(struct uma_cache) * (mp_maxid + 1)); 12378355f576SJeff Roberson args.ctor = zone_ctor; 12389c2cd7e5SJeff Roberson args.dtor = zone_dtor; 12398355f576SJeff Roberson args.uminit = zero_init; 12408355f576SJeff Roberson args.fini = NULL; 12418355f576SJeff Roberson args.align = 32 - 1; 1242b60f5b79SJeff Roberson args.flags = UMA_ZFLAG_INTERNAL; 12438355f576SJeff Roberson /* The initial zone has no Per cpu queues so it's smaller */ 12448355f576SJeff Roberson zone_ctor(zones, sizeof(struct uma_zone), &args); 12458355f576SJeff Roberson 1246d88797c2SBosko Milekic /* Initialize the pcpu cache lock set once and for all */ 1247b6c71225SJohn Baldwin for (i = 0; i <= mp_maxid; i++) 1248d88797c2SBosko Milekic CPU_LOCK_INIT(i); 12498355f576SJeff Roberson #ifdef UMA_DEBUG 12508355f576SJeff Roberson printf("Filling boot free list.\n"); 12518355f576SJeff Roberson #endif 12528355f576SJeff Roberson for (i = 0; i < UMA_BOOT_PAGES; i++) { 12538355f576SJeff Roberson slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE)); 12548355f576SJeff Roberson slab->us_data = (u_int8_t *)slab; 12558355f576SJeff Roberson slab->us_flags = UMA_SLAB_BOOT; 12568355f576SJeff Roberson LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link); 12578355f576SJeff Roberson uma_boot_free++; 12588355f576SJeff Roberson } 12598355f576SJeff Roberson 12608355f576SJeff Roberson #ifdef UMA_DEBUG 12618355f576SJeff Roberson printf("Creating slab zone.\n"); 12628355f576SJeff Roberson #endif 12638355f576SJeff Roberson 12648355f576SJeff Roberson /* 12658355f576SJeff Roberson * This is the max number of free list items we'll have with 12668355f576SJeff Roberson * offpage slabs. 12678355f576SJeff Roberson */ 12688355f576SJeff Roberson slabsize = UMA_SLAB_SIZE - sizeof(struct uma_slab); 12698355f576SJeff Roberson slabsize /= UMA_MAX_WASTE; 12708355f576SJeff Roberson slabsize++; /* In case there it's rounded */ 12718355f576SJeff Roberson slabsize += sizeof(struct uma_slab); 12728355f576SJeff Roberson 12738355f576SJeff Roberson /* Now make a zone for slab headers */ 12748355f576SJeff Roberson slabzone = uma_zcreate("UMA Slabs", 12758355f576SJeff Roberson slabsize, 12768355f576SJeff Roberson NULL, NULL, NULL, NULL, 1277b60f5b79SJeff Roberson UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 12788355f576SJeff Roberson 12798355f576SJeff Roberson hashzone = uma_zcreate("UMA Hash", 12808355f576SJeff Roberson sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 12818355f576SJeff Roberson NULL, NULL, NULL, NULL, 1282b60f5b79SJeff Roberson UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 12838355f576SJeff Roberson 1284cae33c14SJeff Roberson bucket_init(); 12858355f576SJeff Roberson 1286009b6fcbSJeff Roberson #ifdef UMA_MD_SMALL_ALLOC 128748eea375SJeff Roberson booted = 1; 128848eea375SJeff Roberson #endif 12898355f576SJeff Roberson 12908355f576SJeff Roberson #ifdef UMA_DEBUG 12918355f576SJeff Roberson printf("UMA startup complete.\n"); 12928355f576SJeff Roberson #endif 12938355f576SJeff Roberson } 12948355f576SJeff Roberson 12958355f576SJeff Roberson /* see uma.h */ 12968355f576SJeff Roberson void 129799571dc3SJeff Roberson uma_startup2(void) 12988355f576SJeff Roberson { 12998355f576SJeff Roberson booted = 1; 130086bbae32SJeff Roberson bucket_enable(); 13018355f576SJeff Roberson #ifdef UMA_DEBUG 13028355f576SJeff Roberson printf("UMA startup2 complete.\n"); 13038355f576SJeff Roberson #endif 13048355f576SJeff Roberson } 13058355f576SJeff Roberson 13068355f576SJeff Roberson /* 13078355f576SJeff Roberson * Initialize our callout handle 13088355f576SJeff Roberson * 13098355f576SJeff Roberson */ 13108355f576SJeff Roberson 13118355f576SJeff Roberson static void 13128355f576SJeff Roberson uma_startup3(void) 13138355f576SJeff Roberson { 13148355f576SJeff Roberson #ifdef UMA_DEBUG 13158355f576SJeff Roberson printf("Starting callout.\n"); 13168355f576SJeff Roberson #endif 13178355f576SJeff Roberson callout_init(&uma_callout, 0); 13189643769aSJeff Roberson callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 13198355f576SJeff Roberson #ifdef UMA_DEBUG 13208355f576SJeff Roberson printf("UMA startup3 complete.\n"); 13218355f576SJeff Roberson #endif 13228355f576SJeff Roberson } 13238355f576SJeff Roberson 13248355f576SJeff Roberson /* See uma.h */ 13258355f576SJeff Roberson uma_zone_t 1326c3bdc05fSAndrew R. Reiter uma_zcreate(char *name, size_t size, uma_ctor ctor, uma_dtor dtor, 1327c3bdc05fSAndrew R. Reiter uma_init uminit, uma_fini fini, int align, u_int16_t flags) 13288355f576SJeff Roberson 13298355f576SJeff Roberson { 13308355f576SJeff Roberson struct uma_zctor_args args; 13318355f576SJeff Roberson 13328355f576SJeff Roberson /* This stuff is essential for the zone ctor */ 13338355f576SJeff Roberson args.name = name; 13348355f576SJeff Roberson args.size = size; 13358355f576SJeff Roberson args.ctor = ctor; 13368355f576SJeff Roberson args.dtor = dtor; 13378355f576SJeff Roberson args.uminit = uminit; 13388355f576SJeff Roberson args.fini = fini; 13398355f576SJeff Roberson args.align = align; 13408355f576SJeff Roberson args.flags = flags; 13418355f576SJeff Roberson 1342a163d034SWarner Losh return (uma_zalloc_internal(zones, &args, M_WAITOK)); 13438355f576SJeff Roberson } 13448355f576SJeff Roberson 13458355f576SJeff Roberson /* See uma.h */ 13469c2cd7e5SJeff Roberson void 13479c2cd7e5SJeff Roberson uma_zdestroy(uma_zone_t zone) 13489c2cd7e5SJeff Roberson { 13499c2cd7e5SJeff Roberson uma_zfree_internal(zones, zone, NULL, 0); 13509c2cd7e5SJeff Roberson } 13519c2cd7e5SJeff Roberson 13529c2cd7e5SJeff Roberson /* See uma.h */ 13538355f576SJeff Roberson void * 13542cc35ff9SJeff Roberson uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 13558355f576SJeff Roberson { 13568355f576SJeff Roberson void *item; 13578355f576SJeff Roberson uma_cache_t cache; 13588355f576SJeff Roberson uma_bucket_t bucket; 13598355f576SJeff Roberson int cpu; 13608355f576SJeff Roberson 13618355f576SJeff Roberson /* This is the fast path allocation */ 13628355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC_1 13638355f576SJeff Roberson printf("Allocating one item from %s(%p)\n", zone->uz_name, zone); 13648355f576SJeff Roberson #endif 1365a553d4b8SJeff Roberson 13668522511bSHartmut Brandt #ifdef INVARIANTS 13678522511bSHartmut Brandt /* 13688522511bSHartmut Brandt * To make sure that WAITOK or NOWAIT is set, but not more than 13698522511bSHartmut Brandt * one, and check against the API botches that are common. 13708522511bSHartmut Brandt * The uma code implies M_WAITOK if M_NOWAIT is not set, so 13718522511bSHartmut Brandt * we default to waiting if none of the flags is set. 13728522511bSHartmut Brandt */ 13738522511bSHartmut Brandt cpu = flags & (M_WAITOK | M_NOWAIT | M_DONTWAIT | M_TRYWAIT); 13748522511bSHartmut Brandt if (cpu != M_NOWAIT && cpu != M_WAITOK) { 13758522511bSHartmut Brandt static struct timeval lasterr; 13768522511bSHartmut Brandt static int curerr, once; 13778522511bSHartmut Brandt if (once == 0 && ppsratecheck(&lasterr, &curerr, 1)) { 13788522511bSHartmut Brandt printf("Bad uma_zalloc flags: %x\n", cpu); 13798522511bSHartmut Brandt backtrace(); 13808522511bSHartmut Brandt once++; 13818522511bSHartmut Brandt } 13828522511bSHartmut Brandt } 13838522511bSHartmut Brandt #endif 13844c1cc01cSJohn Baldwin if (!(flags & M_NOWAIT)) { 13854c1cc01cSJohn Baldwin KASSERT(curthread->td_intr_nesting_level == 0, 1386a163d034SWarner Losh ("malloc(M_WAITOK) in interrupt context")); 138726306795SJohn Baldwin WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 138826306795SJohn Baldwin "malloc() of \"%s\"", zone->uz_name); 13894c1cc01cSJohn Baldwin } 13904c1cc01cSJohn Baldwin 1391a553d4b8SJeff Roberson zalloc_restart: 13928355f576SJeff Roberson cpu = PCPU_GET(cpuid); 1393d88797c2SBosko Milekic CPU_LOCK(cpu); 13948355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 13958355f576SJeff Roberson 13968355f576SJeff Roberson zalloc_start: 13978355f576SJeff Roberson bucket = cache->uc_allocbucket; 13988355f576SJeff Roberson 13998355f576SJeff Roberson if (bucket) { 1400cae33c14SJeff Roberson if (bucket->ub_cnt > 0) { 1401cae33c14SJeff Roberson bucket->ub_cnt--; 1402cae33c14SJeff Roberson item = bucket->ub_bucket[bucket->ub_cnt]; 14038355f576SJeff Roberson #ifdef INVARIANTS 1404cae33c14SJeff Roberson bucket->ub_bucket[bucket->ub_cnt] = NULL; 14058355f576SJeff Roberson #endif 14068355f576SJeff Roberson KASSERT(item != NULL, 14078355f576SJeff Roberson ("uma_zalloc: Bucket pointer mangled.")); 14088355f576SJeff Roberson cache->uc_allocs++; 1409639c9550SJeff Roberson #ifdef INVARIANTS 141081f71edaSMatt Jacob ZONE_LOCK(zone); 1411639c9550SJeff Roberson uma_dbg_alloc(zone, NULL, item); 141281f71edaSMatt Jacob ZONE_UNLOCK(zone); 1413639c9550SJeff Roberson #endif 1414d88797c2SBosko Milekic CPU_UNLOCK(cpu); 14158355f576SJeff Roberson if (zone->uz_ctor) 14168355f576SJeff Roberson zone->uz_ctor(item, zone->uz_size, udata); 14172cc35ff9SJeff Roberson if (flags & M_ZERO) 14182cc35ff9SJeff Roberson bzero(item, zone->uz_size); 14198355f576SJeff Roberson return (item); 14208355f576SJeff Roberson } else if (cache->uc_freebucket) { 14218355f576SJeff Roberson /* 14228355f576SJeff Roberson * We have run out of items in our allocbucket. 14238355f576SJeff Roberson * See if we can switch with our free bucket. 14248355f576SJeff Roberson */ 1425cae33c14SJeff Roberson if (cache->uc_freebucket->ub_cnt > 0) { 14268355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 142764f051e9SJeff Roberson printf("uma_zalloc: Swapping empty with" 142864f051e9SJeff Roberson " alloc.\n"); 14298355f576SJeff Roberson #endif 1430b983089aSJeff Roberson bucket = cache->uc_freebucket; 14318355f576SJeff Roberson cache->uc_freebucket = cache->uc_allocbucket; 1432b983089aSJeff Roberson cache->uc_allocbucket = bucket; 14338355f576SJeff Roberson 14348355f576SJeff Roberson goto zalloc_start; 14358355f576SJeff Roberson } 14368355f576SJeff Roberson } 14378355f576SJeff Roberson } 1438a553d4b8SJeff Roberson ZONE_LOCK(zone); 1439a553d4b8SJeff Roberson /* Since we have locked the zone we may as well send back our stats */ 1440a553d4b8SJeff Roberson zone->uz_allocs += cache->uc_allocs; 1441a553d4b8SJeff Roberson cache->uc_allocs = 0; 14428355f576SJeff Roberson 1443a553d4b8SJeff Roberson /* Our old one is now a free bucket */ 1444a553d4b8SJeff Roberson if (cache->uc_allocbucket) { 1445cae33c14SJeff Roberson KASSERT(cache->uc_allocbucket->ub_cnt == 0, 1446a553d4b8SJeff Roberson ("uma_zalloc_arg: Freeing a non free bucket.")); 1447a553d4b8SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_bucket, 1448a553d4b8SJeff Roberson cache->uc_allocbucket, ub_link); 1449a553d4b8SJeff Roberson cache->uc_allocbucket = NULL; 1450a553d4b8SJeff Roberson } 14518355f576SJeff Roberson 1452a553d4b8SJeff Roberson /* Check the free list for a new alloc bucket */ 1453a553d4b8SJeff Roberson if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 1454cae33c14SJeff Roberson KASSERT(bucket->ub_cnt != 0, 1455a553d4b8SJeff Roberson ("uma_zalloc_arg: Returning an empty bucket.")); 14568355f576SJeff Roberson 1457a553d4b8SJeff Roberson LIST_REMOVE(bucket, ub_link); 1458a553d4b8SJeff Roberson cache->uc_allocbucket = bucket; 1459a553d4b8SJeff Roberson ZONE_UNLOCK(zone); 14608355f576SJeff Roberson goto zalloc_start; 1461a553d4b8SJeff Roberson } 1462bbee39c6SJeff Roberson /* We are no longer associated with this cpu!!! */ 1463d88797c2SBosko Milekic CPU_UNLOCK(cpu); 1464bbee39c6SJeff Roberson 1465a553d4b8SJeff Roberson /* Bump up our uz_count so we get here less */ 1466cae33c14SJeff Roberson if (zone->uz_count < BUCKET_MAX) 1467a553d4b8SJeff Roberson zone->uz_count++; 14688355f576SJeff Roberson /* 1469a553d4b8SJeff Roberson * Now lets just fill a bucket and put it on the free list. If that 1470a553d4b8SJeff Roberson * works we'll restart the allocation from the begining. 1471bbee39c6SJeff Roberson */ 1472bbee39c6SJeff Roberson if (uma_zalloc_bucket(zone, flags)) { 1473bbee39c6SJeff Roberson ZONE_UNLOCK(zone); 1474bbee39c6SJeff Roberson goto zalloc_restart; 1475bbee39c6SJeff Roberson } 1476bbee39c6SJeff Roberson ZONE_UNLOCK(zone); 1477bbee39c6SJeff Roberson /* 1478bbee39c6SJeff Roberson * We may not be able to get a bucket so return an actual item. 1479bbee39c6SJeff Roberson */ 1480bbee39c6SJeff Roberson #ifdef UMA_DEBUG 1481bbee39c6SJeff Roberson printf("uma_zalloc_arg: Bucketzone returned NULL\n"); 1482bbee39c6SJeff Roberson #endif 1483bbee39c6SJeff Roberson 1484bbee39c6SJeff Roberson return (uma_zalloc_internal(zone, udata, flags)); 1485bbee39c6SJeff Roberson } 1486bbee39c6SJeff Roberson 1487bbee39c6SJeff Roberson static uma_slab_t 1488bbee39c6SJeff Roberson uma_zone_slab(uma_zone_t zone, int flags) 1489bbee39c6SJeff Roberson { 1490bbee39c6SJeff Roberson uma_slab_t slab; 1491bbee39c6SJeff Roberson 1492bbee39c6SJeff Roberson /* 1493bbee39c6SJeff Roberson * This is to prevent us from recursively trying to allocate 1494bbee39c6SJeff Roberson * buckets. The problem is that if an allocation forces us to 1495bbee39c6SJeff Roberson * grab a new bucket we will call page_alloc, which will go off 1496bbee39c6SJeff Roberson * and cause the vm to allocate vm_map_entries. If we need new 1497bbee39c6SJeff Roberson * buckets there too we will recurse in kmem_alloc and bad 1498bbee39c6SJeff Roberson * things happen. So instead we return a NULL bucket, and make 1499bbee39c6SJeff Roberson * the code that allocates buckets smart enough to deal with it 1500bbee39c6SJeff Roberson */ 1501cae33c14SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_INTERNAL && zone->uz_recurse != 0) 1502bbee39c6SJeff Roberson return (NULL); 1503bbee39c6SJeff Roberson 1504bbee39c6SJeff Roberson slab = NULL; 1505bbee39c6SJeff Roberson 1506bbee39c6SJeff Roberson for (;;) { 1507bbee39c6SJeff Roberson /* 1508bbee39c6SJeff Roberson * Find a slab with some space. Prefer slabs that are partially 1509bbee39c6SJeff Roberson * used over those that are totally full. This helps to reduce 1510bbee39c6SJeff Roberson * fragmentation. 1511bbee39c6SJeff Roberson */ 1512bbee39c6SJeff Roberson if (zone->uz_free != 0) { 1513bbee39c6SJeff Roberson if (!LIST_EMPTY(&zone->uz_part_slab)) { 1514bbee39c6SJeff Roberson slab = LIST_FIRST(&zone->uz_part_slab); 1515bbee39c6SJeff Roberson } else { 1516bbee39c6SJeff Roberson slab = LIST_FIRST(&zone->uz_free_slab); 1517bbee39c6SJeff Roberson LIST_REMOVE(slab, us_link); 1518bbee39c6SJeff Roberson LIST_INSERT_HEAD(&zone->uz_part_slab, slab, 1519bbee39c6SJeff Roberson us_link); 1520bbee39c6SJeff Roberson } 1521bbee39c6SJeff Roberson return (slab); 1522bbee39c6SJeff Roberson } 1523bbee39c6SJeff Roberson 1524bbee39c6SJeff Roberson /* 1525bbee39c6SJeff Roberson * M_NOVM means don't ask at all! 1526bbee39c6SJeff Roberson */ 1527bbee39c6SJeff Roberson if (flags & M_NOVM) 1528bbee39c6SJeff Roberson break; 1529bbee39c6SJeff Roberson 1530bbee39c6SJeff Roberson if (zone->uz_maxpages && 1531bbee39c6SJeff Roberson zone->uz_pages >= zone->uz_maxpages) { 1532bbee39c6SJeff Roberson zone->uz_flags |= UMA_ZFLAG_FULL; 1533bbee39c6SJeff Roberson 1534ebc85edfSJeff Roberson if (flags & M_NOWAIT) 1535bbee39c6SJeff Roberson break; 1536ebc85edfSJeff Roberson else 153764f051e9SJeff Roberson msleep(zone, &zone->uz_lock, PVM, 153864f051e9SJeff Roberson "zonelimit", 0); 1539bbee39c6SJeff Roberson continue; 1540bbee39c6SJeff Roberson } 1541bbee39c6SJeff Roberson zone->uz_recurse++; 1542bbee39c6SJeff Roberson slab = slab_zalloc(zone, flags); 1543bbee39c6SJeff Roberson zone->uz_recurse--; 1544bbee39c6SJeff Roberson /* 1545bbee39c6SJeff Roberson * If we got a slab here it's safe to mark it partially used 1546bbee39c6SJeff Roberson * and return. We assume that the caller is going to remove 1547bbee39c6SJeff Roberson * at least one item. 1548bbee39c6SJeff Roberson */ 1549bbee39c6SJeff Roberson if (slab) { 1550bbee39c6SJeff Roberson LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); 1551bbee39c6SJeff Roberson return (slab); 1552bbee39c6SJeff Roberson } 1553bbee39c6SJeff Roberson /* 1554bbee39c6SJeff Roberson * We might not have been able to get a slab but another cpu 1555bbee39c6SJeff Roberson * could have while we were unlocked. Check again before we 1556bbee39c6SJeff Roberson * fail. 1557bbee39c6SJeff Roberson */ 1558ebc85edfSJeff Roberson if (flags & M_NOWAIT) 1559bbee39c6SJeff Roberson flags |= M_NOVM; 1560bbee39c6SJeff Roberson } 1561bbee39c6SJeff Roberson return (slab); 1562bbee39c6SJeff Roberson } 1563bbee39c6SJeff Roberson 1564d56368d7SBosko Milekic static void * 1565bbee39c6SJeff Roberson uma_slab_alloc(uma_zone_t zone, uma_slab_t slab) 1566bbee39c6SJeff Roberson { 1567bbee39c6SJeff Roberson void *item; 1568bbee39c6SJeff Roberson u_int8_t freei; 1569bbee39c6SJeff Roberson 1570bbee39c6SJeff Roberson freei = slab->us_firstfree; 1571bbee39c6SJeff Roberson slab->us_firstfree = slab->us_freelist[freei]; 1572bbee39c6SJeff Roberson item = slab->us_data + (zone->uz_rsize * freei); 1573bbee39c6SJeff Roberson 1574bbee39c6SJeff Roberson slab->us_freecount--; 1575bbee39c6SJeff Roberson zone->uz_free--; 1576bbee39c6SJeff Roberson #ifdef INVARIANTS 1577bbee39c6SJeff Roberson uma_dbg_alloc(zone, slab, item); 1578bbee39c6SJeff Roberson #endif 1579bbee39c6SJeff Roberson /* Move this slab to the full list */ 1580bbee39c6SJeff Roberson if (slab->us_freecount == 0) { 1581bbee39c6SJeff Roberson LIST_REMOVE(slab, us_link); 1582bbee39c6SJeff Roberson LIST_INSERT_HEAD(&zone->uz_full_slab, slab, us_link); 1583bbee39c6SJeff Roberson } 1584bbee39c6SJeff Roberson 1585bbee39c6SJeff Roberson return (item); 1586bbee39c6SJeff Roberson } 1587bbee39c6SJeff Roberson 1588bbee39c6SJeff Roberson static int 1589bbee39c6SJeff Roberson uma_zalloc_bucket(uma_zone_t zone, int flags) 1590bbee39c6SJeff Roberson { 1591bbee39c6SJeff Roberson uma_bucket_t bucket; 1592bbee39c6SJeff Roberson uma_slab_t slab; 159344eca34aSJeff Roberson int max; 1594bbee39c6SJeff Roberson 1595bbee39c6SJeff Roberson /* 1596a553d4b8SJeff Roberson * Try this zone's free list first so we don't allocate extra buckets. 15978355f576SJeff Roberson */ 1598bbee39c6SJeff Roberson if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 1599cae33c14SJeff Roberson KASSERT(bucket->ub_cnt == 0, 1600bbee39c6SJeff Roberson ("uma_zalloc_bucket: Bucket on free list is not empty.")); 1601a553d4b8SJeff Roberson LIST_REMOVE(bucket, ub_link); 1602bbee39c6SJeff Roberson } else { 160318aa2de5SJeff Roberson int bflags; 160418aa2de5SJeff Roberson 1605cae33c14SJeff Roberson bflags = (flags & ~M_ZERO); 160620e8e865SBosko Milekic if (zone->uz_flags & UMA_ZFLAG_CACHEONLY) 160718aa2de5SJeff Roberson bflags |= M_NOVM; 160818aa2de5SJeff Roberson 1609bbee39c6SJeff Roberson ZONE_UNLOCK(zone); 1610cae33c14SJeff Roberson bucket = bucket_alloc(zone->uz_count, bflags); 1611bbee39c6SJeff Roberson ZONE_LOCK(zone); 1612bbee39c6SJeff Roberson } 1613bbee39c6SJeff Roberson 1614bbee39c6SJeff Roberson if (bucket == NULL) 1615bbee39c6SJeff Roberson return (0); 1616bbee39c6SJeff Roberson 1617bbee39c6SJeff Roberson #ifdef SMP 1618a553d4b8SJeff Roberson /* 1619bbee39c6SJeff Roberson * This code is here to limit the number of simultaneous bucket fills 1620bbee39c6SJeff Roberson * for any given zone to the number of per cpu caches in this zone. This 1621bbee39c6SJeff Roberson * is done so that we don't allocate more memory than we really need. 1622a553d4b8SJeff Roberson */ 1623bbee39c6SJeff Roberson if (zone->uz_fills >= mp_ncpus) 1624bbee39c6SJeff Roberson goto done; 1625a553d4b8SJeff Roberson 1626bbee39c6SJeff Roberson #endif 1627bbee39c6SJeff Roberson zone->uz_fills++; 1628bbee39c6SJeff Roberson 162944eca34aSJeff Roberson max = MIN(bucket->ub_entries, zone->uz_count); 1630bbee39c6SJeff Roberson /* Try to keep the buckets totally full */ 163144eca34aSJeff Roberson while (bucket->ub_cnt < max && 1632d11e0ba5SJeff Roberson (slab = uma_zone_slab(zone, flags)) != NULL) { 163344eca34aSJeff Roberson while (slab->us_freecount && bucket->ub_cnt < max) { 1634cae33c14SJeff Roberson bucket->ub_bucket[bucket->ub_cnt++] = 1635bbee39c6SJeff Roberson uma_slab_alloc(zone, slab); 1636bbee39c6SJeff Roberson } 1637bbee39c6SJeff Roberson /* Don't block on the next fill */ 1638bbee39c6SJeff Roberson flags |= M_NOWAIT; 16398355f576SJeff Roberson } 16408355f576SJeff Roberson 1641bbee39c6SJeff Roberson zone->uz_fills--; 1642bbee39c6SJeff Roberson 1643cae33c14SJeff Roberson if (bucket->ub_cnt != 0) { 1644bbee39c6SJeff Roberson LIST_INSERT_HEAD(&zone->uz_full_bucket, 1645bbee39c6SJeff Roberson bucket, ub_link); 1646bbee39c6SJeff Roberson return (1); 1647bbee39c6SJeff Roberson } 1648bbee39c6SJeff Roberson #ifdef SMP 1649bbee39c6SJeff Roberson done: 1650bbee39c6SJeff Roberson #endif 1651cae33c14SJeff Roberson bucket_free(bucket); 1652bbee39c6SJeff Roberson 1653bbee39c6SJeff Roberson return (0); 1654bbee39c6SJeff Roberson } 16558355f576SJeff Roberson /* 1656bbee39c6SJeff Roberson * Allocates an item for an internal zone 16578355f576SJeff Roberson * 16588355f576SJeff Roberson * Arguments 16598355f576SJeff Roberson * zone The zone to alloc for. 16608355f576SJeff Roberson * udata The data to be passed to the constructor. 1661a163d034SWarner Losh * flags M_WAITOK, M_NOWAIT, M_ZERO. 16628355f576SJeff Roberson * 16638355f576SJeff Roberson * Returns 16648355f576SJeff Roberson * NULL if there is no memory and M_NOWAIT is set 1665bbee39c6SJeff Roberson * An item if successful 16668355f576SJeff Roberson */ 16678355f576SJeff Roberson 16688355f576SJeff Roberson static void * 1669bbee39c6SJeff Roberson uma_zalloc_internal(uma_zone_t zone, void *udata, int flags) 16708355f576SJeff Roberson { 16718355f576SJeff Roberson uma_slab_t slab; 16728355f576SJeff Roberson void *item; 16738355f576SJeff Roberson 16748355f576SJeff Roberson item = NULL; 16758355f576SJeff Roberson 16768355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 16778355f576SJeff Roberson printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); 16788355f576SJeff Roberson #endif 16798355f576SJeff Roberson ZONE_LOCK(zone); 16808355f576SJeff Roberson 1681bbee39c6SJeff Roberson slab = uma_zone_slab(zone, flags); 1682bbee39c6SJeff Roberson if (slab == NULL) { 1683bce97791SJeff Roberson ZONE_UNLOCK(zone); 1684a553d4b8SJeff Roberson return (NULL); 1685bce97791SJeff Roberson } 1686a553d4b8SJeff Roberson 1687bbee39c6SJeff Roberson item = uma_slab_alloc(zone, slab); 16888355f576SJeff Roberson 16898355f576SJeff Roberson ZONE_UNLOCK(zone); 16908355f576SJeff Roberson 16913370c5bfSJeff Roberson if (zone->uz_ctor != NULL) 16928355f576SJeff Roberson zone->uz_ctor(item, zone->uz_size, udata); 16932cc35ff9SJeff Roberson if (flags & M_ZERO) 16942cc35ff9SJeff Roberson bzero(item, zone->uz_size); 16958355f576SJeff Roberson 16968355f576SJeff Roberson return (item); 16978355f576SJeff Roberson } 16988355f576SJeff Roberson 16998355f576SJeff Roberson /* See uma.h */ 17008355f576SJeff Roberson void 17018355f576SJeff Roberson uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 17028355f576SJeff Roberson { 17038355f576SJeff Roberson uma_cache_t cache; 17048355f576SJeff Roberson uma_bucket_t bucket; 17054741dcbfSJeff Roberson int bflags; 17068355f576SJeff Roberson int cpu; 17075c133dfaSBosko Milekic int skip; 17088355f576SJeff Roberson 17098355f576SJeff Roberson /* This is the fast path free */ 17105c133dfaSBosko Milekic skip = 0; 17118355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC_1 17128355f576SJeff Roberson printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone); 17138355f576SJeff Roberson #endif 1714af7f9b97SJeff Roberson /* 1715af7f9b97SJeff Roberson * The race here is acceptable. If we miss it we'll just have to wait 1716af7f9b97SJeff Roberson * a little longer for the limits to be reset. 1717af7f9b97SJeff Roberson */ 1718af7f9b97SJeff Roberson 1719af7f9b97SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_FULL) 1720af7f9b97SJeff Roberson goto zfree_internal; 1721af7f9b97SJeff Roberson 17225c133dfaSBosko Milekic if (zone->uz_dtor) { 1723bba739abSJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 17245c133dfaSBosko Milekic skip = 1; 17255c133dfaSBosko Milekic } 1726bba739abSJeff Roberson 1727a553d4b8SJeff Roberson zfree_restart: 17288355f576SJeff Roberson cpu = PCPU_GET(cpuid); 1729d88797c2SBosko Milekic CPU_LOCK(cpu); 17308355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 17318355f576SJeff Roberson 17328355f576SJeff Roberson zfree_start: 17338355f576SJeff Roberson bucket = cache->uc_freebucket; 17348355f576SJeff Roberson 17358355f576SJeff Roberson if (bucket) { 1736a553d4b8SJeff Roberson /* 1737a553d4b8SJeff Roberson * Do we have room in our bucket? It is OK for this uz count 1738a553d4b8SJeff Roberson * check to be slightly out of sync. 1739a553d4b8SJeff Roberson */ 1740a553d4b8SJeff Roberson 1741cae33c14SJeff Roberson if (bucket->ub_cnt < bucket->ub_entries) { 1742cae33c14SJeff Roberson KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL, 17438355f576SJeff Roberson ("uma_zfree: Freeing to non free bucket index.")); 1744cae33c14SJeff Roberson bucket->ub_bucket[bucket->ub_cnt] = item; 1745cae33c14SJeff Roberson bucket->ub_cnt++; 1746b9ba8931SJeff Roberson #ifdef INVARIANTS 174781f71edaSMatt Jacob ZONE_LOCK(zone); 1748b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_MALLOC) 1749b9ba8931SJeff Roberson uma_dbg_free(zone, udata, item); 1750b9ba8931SJeff Roberson else 1751b9ba8931SJeff Roberson uma_dbg_free(zone, NULL, item); 175281f71edaSMatt Jacob ZONE_UNLOCK(zone); 1753b9ba8931SJeff Roberson #endif 1754d88797c2SBosko Milekic CPU_UNLOCK(cpu); 17558355f576SJeff Roberson return; 17568355f576SJeff Roberson } else if (cache->uc_allocbucket) { 17578355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 17588355f576SJeff Roberson printf("uma_zfree: Swapping buckets.\n"); 17598355f576SJeff Roberson #endif 17608355f576SJeff Roberson /* 17618355f576SJeff Roberson * We have run out of space in our freebucket. 17628355f576SJeff Roberson * See if we can switch with our alloc bucket. 17638355f576SJeff Roberson */ 1764cae33c14SJeff Roberson if (cache->uc_allocbucket->ub_cnt < 1765cae33c14SJeff Roberson cache->uc_freebucket->ub_cnt) { 1766b983089aSJeff Roberson bucket = cache->uc_freebucket; 17678355f576SJeff Roberson cache->uc_freebucket = cache->uc_allocbucket; 1768b983089aSJeff Roberson cache->uc_allocbucket = bucket; 17698355f576SJeff Roberson goto zfree_start; 17708355f576SJeff Roberson } 17718355f576SJeff Roberson } 17728355f576SJeff Roberson } 17738355f576SJeff Roberson /* 1774a553d4b8SJeff Roberson * We can get here for two reasons: 17758355f576SJeff Roberson * 17768355f576SJeff Roberson * 1) The buckets are NULL 1777a553d4b8SJeff Roberson * 2) The alloc and free buckets are both somewhat full. 17788355f576SJeff Roberson */ 17798355f576SJeff Roberson 17808355f576SJeff Roberson ZONE_LOCK(zone); 17818355f576SJeff Roberson 17828355f576SJeff Roberson bucket = cache->uc_freebucket; 17838355f576SJeff Roberson cache->uc_freebucket = NULL; 17848355f576SJeff Roberson 17858355f576SJeff Roberson /* Can we throw this on the zone full list? */ 17868355f576SJeff Roberson if (bucket != NULL) { 17878355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 17888355f576SJeff Roberson printf("uma_zfree: Putting old bucket on the free list.\n"); 17898355f576SJeff Roberson #endif 1790cae33c14SJeff Roberson /* ub_cnt is pointing to the last free item */ 1791cae33c14SJeff Roberson KASSERT(bucket->ub_cnt != 0, 17928355f576SJeff Roberson ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n")); 17938355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_full_bucket, 17948355f576SJeff Roberson bucket, ub_link); 17958355f576SJeff Roberson } 1796a553d4b8SJeff Roberson if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 1797a553d4b8SJeff Roberson LIST_REMOVE(bucket, ub_link); 1798a553d4b8SJeff Roberson ZONE_UNLOCK(zone); 1799a553d4b8SJeff Roberson cache->uc_freebucket = bucket; 1800a553d4b8SJeff Roberson goto zfree_start; 1801a553d4b8SJeff Roberson } 1802a553d4b8SJeff Roberson /* We're done with this CPU now */ 1803d88797c2SBosko Milekic CPU_UNLOCK(cpu); 1804a553d4b8SJeff Roberson 1805a553d4b8SJeff Roberson /* And the zone.. */ 1806a553d4b8SJeff Roberson ZONE_UNLOCK(zone); 1807a553d4b8SJeff Roberson 18088355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 18098355f576SJeff Roberson printf("uma_zfree: Allocating new free bucket.\n"); 18108355f576SJeff Roberson #endif 18114741dcbfSJeff Roberson bflags = M_NOWAIT; 18124741dcbfSJeff Roberson 181320e8e865SBosko Milekic if (zone->uz_flags & UMA_ZFLAG_CACHEONLY) 18144741dcbfSJeff Roberson bflags |= M_NOVM; 1815cae33c14SJeff Roberson bucket = bucket_alloc(zone->uz_count, bflags); 18164741dcbfSJeff Roberson if (bucket) { 1817a553d4b8SJeff Roberson ZONE_LOCK(zone); 1818a553d4b8SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_bucket, 1819a553d4b8SJeff Roberson bucket, ub_link); 18208355f576SJeff Roberson ZONE_UNLOCK(zone); 1821a553d4b8SJeff Roberson goto zfree_restart; 18228355f576SJeff Roberson } 18238355f576SJeff Roberson 1824a553d4b8SJeff Roberson /* 1825a553d4b8SJeff Roberson * If nothing else caught this, we'll just do an internal free. 1826a553d4b8SJeff Roberson */ 18278355f576SJeff Roberson 1828af7f9b97SJeff Roberson zfree_internal: 1829af7f9b97SJeff Roberson 183048bf8725SBosko Milekic #ifdef INVARIANTS 183148bf8725SBosko Milekic /* 183264f051e9SJeff Roberson * If we need to skip the dtor and the uma_dbg_free in 183364f051e9SJeff Roberson * uma_zfree_internal because we've already called the dtor 183464f051e9SJeff Roberson * above, but we ended up here, then we need to make sure 183564f051e9SJeff Roberson * that we take care of the uma_dbg_free immediately. 183648bf8725SBosko Milekic */ 183748bf8725SBosko Milekic if (skip) { 183848bf8725SBosko Milekic ZONE_LOCK(zone); 1839b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_MALLOC) 184048bf8725SBosko Milekic uma_dbg_free(zone, udata, item); 184148bf8725SBosko Milekic else 184248bf8725SBosko Milekic uma_dbg_free(zone, NULL, item); 184348bf8725SBosko Milekic ZONE_UNLOCK(zone); 184448bf8725SBosko Milekic } 184548bf8725SBosko Milekic #endif 18465c133dfaSBosko Milekic uma_zfree_internal(zone, item, udata, skip); 18478355f576SJeff Roberson 18488355f576SJeff Roberson return; 18498355f576SJeff Roberson 18508355f576SJeff Roberson } 18518355f576SJeff Roberson 18528355f576SJeff Roberson /* 18538355f576SJeff Roberson * Frees an item to an INTERNAL zone or allocates a free bucket 18548355f576SJeff Roberson * 18558355f576SJeff Roberson * Arguments: 18568355f576SJeff Roberson * zone The zone to free to 18578355f576SJeff Roberson * item The item we're freeing 18588355f576SJeff Roberson * udata User supplied data for the dtor 18598355f576SJeff Roberson * skip Skip the dtor, it was done in uma_zfree_arg 18608355f576SJeff Roberson */ 18618355f576SJeff Roberson static void 18628355f576SJeff Roberson uma_zfree_internal(uma_zone_t zone, void *item, void *udata, int skip) 18638355f576SJeff Roberson { 18648355f576SJeff Roberson uma_slab_t slab; 18658355f576SJeff Roberson u_int8_t *mem; 18668355f576SJeff Roberson u_int8_t freei; 18678355f576SJeff Roberson 1868bba739abSJeff Roberson if (!skip && zone->uz_dtor) 1869bba739abSJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 1870bba739abSJeff Roberson 18718355f576SJeff Roberson ZONE_LOCK(zone); 18728355f576SJeff Roberson 1873b60f5b79SJeff Roberson if (!(zone->uz_flags & UMA_ZONE_MALLOC)) { 18748355f576SJeff Roberson mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); 1875b60f5b79SJeff Roberson if (zone->uz_flags & UMA_ZONE_HASH) 18768355f576SJeff Roberson slab = hash_sfind(&zone->uz_hash, mem); 18778355f576SJeff Roberson else { 18788355f576SJeff Roberson mem += zone->uz_pgoff; 18798355f576SJeff Roberson slab = (uma_slab_t)mem; 18808355f576SJeff Roberson } 18818355f576SJeff Roberson } else { 18828355f576SJeff Roberson slab = (uma_slab_t)udata; 18838355f576SJeff Roberson } 18848355f576SJeff Roberson 18858355f576SJeff Roberson /* Do we need to remove from any lists? */ 18868355f576SJeff Roberson if (slab->us_freecount+1 == zone->uz_ipers) { 18878355f576SJeff Roberson LIST_REMOVE(slab, us_link); 18888355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_slab, slab, us_link); 18898355f576SJeff Roberson } else if (slab->us_freecount == 0) { 18908355f576SJeff Roberson LIST_REMOVE(slab, us_link); 18918355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); 18928355f576SJeff Roberson } 18938355f576SJeff Roberson 18948355f576SJeff Roberson /* Slab management stuff */ 18958355f576SJeff Roberson freei = ((unsigned long)item - (unsigned long)slab->us_data) 18968355f576SJeff Roberson / zone->uz_rsize; 18978355f576SJeff Roberson 1898639c9550SJeff Roberson #ifdef INVARIANTS 1899639c9550SJeff Roberson if (!skip) 1900639c9550SJeff Roberson uma_dbg_free(zone, slab, item); 19018355f576SJeff Roberson #endif 1902639c9550SJeff Roberson 19038355f576SJeff Roberson slab->us_freelist[freei] = slab->us_firstfree; 19048355f576SJeff Roberson slab->us_firstfree = freei; 19058355f576SJeff Roberson slab->us_freecount++; 19068355f576SJeff Roberson 19078355f576SJeff Roberson /* Zone statistics */ 19088355f576SJeff Roberson zone->uz_free++; 19098355f576SJeff Roberson 1910af7f9b97SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_FULL) { 1911af7f9b97SJeff Roberson if (zone->uz_pages < zone->uz_maxpages) 1912af7f9b97SJeff Roberson zone->uz_flags &= ~UMA_ZFLAG_FULL; 1913af7f9b97SJeff Roberson 1914af7f9b97SJeff Roberson /* We can handle one more allocation */ 191574c924b5SJeff Roberson wakeup_one(zone); 1916af7f9b97SJeff Roberson } 1917af7f9b97SJeff Roberson 1918605cbd6aSJeff Roberson ZONE_UNLOCK(zone); 19198355f576SJeff Roberson } 19208355f576SJeff Roberson 19218355f576SJeff Roberson /* See uma.h */ 19228355f576SJeff Roberson void 1923736ee590SJeff Roberson uma_zone_set_max(uma_zone_t zone, int nitems) 1924736ee590SJeff Roberson { 1925736ee590SJeff Roberson ZONE_LOCK(zone); 1926736ee590SJeff Roberson if (zone->uz_ppera > 1) 1927af7f9b97SJeff Roberson zone->uz_maxpages = nitems * zone->uz_ppera; 1928736ee590SJeff Roberson else 1929736ee590SJeff Roberson zone->uz_maxpages = nitems / zone->uz_ipers; 193028bc4419SJeff Roberson 1931d4d6aee5SAndrew R. Reiter if (zone->uz_maxpages * zone->uz_ipers < nitems) 1932d4d6aee5SAndrew R. Reiter zone->uz_maxpages++; 193328bc4419SJeff Roberson 1934736ee590SJeff Roberson ZONE_UNLOCK(zone); 1935736ee590SJeff Roberson } 1936736ee590SJeff Roberson 1937736ee590SJeff Roberson /* See uma.h */ 1938736ee590SJeff Roberson void 19398355f576SJeff Roberson uma_zone_set_freef(uma_zone_t zone, uma_free freef) 19408355f576SJeff Roberson { 19418355f576SJeff Roberson ZONE_LOCK(zone); 19428355f576SJeff Roberson zone->uz_freef = freef; 19438355f576SJeff Roberson ZONE_UNLOCK(zone); 19448355f576SJeff Roberson } 19458355f576SJeff Roberson 19468355f576SJeff Roberson /* See uma.h */ 19478355f576SJeff Roberson void 19488355f576SJeff Roberson uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 19498355f576SJeff Roberson { 19508355f576SJeff Roberson ZONE_LOCK(zone); 19518355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_PRIVALLOC; 19528355f576SJeff Roberson zone->uz_allocf = allocf; 19538355f576SJeff Roberson ZONE_UNLOCK(zone); 19548355f576SJeff Roberson } 19558355f576SJeff Roberson 19568355f576SJeff Roberson /* See uma.h */ 19578355f576SJeff Roberson int 19588355f576SJeff Roberson uma_zone_set_obj(uma_zone_t zone, struct vm_object *obj, int count) 19598355f576SJeff Roberson { 19608355f576SJeff Roberson int pages; 19618355f576SJeff Roberson vm_offset_t kva; 19628355f576SJeff Roberson 19638355f576SJeff Roberson pages = count / zone->uz_ipers; 19648355f576SJeff Roberson 19658355f576SJeff Roberson if (pages * zone->uz_ipers < count) 19668355f576SJeff Roberson pages++; 1967a553d4b8SJeff Roberson 19688355f576SJeff Roberson kva = kmem_alloc_pageable(kernel_map, pages * UMA_SLAB_SIZE); 19698355f576SJeff Roberson 1970d1f42ac2SAlan Cox if (kva == 0) 19718355f576SJeff Roberson return (0); 197264f051e9SJeff Roberson if (obj == NULL) { 1973a553d4b8SJeff Roberson obj = vm_object_allocate(OBJT_DEFAULT, 1974c7173f58SJeff Roberson pages); 197564f051e9SJeff Roberson } else { 197682774d80SAlan Cox VM_OBJECT_LOCK_INIT(obj); 19778355f576SJeff Roberson _vm_object_allocate(OBJT_DEFAULT, 1978c7173f58SJeff Roberson pages, obj); 197982774d80SAlan Cox } 1980a553d4b8SJeff Roberson ZONE_LOCK(zone); 1981a553d4b8SJeff Roberson zone->uz_kva = kva; 1982a553d4b8SJeff Roberson zone->uz_obj = obj; 1983a553d4b8SJeff Roberson zone->uz_maxpages = pages; 19848355f576SJeff Roberson zone->uz_allocf = obj_alloc; 1985b60f5b79SJeff Roberson zone->uz_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC; 19868355f576SJeff Roberson ZONE_UNLOCK(zone); 19878355f576SJeff Roberson return (1); 19888355f576SJeff Roberson } 19898355f576SJeff Roberson 19908355f576SJeff Roberson /* See uma.h */ 19918355f576SJeff Roberson void 19928355f576SJeff Roberson uma_prealloc(uma_zone_t zone, int items) 19938355f576SJeff Roberson { 19948355f576SJeff Roberson int slabs; 19958355f576SJeff Roberson uma_slab_t slab; 19968355f576SJeff Roberson 19978355f576SJeff Roberson ZONE_LOCK(zone); 19988355f576SJeff Roberson slabs = items / zone->uz_ipers; 19998355f576SJeff Roberson if (slabs * zone->uz_ipers < items) 20008355f576SJeff Roberson slabs++; 20018355f576SJeff Roberson while (slabs > 0) { 2002a163d034SWarner Losh slab = slab_zalloc(zone, M_WAITOK); 20038355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_slab, slab, us_link); 20048355f576SJeff Roberson slabs--; 20058355f576SJeff Roberson } 20068355f576SJeff Roberson ZONE_UNLOCK(zone); 20078355f576SJeff Roberson } 20088355f576SJeff Roberson 20098355f576SJeff Roberson /* See uma.h */ 20108355f576SJeff Roberson void 20118355f576SJeff Roberson uma_reclaim(void) 20128355f576SJeff Roberson { 20138355f576SJeff Roberson #ifdef UMA_DEBUG 20148355f576SJeff Roberson printf("UMA: vm asked us to release pages!\n"); 20158355f576SJeff Roberson #endif 201686bbae32SJeff Roberson bucket_enable(); 20178355f576SJeff Roberson zone_foreach(zone_drain); 20188355f576SJeff Roberson /* 20198355f576SJeff Roberson * Some slabs may have been freed but this zone will be visited early 20208355f576SJeff Roberson * we visit again so that we can free pages that are empty once other 20218355f576SJeff Roberson * zones are drained. We have to do the same for buckets. 20228355f576SJeff Roberson */ 20239643769aSJeff Roberson zone_drain(slabzone); 2024cae33c14SJeff Roberson bucket_zone_drain(); 20258355f576SJeff Roberson } 20268355f576SJeff Roberson 20278355f576SJeff Roberson void * 20288355f576SJeff Roberson uma_large_malloc(int size, int wait) 20298355f576SJeff Roberson { 20308355f576SJeff Roberson void *mem; 20318355f576SJeff Roberson uma_slab_t slab; 20328355f576SJeff Roberson u_int8_t flags; 20338355f576SJeff Roberson 2034bbee39c6SJeff Roberson slab = uma_zalloc_internal(slabzone, NULL, wait); 20358355f576SJeff Roberson if (slab == NULL) 20368355f576SJeff Roberson return (NULL); 20378355f576SJeff Roberson mem = page_alloc(NULL, size, &flags, wait); 20388355f576SJeff Roberson if (mem) { 203999571dc3SJeff Roberson vsetslab((vm_offset_t)mem, slab); 20408355f576SJeff Roberson slab->us_data = mem; 20418355f576SJeff Roberson slab->us_flags = flags | UMA_SLAB_MALLOC; 20428355f576SJeff Roberson slab->us_size = size; 20438355f576SJeff Roberson } else { 20448355f576SJeff Roberson uma_zfree_internal(slabzone, slab, NULL, 0); 20458355f576SJeff Roberson } 20468355f576SJeff Roberson 20478355f576SJeff Roberson 20488355f576SJeff Roberson return (mem); 20498355f576SJeff Roberson } 20508355f576SJeff Roberson 20518355f576SJeff Roberson void 20528355f576SJeff Roberson uma_large_free(uma_slab_t slab) 20538355f576SJeff Roberson { 205499571dc3SJeff Roberson vsetobj((vm_offset_t)slab->us_data, kmem_object); 2055125ee0d1STor Egge /* 20565103186cSAlan Cox * XXX: We get a lock order reversal if we don't have Giant: 2057125ee0d1STor Egge * vm_map_remove (locks system map) -> vm_map_delete -> 2058125ee0d1STor Egge * vm_map_entry_unwire -> vm_fault_unwire -> mtx_lock(&Giant) 2059125ee0d1STor Egge */ 2060125ee0d1STor Egge if (!mtx_owned(&Giant)) { 2061125ee0d1STor Egge mtx_lock(&Giant); 2062125ee0d1STor Egge page_free(slab->us_data, slab->us_size, slab->us_flags); 2063125ee0d1STor Egge mtx_unlock(&Giant); 2064125ee0d1STor Egge } else 20658355f576SJeff Roberson page_free(slab->us_data, slab->us_size, slab->us_flags); 20668355f576SJeff Roberson uma_zfree_internal(slabzone, slab, NULL, 0); 20678355f576SJeff Roberson } 20688355f576SJeff Roberson 20698355f576SJeff Roberson void 20708355f576SJeff Roberson uma_print_stats(void) 20718355f576SJeff Roberson { 20728355f576SJeff Roberson zone_foreach(uma_print_zone); 20738355f576SJeff Roberson } 20748355f576SJeff Roberson 2075504d5de3SJeff Roberson static void 2076504d5de3SJeff Roberson slab_print(uma_slab_t slab) 2077504d5de3SJeff Roberson { 2078504d5de3SJeff Roberson printf("slab: zone %p, data %p, freecount %d, firstfree %d\n", 2079504d5de3SJeff Roberson slab->us_zone, slab->us_data, slab->us_freecount, 2080504d5de3SJeff Roberson slab->us_firstfree); 2081504d5de3SJeff Roberson } 2082504d5de3SJeff Roberson 2083504d5de3SJeff Roberson static void 2084504d5de3SJeff Roberson cache_print(uma_cache_t cache) 2085504d5de3SJeff Roberson { 2086504d5de3SJeff Roberson printf("alloc: %p(%d), free: %p(%d)\n", 2087504d5de3SJeff Roberson cache->uc_allocbucket, 2088504d5de3SJeff Roberson cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0, 2089504d5de3SJeff Roberson cache->uc_freebucket, 2090504d5de3SJeff Roberson cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0); 2091504d5de3SJeff Roberson } 2092504d5de3SJeff Roberson 20938355f576SJeff Roberson void 20948355f576SJeff Roberson uma_print_zone(uma_zone_t zone) 20958355f576SJeff Roberson { 2096504d5de3SJeff Roberson uma_cache_t cache; 2097504d5de3SJeff Roberson uma_slab_t slab; 2098504d5de3SJeff Roberson int i; 2099504d5de3SJeff Roberson 21008355f576SJeff Roberson printf("%s(%p) size %d(%d) flags %d ipers %d ppera %d out %d free %d\n", 21018355f576SJeff Roberson zone->uz_name, zone, zone->uz_size, zone->uz_rsize, zone->uz_flags, 21028355f576SJeff Roberson zone->uz_ipers, zone->uz_ppera, 21038355f576SJeff Roberson (zone->uz_ipers * zone->uz_pages) - zone->uz_free, zone->uz_free); 2104504d5de3SJeff Roberson printf("Part slabs:\n"); 2105504d5de3SJeff Roberson LIST_FOREACH(slab, &zone->uz_part_slab, us_link) 2106504d5de3SJeff Roberson slab_print(slab); 2107504d5de3SJeff Roberson printf("Free slabs:\n"); 2108504d5de3SJeff Roberson LIST_FOREACH(slab, &zone->uz_free_slab, us_link) 2109504d5de3SJeff Roberson slab_print(slab); 2110504d5de3SJeff Roberson printf("Full slabs:\n"); 2111504d5de3SJeff Roberson LIST_FOREACH(slab, &zone->uz_full_slab, us_link) 2112504d5de3SJeff Roberson slab_print(slab); 2113b6c71225SJohn Baldwin for (i = 0; i <= mp_maxid; i++) { 2114504d5de3SJeff Roberson if (CPU_ABSENT(i)) 2115504d5de3SJeff Roberson continue; 2116504d5de3SJeff Roberson cache = &zone->uz_cpu[i]; 2117504d5de3SJeff Roberson printf("CPU %d Cache:\n", i); 2118504d5de3SJeff Roberson cache_print(cache); 2119504d5de3SJeff Roberson } 21208355f576SJeff Roberson } 21218355f576SJeff Roberson 21228355f576SJeff Roberson /* 21238355f576SJeff Roberson * Sysctl handler for vm.zone 21248355f576SJeff Roberson * 21258355f576SJeff Roberson * stolen from vm_zone.c 21268355f576SJeff Roberson */ 21278355f576SJeff Roberson static int 21288355f576SJeff Roberson sysctl_vm_zone(SYSCTL_HANDLER_ARGS) 21298355f576SJeff Roberson { 21308355f576SJeff Roberson int error, len, cnt; 21318355f576SJeff Roberson const int linesize = 128; /* conservative */ 21328355f576SJeff Roberson int totalfree; 21338355f576SJeff Roberson char *tmpbuf, *offset; 21348355f576SJeff Roberson uma_zone_t z; 21358355f576SJeff Roberson char *p; 2136f828e5beSJeff Roberson int cpu; 2137f828e5beSJeff Roberson int cachefree; 2138f828e5beSJeff Roberson uma_bucket_t bucket; 2139f828e5beSJeff Roberson uma_cache_t cache; 21408355f576SJeff Roberson 21418355f576SJeff Roberson cnt = 0; 21420da47b2fSJeff Roberson mtx_lock(&uma_mtx); 21438355f576SJeff Roberson LIST_FOREACH(z, &uma_zones, uz_link) 21448355f576SJeff Roberson cnt++; 21450da47b2fSJeff Roberson mtx_unlock(&uma_mtx); 21468355f576SJeff Roberson MALLOC(tmpbuf, char *, (cnt == 0 ? 1 : cnt) * linesize, 2147a163d034SWarner Losh M_TEMP, M_WAITOK); 21488355f576SJeff Roberson len = snprintf(tmpbuf, linesize, 21498355f576SJeff Roberson "\nITEM SIZE LIMIT USED FREE REQUESTS\n\n"); 21508355f576SJeff Roberson if (cnt == 0) 21518355f576SJeff Roberson tmpbuf[len - 1] = '\0'; 21528355f576SJeff Roberson error = SYSCTL_OUT(req, tmpbuf, cnt == 0 ? len-1 : len); 21538355f576SJeff Roberson if (error || cnt == 0) 21548355f576SJeff Roberson goto out; 21558355f576SJeff Roberson offset = tmpbuf; 2156f4af24d5SJeff Roberson mtx_lock(&uma_mtx); 21578355f576SJeff Roberson LIST_FOREACH(z, &uma_zones, uz_link) { 21588355f576SJeff Roberson if (cnt == 0) /* list may have changed size */ 21598355f576SJeff Roberson break; 21601c35e213SBosko Milekic if (!(z->uz_flags & UMA_ZFLAG_INTERNAL)) { 2161b6c71225SJohn Baldwin for (cpu = 0; cpu <= mp_maxid; cpu++) { 2162f828e5beSJeff Roberson if (CPU_ABSENT(cpu)) 2163f828e5beSJeff Roberson continue; 2164f828e5beSJeff Roberson CPU_LOCK(cpu); 2165f828e5beSJeff Roberson } 21661c35e213SBosko Milekic } 21678355f576SJeff Roberson ZONE_LOCK(z); 2168f828e5beSJeff Roberson cachefree = 0; 21691c35e213SBosko Milekic if (!(z->uz_flags & UMA_ZFLAG_INTERNAL)) { 2170b6c71225SJohn Baldwin for (cpu = 0; cpu <= mp_maxid; cpu++) { 2171f828e5beSJeff Roberson if (CPU_ABSENT(cpu)) 2172f828e5beSJeff Roberson continue; 2173f828e5beSJeff Roberson cache = &z->uz_cpu[cpu]; 2174f828e5beSJeff Roberson if (cache->uc_allocbucket != NULL) 2175cae33c14SJeff Roberson cachefree += cache->uc_allocbucket->ub_cnt; 2176f828e5beSJeff Roberson if (cache->uc_freebucket != NULL) 2177cae33c14SJeff Roberson cachefree += cache->uc_freebucket->ub_cnt; 2178f828e5beSJeff Roberson CPU_UNLOCK(cpu); 2179f828e5beSJeff Roberson } 21801c35e213SBosko Milekic } 2181f828e5beSJeff Roberson LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) { 2182cae33c14SJeff Roberson cachefree += bucket->ub_cnt; 2183f828e5beSJeff Roberson } 2184f828e5beSJeff Roberson totalfree = z->uz_free + cachefree; 21858355f576SJeff Roberson len = snprintf(offset, linesize, 21868355f576SJeff Roberson "%-12.12s %6.6u, %8.8u, %6.6u, %6.6u, %8.8llu\n", 21878355f576SJeff Roberson z->uz_name, z->uz_size, 21888355f576SJeff Roberson z->uz_maxpages * z->uz_ipers, 21898355f576SJeff Roberson (z->uz_ipers * (z->uz_pages / z->uz_ppera)) - totalfree, 21908355f576SJeff Roberson totalfree, 21918355f576SJeff Roberson (unsigned long long)z->uz_allocs); 21928355f576SJeff Roberson ZONE_UNLOCK(z); 21938355f576SJeff Roberson for (p = offset + 12; p > offset && *p == ' '; --p) 21948355f576SJeff Roberson /* nothing */ ; 21958355f576SJeff Roberson p[1] = ':'; 21968355f576SJeff Roberson cnt--; 21978355f576SJeff Roberson offset += len; 21988355f576SJeff Roberson } 2199f4af24d5SJeff Roberson mtx_unlock(&uma_mtx); 22008355f576SJeff Roberson *offset++ = '\0'; 22018355f576SJeff Roberson error = SYSCTL_OUT(req, tmpbuf, offset - tmpbuf); 22028355f576SJeff Roberson out: 22038355f576SJeff Roberson FREE(tmpbuf, M_TEMP); 22048355f576SJeff Roberson return (error); 22058355f576SJeff Roberson } 2206