18355f576SJeff Roberson /* 2f461cf22SJeff Roberson * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 38355f576SJeff Roberson * All rights reserved. 48355f576SJeff Roberson * 58355f576SJeff Roberson * Redistribution and use in source and binary forms, with or without 68355f576SJeff Roberson * modification, are permitted provided that the following conditions 78355f576SJeff Roberson * are met: 88355f576SJeff Roberson * 1. Redistributions of source code must retain the above copyright 98355f576SJeff Roberson * notice unmodified, this list of conditions, and the following 108355f576SJeff Roberson * disclaimer. 118355f576SJeff Roberson * 2. Redistributions in binary form must reproduce the above copyright 128355f576SJeff Roberson * notice, this list of conditions and the following disclaimer in the 138355f576SJeff Roberson * documentation and/or other materials provided with the distribution. 148355f576SJeff Roberson * 158355f576SJeff Roberson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 168355f576SJeff Roberson * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 178355f576SJeff Roberson * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 188355f576SJeff Roberson * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 198355f576SJeff Roberson * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 208355f576SJeff Roberson * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 218355f576SJeff Roberson * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 228355f576SJeff Roberson * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 238355f576SJeff Roberson * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 248355f576SJeff Roberson * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 258355f576SJeff Roberson * 268355f576SJeff Roberson * $FreeBSD$ 278355f576SJeff Roberson * 288355f576SJeff Roberson */ 298355f576SJeff Roberson 308355f576SJeff Roberson /* 318355f576SJeff Roberson * uma_core.c Implementation of the Universal Memory allocator 328355f576SJeff Roberson * 338355f576SJeff Roberson * This allocator is intended to replace the multitude of similar object caches 348355f576SJeff Roberson * in the standard FreeBSD kernel. The intent is to be flexible as well as 358355f576SJeff Roberson * effecient. A primary design goal is to return unused memory to the rest of 368355f576SJeff Roberson * the system. This will make the system as a whole more flexible due to the 378355f576SJeff Roberson * ability to move memory to subsystems which most need it instead of leaving 388355f576SJeff Roberson * pools of reserved memory unused. 398355f576SJeff Roberson * 408355f576SJeff Roberson * The basic ideas stem from similar slab/zone based allocators whose algorithms 418355f576SJeff Roberson * are well known. 428355f576SJeff Roberson * 438355f576SJeff Roberson */ 448355f576SJeff Roberson 458355f576SJeff Roberson /* 468355f576SJeff Roberson * TODO: 478355f576SJeff Roberson * - Improve memory usage for large allocations 488355f576SJeff Roberson * - Investigate cache size adjustments 498355f576SJeff Roberson */ 508355f576SJeff Roberson 518355f576SJeff Roberson /* I should really use ktr.. */ 528355f576SJeff Roberson /* 538355f576SJeff Roberson #define UMA_DEBUG 1 548355f576SJeff Roberson #define UMA_DEBUG_ALLOC 1 558355f576SJeff Roberson #define UMA_DEBUG_ALLOC_1 1 568355f576SJeff Roberson */ 578355f576SJeff Roberson 588355f576SJeff Roberson 598355f576SJeff Roberson #include "opt_param.h" 608355f576SJeff Roberson #include <sys/param.h> 618355f576SJeff Roberson #include <sys/systm.h> 628355f576SJeff Roberson #include <sys/kernel.h> 638355f576SJeff Roberson #include <sys/types.h> 648355f576SJeff Roberson #include <sys/queue.h> 658355f576SJeff Roberson #include <sys/malloc.h> 668355f576SJeff Roberson #include <sys/lock.h> 678355f576SJeff Roberson #include <sys/sysctl.h> 688355f576SJeff Roberson #include <sys/mutex.h> 694c1cc01cSJohn Baldwin #include <sys/proc.h> 708355f576SJeff Roberson #include <sys/smp.h> 7186bbae32SJeff Roberson #include <sys/vmmeter.h> 7286bbae32SJeff Roberson 738355f576SJeff Roberson #include <vm/vm.h> 748355f576SJeff Roberson #include <vm/vm_object.h> 758355f576SJeff Roberson #include <vm/vm_page.h> 768355f576SJeff Roberson #include <vm/vm_param.h> 778355f576SJeff Roberson #include <vm/vm_map.h> 788355f576SJeff Roberson #include <vm/vm_kern.h> 798355f576SJeff Roberson #include <vm/vm_extern.h> 808355f576SJeff Roberson #include <vm/uma.h> 818355f576SJeff Roberson #include <vm/uma_int.h> 82639c9550SJeff Roberson #include <vm/uma_dbg.h> 838355f576SJeff Roberson 848355f576SJeff Roberson /* 858355f576SJeff Roberson * This is the zone from which all zones are spawned. The idea is that even 868355f576SJeff Roberson * the zone heads are allocated from the allocator, so we use the bss section 878355f576SJeff Roberson * to bootstrap us. 888355f576SJeff Roberson */ 8986bbae32SJeff Roberson static struct uma_zone masterzone; 9086bbae32SJeff Roberson static uma_zone_t zones = &masterzone; 918355f576SJeff Roberson 928355f576SJeff Roberson /* This is the zone from which all of uma_slab_t's are allocated. */ 938355f576SJeff Roberson static uma_zone_t slabzone; 948355f576SJeff Roberson 958355f576SJeff Roberson /* 968355f576SJeff Roberson * The initial hash tables come out of this zone so they can be allocated 978355f576SJeff Roberson * prior to malloc coming up. 988355f576SJeff Roberson */ 998355f576SJeff Roberson static uma_zone_t hashzone; 1008355f576SJeff Roberson 1018355f576SJeff Roberson /* 1028355f576SJeff Roberson * Zone that buckets come from. 1038355f576SJeff Roberson */ 1048355f576SJeff Roberson static uma_zone_t bucketzone; 1058355f576SJeff Roberson 10686bbae32SJeff Roberson /* 10786bbae32SJeff Roberson * Are we allowed to allocate buckets? 10886bbae32SJeff Roberson */ 10986bbae32SJeff Roberson static int bucketdisable = 1; 11086bbae32SJeff Roberson 1118355f576SJeff Roberson /* Linked list of all zones in the system */ 1128355f576SJeff Roberson static LIST_HEAD(,uma_zone) uma_zones = LIST_HEAD_INITIALIZER(&uma_zones); 1138355f576SJeff Roberson 1148355f576SJeff Roberson /* This mutex protects the zone list */ 1158355f576SJeff Roberson static struct mtx uma_mtx; 1168355f576SJeff Roberson 1178355f576SJeff Roberson /* Linked list of boot time pages */ 1188355f576SJeff Roberson static LIST_HEAD(,uma_slab) uma_boot_pages = 1198355f576SJeff Roberson LIST_HEAD_INITIALIZER(&uma_boot_pages); 1208355f576SJeff Roberson 1218355f576SJeff Roberson /* Count of free boottime pages */ 1228355f576SJeff Roberson static int uma_boot_free = 0; 1238355f576SJeff Roberson 1248355f576SJeff Roberson /* Is the VM done starting up? */ 1258355f576SJeff Roberson static int booted = 0; 1268355f576SJeff Roberson 1278355f576SJeff Roberson /* This is the handle used to schedule our working set calculator */ 1288355f576SJeff Roberson static struct callout uma_callout; 1298355f576SJeff Roberson 1308355f576SJeff Roberson /* This is mp_maxid + 1, for use while looping over each cpu */ 1318355f576SJeff Roberson static int maxcpu; 1328355f576SJeff Roberson 1338355f576SJeff Roberson /* 1348355f576SJeff Roberson * This structure is passed as the zone ctor arg so that I don't have to create 1358355f576SJeff Roberson * a special allocation function just for zones. 1368355f576SJeff Roberson */ 1378355f576SJeff Roberson struct uma_zctor_args { 1388355f576SJeff Roberson char *name; 139c3bdc05fSAndrew R. Reiter size_t size; 1408355f576SJeff Roberson uma_ctor ctor; 1418355f576SJeff Roberson uma_dtor dtor; 1428355f576SJeff Roberson uma_init uminit; 1438355f576SJeff Roberson uma_fini fini; 1448355f576SJeff Roberson int align; 1458355f576SJeff Roberson u_int16_t flags; 1468355f576SJeff Roberson }; 1478355f576SJeff Roberson 1488355f576SJeff Roberson /* Prototypes.. */ 1498355f576SJeff Roberson 1508355f576SJeff Roberson static void *obj_alloc(uma_zone_t, int, u_int8_t *, int); 1518355f576SJeff Roberson static void *page_alloc(uma_zone_t, int, u_int8_t *, int); 1528355f576SJeff Roberson static void page_free(void *, int, u_int8_t); 1538355f576SJeff Roberson static uma_slab_t slab_zalloc(uma_zone_t, int); 1548355f576SJeff Roberson static void cache_drain(uma_zone_t); 1558355f576SJeff Roberson static void bucket_drain(uma_zone_t, uma_bucket_t); 1568355f576SJeff Roberson static void zone_drain(uma_zone_t); 1578355f576SJeff Roberson static void zone_ctor(void *, int, void *); 1589c2cd7e5SJeff Roberson static void zone_dtor(void *, int, void *); 1598355f576SJeff Roberson static void zero_init(void *, int); 1608355f576SJeff Roberson static void zone_small_init(uma_zone_t zone); 1618355f576SJeff Roberson static void zone_large_init(uma_zone_t zone); 1628355f576SJeff Roberson static void zone_foreach(void (*zfunc)(uma_zone_t)); 1638355f576SJeff Roberson static void zone_timeout(uma_zone_t zone); 1640aef6126SJeff Roberson static int hash_alloc(struct uma_hash *); 1650aef6126SJeff Roberson static int hash_expand(struct uma_hash *, struct uma_hash *); 1660aef6126SJeff Roberson static void hash_free(struct uma_hash *hash); 1678355f576SJeff Roberson static void uma_timeout(void *); 1688355f576SJeff Roberson static void uma_startup3(void); 169bbee39c6SJeff Roberson static void *uma_zalloc_internal(uma_zone_t, void *, int); 17086bbae32SJeff Roberson static void uma_zfree_internal(uma_zone_t, void *, void *, int); 17186bbae32SJeff Roberson static void bucket_enable(void); 172bbee39c6SJeff Roberson static int uma_zalloc_bucket(uma_zone_t zone, int flags); 173bbee39c6SJeff Roberson static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags); 174bbee39c6SJeff Roberson static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab); 175bbee39c6SJeff Roberson 1768355f576SJeff Roberson void uma_print_zone(uma_zone_t); 1778355f576SJeff Roberson void uma_print_stats(void); 1788355f576SJeff Roberson static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS); 1798355f576SJeff Roberson 1808355f576SJeff Roberson SYSCTL_OID(_vm, OID_AUTO, zone, CTLTYPE_STRING|CTLFLAG_RD, 1818355f576SJeff Roberson NULL, 0, sysctl_vm_zone, "A", "Zone Info"); 1828355f576SJeff Roberson SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 1838355f576SJeff Roberson 18486bbae32SJeff Roberson /* 18586bbae32SJeff Roberson * This routine checks to see whether or not it's safe to enable buckets. 18686bbae32SJeff Roberson */ 18786bbae32SJeff Roberson 18886bbae32SJeff Roberson static void 18986bbae32SJeff Roberson bucket_enable(void) 19086bbae32SJeff Roberson { 19186bbae32SJeff Roberson if (cnt.v_free_count < cnt.v_free_min) 19286bbae32SJeff Roberson bucketdisable = 1; 19386bbae32SJeff Roberson else 19486bbae32SJeff Roberson bucketdisable = 0; 19586bbae32SJeff Roberson } 19686bbae32SJeff Roberson 1978355f576SJeff Roberson 1988355f576SJeff Roberson /* 1998355f576SJeff Roberson * Routine called by timeout which is used to fire off some time interval 2008355f576SJeff Roberson * based calculations. (working set, stats, etc.) 2018355f576SJeff Roberson * 2028355f576SJeff Roberson * Arguments: 2038355f576SJeff Roberson * arg Unused 2048355f576SJeff Roberson * 2058355f576SJeff Roberson * Returns: 2068355f576SJeff Roberson * Nothing 2078355f576SJeff Roberson */ 2088355f576SJeff Roberson static void 2098355f576SJeff Roberson uma_timeout(void *unused) 2108355f576SJeff Roberson { 21186bbae32SJeff Roberson bucket_enable(); 2128355f576SJeff Roberson zone_foreach(zone_timeout); 2138355f576SJeff Roberson 2148355f576SJeff Roberson /* Reschedule this event */ 2158355f576SJeff Roberson callout_reset(&uma_callout, UMA_WORKING_TIME * hz, uma_timeout, NULL); 2168355f576SJeff Roberson } 2178355f576SJeff Roberson 2188355f576SJeff Roberson /* 2198355f576SJeff Roberson * Routine to perform timeout driven calculations. This does the working set 2208355f576SJeff Roberson * as well as hash expanding, and per cpu statistics aggregation. 2218355f576SJeff Roberson * 2228355f576SJeff Roberson * Arguments: 2238355f576SJeff Roberson * zone The zone to operate on 2248355f576SJeff Roberson * 2258355f576SJeff Roberson * Returns: 2268355f576SJeff Roberson * Nothing 2278355f576SJeff Roberson */ 2288355f576SJeff Roberson static void 2298355f576SJeff Roberson zone_timeout(uma_zone_t zone) 2308355f576SJeff Roberson { 2318355f576SJeff Roberson uma_cache_t cache; 2328355f576SJeff Roberson u_int64_t alloc; 2338355f576SJeff Roberson int free; 2348355f576SJeff Roberson int cpu; 2358355f576SJeff Roberson 2368355f576SJeff Roberson alloc = 0; 2378355f576SJeff Roberson free = 0; 2388355f576SJeff Roberson 2398355f576SJeff Roberson /* 2408355f576SJeff Roberson * Aggregate per cpu cache statistics back to the zone. 2418355f576SJeff Roberson * 2428355f576SJeff Roberson * I may rewrite this to set a flag in the per cpu cache instead of 2438355f576SJeff Roberson * locking. If the flag is not cleared on the next round I will have 2448355f576SJeff Roberson * to lock and do it here instead so that the statistics don't get too 2458355f576SJeff Roberson * far out of sync. 2468355f576SJeff Roberson */ 2478355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) { 2488355f576SJeff Roberson for (cpu = 0; cpu < maxcpu; cpu++) { 2498355f576SJeff Roberson if (CPU_ABSENT(cpu)) 2508355f576SJeff Roberson continue; 2518355f576SJeff Roberson CPU_LOCK(zone, cpu); 2528355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 2538355f576SJeff Roberson /* Add them up, and reset */ 2548355f576SJeff Roberson alloc += cache->uc_allocs; 2558355f576SJeff Roberson cache->uc_allocs = 0; 2568355f576SJeff Roberson if (cache->uc_allocbucket) 2578355f576SJeff Roberson free += cache->uc_allocbucket->ub_ptr + 1; 2588355f576SJeff Roberson if (cache->uc_freebucket) 2598355f576SJeff Roberson free += cache->uc_freebucket->ub_ptr + 1; 2608355f576SJeff Roberson CPU_UNLOCK(zone, cpu); 2618355f576SJeff Roberson } 2628355f576SJeff Roberson } 2638355f576SJeff Roberson 2648355f576SJeff Roberson /* Now push these stats back into the zone.. */ 2658355f576SJeff Roberson ZONE_LOCK(zone); 2668355f576SJeff Roberson zone->uz_allocs += alloc; 2678355f576SJeff Roberson 2688355f576SJeff Roberson /* 2698355f576SJeff Roberson * cachefree is an instantanious snapshot of what is in the per cpu 2708355f576SJeff Roberson * caches, not an accurate counter 2718355f576SJeff Roberson */ 2728355f576SJeff Roberson zone->uz_cachefree = free; 2738355f576SJeff Roberson 2748355f576SJeff Roberson /* 2758355f576SJeff Roberson * Expand the zone hash table. 2768355f576SJeff Roberson * 2778355f576SJeff Roberson * This is done if the number of slabs is larger than the hash size. 2788355f576SJeff Roberson * What I'm trying to do here is completely reduce collisions. This 2798355f576SJeff Roberson * may be a little aggressive. Should I allow for two collisions max? 2808355f576SJeff Roberson */ 2818355f576SJeff Roberson 28299571dc3SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_HASH && 28399571dc3SJeff Roberson zone->uz_pages / zone->uz_ppera >= zone->uz_hash.uh_hashsize) { 2840aef6126SJeff Roberson struct uma_hash newhash; 2850aef6126SJeff Roberson struct uma_hash oldhash; 2860aef6126SJeff Roberson int ret; 2875300d9ddSJeff Roberson 2880aef6126SJeff Roberson /* 2890aef6126SJeff Roberson * This is so involved because allocating and freeing 2900aef6126SJeff Roberson * while the zone lock is held will lead to deadlock. 2910aef6126SJeff Roberson * I have to do everything in stages and check for 2920aef6126SJeff Roberson * races. 2930aef6126SJeff Roberson */ 2940aef6126SJeff Roberson newhash = zone->uz_hash; 2955300d9ddSJeff Roberson ZONE_UNLOCK(zone); 2960aef6126SJeff Roberson ret = hash_alloc(&newhash); 2975300d9ddSJeff Roberson ZONE_LOCK(zone); 2980aef6126SJeff Roberson if (ret) { 2990aef6126SJeff Roberson if (hash_expand(&zone->uz_hash, &newhash)) { 3000aef6126SJeff Roberson oldhash = zone->uz_hash; 3010aef6126SJeff Roberson zone->uz_hash = newhash; 3020aef6126SJeff Roberson } else 3030aef6126SJeff Roberson oldhash = newhash; 3040aef6126SJeff Roberson 3050aef6126SJeff Roberson ZONE_UNLOCK(zone); 3060aef6126SJeff Roberson hash_free(&oldhash); 3070aef6126SJeff Roberson ZONE_LOCK(zone); 3080aef6126SJeff Roberson } 3095300d9ddSJeff Roberson } 3108355f576SJeff Roberson 3118355f576SJeff Roberson /* 3128355f576SJeff Roberson * Here we compute the working set size as the total number of items 3138355f576SJeff Roberson * left outstanding since the last time interval. This is slightly 3148355f576SJeff Roberson * suboptimal. What we really want is the highest number of outstanding 3158355f576SJeff Roberson * items during the last time quantum. This should be close enough. 3168355f576SJeff Roberson * 3178355f576SJeff Roberson * The working set size is used to throttle the zone_drain function. 3188355f576SJeff Roberson * We don't want to return memory that we may need again immediately. 3198355f576SJeff Roberson */ 3208355f576SJeff Roberson alloc = zone->uz_allocs - zone->uz_oallocs; 3218355f576SJeff Roberson zone->uz_oallocs = zone->uz_allocs; 3228355f576SJeff Roberson zone->uz_wssize = alloc; 3238355f576SJeff Roberson 3248355f576SJeff Roberson ZONE_UNLOCK(zone); 3258355f576SJeff Roberson } 3268355f576SJeff Roberson 3278355f576SJeff Roberson /* 3285300d9ddSJeff Roberson * Allocate and zero fill the next sized hash table from the appropriate 3295300d9ddSJeff Roberson * backing store. 3305300d9ddSJeff Roberson * 3315300d9ddSJeff Roberson * Arguments: 3320aef6126SJeff Roberson * hash A new hash structure with the old hash size in uh_hashsize 3335300d9ddSJeff Roberson * 3345300d9ddSJeff Roberson * Returns: 3350aef6126SJeff Roberson * 1 on sucess and 0 on failure. 3365300d9ddSJeff Roberson */ 33737c84183SPoul-Henning Kamp static int 3380aef6126SJeff Roberson hash_alloc(struct uma_hash *hash) 3395300d9ddSJeff Roberson { 3400aef6126SJeff Roberson int oldsize; 3415300d9ddSJeff Roberson int alloc; 3425300d9ddSJeff Roberson 3430aef6126SJeff Roberson oldsize = hash->uh_hashsize; 3440aef6126SJeff Roberson 3455300d9ddSJeff Roberson /* We're just going to go to a power of two greater */ 3460aef6126SJeff Roberson if (oldsize) { 3470aef6126SJeff Roberson hash->uh_hashsize = oldsize * 2; 3480aef6126SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; 3495300d9ddSJeff Roberson /* XXX Shouldn't be abusing DEVBUF here */ 3500aef6126SJeff Roberson hash->uh_slab_hash = (struct slabhead *)malloc(alloc, 3510aef6126SJeff Roberson M_DEVBUF, M_NOWAIT); 3525300d9ddSJeff Roberson } else { 3530aef6126SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; 3540aef6126SJeff Roberson hash->uh_slab_hash = uma_zalloc_internal(hashzone, NULL, 355bbee39c6SJeff Roberson M_WAITOK); 3560aef6126SJeff Roberson hash->uh_hashsize = UMA_HASH_SIZE_INIT; 3575300d9ddSJeff Roberson } 3580aef6126SJeff Roberson if (hash->uh_slab_hash) { 3590aef6126SJeff Roberson bzero(hash->uh_slab_hash, alloc); 3600aef6126SJeff Roberson hash->uh_hashmask = hash->uh_hashsize - 1; 3610aef6126SJeff Roberson return (1); 3620aef6126SJeff Roberson } 3635300d9ddSJeff Roberson 3640aef6126SJeff Roberson return (0); 3655300d9ddSJeff Roberson } 3665300d9ddSJeff Roberson 3675300d9ddSJeff Roberson /* 3688355f576SJeff Roberson * Expands the hash table for OFFPAGE zones. This is done from zone_timeout 3698355f576SJeff Roberson * to reduce collisions. This must not be done in the regular allocation path, 3708355f576SJeff Roberson * otherwise, we can recurse on the vm while allocating pages. 3718355f576SJeff Roberson * 3728355f576SJeff Roberson * Arguments: 3730aef6126SJeff Roberson * oldhash The hash you want to expand 3740aef6126SJeff Roberson * newhash The hash structure for the new table 3758355f576SJeff Roberson * 3768355f576SJeff Roberson * Returns: 3778355f576SJeff Roberson * Nothing 3788355f576SJeff Roberson * 3798355f576SJeff Roberson * Discussion: 3808355f576SJeff Roberson */ 3810aef6126SJeff Roberson static int 3820aef6126SJeff Roberson hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash) 3838355f576SJeff Roberson { 3848355f576SJeff Roberson uma_slab_t slab; 3858355f576SJeff Roberson int hval; 3868355f576SJeff Roberson int i; 3878355f576SJeff Roberson 3880aef6126SJeff Roberson if (!newhash->uh_slab_hash) 3890aef6126SJeff Roberson return (0); 3908355f576SJeff Roberson 3910aef6126SJeff Roberson if (oldhash->uh_hashsize >= newhash->uh_hashsize) 3920aef6126SJeff Roberson return (0); 3938355f576SJeff Roberson 3948355f576SJeff Roberson /* 3958355f576SJeff Roberson * I need to investigate hash algorithms for resizing without a 3968355f576SJeff Roberson * full rehash. 3978355f576SJeff Roberson */ 3988355f576SJeff Roberson 3990aef6126SJeff Roberson for (i = 0; i < oldhash->uh_hashsize; i++) 4000aef6126SJeff Roberson while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) { 4010aef6126SJeff Roberson slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]); 4020aef6126SJeff Roberson SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink); 4030aef6126SJeff Roberson hval = UMA_HASH(newhash, slab->us_data); 4040aef6126SJeff Roberson SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval], 4050aef6126SJeff Roberson slab, us_hlink); 4068355f576SJeff Roberson } 4078355f576SJeff Roberson 4080aef6126SJeff Roberson return (1); 4099c2cd7e5SJeff Roberson } 4109c2cd7e5SJeff Roberson 4115300d9ddSJeff Roberson /* 4125300d9ddSJeff Roberson * Free the hash bucket to the appropriate backing store. 4135300d9ddSJeff Roberson * 4145300d9ddSJeff Roberson * Arguments: 4155300d9ddSJeff Roberson * slab_hash The hash bucket we're freeing 4165300d9ddSJeff Roberson * hashsize The number of entries in that hash bucket 4175300d9ddSJeff Roberson * 4185300d9ddSJeff Roberson * Returns: 4195300d9ddSJeff Roberson * Nothing 4205300d9ddSJeff Roberson */ 4219c2cd7e5SJeff Roberson static void 4220aef6126SJeff Roberson hash_free(struct uma_hash *hash) 4239c2cd7e5SJeff Roberson { 4240aef6126SJeff Roberson if (hash->uh_slab_hash == NULL) 4250aef6126SJeff Roberson return; 4260aef6126SJeff Roberson if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) 4278355f576SJeff Roberson uma_zfree_internal(hashzone, 4280aef6126SJeff Roberson hash->uh_slab_hash, NULL, 0); 4298355f576SJeff Roberson else 4300aef6126SJeff Roberson free(hash->uh_slab_hash, M_DEVBUF); 4318355f576SJeff Roberson } 4328355f576SJeff Roberson 4338355f576SJeff Roberson /* 4348355f576SJeff Roberson * Frees all outstanding items in a bucket 4358355f576SJeff Roberson * 4368355f576SJeff Roberson * Arguments: 4378355f576SJeff Roberson * zone The zone to free to, must be unlocked. 4388355f576SJeff Roberson * bucket The free/alloc bucket with items, cpu queue must be locked. 4398355f576SJeff Roberson * 4408355f576SJeff Roberson * Returns: 4418355f576SJeff Roberson * Nothing 4428355f576SJeff Roberson */ 4438355f576SJeff Roberson 4448355f576SJeff Roberson static void 4458355f576SJeff Roberson bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 4468355f576SJeff Roberson { 4478355f576SJeff Roberson uma_slab_t slab; 4488355f576SJeff Roberson int mzone; 4498355f576SJeff Roberson void *item; 4508355f576SJeff Roberson 4518355f576SJeff Roberson if (bucket == NULL) 4528355f576SJeff Roberson return; 4538355f576SJeff Roberson 4548355f576SJeff Roberson slab = NULL; 4558355f576SJeff Roberson mzone = 0; 4568355f576SJeff Roberson 4578355f576SJeff Roberson /* We have to lookup the slab again for malloc.. */ 4588355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_MALLOC) 4598355f576SJeff Roberson mzone = 1; 4608355f576SJeff Roberson 4618355f576SJeff Roberson while (bucket->ub_ptr > -1) { 4628355f576SJeff Roberson item = bucket->ub_bucket[bucket->ub_ptr]; 4638355f576SJeff Roberson #ifdef INVARIANTS 4648355f576SJeff Roberson bucket->ub_bucket[bucket->ub_ptr] = NULL; 4658355f576SJeff Roberson KASSERT(item != NULL, 4668355f576SJeff Roberson ("bucket_drain: botched ptr, item is NULL")); 4678355f576SJeff Roberson #endif 4688355f576SJeff Roberson bucket->ub_ptr--; 4698355f576SJeff Roberson /* 4708355f576SJeff Roberson * This is extremely inefficient. The slab pointer was passed 4718355f576SJeff Roberson * to uma_zfree_arg, but we lost it because the buckets don't 4728355f576SJeff Roberson * hold them. This will go away when free() gets a size passed 4738355f576SJeff Roberson * to it. 4748355f576SJeff Roberson */ 47599571dc3SJeff Roberson if (mzone) 47699571dc3SJeff Roberson slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK)); 4778355f576SJeff Roberson uma_zfree_internal(zone, item, slab, 1); 4788355f576SJeff Roberson } 4798355f576SJeff Roberson } 4808355f576SJeff Roberson 4818355f576SJeff Roberson /* 4828355f576SJeff Roberson * Drains the per cpu caches for a zone. 4838355f576SJeff Roberson * 4848355f576SJeff Roberson * Arguments: 4858355f576SJeff Roberson * zone The zone to drain, must be unlocked. 4868355f576SJeff Roberson * 4878355f576SJeff Roberson * Returns: 4888355f576SJeff Roberson * Nothing 4898355f576SJeff Roberson * 4908355f576SJeff Roberson * This function returns with the zone locked so that the per cpu queues can 4918355f576SJeff Roberson * not be filled until zone_drain is finished. 4928355f576SJeff Roberson * 4938355f576SJeff Roberson */ 4948355f576SJeff Roberson static void 4958355f576SJeff Roberson cache_drain(uma_zone_t zone) 4968355f576SJeff Roberson { 4978355f576SJeff Roberson uma_bucket_t bucket; 4988355f576SJeff Roberson uma_cache_t cache; 4998355f576SJeff Roberson int cpu; 5008355f576SJeff Roberson 5018355f576SJeff Roberson /* 5028355f576SJeff Roberson * Flush out the per cpu queues. 5038355f576SJeff Roberson * 504157d7b35SAlfred Perlstein * XXX This causes unnecessary thrashing due to immediately having 5058355f576SJeff Roberson * empty per cpu queues. I need to improve this. 5068355f576SJeff Roberson */ 5078355f576SJeff Roberson 5088355f576SJeff Roberson /* 5098355f576SJeff Roberson * We have to lock each cpu cache before locking the zone 5108355f576SJeff Roberson */ 5118355f576SJeff Roberson ZONE_UNLOCK(zone); 5128355f576SJeff Roberson 5138355f576SJeff Roberson for (cpu = 0; cpu < maxcpu; cpu++) { 5148355f576SJeff Roberson if (CPU_ABSENT(cpu)) 5158355f576SJeff Roberson continue; 5168355f576SJeff Roberson CPU_LOCK(zone, cpu); 5178355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 5188355f576SJeff Roberson bucket_drain(zone, cache->uc_allocbucket); 5198355f576SJeff Roberson bucket_drain(zone, cache->uc_freebucket); 5208355f576SJeff Roberson } 5218355f576SJeff Roberson 5228355f576SJeff Roberson /* 5238355f576SJeff Roberson * Drain the bucket queues and free the buckets, we just keep two per 5248355f576SJeff Roberson * cpu (alloc/free). 5258355f576SJeff Roberson */ 5268355f576SJeff Roberson ZONE_LOCK(zone); 5278355f576SJeff Roberson while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 5288355f576SJeff Roberson LIST_REMOVE(bucket, ub_link); 5298355f576SJeff Roberson ZONE_UNLOCK(zone); 5308355f576SJeff Roberson bucket_drain(zone, bucket); 5318355f576SJeff Roberson uma_zfree_internal(bucketzone, bucket, NULL, 0); 5328355f576SJeff Roberson ZONE_LOCK(zone); 5338355f576SJeff Roberson } 5348355f576SJeff Roberson 5358355f576SJeff Roberson /* Now we do the free queue.. */ 5368355f576SJeff Roberson while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 5378355f576SJeff Roberson LIST_REMOVE(bucket, ub_link); 5388355f576SJeff Roberson uma_zfree_internal(bucketzone, bucket, NULL, 0); 5398355f576SJeff Roberson } 5408355f576SJeff Roberson 5418355f576SJeff Roberson /* We unlock here, but they will all block until the zone is unlocked */ 5428355f576SJeff Roberson for (cpu = 0; cpu < maxcpu; cpu++) { 5438355f576SJeff Roberson if (CPU_ABSENT(cpu)) 5448355f576SJeff Roberson continue; 5458355f576SJeff Roberson CPU_UNLOCK(zone, cpu); 5468355f576SJeff Roberson } 547f4af24d5SJeff Roberson 548f4af24d5SJeff Roberson zone->uz_cachefree = 0; 5498355f576SJeff Roberson } 5508355f576SJeff Roberson 5518355f576SJeff Roberson /* 5528355f576SJeff Roberson * Frees pages from a zone back to the system. This is done on demand from 5538355f576SJeff Roberson * the pageout daemon. 5548355f576SJeff Roberson * 5558355f576SJeff Roberson * Arguments: 5568355f576SJeff Roberson * zone The zone to free pages from 5579c2cd7e5SJeff Roberson * all Should we drain all items? 5588355f576SJeff Roberson * 5598355f576SJeff Roberson * Returns: 5608355f576SJeff Roberson * Nothing. 5618355f576SJeff Roberson */ 5628355f576SJeff Roberson static void 5638355f576SJeff Roberson zone_drain(uma_zone_t zone) 5648355f576SJeff Roberson { 565713deb36SJeff Roberson struct slabhead freeslabs = {}; 5668355f576SJeff Roberson uma_slab_t slab; 5678355f576SJeff Roberson uma_slab_t n; 5688355f576SJeff Roberson u_int64_t extra; 5698355f576SJeff Roberson u_int8_t flags; 5708355f576SJeff Roberson u_int8_t *mem; 5718355f576SJeff Roberson int i; 5728355f576SJeff Roberson 5738355f576SJeff Roberson /* 5748355f576SJeff Roberson * We don't want to take pages from staticly allocated zones at this 5758355f576SJeff Roberson * time 5768355f576SJeff Roberson */ 5778355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_NOFREE || zone->uz_freef == NULL) 5788355f576SJeff Roberson return; 5798355f576SJeff Roberson 5808355f576SJeff Roberson ZONE_LOCK(zone); 5818355f576SJeff Roberson 5828355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) 5838355f576SJeff Roberson cache_drain(zone); 5848355f576SJeff Roberson 5858355f576SJeff Roberson if (zone->uz_free < zone->uz_wssize) 5868355f576SJeff Roberson goto finished; 5878355f576SJeff Roberson #ifdef UMA_DEBUG 5888355f576SJeff Roberson printf("%s working set size: %llu free items: %u\n", 5898355f576SJeff Roberson zone->uz_name, (unsigned long long)zone->uz_wssize, zone->uz_free); 5908355f576SJeff Roberson #endif 5919c2cd7e5SJeff Roberson extra = zone->uz_free - zone->uz_wssize; 5928355f576SJeff Roberson extra /= zone->uz_ipers; 5938355f576SJeff Roberson 5948355f576SJeff Roberson /* extra is now the number of extra slabs that we can free */ 5958355f576SJeff Roberson 5968355f576SJeff Roberson if (extra == 0) 5978355f576SJeff Roberson goto finished; 5988355f576SJeff Roberson 5998355f576SJeff Roberson slab = LIST_FIRST(&zone->uz_free_slab); 6008355f576SJeff Roberson while (slab && extra) { 6018355f576SJeff Roberson n = LIST_NEXT(slab, us_link); 6028355f576SJeff Roberson 6038355f576SJeff Roberson /* We have no where to free these to */ 6048355f576SJeff Roberson if (slab->us_flags & UMA_SLAB_BOOT) { 6058355f576SJeff Roberson slab = n; 6068355f576SJeff Roberson continue; 6078355f576SJeff Roberson } 6088355f576SJeff Roberson 6098355f576SJeff Roberson LIST_REMOVE(slab, us_link); 6108355f576SJeff Roberson zone->uz_pages -= zone->uz_ppera; 6118355f576SJeff Roberson zone->uz_free -= zone->uz_ipers; 612713deb36SJeff Roberson 61399571dc3SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_HASH) 614713deb36SJeff Roberson UMA_HASH_REMOVE(&zone->uz_hash, slab, slab->us_data); 615713deb36SJeff Roberson 616713deb36SJeff Roberson SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink); 617713deb36SJeff Roberson 618713deb36SJeff Roberson slab = n; 619713deb36SJeff Roberson extra--; 620713deb36SJeff Roberson } 621713deb36SJeff Roberson finished: 622713deb36SJeff Roberson ZONE_UNLOCK(zone); 623713deb36SJeff Roberson 624713deb36SJeff Roberson while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { 625713deb36SJeff Roberson SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); 6268355f576SJeff Roberson if (zone->uz_fini) 6278355f576SJeff Roberson for (i = 0; i < zone->uz_ipers; i++) 6288355f576SJeff Roberson zone->uz_fini( 6298355f576SJeff Roberson slab->us_data + (zone->uz_rsize * i), 6308355f576SJeff Roberson zone->uz_size); 6318355f576SJeff Roberson flags = slab->us_flags; 6328355f576SJeff Roberson mem = slab->us_data; 63399571dc3SJeff Roberson 63499571dc3SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_OFFPAGE) 6358355f576SJeff Roberson uma_zfree_internal(slabzone, slab, NULL, 0); 63699571dc3SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_MALLOC) 63799571dc3SJeff Roberson for (i = 0; i < zone->uz_ppera; i++) 63899571dc3SJeff Roberson vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), 63999571dc3SJeff Roberson kmem_object); 6408355f576SJeff Roberson #ifdef UMA_DEBUG 6418355f576SJeff Roberson printf("%s: Returning %d bytes.\n", 6428355f576SJeff Roberson zone->uz_name, UMA_SLAB_SIZE * zone->uz_ppera); 6438355f576SJeff Roberson #endif 6448355f576SJeff Roberson zone->uz_freef(mem, UMA_SLAB_SIZE * zone->uz_ppera, flags); 6458355f576SJeff Roberson } 6468355f576SJeff Roberson 6478355f576SJeff Roberson } 6488355f576SJeff Roberson 6498355f576SJeff Roberson /* 6508355f576SJeff Roberson * Allocate a new slab for a zone. This does not insert the slab onto a list. 6518355f576SJeff Roberson * 6528355f576SJeff Roberson * Arguments: 6538355f576SJeff Roberson * zone The zone to allocate slabs for 6548355f576SJeff Roberson * wait Shall we wait? 6558355f576SJeff Roberson * 6568355f576SJeff Roberson * Returns: 6578355f576SJeff Roberson * The slab that was allocated or NULL if there is no memory and the 6588355f576SJeff Roberson * caller specified M_NOWAIT. 6598355f576SJeff Roberson * 6608355f576SJeff Roberson */ 6618355f576SJeff Roberson static uma_slab_t 6628355f576SJeff Roberson slab_zalloc(uma_zone_t zone, int wait) 6638355f576SJeff Roberson { 6648355f576SJeff Roberson uma_slab_t slab; /* Starting slab */ 6658355f576SJeff Roberson u_int8_t *mem; 6668355f576SJeff Roberson u_int8_t flags; 6678355f576SJeff Roberson int i; 6688355f576SJeff Roberson 669a553d4b8SJeff Roberson slab = NULL; 670a553d4b8SJeff Roberson 6718355f576SJeff Roberson #ifdef UMA_DEBUG 6728355f576SJeff Roberson printf("slab_zalloc: Allocating a new slab for %s\n", zone->uz_name); 6738355f576SJeff Roberson #endif 6748355f576SJeff Roberson ZONE_UNLOCK(zone); 675a553d4b8SJeff Roberson 676a553d4b8SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_OFFPAGE) { 677bbee39c6SJeff Roberson slab = uma_zalloc_internal(slabzone, NULL, wait); 678a553d4b8SJeff Roberson if (slab == NULL) { 679a553d4b8SJeff Roberson ZONE_LOCK(zone); 680a553d4b8SJeff Roberson return NULL; 681a553d4b8SJeff Roberson } 682a553d4b8SJeff Roberson } 683a553d4b8SJeff Roberson 6843370c5bfSJeff Roberson /* 6853370c5bfSJeff Roberson * This reproduces the old vm_zone behavior of zero filling pages the 6863370c5bfSJeff Roberson * first time they are added to a zone. 6873370c5bfSJeff Roberson * 6883370c5bfSJeff Roberson * Malloced items are zeroed in uma_zalloc. 6893370c5bfSJeff Roberson */ 6903370c5bfSJeff Roberson 6913370c5bfSJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_MALLOC) == 0) 6923370c5bfSJeff Roberson wait |= M_ZERO; 6933370c5bfSJeff Roberson else 6943370c5bfSJeff Roberson wait &= ~M_ZERO; 6953370c5bfSJeff Roberson 696a553d4b8SJeff Roberson if (booted || (zone->uz_flags & UMA_ZFLAG_PRIVALLOC)) { 6978355f576SJeff Roberson mtx_lock(&Giant); 698a553d4b8SJeff Roberson mem = zone->uz_allocf(zone, 6998355f576SJeff Roberson zone->uz_ppera * UMA_SLAB_SIZE, &flags, wait); 7008355f576SJeff Roberson mtx_unlock(&Giant); 701a553d4b8SJeff Roberson if (mem == NULL) { 7028355f576SJeff Roberson ZONE_LOCK(zone); 7038355f576SJeff Roberson return (NULL); 704a553d4b8SJeff Roberson } 7058355f576SJeff Roberson } else { 706a553d4b8SJeff Roberson uma_slab_t tmps; 7078355f576SJeff Roberson 7088355f576SJeff Roberson if (zone->uz_ppera > 1) 7098355f576SJeff Roberson panic("UMA: Attemping to allocate multiple pages before vm has started.\n"); 7108355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_MALLOC) 7118355f576SJeff Roberson panic("Mallocing before uma_startup2 has been called.\n"); 7128355f576SJeff Roberson if (uma_boot_free == 0) 7138355f576SJeff Roberson panic("UMA: Ran out of pre init pages, increase UMA_BOOT_PAGES\n"); 714a553d4b8SJeff Roberson tmps = LIST_FIRST(&uma_boot_pages); 715a553d4b8SJeff Roberson LIST_REMOVE(tmps, us_link); 7168355f576SJeff Roberson uma_boot_free--; 717a553d4b8SJeff Roberson mem = tmps->us_data; 7188355f576SJeff Roberson } 7198355f576SJeff Roberson 7205c0e403bSJeff Roberson /* Point the slab into the allocated memory */ 72199571dc3SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_OFFPAGE)) 7228355f576SJeff Roberson slab = (uma_slab_t )(mem + zone->uz_pgoff); 7235c0e403bSJeff Roberson 72499571dc3SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_MALLOC) 72599571dc3SJeff Roberson for (i = 0; i < zone->uz_ppera; i++) 72699571dc3SJeff Roberson vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab); 7278355f576SJeff Roberson 7288355f576SJeff Roberson slab->us_zone = zone; 7298355f576SJeff Roberson slab->us_data = mem; 7308355f576SJeff Roberson 7318355f576SJeff Roberson /* 7328355f576SJeff Roberson * This is intended to spread data out across cache lines. 7338355f576SJeff Roberson * 7348355f576SJeff Roberson * This code doesn't seem to work properly on x86, and on alpha 7358355f576SJeff Roberson * it makes absolutely no performance difference. I'm sure it could 7368355f576SJeff Roberson * use some tuning, but sun makes outrageous claims about it's 7378355f576SJeff Roberson * performance. 7388355f576SJeff Roberson */ 7398355f576SJeff Roberson #if 0 7408355f576SJeff Roberson if (zone->uz_cachemax) { 7418355f576SJeff Roberson slab->us_data += zone->uz_cacheoff; 7428355f576SJeff Roberson zone->uz_cacheoff += UMA_CACHE_INC; 7438355f576SJeff Roberson if (zone->uz_cacheoff > zone->uz_cachemax) 7448355f576SJeff Roberson zone->uz_cacheoff = 0; 7458355f576SJeff Roberson } 7468355f576SJeff Roberson #endif 7478355f576SJeff Roberson 7488355f576SJeff Roberson slab->us_freecount = zone->uz_ipers; 7498355f576SJeff Roberson slab->us_firstfree = 0; 7508355f576SJeff Roberson slab->us_flags = flags; 7518355f576SJeff Roberson for (i = 0; i < zone->uz_ipers; i++) 7528355f576SJeff Roberson slab->us_freelist[i] = i+1; 7538355f576SJeff Roberson 7548355f576SJeff Roberson if (zone->uz_init) 7558355f576SJeff Roberson for (i = 0; i < zone->uz_ipers; i++) 7568355f576SJeff Roberson zone->uz_init(slab->us_data + (zone->uz_rsize * i), 7578355f576SJeff Roberson zone->uz_size); 7585c0e403bSJeff Roberson ZONE_LOCK(zone); 7595c0e403bSJeff Roberson 76099571dc3SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_HASH) 7615c0e403bSJeff Roberson UMA_HASH_INSERT(&zone->uz_hash, slab, mem); 7628355f576SJeff Roberson 7638355f576SJeff Roberson zone->uz_pages += zone->uz_ppera; 7648355f576SJeff Roberson zone->uz_free += zone->uz_ipers; 7658355f576SJeff Roberson 7665c0e403bSJeff Roberson 7678355f576SJeff Roberson return (slab); 7688355f576SJeff Roberson } 7698355f576SJeff Roberson 7708355f576SJeff Roberson /* 7718355f576SJeff Roberson * Allocates a number of pages from the system 7728355f576SJeff Roberson * 7738355f576SJeff Roberson * Arguments: 7748355f576SJeff Roberson * zone Unused 7758355f576SJeff Roberson * bytes The number of bytes requested 7768355f576SJeff Roberson * wait Shall we wait? 7778355f576SJeff Roberson * 7788355f576SJeff Roberson * Returns: 7798355f576SJeff Roberson * A pointer to the alloced memory or possibly 7808355f576SJeff Roberson * NULL if M_NOWAIT is set. 7818355f576SJeff Roberson */ 7828355f576SJeff Roberson static void * 7838355f576SJeff Roberson page_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 7848355f576SJeff Roberson { 7858355f576SJeff Roberson void *p; /* Returned page */ 7868355f576SJeff Roberson 7878355f576SJeff Roberson *pflag = UMA_SLAB_KMEM; 7888355f576SJeff Roberson p = (void *) kmem_malloc(kmem_map, bytes, wait); 7898355f576SJeff Roberson 7908355f576SJeff Roberson return (p); 7918355f576SJeff Roberson } 7928355f576SJeff Roberson 7938355f576SJeff Roberson /* 7948355f576SJeff Roberson * Allocates a number of pages from within an object 7958355f576SJeff Roberson * 7968355f576SJeff Roberson * Arguments: 7978355f576SJeff Roberson * zone Unused 7988355f576SJeff Roberson * bytes The number of bytes requested 7998355f576SJeff Roberson * wait Shall we wait? 8008355f576SJeff Roberson * 8018355f576SJeff Roberson * Returns: 8028355f576SJeff Roberson * A pointer to the alloced memory or possibly 8038355f576SJeff Roberson * NULL if M_NOWAIT is set. 804494273beSJeff Roberson * 805494273beSJeff Roberson * TODO: If we fail during a multi-page allocation release the pages that have 806494273beSJeff Roberson * already been allocated. 8078355f576SJeff Roberson */ 8088355f576SJeff Roberson static void * 8098355f576SJeff Roberson obj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 8108355f576SJeff Roberson { 8118355f576SJeff Roberson vm_offset_t zkva; 8128355f576SJeff Roberson vm_offset_t retkva; 8138355f576SJeff Roberson vm_page_t p; 8148355f576SJeff Roberson int pages; 8158355f576SJeff Roberson 81655f7c614SArchie Cobbs retkva = 0; 8178355f576SJeff Roberson pages = zone->uz_pages; 8188355f576SJeff Roberson 8198355f576SJeff Roberson /* 8208355f576SJeff Roberson * This looks a little weird since we're getting one page at a time 8218355f576SJeff Roberson */ 8228355f576SJeff Roberson while (bytes > 0) { 8238355f576SJeff Roberson p = vm_page_alloc(zone->uz_obj, pages, 8248355f576SJeff Roberson VM_ALLOC_INTERRUPT); 8258355f576SJeff Roberson if (p == NULL) 8268355f576SJeff Roberson return (NULL); 8278355f576SJeff Roberson 8288355f576SJeff Roberson zkva = zone->uz_kva + pages * PAGE_SIZE; 82955f7c614SArchie Cobbs if (retkva == 0) 8308355f576SJeff Roberson retkva = zkva; 8318355f576SJeff Roberson pmap_qenter(zkva, &p, 1); 8328355f576SJeff Roberson bytes -= PAGE_SIZE; 8338355f576SJeff Roberson pages += 1; 8348355f576SJeff Roberson } 8358355f576SJeff Roberson 8368355f576SJeff Roberson *flags = UMA_SLAB_PRIV; 8378355f576SJeff Roberson 8388355f576SJeff Roberson return ((void *)retkva); 8398355f576SJeff Roberson } 8408355f576SJeff Roberson 8418355f576SJeff Roberson /* 8428355f576SJeff Roberson * Frees a number of pages to the system 8438355f576SJeff Roberson * 8448355f576SJeff Roberson * Arguments: 8458355f576SJeff Roberson * mem A pointer to the memory to be freed 8468355f576SJeff Roberson * size The size of the memory being freed 8478355f576SJeff Roberson * flags The original p->us_flags field 8488355f576SJeff Roberson * 8498355f576SJeff Roberson * Returns: 8508355f576SJeff Roberson * Nothing 8518355f576SJeff Roberson * 8528355f576SJeff Roberson */ 8538355f576SJeff Roberson static void 8548355f576SJeff Roberson page_free(void *mem, int size, u_int8_t flags) 8558355f576SJeff Roberson { 8568355f576SJeff Roberson vm_map_t map; 8573370c5bfSJeff Roberson 8588355f576SJeff Roberson if (flags & UMA_SLAB_KMEM) 8598355f576SJeff Roberson map = kmem_map; 8608355f576SJeff Roberson else 8618355f576SJeff Roberson panic("UMA: page_free used with invalid flags %d\n", flags); 8628355f576SJeff Roberson 8638355f576SJeff Roberson kmem_free(map, (vm_offset_t)mem, size); 8648355f576SJeff Roberson } 8658355f576SJeff Roberson 8668355f576SJeff Roberson /* 8678355f576SJeff Roberson * Zero fill initializer 8688355f576SJeff Roberson * 8698355f576SJeff Roberson * Arguments/Returns follow uma_init specifications 8708355f576SJeff Roberson * 8718355f576SJeff Roberson */ 8728355f576SJeff Roberson static void 8738355f576SJeff Roberson zero_init(void *mem, int size) 8748355f576SJeff Roberson { 8758355f576SJeff Roberson bzero(mem, size); 8768355f576SJeff Roberson } 8778355f576SJeff Roberson 8788355f576SJeff Roberson /* 8798355f576SJeff Roberson * Finish creating a small uma zone. This calculates ipers, and the zone size. 8808355f576SJeff Roberson * 8818355f576SJeff Roberson * Arguments 8828355f576SJeff Roberson * zone The zone we should initialize 8838355f576SJeff Roberson * 8848355f576SJeff Roberson * Returns 8858355f576SJeff Roberson * Nothing 8868355f576SJeff Roberson */ 8878355f576SJeff Roberson static void 8888355f576SJeff Roberson zone_small_init(uma_zone_t zone) 8898355f576SJeff Roberson { 8908355f576SJeff Roberson int rsize; 8918355f576SJeff Roberson int memused; 8928355f576SJeff Roberson int ipers; 8938355f576SJeff Roberson 8948355f576SJeff Roberson rsize = zone->uz_size; 8958355f576SJeff Roberson 8968355f576SJeff Roberson if (rsize < UMA_SMALLEST_UNIT) 8978355f576SJeff Roberson rsize = UMA_SMALLEST_UNIT; 8988355f576SJeff Roberson 8998355f576SJeff Roberson if (rsize & zone->uz_align) 9008355f576SJeff Roberson rsize = (rsize & ~zone->uz_align) + (zone->uz_align + 1); 9018355f576SJeff Roberson 9028355f576SJeff Roberson zone->uz_rsize = rsize; 9038355f576SJeff Roberson 9048355f576SJeff Roberson rsize += 1; /* Account for the byte of linkage */ 9058355f576SJeff Roberson zone->uz_ipers = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) / rsize; 9068355f576SJeff Roberson zone->uz_ppera = 1; 9078355f576SJeff Roberson 9088355f576SJeff Roberson memused = zone->uz_ipers * zone->uz_rsize; 9098355f576SJeff Roberson 9108355f576SJeff Roberson /* Can we do any better? */ 9118355f576SJeff Roberson if ((UMA_SLAB_SIZE - memused) >= UMA_MAX_WASTE) { 9128355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 9138355f576SJeff Roberson return; 9148355f576SJeff Roberson ipers = UMA_SLAB_SIZE / zone->uz_rsize; 9158355f576SJeff Roberson if (ipers > zone->uz_ipers) { 9168355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_OFFPAGE; 91799571dc3SJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_MALLOC) == 0) 91899571dc3SJeff Roberson zone->uz_flags |= UMA_ZFLAG_HASH; 9198355f576SJeff Roberson zone->uz_ipers = ipers; 9208355f576SJeff Roberson } 9218355f576SJeff Roberson } 9228355f576SJeff Roberson 9238355f576SJeff Roberson } 9248355f576SJeff Roberson 9258355f576SJeff Roberson /* 9268355f576SJeff Roberson * Finish creating a large (> UMA_SLAB_SIZE) uma zone. Just give in and do 9278355f576SJeff Roberson * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be 9288355f576SJeff Roberson * more complicated. 9298355f576SJeff Roberson * 9308355f576SJeff Roberson * Arguments 9318355f576SJeff Roberson * zone The zone we should initialize 9328355f576SJeff Roberson * 9338355f576SJeff Roberson * Returns 9348355f576SJeff Roberson * Nothing 9358355f576SJeff Roberson */ 9368355f576SJeff Roberson static void 9378355f576SJeff Roberson zone_large_init(uma_zone_t zone) 9388355f576SJeff Roberson { 9398355f576SJeff Roberson int pages; 9408355f576SJeff Roberson 9418355f576SJeff Roberson pages = zone->uz_size / UMA_SLAB_SIZE; 9428355f576SJeff Roberson 9438355f576SJeff Roberson /* Account for remainder */ 9448355f576SJeff Roberson if ((pages * UMA_SLAB_SIZE) < zone->uz_size) 9458355f576SJeff Roberson pages++; 9468355f576SJeff Roberson 9478355f576SJeff Roberson zone->uz_ppera = pages; 9488355f576SJeff Roberson zone->uz_ipers = 1; 9498355f576SJeff Roberson 9508355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_OFFPAGE; 95199571dc3SJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_MALLOC) == 0) 95299571dc3SJeff Roberson zone->uz_flags |= UMA_ZFLAG_HASH; 95399571dc3SJeff Roberson 9548355f576SJeff Roberson zone->uz_rsize = zone->uz_size; 9558355f576SJeff Roberson } 9568355f576SJeff Roberson 9578355f576SJeff Roberson /* 9588355f576SJeff Roberson * Zone header ctor. This initializes all fields, locks, etc. And inserts 9598355f576SJeff Roberson * the zone onto the global zone list. 9608355f576SJeff Roberson * 9618355f576SJeff Roberson * Arguments/Returns follow uma_ctor specifications 9628355f576SJeff Roberson * udata Actually uma_zcreat_args 9638355f576SJeff Roberson * 9648355f576SJeff Roberson */ 9658355f576SJeff Roberson 9668355f576SJeff Roberson static void 9678355f576SJeff Roberson zone_ctor(void *mem, int size, void *udata) 9688355f576SJeff Roberson { 9698355f576SJeff Roberson struct uma_zctor_args *arg = udata; 9708355f576SJeff Roberson uma_zone_t zone = mem; 97128bc4419SJeff Roberson int privlc; 9728355f576SJeff Roberson int cplen; 9738355f576SJeff Roberson int cpu; 9748355f576SJeff Roberson 9758355f576SJeff Roberson bzero(zone, size); 9768355f576SJeff Roberson zone->uz_name = arg->name; 9778355f576SJeff Roberson zone->uz_size = arg->size; 9788355f576SJeff Roberson zone->uz_ctor = arg->ctor; 9798355f576SJeff Roberson zone->uz_dtor = arg->dtor; 9808355f576SJeff Roberson zone->uz_init = arg->uminit; 981e221e841SJeff Roberson zone->uz_fini = arg->fini; 9828355f576SJeff Roberson zone->uz_align = arg->align; 9838355f576SJeff Roberson zone->uz_free = 0; 9848355f576SJeff Roberson zone->uz_pages = 0; 9858355f576SJeff Roberson zone->uz_flags = 0; 9868355f576SJeff Roberson zone->uz_allocf = page_alloc; 9878355f576SJeff Roberson zone->uz_freef = page_free; 9888355f576SJeff Roberson 9898355f576SJeff Roberson if (arg->flags & UMA_ZONE_ZINIT) 9908355f576SJeff Roberson zone->uz_init = zero_init; 9918355f576SJeff Roberson 9928355f576SJeff Roberson if (arg->flags & UMA_ZONE_INTERNAL) 9938355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_INTERNAL; 9948355f576SJeff Roberson 9958355f576SJeff Roberson if (arg->flags & UMA_ZONE_MALLOC) 9968355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_MALLOC; 9978355f576SJeff Roberson 9988355f576SJeff Roberson if (arg->flags & UMA_ZONE_NOFREE) 9998355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_NOFREE; 10008355f576SJeff Roberson 100118aa2de5SJeff Roberson if (arg->flags & UMA_ZONE_VM) 100218aa2de5SJeff Roberson zone->uz_flags |= UMA_ZFLAG_BUCKETCACHE; 100318aa2de5SJeff Roberson 10048355f576SJeff Roberson if (zone->uz_size > UMA_SLAB_SIZE) 10058355f576SJeff Roberson zone_large_init(zone); 10068355f576SJeff Roberson else 10078355f576SJeff Roberson zone_small_init(zone); 10088355f576SJeff Roberson 100928bc4419SJeff Roberson if (arg->flags & UMA_ZONE_MTXCLASS) 101028bc4419SJeff Roberson privlc = 1; 101128bc4419SJeff Roberson else 101228bc4419SJeff Roberson privlc = 0; 101328bc4419SJeff Roberson 10148355f576SJeff Roberson /* We do this so that the per cpu lock name is unique for each zone */ 10158355f576SJeff Roberson memcpy(zone->uz_lname, "PCPU ", 5); 10168355f576SJeff Roberson cplen = min(strlen(zone->uz_name) + 1, LOCKNAME_LEN - 6); 10178355f576SJeff Roberson memcpy(zone->uz_lname+5, zone->uz_name, cplen); 10188355f576SJeff Roberson zone->uz_lname[LOCKNAME_LEN - 1] = '\0'; 10198355f576SJeff Roberson 10208355f576SJeff Roberson /* 10218355f576SJeff Roberson * If we're putting the slab header in the actual page we need to 10228355f576SJeff Roberson * figure out where in each page it goes. This calculates a right 10238355f576SJeff Roberson * justified offset into the memory on a ALIGN_PTR boundary. 10248355f576SJeff Roberson */ 10258355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_OFFPAGE)) { 10268355f576SJeff Roberson int totsize; 10278355f576SJeff Roberson int waste; 10288355f576SJeff Roberson 10298355f576SJeff Roberson /* Size of the slab struct and free list */ 10308355f576SJeff Roberson totsize = sizeof(struct uma_slab) + zone->uz_ipers; 10318355f576SJeff Roberson if (totsize & UMA_ALIGN_PTR) 10328355f576SJeff Roberson totsize = (totsize & ~UMA_ALIGN_PTR) + 10338355f576SJeff Roberson (UMA_ALIGN_PTR + 1); 10348355f576SJeff Roberson zone->uz_pgoff = UMA_SLAB_SIZE - totsize; 10358355f576SJeff Roberson 10368355f576SJeff Roberson waste = zone->uz_pgoff; 10378355f576SJeff Roberson waste -= (zone->uz_ipers * zone->uz_rsize); 10388355f576SJeff Roberson 10398355f576SJeff Roberson /* 10408355f576SJeff Roberson * This calculates how much space we have for cache line size 10418355f576SJeff Roberson * optimizations. It works by offseting each slab slightly. 10428355f576SJeff Roberson * Currently it breaks on x86, and so it is disabled. 10438355f576SJeff Roberson */ 10448355f576SJeff Roberson 10458355f576SJeff Roberson if (zone->uz_align < UMA_CACHE_INC && waste > UMA_CACHE_INC) { 10468355f576SJeff Roberson zone->uz_cachemax = waste - UMA_CACHE_INC; 10478355f576SJeff Roberson zone->uz_cacheoff = 0; 10488355f576SJeff Roberson } 10498355f576SJeff Roberson 10508355f576SJeff Roberson totsize = zone->uz_pgoff + sizeof(struct uma_slab) 10518355f576SJeff Roberson + zone->uz_ipers; 10528355f576SJeff Roberson /* I don't think it's possible, but I'll make sure anyway */ 10538355f576SJeff Roberson if (totsize > UMA_SLAB_SIZE) { 10548355f576SJeff Roberson printf("zone %s ipers %d rsize %d size %d\n", 10558355f576SJeff Roberson zone->uz_name, zone->uz_ipers, zone->uz_rsize, 10568355f576SJeff Roberson zone->uz_size); 10578355f576SJeff Roberson panic("UMA slab won't fit.\n"); 10588355f576SJeff Roberson } 10598355f576SJeff Roberson } 10608355f576SJeff Roberson 106199571dc3SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_HASH) 106299571dc3SJeff Roberson hash_alloc(&zone->uz_hash); 106399571dc3SJeff Roberson 10648355f576SJeff Roberson #ifdef UMA_DEBUG 10658355f576SJeff Roberson printf("%s(%p) size = %d ipers = %d ppera = %d pgoff = %d\n", 10668355f576SJeff Roberson zone->uz_name, zone, 10678355f576SJeff Roberson zone->uz_size, zone->uz_ipers, 10688355f576SJeff Roberson zone->uz_ppera, zone->uz_pgoff); 10698355f576SJeff Roberson #endif 107028bc4419SJeff Roberson ZONE_LOCK_INIT(zone, privlc); 10718355f576SJeff Roberson 10728355f576SJeff Roberson mtx_lock(&uma_mtx); 10738355f576SJeff Roberson LIST_INSERT_HEAD(&uma_zones, zone, uz_link); 10748355f576SJeff Roberson mtx_unlock(&uma_mtx); 10758355f576SJeff Roberson 10768355f576SJeff Roberson /* 10778355f576SJeff Roberson * Some internal zones don't have room allocated for the per cpu 10788355f576SJeff Roberson * caches. If we're internal, bail out here. 10798355f576SJeff Roberson */ 10808355f576SJeff Roberson 10818355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 10828355f576SJeff Roberson return; 10838355f576SJeff Roberson 10848355f576SJeff Roberson if (zone->uz_ipers < UMA_BUCKET_SIZE) 1085a553d4b8SJeff Roberson zone->uz_count = zone->uz_ipers - 1; 10868355f576SJeff Roberson else 1087a553d4b8SJeff Roberson zone->uz_count = UMA_BUCKET_SIZE - 1; 1088a553d4b8SJeff Roberson 1089a553d4b8SJeff Roberson for (cpu = 0; cpu < maxcpu; cpu++) 109028bc4419SJeff Roberson CPU_LOCK_INIT(zone, cpu, privlc); 10918355f576SJeff Roberson } 10928355f576SJeff Roberson 10938355f576SJeff Roberson /* 10949c2cd7e5SJeff Roberson * Zone header dtor. This frees all data, destroys locks, frees the hash table 10959c2cd7e5SJeff Roberson * and removes the zone from the global list. 10969c2cd7e5SJeff Roberson * 10979c2cd7e5SJeff Roberson * Arguments/Returns follow uma_dtor specifications 10989c2cd7e5SJeff Roberson * udata unused 10999c2cd7e5SJeff Roberson */ 11009c2cd7e5SJeff Roberson 11019c2cd7e5SJeff Roberson static void 11029c2cd7e5SJeff Roberson zone_dtor(void *arg, int size, void *udata) 11039c2cd7e5SJeff Roberson { 11049c2cd7e5SJeff Roberson uma_zone_t zone; 11059c2cd7e5SJeff Roberson int cpu; 11069c2cd7e5SJeff Roberson 11079c2cd7e5SJeff Roberson zone = (uma_zone_t)arg; 11089c2cd7e5SJeff Roberson 11099c2cd7e5SJeff Roberson ZONE_LOCK(zone); 11109c2cd7e5SJeff Roberson zone->uz_wssize = 0; 11119c2cd7e5SJeff Roberson ZONE_UNLOCK(zone); 11129c2cd7e5SJeff Roberson 111317b9cc49SJeff Roberson mtx_lock(&uma_mtx); 111417b9cc49SJeff Roberson LIST_REMOVE(zone, uz_link); 11159c2cd7e5SJeff Roberson zone_drain(zone); 111617b9cc49SJeff Roberson mtx_unlock(&uma_mtx); 111717b9cc49SJeff Roberson 11189c2cd7e5SJeff Roberson ZONE_LOCK(zone); 11199c2cd7e5SJeff Roberson if (zone->uz_free != 0) 11209c2cd7e5SJeff Roberson printf("Zone %s was not empty. Lost %d pages of memory.\n", 11219c2cd7e5SJeff Roberson zone->uz_name, zone->uz_pages); 11229c2cd7e5SJeff Roberson 1123f97d6ce3SIan Dowse if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) 11249c2cd7e5SJeff Roberson for (cpu = 0; cpu < maxcpu; cpu++) 11259c2cd7e5SJeff Roberson CPU_LOCK_FINI(zone, cpu); 11269c2cd7e5SJeff Roberson 11279c2cd7e5SJeff Roberson ZONE_UNLOCK(zone); 11280aef6126SJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_OFFPAGE) != 0) 11290aef6126SJeff Roberson hash_free(&zone->uz_hash); 11300aef6126SJeff Roberson 11319c2cd7e5SJeff Roberson ZONE_LOCK_FINI(zone); 11329c2cd7e5SJeff Roberson } 11339c2cd7e5SJeff Roberson /* 11348355f576SJeff Roberson * Traverses every zone in the system and calls a callback 11358355f576SJeff Roberson * 11368355f576SJeff Roberson * Arguments: 11378355f576SJeff Roberson * zfunc A pointer to a function which accepts a zone 11388355f576SJeff Roberson * as an argument. 11398355f576SJeff Roberson * 11408355f576SJeff Roberson * Returns: 11418355f576SJeff Roberson * Nothing 11428355f576SJeff Roberson */ 11438355f576SJeff Roberson static void 11448355f576SJeff Roberson zone_foreach(void (*zfunc)(uma_zone_t)) 11458355f576SJeff Roberson { 11468355f576SJeff Roberson uma_zone_t zone; 11478355f576SJeff Roberson 11488355f576SJeff Roberson mtx_lock(&uma_mtx); 11498355f576SJeff Roberson LIST_FOREACH(zone, &uma_zones, uz_link) { 11508355f576SJeff Roberson zfunc(zone); 11518355f576SJeff Roberson } 11528355f576SJeff Roberson mtx_unlock(&uma_mtx); 11538355f576SJeff Roberson } 11548355f576SJeff Roberson 11558355f576SJeff Roberson /* Public functions */ 11568355f576SJeff Roberson /* See uma.h */ 11578355f576SJeff Roberson void 11588355f576SJeff Roberson uma_startup(void *bootmem) 11598355f576SJeff Roberson { 11608355f576SJeff Roberson struct uma_zctor_args args; 11618355f576SJeff Roberson uma_slab_t slab; 11628355f576SJeff Roberson int slabsize; 11638355f576SJeff Roberson int i; 11648355f576SJeff Roberson 11658355f576SJeff Roberson #ifdef UMA_DEBUG 11668355f576SJeff Roberson printf("Creating uma zone headers zone.\n"); 11678355f576SJeff Roberson #endif 11688355f576SJeff Roberson #ifdef SMP 11698355f576SJeff Roberson maxcpu = mp_maxid + 1; 11708355f576SJeff Roberson #else 11718355f576SJeff Roberson maxcpu = 1; 11728355f576SJeff Roberson #endif 11738355f576SJeff Roberson #ifdef UMA_DEBUG 11748355f576SJeff Roberson printf("Max cpu = %d, mp_maxid = %d\n", maxcpu, mp_maxid); 11758355f576SJeff Roberson Debugger("stop"); 11768355f576SJeff Roberson #endif 11776008862bSJohn Baldwin mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF); 11788355f576SJeff Roberson /* "manually" Create the initial zone */ 11798355f576SJeff Roberson args.name = "UMA Zones"; 11808355f576SJeff Roberson args.size = sizeof(struct uma_zone) + 11818355f576SJeff Roberson (sizeof(struct uma_cache) * (maxcpu - 1)); 11828355f576SJeff Roberson args.ctor = zone_ctor; 11839c2cd7e5SJeff Roberson args.dtor = zone_dtor; 11848355f576SJeff Roberson args.uminit = zero_init; 11858355f576SJeff Roberson args.fini = NULL; 11868355f576SJeff Roberson args.align = 32 - 1; 11878355f576SJeff Roberson args.flags = UMA_ZONE_INTERNAL; 11888355f576SJeff Roberson /* The initial zone has no Per cpu queues so it's smaller */ 11898355f576SJeff Roberson zone_ctor(zones, sizeof(struct uma_zone), &args); 11908355f576SJeff Roberson 11918355f576SJeff Roberson #ifdef UMA_DEBUG 11928355f576SJeff Roberson printf("Filling boot free list.\n"); 11938355f576SJeff Roberson #endif 11948355f576SJeff Roberson for (i = 0; i < UMA_BOOT_PAGES; i++) { 11958355f576SJeff Roberson slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE)); 11968355f576SJeff Roberson slab->us_data = (u_int8_t *)slab; 11978355f576SJeff Roberson slab->us_flags = UMA_SLAB_BOOT; 11988355f576SJeff Roberson LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link); 11998355f576SJeff Roberson uma_boot_free++; 12008355f576SJeff Roberson } 12018355f576SJeff Roberson 12028355f576SJeff Roberson #ifdef UMA_DEBUG 12038355f576SJeff Roberson printf("Creating slab zone.\n"); 12048355f576SJeff Roberson #endif 12058355f576SJeff Roberson 12068355f576SJeff Roberson /* 12078355f576SJeff Roberson * This is the max number of free list items we'll have with 12088355f576SJeff Roberson * offpage slabs. 12098355f576SJeff Roberson */ 12108355f576SJeff Roberson 12118355f576SJeff Roberson slabsize = UMA_SLAB_SIZE - sizeof(struct uma_slab); 12128355f576SJeff Roberson slabsize /= UMA_MAX_WASTE; 12138355f576SJeff Roberson slabsize++; /* In case there it's rounded */ 12148355f576SJeff Roberson slabsize += sizeof(struct uma_slab); 12158355f576SJeff Roberson 12168355f576SJeff Roberson /* Now make a zone for slab headers */ 12178355f576SJeff Roberson slabzone = uma_zcreate("UMA Slabs", 12188355f576SJeff Roberson slabsize, 12198355f576SJeff Roberson NULL, NULL, NULL, NULL, 12208355f576SJeff Roberson UMA_ALIGN_PTR, UMA_ZONE_INTERNAL); 12218355f576SJeff Roberson 12228355f576SJeff Roberson hashzone = uma_zcreate("UMA Hash", 12238355f576SJeff Roberson sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 12248355f576SJeff Roberson NULL, NULL, NULL, NULL, 12258355f576SJeff Roberson UMA_ALIGN_PTR, UMA_ZONE_INTERNAL); 12268355f576SJeff Roberson 12278355f576SJeff Roberson bucketzone = uma_zcreate("UMA Buckets", sizeof(struct uma_bucket), 12288355f576SJeff Roberson NULL, NULL, NULL, NULL, 12298355f576SJeff Roberson UMA_ALIGN_PTR, UMA_ZONE_INTERNAL); 12308355f576SJeff Roberson 12318355f576SJeff Roberson 12328355f576SJeff Roberson #ifdef UMA_DEBUG 12338355f576SJeff Roberson printf("UMA startup complete.\n"); 12348355f576SJeff Roberson #endif 12358355f576SJeff Roberson } 12368355f576SJeff Roberson 12378355f576SJeff Roberson /* see uma.h */ 12388355f576SJeff Roberson void 123999571dc3SJeff Roberson uma_startup2(void) 12408355f576SJeff Roberson { 12418355f576SJeff Roberson booted = 1; 124286bbae32SJeff Roberson bucket_enable(); 12438355f576SJeff Roberson #ifdef UMA_DEBUG 12448355f576SJeff Roberson printf("UMA startup2 complete.\n"); 12458355f576SJeff Roberson #endif 12468355f576SJeff Roberson } 12478355f576SJeff Roberson 12488355f576SJeff Roberson /* 12498355f576SJeff Roberson * Initialize our callout handle 12508355f576SJeff Roberson * 12518355f576SJeff Roberson */ 12528355f576SJeff Roberson 12538355f576SJeff Roberson static void 12548355f576SJeff Roberson uma_startup3(void) 12558355f576SJeff Roberson { 12568355f576SJeff Roberson #ifdef UMA_DEBUG 12578355f576SJeff Roberson printf("Starting callout.\n"); 12588355f576SJeff Roberson #endif 12598355f576SJeff Roberson callout_init(&uma_callout, 0); 12608355f576SJeff Roberson callout_reset(&uma_callout, UMA_WORKING_TIME * hz, uma_timeout, NULL); 12618355f576SJeff Roberson #ifdef UMA_DEBUG 12628355f576SJeff Roberson printf("UMA startup3 complete.\n"); 12638355f576SJeff Roberson #endif 12648355f576SJeff Roberson } 12658355f576SJeff Roberson 12668355f576SJeff Roberson /* See uma.h */ 12678355f576SJeff Roberson uma_zone_t 1268c3bdc05fSAndrew R. Reiter uma_zcreate(char *name, size_t size, uma_ctor ctor, uma_dtor dtor, 1269c3bdc05fSAndrew R. Reiter uma_init uminit, uma_fini fini, int align, u_int16_t flags) 12708355f576SJeff Roberson 12718355f576SJeff Roberson { 12728355f576SJeff Roberson struct uma_zctor_args args; 12738355f576SJeff Roberson 12748355f576SJeff Roberson /* This stuff is essential for the zone ctor */ 12758355f576SJeff Roberson args.name = name; 12768355f576SJeff Roberson args.size = size; 12778355f576SJeff Roberson args.ctor = ctor; 12788355f576SJeff Roberson args.dtor = dtor; 12798355f576SJeff Roberson args.uminit = uminit; 12808355f576SJeff Roberson args.fini = fini; 12818355f576SJeff Roberson args.align = align; 12828355f576SJeff Roberson args.flags = flags; 12838355f576SJeff Roberson 1284bbee39c6SJeff Roberson return (uma_zalloc_internal(zones, &args, M_WAITOK)); 12858355f576SJeff Roberson } 12868355f576SJeff Roberson 12878355f576SJeff Roberson /* See uma.h */ 12889c2cd7e5SJeff Roberson void 12899c2cd7e5SJeff Roberson uma_zdestroy(uma_zone_t zone) 12909c2cd7e5SJeff Roberson { 12919c2cd7e5SJeff Roberson uma_zfree_internal(zones, zone, NULL, 0); 12929c2cd7e5SJeff Roberson } 12939c2cd7e5SJeff Roberson 12949c2cd7e5SJeff Roberson /* See uma.h */ 12958355f576SJeff Roberson void * 12962cc35ff9SJeff Roberson uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 12978355f576SJeff Roberson { 12988355f576SJeff Roberson void *item; 12998355f576SJeff Roberson uma_cache_t cache; 13008355f576SJeff Roberson uma_bucket_t bucket; 13018355f576SJeff Roberson int cpu; 13028355f576SJeff Roberson 13038355f576SJeff Roberson /* This is the fast path allocation */ 13048355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC_1 13058355f576SJeff Roberson printf("Allocating one item from %s(%p)\n", zone->uz_name, zone); 13068355f576SJeff Roberson #endif 1307a553d4b8SJeff Roberson 13084c1cc01cSJohn Baldwin if (!(flags & M_NOWAIT)) { 13094c1cc01cSJohn Baldwin KASSERT(curthread->td_intr_nesting_level == 0, 13104c1cc01cSJohn Baldwin ("malloc(M_WAITOK) in interrupt context")); 13114c1cc01cSJohn Baldwin WITNESS_SLEEP(1, NULL); 13124c1cc01cSJohn Baldwin } 13134c1cc01cSJohn Baldwin 1314a553d4b8SJeff Roberson zalloc_restart: 13158355f576SJeff Roberson cpu = PCPU_GET(cpuid); 13168355f576SJeff Roberson CPU_LOCK(zone, cpu); 13178355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 13188355f576SJeff Roberson 13198355f576SJeff Roberson zalloc_start: 13208355f576SJeff Roberson bucket = cache->uc_allocbucket; 13218355f576SJeff Roberson 13228355f576SJeff Roberson if (bucket) { 13238355f576SJeff Roberson if (bucket->ub_ptr > -1) { 13248355f576SJeff Roberson item = bucket->ub_bucket[bucket->ub_ptr]; 13258355f576SJeff Roberson #ifdef INVARIANTS 13268355f576SJeff Roberson bucket->ub_bucket[bucket->ub_ptr] = NULL; 13278355f576SJeff Roberson #endif 13288355f576SJeff Roberson bucket->ub_ptr--; 13298355f576SJeff Roberson KASSERT(item != NULL, 13308355f576SJeff Roberson ("uma_zalloc: Bucket pointer mangled.")); 13318355f576SJeff Roberson cache->uc_allocs++; 1332639c9550SJeff Roberson #ifdef INVARIANTS 1333639c9550SJeff Roberson uma_dbg_alloc(zone, NULL, item); 1334639c9550SJeff Roberson #endif 1335b9ba8931SJeff Roberson CPU_UNLOCK(zone, cpu); 13368355f576SJeff Roberson if (zone->uz_ctor) 13378355f576SJeff Roberson zone->uz_ctor(item, zone->uz_size, udata); 13382cc35ff9SJeff Roberson if (flags & M_ZERO) 13392cc35ff9SJeff Roberson bzero(item, zone->uz_size); 13408355f576SJeff Roberson return (item); 13418355f576SJeff Roberson } else if (cache->uc_freebucket) { 13428355f576SJeff Roberson /* 13438355f576SJeff Roberson * We have run out of items in our allocbucket. 13448355f576SJeff Roberson * See if we can switch with our free bucket. 13458355f576SJeff Roberson */ 13468355f576SJeff Roberson if (cache->uc_freebucket->ub_ptr > -1) { 13478355f576SJeff Roberson uma_bucket_t swap; 13488355f576SJeff Roberson 13498355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 13508355f576SJeff Roberson printf("uma_zalloc: Swapping empty with alloc.\n"); 13518355f576SJeff Roberson #endif 13528355f576SJeff Roberson swap = cache->uc_freebucket; 13538355f576SJeff Roberson cache->uc_freebucket = cache->uc_allocbucket; 13548355f576SJeff Roberson cache->uc_allocbucket = swap; 13558355f576SJeff Roberson 13568355f576SJeff Roberson goto zalloc_start; 13578355f576SJeff Roberson } 13588355f576SJeff Roberson } 13598355f576SJeff Roberson } 1360a553d4b8SJeff Roberson ZONE_LOCK(zone); 1361a553d4b8SJeff Roberson /* Since we have locked the zone we may as well send back our stats */ 1362a553d4b8SJeff Roberson zone->uz_allocs += cache->uc_allocs; 1363a553d4b8SJeff Roberson cache->uc_allocs = 0; 13648355f576SJeff Roberson 1365a553d4b8SJeff Roberson /* Our old one is now a free bucket */ 1366a553d4b8SJeff Roberson if (cache->uc_allocbucket) { 1367a553d4b8SJeff Roberson KASSERT(cache->uc_allocbucket->ub_ptr == -1, 1368a553d4b8SJeff Roberson ("uma_zalloc_arg: Freeing a non free bucket.")); 1369a553d4b8SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_bucket, 1370a553d4b8SJeff Roberson cache->uc_allocbucket, ub_link); 1371a553d4b8SJeff Roberson cache->uc_allocbucket = NULL; 1372a553d4b8SJeff Roberson } 13738355f576SJeff Roberson 1374a553d4b8SJeff Roberson /* Check the free list for a new alloc bucket */ 1375a553d4b8SJeff Roberson if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 1376a553d4b8SJeff Roberson KASSERT(bucket->ub_ptr != -1, 1377a553d4b8SJeff Roberson ("uma_zalloc_arg: Returning an empty bucket.")); 13788355f576SJeff Roberson 1379a553d4b8SJeff Roberson LIST_REMOVE(bucket, ub_link); 1380a553d4b8SJeff Roberson cache->uc_allocbucket = bucket; 1381a553d4b8SJeff Roberson ZONE_UNLOCK(zone); 13828355f576SJeff Roberson goto zalloc_start; 1383a553d4b8SJeff Roberson } 1384bbee39c6SJeff Roberson /* We are no longer associated with this cpu!!! */ 1385bbee39c6SJeff Roberson CPU_UNLOCK(zone, cpu); 1386bbee39c6SJeff Roberson 1387a553d4b8SJeff Roberson /* Bump up our uz_count so we get here less */ 1388a553d4b8SJeff Roberson if (zone->uz_count < UMA_BUCKET_SIZE - 1) 1389a553d4b8SJeff Roberson zone->uz_count++; 1390a553d4b8SJeff Roberson 13918355f576SJeff Roberson /* 1392a553d4b8SJeff Roberson * Now lets just fill a bucket and put it on the free list. If that 1393a553d4b8SJeff Roberson * works we'll restart the allocation from the begining. 1394bbee39c6SJeff Roberson */ 1395bbee39c6SJeff Roberson 1396bbee39c6SJeff Roberson if (uma_zalloc_bucket(zone, flags)) { 1397bbee39c6SJeff Roberson ZONE_UNLOCK(zone); 1398bbee39c6SJeff Roberson goto zalloc_restart; 1399bbee39c6SJeff Roberson } 1400bbee39c6SJeff Roberson ZONE_UNLOCK(zone); 1401bbee39c6SJeff Roberson /* 1402bbee39c6SJeff Roberson * We may not be able to get a bucket so return an actual item. 1403bbee39c6SJeff Roberson */ 1404bbee39c6SJeff Roberson #ifdef UMA_DEBUG 1405bbee39c6SJeff Roberson printf("uma_zalloc_arg: Bucketzone returned NULL\n"); 1406bbee39c6SJeff Roberson #endif 1407bbee39c6SJeff Roberson 1408bbee39c6SJeff Roberson return (uma_zalloc_internal(zone, udata, flags)); 1409bbee39c6SJeff Roberson } 1410bbee39c6SJeff Roberson 1411bbee39c6SJeff Roberson static uma_slab_t 1412bbee39c6SJeff Roberson uma_zone_slab(uma_zone_t zone, int flags) 1413bbee39c6SJeff Roberson { 1414bbee39c6SJeff Roberson uma_slab_t slab; 1415bbee39c6SJeff Roberson 1416bbee39c6SJeff Roberson /* 1417bbee39c6SJeff Roberson * This is to prevent us from recursively trying to allocate 1418bbee39c6SJeff Roberson * buckets. The problem is that if an allocation forces us to 1419bbee39c6SJeff Roberson * grab a new bucket we will call page_alloc, which will go off 1420bbee39c6SJeff Roberson * and cause the vm to allocate vm_map_entries. If we need new 1421bbee39c6SJeff Roberson * buckets there too we will recurse in kmem_alloc and bad 1422bbee39c6SJeff Roberson * things happen. So instead we return a NULL bucket, and make 1423bbee39c6SJeff Roberson * the code that allocates buckets smart enough to deal with it 1424bbee39c6SJeff Roberson */ 1425bbee39c6SJeff Roberson if (zone == bucketzone && zone->uz_recurse != 0) 1426bbee39c6SJeff Roberson return (NULL); 1427bbee39c6SJeff Roberson 1428bbee39c6SJeff Roberson slab = NULL; 1429bbee39c6SJeff Roberson 1430bbee39c6SJeff Roberson for (;;) { 1431bbee39c6SJeff Roberson /* 1432bbee39c6SJeff Roberson * Find a slab with some space. Prefer slabs that are partially 1433bbee39c6SJeff Roberson * used over those that are totally full. This helps to reduce 1434bbee39c6SJeff Roberson * fragmentation. 1435bbee39c6SJeff Roberson */ 1436bbee39c6SJeff Roberson if (zone->uz_free != 0) { 1437bbee39c6SJeff Roberson if (!LIST_EMPTY(&zone->uz_part_slab)) { 1438bbee39c6SJeff Roberson slab = LIST_FIRST(&zone->uz_part_slab); 1439bbee39c6SJeff Roberson } else { 1440bbee39c6SJeff Roberson slab = LIST_FIRST(&zone->uz_free_slab); 1441bbee39c6SJeff Roberson LIST_REMOVE(slab, us_link); 1442bbee39c6SJeff Roberson LIST_INSERT_HEAD(&zone->uz_part_slab, slab, 1443bbee39c6SJeff Roberson us_link); 1444bbee39c6SJeff Roberson } 1445bbee39c6SJeff Roberson return (slab); 1446bbee39c6SJeff Roberson } 1447bbee39c6SJeff Roberson 1448bbee39c6SJeff Roberson /* 1449bbee39c6SJeff Roberson * M_NOVM means don't ask at all! 1450bbee39c6SJeff Roberson */ 1451bbee39c6SJeff Roberson if (flags & M_NOVM) 1452bbee39c6SJeff Roberson break; 1453bbee39c6SJeff Roberson 1454bbee39c6SJeff Roberson if (zone->uz_maxpages && 1455bbee39c6SJeff Roberson zone->uz_pages >= zone->uz_maxpages) { 1456bbee39c6SJeff Roberson zone->uz_flags |= UMA_ZFLAG_FULL; 1457bbee39c6SJeff Roberson 1458bbee39c6SJeff Roberson if (flags & M_WAITOK) 1459bbee39c6SJeff Roberson msleep(zone, &zone->uz_lock, PVM, "zonelimit", 0); 1460bbee39c6SJeff Roberson else 1461bbee39c6SJeff Roberson break; 1462bbee39c6SJeff Roberson continue; 1463bbee39c6SJeff Roberson } 1464bbee39c6SJeff Roberson zone->uz_recurse++; 1465bbee39c6SJeff Roberson slab = slab_zalloc(zone, flags); 1466bbee39c6SJeff Roberson zone->uz_recurse--; 1467bbee39c6SJeff Roberson /* 1468bbee39c6SJeff Roberson * If we got a slab here it's safe to mark it partially used 1469bbee39c6SJeff Roberson * and return. We assume that the caller is going to remove 1470bbee39c6SJeff Roberson * at least one item. 1471bbee39c6SJeff Roberson */ 1472bbee39c6SJeff Roberson if (slab) { 1473bbee39c6SJeff Roberson LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); 1474bbee39c6SJeff Roberson return (slab); 1475bbee39c6SJeff Roberson } 1476bbee39c6SJeff Roberson /* 1477bbee39c6SJeff Roberson * We might not have been able to get a slab but another cpu 1478bbee39c6SJeff Roberson * could have while we were unlocked. Check again before we 1479bbee39c6SJeff Roberson * fail. 1480bbee39c6SJeff Roberson */ 1481bbee39c6SJeff Roberson if ((flags & M_WAITOK) == 0) 1482bbee39c6SJeff Roberson flags |= M_NOVM; 1483bbee39c6SJeff Roberson } 1484bbee39c6SJeff Roberson return (slab); 1485bbee39c6SJeff Roberson } 1486bbee39c6SJeff Roberson 1487bbee39c6SJeff Roberson static __inline void * 1488bbee39c6SJeff Roberson uma_slab_alloc(uma_zone_t zone, uma_slab_t slab) 1489bbee39c6SJeff Roberson { 1490bbee39c6SJeff Roberson void *item; 1491bbee39c6SJeff Roberson u_int8_t freei; 1492bbee39c6SJeff Roberson 1493bbee39c6SJeff Roberson freei = slab->us_firstfree; 1494bbee39c6SJeff Roberson slab->us_firstfree = slab->us_freelist[freei]; 1495bbee39c6SJeff Roberson item = slab->us_data + (zone->uz_rsize * freei); 1496bbee39c6SJeff Roberson 1497bbee39c6SJeff Roberson slab->us_freecount--; 1498bbee39c6SJeff Roberson zone->uz_free--; 1499bbee39c6SJeff Roberson #ifdef INVARIANTS 1500bbee39c6SJeff Roberson uma_dbg_alloc(zone, slab, item); 1501bbee39c6SJeff Roberson #endif 1502bbee39c6SJeff Roberson /* Move this slab to the full list */ 1503bbee39c6SJeff Roberson if (slab->us_freecount == 0) { 1504bbee39c6SJeff Roberson LIST_REMOVE(slab, us_link); 1505bbee39c6SJeff Roberson LIST_INSERT_HEAD(&zone->uz_full_slab, slab, us_link); 1506bbee39c6SJeff Roberson } 1507bbee39c6SJeff Roberson 1508bbee39c6SJeff Roberson return (item); 1509bbee39c6SJeff Roberson } 1510bbee39c6SJeff Roberson 1511bbee39c6SJeff Roberson static int 1512bbee39c6SJeff Roberson uma_zalloc_bucket(uma_zone_t zone, int flags) 1513bbee39c6SJeff Roberson { 1514bbee39c6SJeff Roberson uma_bucket_t bucket; 1515bbee39c6SJeff Roberson uma_slab_t slab; 1516bbee39c6SJeff Roberson 1517bbee39c6SJeff Roberson /* 1518a553d4b8SJeff Roberson * Try this zone's free list first so we don't allocate extra buckets. 15198355f576SJeff Roberson */ 1520a553d4b8SJeff Roberson 1521bbee39c6SJeff Roberson if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 1522bbee39c6SJeff Roberson KASSERT(bucket->ub_ptr == -1, 1523bbee39c6SJeff Roberson ("uma_zalloc_bucket: Bucket on free list is not empty.")); 1524a553d4b8SJeff Roberson LIST_REMOVE(bucket, ub_link); 1525bbee39c6SJeff Roberson } else { 152618aa2de5SJeff Roberson int bflags; 152718aa2de5SJeff Roberson 152818aa2de5SJeff Roberson bflags = flags; 152918aa2de5SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_BUCKETCACHE) 153018aa2de5SJeff Roberson bflags |= M_NOVM; 153118aa2de5SJeff Roberson 1532bbee39c6SJeff Roberson ZONE_UNLOCK(zone); 1533a553d4b8SJeff Roberson bucket = uma_zalloc_internal(bucketzone, 1534bbee39c6SJeff Roberson NULL, bflags); 1535bbee39c6SJeff Roberson ZONE_LOCK(zone); 1536a553d4b8SJeff Roberson if (bucket != NULL) { 1537a553d4b8SJeff Roberson #ifdef INVARIANTS 1538a553d4b8SJeff Roberson bzero(bucket, bucketzone->uz_size); 1539a553d4b8SJeff Roberson #endif 1540a553d4b8SJeff Roberson bucket->ub_ptr = -1; 1541a553d4b8SJeff Roberson } 1542bbee39c6SJeff Roberson } 1543bbee39c6SJeff Roberson 1544bbee39c6SJeff Roberson if (bucket == NULL) 1545bbee39c6SJeff Roberson return (0); 1546bbee39c6SJeff Roberson 1547bbee39c6SJeff Roberson #ifdef SMP 1548a553d4b8SJeff Roberson /* 1549bbee39c6SJeff Roberson * This code is here to limit the number of simultaneous bucket fills 1550bbee39c6SJeff Roberson * for any given zone to the number of per cpu caches in this zone. This 1551bbee39c6SJeff Roberson * is done so that we don't allocate more memory than we really need. 1552a553d4b8SJeff Roberson */ 1553bbee39c6SJeff Roberson if (zone->uz_fills >= mp_ncpus) 1554bbee39c6SJeff Roberson goto done; 1555a553d4b8SJeff Roberson 1556bbee39c6SJeff Roberson #endif 1557bbee39c6SJeff Roberson zone->uz_fills++; 1558bbee39c6SJeff Roberson 1559bbee39c6SJeff Roberson /* Try to keep the buckets totally full */ 1560bbee39c6SJeff Roberson while ((slab = uma_zone_slab(zone, flags)) != NULL && 1561bbee39c6SJeff Roberson bucket->ub_ptr < zone->uz_count) { 1562bbee39c6SJeff Roberson while (slab->us_freecount && 1563bbee39c6SJeff Roberson bucket->ub_ptr < zone->uz_count) { 1564bbee39c6SJeff Roberson bucket->ub_bucket[++bucket->ub_ptr] = 1565bbee39c6SJeff Roberson uma_slab_alloc(zone, slab); 1566bbee39c6SJeff Roberson } 1567bbee39c6SJeff Roberson /* Don't block on the next fill */ 1568bbee39c6SJeff Roberson flags |= M_NOWAIT; 1569bbee39c6SJeff Roberson flags &= ~M_WAITOK; 15708355f576SJeff Roberson } 15718355f576SJeff Roberson 1572bbee39c6SJeff Roberson zone->uz_fills--; 1573bbee39c6SJeff Roberson 1574bbee39c6SJeff Roberson if (bucket->ub_ptr != -1) { 1575bbee39c6SJeff Roberson LIST_INSERT_HEAD(&zone->uz_full_bucket, 1576bbee39c6SJeff Roberson bucket, ub_link); 1577bbee39c6SJeff Roberson return (1); 1578bbee39c6SJeff Roberson } 1579bbee39c6SJeff Roberson #ifdef SMP 1580bbee39c6SJeff Roberson done: 1581bbee39c6SJeff Roberson #endif 1582bbee39c6SJeff Roberson uma_zfree_internal(bucketzone, bucket, NULL, 0); 1583bbee39c6SJeff Roberson 1584bbee39c6SJeff Roberson return (0); 1585bbee39c6SJeff Roberson } 15868355f576SJeff Roberson /* 1587bbee39c6SJeff Roberson * Allocates an item for an internal zone 15888355f576SJeff Roberson * 15898355f576SJeff Roberson * Arguments 15908355f576SJeff Roberson * zone The zone to alloc for. 15918355f576SJeff Roberson * udata The data to be passed to the constructor. 15922cc35ff9SJeff Roberson * flags M_WAITOK, M_NOWAIT, M_ZERO. 15938355f576SJeff Roberson * 15948355f576SJeff Roberson * Returns 15958355f576SJeff Roberson * NULL if there is no memory and M_NOWAIT is set 1596bbee39c6SJeff Roberson * An item if successful 15978355f576SJeff Roberson */ 15988355f576SJeff Roberson 15998355f576SJeff Roberson static void * 1600bbee39c6SJeff Roberson uma_zalloc_internal(uma_zone_t zone, void *udata, int flags) 16018355f576SJeff Roberson { 16028355f576SJeff Roberson uma_slab_t slab; 16038355f576SJeff Roberson void *item; 16048355f576SJeff Roberson 16058355f576SJeff Roberson item = NULL; 16068355f576SJeff Roberson 16078355f576SJeff Roberson /* 1608a553d4b8SJeff Roberson * This is to stop us from allocating per cpu buckets while we're 1609a553d4b8SJeff Roberson * running out of UMA_BOOT_PAGES. Otherwise, we would exhaust the 1610a553d4b8SJeff Roberson * boot pages. 16118355f576SJeff Roberson */ 16128355f576SJeff Roberson 161386bbae32SJeff Roberson if (bucketdisable && zone == bucketzone) 16148355f576SJeff Roberson return (NULL); 16158355f576SJeff Roberson 16168355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 16178355f576SJeff Roberson printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); 16188355f576SJeff Roberson #endif 16198355f576SJeff Roberson ZONE_LOCK(zone); 16208355f576SJeff Roberson 1621bbee39c6SJeff Roberson slab = uma_zone_slab(zone, flags); 1622bbee39c6SJeff Roberson if (slab == NULL) { 1623bce97791SJeff Roberson ZONE_UNLOCK(zone); 1624a553d4b8SJeff Roberson return (NULL); 1625bce97791SJeff Roberson } 1626a553d4b8SJeff Roberson 1627bbee39c6SJeff Roberson item = uma_slab_alloc(zone, slab); 16288355f576SJeff Roberson 16298355f576SJeff Roberson ZONE_UNLOCK(zone); 16308355f576SJeff Roberson 16313370c5bfSJeff Roberson if (zone->uz_ctor != NULL) 16328355f576SJeff Roberson zone->uz_ctor(item, zone->uz_size, udata); 16332cc35ff9SJeff Roberson if (flags & M_ZERO) 16342cc35ff9SJeff Roberson bzero(item, zone->uz_size); 16358355f576SJeff Roberson 16368355f576SJeff Roberson return (item); 16378355f576SJeff Roberson } 16388355f576SJeff Roberson 16398355f576SJeff Roberson /* See uma.h */ 16408355f576SJeff Roberson void 16418355f576SJeff Roberson uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 16428355f576SJeff Roberson { 16438355f576SJeff Roberson uma_cache_t cache; 16448355f576SJeff Roberson uma_bucket_t bucket; 16454741dcbfSJeff Roberson int bflags; 16468355f576SJeff Roberson int cpu; 16478355f576SJeff Roberson 16488355f576SJeff Roberson /* This is the fast path free */ 16498355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC_1 16508355f576SJeff Roberson printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone); 16518355f576SJeff Roberson #endif 1652af7f9b97SJeff Roberson /* 1653af7f9b97SJeff Roberson * The race here is acceptable. If we miss it we'll just have to wait 1654af7f9b97SJeff Roberson * a little longer for the limits to be reset. 1655af7f9b97SJeff Roberson */ 1656af7f9b97SJeff Roberson 1657af7f9b97SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_FULL) 1658af7f9b97SJeff Roberson goto zfree_internal; 1659af7f9b97SJeff Roberson 1660bba739abSJeff Roberson if (zone->uz_dtor) 1661bba739abSJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 1662bba739abSJeff Roberson 1663a553d4b8SJeff Roberson zfree_restart: 16648355f576SJeff Roberson cpu = PCPU_GET(cpuid); 16658355f576SJeff Roberson CPU_LOCK(zone, cpu); 16668355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 16678355f576SJeff Roberson 16688355f576SJeff Roberson zfree_start: 16698355f576SJeff Roberson bucket = cache->uc_freebucket; 16708355f576SJeff Roberson 16718355f576SJeff Roberson if (bucket) { 1672a553d4b8SJeff Roberson /* 1673a553d4b8SJeff Roberson * Do we have room in our bucket? It is OK for this uz count 1674a553d4b8SJeff Roberson * check to be slightly out of sync. 1675a553d4b8SJeff Roberson */ 1676a553d4b8SJeff Roberson 1677a553d4b8SJeff Roberson if (bucket->ub_ptr < zone->uz_count) { 16788355f576SJeff Roberson bucket->ub_ptr++; 16798355f576SJeff Roberson KASSERT(bucket->ub_bucket[bucket->ub_ptr] == NULL, 16808355f576SJeff Roberson ("uma_zfree: Freeing to non free bucket index.")); 16818355f576SJeff Roberson bucket->ub_bucket[bucket->ub_ptr] = item; 1682b9ba8931SJeff Roberson #ifdef INVARIANTS 1683b9ba8931SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_MALLOC) 1684b9ba8931SJeff Roberson uma_dbg_free(zone, udata, item); 1685b9ba8931SJeff Roberson else 1686b9ba8931SJeff Roberson uma_dbg_free(zone, NULL, item); 1687b9ba8931SJeff Roberson #endif 1688605cbd6aSJeff Roberson CPU_UNLOCK(zone, cpu); 16898355f576SJeff Roberson return; 16908355f576SJeff Roberson } else if (cache->uc_allocbucket) { 16918355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 16928355f576SJeff Roberson printf("uma_zfree: Swapping buckets.\n"); 16938355f576SJeff Roberson #endif 16948355f576SJeff Roberson /* 16958355f576SJeff Roberson * We have run out of space in our freebucket. 16968355f576SJeff Roberson * See if we can switch with our alloc bucket. 16978355f576SJeff Roberson */ 16988355f576SJeff Roberson if (cache->uc_allocbucket->ub_ptr < 16998355f576SJeff Roberson cache->uc_freebucket->ub_ptr) { 17008355f576SJeff Roberson uma_bucket_t swap; 17018355f576SJeff Roberson 17028355f576SJeff Roberson swap = cache->uc_freebucket; 17038355f576SJeff Roberson cache->uc_freebucket = cache->uc_allocbucket; 17048355f576SJeff Roberson cache->uc_allocbucket = swap; 17058355f576SJeff Roberson 17068355f576SJeff Roberson goto zfree_start; 17078355f576SJeff Roberson } 17088355f576SJeff Roberson } 17098355f576SJeff Roberson } 17108355f576SJeff Roberson 17118355f576SJeff Roberson /* 1712a553d4b8SJeff Roberson * We can get here for two reasons: 17138355f576SJeff Roberson * 17148355f576SJeff Roberson * 1) The buckets are NULL 1715a553d4b8SJeff Roberson * 2) The alloc and free buckets are both somewhat full. 17168355f576SJeff Roberson * 17178355f576SJeff Roberson */ 17188355f576SJeff Roberson 17198355f576SJeff Roberson ZONE_LOCK(zone); 17208355f576SJeff Roberson 17218355f576SJeff Roberson bucket = cache->uc_freebucket; 17228355f576SJeff Roberson cache->uc_freebucket = NULL; 17238355f576SJeff Roberson 17248355f576SJeff Roberson /* Can we throw this on the zone full list? */ 17258355f576SJeff Roberson if (bucket != NULL) { 17268355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 17278355f576SJeff Roberson printf("uma_zfree: Putting old bucket on the free list.\n"); 17288355f576SJeff Roberson #endif 17298355f576SJeff Roberson /* ub_ptr is pointing to the last free item */ 17308355f576SJeff Roberson KASSERT(bucket->ub_ptr != -1, 17318355f576SJeff Roberson ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n")); 17328355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_full_bucket, 17338355f576SJeff Roberson bucket, ub_link); 17348355f576SJeff Roberson } 1735a553d4b8SJeff Roberson if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 1736a553d4b8SJeff Roberson LIST_REMOVE(bucket, ub_link); 1737a553d4b8SJeff Roberson ZONE_UNLOCK(zone); 1738a553d4b8SJeff Roberson cache->uc_freebucket = bucket; 1739a553d4b8SJeff Roberson goto zfree_start; 1740a553d4b8SJeff Roberson } 1741a553d4b8SJeff Roberson /* We're done with this CPU now */ 1742a553d4b8SJeff Roberson CPU_UNLOCK(zone, cpu); 1743a553d4b8SJeff Roberson 1744a553d4b8SJeff Roberson /* And the zone.. */ 1745a553d4b8SJeff Roberson ZONE_UNLOCK(zone); 1746a553d4b8SJeff Roberson 17478355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 17488355f576SJeff Roberson printf("uma_zfree: Allocating new free bucket.\n"); 17498355f576SJeff Roberson #endif 17504741dcbfSJeff Roberson bflags = M_NOWAIT; 17514741dcbfSJeff Roberson 17524741dcbfSJeff Roberson if (zone->uz_flags & UMA_ZFLAG_BUCKETCACHE) 17534741dcbfSJeff Roberson bflags |= M_NOVM; 17548355f576SJeff Roberson #ifdef INVARIANTS 17554741dcbfSJeff Roberson bflags |= M_ZERO; 17568355f576SJeff Roberson #endif 17574741dcbfSJeff Roberson bucket = uma_zalloc_internal(bucketzone, 1758bbee39c6SJeff Roberson NULL, bflags); 17594741dcbfSJeff Roberson if (bucket) { 17608355f576SJeff Roberson bucket->ub_ptr = -1; 1761a553d4b8SJeff Roberson ZONE_LOCK(zone); 1762a553d4b8SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_bucket, 1763a553d4b8SJeff Roberson bucket, ub_link); 17648355f576SJeff Roberson ZONE_UNLOCK(zone); 1765a553d4b8SJeff Roberson goto zfree_restart; 17668355f576SJeff Roberson } 17678355f576SJeff Roberson 1768a553d4b8SJeff Roberson /* 1769a553d4b8SJeff Roberson * If nothing else caught this, we'll just do an internal free. 1770a553d4b8SJeff Roberson */ 17718355f576SJeff Roberson 1772af7f9b97SJeff Roberson zfree_internal: 1773af7f9b97SJeff Roberson 17748355f576SJeff Roberson uma_zfree_internal(zone, item, udata, 0); 17758355f576SJeff Roberson 17768355f576SJeff Roberson return; 17778355f576SJeff Roberson 17788355f576SJeff Roberson } 17798355f576SJeff Roberson 17808355f576SJeff Roberson /* 17818355f576SJeff Roberson * Frees an item to an INTERNAL zone or allocates a free bucket 17828355f576SJeff Roberson * 17838355f576SJeff Roberson * Arguments: 17848355f576SJeff Roberson * zone The zone to free to 17858355f576SJeff Roberson * item The item we're freeing 17868355f576SJeff Roberson * udata User supplied data for the dtor 17878355f576SJeff Roberson * skip Skip the dtor, it was done in uma_zfree_arg 17888355f576SJeff Roberson */ 17898355f576SJeff Roberson 17908355f576SJeff Roberson static void 17918355f576SJeff Roberson uma_zfree_internal(uma_zone_t zone, void *item, void *udata, int skip) 17928355f576SJeff Roberson { 17938355f576SJeff Roberson uma_slab_t slab; 17948355f576SJeff Roberson u_int8_t *mem; 17958355f576SJeff Roberson u_int8_t freei; 17968355f576SJeff Roberson 1797bba739abSJeff Roberson if (!skip && zone->uz_dtor) 1798bba739abSJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 1799bba739abSJeff Roberson 18008355f576SJeff Roberson ZONE_LOCK(zone); 18018355f576SJeff Roberson 18028355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_MALLOC)) { 18038355f576SJeff Roberson mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); 180499571dc3SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_HASH) 18058355f576SJeff Roberson slab = hash_sfind(&zone->uz_hash, mem); 18068355f576SJeff Roberson else { 18078355f576SJeff Roberson mem += zone->uz_pgoff; 18088355f576SJeff Roberson slab = (uma_slab_t)mem; 18098355f576SJeff Roberson } 18108355f576SJeff Roberson } else { 18118355f576SJeff Roberson slab = (uma_slab_t)udata; 18128355f576SJeff Roberson } 18138355f576SJeff Roberson 18148355f576SJeff Roberson /* Do we need to remove from any lists? */ 18158355f576SJeff Roberson if (slab->us_freecount+1 == zone->uz_ipers) { 18168355f576SJeff Roberson LIST_REMOVE(slab, us_link); 18178355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_slab, slab, us_link); 18188355f576SJeff Roberson } else if (slab->us_freecount == 0) { 18198355f576SJeff Roberson LIST_REMOVE(slab, us_link); 18208355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); 18218355f576SJeff Roberson } 18228355f576SJeff Roberson 18238355f576SJeff Roberson /* Slab management stuff */ 18248355f576SJeff Roberson freei = ((unsigned long)item - (unsigned long)slab->us_data) 18258355f576SJeff Roberson / zone->uz_rsize; 18268355f576SJeff Roberson 1827639c9550SJeff Roberson #ifdef INVARIANTS 1828639c9550SJeff Roberson if (!skip) 1829639c9550SJeff Roberson uma_dbg_free(zone, slab, item); 18308355f576SJeff Roberson #endif 1831639c9550SJeff Roberson 18328355f576SJeff Roberson slab->us_freelist[freei] = slab->us_firstfree; 18338355f576SJeff Roberson slab->us_firstfree = freei; 18348355f576SJeff Roberson slab->us_freecount++; 18358355f576SJeff Roberson 18368355f576SJeff Roberson /* Zone statistics */ 18378355f576SJeff Roberson zone->uz_free++; 18388355f576SJeff Roberson 1839af7f9b97SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_FULL) { 1840af7f9b97SJeff Roberson if (zone->uz_pages < zone->uz_maxpages) 1841af7f9b97SJeff Roberson zone->uz_flags &= ~UMA_ZFLAG_FULL; 1842af7f9b97SJeff Roberson 1843af7f9b97SJeff Roberson /* We can handle one more allocation */ 1844af7f9b97SJeff Roberson wakeup_one(&zone); 1845af7f9b97SJeff Roberson } 1846af7f9b97SJeff Roberson 1847605cbd6aSJeff Roberson ZONE_UNLOCK(zone); 18488355f576SJeff Roberson } 18498355f576SJeff Roberson 18508355f576SJeff Roberson /* See uma.h */ 18518355f576SJeff Roberson void 1852736ee590SJeff Roberson uma_zone_set_max(uma_zone_t zone, int nitems) 1853736ee590SJeff Roberson { 1854736ee590SJeff Roberson ZONE_LOCK(zone); 1855736ee590SJeff Roberson if (zone->uz_ppera > 1) 1856af7f9b97SJeff Roberson zone->uz_maxpages = nitems * zone->uz_ppera; 1857736ee590SJeff Roberson else 1858736ee590SJeff Roberson zone->uz_maxpages = nitems / zone->uz_ipers; 185928bc4419SJeff Roberson 1860d4d6aee5SAndrew R. Reiter if (zone->uz_maxpages * zone->uz_ipers < nitems) 1861d4d6aee5SAndrew R. Reiter zone->uz_maxpages++; 186228bc4419SJeff Roberson 1863736ee590SJeff Roberson ZONE_UNLOCK(zone); 1864736ee590SJeff Roberson } 1865736ee590SJeff Roberson 1866736ee590SJeff Roberson /* See uma.h */ 1867736ee590SJeff Roberson void 18688355f576SJeff Roberson uma_zone_set_freef(uma_zone_t zone, uma_free freef) 18698355f576SJeff Roberson { 18708355f576SJeff Roberson ZONE_LOCK(zone); 18718355f576SJeff Roberson 18728355f576SJeff Roberson zone->uz_freef = freef; 18738355f576SJeff Roberson 18748355f576SJeff Roberson ZONE_UNLOCK(zone); 18758355f576SJeff Roberson } 18768355f576SJeff Roberson 18778355f576SJeff Roberson /* See uma.h */ 18788355f576SJeff Roberson void 18798355f576SJeff Roberson uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 18808355f576SJeff Roberson { 18818355f576SJeff Roberson ZONE_LOCK(zone); 18828355f576SJeff Roberson 18838355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_PRIVALLOC; 18848355f576SJeff Roberson zone->uz_allocf = allocf; 18858355f576SJeff Roberson 18868355f576SJeff Roberson ZONE_UNLOCK(zone); 18878355f576SJeff Roberson } 18888355f576SJeff Roberson 18898355f576SJeff Roberson /* See uma.h */ 18908355f576SJeff Roberson int 18918355f576SJeff Roberson uma_zone_set_obj(uma_zone_t zone, struct vm_object *obj, int count) 18928355f576SJeff Roberson { 18938355f576SJeff Roberson int pages; 18948355f576SJeff Roberson vm_offset_t kva; 18958355f576SJeff Roberson 18968355f576SJeff Roberson mtx_lock(&Giant); 18978355f576SJeff Roberson 18988355f576SJeff Roberson pages = count / zone->uz_ipers; 18998355f576SJeff Roberson 19008355f576SJeff Roberson if (pages * zone->uz_ipers < count) 19018355f576SJeff Roberson pages++; 1902a553d4b8SJeff Roberson 19038355f576SJeff Roberson kva = kmem_alloc_pageable(kernel_map, pages * UMA_SLAB_SIZE); 19048355f576SJeff Roberson 1905a553d4b8SJeff Roberson if (kva == 0) { 1906a553d4b8SJeff Roberson mtx_unlock(&Giant); 19078355f576SJeff Roberson return (0); 19088355f576SJeff Roberson } 19098355f576SJeff Roberson 19108355f576SJeff Roberson 1911a553d4b8SJeff Roberson if (obj == NULL) 1912a553d4b8SJeff Roberson obj = vm_object_allocate(OBJT_DEFAULT, 1913c7173f58SJeff Roberson pages); 19148355f576SJeff Roberson else 19158355f576SJeff Roberson _vm_object_allocate(OBJT_DEFAULT, 1916c7173f58SJeff Roberson pages, obj); 1917a553d4b8SJeff Roberson 1918a553d4b8SJeff Roberson ZONE_LOCK(zone); 1919a553d4b8SJeff Roberson zone->uz_kva = kva; 1920a553d4b8SJeff Roberson zone->uz_obj = obj; 1921a553d4b8SJeff Roberson zone->uz_maxpages = pages; 19228355f576SJeff Roberson 19238355f576SJeff Roberson zone->uz_allocf = obj_alloc; 19248355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_NOFREE | UMA_ZFLAG_PRIVALLOC; 19258355f576SJeff Roberson 19268355f576SJeff Roberson ZONE_UNLOCK(zone); 1927a553d4b8SJeff Roberson mtx_unlock(&Giant); 19288355f576SJeff Roberson 19298355f576SJeff Roberson return (1); 19308355f576SJeff Roberson } 19318355f576SJeff Roberson 19328355f576SJeff Roberson /* See uma.h */ 19338355f576SJeff Roberson void 19348355f576SJeff Roberson uma_prealloc(uma_zone_t zone, int items) 19358355f576SJeff Roberson { 19368355f576SJeff Roberson int slabs; 19378355f576SJeff Roberson uma_slab_t slab; 19388355f576SJeff Roberson 19398355f576SJeff Roberson ZONE_LOCK(zone); 19408355f576SJeff Roberson slabs = items / zone->uz_ipers; 19418355f576SJeff Roberson if (slabs * zone->uz_ipers < items) 19428355f576SJeff Roberson slabs++; 19438355f576SJeff Roberson 19448355f576SJeff Roberson while (slabs > 0) { 19458355f576SJeff Roberson slab = slab_zalloc(zone, M_WAITOK); 19468355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_slab, slab, us_link); 19478355f576SJeff Roberson slabs--; 19488355f576SJeff Roberson } 19498355f576SJeff Roberson ZONE_UNLOCK(zone); 19508355f576SJeff Roberson } 19518355f576SJeff Roberson 19528355f576SJeff Roberson /* See uma.h */ 19538355f576SJeff Roberson void 19548355f576SJeff Roberson uma_reclaim(void) 19558355f576SJeff Roberson { 19568355f576SJeff Roberson /* 19578355f576SJeff Roberson * You might think that the delay below would improve performance since 19588355f576SJeff Roberson * the allocator will give away memory that it may ask for immediately. 19598355f576SJeff Roberson * Really, it makes things worse, since cpu cycles are so much cheaper 19608355f576SJeff Roberson * than disk activity. 19618355f576SJeff Roberson */ 19628355f576SJeff Roberson #if 0 19638355f576SJeff Roberson static struct timeval tv = {0}; 19648355f576SJeff Roberson struct timeval now; 19658355f576SJeff Roberson getmicrouptime(&now); 19668355f576SJeff Roberson if (now.tv_sec > tv.tv_sec + 30) 19678355f576SJeff Roberson tv = now; 19688355f576SJeff Roberson else 19698355f576SJeff Roberson return; 19708355f576SJeff Roberson #endif 19718355f576SJeff Roberson #ifdef UMA_DEBUG 19728355f576SJeff Roberson printf("UMA: vm asked us to release pages!\n"); 19738355f576SJeff Roberson #endif 197486bbae32SJeff Roberson bucket_enable(); 19758355f576SJeff Roberson zone_foreach(zone_drain); 19768355f576SJeff Roberson 19778355f576SJeff Roberson /* 19788355f576SJeff Roberson * Some slabs may have been freed but this zone will be visited early 19798355f576SJeff Roberson * we visit again so that we can free pages that are empty once other 19808355f576SJeff Roberson * zones are drained. We have to do the same for buckets. 19818355f576SJeff Roberson */ 19828355f576SJeff Roberson zone_drain(slabzone); 19838355f576SJeff Roberson zone_drain(bucketzone); 19848355f576SJeff Roberson } 19858355f576SJeff Roberson 19868355f576SJeff Roberson void * 19878355f576SJeff Roberson uma_large_malloc(int size, int wait) 19888355f576SJeff Roberson { 19898355f576SJeff Roberson void *mem; 19908355f576SJeff Roberson uma_slab_t slab; 19918355f576SJeff Roberson u_int8_t flags; 19928355f576SJeff Roberson 1993bbee39c6SJeff Roberson slab = uma_zalloc_internal(slabzone, NULL, wait); 19948355f576SJeff Roberson if (slab == NULL) 19958355f576SJeff Roberson return (NULL); 19968355f576SJeff Roberson 19978355f576SJeff Roberson mem = page_alloc(NULL, size, &flags, wait); 19988355f576SJeff Roberson if (mem) { 199999571dc3SJeff Roberson vsetslab((vm_offset_t)mem, slab); 20008355f576SJeff Roberson slab->us_data = mem; 20018355f576SJeff Roberson slab->us_flags = flags | UMA_SLAB_MALLOC; 20028355f576SJeff Roberson slab->us_size = size; 20038355f576SJeff Roberson } else { 20048355f576SJeff Roberson uma_zfree_internal(slabzone, slab, NULL, 0); 20058355f576SJeff Roberson } 20068355f576SJeff Roberson 20078355f576SJeff Roberson 20088355f576SJeff Roberson return (mem); 20098355f576SJeff Roberson } 20108355f576SJeff Roberson 20118355f576SJeff Roberson void 20128355f576SJeff Roberson uma_large_free(uma_slab_t slab) 20138355f576SJeff Roberson { 201499571dc3SJeff Roberson vsetobj((vm_offset_t)slab->us_data, kmem_object); 20158355f576SJeff Roberson page_free(slab->us_data, slab->us_size, slab->us_flags); 20168355f576SJeff Roberson uma_zfree_internal(slabzone, slab, NULL, 0); 20178355f576SJeff Roberson } 20188355f576SJeff Roberson 20198355f576SJeff Roberson void 20208355f576SJeff Roberson uma_print_stats(void) 20218355f576SJeff Roberson { 20228355f576SJeff Roberson zone_foreach(uma_print_zone); 20238355f576SJeff Roberson } 20248355f576SJeff Roberson 20258355f576SJeff Roberson void 20268355f576SJeff Roberson uma_print_zone(uma_zone_t zone) 20278355f576SJeff Roberson { 20288355f576SJeff Roberson printf("%s(%p) size %d(%d) flags %d ipers %d ppera %d out %d free %d\n", 20298355f576SJeff Roberson zone->uz_name, zone, zone->uz_size, zone->uz_rsize, zone->uz_flags, 20308355f576SJeff Roberson zone->uz_ipers, zone->uz_ppera, 20318355f576SJeff Roberson (zone->uz_ipers * zone->uz_pages) - zone->uz_free, zone->uz_free); 20328355f576SJeff Roberson } 20338355f576SJeff Roberson 20348355f576SJeff Roberson /* 20358355f576SJeff Roberson * Sysctl handler for vm.zone 20368355f576SJeff Roberson * 20378355f576SJeff Roberson * stolen from vm_zone.c 20388355f576SJeff Roberson */ 20398355f576SJeff Roberson static int 20408355f576SJeff Roberson sysctl_vm_zone(SYSCTL_HANDLER_ARGS) 20418355f576SJeff Roberson { 20428355f576SJeff Roberson int error, len, cnt; 20438355f576SJeff Roberson const int linesize = 128; /* conservative */ 20448355f576SJeff Roberson int totalfree; 20458355f576SJeff Roberson char *tmpbuf, *offset; 20468355f576SJeff Roberson uma_zone_t z; 20478355f576SJeff Roberson char *p; 20488355f576SJeff Roberson 20498355f576SJeff Roberson cnt = 0; 20500da47b2fSJeff Roberson mtx_lock(&uma_mtx); 20518355f576SJeff Roberson LIST_FOREACH(z, &uma_zones, uz_link) 20528355f576SJeff Roberson cnt++; 20530da47b2fSJeff Roberson mtx_unlock(&uma_mtx); 20548355f576SJeff Roberson MALLOC(tmpbuf, char *, (cnt == 0 ? 1 : cnt) * linesize, 20558355f576SJeff Roberson M_TEMP, M_WAITOK); 20568355f576SJeff Roberson len = snprintf(tmpbuf, linesize, 20578355f576SJeff Roberson "\nITEM SIZE LIMIT USED FREE REQUESTS\n\n"); 20588355f576SJeff Roberson if (cnt == 0) 20598355f576SJeff Roberson tmpbuf[len - 1] = '\0'; 20608355f576SJeff Roberson error = SYSCTL_OUT(req, tmpbuf, cnt == 0 ? len-1 : len); 20618355f576SJeff Roberson if (error || cnt == 0) 20628355f576SJeff Roberson goto out; 20638355f576SJeff Roberson offset = tmpbuf; 2064f4af24d5SJeff Roberson mtx_lock(&uma_mtx); 20658355f576SJeff Roberson LIST_FOREACH(z, &uma_zones, uz_link) { 20668355f576SJeff Roberson if (cnt == 0) /* list may have changed size */ 20678355f576SJeff Roberson break; 20688355f576SJeff Roberson ZONE_LOCK(z); 20698355f576SJeff Roberson totalfree = z->uz_free + z->uz_cachefree; 20708355f576SJeff Roberson len = snprintf(offset, linesize, 20718355f576SJeff Roberson "%-12.12s %6.6u, %8.8u, %6.6u, %6.6u, %8.8llu\n", 20728355f576SJeff Roberson z->uz_name, z->uz_size, 20738355f576SJeff Roberson z->uz_maxpages * z->uz_ipers, 20748355f576SJeff Roberson (z->uz_ipers * (z->uz_pages / z->uz_ppera)) - totalfree, 20758355f576SJeff Roberson totalfree, 20768355f576SJeff Roberson (unsigned long long)z->uz_allocs); 20778355f576SJeff Roberson ZONE_UNLOCK(z); 20788355f576SJeff Roberson for (p = offset + 12; p > offset && *p == ' '; --p) 20798355f576SJeff Roberson /* nothing */ ; 20808355f576SJeff Roberson p[1] = ':'; 20818355f576SJeff Roberson cnt--; 20828355f576SJeff Roberson offset += len; 20838355f576SJeff Roberson } 2084f4af24d5SJeff Roberson mtx_unlock(&uma_mtx); 20858355f576SJeff Roberson *offset++ = '\0'; 20868355f576SJeff Roberson error = SYSCTL_OUT(req, tmpbuf, offset - tmpbuf); 20878355f576SJeff Roberson out: 20888355f576SJeff Roberson FREE(tmpbuf, M_TEMP); 20898355f576SJeff Roberson return (error); 20908355f576SJeff Roberson } 2091