18355f576SJeff Roberson /* 28355f576SJeff Roberson * Copyright (c) 2002, Jeffrey Roberson <jroberson@chesapeake.net> 38355f576SJeff Roberson * All rights reserved. 48355f576SJeff Roberson * 58355f576SJeff Roberson * Redistribution and use in source and binary forms, with or without 68355f576SJeff Roberson * modification, are permitted provided that the following conditions 78355f576SJeff Roberson * are met: 88355f576SJeff Roberson * 1. Redistributions of source code must retain the above copyright 98355f576SJeff Roberson * notice unmodified, this list of conditions, and the following 108355f576SJeff Roberson * disclaimer. 118355f576SJeff Roberson * 2. Redistributions in binary form must reproduce the above copyright 128355f576SJeff Roberson * notice, this list of conditions and the following disclaimer in the 138355f576SJeff Roberson * documentation and/or other materials provided with the distribution. 148355f576SJeff Roberson * 158355f576SJeff Roberson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 168355f576SJeff Roberson * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 178355f576SJeff Roberson * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 188355f576SJeff Roberson * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 198355f576SJeff Roberson * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 208355f576SJeff Roberson * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 218355f576SJeff Roberson * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 228355f576SJeff Roberson * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 238355f576SJeff Roberson * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 248355f576SJeff Roberson * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 258355f576SJeff Roberson * 268355f576SJeff Roberson * $FreeBSD$ 278355f576SJeff Roberson * 288355f576SJeff Roberson */ 298355f576SJeff Roberson 308355f576SJeff Roberson /* 318355f576SJeff Roberson * uma_core.c Implementation of the Universal Memory allocator 328355f576SJeff Roberson * 338355f576SJeff Roberson * This allocator is intended to replace the multitude of similar object caches 348355f576SJeff Roberson * in the standard FreeBSD kernel. The intent is to be flexible as well as 358355f576SJeff Roberson * effecient. A primary design goal is to return unused memory to the rest of 368355f576SJeff Roberson * the system. This will make the system as a whole more flexible due to the 378355f576SJeff Roberson * ability to move memory to subsystems which most need it instead of leaving 388355f576SJeff Roberson * pools of reserved memory unused. 398355f576SJeff Roberson * 408355f576SJeff Roberson * The basic ideas stem from similar slab/zone based allocators whose algorithms 418355f576SJeff Roberson * are well known. 428355f576SJeff Roberson * 438355f576SJeff Roberson */ 448355f576SJeff Roberson 458355f576SJeff Roberson /* 468355f576SJeff Roberson * TODO: 478355f576SJeff Roberson * - Improve memory usage for large allocations 488355f576SJeff Roberson * - Improve INVARIANTS (0xdeadc0de write out) 498355f576SJeff Roberson * - Investigate cache size adjustments 508355f576SJeff Roberson */ 518355f576SJeff Roberson 528355f576SJeff Roberson /* I should really use ktr.. */ 538355f576SJeff Roberson /* 548355f576SJeff Roberson #define UMA_DEBUG 1 558355f576SJeff Roberson #define UMA_DEBUG_ALLOC 1 568355f576SJeff Roberson #define UMA_DEBUG_ALLOC_1 1 578355f576SJeff Roberson */ 588355f576SJeff Roberson 598355f576SJeff Roberson 608355f576SJeff Roberson #include "opt_param.h" 618355f576SJeff Roberson #include <sys/param.h> 628355f576SJeff Roberson #include <sys/systm.h> 638355f576SJeff Roberson #include <sys/kernel.h> 648355f576SJeff Roberson #include <sys/types.h> 658355f576SJeff Roberson #include <sys/queue.h> 668355f576SJeff Roberson #include <sys/malloc.h> 678355f576SJeff Roberson #include <sys/lock.h> 688355f576SJeff Roberson #include <sys/sysctl.h> 698355f576SJeff Roberson #include <machine/types.h> 708355f576SJeff Roberson #include <sys/mutex.h> 718355f576SJeff Roberson #include <sys/smp.h> 728355f576SJeff Roberson 738355f576SJeff Roberson #include <vm/vm.h> 748355f576SJeff Roberson #include <vm/vm_object.h> 758355f576SJeff Roberson #include <vm/vm_page.h> 768355f576SJeff Roberson #include <vm/vm_param.h> 778355f576SJeff Roberson #include <vm/vm_map.h> 788355f576SJeff Roberson #include <vm/vm_kern.h> 798355f576SJeff Roberson #include <vm/vm_extern.h> 808355f576SJeff Roberson #include <vm/uma.h> 818355f576SJeff Roberson #include <vm/uma_int.h> 828355f576SJeff Roberson 838355f576SJeff Roberson /* 848355f576SJeff Roberson * This is the zone from which all zones are spawned. The idea is that even 858355f576SJeff Roberson * the zone heads are allocated from the allocator, so we use the bss section 868355f576SJeff Roberson * to bootstrap us. 878355f576SJeff Roberson */ 888355f576SJeff Roberson static struct uma_zone master_zone; 898355f576SJeff Roberson static uma_zone_t zones = &master_zone; 908355f576SJeff Roberson 918355f576SJeff Roberson /* This is the zone from which all of uma_slab_t's are allocated. */ 928355f576SJeff Roberson static uma_zone_t slabzone; 938355f576SJeff Roberson 948355f576SJeff Roberson /* 958355f576SJeff Roberson * The initial hash tables come out of this zone so they can be allocated 968355f576SJeff Roberson * prior to malloc coming up. 978355f576SJeff Roberson */ 988355f576SJeff Roberson static uma_zone_t hashzone; 998355f576SJeff Roberson 1008355f576SJeff Roberson /* 1018355f576SJeff Roberson * Zone that buckets come from. 1028355f576SJeff Roberson */ 1038355f576SJeff Roberson static uma_zone_t bucketzone; 1048355f576SJeff Roberson 1058355f576SJeff Roberson /* Linked list of all zones in the system */ 1068355f576SJeff Roberson static LIST_HEAD(,uma_zone) uma_zones = LIST_HEAD_INITIALIZER(&uma_zones); 1078355f576SJeff Roberson 1088355f576SJeff Roberson /* This mutex protects the zone list */ 1098355f576SJeff Roberson static struct mtx uma_mtx; 1108355f576SJeff Roberson 1118355f576SJeff Roberson /* Linked list of boot time pages */ 1128355f576SJeff Roberson static LIST_HEAD(,uma_slab) uma_boot_pages = 1138355f576SJeff Roberson LIST_HEAD_INITIALIZER(&uma_boot_pages); 1148355f576SJeff Roberson 1158355f576SJeff Roberson /* Count of free boottime pages */ 1168355f576SJeff Roberson static int uma_boot_free = 0; 1178355f576SJeff Roberson 1188355f576SJeff Roberson /* Is the VM done starting up? */ 1198355f576SJeff Roberson static int booted = 0; 1208355f576SJeff Roberson 1218355f576SJeff Roberson /* This is the handle used to schedule our working set calculator */ 1228355f576SJeff Roberson static struct callout uma_callout; 1238355f576SJeff Roberson 1248355f576SJeff Roberson /* This is mp_maxid + 1, for use while looping over each cpu */ 1258355f576SJeff Roberson static int maxcpu; 1268355f576SJeff Roberson 1278355f576SJeff Roberson /* 1288355f576SJeff Roberson * This structure is passed as the zone ctor arg so that I don't have to create 1298355f576SJeff Roberson * a special allocation function just for zones. 1308355f576SJeff Roberson */ 1318355f576SJeff Roberson struct uma_zctor_args { 1328355f576SJeff Roberson char *name; 1338355f576SJeff Roberson int size; 1348355f576SJeff Roberson uma_ctor ctor; 1358355f576SJeff Roberson uma_dtor dtor; 1368355f576SJeff Roberson uma_init uminit; 1378355f576SJeff Roberson uma_fini fini; 1388355f576SJeff Roberson int align; 1398355f576SJeff Roberson u_int16_t flags; 1408355f576SJeff Roberson }; 1418355f576SJeff Roberson 1428355f576SJeff Roberson /* 1438355f576SJeff Roberson * This is the malloc hash table which is used to find the zone that a 1448355f576SJeff Roberson * malloc allocation came from. It is not currently resizeable. The 1458355f576SJeff Roberson * memory for the actual hash bucket is allocated in kmeminit. 1468355f576SJeff Roberson */ 1478355f576SJeff Roberson struct uma_hash mhash; 1488355f576SJeff Roberson struct uma_hash *mallochash = &mhash; 1498355f576SJeff Roberson 1508355f576SJeff Roberson /* Prototypes.. */ 1518355f576SJeff Roberson 1528355f576SJeff Roberson static void *obj_alloc(uma_zone_t, int, u_int8_t *, int); 1538355f576SJeff Roberson static void *page_alloc(uma_zone_t, int, u_int8_t *, int); 1548355f576SJeff Roberson static void page_free(void *, int, u_int8_t); 1558355f576SJeff Roberson static uma_slab_t slab_zalloc(uma_zone_t, int); 1568355f576SJeff Roberson static void cache_drain(uma_zone_t); 1578355f576SJeff Roberson static void bucket_drain(uma_zone_t, uma_bucket_t); 1588355f576SJeff Roberson static void zone_drain(uma_zone_t); 1598355f576SJeff Roberson static void zone_ctor(void *, int, void *); 1608355f576SJeff Roberson static void zero_init(void *, int); 1618355f576SJeff Roberson static void zone_small_init(uma_zone_t zone); 1628355f576SJeff Roberson static void zone_large_init(uma_zone_t zone); 1638355f576SJeff Roberson static void zone_foreach(void (*zfunc)(uma_zone_t)); 1648355f576SJeff Roberson static void zone_timeout(uma_zone_t zone); 1658355f576SJeff Roberson static void hash_expand(struct uma_hash *); 1668355f576SJeff Roberson static void uma_timeout(void *); 1678355f576SJeff Roberson static void uma_startup3(void); 1688355f576SJeff Roberson static void *uma_zalloc_internal(uma_zone_t, void *, int, int *, int); 1698355f576SJeff Roberson static void uma_zfree_internal(uma_zone_t, 1708355f576SJeff Roberson void *, void *, int); 1718355f576SJeff Roberson void uma_print_zone(uma_zone_t); 1728355f576SJeff Roberson void uma_print_stats(void); 1738355f576SJeff Roberson static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS); 1748355f576SJeff Roberson 1758355f576SJeff Roberson SYSCTL_OID(_vm, OID_AUTO, zone, CTLTYPE_STRING|CTLFLAG_RD, 1768355f576SJeff Roberson NULL, 0, sysctl_vm_zone, "A", "Zone Info"); 1778355f576SJeff Roberson SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 1788355f576SJeff Roberson 1798355f576SJeff Roberson 1808355f576SJeff Roberson /* 1818355f576SJeff Roberson * Routine called by timeout which is used to fire off some time interval 1828355f576SJeff Roberson * based calculations. (working set, stats, etc.) 1838355f576SJeff Roberson * 1848355f576SJeff Roberson * Arguments: 1858355f576SJeff Roberson * arg Unused 1868355f576SJeff Roberson * 1878355f576SJeff Roberson * Returns: 1888355f576SJeff Roberson * Nothing 1898355f576SJeff Roberson */ 1908355f576SJeff Roberson static void 1918355f576SJeff Roberson uma_timeout(void *unused) 1928355f576SJeff Roberson { 1938355f576SJeff Roberson zone_foreach(zone_timeout); 1948355f576SJeff Roberson 1958355f576SJeff Roberson /* Reschedule this event */ 1968355f576SJeff Roberson callout_reset(&uma_callout, UMA_WORKING_TIME * hz, uma_timeout, NULL); 1978355f576SJeff Roberson } 1988355f576SJeff Roberson 1998355f576SJeff Roberson /* 2008355f576SJeff Roberson * Routine to perform timeout driven calculations. This does the working set 2018355f576SJeff Roberson * as well as hash expanding, and per cpu statistics aggregation. 2028355f576SJeff Roberson * 2038355f576SJeff Roberson * Arguments: 2048355f576SJeff Roberson * zone The zone to operate on 2058355f576SJeff Roberson * 2068355f576SJeff Roberson * Returns: 2078355f576SJeff Roberson * Nothing 2088355f576SJeff Roberson */ 2098355f576SJeff Roberson static void 2108355f576SJeff Roberson zone_timeout(uma_zone_t zone) 2118355f576SJeff Roberson { 2128355f576SJeff Roberson uma_cache_t cache; 2138355f576SJeff Roberson u_int64_t alloc; 2148355f576SJeff Roberson int free; 2158355f576SJeff Roberson int cpu; 2168355f576SJeff Roberson 2178355f576SJeff Roberson alloc = 0; 2188355f576SJeff Roberson free = 0; 2198355f576SJeff Roberson 2208355f576SJeff Roberson /* 2218355f576SJeff Roberson * Aggregate per cpu cache statistics back to the zone. 2228355f576SJeff Roberson * 2238355f576SJeff Roberson * I may rewrite this to set a flag in the per cpu cache instead of 2248355f576SJeff Roberson * locking. If the flag is not cleared on the next round I will have 2258355f576SJeff Roberson * to lock and do it here instead so that the statistics don't get too 2268355f576SJeff Roberson * far out of sync. 2278355f576SJeff Roberson */ 2288355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) { 2298355f576SJeff Roberson for (cpu = 0; cpu < maxcpu; cpu++) { 2308355f576SJeff Roberson if (CPU_ABSENT(cpu)) 2318355f576SJeff Roberson continue; 2328355f576SJeff Roberson CPU_LOCK(zone, cpu); 2338355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 2348355f576SJeff Roberson /* Add them up, and reset */ 2358355f576SJeff Roberson alloc += cache->uc_allocs; 2368355f576SJeff Roberson cache->uc_allocs = 0; 2378355f576SJeff Roberson if (cache->uc_allocbucket) 2388355f576SJeff Roberson free += cache->uc_allocbucket->ub_ptr + 1; 2398355f576SJeff Roberson if (cache->uc_freebucket) 2408355f576SJeff Roberson free += cache->uc_freebucket->ub_ptr + 1; 2418355f576SJeff Roberson CPU_UNLOCK(zone, cpu); 2428355f576SJeff Roberson } 2438355f576SJeff Roberson } 2448355f576SJeff Roberson 2458355f576SJeff Roberson /* Now push these stats back into the zone.. */ 2468355f576SJeff Roberson ZONE_LOCK(zone); 2478355f576SJeff Roberson zone->uz_allocs += alloc; 2488355f576SJeff Roberson 2498355f576SJeff Roberson /* 2508355f576SJeff Roberson * cachefree is an instantanious snapshot of what is in the per cpu 2518355f576SJeff Roberson * caches, not an accurate counter 2528355f576SJeff Roberson */ 2538355f576SJeff Roberson zone->uz_cachefree = free; 2548355f576SJeff Roberson 2558355f576SJeff Roberson /* 2568355f576SJeff Roberson * Expand the zone hash table. 2578355f576SJeff Roberson * 2588355f576SJeff Roberson * This is done if the number of slabs is larger than the hash size. 2598355f576SJeff Roberson * What I'm trying to do here is completely reduce collisions. This 2608355f576SJeff Roberson * may be a little aggressive. Should I allow for two collisions max? 2618355f576SJeff Roberson */ 2628355f576SJeff Roberson 2638355f576SJeff Roberson if ((zone->uz_flags & UMA_ZFLAG_OFFPAGE) && 2648355f576SJeff Roberson !(zone->uz_flags & UMA_ZFLAG_MALLOC)) { 2658355f576SJeff Roberson if (zone->uz_pages / zone->uz_ppera 2668355f576SJeff Roberson >= zone->uz_hash.uh_hashsize) 2678355f576SJeff Roberson hash_expand(&zone->uz_hash); 2688355f576SJeff Roberson } 2698355f576SJeff Roberson 2708355f576SJeff Roberson /* 2718355f576SJeff Roberson * Here we compute the working set size as the total number of items 2728355f576SJeff Roberson * left outstanding since the last time interval. This is slightly 2738355f576SJeff Roberson * suboptimal. What we really want is the highest number of outstanding 2748355f576SJeff Roberson * items during the last time quantum. This should be close enough. 2758355f576SJeff Roberson * 2768355f576SJeff Roberson * The working set size is used to throttle the zone_drain function. 2778355f576SJeff Roberson * We don't want to return memory that we may need again immediately. 2788355f576SJeff Roberson */ 2798355f576SJeff Roberson alloc = zone->uz_allocs - zone->uz_oallocs; 2808355f576SJeff Roberson zone->uz_oallocs = zone->uz_allocs; 2818355f576SJeff Roberson zone->uz_wssize = alloc; 2828355f576SJeff Roberson 2838355f576SJeff Roberson ZONE_UNLOCK(zone); 2848355f576SJeff Roberson } 2858355f576SJeff Roberson 2868355f576SJeff Roberson /* 2878355f576SJeff Roberson * Expands the hash table for OFFPAGE zones. This is done from zone_timeout 2888355f576SJeff Roberson * to reduce collisions. This must not be done in the regular allocation path, 2898355f576SJeff Roberson * otherwise, we can recurse on the vm while allocating pages. 2908355f576SJeff Roberson * 2918355f576SJeff Roberson * Arguments: 2928355f576SJeff Roberson * hash The hash you want to expand by a factor of two. 2938355f576SJeff Roberson * 2948355f576SJeff Roberson * Returns: 2958355f576SJeff Roberson * Nothing 2968355f576SJeff Roberson * 2978355f576SJeff Roberson * Discussion: 2988355f576SJeff Roberson */ 2998355f576SJeff Roberson static void 3008355f576SJeff Roberson hash_expand(struct uma_hash *hash) 3018355f576SJeff Roberson { 3028355f576SJeff Roberson struct slabhead *newhash; 3038355f576SJeff Roberson struct slabhead *oldhash; 3048355f576SJeff Roberson uma_slab_t slab; 3058355f576SJeff Roberson int hzonefree; 3068355f576SJeff Roberson int hashsize; 3078355f576SJeff Roberson int alloc; 3088355f576SJeff Roberson int hval; 3098355f576SJeff Roberson int i; 3108355f576SJeff Roberson 3118355f576SJeff Roberson 3128355f576SJeff Roberson /* 3138355f576SJeff Roberson * Remember the old hash size and see if it has to go back to the 3148355f576SJeff Roberson * hash zone, or malloc. The hash zone is used for the initial hash 3158355f576SJeff Roberson */ 3168355f576SJeff Roberson 3178355f576SJeff Roberson hashsize = hash->uh_hashsize; 3188355f576SJeff Roberson oldhash = hash->uh_slab_hash; 3198355f576SJeff Roberson 3208355f576SJeff Roberson if (hashsize == UMA_HASH_SIZE_INIT) 3218355f576SJeff Roberson hzonefree = 1; 3228355f576SJeff Roberson else 3238355f576SJeff Roberson hzonefree = 0; 3248355f576SJeff Roberson 3258355f576SJeff Roberson 3268355f576SJeff Roberson /* We're just going to go to a power of two greater */ 3278355f576SJeff Roberson if (hash->uh_hashsize) { 3288355f576SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * (hash->uh_hashsize * 2); 3298355f576SJeff Roberson /* XXX Shouldn't be abusing DEVBUF here */ 3308355f576SJeff Roberson newhash = (struct slabhead *)malloc(alloc, M_DEVBUF, M_NOWAIT); 3318355f576SJeff Roberson if (newhash == NULL) { 3328355f576SJeff Roberson return; 3338355f576SJeff Roberson } 3348355f576SJeff Roberson hash->uh_hashsize *= 2; 3358355f576SJeff Roberson } else { 3368355f576SJeff Roberson alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; 3378355f576SJeff Roberson newhash = uma_zalloc_internal(hashzone, NULL, M_WAITOK, NULL, -1); 3388355f576SJeff Roberson hash->uh_hashsize = UMA_HASH_SIZE_INIT; 3398355f576SJeff Roberson } 3408355f576SJeff Roberson 3418355f576SJeff Roberson bzero(newhash, alloc); 3428355f576SJeff Roberson 3438355f576SJeff Roberson hash->uh_hashmask = hash->uh_hashsize - 1; 3448355f576SJeff Roberson 3458355f576SJeff Roberson /* 3468355f576SJeff Roberson * I need to investigate hash algorithms for resizing without a 3478355f576SJeff Roberson * full rehash. 3488355f576SJeff Roberson */ 3498355f576SJeff Roberson 3508355f576SJeff Roberson for (i = 0; i < hashsize; i++) 3518355f576SJeff Roberson while (!SLIST_EMPTY(&hash->uh_slab_hash[i])) { 3528355f576SJeff Roberson slab = SLIST_FIRST(&hash->uh_slab_hash[i]); 3538355f576SJeff Roberson SLIST_REMOVE_HEAD(&hash->uh_slab_hash[i], us_hlink); 3548355f576SJeff Roberson hval = UMA_HASH(hash, slab->us_data); 3558355f576SJeff Roberson SLIST_INSERT_HEAD(&newhash[hval], slab, us_hlink); 3568355f576SJeff Roberson } 3578355f576SJeff Roberson 3588355f576SJeff Roberson if (hash->uh_slab_hash) { 3598355f576SJeff Roberson if (hzonefree) 3608355f576SJeff Roberson uma_zfree_internal(hashzone, 3618355f576SJeff Roberson hash->uh_slab_hash, NULL, 0); 3628355f576SJeff Roberson else 3638355f576SJeff Roberson free(hash->uh_slab_hash, M_DEVBUF); 3648355f576SJeff Roberson } 3658355f576SJeff Roberson hash->uh_slab_hash = newhash; 3668355f576SJeff Roberson 3678355f576SJeff Roberson return; 3688355f576SJeff Roberson } 3698355f576SJeff Roberson 3708355f576SJeff Roberson /* 3718355f576SJeff Roberson * Frees all outstanding items in a bucket 3728355f576SJeff Roberson * 3738355f576SJeff Roberson * Arguments: 3748355f576SJeff Roberson * zone The zone to free to, must be unlocked. 3758355f576SJeff Roberson * bucket The free/alloc bucket with items, cpu queue must be locked. 3768355f576SJeff Roberson * 3778355f576SJeff Roberson * Returns: 3788355f576SJeff Roberson * Nothing 3798355f576SJeff Roberson */ 3808355f576SJeff Roberson 3818355f576SJeff Roberson static void 3828355f576SJeff Roberson bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 3838355f576SJeff Roberson { 3848355f576SJeff Roberson uma_slab_t slab; 3858355f576SJeff Roberson int mzone; 3868355f576SJeff Roberson void *item; 3878355f576SJeff Roberson 3888355f576SJeff Roberson if (bucket == NULL) 3898355f576SJeff Roberson return; 3908355f576SJeff Roberson 3918355f576SJeff Roberson slab = NULL; 3928355f576SJeff Roberson mzone = 0; 3938355f576SJeff Roberson 3948355f576SJeff Roberson /* We have to lookup the slab again for malloc.. */ 3958355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_MALLOC) 3968355f576SJeff Roberson mzone = 1; 3978355f576SJeff Roberson 3988355f576SJeff Roberson while (bucket->ub_ptr > -1) { 3998355f576SJeff Roberson item = bucket->ub_bucket[bucket->ub_ptr]; 4008355f576SJeff Roberson #ifdef INVARIANTS 4018355f576SJeff Roberson bucket->ub_bucket[bucket->ub_ptr] = NULL; 4028355f576SJeff Roberson KASSERT(item != NULL, 4038355f576SJeff Roberson ("bucket_drain: botched ptr, item is NULL")); 4048355f576SJeff Roberson #endif 4058355f576SJeff Roberson bucket->ub_ptr--; 4068355f576SJeff Roberson /* 4078355f576SJeff Roberson * This is extremely inefficient. The slab pointer was passed 4088355f576SJeff Roberson * to uma_zfree_arg, but we lost it because the buckets don't 4098355f576SJeff Roberson * hold them. This will go away when free() gets a size passed 4108355f576SJeff Roberson * to it. 4118355f576SJeff Roberson */ 4128355f576SJeff Roberson if (mzone) 4138355f576SJeff Roberson slab = hash_sfind(mallochash, 4148355f576SJeff Roberson (u_int8_t *)((unsigned long)item & 4158355f576SJeff Roberson (~UMA_SLAB_MASK))); 4168355f576SJeff Roberson uma_zfree_internal(zone, item, slab, 1); 4178355f576SJeff Roberson } 4188355f576SJeff Roberson } 4198355f576SJeff Roberson 4208355f576SJeff Roberson /* 4218355f576SJeff Roberson * Drains the per cpu caches for a zone. 4228355f576SJeff Roberson * 4238355f576SJeff Roberson * Arguments: 4248355f576SJeff Roberson * zone The zone to drain, must be unlocked. 4258355f576SJeff Roberson * 4268355f576SJeff Roberson * Returns: 4278355f576SJeff Roberson * Nothing 4288355f576SJeff Roberson * 4298355f576SJeff Roberson * This function returns with the zone locked so that the per cpu queues can 4308355f576SJeff Roberson * not be filled until zone_drain is finished. 4318355f576SJeff Roberson * 4328355f576SJeff Roberson */ 4338355f576SJeff Roberson static void 4348355f576SJeff Roberson cache_drain(uma_zone_t zone) 4358355f576SJeff Roberson { 4368355f576SJeff Roberson uma_bucket_t bucket; 4378355f576SJeff Roberson uma_cache_t cache; 4388355f576SJeff Roberson int cpu; 4398355f576SJeff Roberson 4408355f576SJeff Roberson /* 4418355f576SJeff Roberson * Flush out the per cpu queues. 4428355f576SJeff Roberson * 4438355f576SJeff Roberson * XXX This causes unneccisary thrashing due to immediately having 4448355f576SJeff Roberson * empty per cpu queues. I need to improve this. 4458355f576SJeff Roberson */ 4468355f576SJeff Roberson 4478355f576SJeff Roberson /* 4488355f576SJeff Roberson * We have to lock each cpu cache before locking the zone 4498355f576SJeff Roberson */ 4508355f576SJeff Roberson ZONE_UNLOCK(zone); 4518355f576SJeff Roberson 4528355f576SJeff Roberson for (cpu = 0; cpu < maxcpu; cpu++) { 4538355f576SJeff Roberson if (CPU_ABSENT(cpu)) 4548355f576SJeff Roberson continue; 4558355f576SJeff Roberson CPU_LOCK(zone, cpu); 4568355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 4578355f576SJeff Roberson bucket_drain(zone, cache->uc_allocbucket); 4588355f576SJeff Roberson bucket_drain(zone, cache->uc_freebucket); 4598355f576SJeff Roberson } 4608355f576SJeff Roberson 4618355f576SJeff Roberson /* 4628355f576SJeff Roberson * Drain the bucket queues and free the buckets, we just keep two per 4638355f576SJeff Roberson * cpu (alloc/free). 4648355f576SJeff Roberson */ 4658355f576SJeff Roberson ZONE_LOCK(zone); 4668355f576SJeff Roberson while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 4678355f576SJeff Roberson LIST_REMOVE(bucket, ub_link); 4688355f576SJeff Roberson ZONE_UNLOCK(zone); 4698355f576SJeff Roberson bucket_drain(zone, bucket); 4708355f576SJeff Roberson uma_zfree_internal(bucketzone, bucket, NULL, 0); 4718355f576SJeff Roberson ZONE_LOCK(zone); 4728355f576SJeff Roberson } 4738355f576SJeff Roberson 4748355f576SJeff Roberson /* Now we do the free queue.. */ 4758355f576SJeff Roberson while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 4768355f576SJeff Roberson LIST_REMOVE(bucket, ub_link); 4778355f576SJeff Roberson uma_zfree_internal(bucketzone, bucket, NULL, 0); 4788355f576SJeff Roberson } 4798355f576SJeff Roberson 4808355f576SJeff Roberson /* We unlock here, but they will all block until the zone is unlocked */ 4818355f576SJeff Roberson for (cpu = 0; cpu < maxcpu; cpu++) { 4828355f576SJeff Roberson if (CPU_ABSENT(cpu)) 4838355f576SJeff Roberson continue; 4848355f576SJeff Roberson CPU_UNLOCK(zone, cpu); 4858355f576SJeff Roberson } 4868355f576SJeff Roberson } 4878355f576SJeff Roberson 4888355f576SJeff Roberson /* 4898355f576SJeff Roberson * Frees pages from a zone back to the system. This is done on demand from 4908355f576SJeff Roberson * the pageout daemon. 4918355f576SJeff Roberson * 4928355f576SJeff Roberson * Arguments: 4938355f576SJeff Roberson * zone The zone to free pages from 4948355f576SJeff Roberson * 4958355f576SJeff Roberson * Returns: 4968355f576SJeff Roberson * Nothing. 4978355f576SJeff Roberson */ 4988355f576SJeff Roberson static void 4998355f576SJeff Roberson zone_drain(uma_zone_t zone) 5008355f576SJeff Roberson { 5018355f576SJeff Roberson uma_slab_t slab; 5028355f576SJeff Roberson uma_slab_t n; 5038355f576SJeff Roberson u_int64_t extra; 5048355f576SJeff Roberson u_int8_t flags; 5058355f576SJeff Roberson u_int8_t *mem; 5068355f576SJeff Roberson int i; 5078355f576SJeff Roberson 5088355f576SJeff Roberson /* 5098355f576SJeff Roberson * We don't want to take pages from staticly allocated zones at this 5108355f576SJeff Roberson * time 5118355f576SJeff Roberson */ 5128355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_NOFREE || zone->uz_freef == NULL) 5138355f576SJeff Roberson return; 5148355f576SJeff Roberson 5158355f576SJeff Roberson ZONE_LOCK(zone); 5168355f576SJeff Roberson 5178355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) 5188355f576SJeff Roberson cache_drain(zone); 5198355f576SJeff Roberson 5208355f576SJeff Roberson if (zone->uz_free < zone->uz_wssize) 5218355f576SJeff Roberson goto finished; 5228355f576SJeff Roberson #ifdef UMA_DEBUG 5238355f576SJeff Roberson printf("%s working set size: %llu free items: %u\n", 5248355f576SJeff Roberson zone->uz_name, (unsigned long long)zone->uz_wssize, zone->uz_free); 5258355f576SJeff Roberson #endif 5268355f576SJeff Roberson extra = zone->uz_wssize - zone->uz_free; 5278355f576SJeff Roberson extra /= zone->uz_ipers; 5288355f576SJeff Roberson 5298355f576SJeff Roberson /* extra is now the number of extra slabs that we can free */ 5308355f576SJeff Roberson 5318355f576SJeff Roberson if (extra == 0) 5328355f576SJeff Roberson goto finished; 5338355f576SJeff Roberson 5348355f576SJeff Roberson slab = LIST_FIRST(&zone->uz_free_slab); 5358355f576SJeff Roberson while (slab && extra) { 5368355f576SJeff Roberson n = LIST_NEXT(slab, us_link); 5378355f576SJeff Roberson 5388355f576SJeff Roberson /* We have no where to free these to */ 5398355f576SJeff Roberson if (slab->us_flags & UMA_SLAB_BOOT) { 5408355f576SJeff Roberson slab = n; 5418355f576SJeff Roberson continue; 5428355f576SJeff Roberson } 5438355f576SJeff Roberson 5448355f576SJeff Roberson LIST_REMOVE(slab, us_link); 5458355f576SJeff Roberson zone->uz_pages -= zone->uz_ppera; 5468355f576SJeff Roberson zone->uz_free -= zone->uz_ipers; 5478355f576SJeff Roberson if (zone->uz_fini) 5488355f576SJeff Roberson for (i = 0; i < zone->uz_ipers; i++) 5498355f576SJeff Roberson zone->uz_fini( 5508355f576SJeff Roberson slab->us_data + (zone->uz_rsize * i), 5518355f576SJeff Roberson zone->uz_size); 5528355f576SJeff Roberson flags = slab->us_flags; 5538355f576SJeff Roberson mem = slab->us_data; 5548355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_OFFPAGE) { 5558355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_MALLOC) { 5568355f576SJeff Roberson UMA_HASH_REMOVE(mallochash, 5578355f576SJeff Roberson slab, slab->us_data); 5588355f576SJeff Roberson } else { 5598355f576SJeff Roberson UMA_HASH_REMOVE(&zone->uz_hash, 5608355f576SJeff Roberson slab, slab->us_data); 5618355f576SJeff Roberson } 5628355f576SJeff Roberson uma_zfree_internal(slabzone, slab, NULL, 0); 5638355f576SJeff Roberson } else if (zone->uz_flags & UMA_ZFLAG_MALLOC) 5648355f576SJeff Roberson UMA_HASH_REMOVE(mallochash, slab, slab->us_data); 5658355f576SJeff Roberson #ifdef UMA_DEBUG 5668355f576SJeff Roberson printf("%s: Returning %d bytes.\n", 5678355f576SJeff Roberson zone->uz_name, UMA_SLAB_SIZE * zone->uz_ppera); 5688355f576SJeff Roberson #endif 5698355f576SJeff Roberson zone->uz_freef(mem, UMA_SLAB_SIZE * zone->uz_ppera, flags); 5708355f576SJeff Roberson 5718355f576SJeff Roberson slab = n; 5728355f576SJeff Roberson extra--; 5738355f576SJeff Roberson } 5748355f576SJeff Roberson 5758355f576SJeff Roberson finished: 5768355f576SJeff Roberson ZONE_UNLOCK(zone); 5778355f576SJeff Roberson } 5788355f576SJeff Roberson 5798355f576SJeff Roberson /* 5808355f576SJeff Roberson * Allocate a new slab for a zone. This does not insert the slab onto a list. 5818355f576SJeff Roberson * 5828355f576SJeff Roberson * Arguments: 5838355f576SJeff Roberson * zone The zone to allocate slabs for 5848355f576SJeff Roberson * wait Shall we wait? 5858355f576SJeff Roberson * 5868355f576SJeff Roberson * Returns: 5878355f576SJeff Roberson * The slab that was allocated or NULL if there is no memory and the 5888355f576SJeff Roberson * caller specified M_NOWAIT. 5898355f576SJeff Roberson * 5908355f576SJeff Roberson */ 5918355f576SJeff Roberson static uma_slab_t 5928355f576SJeff Roberson slab_zalloc(uma_zone_t zone, int wait) 5938355f576SJeff Roberson { 5948355f576SJeff Roberson uma_slab_t slab; /* Starting slab */ 5958355f576SJeff Roberson u_int8_t *mem; 5968355f576SJeff Roberson u_int8_t flags; 5978355f576SJeff Roberson int i; 5988355f576SJeff Roberson 5998355f576SJeff Roberson #ifdef UMA_DEBUG 6008355f576SJeff Roberson printf("slab_zalloc: Allocating a new slab for %s\n", zone->uz_name); 6018355f576SJeff Roberson #endif 602736ee590SJeff Roberson if (zone->uz_maxpages && 603736ee590SJeff Roberson zone->uz_pages + zone->uz_ppera > zone->uz_maxpages) 604736ee590SJeff Roberson return (NULL); 6058355f576SJeff Roberson 6068355f576SJeff Roberson if (booted || (zone->uz_flags & UMA_ZFLAG_PRIVALLOC)) { 6078355f576SJeff Roberson ZONE_UNLOCK(zone); 6088355f576SJeff Roberson mtx_lock(&Giant); 6098355f576SJeff Roberson slab = (uma_slab_t )zone->uz_allocf(zone, 6108355f576SJeff Roberson zone->uz_ppera * UMA_SLAB_SIZE, &flags, wait); 6118355f576SJeff Roberson mtx_unlock(&Giant); 6128355f576SJeff Roberson ZONE_LOCK(zone); 6138355f576SJeff Roberson if (slab != NULL) 6148355f576SJeff Roberson slab->us_data = (u_int8_t *)slab; 6158355f576SJeff Roberson else 6168355f576SJeff Roberson return (NULL); 6178355f576SJeff Roberson } else { 6188355f576SJeff Roberson 6198355f576SJeff Roberson if (zone->uz_ppera > 1) 6208355f576SJeff Roberson panic("UMA: Attemping to allocate multiple pages before vm has started.\n"); 6218355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_MALLOC) 6228355f576SJeff Roberson panic("Mallocing before uma_startup2 has been called.\n"); 6238355f576SJeff Roberson if (uma_boot_free == 0) 6248355f576SJeff Roberson panic("UMA: Ran out of pre init pages, increase UMA_BOOT_PAGES\n"); 6258355f576SJeff Roberson slab = LIST_FIRST(&uma_boot_pages); 6268355f576SJeff Roberson LIST_REMOVE(slab, us_link); 6278355f576SJeff Roberson uma_boot_free--; 6288355f576SJeff Roberson } 6298355f576SJeff Roberson 6308355f576SJeff Roberson mem = slab->us_data; 6318355f576SJeff Roberson 6328355f576SJeff Roberson /* Alloc slab structure for offpage, otherwise adjust it's position */ 6338355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_OFFPAGE)) { 6348355f576SJeff Roberson slab = (uma_slab_t )(mem + zone->uz_pgoff); 6358355f576SJeff Roberson } else { 6368355f576SJeff Roberson slab = uma_zalloc_internal(slabzone, NULL, wait, NULL, -1); 6378355f576SJeff Roberson if (slab == NULL) /* XXX This should go away */ 6388355f576SJeff Roberson panic("UMA: No free slab structures"); 6398355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_MALLOC)) 6408355f576SJeff Roberson UMA_HASH_INSERT(&zone->uz_hash, slab, mem); 6418355f576SJeff Roberson } 6428355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_MALLOC) { 6438355f576SJeff Roberson #ifdef UMA_DEBUG 6448355f576SJeff Roberson printf("Inserting %p into malloc hash from slab %p\n", 6458355f576SJeff Roberson mem, slab); 6468355f576SJeff Roberson #endif 6478355f576SJeff Roberson UMA_HASH_INSERT(mallochash, slab, mem); 6488355f576SJeff Roberson } 6498355f576SJeff Roberson 6508355f576SJeff Roberson slab->us_zone = zone; 6518355f576SJeff Roberson slab->us_data = mem; 6528355f576SJeff Roberson 6538355f576SJeff Roberson /* 6548355f576SJeff Roberson * This is intended to spread data out across cache lines. 6558355f576SJeff Roberson * 6568355f576SJeff Roberson * This code doesn't seem to work properly on x86, and on alpha 6578355f576SJeff Roberson * it makes absolutely no performance difference. I'm sure it could 6588355f576SJeff Roberson * use some tuning, but sun makes outrageous claims about it's 6598355f576SJeff Roberson * performance. 6608355f576SJeff Roberson */ 6618355f576SJeff Roberson #if 0 6628355f576SJeff Roberson if (zone->uz_cachemax) { 6638355f576SJeff Roberson slab->us_data += zone->uz_cacheoff; 6648355f576SJeff Roberson zone->uz_cacheoff += UMA_CACHE_INC; 6658355f576SJeff Roberson if (zone->uz_cacheoff > zone->uz_cachemax) 6668355f576SJeff Roberson zone->uz_cacheoff = 0; 6678355f576SJeff Roberson } 6688355f576SJeff Roberson #endif 6698355f576SJeff Roberson 6708355f576SJeff Roberson slab->us_freecount = zone->uz_ipers; 6718355f576SJeff Roberson slab->us_firstfree = 0; 6728355f576SJeff Roberson slab->us_flags = flags; 6738355f576SJeff Roberson for (i = 0; i < zone->uz_ipers; i++) 6748355f576SJeff Roberson slab->us_freelist[i] = i+1; 6758355f576SJeff Roberson 6768355f576SJeff Roberson if (zone->uz_init) 6778355f576SJeff Roberson for (i = 0; i < zone->uz_ipers; i++) 6788355f576SJeff Roberson zone->uz_init(slab->us_data + (zone->uz_rsize * i), 6798355f576SJeff Roberson zone->uz_size); 6808355f576SJeff Roberson 6818355f576SJeff Roberson zone->uz_pages += zone->uz_ppera; 6828355f576SJeff Roberson zone->uz_free += zone->uz_ipers; 6838355f576SJeff Roberson 6848355f576SJeff Roberson return (slab); 6858355f576SJeff Roberson } 6868355f576SJeff Roberson 6878355f576SJeff Roberson /* 6888355f576SJeff Roberson * Allocates a number of pages from the system 6898355f576SJeff Roberson * 6908355f576SJeff Roberson * Arguments: 6918355f576SJeff Roberson * zone Unused 6928355f576SJeff Roberson * bytes The number of bytes requested 6938355f576SJeff Roberson * wait Shall we wait? 6948355f576SJeff Roberson * 6958355f576SJeff Roberson * Returns: 6968355f576SJeff Roberson * A pointer to the alloced memory or possibly 6978355f576SJeff Roberson * NULL if M_NOWAIT is set. 6988355f576SJeff Roberson */ 6998355f576SJeff Roberson static void * 7008355f576SJeff Roberson page_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 7018355f576SJeff Roberson { 7028355f576SJeff Roberson void *p; /* Returned page */ 7038355f576SJeff Roberson 7048355f576SJeff Roberson /* 7058355f576SJeff Roberson * XXX The original zone allocator did this, but I don't think it's 7068355f576SJeff Roberson * neccisary in current. 7078355f576SJeff Roberson */ 7088355f576SJeff Roberson 7098355f576SJeff Roberson if (lockstatus(&kernel_map->lock, NULL)) { 7108355f576SJeff Roberson *pflag = UMA_SLAB_KMEM; 7118355f576SJeff Roberson p = (void *) kmem_malloc(kmem_map, bytes, wait); 7128355f576SJeff Roberson } else { 7138355f576SJeff Roberson *pflag = UMA_SLAB_KMAP; 7148355f576SJeff Roberson p = (void *) kmem_alloc(kernel_map, bytes); 7158355f576SJeff Roberson } 7168355f576SJeff Roberson 7178355f576SJeff Roberson return (p); 7188355f576SJeff Roberson } 7198355f576SJeff Roberson 7208355f576SJeff Roberson /* 7218355f576SJeff Roberson * Allocates a number of pages from within an object 7228355f576SJeff Roberson * 7238355f576SJeff Roberson * Arguments: 7248355f576SJeff Roberson * zone Unused 7258355f576SJeff Roberson * bytes The number of bytes requested 7268355f576SJeff Roberson * wait Shall we wait? 7278355f576SJeff Roberson * 7288355f576SJeff Roberson * Returns: 7298355f576SJeff Roberson * A pointer to the alloced memory or possibly 7308355f576SJeff Roberson * NULL if M_NOWAIT is set. 7318355f576SJeff Roberson */ 7328355f576SJeff Roberson static void * 7338355f576SJeff Roberson obj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 7348355f576SJeff Roberson { 7358355f576SJeff Roberson vm_offset_t zkva; 7368355f576SJeff Roberson vm_offset_t retkva; 7378355f576SJeff Roberson vm_page_t p; 7388355f576SJeff Roberson int pages; 7398355f576SJeff Roberson 7408355f576SJeff Roberson retkva = NULL; 7418355f576SJeff Roberson pages = zone->uz_pages; 7428355f576SJeff Roberson 7438355f576SJeff Roberson /* 7448355f576SJeff Roberson * This looks a little weird since we're getting one page at a time 7458355f576SJeff Roberson */ 7468355f576SJeff Roberson while (bytes > 0) { 7478355f576SJeff Roberson p = vm_page_alloc(zone->uz_obj, pages, 7488355f576SJeff Roberson VM_ALLOC_INTERRUPT); 7498355f576SJeff Roberson if (p == NULL) 7508355f576SJeff Roberson return (NULL); 7518355f576SJeff Roberson 7528355f576SJeff Roberson zkva = zone->uz_kva + pages * PAGE_SIZE; 7538355f576SJeff Roberson if (retkva == NULL) 7548355f576SJeff Roberson retkva = zkva; 7558355f576SJeff Roberson pmap_qenter(zkva, &p, 1); 7568355f576SJeff Roberson bytes -= PAGE_SIZE; 7578355f576SJeff Roberson pages += 1; 7588355f576SJeff Roberson } 7598355f576SJeff Roberson 7608355f576SJeff Roberson *flags = UMA_SLAB_PRIV; 7618355f576SJeff Roberson 7628355f576SJeff Roberson return ((void *)retkva); 7638355f576SJeff Roberson } 7648355f576SJeff Roberson 7658355f576SJeff Roberson /* 7668355f576SJeff Roberson * Frees a number of pages to the system 7678355f576SJeff Roberson * 7688355f576SJeff Roberson * Arguments: 7698355f576SJeff Roberson * mem A pointer to the memory to be freed 7708355f576SJeff Roberson * size The size of the memory being freed 7718355f576SJeff Roberson * flags The original p->us_flags field 7728355f576SJeff Roberson * 7738355f576SJeff Roberson * Returns: 7748355f576SJeff Roberson * Nothing 7758355f576SJeff Roberson * 7768355f576SJeff Roberson */ 7778355f576SJeff Roberson static void 7788355f576SJeff Roberson page_free(void *mem, int size, u_int8_t flags) 7798355f576SJeff Roberson { 7808355f576SJeff Roberson vm_map_t map; 7818355f576SJeff Roberson if (flags & UMA_SLAB_KMEM) 7828355f576SJeff Roberson map = kmem_map; 7838355f576SJeff Roberson else if (flags & UMA_SLAB_KMAP) 7848355f576SJeff Roberson map = kernel_map; 7858355f576SJeff Roberson else 7868355f576SJeff Roberson panic("UMA: page_free used with invalid flags %d\n", flags); 7878355f576SJeff Roberson 7888355f576SJeff Roberson kmem_free(map, (vm_offset_t)mem, size); 7898355f576SJeff Roberson } 7908355f576SJeff Roberson 7918355f576SJeff Roberson /* 7928355f576SJeff Roberson * Zero fill initializer 7938355f576SJeff Roberson * 7948355f576SJeff Roberson * Arguments/Returns follow uma_init specifications 7958355f576SJeff Roberson * 7968355f576SJeff Roberson */ 7978355f576SJeff Roberson static void 7988355f576SJeff Roberson zero_init(void *mem, int size) 7998355f576SJeff Roberson { 8008355f576SJeff Roberson bzero(mem, size); 8018355f576SJeff Roberson } 8028355f576SJeff Roberson 8038355f576SJeff Roberson /* 8048355f576SJeff Roberson * Finish creating a small uma zone. This calculates ipers, and the zone size. 8058355f576SJeff Roberson * 8068355f576SJeff Roberson * Arguments 8078355f576SJeff Roberson * zone The zone we should initialize 8088355f576SJeff Roberson * 8098355f576SJeff Roberson * Returns 8108355f576SJeff Roberson * Nothing 8118355f576SJeff Roberson */ 8128355f576SJeff Roberson static void 8138355f576SJeff Roberson zone_small_init(uma_zone_t zone) 8148355f576SJeff Roberson { 8158355f576SJeff Roberson int rsize; 8168355f576SJeff Roberson int memused; 8178355f576SJeff Roberson int ipers; 8188355f576SJeff Roberson 8198355f576SJeff Roberson rsize = zone->uz_size; 8208355f576SJeff Roberson 8218355f576SJeff Roberson if (rsize < UMA_SMALLEST_UNIT) 8228355f576SJeff Roberson rsize = UMA_SMALLEST_UNIT; 8238355f576SJeff Roberson 8248355f576SJeff Roberson if (rsize & zone->uz_align) 8258355f576SJeff Roberson rsize = (rsize & ~zone->uz_align) + (zone->uz_align + 1); 8268355f576SJeff Roberson 8278355f576SJeff Roberson zone->uz_rsize = rsize; 8288355f576SJeff Roberson 8298355f576SJeff Roberson rsize += 1; /* Account for the byte of linkage */ 8308355f576SJeff Roberson zone->uz_ipers = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) / rsize; 8318355f576SJeff Roberson zone->uz_ppera = 1; 8328355f576SJeff Roberson 8338355f576SJeff Roberson memused = zone->uz_ipers * zone->uz_rsize; 8348355f576SJeff Roberson 8358355f576SJeff Roberson /* Can we do any better? */ 8368355f576SJeff Roberson if ((UMA_SLAB_SIZE - memused) >= UMA_MAX_WASTE) { 8378355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 8388355f576SJeff Roberson return; 8398355f576SJeff Roberson ipers = UMA_SLAB_SIZE / zone->uz_rsize; 8408355f576SJeff Roberson if (ipers > zone->uz_ipers) { 8418355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_OFFPAGE; 8428355f576SJeff Roberson zone->uz_ipers = ipers; 8438355f576SJeff Roberson } 8448355f576SJeff Roberson } 8458355f576SJeff Roberson 8468355f576SJeff Roberson } 8478355f576SJeff Roberson 8488355f576SJeff Roberson /* 8498355f576SJeff Roberson * Finish creating a large (> UMA_SLAB_SIZE) uma zone. Just give in and do 8508355f576SJeff Roberson * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be 8518355f576SJeff Roberson * more complicated. 8528355f576SJeff Roberson * 8538355f576SJeff Roberson * Arguments 8548355f576SJeff Roberson * zone The zone we should initialize 8558355f576SJeff Roberson * 8568355f576SJeff Roberson * Returns 8578355f576SJeff Roberson * Nothing 8588355f576SJeff Roberson */ 8598355f576SJeff Roberson static void 8608355f576SJeff Roberson zone_large_init(uma_zone_t zone) 8618355f576SJeff Roberson { 8628355f576SJeff Roberson int pages; 8638355f576SJeff Roberson 8648355f576SJeff Roberson pages = zone->uz_size / UMA_SLAB_SIZE; 8658355f576SJeff Roberson 8668355f576SJeff Roberson /* Account for remainder */ 8678355f576SJeff Roberson if ((pages * UMA_SLAB_SIZE) < zone->uz_size) 8688355f576SJeff Roberson pages++; 8698355f576SJeff Roberson 8708355f576SJeff Roberson zone->uz_ppera = pages; 8718355f576SJeff Roberson zone->uz_ipers = 1; 8728355f576SJeff Roberson 8738355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_OFFPAGE; 8748355f576SJeff Roberson zone->uz_rsize = zone->uz_size; 8758355f576SJeff Roberson } 8768355f576SJeff Roberson 8778355f576SJeff Roberson /* 8788355f576SJeff Roberson * Zone header ctor. This initializes all fields, locks, etc. And inserts 8798355f576SJeff Roberson * the zone onto the global zone list. 8808355f576SJeff Roberson * 8818355f576SJeff Roberson * Arguments/Returns follow uma_ctor specifications 8828355f576SJeff Roberson * udata Actually uma_zcreat_args 8838355f576SJeff Roberson * 8848355f576SJeff Roberson */ 8858355f576SJeff Roberson 8868355f576SJeff Roberson static void 8878355f576SJeff Roberson zone_ctor(void *mem, int size, void *udata) 8888355f576SJeff Roberson { 8898355f576SJeff Roberson struct uma_zctor_args *arg = udata; 8908355f576SJeff Roberson uma_zone_t zone = mem; 8918355f576SJeff Roberson int cplen; 8928355f576SJeff Roberson int cpu; 8938355f576SJeff Roberson 8948355f576SJeff Roberson bzero(zone, size); 8958355f576SJeff Roberson zone->uz_name = arg->name; 8968355f576SJeff Roberson zone->uz_size = arg->size; 8978355f576SJeff Roberson zone->uz_ctor = arg->ctor; 8988355f576SJeff Roberson zone->uz_dtor = arg->dtor; 8998355f576SJeff Roberson zone->uz_init = arg->uminit; 9008355f576SJeff Roberson zone->uz_align = arg->align; 9018355f576SJeff Roberson zone->uz_free = 0; 9028355f576SJeff Roberson zone->uz_pages = 0; 9038355f576SJeff Roberson zone->uz_flags = 0; 9048355f576SJeff Roberson zone->uz_allocf = page_alloc; 9058355f576SJeff Roberson zone->uz_freef = page_free; 9068355f576SJeff Roberson 9078355f576SJeff Roberson if (arg->flags & UMA_ZONE_ZINIT) 9088355f576SJeff Roberson zone->uz_init = zero_init; 9098355f576SJeff Roberson 9108355f576SJeff Roberson if (arg->flags & UMA_ZONE_INTERNAL) 9118355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_INTERNAL; 9128355f576SJeff Roberson 9138355f576SJeff Roberson if (arg->flags & UMA_ZONE_MALLOC) 9148355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_MALLOC; 9158355f576SJeff Roberson 9168355f576SJeff Roberson if (arg->flags & UMA_ZONE_NOFREE) 9178355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_NOFREE; 9188355f576SJeff Roberson 9198355f576SJeff Roberson if (zone->uz_size > UMA_SLAB_SIZE) 9208355f576SJeff Roberson zone_large_init(zone); 9218355f576SJeff Roberson else 9228355f576SJeff Roberson zone_small_init(zone); 9238355f576SJeff Roberson 9248355f576SJeff Roberson /* We do this so that the per cpu lock name is unique for each zone */ 9258355f576SJeff Roberson memcpy(zone->uz_lname, "PCPU ", 5); 9268355f576SJeff Roberson cplen = min(strlen(zone->uz_name) + 1, LOCKNAME_LEN - 6); 9278355f576SJeff Roberson memcpy(zone->uz_lname+5, zone->uz_name, cplen); 9288355f576SJeff Roberson zone->uz_lname[LOCKNAME_LEN - 1] = '\0'; 9298355f576SJeff Roberson 9308355f576SJeff Roberson /* 9318355f576SJeff Roberson * If we're putting the slab header in the actual page we need to 9328355f576SJeff Roberson * figure out where in each page it goes. This calculates a right 9338355f576SJeff Roberson * justified offset into the memory on a ALIGN_PTR boundary. 9348355f576SJeff Roberson */ 9358355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_OFFPAGE)) { 9368355f576SJeff Roberson int totsize; 9378355f576SJeff Roberson int waste; 9388355f576SJeff Roberson 9398355f576SJeff Roberson /* Size of the slab struct and free list */ 9408355f576SJeff Roberson totsize = sizeof(struct uma_slab) + zone->uz_ipers; 9418355f576SJeff Roberson if (totsize & UMA_ALIGN_PTR) 9428355f576SJeff Roberson totsize = (totsize & ~UMA_ALIGN_PTR) + 9438355f576SJeff Roberson (UMA_ALIGN_PTR + 1); 9448355f576SJeff Roberson zone->uz_pgoff = UMA_SLAB_SIZE - totsize; 9458355f576SJeff Roberson 9468355f576SJeff Roberson waste = zone->uz_pgoff; 9478355f576SJeff Roberson waste -= (zone->uz_ipers * zone->uz_rsize); 9488355f576SJeff Roberson 9498355f576SJeff Roberson /* 9508355f576SJeff Roberson * This calculates how much space we have for cache line size 9518355f576SJeff Roberson * optimizations. It works by offseting each slab slightly. 9528355f576SJeff Roberson * Currently it breaks on x86, and so it is disabled. 9538355f576SJeff Roberson */ 9548355f576SJeff Roberson 9558355f576SJeff Roberson if (zone->uz_align < UMA_CACHE_INC && waste > UMA_CACHE_INC) { 9568355f576SJeff Roberson zone->uz_cachemax = waste - UMA_CACHE_INC; 9578355f576SJeff Roberson zone->uz_cacheoff = 0; 9588355f576SJeff Roberson } 9598355f576SJeff Roberson 9608355f576SJeff Roberson totsize = zone->uz_pgoff + sizeof(struct uma_slab) 9618355f576SJeff Roberson + zone->uz_ipers; 9628355f576SJeff Roberson /* I don't think it's possible, but I'll make sure anyway */ 9638355f576SJeff Roberson if (totsize > UMA_SLAB_SIZE) { 9648355f576SJeff Roberson printf("zone %s ipers %d rsize %d size %d\n", 9658355f576SJeff Roberson zone->uz_name, zone->uz_ipers, zone->uz_rsize, 9668355f576SJeff Roberson zone->uz_size); 9678355f576SJeff Roberson panic("UMA slab won't fit.\n"); 9688355f576SJeff Roberson } 9698355f576SJeff Roberson } else { 9708355f576SJeff Roberson /* hash_expand here to allocate the initial hash table */ 9718355f576SJeff Roberson hash_expand(&zone->uz_hash); 9728355f576SJeff Roberson zone->uz_pgoff = 0; 9738355f576SJeff Roberson } 9748355f576SJeff Roberson 9758355f576SJeff Roberson #ifdef UMA_DEBUG 9768355f576SJeff Roberson printf("%s(%p) size = %d ipers = %d ppera = %d pgoff = %d\n", 9778355f576SJeff Roberson zone->uz_name, zone, 9788355f576SJeff Roberson zone->uz_size, zone->uz_ipers, 9798355f576SJeff Roberson zone->uz_ppera, zone->uz_pgoff); 9808355f576SJeff Roberson #endif 9818355f576SJeff Roberson ZONE_LOCK_INIT(zone); 9828355f576SJeff Roberson 9838355f576SJeff Roberson mtx_lock(&uma_mtx); 9848355f576SJeff Roberson LIST_INSERT_HEAD(&uma_zones, zone, uz_link); 9858355f576SJeff Roberson mtx_unlock(&uma_mtx); 9868355f576SJeff Roberson 9878355f576SJeff Roberson /* 9888355f576SJeff Roberson * Some internal zones don't have room allocated for the per cpu 9898355f576SJeff Roberson * caches. If we're internal, bail out here. 9908355f576SJeff Roberson */ 9918355f576SJeff Roberson 9928355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 9938355f576SJeff Roberson return; 9948355f576SJeff Roberson 9958355f576SJeff Roberson for (cpu = 0; cpu < maxcpu; cpu++) { 9968355f576SJeff Roberson if (zone->uz_ipers < UMA_BUCKET_SIZE) 9978355f576SJeff Roberson zone->uz_cpu[cpu].uc_count = zone->uz_ipers - 1; 9988355f576SJeff Roberson else 9998355f576SJeff Roberson zone->uz_cpu[cpu].uc_count = UMA_BUCKET_SIZE - 1; 10008355f576SJeff Roberson CPU_LOCK_INIT(zone, cpu); 10018355f576SJeff Roberson } 10028355f576SJeff Roberson } 10038355f576SJeff Roberson 10048355f576SJeff Roberson /* 10058355f576SJeff Roberson * Traverses every zone in the system and calls a callback 10068355f576SJeff Roberson * 10078355f576SJeff Roberson * Arguments: 10088355f576SJeff Roberson * zfunc A pointer to a function which accepts a zone 10098355f576SJeff Roberson * as an argument. 10108355f576SJeff Roberson * 10118355f576SJeff Roberson * Returns: 10128355f576SJeff Roberson * Nothing 10138355f576SJeff Roberson */ 10148355f576SJeff Roberson static void 10158355f576SJeff Roberson zone_foreach(void (*zfunc)(uma_zone_t)) 10168355f576SJeff Roberson { 10178355f576SJeff Roberson uma_zone_t zone; 10188355f576SJeff Roberson 10198355f576SJeff Roberson mtx_lock(&uma_mtx); 10208355f576SJeff Roberson LIST_FOREACH(zone, &uma_zones, uz_link) { 10218355f576SJeff Roberson zfunc(zone); 10228355f576SJeff Roberson } 10238355f576SJeff Roberson mtx_unlock(&uma_mtx); 10248355f576SJeff Roberson } 10258355f576SJeff Roberson 10268355f576SJeff Roberson /* Public functions */ 10278355f576SJeff Roberson /* See uma.h */ 10288355f576SJeff Roberson void 10298355f576SJeff Roberson uma_startup(void *bootmem) 10308355f576SJeff Roberson { 10318355f576SJeff Roberson struct uma_zctor_args args; 10328355f576SJeff Roberson uma_slab_t slab; 10338355f576SJeff Roberson int slabsize; 10348355f576SJeff Roberson int i; 10358355f576SJeff Roberson 10368355f576SJeff Roberson #ifdef UMA_DEBUG 10378355f576SJeff Roberson printf("Creating uma zone headers zone.\n"); 10388355f576SJeff Roberson #endif 10398355f576SJeff Roberson #ifdef SMP 10408355f576SJeff Roberson maxcpu = mp_maxid + 1; 10418355f576SJeff Roberson #else 10428355f576SJeff Roberson maxcpu = 1; 10438355f576SJeff Roberson #endif 10448355f576SJeff Roberson #ifdef UMA_DEBUG 10458355f576SJeff Roberson printf("Max cpu = %d, mp_maxid = %d\n", maxcpu, mp_maxid); 10468355f576SJeff Roberson Debugger("stop"); 10478355f576SJeff Roberson #endif 10488355f576SJeff Roberson mtx_init(&uma_mtx, "UMA lock", MTX_DEF); 10498355f576SJeff Roberson /* "manually" Create the initial zone */ 10508355f576SJeff Roberson args.name = "UMA Zones"; 10518355f576SJeff Roberson args.size = sizeof(struct uma_zone) + 10528355f576SJeff Roberson (sizeof(struct uma_cache) * (maxcpu - 1)); 10538355f576SJeff Roberson args.ctor = zone_ctor; 10548355f576SJeff Roberson args.dtor = NULL; 10558355f576SJeff Roberson args.uminit = zero_init; 10568355f576SJeff Roberson args.fini = NULL; 10578355f576SJeff Roberson args.align = 32 - 1; 10588355f576SJeff Roberson args.flags = UMA_ZONE_INTERNAL; 10598355f576SJeff Roberson /* The initial zone has no Per cpu queues so it's smaller */ 10608355f576SJeff Roberson zone_ctor(zones, sizeof(struct uma_zone), &args); 10618355f576SJeff Roberson 10628355f576SJeff Roberson #ifdef UMA_DEBUG 10638355f576SJeff Roberson printf("Filling boot free list.\n"); 10648355f576SJeff Roberson #endif 10658355f576SJeff Roberson for (i = 0; i < UMA_BOOT_PAGES; i++) { 10668355f576SJeff Roberson slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE)); 10678355f576SJeff Roberson slab->us_data = (u_int8_t *)slab; 10688355f576SJeff Roberson slab->us_flags = UMA_SLAB_BOOT; 10698355f576SJeff Roberson LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link); 10708355f576SJeff Roberson uma_boot_free++; 10718355f576SJeff Roberson } 10728355f576SJeff Roberson 10738355f576SJeff Roberson #ifdef UMA_DEBUG 10748355f576SJeff Roberson printf("Creating slab zone.\n"); 10758355f576SJeff Roberson #endif 10768355f576SJeff Roberson 10778355f576SJeff Roberson /* 10788355f576SJeff Roberson * This is the max number of free list items we'll have with 10798355f576SJeff Roberson * offpage slabs. 10808355f576SJeff Roberson */ 10818355f576SJeff Roberson 10828355f576SJeff Roberson slabsize = UMA_SLAB_SIZE - sizeof(struct uma_slab); 10838355f576SJeff Roberson slabsize /= UMA_MAX_WASTE; 10848355f576SJeff Roberson slabsize++; /* In case there it's rounded */ 10858355f576SJeff Roberson slabsize += sizeof(struct uma_slab); 10868355f576SJeff Roberson 10878355f576SJeff Roberson /* Now make a zone for slab headers */ 10888355f576SJeff Roberson slabzone = uma_zcreate("UMA Slabs", 10898355f576SJeff Roberson slabsize, 10908355f576SJeff Roberson NULL, NULL, NULL, NULL, 10918355f576SJeff Roberson UMA_ALIGN_PTR, UMA_ZONE_INTERNAL); 10928355f576SJeff Roberson 10938355f576SJeff Roberson hashzone = uma_zcreate("UMA Hash", 10948355f576SJeff Roberson sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 10958355f576SJeff Roberson NULL, NULL, NULL, NULL, 10968355f576SJeff Roberson UMA_ALIGN_PTR, UMA_ZONE_INTERNAL); 10978355f576SJeff Roberson 10988355f576SJeff Roberson bucketzone = uma_zcreate("UMA Buckets", sizeof(struct uma_bucket), 10998355f576SJeff Roberson NULL, NULL, NULL, NULL, 11008355f576SJeff Roberson UMA_ALIGN_PTR, UMA_ZONE_INTERNAL); 11018355f576SJeff Roberson 11028355f576SJeff Roberson 11038355f576SJeff Roberson #ifdef UMA_DEBUG 11048355f576SJeff Roberson printf("UMA startup complete.\n"); 11058355f576SJeff Roberson #endif 11068355f576SJeff Roberson } 11078355f576SJeff Roberson 11088355f576SJeff Roberson /* see uma.h */ 11098355f576SJeff Roberson void 11108355f576SJeff Roberson uma_startup2(void *hashmem, u_long elems) 11118355f576SJeff Roberson { 11128355f576SJeff Roberson bzero(hashmem, elems * sizeof(void *)); 11138355f576SJeff Roberson mallochash->uh_slab_hash = hashmem; 11148355f576SJeff Roberson mallochash->uh_hashsize = elems; 11158355f576SJeff Roberson mallochash->uh_hashmask = elems - 1; 11168355f576SJeff Roberson booted = 1; 11178355f576SJeff Roberson #ifdef UMA_DEBUG 11188355f576SJeff Roberson printf("UMA startup2 complete.\n"); 11198355f576SJeff Roberson #endif 11208355f576SJeff Roberson } 11218355f576SJeff Roberson 11228355f576SJeff Roberson /* 11238355f576SJeff Roberson * Initialize our callout handle 11248355f576SJeff Roberson * 11258355f576SJeff Roberson */ 11268355f576SJeff Roberson 11278355f576SJeff Roberson static void 11288355f576SJeff Roberson uma_startup3(void) 11298355f576SJeff Roberson { 11308355f576SJeff Roberson #ifdef UMA_DEBUG 11318355f576SJeff Roberson printf("Starting callout.\n"); 11328355f576SJeff Roberson #endif 11338355f576SJeff Roberson /* We'll be mpsafe once the vm is locked. */ 11348355f576SJeff Roberson callout_init(&uma_callout, 0); 11358355f576SJeff Roberson callout_reset(&uma_callout, UMA_WORKING_TIME * hz, uma_timeout, NULL); 11368355f576SJeff Roberson #ifdef UMA_DEBUG 11378355f576SJeff Roberson printf("UMA startup3 complete.\n"); 11388355f576SJeff Roberson #endif 11398355f576SJeff Roberson } 11408355f576SJeff Roberson 11418355f576SJeff Roberson /* See uma.h */ 11428355f576SJeff Roberson uma_zone_t 11438355f576SJeff Roberson uma_zcreate(char *name, int size, uma_ctor ctor, uma_dtor dtor, uma_init uminit, 11448355f576SJeff Roberson uma_fini fini, int align, u_int16_t flags) 11458355f576SJeff Roberson 11468355f576SJeff Roberson { 11478355f576SJeff Roberson struct uma_zctor_args args; 11488355f576SJeff Roberson 11498355f576SJeff Roberson /* This stuff is essential for the zone ctor */ 11508355f576SJeff Roberson args.name = name; 11518355f576SJeff Roberson args.size = size; 11528355f576SJeff Roberson args.ctor = ctor; 11538355f576SJeff Roberson args.dtor = dtor; 11548355f576SJeff Roberson args.uminit = uminit; 11558355f576SJeff Roberson args.fini = fini; 11568355f576SJeff Roberson args.align = align; 11578355f576SJeff Roberson args.flags = flags; 11588355f576SJeff Roberson 11598355f576SJeff Roberson return (uma_zalloc_internal(zones, &args, M_WAITOK, NULL, -1)); 11608355f576SJeff Roberson } 11618355f576SJeff Roberson 11628355f576SJeff Roberson /* See uma.h */ 11638355f576SJeff Roberson void * 11648355f576SJeff Roberson uma_zalloc_arg(uma_zone_t zone, void *udata, int wait) 11658355f576SJeff Roberson { 11668355f576SJeff Roberson void *item; 11678355f576SJeff Roberson uma_cache_t cache; 11688355f576SJeff Roberson uma_bucket_t bucket; 11698355f576SJeff Roberson int isitem; 11708355f576SJeff Roberson int cpu; 11718355f576SJeff Roberson 11728355f576SJeff Roberson /* This is the fast path allocation */ 11738355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC_1 11748355f576SJeff Roberson printf("Allocating one item from %s(%p)\n", zone->uz_name, zone); 11758355f576SJeff Roberson #endif 11768355f576SJeff Roberson cpu = PCPU_GET(cpuid); 11778355f576SJeff Roberson CPU_LOCK(zone, cpu); 11788355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 11798355f576SJeff Roberson cache->uc_allocs++; 11808355f576SJeff Roberson 11818355f576SJeff Roberson zalloc_start: 11828355f576SJeff Roberson bucket = cache->uc_allocbucket; 11838355f576SJeff Roberson 11848355f576SJeff Roberson if (bucket) { 11858355f576SJeff Roberson if (bucket->ub_ptr > -1) { 11868355f576SJeff Roberson item = bucket->ub_bucket[bucket->ub_ptr]; 11878355f576SJeff Roberson #ifdef INVARIANTS 11888355f576SJeff Roberson bucket->ub_bucket[bucket->ub_ptr] = NULL; 11898355f576SJeff Roberson #endif 11908355f576SJeff Roberson bucket->ub_ptr--; 11918355f576SJeff Roberson KASSERT(item != NULL, 11928355f576SJeff Roberson ("uma_zalloc: Bucket pointer mangled.")); 11938355f576SJeff Roberson cache->uc_allocs++; 11948355f576SJeff Roberson CPU_UNLOCK(zone, cpu); 11958355f576SJeff Roberson if (zone->uz_ctor) 11968355f576SJeff Roberson zone->uz_ctor(item, zone->uz_size, udata); 11978355f576SJeff Roberson return (item); 11988355f576SJeff Roberson } else if (cache->uc_freebucket) { 11998355f576SJeff Roberson /* 12008355f576SJeff Roberson * We have run out of items in our allocbucket. 12018355f576SJeff Roberson * See if we can switch with our free bucket. 12028355f576SJeff Roberson */ 12038355f576SJeff Roberson if (cache->uc_freebucket->ub_ptr > -1) { 12048355f576SJeff Roberson uma_bucket_t swap; 12058355f576SJeff Roberson 12068355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 12078355f576SJeff Roberson printf("uma_zalloc: Swapping empty with alloc.\n"); 12088355f576SJeff Roberson #endif 12098355f576SJeff Roberson swap = cache->uc_freebucket; 12108355f576SJeff Roberson cache->uc_freebucket = cache->uc_allocbucket; 12118355f576SJeff Roberson cache->uc_allocbucket = swap; 12128355f576SJeff Roberson 12138355f576SJeff Roberson goto zalloc_start; 12148355f576SJeff Roberson } 12158355f576SJeff Roberson } 12168355f576SJeff Roberson } 12178355f576SJeff Roberson /* 12188355f576SJeff Roberson * We can get here for three reasons: 12198355f576SJeff Roberson * 12208355f576SJeff Roberson * 1) The buckets are NULL 12218355f576SJeff Roberson * 2) The zone is INTERNAL, and so it has no buckets. 12228355f576SJeff Roberson * 3) The alloc and free buckets are both empty. 12238355f576SJeff Roberson * 12248355f576SJeff Roberson * Just handoff to uma_zalloc_internal to do the hard stuff 12258355f576SJeff Roberson * 12268355f576SJeff Roberson */ 12278355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 12288355f576SJeff Roberson printf("uma_zalloc: Falling back to zalloc_internal.\n"); 12298355f576SJeff Roberson #endif 12308355f576SJeff Roberson 12318355f576SJeff Roberson item = uma_zalloc_internal(zone, udata, wait, &isitem, cpu); 12328355f576SJeff Roberson 12338355f576SJeff Roberson #ifdef UMA_DEBUG 12348355f576SJeff Roberson printf("uma_zalloc: zalloc_internal completed.\n"); 12358355f576SJeff Roberson #endif 12368355f576SJeff Roberson 12378355f576SJeff Roberson if (item && isitem == 0) 12388355f576SJeff Roberson goto zalloc_start; 12398355f576SJeff Roberson 12408355f576SJeff Roberson /* 12418355f576SJeff Roberson * If isitem is set then we should just return it. The cpu lock 12428355f576SJeff Roberson * was unlocked when we couldn't get a bucket. 12438355f576SJeff Roberson */ 12448355f576SJeff Roberson return item; 12458355f576SJeff Roberson } 12468355f576SJeff Roberson 12478355f576SJeff Roberson /* 12488355f576SJeff Roberson * Allocates an item for an internal zone OR fills a bucket 12498355f576SJeff Roberson * 12508355f576SJeff Roberson * Arguments 12518355f576SJeff Roberson * zone The zone to alloc for. 12528355f576SJeff Roberson * udata The data to be passed to the constructor. 12538355f576SJeff Roberson * wait M_WAITOK or M_NOWAIT. 12548355f576SJeff Roberson * isitem The returned value is an item if this is true. 12558355f576SJeff Roberson * cpu The cpu # of the cache that we should use, or -1. 12568355f576SJeff Roberson * 12578355f576SJeff Roberson * Returns 12588355f576SJeff Roberson * NULL if there is no memory and M_NOWAIT is set 12598355f576SJeff Roberson * An item if called on an interal zone 12608355f576SJeff Roberson * Non NULL if called to fill a bucket and it was successful. 12618355f576SJeff Roberson * 12628355f576SJeff Roberson * Discussion: 12638355f576SJeff Roberson * This was much cleaner before it had to do per cpu caches. It is 12648355f576SJeff Roberson * complicated now because it has to handle the simple internal case, and 12658355f576SJeff Roberson * the more involved bucket filling and allocation. The isitem is there 12668355f576SJeff Roberson * to remove a failure case. You shouldn't fail on allocating from a zone 12678355f576SJeff Roberson * because there were no buckets. This allows the exported zalloc to just 12688355f576SJeff Roberson * return the item. 12698355f576SJeff Roberson * 12708355f576SJeff Roberson */ 12718355f576SJeff Roberson 12728355f576SJeff Roberson static void * 12738355f576SJeff Roberson uma_zalloc_internal(uma_zone_t zone, void *udata, int wait, int *isitem, int cpu) 12748355f576SJeff Roberson { 12758355f576SJeff Roberson uma_bucket_t bucket; 12768355f576SJeff Roberson uma_cache_t cache; 12778355f576SJeff Roberson uma_slab_t slab; 12788355f576SJeff Roberson u_int8_t freei; 12798355f576SJeff Roberson void *item; 12808355f576SJeff Roberson 12818355f576SJeff Roberson bucket = NULL; 12828355f576SJeff Roberson cache = NULL; 12838355f576SJeff Roberson item = NULL; 12848355f576SJeff Roberson 12858355f576SJeff Roberson /* 12868355f576SJeff Roberson * This is to stop us from allocating per cpu buckets while we're running 12878355f576SJeff Roberson * out of UMA_BOOT_PAGES. Otherwise, we would exhaust the boot pages. 12888355f576SJeff Roberson */ 12898355f576SJeff Roberson 12908355f576SJeff Roberson if (!booted && zone == bucketzone) 12918355f576SJeff Roberson return (NULL); 12928355f576SJeff Roberson 12938355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 12948355f576SJeff Roberson printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); 12958355f576SJeff Roberson #endif 12968355f576SJeff Roberson if (isitem != NULL) 12978355f576SJeff Roberson *isitem = 0; 12988355f576SJeff Roberson 12998355f576SJeff Roberson ZONE_LOCK(zone); 13008355f576SJeff Roberson 13018355f576SJeff Roberson /* We got here because we need to fill some buckets */ 13028355f576SJeff Roberson if (cpu != -1) { 13038355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 13048355f576SJeff Roberson 13058355f576SJeff Roberson zone->uz_allocs += cache->uc_allocs; 13068355f576SJeff Roberson /* Check the free list */ 13078355f576SJeff Roberson bucket = LIST_FIRST(&zone->uz_full_bucket); 13088355f576SJeff Roberson if (bucket) { 13098355f576SJeff Roberson LIST_REMOVE(bucket, ub_link); 13108355f576SJeff Roberson /* Our old one is now a free bucket */ 13118355f576SJeff Roberson if (cache->uc_allocbucket) { 13128355f576SJeff Roberson KASSERT(cache->uc_allocbucket->ub_ptr == -1, 13138355f576SJeff Roberson ("uma_zalloc_internal: Freeing a non free bucket.")); 13148355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_bucket, 13158355f576SJeff Roberson cache->uc_allocbucket, ub_link); 13168355f576SJeff Roberson } 13178355f576SJeff Roberson KASSERT(bucket->ub_ptr != -1, 13188355f576SJeff Roberson ("uma_zalloc_internal: Returning an empty bucket.")); 13198355f576SJeff Roberson /*zone->uz_free -= bucket->ub_ptr + 1;*/ 13208355f576SJeff Roberson cache->uc_allocbucket = bucket; 13218355f576SJeff Roberson ZONE_UNLOCK(zone); 13228355f576SJeff Roberson return (bucket); 13238355f576SJeff Roberson } 13248355f576SJeff Roberson /* Bump up our uc_count so we get here less */ 13258355f576SJeff Roberson if (cache->uc_count < UMA_BUCKET_SIZE - 1) 13268355f576SJeff Roberson cache->uc_count++; 13278355f576SJeff Roberson /* Nothing on the free list, try to re-use the old one */ 13288355f576SJeff Roberson bucket = cache->uc_allocbucket; 13298355f576SJeff Roberson if (bucket == NULL) { 13308355f576SJeff Roberson /* Nope, we need a new one */ 13318355f576SJeff Roberson CPU_UNLOCK(zone, cpu); 13328355f576SJeff Roberson ZONE_UNLOCK(zone); 13338355f576SJeff Roberson bucket = uma_zalloc_internal(bucketzone, 13348355f576SJeff Roberson NULL, wait, NULL, -1); 13358355f576SJeff Roberson CPU_LOCK(zone, cpu); 13368355f576SJeff Roberson ZONE_LOCK(zone); 13378355f576SJeff Roberson /* Did we lose the race? */ 13388355f576SJeff Roberson if (cache->uc_allocbucket) { 13398355f576SJeff Roberson #ifdef UMA_DEBUG 13408355f576SJeff Roberson printf("uma_zalloc_internal: Lost race with another CPU.\n"); 13418355f576SJeff Roberson #endif 13428355f576SJeff Roberson if (bucket) 13438355f576SJeff Roberson uma_zfree_internal(bucketzone, 13448355f576SJeff Roberson bucket, NULL, 0); 13458355f576SJeff Roberson ZONE_UNLOCK(zone); 13468355f576SJeff Roberson return (cache->uc_allocbucket); 13478355f576SJeff Roberson } 13488355f576SJeff Roberson cache->uc_allocbucket = bucket; 13498355f576SJeff Roberson 13508355f576SJeff Roberson if (bucket) { 13518355f576SJeff Roberson #ifdef INVARIANTS 13528355f576SJeff Roberson bzero(bucket, bucketzone->uz_size); 13538355f576SJeff Roberson #endif 13548355f576SJeff Roberson bucket->ub_ptr = -1; 13558355f576SJeff Roberson } else { 13568355f576SJeff Roberson /* 13578355f576SJeff Roberson * We may not get a bucket if we recurse, so 13588355f576SJeff Roberson * return an actual item. The rest of this code 13598355f576SJeff Roberson * does the right thing if the cache is NULL. 13608355f576SJeff Roberson */ 13618355f576SJeff Roberson #ifdef UMA_DEBUG 13628355f576SJeff Roberson printf("uma_zalloc_internal: Bucketzone returned NULL\n"); 13638355f576SJeff Roberson #endif 13648355f576SJeff Roberson CPU_UNLOCK(zone, cpu); 13658355f576SJeff Roberson cache = NULL; 13668355f576SJeff Roberson cpu = -1; 13678355f576SJeff Roberson } 13688355f576SJeff Roberson } 13698355f576SJeff Roberson } 13708355f576SJeff Roberson 13718355f576SJeff Roberson new_slab: 13728355f576SJeff Roberson 13738355f576SJeff Roberson /* Find a slab with some space */ 13748355f576SJeff Roberson if (zone->uz_free) { 13758355f576SJeff Roberson if (!LIST_EMPTY(&zone->uz_part_slab)) { 13768355f576SJeff Roberson slab = LIST_FIRST(&zone->uz_part_slab); 13778355f576SJeff Roberson } else { 13788355f576SJeff Roberson slab = LIST_FIRST(&zone->uz_free_slab); 13798355f576SJeff Roberson LIST_REMOVE(slab, us_link); 13808355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); 13818355f576SJeff Roberson } 13828355f576SJeff Roberson } else { 13838355f576SJeff Roberson /* 13848355f576SJeff Roberson * This is to prevent us from recursively trying to allocate 13858355f576SJeff Roberson * buckets. The problem is that if an allocation forces us to 13868355f576SJeff Roberson * grab a new bucket we will call page_alloc, which will go off 13878355f576SJeff Roberson * and cause the vm to allocate vm_map_entries. If we need new 13888355f576SJeff Roberson * buckets there too we will recurse in kmem_alloc and bad 13898355f576SJeff Roberson * things happen. So instead we return a NULL bucket, and make 13908355f576SJeff Roberson * the code that allocates buckets smart enough to deal with it */ 13918355f576SJeff Roberson if (zone == bucketzone && zone->uz_recurse != 0) { 13928355f576SJeff Roberson ZONE_UNLOCK(zone); 13938355f576SJeff Roberson return (NULL); 13948355f576SJeff Roberson } 13958355f576SJeff Roberson zone->uz_recurse++; 13968355f576SJeff Roberson slab = slab_zalloc(zone, wait); 13978355f576SJeff Roberson zone->uz_recurse--; 13988355f576SJeff Roberson if (slab) { 13998355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); 14008355f576SJeff Roberson /* 14018355f576SJeff Roberson * We might not have been able to get a page, but another cpu 14028355f576SJeff Roberson * could have while we were unlocked. 14038355f576SJeff Roberson */ 14048355f576SJeff Roberson } else if (zone->uz_free == 0) { 14058355f576SJeff Roberson ZONE_UNLOCK(zone); 14068355f576SJeff Roberson /* If we're filling a bucket return what we have */ 14078355f576SJeff Roberson if (bucket != NULL && bucket->ub_ptr != -1) { 14088355f576SJeff Roberson return (bucket); 14098355f576SJeff Roberson } else 14108355f576SJeff Roberson return (NULL); 14118355f576SJeff Roberson } else { 14128355f576SJeff Roberson /* Another cpu must have succeeded */ 14138355f576SJeff Roberson if ((slab = LIST_FIRST(&zone->uz_part_slab)) == NULL) { 14148355f576SJeff Roberson slab = LIST_FIRST(&zone->uz_free_slab); 14158355f576SJeff Roberson LIST_REMOVE(slab, us_link); 14168355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_part_slab, 14178355f576SJeff Roberson slab, us_link); 14188355f576SJeff Roberson } 14198355f576SJeff Roberson } 14208355f576SJeff Roberson } 14218355f576SJeff Roberson 14228355f576SJeff Roberson while (slab->us_freecount) { 14238355f576SJeff Roberson freei = slab->us_firstfree; 14248355f576SJeff Roberson slab->us_firstfree = slab->us_freelist[freei]; 14258355f576SJeff Roberson #ifdef INVARIANTS 14268355f576SJeff Roberson slab->us_freelist[freei] = 255; 14278355f576SJeff Roberson #endif 14288355f576SJeff Roberson slab->us_freecount--; 14298355f576SJeff Roberson zone->uz_free--; 14308355f576SJeff Roberson item = slab->us_data + (zone->uz_rsize * freei); 14318355f576SJeff Roberson 14328355f576SJeff Roberson if (cache == NULL) { 14338355f576SJeff Roberson zone->uz_allocs++; 14348355f576SJeff Roberson break; 14358355f576SJeff Roberson } 14368355f576SJeff Roberson 14378355f576SJeff Roberson bucket->ub_bucket[++bucket->ub_ptr] = item; 14388355f576SJeff Roberson 14398355f576SJeff Roberson /* Don't overfill the bucket! */ 14408355f576SJeff Roberson if (bucket->ub_ptr == cache->uc_count) 14418355f576SJeff Roberson break; 14428355f576SJeff Roberson } 14438355f576SJeff Roberson 14448355f576SJeff Roberson /* Move this slab to the full list */ 14458355f576SJeff Roberson if (slab->us_freecount == 0) { 14468355f576SJeff Roberson LIST_REMOVE(slab, us_link); 14478355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_full_slab, slab, us_link); 14488355f576SJeff Roberson } 14498355f576SJeff Roberson 14508355f576SJeff Roberson if (cache != NULL) { 14518355f576SJeff Roberson /* Try to keep the buckets totally full, but don't block */ 14528355f576SJeff Roberson if (bucket->ub_ptr < cache->uc_count) { 14538355f576SJeff Roberson wait = M_NOWAIT; 14548355f576SJeff Roberson goto new_slab; 14558355f576SJeff Roberson } 14568355f576SJeff Roberson } 14578355f576SJeff Roberson 14588355f576SJeff Roberson ZONE_UNLOCK(zone); 14598355f576SJeff Roberson 14608355f576SJeff Roberson /* Only construct at this time if we're not filling a bucket */ 14618355f576SJeff Roberson if (cache == NULL) { 14628355f576SJeff Roberson if (zone->uz_ctor) 14638355f576SJeff Roberson zone->uz_ctor(item, zone->uz_size, udata); 14648355f576SJeff Roberson 14658355f576SJeff Roberson if (isitem != NULL) 14668355f576SJeff Roberson *isitem = 1; 14678355f576SJeff Roberson } 14688355f576SJeff Roberson 14698355f576SJeff Roberson return (item); 14708355f576SJeff Roberson } 14718355f576SJeff Roberson 14728355f576SJeff Roberson /* See uma.h */ 14738355f576SJeff Roberson void 14748355f576SJeff Roberson uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 14758355f576SJeff Roberson { 14768355f576SJeff Roberson uma_cache_t cache; 14778355f576SJeff Roberson uma_bucket_t bucket; 14788355f576SJeff Roberson int cpu; 14798355f576SJeff Roberson 14808355f576SJeff Roberson /* This is the fast path free */ 14818355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC_1 14828355f576SJeff Roberson printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone); 14838355f576SJeff Roberson #endif 14848355f576SJeff Roberson cpu = PCPU_GET(cpuid); 14858355f576SJeff Roberson CPU_LOCK(zone, cpu); 14868355f576SJeff Roberson cache = &zone->uz_cpu[cpu]; 14878355f576SJeff Roberson 14888355f576SJeff Roberson zfree_start: 14898355f576SJeff Roberson bucket = cache->uc_freebucket; 14908355f576SJeff Roberson 14918355f576SJeff Roberson if (bucket) { 14928355f576SJeff Roberson /* Do we have room in our bucket? */ 14938355f576SJeff Roberson if (bucket->ub_ptr < cache->uc_count) { 14948355f576SJeff Roberson bucket->ub_ptr++; 14958355f576SJeff Roberson KASSERT(bucket->ub_bucket[bucket->ub_ptr] == NULL, 14968355f576SJeff Roberson ("uma_zfree: Freeing to non free bucket index.")); 14978355f576SJeff Roberson bucket->ub_bucket[bucket->ub_ptr] = item; 14988355f576SJeff Roberson CPU_UNLOCK(zone, cpu); 14998355f576SJeff Roberson if (zone->uz_dtor) 15008355f576SJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 15018355f576SJeff Roberson return; 15028355f576SJeff Roberson } else if (cache->uc_allocbucket) { 15038355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 15048355f576SJeff Roberson printf("uma_zfree: Swapping buckets.\n"); 15058355f576SJeff Roberson #endif 15068355f576SJeff Roberson /* 15078355f576SJeff Roberson * We have run out of space in our freebucket. 15088355f576SJeff Roberson * See if we can switch with our alloc bucket. 15098355f576SJeff Roberson */ 15108355f576SJeff Roberson if (cache->uc_allocbucket->ub_ptr < 15118355f576SJeff Roberson cache->uc_freebucket->ub_ptr) { 15128355f576SJeff Roberson uma_bucket_t swap; 15138355f576SJeff Roberson 15148355f576SJeff Roberson swap = cache->uc_freebucket; 15158355f576SJeff Roberson cache->uc_freebucket = cache->uc_allocbucket; 15168355f576SJeff Roberson cache->uc_allocbucket = swap; 15178355f576SJeff Roberson 15188355f576SJeff Roberson goto zfree_start; 15198355f576SJeff Roberson } 15208355f576SJeff Roberson } 15218355f576SJeff Roberson } 15228355f576SJeff Roberson 15238355f576SJeff Roberson /* 15248355f576SJeff Roberson * We can get here for three reasons: 15258355f576SJeff Roberson * 15268355f576SJeff Roberson * 1) The buckets are NULL 15278355f576SJeff Roberson * 2) The zone is INTERNAL, and so it has no buckets. 15288355f576SJeff Roberson * 3) The alloc and free buckets are both somewhat full. 15298355f576SJeff Roberson * 15308355f576SJeff Roberson */ 15318355f576SJeff Roberson 15328355f576SJeff Roberson ZONE_LOCK(zone); 15338355f576SJeff Roberson 15348355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) { 15358355f576SJeff Roberson bucket = cache->uc_freebucket; 15368355f576SJeff Roberson cache->uc_freebucket = NULL; 15378355f576SJeff Roberson 15388355f576SJeff Roberson /* Can we throw this on the zone full list? */ 15398355f576SJeff Roberson if (bucket != NULL) { 15408355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 15418355f576SJeff Roberson printf("uma_zfree: Putting old bucket on the free list.\n"); 15428355f576SJeff Roberson #endif 15438355f576SJeff Roberson /* ub_ptr is pointing to the last free item */ 15448355f576SJeff Roberson KASSERT(bucket->ub_ptr != -1, 15458355f576SJeff Roberson ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n")); 15468355f576SJeff Roberson /*zone->uz_free += bucket->ub_ptr + 1;*/ 15478355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_full_bucket, 15488355f576SJeff Roberson bucket, ub_link); 15498355f576SJeff Roberson bucket = LIST_FIRST(&zone->uz_free_bucket); 15508355f576SJeff Roberson if (bucket) 15518355f576SJeff Roberson LIST_REMOVE(bucket, ub_link); 15528355f576SJeff Roberson } 15538355f576SJeff Roberson /* 15548355f576SJeff Roberson * Do we need to alloc one? Either the freebucket was NULL 15558355f576SJeff Roberson * or the free_bucket list was empty. 15568355f576SJeff Roberson */ 15578355f576SJeff Roberson if (bucket == NULL) { 15588355f576SJeff Roberson #ifdef UMA_DEBUG_ALLOC 15598355f576SJeff Roberson printf("uma_zfree: Allocating new free bucket.\n"); 15608355f576SJeff Roberson #endif 15618355f576SJeff Roberson /* This has to be done so we don't recurse on a lock */ 15628355f576SJeff Roberson ZONE_UNLOCK(zone); 15638355f576SJeff Roberson CPU_UNLOCK(zone, cpu); 15648355f576SJeff Roberson bucket = uma_zalloc_internal(bucketzone, 15658355f576SJeff Roberson NULL, M_NOWAIT, NULL, -1); 15668355f576SJeff Roberson CPU_LOCK(zone, cpu); 15678355f576SJeff Roberson ZONE_LOCK(zone); 15688355f576SJeff Roberson if (bucket) { 15698355f576SJeff Roberson #ifdef INVARIANTS 15708355f576SJeff Roberson bzero(bucket, bucketzone->uz_size); 15718355f576SJeff Roberson #endif 15728355f576SJeff Roberson bucket->ub_ptr = -1; 15738355f576SJeff Roberson } 15748355f576SJeff Roberson /* Did we lose the race? */ 15758355f576SJeff Roberson if (cache->uc_freebucket != NULL) { 15768355f576SJeff Roberson if (bucket) 15778355f576SJeff Roberson uma_zfree_internal(bucketzone, 15788355f576SJeff Roberson bucket, NULL, 0); 15798355f576SJeff Roberson ZONE_UNLOCK(zone); 15808355f576SJeff Roberson goto zfree_start; 15818355f576SJeff Roberson } 15828355f576SJeff Roberson /* If we couldn't get one just free directly */ 15838355f576SJeff Roberson if (bucket == NULL) 15848355f576SJeff Roberson goto zfree_internal; 15858355f576SJeff Roberson } 15868355f576SJeff Roberson cache->uc_freebucket = bucket; 15878355f576SJeff Roberson ZONE_UNLOCK(zone); 15888355f576SJeff Roberson goto zfree_start; 15898355f576SJeff Roberson } 15908355f576SJeff Roberson 15918355f576SJeff Roberson zfree_internal: 15928355f576SJeff Roberson 15938355f576SJeff Roberson CPU_UNLOCK(zone, cpu); 15948355f576SJeff Roberson ZONE_UNLOCK(zone); 15958355f576SJeff Roberson uma_zfree_internal(zone, item, udata, 0); 15968355f576SJeff Roberson 15978355f576SJeff Roberson return; 15988355f576SJeff Roberson 15998355f576SJeff Roberson } 16008355f576SJeff Roberson 16018355f576SJeff Roberson /* 16028355f576SJeff Roberson * Frees an item to an INTERNAL zone or allocates a free bucket 16038355f576SJeff Roberson * 16048355f576SJeff Roberson * Arguments: 16058355f576SJeff Roberson * zone The zone to free to 16068355f576SJeff Roberson * item The item we're freeing 16078355f576SJeff Roberson * udata User supplied data for the dtor 16088355f576SJeff Roberson * skip Skip the dtor, it was done in uma_zfree_arg 16098355f576SJeff Roberson */ 16108355f576SJeff Roberson 16118355f576SJeff Roberson static void 16128355f576SJeff Roberson uma_zfree_internal(uma_zone_t zone, void *item, void *udata, int skip) 16138355f576SJeff Roberson { 16148355f576SJeff Roberson uma_slab_t slab; 16158355f576SJeff Roberson u_int8_t *mem; 16168355f576SJeff Roberson u_int8_t freei; 16178355f576SJeff Roberson 16188355f576SJeff Roberson ZONE_LOCK(zone); 16198355f576SJeff Roberson 16208355f576SJeff Roberson if (!(zone->uz_flags & UMA_ZFLAG_MALLOC)) { 16218355f576SJeff Roberson mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); 16228355f576SJeff Roberson if (zone->uz_flags & UMA_ZFLAG_OFFPAGE) 16238355f576SJeff Roberson slab = hash_sfind(&zone->uz_hash, mem); 16248355f576SJeff Roberson else { 16258355f576SJeff Roberson mem += zone->uz_pgoff; 16268355f576SJeff Roberson slab = (uma_slab_t)mem; 16278355f576SJeff Roberson } 16288355f576SJeff Roberson } else { 16298355f576SJeff Roberson slab = (uma_slab_t)udata; 16308355f576SJeff Roberson } 16318355f576SJeff Roberson 16328355f576SJeff Roberson /* Do we need to remove from any lists? */ 16338355f576SJeff Roberson if (slab->us_freecount+1 == zone->uz_ipers) { 16348355f576SJeff Roberson LIST_REMOVE(slab, us_link); 16358355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_slab, slab, us_link); 16368355f576SJeff Roberson } else if (slab->us_freecount == 0) { 16378355f576SJeff Roberson LIST_REMOVE(slab, us_link); 16388355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); 16398355f576SJeff Roberson } 16408355f576SJeff Roberson 16418355f576SJeff Roberson /* Slab management stuff */ 16428355f576SJeff Roberson freei = ((unsigned long)item - (unsigned long)slab->us_data) 16438355f576SJeff Roberson / zone->uz_rsize; 16448355f576SJeff Roberson #ifdef INVARIANTS 16458355f576SJeff Roberson if (((freei * zone->uz_rsize) + slab->us_data) != item) 16468355f576SJeff Roberson panic("zone: %s(%p) slab %p freed address %p unaligned.\n", 16478355f576SJeff Roberson zone->uz_name, zone, slab, item); 16488355f576SJeff Roberson if (freei >= zone->uz_ipers) 16498355f576SJeff Roberson panic("zone: %s(%p) slab %p freelist %i out of range 0-%d\n", 16508355f576SJeff Roberson zone->uz_name, zone, slab, freei, zone->uz_ipers-1); 16518355f576SJeff Roberson 16528355f576SJeff Roberson if (slab->us_freelist[freei] != 255) { 16538355f576SJeff Roberson printf("Slab at %p, freei %d = %d.\n", 16548355f576SJeff Roberson slab, freei, slab->us_freelist[freei]); 16558355f576SJeff Roberson panic("Duplicate free of item %p from zone %p(%s)\n", 16568355f576SJeff Roberson item, zone, zone->uz_name); 16578355f576SJeff Roberson } 16588355f576SJeff Roberson #endif 16598355f576SJeff Roberson slab->us_freelist[freei] = slab->us_firstfree; 16608355f576SJeff Roberson slab->us_firstfree = freei; 16618355f576SJeff Roberson slab->us_freecount++; 16628355f576SJeff Roberson 16638355f576SJeff Roberson /* Zone statistics */ 16648355f576SJeff Roberson zone->uz_free++; 16658355f576SJeff Roberson 16668355f576SJeff Roberson ZONE_UNLOCK(zone); 16678355f576SJeff Roberson 16688355f576SJeff Roberson if (!skip && zone->uz_dtor) 16698355f576SJeff Roberson zone->uz_dtor(item, zone->uz_size, udata); 16708355f576SJeff Roberson } 16718355f576SJeff Roberson 16728355f576SJeff Roberson /* See uma.h */ 16738355f576SJeff Roberson void 1674736ee590SJeff Roberson uma_zone_set_max(uma_zone_t zone, int nitems) 1675736ee590SJeff Roberson { 1676736ee590SJeff Roberson ZONE_LOCK(zone); 1677736ee590SJeff Roberson if (zone->uz_ppera > 1) 1678736ee590SJeff Roberson zone->uz_maxpages = nitems / zone->uz_ppera; 1679736ee590SJeff Roberson else 1680736ee590SJeff Roberson zone->uz_maxpages = nitems / zone->uz_ipers; 1681736ee590SJeff Roberson ZONE_UNLOCK(zone); 1682736ee590SJeff Roberson } 1683736ee590SJeff Roberson 1684736ee590SJeff Roberson /* See uma.h */ 1685736ee590SJeff Roberson void 16868355f576SJeff Roberson uma_zone_set_freef(uma_zone_t zone, uma_free freef) 16878355f576SJeff Roberson { 16888355f576SJeff Roberson ZONE_LOCK(zone); 16898355f576SJeff Roberson 16908355f576SJeff Roberson zone->uz_freef = freef; 16918355f576SJeff Roberson 16928355f576SJeff Roberson ZONE_UNLOCK(zone); 16938355f576SJeff Roberson } 16948355f576SJeff Roberson 16958355f576SJeff Roberson /* See uma.h */ 16968355f576SJeff Roberson void 16978355f576SJeff Roberson uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 16988355f576SJeff Roberson { 16998355f576SJeff Roberson ZONE_LOCK(zone); 17008355f576SJeff Roberson 17018355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_PRIVALLOC; 17028355f576SJeff Roberson zone->uz_allocf = allocf; 17038355f576SJeff Roberson 17048355f576SJeff Roberson ZONE_UNLOCK(zone); 17058355f576SJeff Roberson } 17068355f576SJeff Roberson 17078355f576SJeff Roberson /* See uma.h */ 17088355f576SJeff Roberson int 17098355f576SJeff Roberson uma_zone_set_obj(uma_zone_t zone, struct vm_object *obj, int count) 17108355f576SJeff Roberson { 17118355f576SJeff Roberson int pages; 17128355f576SJeff Roberson vm_offset_t kva; 17138355f576SJeff Roberson 17148355f576SJeff Roberson ZONE_LOCK(zone); 17158355f576SJeff Roberson mtx_lock(&Giant); 17168355f576SJeff Roberson 17178355f576SJeff Roberson zone->uz_obj = obj; 17188355f576SJeff Roberson pages = count / zone->uz_ipers; 17198355f576SJeff Roberson 17208355f576SJeff Roberson if (pages * zone->uz_ipers < count) 17218355f576SJeff Roberson pages++; 17228355f576SJeff Roberson zone->uz_kva = NULL; 17238355f576SJeff Roberson ZONE_UNLOCK(zone); 17248355f576SJeff Roberson kva = kmem_alloc_pageable(kernel_map, pages * UMA_SLAB_SIZE); 17258355f576SJeff Roberson ZONE_LOCK(zone); 17268355f576SJeff Roberson 17278355f576SJeff Roberson zone->uz_kva = kva; 17288355f576SJeff Roberson 17298355f576SJeff Roberson if (zone->uz_kva == 0) { 17308355f576SJeff Roberson ZONE_UNLOCK(zone); 17318355f576SJeff Roberson return (0); 17328355f576SJeff Roberson } 17338355f576SJeff Roberson 17348355f576SJeff Roberson zone->uz_maxpages = pages; 17358355f576SJeff Roberson 17368355f576SJeff Roberson if (zone->uz_obj == NULL) 17378355f576SJeff Roberson zone->uz_obj = vm_object_allocate(OBJT_DEFAULT, 17388355f576SJeff Roberson zone->uz_maxpages); 17398355f576SJeff Roberson else 17408355f576SJeff Roberson _vm_object_allocate(OBJT_DEFAULT, 17418355f576SJeff Roberson zone->uz_maxpages, zone->uz_obj); 17428355f576SJeff Roberson 17438355f576SJeff Roberson zone->uz_allocf = obj_alloc; 17448355f576SJeff Roberson zone->uz_flags |= UMA_ZFLAG_NOFREE | UMA_ZFLAG_PRIVALLOC; 17458355f576SJeff Roberson 17468355f576SJeff Roberson mtx_unlock(&Giant); 17478355f576SJeff Roberson ZONE_UNLOCK(zone); 17488355f576SJeff Roberson 17498355f576SJeff Roberson return (1); 17508355f576SJeff Roberson } 17518355f576SJeff Roberson 17528355f576SJeff Roberson /* See uma.h */ 17538355f576SJeff Roberson void 17548355f576SJeff Roberson uma_prealloc(uma_zone_t zone, int items) 17558355f576SJeff Roberson { 17568355f576SJeff Roberson int slabs; 17578355f576SJeff Roberson uma_slab_t slab; 17588355f576SJeff Roberson 17598355f576SJeff Roberson ZONE_LOCK(zone); 17608355f576SJeff Roberson slabs = items / zone->uz_ipers; 17618355f576SJeff Roberson if (slabs * zone->uz_ipers < items) 17628355f576SJeff Roberson slabs++; 17638355f576SJeff Roberson 17648355f576SJeff Roberson while (slabs > 0) { 17658355f576SJeff Roberson slab = slab_zalloc(zone, M_WAITOK); 17668355f576SJeff Roberson LIST_INSERT_HEAD(&zone->uz_free_slab, slab, us_link); 17678355f576SJeff Roberson slabs--; 17688355f576SJeff Roberson } 17698355f576SJeff Roberson ZONE_UNLOCK(zone); 17708355f576SJeff Roberson } 17718355f576SJeff Roberson 17728355f576SJeff Roberson /* See uma.h */ 17738355f576SJeff Roberson void 17748355f576SJeff Roberson uma_reclaim(void) 17758355f576SJeff Roberson { 17768355f576SJeff Roberson /* 17778355f576SJeff Roberson * You might think that the delay below would improve performance since 17788355f576SJeff Roberson * the allocator will give away memory that it may ask for immediately. 17798355f576SJeff Roberson * Really, it makes things worse, since cpu cycles are so much cheaper 17808355f576SJeff Roberson * than disk activity. 17818355f576SJeff Roberson */ 17828355f576SJeff Roberson #if 0 17838355f576SJeff Roberson static struct timeval tv = {0}; 17848355f576SJeff Roberson struct timeval now; 17858355f576SJeff Roberson getmicrouptime(&now); 17868355f576SJeff Roberson if (now.tv_sec > tv.tv_sec + 30) 17878355f576SJeff Roberson tv = now; 17888355f576SJeff Roberson else 17898355f576SJeff Roberson return; 17908355f576SJeff Roberson #endif 17918355f576SJeff Roberson #ifdef UMA_DEBUG 17928355f576SJeff Roberson printf("UMA: vm asked us to release pages!\n"); 17938355f576SJeff Roberson #endif 17948355f576SJeff Roberson zone_foreach(zone_drain); 17958355f576SJeff Roberson 17968355f576SJeff Roberson /* 17978355f576SJeff Roberson * Some slabs may have been freed but this zone will be visited early 17988355f576SJeff Roberson * we visit again so that we can free pages that are empty once other 17998355f576SJeff Roberson * zones are drained. We have to do the same for buckets. 18008355f576SJeff Roberson */ 18018355f576SJeff Roberson zone_drain(slabzone); 18028355f576SJeff Roberson zone_drain(bucketzone); 18038355f576SJeff Roberson } 18048355f576SJeff Roberson 18058355f576SJeff Roberson void * 18068355f576SJeff Roberson uma_large_malloc(int size, int wait) 18078355f576SJeff Roberson { 18088355f576SJeff Roberson void *mem; 18098355f576SJeff Roberson uma_slab_t slab; 18108355f576SJeff Roberson u_int8_t flags; 18118355f576SJeff Roberson 18128355f576SJeff Roberson slab = uma_zalloc_internal(slabzone, NULL, wait, NULL, -1); 18138355f576SJeff Roberson if (slab == NULL) 18148355f576SJeff Roberson return (NULL); 18158355f576SJeff Roberson 18168355f576SJeff Roberson mem = page_alloc(NULL, size, &flags, wait); 18178355f576SJeff Roberson if (mem) { 18188355f576SJeff Roberson slab->us_data = mem; 18198355f576SJeff Roberson slab->us_flags = flags | UMA_SLAB_MALLOC; 18208355f576SJeff Roberson slab->us_size = size; 18218355f576SJeff Roberson UMA_HASH_INSERT(mallochash, slab, mem); 18228355f576SJeff Roberson } else { 18238355f576SJeff Roberson uma_zfree_internal(slabzone, slab, NULL, 0); 18248355f576SJeff Roberson } 18258355f576SJeff Roberson 18268355f576SJeff Roberson 18278355f576SJeff Roberson return (mem); 18288355f576SJeff Roberson } 18298355f576SJeff Roberson 18308355f576SJeff Roberson void 18318355f576SJeff Roberson uma_large_free(uma_slab_t slab) 18328355f576SJeff Roberson { 18338355f576SJeff Roberson UMA_HASH_REMOVE(mallochash, slab, slab->us_data); 18348355f576SJeff Roberson page_free(slab->us_data, slab->us_size, slab->us_flags); 18358355f576SJeff Roberson uma_zfree_internal(slabzone, slab, NULL, 0); 18368355f576SJeff Roberson } 18378355f576SJeff Roberson 18388355f576SJeff Roberson void 18398355f576SJeff Roberson uma_print_stats(void) 18408355f576SJeff Roberson { 18418355f576SJeff Roberson zone_foreach(uma_print_zone); 18428355f576SJeff Roberson } 18438355f576SJeff Roberson 18448355f576SJeff Roberson void 18458355f576SJeff Roberson uma_print_zone(uma_zone_t zone) 18468355f576SJeff Roberson { 18478355f576SJeff Roberson printf("%s(%p) size %d(%d) flags %d ipers %d ppera %d out %d free %d\n", 18488355f576SJeff Roberson zone->uz_name, zone, zone->uz_size, zone->uz_rsize, zone->uz_flags, 18498355f576SJeff Roberson zone->uz_ipers, zone->uz_ppera, 18508355f576SJeff Roberson (zone->uz_ipers * zone->uz_pages) - zone->uz_free, zone->uz_free); 18518355f576SJeff Roberson } 18528355f576SJeff Roberson 18538355f576SJeff Roberson /* 18548355f576SJeff Roberson * Sysctl handler for vm.zone 18558355f576SJeff Roberson * 18568355f576SJeff Roberson * stolen from vm_zone.c 18578355f576SJeff Roberson */ 18588355f576SJeff Roberson static int 18598355f576SJeff Roberson sysctl_vm_zone(SYSCTL_HANDLER_ARGS) 18608355f576SJeff Roberson { 18618355f576SJeff Roberson int error, len, cnt; 18628355f576SJeff Roberson const int linesize = 128; /* conservative */ 18638355f576SJeff Roberson int totalfree; 18648355f576SJeff Roberson char *tmpbuf, *offset; 18658355f576SJeff Roberson uma_zone_t z; 18668355f576SJeff Roberson char *p; 18678355f576SJeff Roberson 18688355f576SJeff Roberson cnt = 0; 18698355f576SJeff Roberson LIST_FOREACH(z, &uma_zones, uz_link) 18708355f576SJeff Roberson cnt++; 18718355f576SJeff Roberson MALLOC(tmpbuf, char *, (cnt == 0 ? 1 : cnt) * linesize, 18728355f576SJeff Roberson M_TEMP, M_WAITOK); 18738355f576SJeff Roberson len = snprintf(tmpbuf, linesize, 18748355f576SJeff Roberson "\nITEM SIZE LIMIT USED FREE REQUESTS\n\n"); 18758355f576SJeff Roberson if (cnt == 0) 18768355f576SJeff Roberson tmpbuf[len - 1] = '\0'; 18778355f576SJeff Roberson error = SYSCTL_OUT(req, tmpbuf, cnt == 0 ? len-1 : len); 18788355f576SJeff Roberson if (error || cnt == 0) 18798355f576SJeff Roberson goto out; 18808355f576SJeff Roberson offset = tmpbuf; 18818355f576SJeff Roberson LIST_FOREACH(z, &uma_zones, uz_link) { 18828355f576SJeff Roberson if (cnt == 0) /* list may have changed size */ 18838355f576SJeff Roberson break; 18848355f576SJeff Roberson ZONE_LOCK(z); 18858355f576SJeff Roberson totalfree = z->uz_free + z->uz_cachefree; 18868355f576SJeff Roberson len = snprintf(offset, linesize, 18878355f576SJeff Roberson "%-12.12s %6.6u, %8.8u, %6.6u, %6.6u, %8.8llu\n", 18888355f576SJeff Roberson z->uz_name, z->uz_size, 18898355f576SJeff Roberson z->uz_maxpages * z->uz_ipers, 18908355f576SJeff Roberson (z->uz_ipers * (z->uz_pages / z->uz_ppera)) - totalfree, 18918355f576SJeff Roberson totalfree, 18928355f576SJeff Roberson (unsigned long long)z->uz_allocs); 18938355f576SJeff Roberson ZONE_UNLOCK(z); 18948355f576SJeff Roberson for (p = offset + 12; p > offset && *p == ' '; --p) 18958355f576SJeff Roberson /* nothing */ ; 18968355f576SJeff Roberson p[1] = ':'; 18978355f576SJeff Roberson cnt--; 18988355f576SJeff Roberson offset += len; 18998355f576SJeff Roberson } 19008355f576SJeff Roberson *offset++ = '\0'; 19018355f576SJeff Roberson error = SYSCTL_OUT(req, tmpbuf, offset - tmpbuf); 19028355f576SJeff Roberson out: 19038355f576SJeff Roberson FREE(tmpbuf, M_TEMP); 19048355f576SJeff Roberson return (error); 19058355f576SJeff Roberson } 1906