160727d8bSWarner Losh /*- 2fe267a55SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3fe267a55SPedro F. Giffuni * 4584061b4SJeff Roberson * Copyright (c) 2002-2019 Jeffrey Roberson <jeff@FreeBSD.org> 508ecce74SRobert Watson * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 608ecce74SRobert Watson * All rights reserved. 78355f576SJeff Roberson * 88355f576SJeff Roberson * Redistribution and use in source and binary forms, with or without 98355f576SJeff Roberson * modification, are permitted provided that the following conditions 108355f576SJeff Roberson * are met: 118355f576SJeff Roberson * 1. Redistributions of source code must retain the above copyright 128355f576SJeff Roberson * notice unmodified, this list of conditions, and the following 138355f576SJeff Roberson * disclaimer. 148355f576SJeff Roberson * 2. Redistributions in binary form must reproduce the above copyright 158355f576SJeff Roberson * notice, this list of conditions and the following disclaimer in the 168355f576SJeff Roberson * documentation and/or other materials provided with the distribution. 178355f576SJeff Roberson * 188355f576SJeff Roberson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 198355f576SJeff Roberson * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 208355f576SJeff Roberson * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 218355f576SJeff Roberson * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 228355f576SJeff Roberson * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 238355f576SJeff Roberson * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 248355f576SJeff Roberson * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 258355f576SJeff Roberson * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 268355f576SJeff Roberson * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 278355f576SJeff Roberson * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 288355f576SJeff Roberson * 298355f576SJeff Roberson * $FreeBSD$ 308355f576SJeff Roberson * 318355f576SJeff Roberson */ 328355f576SJeff Roberson 3339669415SGleb Smirnoff #include <sys/counter.h> 34e04223bfSMark Johnston #include <sys/_bitset.h> 357571e249SMark Johnston #include <sys/_domainset.h> 36b28cc462SGleb Smirnoff #include <sys/_task.h> 37b28cc462SGleb Smirnoff 388355f576SJeff Roberson /* 398355f576SJeff Roberson * This file includes definitions, structures, prototypes, and inlines that 408355f576SJeff Roberson * should not be used outside of the actual implementation of UMA. 418355f576SJeff Roberson */ 428355f576SJeff Roberson 438355f576SJeff Roberson /* 44ab3185d1SJeff Roberson * The brief summary; Zones describe unique allocation types. Zones are 45ab3185d1SJeff Roberson * organized into per-CPU caches which are filled by buckets. Buckets are 46ab3185d1SJeff Roberson * organized according to memory domains. Buckets are filled from kegs which 47ab3185d1SJeff Roberson * are also organized according to memory domains. Kegs describe a unique 48ab3185d1SJeff Roberson * allocation type, backend memory provider, and layout. Kegs are associated 49ab3185d1SJeff Roberson * with one or more zones and zones reference one or more kegs. Kegs provide 50ab3185d1SJeff Roberson * slabs which are virtually contiguous collections of pages. Each slab is 51ab3185d1SJeff Roberson * broken down int one or more items that will satisfy an individual allocation. 52ab3185d1SJeff Roberson * 53ab3185d1SJeff Roberson * Allocation is satisfied in the following order: 54ab3185d1SJeff Roberson * 1) Per-CPU cache 55ab3185d1SJeff Roberson * 2) Per-domain cache of buckets 56ab3185d1SJeff Roberson * 3) Slab from any of N kegs 57ab3185d1SJeff Roberson * 4) Backend page provider 58ab3185d1SJeff Roberson * 59ab3185d1SJeff Roberson * More detail on individual objects is contained below: 608355f576SJeff Roberson * 61099a0e58SBosko Milekic * Kegs contain lists of slabs which are stored in either the full bin, empty 628355f576SJeff Roberson * bin, or partially allocated bin, to reduce fragmentation. They also contain 638355f576SJeff Roberson * the user supplied value for size, which is adjusted for alignment purposes 64099a0e58SBosko Milekic * and rsize is the result of that. The Keg also stores information for 658355f576SJeff Roberson * managing a hash of page addresses that maps pages to uma_slab_t structures 668355f576SJeff Roberson * for pages that don't have embedded uma_slab_t's. 678355f576SJeff Roberson * 68ab3185d1SJeff Roberson * Keg slab lists are organized by memory domain to support NUMA allocation 69ab3185d1SJeff Roberson * policies. By default allocations are spread across domains to reduce the 70ab3185d1SJeff Roberson * potential for hotspots. Special keg creation flags may be specified to 71ab3185d1SJeff Roberson * prefer location allocation. However there is no strict enforcement as frees 72ab3185d1SJeff Roberson * may happen on any CPU and these are returned to the CPU-local cache 73ab3185d1SJeff Roberson * regardless of the originating domain. 74ab3185d1SJeff Roberson * 758355f576SJeff Roberson * The uma_slab_t may be embedded in a UMA_SLAB_SIZE chunk of memory or it may 768355f576SJeff Roberson * be allocated off the page from a special slab zone. The free list within a 77ef72505eSJeff Roberson * slab is managed with a bitmask. For item sizes that would yield more than 78ef72505eSJeff Roberson * 10% memory waste we potentially allocate a separate uma_slab_t if this will 79ef72505eSJeff Roberson * improve the number of items per slab that will fit. 808355f576SJeff Roberson * 818355f576SJeff Roberson * The only really gross cases, with regards to memory waste, are for those 828355f576SJeff Roberson * items that are just over half the page size. You can get nearly 50% waste, 838355f576SJeff Roberson * so you fall back to the memory footprint of the power of two allocator. I 848355f576SJeff Roberson * have looked at memory allocation sizes on many of the machines available to 858355f576SJeff Roberson * me, and there does not seem to be an abundance of allocations at this range 868355f576SJeff Roberson * so at this time it may not make sense to optimize for it. This can, of 878355f576SJeff Roberson * course, be solved with dynamic slab sizes. 888355f576SJeff Roberson * 89099a0e58SBosko Milekic * Kegs may serve multiple Zones but by far most of the time they only serve 90099a0e58SBosko Milekic * one. When a Zone is created, a Keg is allocated and setup for it. While 91099a0e58SBosko Milekic * the backing Keg stores slabs, the Zone caches Buckets of items allocated 92099a0e58SBosko Milekic * from the slabs. Each Zone is equipped with an init/fini and ctor/dtor 93099a0e58SBosko Milekic * pair, as well as with its own set of small per-CPU caches, layered above 94099a0e58SBosko Milekic * the Zone's general Bucket cache. 95099a0e58SBosko Milekic * 966ab3b958SRobert Watson * The PCPU caches are protected by critical sections, and may be accessed 976ab3b958SRobert Watson * safely only from their associated CPU, while the Zones backed by the same 986ab3b958SRobert Watson * Keg all share a common Keg lock (to coalesce contention on the backing 996ab3b958SRobert Watson * slabs). The backing Keg typically only serves one Zone but in the case of 1006ab3b958SRobert Watson * multiple Zones, one of the Zones is considered the Master Zone and all 1016ab3b958SRobert Watson * Zone-related stats from the Keg are done in the Master Zone. For an 1026ab3b958SRobert Watson * example of a Multi-Zone setup, refer to the Mbuf allocation code. 1038355f576SJeff Roberson */ 1048355f576SJeff Roberson 1058355f576SJeff Roberson /* 1068355f576SJeff Roberson * This is the representation for normal (Non OFFPAGE slab) 1078355f576SJeff Roberson * 1088355f576SJeff Roberson * i == item 1098355f576SJeff Roberson * s == slab pointer 1108355f576SJeff Roberson * 1118355f576SJeff Roberson * <---------------- Page (UMA_SLAB_SIZE) ------------------> 1128355f576SJeff Roberson * ___________________________________________________________ 1138355f576SJeff Roberson * | _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ ___________ | 1148355f576SJeff Roberson * ||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i| |slab header|| 1158355f576SJeff Roberson * ||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_| |___________|| 1168355f576SJeff Roberson * |___________________________________________________________| 1178355f576SJeff Roberson * 1188355f576SJeff Roberson * 1198355f576SJeff Roberson * This is an OFFPAGE slab. These can be larger than UMA_SLAB_SIZE. 1208355f576SJeff Roberson * 1218355f576SJeff Roberson * ___________________________________________________________ 1228355f576SJeff Roberson * | _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | 1238355f576SJeff Roberson * ||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i| | 1248355f576SJeff Roberson * ||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_| | 1258355f576SJeff Roberson * |___________________________________________________________| 1268355f576SJeff Roberson * ___________ ^ 1278355f576SJeff Roberson * |slab header| | 1288355f576SJeff Roberson * |___________|---* 1298355f576SJeff Roberson * 1308355f576SJeff Roberson */ 1318355f576SJeff Roberson 1328355f576SJeff Roberson #ifndef VM_UMA_INT_H 1338355f576SJeff Roberson #define VM_UMA_INT_H 1348355f576SJeff Roberson 1358355f576SJeff Roberson #define UMA_SLAB_SIZE PAGE_SIZE /* How big are our slabs? */ 1368355f576SJeff Roberson #define UMA_SLAB_MASK (PAGE_SIZE - 1) /* Mask to get back to the page */ 1378355f576SJeff Roberson #define UMA_SLAB_SHIFT PAGE_SHIFT /* Number of bits PAGE_MASK */ 1388355f576SJeff Roberson 139ad97af7eSGleb Smirnoff /* Max waste percentage before going to off page slab management */ 140ad97af7eSGleb Smirnoff #define UMA_MAX_WASTE 10 1418355f576SJeff Roberson 1428355f576SJeff Roberson /* 1438355f576SJeff Roberson * I doubt there will be many cases where this is exceeded. This is the initial 1448355f576SJeff Roberson * size of the hash table for uma_slabs that are managed off page. This hash 1458355f576SJeff Roberson * does expand by powers of two. Currently it doesn't get smaller. 1468355f576SJeff Roberson */ 1478355f576SJeff Roberson #define UMA_HASH_SIZE_INIT 32 1488355f576SJeff Roberson 1498355f576SJeff Roberson /* 1508355f576SJeff Roberson * I should investigate other hashing algorithms. This should yield a low 1518355f576SJeff Roberson * number of collisions if the pages are relatively contiguous. 1528355f576SJeff Roberson */ 1538355f576SJeff Roberson 154ef72505eSJeff Roberson #define UMA_HASH(h, s) ((((uintptr_t)s) >> UMA_SLAB_SHIFT) & (h)->uh_hashmask) 1558355f576SJeff Roberson 1568355f576SJeff Roberson #define UMA_HASH_INSERT(h, s, mem) \ 1578355f576SJeff Roberson SLIST_INSERT_HEAD(&(h)->uh_slab_hash[UMA_HASH((h), \ 1584e2d83fcSAntoine Brodin (mem))], (s), us_hlink) 1598355f576SJeff Roberson #define UMA_HASH_REMOVE(h, s, mem) \ 1608355f576SJeff Roberson SLIST_REMOVE(&(h)->uh_slab_hash[UMA_HASH((h), \ 1614e2d83fcSAntoine Brodin (mem))], (s), uma_slab, us_hlink) 1628355f576SJeff Roberson 1638355f576SJeff Roberson /* Hash table for freed address -> slab translation */ 1648355f576SJeff Roberson 1658355f576SJeff Roberson SLIST_HEAD(slabhead, uma_slab); 1668355f576SJeff Roberson 1678355f576SJeff Roberson struct uma_hash { 1688355f576SJeff Roberson struct slabhead *uh_slab_hash; /* Hash table for slabs */ 1696929b7d1SPedro F. Giffuni u_int uh_hashsize; /* Current size of the hash table */ 1706929b7d1SPedro F. Giffuni u_int uh_hashmask; /* Mask used during hashing */ 1718355f576SJeff Roberson }; 1728355f576SJeff Roberson 1738355f576SJeff Roberson /* 1745e4bb93cSKip Macy * align field or structure to cache line 1755e4bb93cSKip Macy */ 17612f69195SJustin Hibbits #if defined(__amd64__) || defined(__powerpc64__) 177782e38aaSMateusz Guzik #define UMA_ALIGN __aligned(128) 1781a23373cSKip Macy #else 1796b4391d7SKip Macy #define UMA_ALIGN 1801a23373cSKip Macy #endif 1815e4bb93cSKip Macy 1825e4bb93cSKip Macy /* 1838355f576SJeff Roberson * Structures for per cpu queues. 1848355f576SJeff Roberson */ 1858355f576SJeff Roberson 1868355f576SJeff Roberson struct uma_bucket { 18708cfa56eSMark Johnston TAILQ_ENTRY(uma_bucket) ub_link; /* Link into the zone */ 188306abf0fSGleb Smirnoff int16_t ub_cnt; /* Count of items in bucket. */ 189cae33c14SJeff Roberson int16_t ub_entries; /* Max items. */ 190cae33c14SJeff Roberson void *ub_bucket[]; /* actual allocation storage */ 1911a23373cSKip Macy }; 1928355f576SJeff Roberson 1938355f576SJeff Roberson typedef struct uma_bucket * uma_bucket_t; 1948355f576SJeff Roberson 1958355f576SJeff Roberson struct uma_cache { 1968355f576SJeff Roberson uma_bucket_t uc_freebucket; /* Bucket we're freeing to */ 1978355f576SJeff Roberson uma_bucket_t uc_allocbucket; /* Bucket to allocate from */ 198c1685086SJeff Roberson uma_bucket_t uc_crossbucket; /* cross domain bucket */ 19985dcf349SGleb Smirnoff uint64_t uc_allocs; /* Count of allocations */ 20085dcf349SGleb Smirnoff uint64_t uc_frees; /* Count of frees */ 2015e4bb93cSKip Macy } UMA_ALIGN; 2028355f576SJeff Roberson 2038355f576SJeff Roberson typedef struct uma_cache * uma_cache_t; 2048355f576SJeff Roberson 2058355f576SJeff Roberson /* 206ab3185d1SJeff Roberson * Per-domain memory list. Embedded in the kegs. 207ab3185d1SJeff Roberson */ 208ab3185d1SJeff Roberson struct uma_domain { 209ab3185d1SJeff Roberson LIST_HEAD(,uma_slab) ud_part_slab; /* partially allocated slabs */ 210ab3185d1SJeff Roberson LIST_HEAD(,uma_slab) ud_free_slab; /* empty slab list */ 211ab3185d1SJeff Roberson LIST_HEAD(,uma_slab) ud_full_slab; /* full slabs */ 212ab3185d1SJeff Roberson }; 213ab3185d1SJeff Roberson 214ab3185d1SJeff Roberson typedef struct uma_domain * uma_domain_t; 215ab3185d1SJeff Roberson 216ab3185d1SJeff Roberson /* 217099a0e58SBosko Milekic * Keg management structure 218099a0e58SBosko Milekic * 219099a0e58SBosko Milekic * TODO: Optimize for cache line size 220099a0e58SBosko Milekic * 221099a0e58SBosko Milekic */ 222099a0e58SBosko Milekic struct uma_keg { 223bb15d1c7SGleb Smirnoff struct mtx uk_lock; /* Lock for the keg must be first. 224bb15d1c7SGleb Smirnoff * See shared uz_keg/uz_lockptr 225bb15d1c7SGleb Smirnoff * member of struct uma_zone. */ 226099a0e58SBosko Milekic struct uma_hash uk_hash; 227099a0e58SBosko Milekic LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */ 228099a0e58SBosko Milekic 229194a979eSMark Johnston struct domainset_ref uk_dr; /* Domain selection policy. */ 23085dcf349SGleb Smirnoff uint32_t uk_align; /* Alignment mask */ 23185dcf349SGleb Smirnoff uint32_t uk_pages; /* Total page count */ 23285dcf349SGleb Smirnoff uint32_t uk_free; /* Count of items free in slabs */ 2336fd34d6fSJeff Roberson uint32_t uk_reserve; /* Number of reserved items. */ 23485dcf349SGleb Smirnoff uint32_t uk_size; /* Requested size of each item */ 23585dcf349SGleb Smirnoff uint32_t uk_rsize; /* Real size of each item */ 236099a0e58SBosko Milekic 237099a0e58SBosko Milekic uma_init uk_init; /* Keg's init routine */ 238099a0e58SBosko Milekic uma_fini uk_fini; /* Keg's fini routine */ 239099a0e58SBosko Milekic uma_alloc uk_allocf; /* Allocation function */ 240099a0e58SBosko Milekic uma_free uk_freef; /* Free routine */ 241099a0e58SBosko Milekic 242a4915c21SAttilio Rao u_long uk_offset; /* Next free offset from base KVA */ 243a4915c21SAttilio Rao vm_offset_t uk_kva; /* Zone base KVA */ 244099a0e58SBosko Milekic uma_zone_t uk_slabzone; /* Slab zone backing us, if OFFPAGE */ 245099a0e58SBosko Milekic 2462d54d4bbSMark Johnston uint32_t uk_pgoff; /* Offset to uma_slab struct */ 24785dcf349SGleb Smirnoff uint16_t uk_ppera; /* pages per allocation from backend */ 24885dcf349SGleb Smirnoff uint16_t uk_ipers; /* Items per slab */ 24985dcf349SGleb Smirnoff uint32_t uk_flags; /* Internal flags */ 250ad97af7eSGleb Smirnoff 251ad97af7eSGleb Smirnoff /* Least used fields go to the last cache line. */ 252ad97af7eSGleb Smirnoff const char *uk_name; /* Name of creating zone. */ 253ad97af7eSGleb Smirnoff LIST_ENTRY(uma_keg) uk_link; /* List of all kegs */ 254ab3185d1SJeff Roberson 255ab3185d1SJeff Roberson /* Must be last, variable sized. */ 256ab3185d1SJeff Roberson struct uma_domain uk_domain[]; /* Keg's slab lists. */ 257099a0e58SBosko Milekic }; 258099a0e58SBosko Milekic typedef struct uma_keg * uma_keg_t; 259099a0e58SBosko Milekic 260ef72505eSJeff Roberson /* 261ef72505eSJeff Roberson * Free bits per-slab. 262ef72505eSJeff Roberson */ 263*9b78b1f4SJeff Roberson #define SLAB_MAX_SETSIZE (PAGE_SIZE / UMA_SMALLEST_UNIT) 264*9b78b1f4SJeff Roberson #define SLAB_MIN_SETSIZE _BITSET_BITS 265*9b78b1f4SJeff Roberson BITSET_DEFINE(slabbits, SLAB_MAX_SETSIZE); 266*9b78b1f4SJeff Roberson BITSET_DEFINE(noslabbits, 0); 267099a0e58SBosko Milekic 268ef72505eSJeff Roberson /* 269ef72505eSJeff Roberson * The slab structure manages a single contiguous allocation from backing 270ef72505eSJeff Roberson * store and subdivides it into individually allocatable items. 271ef72505eSJeff Roberson */ 272ef72505eSJeff Roberson struct uma_slab { 2736d6a03d7SJeff Roberson LIST_ENTRY(uma_slab) us_link; /* slabs in zone */ 274099a0e58SBosko Milekic SLIST_ENTRY(uma_slab) us_hlink; /* Link for hash table */ 27585dcf349SGleb Smirnoff uint8_t *us_data; /* First item */ 27685dcf349SGleb Smirnoff uint16_t us_freecount; /* How many are free? */ 27785dcf349SGleb Smirnoff uint8_t us_flags; /* Page flags see uma.h */ 278ab3185d1SJeff Roberson uint8_t us_domain; /* Backing NUMA domain. */ 279*9b78b1f4SJeff Roberson #ifdef INVARIANTS 280*9b78b1f4SJeff Roberson struct slabbits us_debugfree; /* Debug bitmask. */ 281*9b78b1f4SJeff Roberson #endif 282*9b78b1f4SJeff Roberson struct noslabbits us_free; /* Free bitmask. */ 283099a0e58SBosko Milekic }; 284099a0e58SBosko Milekic 285ab3185d1SJeff Roberson #if MAXMEMDOM >= 255 286ab3185d1SJeff Roberson #error "Slab domain type insufficient" 287ab3185d1SJeff Roberson #endif 288ab3185d1SJeff Roberson 289099a0e58SBosko Milekic typedef struct uma_slab * uma_slab_t; 290e20a199fSJeff Roberson 291*9b78b1f4SJeff Roberson /* These three functions are for embedded (!OFFPAGE) use only. */ 292*9b78b1f4SJeff Roberson size_t slab_sizeof(int nitems); 293*9b78b1f4SJeff Roberson size_t slab_space(int nitems); 294*9b78b1f4SJeff Roberson int slab_ipers(size_t size, int align); 295*9b78b1f4SJeff Roberson 29608cfa56eSMark Johnston TAILQ_HEAD(uma_bucketlist, uma_bucket); 29708cfa56eSMark Johnston 298ab3185d1SJeff Roberson struct uma_zone_domain { 29908cfa56eSMark Johnston struct uma_bucketlist uzd_buckets; /* full buckets */ 3000f9b7bf3SMark Johnston long uzd_nitems; /* total item count */ 3010f9b7bf3SMark Johnston long uzd_imax; /* maximum item count this period */ 3020f9b7bf3SMark Johnston long uzd_imin; /* minimum item count this period */ 3030f9b7bf3SMark Johnston long uzd_wss; /* working set size estimate */ 304ab3185d1SJeff Roberson }; 305ab3185d1SJeff Roberson 306ab3185d1SJeff Roberson typedef struct uma_zone_domain * uma_zone_domain_t; 307ab3185d1SJeff Roberson 308244f4554SBosko Milekic /* 3098355f576SJeff Roberson * Zone management structure 3108355f576SJeff Roberson * 3118355f576SJeff Roberson * TODO: Optimize for cache line size 3128355f576SJeff Roberson * 3138355f576SJeff Roberson */ 3148355f576SJeff Roberson struct uma_zone { 31563b5557bSJeff Roberson /* Offset 0, used in alloc/free fast/medium fast path and const. */ 316bb15d1c7SGleb Smirnoff union { 317bb15d1c7SGleb Smirnoff uma_keg_t uz_keg; /* This zone's keg */ 318bb15d1c7SGleb Smirnoff struct mtx *uz_lockptr; /* To keg or to self */ 319bb15d1c7SGleb Smirnoff }; 320ab3185d1SJeff Roberson struct uma_zone_domain *uz_domain; /* per-domain buckets */ 32163b5557bSJeff Roberson uint32_t uz_flags; /* Flags inherited from kegs */ 32263b5557bSJeff Roberson uint32_t uz_size; /* Size inherited from kegs */ 3238355f576SJeff Roberson uma_ctor uz_ctor; /* Constructor for each allocation */ 3248355f576SJeff Roberson uma_dtor uz_dtor; /* Destructor */ 325bb15d1c7SGleb Smirnoff uint64_t uz_items; /* Total items count */ 326bb15d1c7SGleb Smirnoff uint64_t uz_max_items; /* Maximum number of items to alloc */ 327bb15d1c7SGleb Smirnoff uint32_t uz_sleepers; /* Number of sleepers on memory */ 32820a4e154SJeff Roberson uint16_t uz_bucket_size; /* Number of items in full bucket */ 32920a4e154SJeff Roberson uint16_t uz_bucket_size_max; /* Maximum number of bucket items */ 33063b5557bSJeff Roberson 33163b5557bSJeff Roberson /* Offset 64, used in bucket replenish. */ 3320095a784SJeff Roberson uma_import uz_import; /* Import new memory to cache. */ 3330095a784SJeff Roberson uma_release uz_release; /* Release memory from cache. */ 3340095a784SJeff Roberson void *uz_arg; /* Import/release argument. */ 335bb15d1c7SGleb Smirnoff uma_init uz_init; /* Initializer for each item */ 336bb15d1c7SGleb Smirnoff uma_fini uz_fini; /* Finalizer for each item. */ 337ad66f958SGleb Smirnoff void *uz_spare; 338bb15d1c7SGleb Smirnoff uint64_t uz_bkt_count; /* Items in bucket cache */ 339bb15d1c7SGleb Smirnoff uint64_t uz_bkt_max; /* Maximum bucket cache size */ 340099a0e58SBosko Milekic 34163b5557bSJeff Roberson /* Offset 128 Rare. */ 34263b5557bSJeff Roberson /* 34363b5557bSJeff Roberson * The lock is placed here to avoid adjacent line prefetcher 34463b5557bSJeff Roberson * in fast paths and to take up space near infrequently accessed 34563b5557bSJeff Roberson * members to reduce alignment overhead. 34663b5557bSJeff Roberson */ 34763b5557bSJeff Roberson struct mtx uz_lock; /* Lock for the zone */ 348bb15d1c7SGleb Smirnoff LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ 349bb15d1c7SGleb Smirnoff const char *uz_name; /* Text name of the zone */ 35063b5557bSJeff Roberson /* The next two fields are used to print a rate-limited warnings. */ 35163b5557bSJeff Roberson const char *uz_warning; /* Warning to print on failure */ 35263b5557bSJeff Roberson struct timeval uz_ratecheck; /* Warnings rate-limiting */ 35363b5557bSJeff Roberson struct task uz_maxaction; /* Task to run when at limit */ 35420a4e154SJeff Roberson uint16_t uz_bucket_size_min; /* Min number of items in bucket */ 3555e4bb93cSKip Macy 35620a4e154SJeff Roberson /* Offset 256+, stats and misc. */ 3572efcc8cbSGleb Smirnoff counter_u64_t uz_allocs; /* Total number of allocations */ 3582efcc8cbSGleb Smirnoff counter_u64_t uz_frees; /* Total number of frees */ 3592efcc8cbSGleb Smirnoff counter_u64_t uz_fails; /* Total number of alloc failures */ 36085dcf349SGleb Smirnoff uint64_t uz_sleeps; /* Total number of alloc sleeps */ 361c1685086SJeff Roberson uint64_t uz_xdomain; /* Total number of cross-domain frees */ 36220a4e154SJeff Roberson char *uz_ctlname; /* sysctl safe name string. */ 36320a4e154SJeff Roberson struct sysctl_oid *uz_oid; /* sysctl oid pointer. */ 36420a4e154SJeff Roberson int uz_namecnt; /* duplicate name count. */ 36554503a13SJonathan T. Looney 3668355f576SJeff Roberson /* 3678355f576SJeff Roberson * This HAS to be the last item because we adjust the zone size 3688355f576SJeff Roberson * based on NCPU and then allocate the space for the zones. 3698355f576SJeff Roberson */ 370ab3185d1SJeff Roberson struct uma_cache uz_cpu[]; /* Per cpu caches */ 371ab3185d1SJeff Roberson 372ab3185d1SJeff Roberson /* uz_domain follows here. */ 3738355f576SJeff Roberson }; 3748355f576SJeff Roberson 375b60f5b79SJeff Roberson /* 376b60f5b79SJeff Roberson * These flags must not overlap with the UMA_ZONE flags specified in uma.h. 377b60f5b79SJeff Roberson */ 378bb15d1c7SGleb Smirnoff #define UMA_ZFLAG_CACHE 0x04000000 /* uma_zcache_create()d it */ 37908cfa56eSMark Johnston #define UMA_ZFLAG_RECLAIMING 0x08000000 /* Running zone_reclaim(). */ 3806fd34d6fSJeff Roberson #define UMA_ZFLAG_BUCKET 0x10000000 /* Bucket zone. */ 3812018f30cSMike Silbersack #define UMA_ZFLAG_INTERNAL 0x20000000 /* No offpage no PCPU. */ 382ca293436SRyan Libby #define UMA_ZFLAG_TRASH 0x40000000 /* Add trash ctor/dtor. */ 3832018f30cSMike Silbersack #define UMA_ZFLAG_CACHEONLY 0x80000000 /* Don't ask VM for buckets. */ 3848355f576SJeff Roberson 3856fd34d6fSJeff Roberson #define UMA_ZFLAG_INHERIT \ 3866fd34d6fSJeff Roberson (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY | UMA_ZFLAG_BUCKET) 387e20a199fSJeff Roberson 3885e4bb93cSKip Macy #undef UMA_ALIGN 3895e4bb93cSKip Macy 390af17e9a9SRobert Watson #ifdef _KERNEL 3918355f576SJeff Roberson /* Internal prototypes */ 39285dcf349SGleb Smirnoff static __inline uma_slab_t hash_sfind(struct uma_hash *hash, uint8_t *data); 3938355f576SJeff Roberson 3948355f576SJeff Roberson /* Lock Macros */ 3958355f576SJeff Roberson 396e20a199fSJeff Roberson #define KEG_LOCK_INIT(k, lc) \ 39728bc4419SJeff Roberson do { \ 39828bc4419SJeff Roberson if ((lc)) \ 399e20a199fSJeff Roberson mtx_init(&(k)->uk_lock, (k)->uk_name, \ 400e20a199fSJeff Roberson (k)->uk_name, MTX_DEF | MTX_DUPOK); \ 40128bc4419SJeff Roberson else \ 402e20a199fSJeff Roberson mtx_init(&(k)->uk_lock, (k)->uk_name, \ 40328bc4419SJeff Roberson "UMA zone", MTX_DEF | MTX_DUPOK); \ 40428bc4419SJeff Roberson } while (0) 40528bc4419SJeff Roberson 406e20a199fSJeff Roberson #define KEG_LOCK_FINI(k) mtx_destroy(&(k)->uk_lock) 407e20a199fSJeff Roberson #define KEG_LOCK(k) mtx_lock(&(k)->uk_lock) 408e20a199fSJeff Roberson #define KEG_UNLOCK(k) mtx_unlock(&(k)->uk_lock) 409bb15d1c7SGleb Smirnoff #define KEG_LOCK_ASSERT(k) mtx_assert(&(k)->uk_lock, MA_OWNED) 410bb15d1c7SGleb Smirnoff 411bb15d1c7SGleb Smirnoff #define KEG_GET(zone, keg) do { \ 412bb15d1c7SGleb Smirnoff (keg) = (zone)->uz_keg; \ 413bb15d1c7SGleb Smirnoff KASSERT((void *)(keg) != (void *)&(zone)->uz_lock, \ 414bb15d1c7SGleb Smirnoff ("%s: Invalid zone %p type", __func__, (zone))); \ 415bb15d1c7SGleb Smirnoff } while (0) 416af526374SJeff Roberson 417af526374SJeff Roberson #define ZONE_LOCK_INIT(z, lc) \ 418af526374SJeff Roberson do { \ 419af526374SJeff Roberson if ((lc)) \ 420af526374SJeff Roberson mtx_init(&(z)->uz_lock, (z)->uz_name, \ 421af526374SJeff Roberson (z)->uz_name, MTX_DEF | MTX_DUPOK); \ 422af526374SJeff Roberson else \ 423af526374SJeff Roberson mtx_init(&(z)->uz_lock, (z)->uz_name, \ 424af526374SJeff Roberson "UMA zone", MTX_DEF | MTX_DUPOK); \ 425af526374SJeff Roberson } while (0) 426af526374SJeff Roberson 427af526374SJeff Roberson #define ZONE_LOCK(z) mtx_lock((z)->uz_lockptr) 428af526374SJeff Roberson #define ZONE_TRYLOCK(z) mtx_trylock((z)->uz_lockptr) 429af526374SJeff Roberson #define ZONE_UNLOCK(z) mtx_unlock((z)->uz_lockptr) 430af526374SJeff Roberson #define ZONE_LOCK_FINI(z) mtx_destroy(&(z)->uz_lock) 4310f9b7bf3SMark Johnston #define ZONE_LOCK_ASSERT(z) mtx_assert((z)->uz_lockptr, MA_OWNED) 4328355f576SJeff Roberson 4338355f576SJeff Roberson /* 4348355f576SJeff Roberson * Find a slab within a hash table. This is used for OFFPAGE zones to lookup 4358355f576SJeff Roberson * the slab structure. 4368355f576SJeff Roberson * 4378355f576SJeff Roberson * Arguments: 4388355f576SJeff Roberson * hash The hash table to search. 4398355f576SJeff Roberson * data The base page of the item. 4408355f576SJeff Roberson * 4418355f576SJeff Roberson * Returns: 4428355f576SJeff Roberson * A pointer to a slab if successful, else NULL. 4438355f576SJeff Roberson */ 4448355f576SJeff Roberson static __inline uma_slab_t 44585dcf349SGleb Smirnoff hash_sfind(struct uma_hash *hash, uint8_t *data) 4468355f576SJeff Roberson { 4478355f576SJeff Roberson uma_slab_t slab; 4486929b7d1SPedro F. Giffuni u_int hval; 4498355f576SJeff Roberson 4508355f576SJeff Roberson hval = UMA_HASH(hash, data); 4518355f576SJeff Roberson 4528355f576SJeff Roberson SLIST_FOREACH(slab, &hash->uh_slab_hash[hval], us_hlink) { 45385dcf349SGleb Smirnoff if ((uint8_t *)slab->us_data == data) 4548355f576SJeff Roberson return (slab); 4558355f576SJeff Roberson } 4568355f576SJeff Roberson return (NULL); 4578355f576SJeff Roberson } 4588355f576SJeff Roberson 45999571dc3SJeff Roberson static __inline uma_slab_t 46099571dc3SJeff Roberson vtoslab(vm_offset_t va) 46199571dc3SJeff Roberson { 46299571dc3SJeff Roberson vm_page_t p; 46399571dc3SJeff Roberson 46499571dc3SJeff Roberson p = PHYS_TO_VM_PAGE(pmap_kextract(va)); 465584061b4SJeff Roberson return (p->plinks.uma.slab); 46699571dc3SJeff Roberson } 46799571dc3SJeff Roberson 46899571dc3SJeff Roberson static __inline void 469584061b4SJeff Roberson vtozoneslab(vm_offset_t va, uma_zone_t *zone, uma_slab_t *slab) 47099571dc3SJeff Roberson { 47199571dc3SJeff Roberson vm_page_t p; 47299571dc3SJeff Roberson 4736fc96493SOlivier Houchard p = PHYS_TO_VM_PAGE(pmap_kextract(va)); 474584061b4SJeff Roberson *slab = p->plinks.uma.slab; 475584061b4SJeff Roberson *zone = p->plinks.uma.zone; 476584061b4SJeff Roberson } 477584061b4SJeff Roberson 478584061b4SJeff Roberson static __inline void 479584061b4SJeff Roberson vsetzoneslab(vm_offset_t va, uma_zone_t zone, uma_slab_t slab) 480584061b4SJeff Roberson { 481584061b4SJeff Roberson vm_page_t p; 482584061b4SJeff Roberson 483584061b4SJeff Roberson p = PHYS_TO_VM_PAGE(pmap_kextract(va)); 484584061b4SJeff Roberson p->plinks.uma.slab = slab; 485584061b4SJeff Roberson p->plinks.uma.zone = zone; 48699571dc3SJeff Roberson } 48799571dc3SJeff Roberson 4886d6a03d7SJeff Roberson extern unsigned long uma_kmem_limit; 4896d6a03d7SJeff Roberson extern unsigned long uma_kmem_total; 4906d6a03d7SJeff Roberson 4916d6a03d7SJeff Roberson /* Adjust bytes under management by UMA. */ 4926d6a03d7SJeff Roberson static inline void 4936d6a03d7SJeff Roberson uma_total_dec(unsigned long size) 4946d6a03d7SJeff Roberson { 4956d6a03d7SJeff Roberson 4966d6a03d7SJeff Roberson atomic_subtract_long(&uma_kmem_total, size); 4976d6a03d7SJeff Roberson } 4986d6a03d7SJeff Roberson 4996d6a03d7SJeff Roberson static inline void 5006d6a03d7SJeff Roberson uma_total_inc(unsigned long size) 5016d6a03d7SJeff Roberson { 5026d6a03d7SJeff Roberson 5036d6a03d7SJeff Roberson if (atomic_fetchadd_long(&uma_kmem_total, size) > uma_kmem_limit) 5046d6a03d7SJeff Roberson uma_reclaim_wakeup(); 5056d6a03d7SJeff Roberson } 5066d6a03d7SJeff Roberson 50748eea375SJeff Roberson /* 50848eea375SJeff Roberson * The following two functions may be defined by architecture specific code 509763df3ecSPedro F. Giffuni * if they can provide more efficient allocation functions. This is useful 51048eea375SJeff Roberson * for using direct mapped addresses. 51148eea375SJeff Roberson */ 512ab3185d1SJeff Roberson void *uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, 513ab3185d1SJeff Roberson uint8_t *pflag, int wait); 514f2c2231eSRyan Stone void uma_small_free(void *mem, vm_size_t size, uint8_t flags); 5152e47807cSJeff Roberson 5162e47807cSJeff Roberson /* Set a global soft limit on UMA managed memory. */ 5172e47807cSJeff Roberson void uma_set_limit(unsigned long limit); 518af17e9a9SRobert Watson #endif /* _KERNEL */ 51948eea375SJeff Roberson 5208355f576SJeff Roberson #endif /* VM_UMA_INT_H */ 521