xref: /freebsd/sys/vm/uma_int.h (revision bbd421cdf6d8c6102e6fd3979c5bec21ace3c2e3)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2002-2019 Jeffrey Roberson <jeff@FreeBSD.org>
5  * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice unmodified, this list of conditions, and the following
13  *    disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  *
31  */
32 
33 #include <sys/counter.h>
34 #include <sys/_bitset.h>
35 #include <sys/_domainset.h>
36 #include <sys/_task.h>
37 
38 /*
39  * This file includes definitions, structures, prototypes, and inlines that
40  * should not be used outside of the actual implementation of UMA.
41  */
42 
43 /*
44  * The brief summary;  Zones describe unique allocation types.  Zones are
45  * organized into per-CPU caches which are filled by buckets.  Buckets are
46  * organized according to memory domains.  Buckets are filled from kegs which
47  * are also organized according to memory domains.  Kegs describe a unique
48  * allocation type, backend memory provider, and layout.  Kegs are associated
49  * with one or more zones and zones reference one or more kegs.  Kegs provide
50  * slabs which are virtually contiguous collections of pages.  Each slab is
51  * broken down int one or more items that will satisfy an individual allocation.
52  *
53  * Allocation is satisfied in the following order:
54  * 1) Per-CPU cache
55  * 2) Per-domain cache of buckets
56  * 3) Slab from any of N kegs
57  * 4) Backend page provider
58  *
59  * More detail on individual objects is contained below:
60  *
61  * Kegs contain lists of slabs which are stored in either the full bin, empty
62  * bin, or partially allocated bin, to reduce fragmentation.  They also contain
63  * the user supplied value for size, which is adjusted for alignment purposes
64  * and rsize is the result of that.  The Keg also stores information for
65  * managing a hash of page addresses that maps pages to uma_slab_t structures
66  * for pages that don't have embedded uma_slab_t's.
67  *
68  * Keg slab lists are organized by memory domain to support NUMA allocation
69  * policies.  By default allocations are spread across domains to reduce the
70  * potential for hotspots.  Special keg creation flags may be specified to
71  * prefer location allocation.  However there is no strict enforcement as frees
72  * may happen on any CPU and these are returned to the CPU-local cache
73  * regardless of the originating domain.
74  *
75  * The uma_slab_t may be embedded in a UMA_SLAB_SIZE chunk of memory or it may
76  * be allocated off the page from a special slab zone.  The free list within a
77  * slab is managed with a bitmask.  For item sizes that would yield more than
78  * 10% memory waste we potentially allocate a separate uma_slab_t if this will
79  * improve the number of items per slab that will fit.
80  *
81  * The only really gross cases, with regards to memory waste, are for those
82  * items that are just over half the page size.   You can get nearly 50% waste,
83  * so you fall back to the memory footprint of the power of two allocator. I
84  * have looked at memory allocation sizes on many of the machines available to
85  * me, and there does not seem to be an abundance of allocations at this range
86  * so at this time it may not make sense to optimize for it.  This can, of
87  * course, be solved with dynamic slab sizes.
88  *
89  * Kegs may serve multiple Zones but by far most of the time they only serve
90  * one.  When a Zone is created, a Keg is allocated and setup for it.  While
91  * the backing Keg stores slabs, the Zone caches Buckets of items allocated
92  * from the slabs.  Each Zone is equipped with an init/fini and ctor/dtor
93  * pair, as well as with its own set of small per-CPU caches, layered above
94  * the Zone's general Bucket cache.
95  *
96  * The PCPU caches are protected by critical sections, and may be accessed
97  * safely only from their associated CPU, while the Zones backed by the same
98  * Keg all share a common Keg lock (to coalesce contention on the backing
99  * slabs).  The backing Keg typically only serves one Zone but in the case of
100  * multiple Zones, one of the Zones is considered the Primary Zone and all
101  * Zone-related stats from the Keg are done in the Primary Zone.  For an
102  * example of a Multi-Zone setup, refer to the Mbuf allocation code.
103  */
104 
105 /*
106  *	This is the representation for normal (Non OFFPAGE slab)
107  *
108  *	i == item
109  *	s == slab pointer
110  *
111  *	<----------------  Page (UMA_SLAB_SIZE) ------------------>
112  *	___________________________________________________________
113  *     | _  _  _  _  _  _  _  _  _  _  _  _  _  _  _   ___________ |
114  *     ||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i| |slab header||
115  *     ||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_| |___________||
116  *     |___________________________________________________________|
117  *
118  *
119  *	This is an OFFPAGE slab. These can be larger than UMA_SLAB_SIZE.
120  *
121  *	___________________________________________________________
122  *     | _  _  _  _  _  _  _  _  _  _  _  _  _  _  _  _  _  _  _   |
123  *     ||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i||i|  |
124  *     ||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_||_|  |
125  *     |___________________________________________________________|
126  *       ___________    ^
127  *	|slab header|   |
128  *	|___________|---*
129  *
130  */
131 
132 #ifndef VM_UMA_INT_H
133 #define VM_UMA_INT_H
134 
135 #define UMA_SLAB_SIZE	PAGE_SIZE	/* How big are our slabs? */
136 #define UMA_SLAB_MASK	(PAGE_SIZE - 1)	/* Mask to get back to the page */
137 #define UMA_SLAB_SHIFT	PAGE_SHIFT	/* Number of bits PAGE_MASK */
138 
139 /* Max waste percentage before going to off page slab management */
140 #define UMA_MAX_WASTE	10
141 
142 /* Max size of a CACHESPREAD slab. */
143 #define	UMA_CACHESPREAD_MAX_SIZE	(128 * 1024)
144 
145 /*
146  * These flags must not overlap with the UMA_ZONE flags specified in uma.h.
147  */
148 #define	UMA_ZFLAG_OFFPAGE	0x00200000	/*
149 						 * Force the slab structure
150 						 * allocation off of the real
151 						 * memory.
152 						 */
153 #define	UMA_ZFLAG_HASH		0x00400000	/*
154 						 * Use a hash table instead of
155 						 * caching information in the
156 						 * vm_page.
157 						 */
158 #define	UMA_ZFLAG_VTOSLAB	0x00800000	/*
159 						 * Zone uses vtoslab for
160 						 * lookup.
161 						 */
162 #define	UMA_ZFLAG_CTORDTOR	0x01000000	/* Zone has ctor/dtor set. */
163 #define	UMA_ZFLAG_LIMIT		0x02000000	/* Zone has limit set. */
164 #define	UMA_ZFLAG_CACHE		0x04000000	/* uma_zcache_create()d it */
165 #define	UMA_ZFLAG_BUCKET	0x10000000	/* Bucket zone. */
166 #define	UMA_ZFLAG_INTERNAL	0x20000000	/* No offpage no PCPU. */
167 #define	UMA_ZFLAG_TRASH		0x40000000	/* Add trash ctor/dtor. */
168 
169 #define	UMA_ZFLAG_INHERIT						\
170     (UMA_ZFLAG_OFFPAGE | UMA_ZFLAG_HASH | UMA_ZFLAG_VTOSLAB |		\
171      UMA_ZFLAG_BUCKET | UMA_ZFLAG_INTERNAL)
172 
173 #define	PRINT_UMA_ZFLAGS	"\20"	\
174     "\37TRASH"				\
175     "\36INTERNAL"			\
176     "\35BUCKET"				\
177     "\33CACHE"				\
178     "\32LIMIT"				\
179     "\31CTORDTOR"			\
180     "\30VTOSLAB"			\
181     "\27HASH"				\
182     "\26OFFPAGE"			\
183     "\23SMR"				\
184     "\22ROUNDROBIN"			\
185     "\21FIRSTTOUCH"			\
186     "\20PCPU"				\
187     "\17NODUMP"				\
188     "\16CACHESPREAD"			\
189     "\14MAXBUCKET"			\
190     "\13NOBUCKET"			\
191     "\12SECONDARY"			\
192     "\11NOTPAGE"			\
193     "\10VM"				\
194     "\7MTXCLASS"			\
195     "\6NOFREE"				\
196     "\5MALLOC"				\
197     "\4NOTOUCH"				\
198     "\3CONTIG"				\
199     "\2ZINIT"
200 
201 /*
202  * Hash table for freed address -> slab translation.
203  *
204  * Only zones with memory not touchable by the allocator use the
205  * hash table.  Otherwise slabs are found with vtoslab().
206  */
207 #define UMA_HASH_SIZE_INIT	32
208 
209 #define UMA_HASH(h, s) ((((uintptr_t)s) >> UMA_SLAB_SHIFT) & (h)->uh_hashmask)
210 
211 #define UMA_HASH_INSERT(h, s, mem)					\
212 	LIST_INSERT_HEAD(&(h)->uh_slab_hash[UMA_HASH((h),		\
213 	    (mem))], slab_tohashslab(s), uhs_hlink)
214 
215 #define UMA_HASH_REMOVE(h, s)						\
216 	LIST_REMOVE(slab_tohashslab(s), uhs_hlink)
217 
218 LIST_HEAD(slabhashhead, uma_hash_slab);
219 
220 struct uma_hash {
221 	struct slabhashhead	*uh_slab_hash;	/* Hash table for slabs */
222 	u_int		uh_hashsize;	/* Current size of the hash table */
223 	u_int		uh_hashmask;	/* Mask used during hashing */
224 };
225 
226 /*
227  * Align field or structure to cache 'sector' in intel terminology.  This
228  * is more efficient with adjacent line prefetch.
229  */
230 #if defined(__amd64__) || defined(__powerpc64__)
231 #define UMA_SUPER_ALIGN	(CACHE_LINE_SIZE * 2)
232 #else
233 #define UMA_SUPER_ALIGN	CACHE_LINE_SIZE
234 #endif
235 
236 #define	UMA_ALIGN	__aligned(UMA_SUPER_ALIGN)
237 
238 /*
239  * The uma_bucket structure is used to queue and manage buckets divorced
240  * from per-cpu caches.  They are loaded into uma_cache_bucket structures
241  * for use.
242  */
243 struct uma_bucket {
244 	STAILQ_ENTRY(uma_bucket)	ub_link; /* Link into the zone */
245 	int16_t		ub_cnt;			/* Count of items in bucket. */
246 	int16_t		ub_entries;		/* Max items. */
247 	smr_seq_t	ub_seq;			/* SMR sequence number. */
248 	void		*ub_bucket[];		/* actual allocation storage */
249 };
250 
251 typedef struct uma_bucket * uma_bucket_t;
252 
253 /*
254  * The uma_cache_bucket structure is statically allocated on each per-cpu
255  * cache.  Its use reduces branches and cache misses in the fast path.
256  */
257 struct uma_cache_bucket {
258 	uma_bucket_t	ucb_bucket;
259 	int16_t		ucb_cnt;
260 	int16_t		ucb_entries;
261 	uint32_t	ucb_spare;
262 };
263 
264 typedef struct uma_cache_bucket * uma_cache_bucket_t;
265 
266 /*
267  * The uma_cache structure is allocated for each cpu for every zone
268  * type.  This optimizes synchronization out of the allocator fast path.
269  */
270 struct uma_cache {
271 	struct uma_cache_bucket	uc_freebucket;	/* Bucket we're freeing to */
272 	struct uma_cache_bucket	uc_allocbucket;	/* Bucket to allocate from */
273 	struct uma_cache_bucket	uc_crossbucket;	/* cross domain bucket */
274 	uint64_t		uc_allocs;	/* Count of allocations */
275 	uint64_t		uc_frees;	/* Count of frees */
276 } UMA_ALIGN;
277 
278 typedef struct uma_cache * uma_cache_t;
279 
280 LIST_HEAD(slabhead, uma_slab);
281 
282 /*
283  * The cache structure pads perfectly into 64 bytes so we use spare
284  * bits from the embedded cache buckets to store information from the zone
285  * and keep all fast-path allocations accessing a single per-cpu line.
286  */
287 static inline void
288 cache_set_uz_flags(uma_cache_t cache, uint32_t flags)
289 {
290 
291 	cache->uc_freebucket.ucb_spare = flags;
292 }
293 
294 static inline void
295 cache_set_uz_size(uma_cache_t cache, uint32_t size)
296 {
297 
298 	cache->uc_allocbucket.ucb_spare = size;
299 }
300 
301 static inline uint32_t
302 cache_uz_flags(uma_cache_t cache)
303 {
304 
305 	return (cache->uc_freebucket.ucb_spare);
306 }
307 
308 static inline uint32_t
309 cache_uz_size(uma_cache_t cache)
310 {
311 
312 	return (cache->uc_allocbucket.ucb_spare);
313 }
314 
315 /*
316  * Per-domain slab lists.  Embedded in the kegs.
317  */
318 struct uma_domain {
319 	struct mtx_padalign ud_lock;	/* Lock for the domain lists. */
320 	struct slabhead	ud_part_slab;	/* partially allocated slabs */
321 	struct slabhead	ud_free_slab;	/* completely unallocated slabs */
322 	struct slabhead ud_full_slab;	/* fully allocated slabs */
323 	uint32_t	ud_pages;	/* Total page count */
324 	uint32_t	ud_free_items;	/* Count of items free in all slabs */
325 	uint32_t	ud_free_slabs;	/* Count of free slabs */
326 } __aligned(CACHE_LINE_SIZE);
327 
328 typedef struct uma_domain * uma_domain_t;
329 
330 /*
331  * Keg management structure
332  *
333  * TODO: Optimize for cache line size
334  *
335  */
336 struct uma_keg {
337 	struct uma_hash	uk_hash;
338 	LIST_HEAD(,uma_zone)	uk_zones;	/* Keg's zones */
339 
340 	struct domainset_ref uk_dr;	/* Domain selection policy. */
341 	uint32_t	uk_align;	/* Alignment mask */
342 	uint32_t	uk_reserve;	/* Number of reserved items. */
343 	uint32_t	uk_size;	/* Requested size of each item */
344 	uint32_t	uk_rsize;	/* Real size of each item */
345 
346 	uma_init	uk_init;	/* Keg's init routine */
347 	uma_fini	uk_fini;	/* Keg's fini routine */
348 	uma_alloc	uk_allocf;	/* Allocation function */
349 	uma_free	uk_freef;	/* Free routine */
350 
351 	u_long		uk_offset;	/* Next free offset from base KVA */
352 	vm_offset_t	uk_kva;		/* Zone base KVA */
353 
354 	uint32_t	uk_pgoff;	/* Offset to uma_slab struct */
355 	uint16_t	uk_ppera;	/* pages per allocation from backend */
356 	uint16_t	uk_ipers;	/* Items per slab */
357 	uint32_t	uk_flags;	/* Internal flags */
358 
359 	/* Least used fields go to the last cache line. */
360 	const char	*uk_name;		/* Name of creating zone. */
361 	LIST_ENTRY(uma_keg)	uk_link;	/* List of all kegs */
362 
363 	/* Must be last, variable sized. */
364 	struct uma_domain	uk_domain[];	/* Keg's slab lists. */
365 };
366 typedef struct uma_keg	* uma_keg_t;
367 
368 /*
369  * Free bits per-slab.
370  */
371 #define	SLAB_MAX_SETSIZE	(PAGE_SIZE / UMA_SMALLEST_UNIT)
372 #define	SLAB_MIN_SETSIZE	_BITSET_BITS
373 BITSET_DEFINE(noslabbits, 0);
374 
375 /*
376  * The slab structure manages a single contiguous allocation from backing
377  * store and subdivides it into individually allocatable items.
378  */
379 struct uma_slab {
380 	LIST_ENTRY(uma_slab)	us_link;	/* slabs in zone */
381 	uint16_t	us_freecount;		/* How many are free? */
382 	uint8_t		us_flags;		/* Page flags see uma.h */
383 	uint8_t		us_domain;		/* Backing NUMA domain. */
384 	struct noslabbits us_free;		/* Free bitmask, flexible. */
385 };
386 _Static_assert(sizeof(struct uma_slab) == __offsetof(struct uma_slab, us_free),
387     "us_free field must be last");
388 _Static_assert(MAXMEMDOM < 255,
389     "us_domain field is not wide enough");
390 
391 typedef struct uma_slab * uma_slab_t;
392 
393 /*
394  * Slab structure with a full sized bitset and hash link for both
395  * HASH and OFFPAGE zones.
396  */
397 struct uma_hash_slab {
398 	LIST_ENTRY(uma_hash_slab) uhs_hlink;	/* Link for hash table */
399 	uint8_t			*uhs_data;	/* First item */
400 	struct uma_slab		uhs_slab;	/* Must be last. */
401 };
402 
403 typedef struct uma_hash_slab * uma_hash_slab_t;
404 
405 static inline uma_hash_slab_t
406 slab_tohashslab(uma_slab_t slab)
407 {
408 
409 	return (__containerof(slab, struct uma_hash_slab, uhs_slab));
410 }
411 
412 static inline void *
413 slab_data(uma_slab_t slab, uma_keg_t keg)
414 {
415 
416 	if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) == 0)
417 		return ((void *)((uintptr_t)slab - keg->uk_pgoff));
418 	else
419 		return (slab_tohashslab(slab)->uhs_data);
420 }
421 
422 static inline void *
423 slab_item(uma_slab_t slab, uma_keg_t keg, int index)
424 {
425 	uintptr_t data;
426 
427 	data = (uintptr_t)slab_data(slab, keg);
428 	return ((void *)(data + keg->uk_rsize * index));
429 }
430 
431 static inline int
432 slab_item_index(uma_slab_t slab, uma_keg_t keg, void *item)
433 {
434 	uintptr_t data;
435 
436 	data = (uintptr_t)slab_data(slab, keg);
437 	return (((uintptr_t)item - data) / keg->uk_rsize);
438 }
439 
440 STAILQ_HEAD(uma_bucketlist, uma_bucket);
441 
442 struct uma_zone_domain {
443 	struct uma_bucketlist uzd_buckets; /* full buckets */
444 	uma_bucket_t	uzd_cross;	/* Fills from cross buckets. */
445 	long		uzd_nitems;	/* total item count */
446 	long		uzd_imax;	/* maximum item count this period */
447 	long		uzd_imin;	/* minimum item count this period */
448 	long		uzd_wss;	/* working set size estimate */
449 	smr_seq_t	uzd_seq;	/* Lowest queued seq. */
450 	struct mtx	uzd_lock;	/* Lock for the domain */
451 } __aligned(CACHE_LINE_SIZE);
452 
453 typedef struct uma_zone_domain * uma_zone_domain_t;
454 
455 /*
456  * Zone structure - per memory type.
457  */
458 struct uma_zone {
459 	/* Offset 0, used in alloc/free fast/medium fast path and const. */
460 	uint32_t	uz_flags;	/* Flags inherited from kegs */
461 	uint32_t	uz_size;	/* Size inherited from kegs */
462 	uma_ctor	uz_ctor;	/* Constructor for each allocation */
463 	uma_dtor	uz_dtor;	/* Destructor */
464 	smr_t		uz_smr;		/* Safe memory reclaim context. */
465 	uint64_t	uz_max_items;	/* Maximum number of items to alloc */
466 	uint64_t	uz_bucket_max;	/* Maximum bucket cache size */
467 	uint16_t	uz_bucket_size;	/* Number of items in full bucket */
468 	uint16_t	uz_bucket_size_max; /* Maximum number of bucket items */
469 	uint32_t	uz_sleepers;	/* Threads sleeping on limit */
470 	counter_u64_t	uz_xdomain;	/* Total number of cross-domain frees */
471 
472 	/* Offset 64, used in bucket replenish. */
473 	uma_keg_t	uz_keg;		/* This zone's keg if !CACHE */
474 	uma_import	uz_import;	/* Import new memory to cache. */
475 	uma_release	uz_release;	/* Release memory from cache. */
476 	void		*uz_arg;	/* Import/release argument. */
477 	uma_init	uz_init;	/* Initializer for each item */
478 	uma_fini	uz_fini;	/* Finalizer for each item. */
479 	volatile uint64_t uz_items;	/* Total items count & sleepers */
480 	uint64_t	uz_sleeps;	/* Total number of alloc sleeps */
481 
482 	/* Offset 128 Rare stats, misc read-only. */
483 	LIST_ENTRY(uma_zone) uz_link;	/* List of all zones in keg */
484 	counter_u64_t	uz_allocs;	/* Total number of allocations */
485 	counter_u64_t	uz_frees;	/* Total number of frees */
486 	counter_u64_t	uz_fails;	/* Total number of alloc failures */
487 	const char	*uz_name;	/* Text name of the zone */
488 	char		*uz_ctlname;	/* sysctl safe name string. */
489 	int		uz_namecnt;	/* duplicate name count. */
490 	uint16_t	uz_bucket_size_min; /* Min number of items in bucket */
491 	uint16_t	uz_reclaimers;	/* pending reclaim operations. */
492 
493 	/* Offset 192, rare read-only. */
494 	struct sysctl_oid *uz_oid;	/* sysctl oid pointer. */
495 	const char	*uz_warning;	/* Warning to print on failure */
496 	struct timeval	uz_ratecheck;	/* Warnings rate-limiting */
497 	struct task	uz_maxaction;	/* Task to run when at limit */
498 
499 	/* Offset 256. */
500 	struct mtx	uz_cross_lock;	/* Cross domain free lock */
501 
502 	/*
503 	 * This HAS to be the last item because we adjust the zone size
504 	 * based on NCPU and then allocate the space for the zones.
505 	 */
506 	struct uma_cache	uz_cpu[]; /* Per cpu caches */
507 
508 	/* domains follow here. */
509 };
510 
511 /*
512  * Macros for interpreting the uz_items field.  20 bits of sleeper count
513  * and 44 bit of item count.
514  */
515 #define	UZ_ITEMS_SLEEPER_SHIFT	44LL
516 #define	UZ_ITEMS_SLEEPERS_MAX	((1 << (64 - UZ_ITEMS_SLEEPER_SHIFT)) - 1)
517 #define	UZ_ITEMS_COUNT_MASK	((1LL << UZ_ITEMS_SLEEPER_SHIFT) - 1)
518 #define	UZ_ITEMS_COUNT(x)	((x) & UZ_ITEMS_COUNT_MASK)
519 #define	UZ_ITEMS_SLEEPERS(x)	((x) >> UZ_ITEMS_SLEEPER_SHIFT)
520 #define	UZ_ITEMS_SLEEPER	(1LL << UZ_ITEMS_SLEEPER_SHIFT)
521 
522 #define	ZONE_ASSERT_COLD(z)						\
523 	KASSERT(uma_zone_get_allocs((z)) == 0,				\
524 	    ("zone %s initialization after use.", (z)->uz_name))
525 
526 /* Domains are contiguous after the last CPU */
527 #define	ZDOM_GET(z, n)							\
528 	(&((uma_zone_domain_t)&(z)->uz_cpu[mp_maxid + 1])[n])
529 
530 #undef	UMA_ALIGN
531 
532 #ifdef _KERNEL
533 /* Internal prototypes */
534 static __inline uma_slab_t hash_sfind(struct uma_hash *hash, uint8_t *data);
535 
536 /* Lock Macros */
537 
538 #define	KEG_LOCKPTR(k, d)	(struct mtx *)&(k)->uk_domain[(d)].ud_lock
539 #define	KEG_LOCK_INIT(k, d, lc)						\
540 	do {								\
541 		if ((lc))						\
542 			mtx_init(KEG_LOCKPTR(k, d), (k)->uk_name,	\
543 			    (k)->uk_name, MTX_DEF | MTX_DUPOK);		\
544 		else							\
545 			mtx_init(KEG_LOCKPTR(k, d), (k)->uk_name,	\
546 			    "UMA zone", MTX_DEF | MTX_DUPOK);		\
547 	} while (0)
548 
549 #define	KEG_LOCK_FINI(k, d)	mtx_destroy(KEG_LOCKPTR(k, d))
550 #define	KEG_LOCK(k, d)							\
551 	({ mtx_lock(KEG_LOCKPTR(k, d)); KEG_LOCKPTR(k, d); })
552 #define	KEG_UNLOCK(k, d)	mtx_unlock(KEG_LOCKPTR(k, d))
553 #define	KEG_LOCK_ASSERT(k, d)	mtx_assert(KEG_LOCKPTR(k, d), MA_OWNED)
554 
555 #define	KEG_GET(zone, keg) do {					\
556 	(keg) = (zone)->uz_keg;					\
557 	KASSERT((void *)(keg) != NULL,				\
558 	    ("%s: Invalid zone %p type", __func__, (zone)));	\
559 	} while (0)
560 
561 #define	KEG_ASSERT_COLD(k)						\
562 	KASSERT(uma_keg_get_allocs((k)) == 0,				\
563 	    ("keg %s initialization after use.", (k)->uk_name))
564 
565 #define	ZDOM_LOCK_INIT(z, zdom, lc)					\
566 	do {								\
567 		if ((lc))						\
568 			mtx_init(&(zdom)->uzd_lock, (z)->uz_name,	\
569 			    (z)->uz_name, MTX_DEF | MTX_DUPOK);		\
570 		else							\
571 			mtx_init(&(zdom)->uzd_lock, (z)->uz_name,	\
572 			    "UMA zone", MTX_DEF | MTX_DUPOK);		\
573 	} while (0)
574 #define	ZDOM_LOCK_FINI(z)	mtx_destroy(&(z)->uzd_lock)
575 #define	ZDOM_LOCK_ASSERT(z)	mtx_assert(&(z)->uzd_lock, MA_OWNED)
576 
577 #define	ZDOM_LOCK(z)	mtx_lock(&(z)->uzd_lock)
578 #define	ZDOM_OWNED(z)	(mtx_owner(&(z)->uzd_lock) != NULL)
579 #define	ZDOM_UNLOCK(z)	mtx_unlock(&(z)->uzd_lock)
580 
581 #define	ZONE_LOCK(z)	ZDOM_LOCK(ZDOM_GET((z), 0))
582 #define	ZONE_UNLOCK(z)	ZDOM_UNLOCK(ZDOM_GET((z), 0))
583 #define	ZONE_LOCKPTR(z)	(&ZDOM_GET((z), 0)->uzd_lock)
584 
585 #define	ZONE_CROSS_LOCK_INIT(z)					\
586 	mtx_init(&(z)->uz_cross_lock, "UMA Cross", NULL, MTX_DEF)
587 #define	ZONE_CROSS_LOCK(z)	mtx_lock(&(z)->uz_cross_lock)
588 #define	ZONE_CROSS_UNLOCK(z)	mtx_unlock(&(z)->uz_cross_lock)
589 #define	ZONE_CROSS_LOCK_FINI(z)	mtx_destroy(&(z)->uz_cross_lock)
590 
591 /*
592  * Find a slab within a hash table.  This is used for OFFPAGE zones to lookup
593  * the slab structure.
594  *
595  * Arguments:
596  *	hash  The hash table to search.
597  *	data  The base page of the item.
598  *
599  * Returns:
600  *	A pointer to a slab if successful, else NULL.
601  */
602 static __inline uma_slab_t
603 hash_sfind(struct uma_hash *hash, uint8_t *data)
604 {
605         uma_hash_slab_t slab;
606         u_int hval;
607 
608         hval = UMA_HASH(hash, data);
609 
610         LIST_FOREACH(slab, &hash->uh_slab_hash[hval], uhs_hlink) {
611                 if ((uint8_t *)slab->uhs_data == data)
612                         return (&slab->uhs_slab);
613         }
614         return (NULL);
615 }
616 
617 static __inline uma_slab_t
618 vtoslab(vm_offset_t va)
619 {
620 	vm_page_t p;
621 
622 	p = PHYS_TO_VM_PAGE(pmap_kextract(va));
623 	return (p->plinks.uma.slab);
624 }
625 
626 static __inline void
627 vtozoneslab(vm_offset_t va, uma_zone_t *zone, uma_slab_t *slab)
628 {
629 	vm_page_t p;
630 
631 	p = PHYS_TO_VM_PAGE(pmap_kextract(va));
632 	*slab = p->plinks.uma.slab;
633 	*zone = p->plinks.uma.zone;
634 }
635 
636 static __inline void
637 vsetzoneslab(vm_offset_t va, uma_zone_t zone, uma_slab_t slab)
638 {
639 	vm_page_t p;
640 
641 	p = PHYS_TO_VM_PAGE(pmap_kextract(va));
642 	p->plinks.uma.slab = slab;
643 	p->plinks.uma.zone = zone;
644 }
645 
646 extern unsigned long uma_kmem_limit;
647 extern unsigned long uma_kmem_total;
648 
649 /* Adjust bytes under management by UMA. */
650 static inline void
651 uma_total_dec(unsigned long size)
652 {
653 
654 	atomic_subtract_long(&uma_kmem_total, size);
655 }
656 
657 static inline void
658 uma_total_inc(unsigned long size)
659 {
660 
661 	if (atomic_fetchadd_long(&uma_kmem_total, size) > uma_kmem_limit)
662 		uma_reclaim_wakeup();
663 }
664 
665 /*
666  * The following two functions may be defined by architecture specific code
667  * if they can provide more efficient allocation functions.  This is useful
668  * for using direct mapped addresses.
669  */
670 void *uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain,
671     uint8_t *pflag, int wait);
672 void uma_small_free(void *mem, vm_size_t size, uint8_t flags);
673 
674 /* Set a global soft limit on UMA managed memory. */
675 void uma_set_limit(unsigned long limit);
676 
677 #endif /* _KERNEL */
678 
679 #endif /* VM_UMA_INT_H */
680