xref: /titanic_52/usr/src/lib/libumem/common/umem.c (revision b60f2a0b921611326383e4789e0874e9e8a2e708)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * based on usr/src/uts/common/os/kmem.c r1.64 from 2001/12/18
31  *
32  * The slab allocator, as described in the following two papers:
33  *
34  *	Jeff Bonwick,
35  *	The Slab Allocator: An Object-Caching Kernel Memory Allocator.
36  *	Proceedings of the Summer 1994 Usenix Conference.
37  *	Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
38  *
39  *	Jeff Bonwick and Jonathan Adams,
40  *	Magazines and vmem: Extending the Slab Allocator to Many CPUs and
41  *	Arbitrary Resources.
42  *	Proceedings of the 2001 Usenix Conference.
43  *	Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
44  *
45  * 1. Overview
46  * -----------
47  * umem is very close to kmem in implementation.  There are four major
48  * areas of divergence:
49  *
50  *	* Initialization
51  *
52  *	* CPU handling
53  *
54  *	* umem_update()
55  *
56  *	* KM_SLEEP v.s. UMEM_NOFAIL
57  *
58  *	* lock ordering
59  *
60  * 2. Initialization
61  * -----------------
62  * kmem is initialized early on in boot, and knows that no one will call
63  * into it before it is ready.  umem does not have these luxuries. Instead,
64  * initialization is divided into two phases:
65  *
66  *	* library initialization, and
67  *
68  *	* first use
69  *
70  * umem's full initialization happens at the time of the first allocation
71  * request (via malloc() and friends, umem_alloc(), or umem_zalloc()),
72  * or the first call to umem_cache_create().
73  *
74  * umem_free(), and umem_cache_alloc() do not require special handling,
75  * since the only way to get valid arguments for them is to successfully
76  * call a function from the first group.
77  *
78  * 2.1. Library Initialization: umem_startup()
79  * -------------------------------------------
80  * umem_startup() is libumem.so's .init section.  It calls pthread_atfork()
81  * to install the handlers necessary for umem's Fork1-Safety.  Because of
82  * race condition issues, all other pre-umem_init() initialization is done
83  * statically (i.e. by the dynamic linker).
84  *
85  * For standalone use, umem_startup() returns everything to its initial
86  * state.
87  *
88  * 2.2. First use: umem_init()
89  * ------------------------------
90  * The first time any memory allocation function is used, we have to
91  * create the backing caches and vmem arenas which are needed for it.
92  * umem_init() is the central point for that task.  When it completes,
93  * umem_ready is either UMEM_READY (all set) or UMEM_READY_INIT_FAILED (unable
94  * to initialize, probably due to lack of memory).
95  *
96  * There are four different paths from which umem_init() is called:
97  *
98  *	* from umem_alloc() or umem_zalloc(), with 0 < size < UMEM_MAXBUF,
99  *
100  *	* from umem_alloc() or umem_zalloc(), with size > UMEM_MAXBUF,
101  *
102  *	* from umem_cache_create(), and
103  *
104  *	* from memalign(), with align > UMEM_ALIGN.
105  *
106  * The last three just check if umem is initialized, and call umem_init()
107  * if it is not.  For performance reasons, the first case is more complicated.
108  *
109  * 2.2.1. umem_alloc()/umem_zalloc(), with 0 < size < UMEM_MAXBUF
110  * -----------------------------------------------------------------
111  * In this case, umem_cache_alloc(&umem_null_cache, ...) is called.
112  * There is special case code in which causes any allocation on
113  * &umem_null_cache to fail by returning (NULL), regardless of the
114  * flags argument.
115  *
116  * So umem_cache_alloc() returns NULL, and umem_alloc()/umem_zalloc() call
117  * umem_alloc_retry().  umem_alloc_retry() sees that the allocation
118  * was agains &umem_null_cache, and calls umem_init().
119  *
120  * If initialization is successful, umem_alloc_retry() returns 1, which
121  * causes umem_alloc()/umem_zalloc() to start over, which causes it to load
122  * the (now valid) cache pointer from umem_alloc_table.
123  *
124  * 2.2.2. Dealing with race conditions
125  * -----------------------------------
126  * There are a couple race conditions resulting from the initialization
127  * code that we have to guard against:
128  *
129  *	* In umem_cache_create(), there is a special UMC_INTERNAL cflag
130  *	that is passed for caches created during initialization.  It
131  *	is illegal for a user to try to create a UMC_INTERNAL cache.
132  *	This allows initialization to proceed, but any other
133  *	umem_cache_create()s will block by calling umem_init().
134  *
135  *	* Since umem_null_cache has a 1-element cache_cpu, it's cache_cpu_mask
136  *	is always zero.  umem_cache_alloc uses cp->cache_cpu_mask to
137  *	mask the cpu number.  This prevents a race between grabbing a
138  *	cache pointer out of umem_alloc_table and growing the cpu array.
139  *
140  *
141  * 3. CPU handling
142  * ---------------
143  * kmem uses the CPU's sequence number to determine which "cpu cache" to
144  * use for an allocation.  Currently, there is no way to get the sequence
145  * number in userspace.
146  *
147  * umem keeps track of cpu information in umem_cpus, an array of umem_max_ncpus
148  * umem_cpu_t structures.  CURCPU() is a a "hint" function, which we then mask
149  * with either umem_cpu_mask or cp->cache_cpu_mask to find the actual "cpu" id.
150  * The mechanics of this is all in the CPU(mask) macro.
151  *
152  * Currently, umem uses _lwp_self() as its hint.
153  *
154  *
155  * 4. The update thread
156  * --------------------
157  * kmem uses a task queue, kmem_taskq, to do periodic maintenance on
158  * every kmem cache.  vmem has a periodic timeout for hash table resizing.
159  * The kmem_taskq also provides a separate context for kmem_cache_reap()'s
160  * to be done in, avoiding issues of the context of kmem_reap() callers.
161  *
162  * Instead, umem has the concept of "updates", which are asynchronous requests
163  * for work attached to single caches.  All caches with pending work are
164  * on a doubly linked list rooted at the umem_null_cache.  All update state
165  * is protected by the umem_update_lock mutex, and the umem_update_cv is used
166  * for notification between threads.
167  *
168  * 4.1. Cache states with regards to updates
169  * -----------------------------------------
170  * A given cache is in one of three states:
171  *
172  * Inactive		cache_uflags is zero, cache_u{next,prev} are NULL
173  *
174  * Work Requested	cache_uflags is non-zero (but UMU_ACTIVE is not set),
175  *			cache_u{next,prev} link the cache onto the global
176  *			update list
177  *
178  * Active		cache_uflags has UMU_ACTIVE set, cache_u{next,prev}
179  *			are NULL, and either umem_update_thr or
180  *			umem_st_update_thr are actively doing work on the
181  *			cache.
182  *
183  * An update can be added to any cache in any state -- if the cache is
184  * Inactive, it transitions to being Work Requested.  If the cache is
185  * Active, the worker will notice the new update and act on it before
186  * transitioning the cache to the Inactive state.
187  *
188  * If a cache is in the Active state, UMU_NOTIFY can be set, which asks
189  * the worker to broadcast the umem_update_cv when it has finished.
190  *
191  * 4.2. Update interface
192  * ---------------------
193  * umem_add_update() adds an update to a particular cache.
194  * umem_updateall() adds an update to all caches.
195  * umem_remove_updates() returns a cache to the Inactive state.
196  *
197  * umem_process_updates() process all caches in the Work Requested state.
198  *
199  * 4.3. Reaping
200  * ------------
201  * When umem_reap() is called (at the time of heap growth), it schedule
202  * UMU_REAP updates on every cache.  It then checks to see if the update
203  * thread exists (umem_update_thr != 0).  If it is, it broadcasts
204  * the umem_update_cv to wake the update thread up, and returns.
205  *
206  * If the update thread does not exist (umem_update_thr == 0), and the
207  * program currently has multiple threads, umem_reap() attempts to create
208  * a new update thread.
209  *
210  * If the process is not multithreaded, or the creation fails, umem_reap()
211  * calls umem_st_update() to do an inline update.
212  *
213  * 4.4. The update thread
214  * ----------------------
215  * The update thread spends most of its time in cond_timedwait() on the
216  * umem_update_cv.  It wakes up under two conditions:
217  *
218  *	* The timedwait times out, in which case it needs to run a global
219  *	update, or
220  *
221  *	* someone cond_broadcast(3THR)s the umem_update_cv, in which case
222  *	it needs to check if there are any caches in the Work Requested
223  *	state.
224  *
225  * When it is time for another global update, umem calls umem_cache_update()
226  * on every cache, then calls vmem_update(), which tunes the vmem structures.
227  * umem_cache_update() can request further work using umem_add_update().
228  *
229  * After any work from the global update completes, the update timer is
230  * reset to umem_reap_interval seconds in the future.  This makes the
231  * updates self-throttling.
232  *
233  * Reaps are similarly self-throttling.  After a UMU_REAP update has
234  * been scheduled on all caches, umem_reap() sets a flag and wakes up the
235  * update thread.  The update thread notices the flag, and resets the
236  * reap state.
237  *
238  * 4.5. Inline updates
239  * -------------------
240  * If the update thread is not running, umem_st_update() is used instead.  It
241  * immediately does a global update (as above), then calls
242  * umem_process_updates() to process both the reaps that umem_reap() added and
243  * any work generated by the global update.  Afterwards, it resets the reap
244  * state.
245  *
246  * While the umem_st_update() is running, umem_st_update_thr holds the thread
247  * id of the thread performing the update.
248  *
249  * 4.6. Updates and fork1()
250  * ------------------------
251  * umem has fork1() pre- and post-handlers which lock up (and release) every
252  * mutex in every cache.  They also lock up the umem_update_lock.  Since
253  * fork1() only copies over a single lwp, other threads (including the update
254  * thread) could have been actively using a cache in the parent.  This
255  * can lead to inconsistencies in the child process.
256  *
257  * Because we locked all of the mutexes, the only possible inconsistancies are:
258  *
259  *	* a umem_cache_alloc() could leak its buffer.
260  *
261  *	* a caller of umem_depot_alloc() could leak a magazine, and all the
262  *	buffers contained in it.
263  *
264  *	* a cache could be in the Active update state.  In the child, there
265  *	would be no thread actually working on it.
266  *
267  *	* a umem_hash_rescale() could leak the new hash table.
268  *
269  *	* a umem_magazine_resize() could be in progress.
270  *
271  *	* a umem_reap() could be in progress.
272  *
273  * The memory leaks we can't do anything about.  umem_release_child() resets
274  * the update state, moves any caches in the Active state to the Work Requested
275  * state.  This might cause some updates to be re-run, but UMU_REAP and
276  * UMU_HASH_RESCALE are effectively idempotent, and the worst that can
277  * happen from umem_magazine_resize() is resizing the magazine twice in close
278  * succession.
279  *
280  * Much of the cleanup in umem_release_child() is skipped if
281  * umem_st_update_thr == thr_self().  This is so that applications which call
282  * fork1() from a cache callback does not break.  Needless to say, any such
283  * application is tremendously broken.
284  *
285  *
286  * 5. KM_SLEEP v.s. UMEM_NOFAIL
287  * ----------------------------
288  * Allocations against kmem and vmem have two basic modes:  SLEEP and
289  * NOSLEEP.  A sleeping allocation is will go to sleep (waiting for
290  * more memory) instead of failing (returning NULL).
291  *
292  * SLEEP allocations presume an extremely multithreaded model, with
293  * a lot of allocation and deallocation activity.  umem cannot presume
294  * that its clients have any particular type of behavior.  Instead,
295  * it provides two types of allocations:
296  *
297  *	* UMEM_DEFAULT, equivalent to KM_NOSLEEP (i.e. return NULL on
298  *	failure)
299  *
300  *	* UMEM_NOFAIL, which, on failure, calls an optional callback
301  *	(registered with umem_nofail_callback()).
302  *
303  * The callback is invoked with no locks held, and can do an arbitrary
304  * amount of work.  It then has a choice between:
305  *
306  *	* Returning UMEM_CALLBACK_RETRY, which will cause the allocation
307  *	to be restarted.
308  *
309  *	* Returning UMEM_CALLBACK_EXIT(status), which will cause exit(2)
310  *	to be invoked with status.  If multiple threads attempt to do
311  *	this simultaneously, only one will call exit(2).
312  *
313  *	* Doing some kind of non-local exit (thr_exit(3thr), longjmp(3C),
314  *	etc.)
315  *
316  * The default callback returns UMEM_CALLBACK_EXIT(255).
317  *
318  * To have these callbacks without risk of state corruption (in the case of
319  * a non-local exit), we have to ensure that the callbacks get invoked
320  * close to the original allocation, with no inconsistent state or held
321  * locks.  The following steps are taken:
322  *
323  *	* All invocations of vmem are VM_NOSLEEP.
324  *
325  *	* All constructor callbacks (which can themselves to allocations)
326  *	are passed UMEM_DEFAULT as their required allocation argument.  This
327  *	way, the constructor will fail, allowing the highest-level allocation
328  *	invoke the nofail callback.
329  *
330  *	If a constructor callback _does_ do a UMEM_NOFAIL allocation, and
331  *	the nofail callback does a non-local exit, we will leak the
332  *	partially-constructed buffer.
333  *
334  *
335  * 6. Lock Ordering
336  * ----------------
337  * umem has a few more locks than kmem does, mostly in the update path.  The
338  * overall lock ordering (earlier locks must be acquired first) is:
339  *
340  *	umem_init_lock
341  *
342  *	vmem_list_lock
343  *	vmem_nosleep_lock.vmpl_mutex
344  *	vmem_t's:
345  *		vm_lock
346  *	sbrk_lock
347  *
348  *	umem_cache_lock
349  *	umem_update_lock
350  *	umem_flags_lock
351  *	umem_cache_t's:
352  *		cache_cpu[*].cc_lock
353  *		cache_depot_lock
354  *		cache_lock
355  *	umem_log_header_t's:
356  *		lh_cpu[*].clh_lock
357  *		lh_lock
358  */
359 
360 #include "c_synonyms.h"
361 #include <umem_impl.h>
362 #include <sys/vmem_impl_user.h>
363 #include "umem_base.h"
364 #include "vmem_base.h"
365 
366 #include <sys/processor.h>
367 #include <sys/sysmacros.h>
368 
369 #include <alloca.h>
370 #include <errno.h>
371 #include <limits.h>
372 #include <stdio.h>
373 #include <stdlib.h>
374 #include <string.h>
375 #include <strings.h>
376 #include <signal.h>
377 #include <unistd.h>
378 #include <atomic.h>
379 
380 #include "misc.h"
381 
382 #define	UMEM_VMFLAGS(umflag)	(VM_NOSLEEP)
383 
384 size_t pagesize;
385 
386 /*
387  * The default set of caches to back umem_alloc().
388  * These sizes should be reevaluated periodically.
389  *
390  * We want allocations that are multiples of the coherency granularity
391  * (64 bytes) to be satisfied from a cache which is a multiple of 64
392  * bytes, so that it will be 64-byte aligned.  For all multiples of 64,
393  * the next kmem_cache_size greater than or equal to it must be a
394  * multiple of 64.
395  *
396  * This table must be in sorted order, from smallest to highest.  The
397  * highest slot must be UMEM_MAXBUF, and every slot afterwards must be
398  * zero.
399  */
400 static int umem_alloc_sizes[] = {
401 #ifdef _LP64
402 	1 * 8,
403 	1 * 16,
404 	2 * 16,
405 	3 * 16,
406 #else
407 	1 * 8,
408 	2 * 8,
409 	3 * 8,
410 	4 * 8,		5 * 8,		6 * 8,		7 * 8,
411 #endif
412 	4 * 16,		5 * 16,		6 * 16,		7 * 16,
413 	4 * 32,		5 * 32,		6 * 32,		7 * 32,
414 	4 * 64,		5 * 64,		6 * 64,		7 * 64,
415 	4 * 128,	5 * 128,	6 * 128,	7 * 128,
416 	P2ALIGN(8192 / 7, 64),
417 	P2ALIGN(8192 / 6, 64),
418 	P2ALIGN(8192 / 5, 64),
419 	P2ALIGN(8192 / 4, 64), 2304,
420 	P2ALIGN(8192 / 3, 64),
421 	P2ALIGN(8192 / 2, 64), 4544,
422 	P2ALIGN(8192 / 1, 64), 9216,
423 	4096 * 3,
424 	UMEM_MAXBUF,				/* = 8192 * 2 */
425 	/* 24 slots for user expansion */
426 	0, 0, 0, 0, 0, 0, 0, 0,
427 	0, 0, 0, 0, 0, 0, 0, 0,
428 	0, 0, 0, 0, 0, 0, 0, 0,
429 };
430 #define	NUM_ALLOC_SIZES (sizeof (umem_alloc_sizes) / sizeof (*umem_alloc_sizes))
431 
432 static umem_magtype_t umem_magtype[] = {
433 	{ 1,	8,	3200,	65536	},
434 	{ 3,	16,	256,	32768	},
435 	{ 7,	32,	64,	16384	},
436 	{ 15,	64,	0,	8192	},
437 	{ 31,	64,	0,	4096	},
438 	{ 47,	64,	0,	2048	},
439 	{ 63,	64,	0,	1024	},
440 	{ 95,	64,	0,	512	},
441 	{ 143,	64,	0,	0	},
442 };
443 
444 /*
445  * umem tunables
446  */
447 uint32_t umem_max_ncpus;	/* # of CPU caches. */
448 
449 uint32_t umem_stack_depth = 15; /* # stack frames in a bufctl_audit */
450 uint32_t umem_reap_interval = 10; /* max reaping rate (seconds) */
451 uint_t umem_depot_contention = 2; /* max failed trylocks per real interval */
452 uint_t umem_abort = 1;		/* whether to abort on error */
453 uint_t umem_output = 0;		/* whether to write to standard error */
454 uint_t umem_logging = 0;	/* umem_log_enter() override */
455 uint32_t umem_mtbf = 0;		/* mean time between failures [default: off] */
456 size_t umem_transaction_log_size; /* size of transaction log */
457 size_t umem_content_log_size;	/* size of content log */
458 size_t umem_failure_log_size;	/* failure log [4 pages per CPU] */
459 size_t umem_slab_log_size;	/* slab create log [4 pages per CPU] */
460 size_t umem_content_maxsave = 256; /* UMF_CONTENTS max bytes to log */
461 size_t umem_lite_minsize = 0;	/* minimum buffer size for UMF_LITE */
462 size_t umem_lite_maxalign = 1024; /* maximum buffer alignment for UMF_LITE */
463 size_t umem_maxverify;		/* maximum bytes to inspect in debug routines */
464 size_t umem_minfirewall;	/* hardware-enforced redzone threshold */
465 
466 uint_t umem_flags = 0;
467 
468 mutex_t			umem_init_lock;		/* locks initialization */
469 cond_t			umem_init_cv;		/* initialization CV */
470 thread_t		umem_init_thr;		/* thread initializing */
471 int			umem_init_env_ready;	/* environ pre-initted */
472 int			umem_ready = UMEM_READY_STARTUP;
473 
474 static umem_nofail_callback_t *nofail_callback;
475 static mutex_t		umem_nofail_exit_lock;
476 static thread_t		umem_nofail_exit_thr;
477 
478 static umem_cache_t	*umem_slab_cache;
479 static umem_cache_t	*umem_bufctl_cache;
480 static umem_cache_t	*umem_bufctl_audit_cache;
481 
482 mutex_t			umem_flags_lock;
483 
484 static vmem_t		*heap_arena;
485 static vmem_alloc_t	*heap_alloc;
486 static vmem_free_t	*heap_free;
487 
488 static vmem_t		*umem_internal_arena;
489 static vmem_t		*umem_cache_arena;
490 static vmem_t		*umem_hash_arena;
491 static vmem_t		*umem_log_arena;
492 static vmem_t		*umem_oversize_arena;
493 static vmem_t		*umem_va_arena;
494 static vmem_t		*umem_default_arena;
495 static vmem_t		*umem_firewall_va_arena;
496 static vmem_t		*umem_firewall_arena;
497 
498 vmem_t			*umem_memalign_arena;
499 
500 umem_log_header_t *umem_transaction_log;
501 umem_log_header_t *umem_content_log;
502 umem_log_header_t *umem_failure_log;
503 umem_log_header_t *umem_slab_log;
504 
505 extern thread_t _thr_self(void);
506 #define	CPUHINT()		(_thr_self())
507 #define	CPUHINT_MAX()		INT_MAX
508 
509 #define	CPU(mask)		(umem_cpus + (CPUHINT() & (mask)))
510 static umem_cpu_t umem_startup_cpu = {	/* initial, single, cpu */
511 	UMEM_CACHE_SIZE(0),
512 	0
513 };
514 
515 static uint32_t umem_cpu_mask = 0;			/* global cpu mask */
516 static umem_cpu_t *umem_cpus = &umem_startup_cpu;	/* cpu list */
517 
518 volatile uint32_t umem_reaping;
519 
520 thread_t		umem_update_thr;
521 struct timeval		umem_update_next;	/* timeofday of next update */
522 volatile thread_t	umem_st_update_thr;	/* only used when single-thd */
523 
524 #define	IN_UPDATE()	(thr_self() == umem_update_thr || \
525 			    thr_self() == umem_st_update_thr)
526 #define	IN_REAP()	IN_UPDATE()
527 
528 mutex_t			umem_update_lock;	/* cache_u{next,prev,flags} */
529 cond_t			umem_update_cv;
530 
531 volatile hrtime_t umem_reap_next;	/* min hrtime of next reap */
532 
533 mutex_t			umem_cache_lock;	/* inter-cache linkage only */
534 
535 #ifdef UMEM_STANDALONE
536 umem_cache_t		umem_null_cache;
537 static const umem_cache_t umem_null_cache_template = {
538 #else
539 umem_cache_t		umem_null_cache = {
540 #endif
541 	0, 0, 0, 0, 0,
542 	0, 0,
543 	0, 0,
544 	0, 0,
545 	"invalid_cache",
546 	0, 0,
547 	NULL, NULL, NULL, NULL,
548 	NULL,
549 	0, 0, 0, 0,
550 	&umem_null_cache, &umem_null_cache,
551 	&umem_null_cache, &umem_null_cache,
552 	0,
553 	DEFAULTMUTEX,				/* start of slab layer */
554 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
555 	&umem_null_cache.cache_nullslab,
556 	{
557 		&umem_null_cache,
558 		NULL,
559 		&umem_null_cache.cache_nullslab,
560 		&umem_null_cache.cache_nullslab,
561 		NULL,
562 		-1,
563 		0
564 	},
565 	NULL,
566 	NULL,
567 	DEFAULTMUTEX,				/* start of depot layer */
568 	NULL, {
569 		NULL, 0, 0, 0, 0
570 	}, {
571 		NULL, 0, 0, 0, 0
572 	}, {
573 		{
574 			DEFAULTMUTEX,		/* start of CPU cache */
575 			0, 0, NULL, NULL, -1, -1, 0
576 		}
577 	}
578 };
579 
580 #define	ALLOC_TABLE_4 \
581 	&umem_null_cache, &umem_null_cache, &umem_null_cache, &umem_null_cache
582 
583 #define	ALLOC_TABLE_64 \
584 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
585 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
586 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, \
587 	ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4, ALLOC_TABLE_4
588 
589 #define	ALLOC_TABLE_1024 \
590 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
591 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
592 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, \
593 	ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64, ALLOC_TABLE_64
594 
595 static umem_cache_t *umem_alloc_table[UMEM_MAXBUF >> UMEM_ALIGN_SHIFT] = {
596 	ALLOC_TABLE_1024,
597 	ALLOC_TABLE_1024
598 };
599 
600 
601 /* Used to constrain audit-log stack traces */
602 caddr_t			umem_min_stack;
603 caddr_t			umem_max_stack;
604 
605 
606 #define	UMERR_MODIFIED	0	/* buffer modified while on freelist */
607 #define	UMERR_REDZONE	1	/* redzone violation (write past end of buf) */
608 #define	UMERR_DUPFREE	2	/* freed a buffer twice */
609 #define	UMERR_BADADDR	3	/* freed a bad (unallocated) address */
610 #define	UMERR_BADBUFTAG	4	/* buftag corrupted */
611 #define	UMERR_BADBUFCTL	5	/* bufctl corrupted */
612 #define	UMERR_BADCACHE	6	/* freed a buffer to the wrong cache */
613 #define	UMERR_BADSIZE	7	/* alloc size != free size */
614 #define	UMERR_BADBASE	8	/* buffer base address wrong */
615 
616 struct {
617 	hrtime_t	ump_timestamp;	/* timestamp of error */
618 	int		ump_error;	/* type of umem error (UMERR_*) */
619 	void		*ump_buffer;	/* buffer that induced abort */
620 	void		*ump_realbuf;	/* real start address for buffer */
621 	umem_cache_t	*ump_cache;	/* buffer's cache according to client */
622 	umem_cache_t	*ump_realcache;	/* actual cache containing buffer */
623 	umem_slab_t	*ump_slab;	/* slab accoring to umem_findslab() */
624 	umem_bufctl_t	*ump_bufctl;	/* bufctl */
625 } umem_abort_info;
626 
627 static void
628 copy_pattern(uint64_t pattern, void *buf_arg, size_t size)
629 {
630 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
631 	uint64_t *buf = buf_arg;
632 
633 	while (buf < bufend)
634 		*buf++ = pattern;
635 }
636 
637 static void *
638 verify_pattern(uint64_t pattern, void *buf_arg, size_t size)
639 {
640 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
641 	uint64_t *buf;
642 
643 	for (buf = buf_arg; buf < bufend; buf++)
644 		if (*buf != pattern)
645 			return (buf);
646 	return (NULL);
647 }
648 
649 static void *
650 verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size)
651 {
652 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
653 	uint64_t *buf;
654 
655 	for (buf = buf_arg; buf < bufend; buf++) {
656 		if (*buf != old) {
657 			copy_pattern(old, buf_arg,
658 			    (char *)buf - (char *)buf_arg);
659 			return (buf);
660 		}
661 		*buf = new;
662 	}
663 
664 	return (NULL);
665 }
666 
667 void
668 umem_cache_applyall(void (*func)(umem_cache_t *))
669 {
670 	umem_cache_t *cp;
671 
672 	(void) mutex_lock(&umem_cache_lock);
673 	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
674 	    cp = cp->cache_next)
675 		func(cp);
676 	(void) mutex_unlock(&umem_cache_lock);
677 }
678 
679 static void
680 umem_add_update_unlocked(umem_cache_t *cp, int flags)
681 {
682 	umem_cache_t *cnext, *cprev;
683 
684 	flags &= ~UMU_ACTIVE;
685 
686 	if (!flags)
687 		return;
688 
689 	if (cp->cache_uflags & UMU_ACTIVE) {
690 		cp->cache_uflags |= flags;
691 	} else {
692 		if (cp->cache_unext != NULL) {
693 			ASSERT(cp->cache_uflags != 0);
694 			cp->cache_uflags |= flags;
695 		} else {
696 			ASSERT(cp->cache_uflags == 0);
697 			cp->cache_uflags = flags;
698 			cp->cache_unext = cnext = &umem_null_cache;
699 			cp->cache_uprev = cprev = umem_null_cache.cache_uprev;
700 			cnext->cache_uprev = cp;
701 			cprev->cache_unext = cp;
702 		}
703 	}
704 }
705 
706 static void
707 umem_add_update(umem_cache_t *cp, int flags)
708 {
709 	(void) mutex_lock(&umem_update_lock);
710 
711 	umem_add_update_unlocked(cp, flags);
712 
713 	if (!IN_UPDATE())
714 		(void) cond_broadcast(&umem_update_cv);
715 
716 	(void) mutex_unlock(&umem_update_lock);
717 }
718 
719 /*
720  * Remove a cache from the update list, waiting for any in-progress work to
721  * complete first.
722  */
723 static void
724 umem_remove_updates(umem_cache_t *cp)
725 {
726 	(void) mutex_lock(&umem_update_lock);
727 
728 	/*
729 	 * Get it out of the active state
730 	 */
731 	while (cp->cache_uflags & UMU_ACTIVE) {
732 		int cancel_state;
733 
734 		ASSERT(cp->cache_unext == NULL);
735 
736 		cp->cache_uflags |= UMU_NOTIFY;
737 
738 		/*
739 		 * Make sure the update state is sane, before we wait
740 		 */
741 		ASSERT(umem_update_thr != 0 || umem_st_update_thr != 0);
742 		ASSERT(umem_update_thr != thr_self() &&
743 		    umem_st_update_thr != thr_self());
744 
745 		(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
746 		    &cancel_state);
747 		(void) cond_wait(&umem_update_cv, &umem_update_lock);
748 		(void) pthread_setcancelstate(cancel_state, NULL);
749 	}
750 	/*
751 	 * Get it out of the Work Requested state
752 	 */
753 	if (cp->cache_unext != NULL) {
754 		cp->cache_uprev->cache_unext = cp->cache_unext;
755 		cp->cache_unext->cache_uprev = cp->cache_uprev;
756 		cp->cache_uprev = cp->cache_unext = NULL;
757 		cp->cache_uflags = 0;
758 	}
759 	/*
760 	 * Make sure it is in the Inactive state
761 	 */
762 	ASSERT(cp->cache_unext == NULL && cp->cache_uflags == 0);
763 	(void) mutex_unlock(&umem_update_lock);
764 }
765 
766 static void
767 umem_updateall(int flags)
768 {
769 	umem_cache_t *cp;
770 
771 	/*
772 	 * NOTE:  To prevent deadlock, umem_cache_lock is always acquired first.
773 	 *
774 	 * (umem_add_update is called from things run via umem_cache_applyall)
775 	 */
776 	(void) mutex_lock(&umem_cache_lock);
777 	(void) mutex_lock(&umem_update_lock);
778 
779 	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
780 	    cp = cp->cache_next)
781 		umem_add_update_unlocked(cp, flags);
782 
783 	if (!IN_UPDATE())
784 		(void) cond_broadcast(&umem_update_cv);
785 
786 	(void) mutex_unlock(&umem_update_lock);
787 	(void) mutex_unlock(&umem_cache_lock);
788 }
789 
790 /*
791  * Debugging support.  Given a buffer address, find its slab.
792  */
793 static umem_slab_t *
794 umem_findslab(umem_cache_t *cp, void *buf)
795 {
796 	umem_slab_t *sp;
797 
798 	(void) mutex_lock(&cp->cache_lock);
799 	for (sp = cp->cache_nullslab.slab_next;
800 	    sp != &cp->cache_nullslab; sp = sp->slab_next) {
801 		if (UMEM_SLAB_MEMBER(sp, buf)) {
802 			(void) mutex_unlock(&cp->cache_lock);
803 			return (sp);
804 		}
805 	}
806 	(void) mutex_unlock(&cp->cache_lock);
807 
808 	return (NULL);
809 }
810 
811 static void
812 umem_error(int error, umem_cache_t *cparg, void *bufarg)
813 {
814 	umem_buftag_t *btp = NULL;
815 	umem_bufctl_t *bcp = NULL;
816 	umem_cache_t *cp = cparg;
817 	umem_slab_t *sp;
818 	uint64_t *off;
819 	void *buf = bufarg;
820 
821 	int old_logging = umem_logging;
822 
823 	umem_logging = 0;	/* stop logging when a bad thing happens */
824 
825 	umem_abort_info.ump_timestamp = gethrtime();
826 
827 	sp = umem_findslab(cp, buf);
828 	if (sp == NULL) {
829 		for (cp = umem_null_cache.cache_prev; cp != &umem_null_cache;
830 		    cp = cp->cache_prev) {
831 			if ((sp = umem_findslab(cp, buf)) != NULL)
832 				break;
833 		}
834 	}
835 
836 	if (sp == NULL) {
837 		cp = NULL;
838 		error = UMERR_BADADDR;
839 	} else {
840 		if (cp != cparg)
841 			error = UMERR_BADCACHE;
842 		else
843 			buf = (char *)bufarg - ((uintptr_t)bufarg -
844 			    (uintptr_t)sp->slab_base) % cp->cache_chunksize;
845 		if (buf != bufarg)
846 			error = UMERR_BADBASE;
847 		if (cp->cache_flags & UMF_BUFTAG)
848 			btp = UMEM_BUFTAG(cp, buf);
849 		if (cp->cache_flags & UMF_HASH) {
850 			(void) mutex_lock(&cp->cache_lock);
851 			for (bcp = *UMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next)
852 				if (bcp->bc_addr == buf)
853 					break;
854 			(void) mutex_unlock(&cp->cache_lock);
855 			if (bcp == NULL && btp != NULL)
856 				bcp = btp->bt_bufctl;
857 			if (umem_findslab(cp->cache_bufctl_cache, bcp) ==
858 			    NULL || P2PHASE((uintptr_t)bcp, UMEM_ALIGN) ||
859 			    bcp->bc_addr != buf) {
860 				error = UMERR_BADBUFCTL;
861 				bcp = NULL;
862 			}
863 		}
864 	}
865 
866 	umem_abort_info.ump_error = error;
867 	umem_abort_info.ump_buffer = bufarg;
868 	umem_abort_info.ump_realbuf = buf;
869 	umem_abort_info.ump_cache = cparg;
870 	umem_abort_info.ump_realcache = cp;
871 	umem_abort_info.ump_slab = sp;
872 	umem_abort_info.ump_bufctl = bcp;
873 
874 	umem_printf("umem allocator: ");
875 
876 	switch (error) {
877 
878 	case UMERR_MODIFIED:
879 		umem_printf("buffer modified after being freed\n");
880 		off = verify_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify);
881 		if (off == NULL)	/* shouldn't happen */
882 			off = buf;
883 		umem_printf("modification occurred at offset 0x%lx "
884 		    "(0x%llx replaced by 0x%llx)\n",
885 		    (uintptr_t)off - (uintptr_t)buf,
886 		    (longlong_t)UMEM_FREE_PATTERN, (longlong_t)*off);
887 		break;
888 
889 	case UMERR_REDZONE:
890 		umem_printf("redzone violation: write past end of buffer\n");
891 		break;
892 
893 	case UMERR_BADADDR:
894 		umem_printf("invalid free: buffer not in cache\n");
895 		break;
896 
897 	case UMERR_DUPFREE:
898 		umem_printf("duplicate free: buffer freed twice\n");
899 		break;
900 
901 	case UMERR_BADBUFTAG:
902 		umem_printf("boundary tag corrupted\n");
903 		umem_printf("bcp ^ bxstat = %lx, should be %lx\n",
904 		    (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat,
905 		    UMEM_BUFTAG_FREE);
906 		break;
907 
908 	case UMERR_BADBUFCTL:
909 		umem_printf("bufctl corrupted\n");
910 		break;
911 
912 	case UMERR_BADCACHE:
913 		umem_printf("buffer freed to wrong cache\n");
914 		umem_printf("buffer was allocated from %s,\n", cp->cache_name);
915 		umem_printf("caller attempting free to %s.\n",
916 		    cparg->cache_name);
917 		break;
918 
919 	case UMERR_BADSIZE:
920 		umem_printf("bad free: free size (%u) != alloc size (%u)\n",
921 		    UMEM_SIZE_DECODE(((uint32_t *)btp)[0]),
922 		    UMEM_SIZE_DECODE(((uint32_t *)btp)[1]));
923 		break;
924 
925 	case UMERR_BADBASE:
926 		umem_printf("bad free: free address (%p) != alloc address "
927 		    "(%p)\n", bufarg, buf);
928 		break;
929 	}
930 
931 	umem_printf("buffer=%p  bufctl=%p  cache: %s\n",
932 	    bufarg, (void *)bcp, cparg->cache_name);
933 
934 	if (bcp != NULL && (cp->cache_flags & UMF_AUDIT) &&
935 	    error != UMERR_BADBUFCTL) {
936 		int d;
937 		timespec_t ts;
938 		hrtime_t diff;
939 		umem_bufctl_audit_t *bcap = (umem_bufctl_audit_t *)bcp;
940 
941 		diff = umem_abort_info.ump_timestamp - bcap->bc_timestamp;
942 		ts.tv_sec = diff / NANOSEC;
943 		ts.tv_nsec = diff % NANOSEC;
944 
945 		umem_printf("previous transaction on buffer %p:\n", buf);
946 		umem_printf("thread=%p  time=T-%ld.%09ld  slab=%p  cache: %s\n",
947 		    (void *)(intptr_t)bcap->bc_thread, ts.tv_sec, ts.tv_nsec,
948 		    (void *)sp, cp->cache_name);
949 		for (d = 0; d < MIN(bcap->bc_depth, umem_stack_depth); d++) {
950 			(void) print_sym((void *)bcap->bc_stack[d]);
951 			umem_printf("\n");
952 		}
953 	}
954 
955 	umem_err_recoverable("umem: heap corruption detected");
956 
957 	umem_logging = old_logging;	/* resume logging */
958 }
959 
960 void
961 umem_nofail_callback(umem_nofail_callback_t *cb)
962 {
963 	nofail_callback = cb;
964 }
965 
966 static int
967 umem_alloc_retry(umem_cache_t *cp, int umflag)
968 {
969 	if (cp == &umem_null_cache) {
970 		if (umem_init())
971 			return (1);				/* retry */
972 		/*
973 		 * Initialization failed.  Do normal failure processing.
974 		 */
975 	}
976 	if (umflag & UMEM_NOFAIL) {
977 		int def_result = UMEM_CALLBACK_EXIT(255);
978 		int result = def_result;
979 		umem_nofail_callback_t *callback = nofail_callback;
980 
981 		if (callback != NULL)
982 			result = callback();
983 
984 		if (result == UMEM_CALLBACK_RETRY)
985 			return (1);
986 
987 		if ((result & ~0xFF) != UMEM_CALLBACK_EXIT(0)) {
988 			log_message("nofail callback returned %x\n", result);
989 			result = def_result;
990 		}
991 
992 		/*
993 		 * only one thread will call exit
994 		 */
995 		if (umem_nofail_exit_thr == thr_self())
996 			umem_panic("recursive UMEM_CALLBACK_EXIT()\n");
997 
998 		(void) mutex_lock(&umem_nofail_exit_lock);
999 		umem_nofail_exit_thr = thr_self();
1000 		exit(result & 0xFF);
1001 		/*NOTREACHED*/
1002 	}
1003 	return (0);
1004 }
1005 
1006 static umem_log_header_t *
1007 umem_log_init(size_t logsize)
1008 {
1009 	umem_log_header_t *lhp;
1010 	int nchunks = 4 * umem_max_ncpus;
1011 	size_t lhsize = offsetof(umem_log_header_t, lh_cpu[umem_max_ncpus]);
1012 	int i;
1013 
1014 	if (logsize == 0)
1015 		return (NULL);
1016 
1017 	/*
1018 	 * Make sure that lhp->lh_cpu[] is nicely aligned
1019 	 * to prevent false sharing of cache lines.
1020 	 */
1021 	lhsize = P2ROUNDUP(lhsize, UMEM_ALIGN);
1022 	lhp = vmem_xalloc(umem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0,
1023 	    NULL, NULL, VM_NOSLEEP);
1024 	if (lhp == NULL)
1025 		goto fail;
1026 
1027 	bzero(lhp, lhsize);
1028 
1029 	(void) mutex_init(&lhp->lh_lock, USYNC_THREAD, NULL);
1030 	lhp->lh_nchunks = nchunks;
1031 	lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks, PAGESIZE);
1032 	if (lhp->lh_chunksize == 0)
1033 		lhp->lh_chunksize = PAGESIZE;
1034 
1035 	lhp->lh_base = vmem_alloc(umem_log_arena,
1036 	    lhp->lh_chunksize * nchunks, VM_NOSLEEP);
1037 	if (lhp->lh_base == NULL)
1038 		goto fail;
1039 
1040 	lhp->lh_free = vmem_alloc(umem_log_arena,
1041 	    nchunks * sizeof (int), VM_NOSLEEP);
1042 	if (lhp->lh_free == NULL)
1043 		goto fail;
1044 
1045 	bzero(lhp->lh_base, lhp->lh_chunksize * nchunks);
1046 
1047 	for (i = 0; i < umem_max_ncpus; i++) {
1048 		umem_cpu_log_header_t *clhp = &lhp->lh_cpu[i];
1049 		(void) mutex_init(&clhp->clh_lock, USYNC_THREAD, NULL);
1050 		clhp->clh_chunk = i;
1051 	}
1052 
1053 	for (i = umem_max_ncpus; i < nchunks; i++)
1054 		lhp->lh_free[i] = i;
1055 
1056 	lhp->lh_head = umem_max_ncpus;
1057 	lhp->lh_tail = 0;
1058 
1059 	return (lhp);
1060 
1061 fail:
1062 	if (lhp != NULL) {
1063 		if (lhp->lh_base != NULL)
1064 			vmem_free(umem_log_arena, lhp->lh_base,
1065 			    lhp->lh_chunksize * nchunks);
1066 
1067 		vmem_xfree(umem_log_arena, lhp, lhsize);
1068 	}
1069 	return (NULL);
1070 }
1071 
1072 static void *
1073 umem_log_enter(umem_log_header_t *lhp, void *data, size_t size)
1074 {
1075 	void *logspace;
1076 	umem_cpu_log_header_t *clhp =
1077 	    &lhp->lh_cpu[CPU(umem_cpu_mask)->cpu_number];
1078 
1079 	if (lhp == NULL || umem_logging == 0)
1080 		return (NULL);
1081 
1082 	(void) mutex_lock(&clhp->clh_lock);
1083 	clhp->clh_hits++;
1084 	if (size > clhp->clh_avail) {
1085 		(void) mutex_lock(&lhp->lh_lock);
1086 		lhp->lh_hits++;
1087 		lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk;
1088 		lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks;
1089 		clhp->clh_chunk = lhp->lh_free[lhp->lh_head];
1090 		lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks;
1091 		clhp->clh_current = lhp->lh_base +
1092 		    clhp->clh_chunk * lhp->lh_chunksize;
1093 		clhp->clh_avail = lhp->lh_chunksize;
1094 		if (size > lhp->lh_chunksize)
1095 			size = lhp->lh_chunksize;
1096 		(void) mutex_unlock(&lhp->lh_lock);
1097 	}
1098 	logspace = clhp->clh_current;
1099 	clhp->clh_current += size;
1100 	clhp->clh_avail -= size;
1101 	bcopy(data, logspace, size);
1102 	(void) mutex_unlock(&clhp->clh_lock);
1103 	return (logspace);
1104 }
1105 
1106 #define	UMEM_AUDIT(lp, cp, bcp)						\
1107 {									\
1108 	umem_bufctl_audit_t *_bcp = (umem_bufctl_audit_t *)(bcp);	\
1109 	_bcp->bc_timestamp = gethrtime();				\
1110 	_bcp->bc_thread = thr_self();					\
1111 	_bcp->bc_depth = getpcstack(_bcp->bc_stack, umem_stack_depth,	\
1112 	    (cp != NULL) && (cp->cache_flags & UMF_CHECKSIGNAL));	\
1113 	_bcp->bc_lastlog = umem_log_enter((lp), _bcp,			\
1114 	    UMEM_BUFCTL_AUDIT_SIZE);					\
1115 }
1116 
1117 static void
1118 umem_log_event(umem_log_header_t *lp, umem_cache_t *cp,
1119 	umem_slab_t *sp, void *addr)
1120 {
1121 	umem_bufctl_audit_t *bcp;
1122 	UMEM_LOCAL_BUFCTL_AUDIT(&bcp);
1123 
1124 	bzero(bcp, UMEM_BUFCTL_AUDIT_SIZE);
1125 	bcp->bc_addr = addr;
1126 	bcp->bc_slab = sp;
1127 	bcp->bc_cache = cp;
1128 	UMEM_AUDIT(lp, cp, bcp);
1129 }
1130 
1131 /*
1132  * Create a new slab for cache cp.
1133  */
1134 static umem_slab_t *
1135 umem_slab_create(umem_cache_t *cp, int umflag)
1136 {
1137 	size_t slabsize = cp->cache_slabsize;
1138 	size_t chunksize = cp->cache_chunksize;
1139 	int cache_flags = cp->cache_flags;
1140 	size_t color, chunks;
1141 	char *buf, *slab;
1142 	umem_slab_t *sp;
1143 	umem_bufctl_t *bcp;
1144 	vmem_t *vmp = cp->cache_arena;
1145 
1146 	color = cp->cache_color + cp->cache_align;
1147 	if (color > cp->cache_maxcolor)
1148 		color = cp->cache_mincolor;
1149 	cp->cache_color = color;
1150 
1151 	slab = vmem_alloc(vmp, slabsize, UMEM_VMFLAGS(umflag));
1152 
1153 	if (slab == NULL)
1154 		goto vmem_alloc_failure;
1155 
1156 	ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0);
1157 
1158 	if (!(cp->cache_cflags & UMC_NOTOUCH) &&
1159 	    (cp->cache_flags & UMF_DEADBEEF))
1160 		copy_pattern(UMEM_UNINITIALIZED_PATTERN, slab, slabsize);
1161 
1162 	if (cache_flags & UMF_HASH) {
1163 		if ((sp = _umem_cache_alloc(umem_slab_cache, umflag)) == NULL)
1164 			goto slab_alloc_failure;
1165 		chunks = (slabsize - color) / chunksize;
1166 	} else {
1167 		sp = UMEM_SLAB(cp, slab);
1168 		chunks = (slabsize - sizeof (umem_slab_t) - color) / chunksize;
1169 	}
1170 
1171 	sp->slab_cache	= cp;
1172 	sp->slab_head	= NULL;
1173 	sp->slab_refcnt	= 0;
1174 	sp->slab_base	= buf = slab + color;
1175 	sp->slab_chunks	= chunks;
1176 
1177 	ASSERT(chunks > 0);
1178 	while (chunks-- != 0) {
1179 		if (cache_flags & UMF_HASH) {
1180 			bcp = _umem_cache_alloc(cp->cache_bufctl_cache, umflag);
1181 			if (bcp == NULL)
1182 				goto bufctl_alloc_failure;
1183 			if (cache_flags & UMF_AUDIT) {
1184 				umem_bufctl_audit_t *bcap =
1185 				    (umem_bufctl_audit_t *)bcp;
1186 				bzero(bcap, UMEM_BUFCTL_AUDIT_SIZE);
1187 				bcap->bc_cache = cp;
1188 			}
1189 			bcp->bc_addr = buf;
1190 			bcp->bc_slab = sp;
1191 		} else {
1192 			bcp = UMEM_BUFCTL(cp, buf);
1193 		}
1194 		if (cache_flags & UMF_BUFTAG) {
1195 			umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
1196 			btp->bt_redzone = UMEM_REDZONE_PATTERN;
1197 			btp->bt_bufctl = bcp;
1198 			btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE;
1199 			if (cache_flags & UMF_DEADBEEF) {
1200 				copy_pattern(UMEM_FREE_PATTERN, buf,
1201 				    cp->cache_verify);
1202 			}
1203 		}
1204 		bcp->bc_next = sp->slab_head;
1205 		sp->slab_head = bcp;
1206 		buf += chunksize;
1207 	}
1208 
1209 	umem_log_event(umem_slab_log, cp, sp, slab);
1210 
1211 	return (sp);
1212 
1213 bufctl_alloc_failure:
1214 
1215 	while ((bcp = sp->slab_head) != NULL) {
1216 		sp->slab_head = bcp->bc_next;
1217 		_umem_cache_free(cp->cache_bufctl_cache, bcp);
1218 	}
1219 	_umem_cache_free(umem_slab_cache, sp);
1220 
1221 slab_alloc_failure:
1222 
1223 	vmem_free(vmp, slab, slabsize);
1224 
1225 vmem_alloc_failure:
1226 
1227 	umem_log_event(umem_failure_log, cp, NULL, NULL);
1228 	atomic_add_64(&cp->cache_alloc_fail, 1);
1229 
1230 	return (NULL);
1231 }
1232 
1233 /*
1234  * Destroy a slab.
1235  */
1236 static void
1237 umem_slab_destroy(umem_cache_t *cp, umem_slab_t *sp)
1238 {
1239 	vmem_t *vmp = cp->cache_arena;
1240 	void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum);
1241 
1242 	if (cp->cache_flags & UMF_HASH) {
1243 		umem_bufctl_t *bcp;
1244 		while ((bcp = sp->slab_head) != NULL) {
1245 			sp->slab_head = bcp->bc_next;
1246 			_umem_cache_free(cp->cache_bufctl_cache, bcp);
1247 		}
1248 		_umem_cache_free(umem_slab_cache, sp);
1249 	}
1250 	vmem_free(vmp, slab, cp->cache_slabsize);
1251 }
1252 
1253 /*
1254  * Allocate a raw (unconstructed) buffer from cp's slab layer.
1255  */
1256 static void *
1257 umem_slab_alloc(umem_cache_t *cp, int umflag)
1258 {
1259 	umem_bufctl_t *bcp, **hash_bucket;
1260 	umem_slab_t *sp;
1261 	void *buf;
1262 
1263 	(void) mutex_lock(&cp->cache_lock);
1264 	cp->cache_slab_alloc++;
1265 	sp = cp->cache_freelist;
1266 	ASSERT(sp->slab_cache == cp);
1267 	if (sp->slab_head == NULL) {
1268 		/*
1269 		 * The freelist is empty.  Create a new slab.
1270 		 */
1271 		(void) mutex_unlock(&cp->cache_lock);
1272 		if (cp == &umem_null_cache)
1273 			return (NULL);
1274 		if ((sp = umem_slab_create(cp, umflag)) == NULL)
1275 			return (NULL);
1276 		(void) mutex_lock(&cp->cache_lock);
1277 		cp->cache_slab_create++;
1278 		if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax)
1279 			cp->cache_bufmax = cp->cache_buftotal;
1280 		sp->slab_next = cp->cache_freelist;
1281 		sp->slab_prev = cp->cache_freelist->slab_prev;
1282 		sp->slab_next->slab_prev = sp;
1283 		sp->slab_prev->slab_next = sp;
1284 		cp->cache_freelist = sp;
1285 	}
1286 
1287 	sp->slab_refcnt++;
1288 	ASSERT(sp->slab_refcnt <= sp->slab_chunks);
1289 
1290 	/*
1291 	 * If we're taking the last buffer in the slab,
1292 	 * remove the slab from the cache's freelist.
1293 	 */
1294 	bcp = sp->slab_head;
1295 	if ((sp->slab_head = bcp->bc_next) == NULL) {
1296 		cp->cache_freelist = sp->slab_next;
1297 		ASSERT(sp->slab_refcnt == sp->slab_chunks);
1298 	}
1299 
1300 	if (cp->cache_flags & UMF_HASH) {
1301 		/*
1302 		 * Add buffer to allocated-address hash table.
1303 		 */
1304 		buf = bcp->bc_addr;
1305 		hash_bucket = UMEM_HASH(cp, buf);
1306 		bcp->bc_next = *hash_bucket;
1307 		*hash_bucket = bcp;
1308 		if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) {
1309 			UMEM_AUDIT(umem_transaction_log, cp, bcp);
1310 		}
1311 	} else {
1312 		buf = UMEM_BUF(cp, bcp);
1313 	}
1314 
1315 	ASSERT(UMEM_SLAB_MEMBER(sp, buf));
1316 
1317 	(void) mutex_unlock(&cp->cache_lock);
1318 
1319 	return (buf);
1320 }
1321 
1322 /*
1323  * Free a raw (unconstructed) buffer to cp's slab layer.
1324  */
1325 static void
1326 umem_slab_free(umem_cache_t *cp, void *buf)
1327 {
1328 	umem_slab_t *sp;
1329 	umem_bufctl_t *bcp, **prev_bcpp;
1330 
1331 	ASSERT(buf != NULL);
1332 
1333 	(void) mutex_lock(&cp->cache_lock);
1334 	cp->cache_slab_free++;
1335 
1336 	if (cp->cache_flags & UMF_HASH) {
1337 		/*
1338 		 * Look up buffer in allocated-address hash table.
1339 		 */
1340 		prev_bcpp = UMEM_HASH(cp, buf);
1341 		while ((bcp = *prev_bcpp) != NULL) {
1342 			if (bcp->bc_addr == buf) {
1343 				*prev_bcpp = bcp->bc_next;
1344 				sp = bcp->bc_slab;
1345 				break;
1346 			}
1347 			cp->cache_lookup_depth++;
1348 			prev_bcpp = &bcp->bc_next;
1349 		}
1350 	} else {
1351 		bcp = UMEM_BUFCTL(cp, buf);
1352 		sp = UMEM_SLAB(cp, buf);
1353 	}
1354 
1355 	if (bcp == NULL || sp->slab_cache != cp || !UMEM_SLAB_MEMBER(sp, buf)) {
1356 		(void) mutex_unlock(&cp->cache_lock);
1357 		umem_error(UMERR_BADADDR, cp, buf);
1358 		return;
1359 	}
1360 
1361 	if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) {
1362 		if (cp->cache_flags & UMF_CONTENTS)
1363 			((umem_bufctl_audit_t *)bcp)->bc_contents =
1364 			    umem_log_enter(umem_content_log, buf,
1365 			    cp->cache_contents);
1366 		UMEM_AUDIT(umem_transaction_log, cp, bcp);
1367 	}
1368 
1369 	/*
1370 	 * If this slab isn't currently on the freelist, put it there.
1371 	 */
1372 	if (sp->slab_head == NULL) {
1373 		ASSERT(sp->slab_refcnt == sp->slab_chunks);
1374 		ASSERT(cp->cache_freelist != sp);
1375 		sp->slab_next->slab_prev = sp->slab_prev;
1376 		sp->slab_prev->slab_next = sp->slab_next;
1377 		sp->slab_next = cp->cache_freelist;
1378 		sp->slab_prev = cp->cache_freelist->slab_prev;
1379 		sp->slab_next->slab_prev = sp;
1380 		sp->slab_prev->slab_next = sp;
1381 		cp->cache_freelist = sp;
1382 	}
1383 
1384 	bcp->bc_next = sp->slab_head;
1385 	sp->slab_head = bcp;
1386 
1387 	ASSERT(sp->slab_refcnt >= 1);
1388 	if (--sp->slab_refcnt == 0) {
1389 		/*
1390 		 * There are no outstanding allocations from this slab,
1391 		 * so we can reclaim the memory.
1392 		 */
1393 		sp->slab_next->slab_prev = sp->slab_prev;
1394 		sp->slab_prev->slab_next = sp->slab_next;
1395 		if (sp == cp->cache_freelist)
1396 			cp->cache_freelist = sp->slab_next;
1397 		cp->cache_slab_destroy++;
1398 		cp->cache_buftotal -= sp->slab_chunks;
1399 		(void) mutex_unlock(&cp->cache_lock);
1400 		umem_slab_destroy(cp, sp);
1401 		return;
1402 	}
1403 	(void) mutex_unlock(&cp->cache_lock);
1404 }
1405 
1406 static int
1407 umem_cache_alloc_debug(umem_cache_t *cp, void *buf, int umflag)
1408 {
1409 	umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
1410 	umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl;
1411 	uint32_t mtbf;
1412 	int flags_nfatal;
1413 
1414 	if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) {
1415 		umem_error(UMERR_BADBUFTAG, cp, buf);
1416 		return (-1);
1417 	}
1418 
1419 	btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_ALLOC;
1420 
1421 	if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) {
1422 		umem_error(UMERR_BADBUFCTL, cp, buf);
1423 		return (-1);
1424 	}
1425 
1426 	btp->bt_redzone = UMEM_REDZONE_PATTERN;
1427 
1428 	if (cp->cache_flags & UMF_DEADBEEF) {
1429 		if (verify_and_copy_pattern(UMEM_FREE_PATTERN,
1430 		    UMEM_UNINITIALIZED_PATTERN, buf, cp->cache_verify)) {
1431 			umem_error(UMERR_MODIFIED, cp, buf);
1432 			return (-1);
1433 		}
1434 	}
1435 
1436 	if ((mtbf = umem_mtbf | cp->cache_mtbf) != 0 &&
1437 	    gethrtime() % mtbf == 0 &&
1438 	    (umflag & (UMEM_FATAL_FLAGS)) == 0) {
1439 		umem_log_event(umem_failure_log, cp, NULL, NULL);
1440 	} else {
1441 		mtbf = 0;
1442 	}
1443 
1444 	/*
1445 	 * We do not pass fatal flags on to the constructor.  This prevents
1446 	 * leaking buffers in the event of a subordinate constructor failing.
1447 	 */
1448 	flags_nfatal = UMEM_DEFAULT;
1449 	if (mtbf || (cp->cache_constructor != NULL &&
1450 	    cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0)) {
1451 		atomic_add_64(&cp->cache_alloc_fail, 1);
1452 		btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE;
1453 		copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify);
1454 		umem_slab_free(cp, buf);
1455 		return (-1);
1456 	}
1457 
1458 	if (cp->cache_flags & UMF_AUDIT) {
1459 		UMEM_AUDIT(umem_transaction_log, cp, bcp);
1460 	}
1461 
1462 	return (0);
1463 }
1464 
1465 static int
1466 umem_cache_free_debug(umem_cache_t *cp, void *buf)
1467 {
1468 	umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
1469 	umem_bufctl_audit_t *bcp = (umem_bufctl_audit_t *)btp->bt_bufctl;
1470 	umem_slab_t *sp;
1471 
1472 	if (btp->bt_bxstat != ((intptr_t)bcp ^ UMEM_BUFTAG_ALLOC)) {
1473 		if (btp->bt_bxstat == ((intptr_t)bcp ^ UMEM_BUFTAG_FREE)) {
1474 			umem_error(UMERR_DUPFREE, cp, buf);
1475 			return (-1);
1476 		}
1477 		sp = umem_findslab(cp, buf);
1478 		if (sp == NULL || sp->slab_cache != cp)
1479 			umem_error(UMERR_BADADDR, cp, buf);
1480 		else
1481 			umem_error(UMERR_REDZONE, cp, buf);
1482 		return (-1);
1483 	}
1484 
1485 	btp->bt_bxstat = (intptr_t)bcp ^ UMEM_BUFTAG_FREE;
1486 
1487 	if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) {
1488 		umem_error(UMERR_BADBUFCTL, cp, buf);
1489 		return (-1);
1490 	}
1491 
1492 	if (btp->bt_redzone != UMEM_REDZONE_PATTERN) {
1493 		umem_error(UMERR_REDZONE, cp, buf);
1494 		return (-1);
1495 	}
1496 
1497 	if (cp->cache_flags & UMF_AUDIT) {
1498 		if (cp->cache_flags & UMF_CONTENTS)
1499 			bcp->bc_contents = umem_log_enter(umem_content_log,
1500 			    buf, cp->cache_contents);
1501 		UMEM_AUDIT(umem_transaction_log, cp, bcp);
1502 	}
1503 
1504 	if (cp->cache_destructor != NULL)
1505 		cp->cache_destructor(buf, cp->cache_private);
1506 
1507 	if (cp->cache_flags & UMF_DEADBEEF)
1508 		copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify);
1509 
1510 	return (0);
1511 }
1512 
1513 /*
1514  * Free each object in magazine mp to cp's slab layer, and free mp itself.
1515  */
1516 static void
1517 umem_magazine_destroy(umem_cache_t *cp, umem_magazine_t *mp, int nrounds)
1518 {
1519 	int round;
1520 
1521 	ASSERT(cp->cache_next == NULL || IN_UPDATE());
1522 
1523 	for (round = 0; round < nrounds; round++) {
1524 		void *buf = mp->mag_round[round];
1525 
1526 		if ((cp->cache_flags & UMF_DEADBEEF) &&
1527 		    verify_pattern(UMEM_FREE_PATTERN, buf,
1528 		    cp->cache_verify) != NULL) {
1529 			umem_error(UMERR_MODIFIED, cp, buf);
1530 			continue;
1531 		}
1532 
1533 		if (!(cp->cache_flags & UMF_BUFTAG) &&
1534 		    cp->cache_destructor != NULL)
1535 			cp->cache_destructor(buf, cp->cache_private);
1536 
1537 		umem_slab_free(cp, buf);
1538 	}
1539 	ASSERT(UMEM_MAGAZINE_VALID(cp, mp));
1540 	_umem_cache_free(cp->cache_magtype->mt_cache, mp);
1541 }
1542 
1543 /*
1544  * Allocate a magazine from the depot.
1545  */
1546 static umem_magazine_t *
1547 umem_depot_alloc(umem_cache_t *cp, umem_maglist_t *mlp)
1548 {
1549 	umem_magazine_t *mp;
1550 
1551 	/*
1552 	 * If we can't get the depot lock without contention,
1553 	 * update our contention count.  We use the depot
1554 	 * contention rate to determine whether we need to
1555 	 * increase the magazine size for better scalability.
1556 	 */
1557 	if (mutex_trylock(&cp->cache_depot_lock) != 0) {
1558 		(void) mutex_lock(&cp->cache_depot_lock);
1559 		cp->cache_depot_contention++;
1560 	}
1561 
1562 	if ((mp = mlp->ml_list) != NULL) {
1563 		ASSERT(UMEM_MAGAZINE_VALID(cp, mp));
1564 		mlp->ml_list = mp->mag_next;
1565 		if (--mlp->ml_total < mlp->ml_min)
1566 			mlp->ml_min = mlp->ml_total;
1567 		mlp->ml_alloc++;
1568 	}
1569 
1570 	(void) mutex_unlock(&cp->cache_depot_lock);
1571 
1572 	return (mp);
1573 }
1574 
1575 /*
1576  * Free a magazine to the depot.
1577  */
1578 static void
1579 umem_depot_free(umem_cache_t *cp, umem_maglist_t *mlp, umem_magazine_t *mp)
1580 {
1581 	(void) mutex_lock(&cp->cache_depot_lock);
1582 	ASSERT(UMEM_MAGAZINE_VALID(cp, mp));
1583 	mp->mag_next = mlp->ml_list;
1584 	mlp->ml_list = mp;
1585 	mlp->ml_total++;
1586 	(void) mutex_unlock(&cp->cache_depot_lock);
1587 }
1588 
1589 /*
1590  * Update the working set statistics for cp's depot.
1591  */
1592 static void
1593 umem_depot_ws_update(umem_cache_t *cp)
1594 {
1595 	(void) mutex_lock(&cp->cache_depot_lock);
1596 	cp->cache_full.ml_reaplimit = cp->cache_full.ml_min;
1597 	cp->cache_full.ml_min = cp->cache_full.ml_total;
1598 	cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min;
1599 	cp->cache_empty.ml_min = cp->cache_empty.ml_total;
1600 	(void) mutex_unlock(&cp->cache_depot_lock);
1601 }
1602 
1603 /*
1604  * Reap all magazines that have fallen out of the depot's working set.
1605  */
1606 static void
1607 umem_depot_ws_reap(umem_cache_t *cp)
1608 {
1609 	long reap;
1610 	umem_magazine_t *mp;
1611 
1612 	ASSERT(cp->cache_next == NULL || IN_REAP());
1613 
1614 	reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
1615 	while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_full)) != NULL)
1616 		umem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize);
1617 
1618 	reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min);
1619 	while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_empty)) != NULL)
1620 		umem_magazine_destroy(cp, mp, 0);
1621 }
1622 
1623 static void
1624 umem_cpu_reload(umem_cpu_cache_t *ccp, umem_magazine_t *mp, int rounds)
1625 {
1626 	ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) ||
1627 	    (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize));
1628 	ASSERT(ccp->cc_magsize > 0);
1629 
1630 	ccp->cc_ploaded = ccp->cc_loaded;
1631 	ccp->cc_prounds = ccp->cc_rounds;
1632 	ccp->cc_loaded = mp;
1633 	ccp->cc_rounds = rounds;
1634 }
1635 
1636 /*
1637  * Allocate a constructed object from cache cp.
1638  */
1639 #pragma weak umem_cache_alloc = _umem_cache_alloc
1640 void *
1641 _umem_cache_alloc(umem_cache_t *cp, int umflag)
1642 {
1643 	umem_cpu_cache_t *ccp;
1644 	umem_magazine_t *fmp;
1645 	void *buf;
1646 	int flags_nfatal;
1647 
1648 retry:
1649 	ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask));
1650 	(void) mutex_lock(&ccp->cc_lock);
1651 	for (;;) {
1652 		/*
1653 		 * If there's an object available in the current CPU's
1654 		 * loaded magazine, just take it and return.
1655 		 */
1656 		if (ccp->cc_rounds > 0) {
1657 			buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds];
1658 			ccp->cc_alloc++;
1659 			(void) mutex_unlock(&ccp->cc_lock);
1660 			if ((ccp->cc_flags & UMF_BUFTAG) &&
1661 			    umem_cache_alloc_debug(cp, buf, umflag) == -1) {
1662 				if (umem_alloc_retry(cp, umflag)) {
1663 					goto retry;
1664 				}
1665 
1666 				return (NULL);
1667 			}
1668 			return (buf);
1669 		}
1670 
1671 		/*
1672 		 * The loaded magazine is empty.  If the previously loaded
1673 		 * magazine was full, exchange them and try again.
1674 		 */
1675 		if (ccp->cc_prounds > 0) {
1676 			umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
1677 			continue;
1678 		}
1679 
1680 		/*
1681 		 * If the magazine layer is disabled, break out now.
1682 		 */
1683 		if (ccp->cc_magsize == 0)
1684 			break;
1685 
1686 		/*
1687 		 * Try to get a full magazine from the depot.
1688 		 */
1689 		fmp = umem_depot_alloc(cp, &cp->cache_full);
1690 		if (fmp != NULL) {
1691 			if (ccp->cc_ploaded != NULL)
1692 				umem_depot_free(cp, &cp->cache_empty,
1693 				    ccp->cc_ploaded);
1694 			umem_cpu_reload(ccp, fmp, ccp->cc_magsize);
1695 			continue;
1696 		}
1697 
1698 		/*
1699 		 * There are no full magazines in the depot,
1700 		 * so fall through to the slab layer.
1701 		 */
1702 		break;
1703 	}
1704 	(void) mutex_unlock(&ccp->cc_lock);
1705 
1706 	/*
1707 	 * We couldn't allocate a constructed object from the magazine layer,
1708 	 * so get a raw buffer from the slab layer and apply its constructor.
1709 	 */
1710 	buf = umem_slab_alloc(cp, umflag);
1711 
1712 	if (buf == NULL) {
1713 		if (cp == &umem_null_cache)
1714 			return (NULL);
1715 		if (umem_alloc_retry(cp, umflag)) {
1716 			goto retry;
1717 		}
1718 
1719 		return (NULL);
1720 	}
1721 
1722 	if (cp->cache_flags & UMF_BUFTAG) {
1723 		/*
1724 		 * Let umem_cache_alloc_debug() apply the constructor for us.
1725 		 */
1726 		if (umem_cache_alloc_debug(cp, buf, umflag) == -1) {
1727 			if (umem_alloc_retry(cp, umflag)) {
1728 				goto retry;
1729 			}
1730 			return (NULL);
1731 		}
1732 		return (buf);
1733 	}
1734 
1735 	/*
1736 	 * We do not pass fatal flags on to the constructor.  This prevents
1737 	 * leaking buffers in the event of a subordinate constructor failing.
1738 	 */
1739 	flags_nfatal = UMEM_DEFAULT;
1740 	if (cp->cache_constructor != NULL &&
1741 	    cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0) {
1742 		atomic_add_64(&cp->cache_alloc_fail, 1);
1743 		umem_slab_free(cp, buf);
1744 
1745 		if (umem_alloc_retry(cp, umflag)) {
1746 			goto retry;
1747 		}
1748 		return (NULL);
1749 	}
1750 
1751 	return (buf);
1752 }
1753 
1754 /*
1755  * Free a constructed object to cache cp.
1756  */
1757 #pragma weak umem_cache_free = _umem_cache_free
1758 void
1759 _umem_cache_free(umem_cache_t *cp, void *buf)
1760 {
1761 	umem_cpu_cache_t *ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask));
1762 	umem_magazine_t *emp;
1763 	umem_magtype_t *mtp;
1764 
1765 	if (ccp->cc_flags & UMF_BUFTAG)
1766 		if (umem_cache_free_debug(cp, buf) == -1)
1767 			return;
1768 
1769 	(void) mutex_lock(&ccp->cc_lock);
1770 	for (;;) {
1771 		/*
1772 		 * If there's a slot available in the current CPU's
1773 		 * loaded magazine, just put the object there and return.
1774 		 */
1775 		if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) {
1776 			ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf;
1777 			ccp->cc_free++;
1778 			(void) mutex_unlock(&ccp->cc_lock);
1779 			return;
1780 		}
1781 
1782 		/*
1783 		 * The loaded magazine is full.  If the previously loaded
1784 		 * magazine was empty, exchange them and try again.
1785 		 */
1786 		if (ccp->cc_prounds == 0) {
1787 			umem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
1788 			continue;
1789 		}
1790 
1791 		/*
1792 		 * If the magazine layer is disabled, break out now.
1793 		 */
1794 		if (ccp->cc_magsize == 0)
1795 			break;
1796 
1797 		/*
1798 		 * Try to get an empty magazine from the depot.
1799 		 */
1800 		emp = umem_depot_alloc(cp, &cp->cache_empty);
1801 		if (emp != NULL) {
1802 			if (ccp->cc_ploaded != NULL)
1803 				umem_depot_free(cp, &cp->cache_full,
1804 				    ccp->cc_ploaded);
1805 			umem_cpu_reload(ccp, emp, 0);
1806 			continue;
1807 		}
1808 
1809 		/*
1810 		 * There are no empty magazines in the depot,
1811 		 * so try to allocate a new one.  We must drop all locks
1812 		 * across umem_cache_alloc() because lower layers may
1813 		 * attempt to allocate from this cache.
1814 		 */
1815 		mtp = cp->cache_magtype;
1816 		(void) mutex_unlock(&ccp->cc_lock);
1817 		emp = _umem_cache_alloc(mtp->mt_cache, UMEM_DEFAULT);
1818 		(void) mutex_lock(&ccp->cc_lock);
1819 
1820 		if (emp != NULL) {
1821 			/*
1822 			 * We successfully allocated an empty magazine.
1823 			 * However, we had to drop ccp->cc_lock to do it,
1824 			 * so the cache's magazine size may have changed.
1825 			 * If so, free the magazine and try again.
1826 			 */
1827 			if (ccp->cc_magsize != mtp->mt_magsize) {
1828 				(void) mutex_unlock(&ccp->cc_lock);
1829 				_umem_cache_free(mtp->mt_cache, emp);
1830 				(void) mutex_lock(&ccp->cc_lock);
1831 				continue;
1832 			}
1833 
1834 			/*
1835 			 * We got a magazine of the right size.  Add it to
1836 			 * the depot and try the whole dance again.
1837 			 */
1838 			umem_depot_free(cp, &cp->cache_empty, emp);
1839 			continue;
1840 		}
1841 
1842 		/*
1843 		 * We couldn't allocate an empty magazine,
1844 		 * so fall through to the slab layer.
1845 		 */
1846 		break;
1847 	}
1848 	(void) mutex_unlock(&ccp->cc_lock);
1849 
1850 	/*
1851 	 * We couldn't free our constructed object to the magazine layer,
1852 	 * so apply its destructor and free it to the slab layer.
1853 	 * Note that if UMF_BUFTAG is in effect, umem_cache_free_debug()
1854 	 * will have already applied the destructor.
1855 	 */
1856 	if (!(cp->cache_flags & UMF_BUFTAG) && cp->cache_destructor != NULL)
1857 		cp->cache_destructor(buf, cp->cache_private);
1858 
1859 	umem_slab_free(cp, buf);
1860 }
1861 
1862 #pragma weak umem_zalloc = _umem_zalloc
1863 void *
1864 _umem_zalloc(size_t size, int umflag)
1865 {
1866 	size_t index = (size - 1) >> UMEM_ALIGN_SHIFT;
1867 	void *buf;
1868 
1869 retry:
1870 	if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) {
1871 		umem_cache_t *cp = umem_alloc_table[index];
1872 		buf = _umem_cache_alloc(cp, umflag);
1873 		if (buf != NULL) {
1874 			if (cp->cache_flags & UMF_BUFTAG) {
1875 				umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
1876 				((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE;
1877 				((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size);
1878 			}
1879 			bzero(buf, size);
1880 		} else if (umem_alloc_retry(cp, umflag))
1881 			goto retry;
1882 	} else {
1883 		buf = _umem_alloc(size, umflag);	/* handles failure */
1884 		if (buf != NULL)
1885 			bzero(buf, size);
1886 	}
1887 	return (buf);
1888 }
1889 
1890 #pragma weak umem_alloc = _umem_alloc
1891 void *
1892 _umem_alloc(size_t size, int umflag)
1893 {
1894 	size_t index = (size - 1) >> UMEM_ALIGN_SHIFT;
1895 	void *buf;
1896 umem_alloc_retry:
1897 	if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) {
1898 		umem_cache_t *cp = umem_alloc_table[index];
1899 		buf = _umem_cache_alloc(cp, umflag);
1900 		if ((cp->cache_flags & UMF_BUFTAG) && buf != NULL) {
1901 			umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
1902 			((uint8_t *)buf)[size] = UMEM_REDZONE_BYTE;
1903 			((uint32_t *)btp)[1] = UMEM_SIZE_ENCODE(size);
1904 		}
1905 		if (buf == NULL && umem_alloc_retry(cp, umflag))
1906 			goto umem_alloc_retry;
1907 		return (buf);
1908 	}
1909 	if (size == 0)
1910 		return (NULL);
1911 	if (umem_oversize_arena == NULL) {
1912 		if (umem_init())
1913 			ASSERT(umem_oversize_arena != NULL);
1914 		else
1915 			return (NULL);
1916 	}
1917 	buf = vmem_alloc(umem_oversize_arena, size, UMEM_VMFLAGS(umflag));
1918 	if (buf == NULL) {
1919 		umem_log_event(umem_failure_log, NULL, NULL, (void *)size);
1920 		if (umem_alloc_retry(NULL, umflag))
1921 			goto umem_alloc_retry;
1922 	}
1923 	return (buf);
1924 }
1925 
1926 #pragma weak umem_alloc_align = _umem_alloc_align
1927 void *
1928 _umem_alloc_align(size_t size, size_t align, int umflag)
1929 {
1930 	void *buf;
1931 
1932 	if (size == 0)
1933 		return (NULL);
1934 	if ((align & (align - 1)) != 0)
1935 		return (NULL);
1936 	if (align < UMEM_ALIGN)
1937 		align = UMEM_ALIGN;
1938 
1939 umem_alloc_align_retry:
1940 	if (umem_memalign_arena == NULL) {
1941 		if (umem_init())
1942 			ASSERT(umem_oversize_arena != NULL);
1943 		else
1944 			return (NULL);
1945 	}
1946 	buf = vmem_xalloc(umem_memalign_arena, size, align, 0, 0, NULL, NULL,
1947 	    UMEM_VMFLAGS(umflag));
1948 	if (buf == NULL) {
1949 		umem_log_event(umem_failure_log, NULL, NULL, (void *)size);
1950 		if (umem_alloc_retry(NULL, umflag))
1951 			goto umem_alloc_align_retry;
1952 	}
1953 	return (buf);
1954 }
1955 
1956 #pragma weak umem_free = _umem_free
1957 void
1958 _umem_free(void *buf, size_t size)
1959 {
1960 	size_t index = (size - 1) >> UMEM_ALIGN_SHIFT;
1961 
1962 	if (index < UMEM_MAXBUF >> UMEM_ALIGN_SHIFT) {
1963 		umem_cache_t *cp = umem_alloc_table[index];
1964 		if (cp->cache_flags & UMF_BUFTAG) {
1965 			umem_buftag_t *btp = UMEM_BUFTAG(cp, buf);
1966 			uint32_t *ip = (uint32_t *)btp;
1967 			if (ip[1] != UMEM_SIZE_ENCODE(size)) {
1968 				if (*(uint64_t *)buf == UMEM_FREE_PATTERN) {
1969 					umem_error(UMERR_DUPFREE, cp, buf);
1970 					return;
1971 				}
1972 				if (UMEM_SIZE_VALID(ip[1])) {
1973 					ip[0] = UMEM_SIZE_ENCODE(size);
1974 					umem_error(UMERR_BADSIZE, cp, buf);
1975 				} else {
1976 					umem_error(UMERR_REDZONE, cp, buf);
1977 				}
1978 				return;
1979 			}
1980 			if (((uint8_t *)buf)[size] != UMEM_REDZONE_BYTE) {
1981 				umem_error(UMERR_REDZONE, cp, buf);
1982 				return;
1983 			}
1984 			btp->bt_redzone = UMEM_REDZONE_PATTERN;
1985 		}
1986 		_umem_cache_free(cp, buf);
1987 	} else {
1988 		if (buf == NULL && size == 0)
1989 			return;
1990 		vmem_free(umem_oversize_arena, buf, size);
1991 	}
1992 }
1993 
1994 #pragma weak umem_free_align = _umem_free_align
1995 void
1996 _umem_free_align(void *buf, size_t size)
1997 {
1998 	if (buf == NULL && size == 0)
1999 		return;
2000 	vmem_xfree(umem_memalign_arena, buf, size);
2001 }
2002 
2003 static void *
2004 umem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag)
2005 {
2006 	size_t realsize = size + vmp->vm_quantum;
2007 
2008 	/*
2009 	 * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding
2010 	 * vm_quantum will cause integer wraparound.  Check for this, and
2011 	 * blow off the firewall page in this case.  Note that such a
2012 	 * giant allocation (the entire address space) can never be
2013 	 * satisfied, so it will either fail immediately (VM_NOSLEEP)
2014 	 * or sleep forever (VM_SLEEP).  Thus, there is no need for a
2015 	 * corresponding check in umem_firewall_va_free().
2016 	 */
2017 	if (realsize < size)
2018 		realsize = size;
2019 
2020 	return (vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT));
2021 }
2022 
2023 static void
2024 umem_firewall_va_free(vmem_t *vmp, void *addr, size_t size)
2025 {
2026 	vmem_free(vmp, addr, size + vmp->vm_quantum);
2027 }
2028 
2029 /*
2030  * Reclaim all unused memory from a cache.
2031  */
2032 static void
2033 umem_cache_reap(umem_cache_t *cp)
2034 {
2035 	/*
2036 	 * Ask the cache's owner to free some memory if possible.
2037 	 * The idea is to handle things like the inode cache, which
2038 	 * typically sits on a bunch of memory that it doesn't truly
2039 	 * *need*.  Reclaim policy is entirely up to the owner; this
2040 	 * callback is just an advisory plea for help.
2041 	 */
2042 	if (cp->cache_reclaim != NULL)
2043 		cp->cache_reclaim(cp->cache_private);
2044 
2045 	umem_depot_ws_reap(cp);
2046 }
2047 
2048 /*
2049  * Purge all magazines from a cache and set its magazine limit to zero.
2050  * All calls are serialized by being done by the update thread, except for
2051  * the final call from umem_cache_destroy().
2052  */
2053 static void
2054 umem_cache_magazine_purge(umem_cache_t *cp)
2055 {
2056 	umem_cpu_cache_t *ccp;
2057 	umem_magazine_t *mp, *pmp;
2058 	int rounds, prounds, cpu_seqid;
2059 
2060 	ASSERT(cp->cache_next == NULL || IN_UPDATE());
2061 
2062 	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) {
2063 		ccp = &cp->cache_cpu[cpu_seqid];
2064 
2065 		(void) mutex_lock(&ccp->cc_lock);
2066 		mp = ccp->cc_loaded;
2067 		pmp = ccp->cc_ploaded;
2068 		rounds = ccp->cc_rounds;
2069 		prounds = ccp->cc_prounds;
2070 		ccp->cc_loaded = NULL;
2071 		ccp->cc_ploaded = NULL;
2072 		ccp->cc_rounds = -1;
2073 		ccp->cc_prounds = -1;
2074 		ccp->cc_magsize = 0;
2075 		(void) mutex_unlock(&ccp->cc_lock);
2076 
2077 		if (mp)
2078 			umem_magazine_destroy(cp, mp, rounds);
2079 		if (pmp)
2080 			umem_magazine_destroy(cp, pmp, prounds);
2081 	}
2082 
2083 	/*
2084 	 * Updating the working set statistics twice in a row has the
2085 	 * effect of setting the working set size to zero, so everything
2086 	 * is eligible for reaping.
2087 	 */
2088 	umem_depot_ws_update(cp);
2089 	umem_depot_ws_update(cp);
2090 
2091 	umem_depot_ws_reap(cp);
2092 }
2093 
2094 /*
2095  * Enable per-cpu magazines on a cache.
2096  */
2097 static void
2098 umem_cache_magazine_enable(umem_cache_t *cp)
2099 {
2100 	int cpu_seqid;
2101 
2102 	if (cp->cache_flags & UMF_NOMAGAZINE)
2103 		return;
2104 
2105 	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) {
2106 		umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
2107 		(void) mutex_lock(&ccp->cc_lock);
2108 		ccp->cc_magsize = cp->cache_magtype->mt_magsize;
2109 		(void) mutex_unlock(&ccp->cc_lock);
2110 	}
2111 
2112 }
2113 
2114 /*
2115  * Recompute a cache's magazine size.  The trade-off is that larger magazines
2116  * provide a higher transfer rate with the depot, while smaller magazines
2117  * reduce memory consumption.  Magazine resizing is an expensive operation;
2118  * it should not be done frequently.
2119  *
2120  * Changes to the magazine size are serialized by only having one thread
2121  * doing updates. (the update thread)
2122  *
2123  * Note: at present this only grows the magazine size.  It might be useful
2124  * to allow shrinkage too.
2125  */
2126 static void
2127 umem_cache_magazine_resize(umem_cache_t *cp)
2128 {
2129 	umem_magtype_t *mtp = cp->cache_magtype;
2130 
2131 	ASSERT(IN_UPDATE());
2132 
2133 	if (cp->cache_chunksize < mtp->mt_maxbuf) {
2134 		umem_cache_magazine_purge(cp);
2135 		(void) mutex_lock(&cp->cache_depot_lock);
2136 		cp->cache_magtype = ++mtp;
2137 		cp->cache_depot_contention_prev =
2138 		    cp->cache_depot_contention + INT_MAX;
2139 		(void) mutex_unlock(&cp->cache_depot_lock);
2140 		umem_cache_magazine_enable(cp);
2141 	}
2142 }
2143 
2144 /*
2145  * Rescale a cache's hash table, so that the table size is roughly the
2146  * cache size.  We want the average lookup time to be extremely small.
2147  */
2148 static void
2149 umem_hash_rescale(umem_cache_t *cp)
2150 {
2151 	umem_bufctl_t **old_table, **new_table, *bcp;
2152 	size_t old_size, new_size, h;
2153 
2154 	ASSERT(IN_UPDATE());
2155 
2156 	new_size = MAX(UMEM_HASH_INITIAL,
2157 	    1 << (highbit(3 * cp->cache_buftotal + 4) - 2));
2158 	old_size = cp->cache_hash_mask + 1;
2159 
2160 	if ((old_size >> 1) <= new_size && new_size <= (old_size << 1))
2161 		return;
2162 
2163 	new_table = vmem_alloc(umem_hash_arena, new_size * sizeof (void *),
2164 	    VM_NOSLEEP);
2165 	if (new_table == NULL)
2166 		return;
2167 	bzero(new_table, new_size * sizeof (void *));
2168 
2169 	(void) mutex_lock(&cp->cache_lock);
2170 
2171 	old_size = cp->cache_hash_mask + 1;
2172 	old_table = cp->cache_hash_table;
2173 
2174 	cp->cache_hash_mask = new_size - 1;
2175 	cp->cache_hash_table = new_table;
2176 	cp->cache_rescale++;
2177 
2178 	for (h = 0; h < old_size; h++) {
2179 		bcp = old_table[h];
2180 		while (bcp != NULL) {
2181 			void *addr = bcp->bc_addr;
2182 			umem_bufctl_t *next_bcp = bcp->bc_next;
2183 			umem_bufctl_t **hash_bucket = UMEM_HASH(cp, addr);
2184 			bcp->bc_next = *hash_bucket;
2185 			*hash_bucket = bcp;
2186 			bcp = next_bcp;
2187 		}
2188 	}
2189 
2190 	(void) mutex_unlock(&cp->cache_lock);
2191 
2192 	vmem_free(umem_hash_arena, old_table, old_size * sizeof (void *));
2193 }
2194 
2195 /*
2196  * Perform periodic maintenance on a cache: hash rescaling,
2197  * depot working-set update, and magazine resizing.
2198  */
2199 void
2200 umem_cache_update(umem_cache_t *cp)
2201 {
2202 	int update_flags = 0;
2203 
2204 	ASSERT(MUTEX_HELD(&umem_cache_lock));
2205 
2206 	/*
2207 	 * If the cache has become much larger or smaller than its hash table,
2208 	 * fire off a request to rescale the hash table.
2209 	 */
2210 	(void) mutex_lock(&cp->cache_lock);
2211 
2212 	if ((cp->cache_flags & UMF_HASH) &&
2213 	    (cp->cache_buftotal > (cp->cache_hash_mask << 1) ||
2214 	    (cp->cache_buftotal < (cp->cache_hash_mask >> 1) &&
2215 	    cp->cache_hash_mask > UMEM_HASH_INITIAL)))
2216 		update_flags |= UMU_HASH_RESCALE;
2217 
2218 	(void) mutex_unlock(&cp->cache_lock);
2219 
2220 	/*
2221 	 * Update the depot working set statistics.
2222 	 */
2223 	umem_depot_ws_update(cp);
2224 
2225 	/*
2226 	 * If there's a lot of contention in the depot,
2227 	 * increase the magazine size.
2228 	 */
2229 	(void) mutex_lock(&cp->cache_depot_lock);
2230 
2231 	if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf &&
2232 	    (int)(cp->cache_depot_contention -
2233 	    cp->cache_depot_contention_prev) > umem_depot_contention)
2234 		update_flags |= UMU_MAGAZINE_RESIZE;
2235 
2236 	cp->cache_depot_contention_prev = cp->cache_depot_contention;
2237 
2238 	(void) mutex_unlock(&cp->cache_depot_lock);
2239 
2240 	if (update_flags)
2241 		umem_add_update(cp, update_flags);
2242 }
2243 
2244 /*
2245  * Runs all pending updates.
2246  *
2247  * The update lock must be held on entrance, and will be held on exit.
2248  */
2249 void
2250 umem_process_updates(void)
2251 {
2252 	ASSERT(MUTEX_HELD(&umem_update_lock));
2253 
2254 	while (umem_null_cache.cache_unext != &umem_null_cache) {
2255 		int notify = 0;
2256 		umem_cache_t *cp = umem_null_cache.cache_unext;
2257 
2258 		cp->cache_uprev->cache_unext = cp->cache_unext;
2259 		cp->cache_unext->cache_uprev = cp->cache_uprev;
2260 		cp->cache_uprev = cp->cache_unext = NULL;
2261 
2262 		ASSERT(!(cp->cache_uflags & UMU_ACTIVE));
2263 
2264 		while (cp->cache_uflags) {
2265 			int uflags = (cp->cache_uflags |= UMU_ACTIVE);
2266 			(void) mutex_unlock(&umem_update_lock);
2267 
2268 			/*
2269 			 * The order here is important.  Each step can speed up
2270 			 * later steps.
2271 			 */
2272 
2273 			if (uflags & UMU_HASH_RESCALE)
2274 				umem_hash_rescale(cp);
2275 
2276 			if (uflags & UMU_MAGAZINE_RESIZE)
2277 				umem_cache_magazine_resize(cp);
2278 
2279 			if (uflags & UMU_REAP)
2280 				umem_cache_reap(cp);
2281 
2282 			(void) mutex_lock(&umem_update_lock);
2283 
2284 			/*
2285 			 * check if anyone has requested notification
2286 			 */
2287 			if (cp->cache_uflags & UMU_NOTIFY) {
2288 				uflags |= UMU_NOTIFY;
2289 				notify = 1;
2290 			}
2291 			cp->cache_uflags &= ~uflags;
2292 		}
2293 		if (notify)
2294 			(void) cond_broadcast(&umem_update_cv);
2295 	}
2296 }
2297 
2298 #ifndef UMEM_STANDALONE
2299 static void
2300 umem_st_update(void)
2301 {
2302 	ASSERT(MUTEX_HELD(&umem_update_lock));
2303 	ASSERT(umem_update_thr == 0 && umem_st_update_thr == 0);
2304 
2305 	umem_st_update_thr = thr_self();
2306 
2307 	(void) mutex_unlock(&umem_update_lock);
2308 
2309 	vmem_update(NULL);
2310 	umem_cache_applyall(umem_cache_update);
2311 
2312 	(void) mutex_lock(&umem_update_lock);
2313 
2314 	umem_process_updates();	/* does all of the requested work */
2315 
2316 	umem_reap_next = gethrtime() +
2317 	    (hrtime_t)umem_reap_interval * NANOSEC;
2318 
2319 	umem_reaping = UMEM_REAP_DONE;
2320 
2321 	umem_st_update_thr = 0;
2322 }
2323 #endif
2324 
2325 /*
2326  * Reclaim all unused memory from all caches.  Called from vmem when memory
2327  * gets tight.  Must be called with no locks held.
2328  *
2329  * This just requests a reap on all caches, and notifies the update thread.
2330  */
2331 void
2332 umem_reap(void)
2333 {
2334 #ifndef UMEM_STANDALONE
2335 	extern int __nthreads(void);
2336 #endif
2337 
2338 	if (umem_ready != UMEM_READY || umem_reaping != UMEM_REAP_DONE ||
2339 	    gethrtime() < umem_reap_next)
2340 		return;
2341 
2342 	(void) mutex_lock(&umem_update_lock);
2343 
2344 	if (umem_reaping != UMEM_REAP_DONE || gethrtime() < umem_reap_next) {
2345 		(void) mutex_unlock(&umem_update_lock);
2346 		return;
2347 	}
2348 	umem_reaping = UMEM_REAP_ADDING;	/* lock out other reaps */
2349 
2350 	(void) mutex_unlock(&umem_update_lock);
2351 
2352 	umem_updateall(UMU_REAP);
2353 
2354 	(void) mutex_lock(&umem_update_lock);
2355 
2356 	umem_reaping = UMEM_REAP_ACTIVE;
2357 
2358 	/* Standalone is single-threaded */
2359 #ifndef UMEM_STANDALONE
2360 	if (umem_update_thr == 0) {
2361 		/*
2362 		 * The update thread does not exist.  If the process is
2363 		 * multi-threaded, create it.  If not, or the creation fails,
2364 		 * do the update processing inline.
2365 		 */
2366 		ASSERT(umem_st_update_thr == 0);
2367 
2368 		if (__nthreads() <= 1 || umem_create_update_thread() == 0)
2369 			umem_st_update();
2370 	}
2371 
2372 	(void) cond_broadcast(&umem_update_cv);	/* wake up the update thread */
2373 #endif
2374 
2375 	(void) mutex_unlock(&umem_update_lock);
2376 }
2377 
2378 umem_cache_t *
2379 umem_cache_create(
2380 	char *name,		/* descriptive name for this cache */
2381 	size_t bufsize,		/* size of the objects it manages */
2382 	size_t align,		/* required object alignment */
2383 	umem_constructor_t *constructor, /* object constructor */
2384 	umem_destructor_t *destructor, /* object destructor */
2385 	umem_reclaim_t *reclaim, /* memory reclaim callback */
2386 	void *private,		/* pass-thru arg for constr/destr/reclaim */
2387 	vmem_t *vmp,		/* vmem source for slab allocation */
2388 	int cflags)		/* cache creation flags */
2389 {
2390 	int cpu_seqid;
2391 	size_t chunksize;
2392 	umem_cache_t *cp, *cnext, *cprev;
2393 	umem_magtype_t *mtp;
2394 	size_t csize;
2395 	size_t phase;
2396 
2397 	/*
2398 	 * The init thread is allowed to create internal and quantum caches.
2399 	 *
2400 	 * Other threads must wait until until initialization is complete.
2401 	 */
2402 	if (umem_init_thr == thr_self())
2403 		ASSERT((cflags & (UMC_INTERNAL | UMC_QCACHE)) != 0);
2404 	else {
2405 		ASSERT(!(cflags & UMC_INTERNAL));
2406 		if (umem_ready != UMEM_READY && umem_init() == 0) {
2407 			errno = EAGAIN;
2408 			return (NULL);
2409 		}
2410 	}
2411 
2412 	csize = UMEM_CACHE_SIZE(umem_max_ncpus);
2413 	phase = P2NPHASE(csize, UMEM_CPU_CACHE_SIZE);
2414 
2415 	if (vmp == NULL)
2416 		vmp = umem_default_arena;
2417 
2418 	ASSERT(P2PHASE(phase, UMEM_ALIGN) == 0);
2419 
2420 	/*
2421 	 * Check that the arguments are reasonable
2422 	 */
2423 	if ((align & (align - 1)) != 0 || align > vmp->vm_quantum ||
2424 	    ((cflags & UMC_NOHASH) && (cflags & UMC_NOTOUCH)) ||
2425 	    name == NULL || bufsize == 0) {
2426 		errno = EINVAL;
2427 		return (NULL);
2428 	}
2429 
2430 	/*
2431 	 * If align == 0, we set it to the minimum required alignment.
2432 	 *
2433 	 * If align < UMEM_ALIGN, we round it up to UMEM_ALIGN, unless
2434 	 * UMC_NOTOUCH was passed.
2435 	 */
2436 	if (align == 0) {
2437 		if (P2ROUNDUP(bufsize, UMEM_ALIGN) >= UMEM_SECOND_ALIGN)
2438 			align = UMEM_SECOND_ALIGN;
2439 		else
2440 			align = UMEM_ALIGN;
2441 	} else if (align < UMEM_ALIGN && (cflags & UMC_NOTOUCH) == 0)
2442 		align = UMEM_ALIGN;
2443 
2444 
2445 	/*
2446 	 * Get a umem_cache structure.  We arrange that cp->cache_cpu[]
2447 	 * is aligned on a UMEM_CPU_CACHE_SIZE boundary to prevent
2448 	 * false sharing of per-CPU data.
2449 	 */
2450 	cp = vmem_xalloc(umem_cache_arena, csize, UMEM_CPU_CACHE_SIZE, phase,
2451 	    0, NULL, NULL, VM_NOSLEEP);
2452 
2453 	if (cp == NULL) {
2454 		errno = EAGAIN;
2455 		return (NULL);
2456 	}
2457 
2458 	bzero(cp, csize);
2459 
2460 	(void) mutex_lock(&umem_flags_lock);
2461 	if (umem_flags & UMF_RANDOMIZE)
2462 		umem_flags = (((umem_flags | ~UMF_RANDOM) + 1) & UMF_RANDOM) |
2463 		    UMF_RANDOMIZE;
2464 	cp->cache_flags = umem_flags | (cflags & UMF_DEBUG);
2465 	(void) mutex_unlock(&umem_flags_lock);
2466 
2467 	/*
2468 	 * Make sure all the various flags are reasonable.
2469 	 */
2470 	if (cp->cache_flags & UMF_LITE) {
2471 		if (bufsize >= umem_lite_minsize &&
2472 		    align <= umem_lite_maxalign &&
2473 		    P2PHASE(bufsize, umem_lite_maxalign) != 0) {
2474 			cp->cache_flags |= UMF_BUFTAG;
2475 			cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL);
2476 		} else {
2477 			cp->cache_flags &= ~UMF_DEBUG;
2478 		}
2479 	}
2480 
2481 	if ((cflags & UMC_QCACHE) && (cp->cache_flags & UMF_AUDIT))
2482 		cp->cache_flags |= UMF_NOMAGAZINE;
2483 
2484 	if (cflags & UMC_NODEBUG)
2485 		cp->cache_flags &= ~UMF_DEBUG;
2486 
2487 	if (cflags & UMC_NOTOUCH)
2488 		cp->cache_flags &= ~UMF_TOUCH;
2489 
2490 	if (cflags & UMC_NOHASH)
2491 		cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL);
2492 
2493 	if (cflags & UMC_NOMAGAZINE)
2494 		cp->cache_flags |= UMF_NOMAGAZINE;
2495 
2496 	if ((cp->cache_flags & UMF_AUDIT) && !(cflags & UMC_NOTOUCH))
2497 		cp->cache_flags |= UMF_REDZONE;
2498 
2499 	if ((cp->cache_flags & UMF_BUFTAG) && bufsize >= umem_minfirewall &&
2500 	    !(cp->cache_flags & UMF_LITE) && !(cflags & UMC_NOHASH))
2501 		cp->cache_flags |= UMF_FIREWALL;
2502 
2503 	if (vmp != umem_default_arena || umem_firewall_arena == NULL)
2504 		cp->cache_flags &= ~UMF_FIREWALL;
2505 
2506 	if (cp->cache_flags & UMF_FIREWALL) {
2507 		cp->cache_flags &= ~UMF_BUFTAG;
2508 		cp->cache_flags |= UMF_NOMAGAZINE;
2509 		ASSERT(vmp == umem_default_arena);
2510 		vmp = umem_firewall_arena;
2511 	}
2512 
2513 	/*
2514 	 * Set cache properties.
2515 	 */
2516 	(void) strncpy(cp->cache_name, name, sizeof (cp->cache_name) - 1);
2517 	cp->cache_bufsize = bufsize;
2518 	cp->cache_align = align;
2519 	cp->cache_constructor = constructor;
2520 	cp->cache_destructor = destructor;
2521 	cp->cache_reclaim = reclaim;
2522 	cp->cache_private = private;
2523 	cp->cache_arena = vmp;
2524 	cp->cache_cflags = cflags;
2525 	cp->cache_cpu_mask = umem_cpu_mask;
2526 
2527 	/*
2528 	 * Determine the chunk size.
2529 	 */
2530 	chunksize = bufsize;
2531 
2532 	if (align >= UMEM_ALIGN) {
2533 		chunksize = P2ROUNDUP(chunksize, UMEM_ALIGN);
2534 		cp->cache_bufctl = chunksize - UMEM_ALIGN;
2535 	}
2536 
2537 	if (cp->cache_flags & UMF_BUFTAG) {
2538 		cp->cache_bufctl = chunksize;
2539 		cp->cache_buftag = chunksize;
2540 		chunksize += sizeof (umem_buftag_t);
2541 	}
2542 
2543 	if (cp->cache_flags & UMF_DEADBEEF) {
2544 		cp->cache_verify = MIN(cp->cache_buftag, umem_maxverify);
2545 		if (cp->cache_flags & UMF_LITE)
2546 			cp->cache_verify = MIN(cp->cache_verify, UMEM_ALIGN);
2547 	}
2548 
2549 	cp->cache_contents = MIN(cp->cache_bufctl, umem_content_maxsave);
2550 
2551 	cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align);
2552 
2553 	if (chunksize < bufsize) {
2554 		errno = ENOMEM;
2555 		goto fail;
2556 	}
2557 
2558 	/*
2559 	 * Now that we know the chunk size, determine the optimal slab size.
2560 	 */
2561 	if (vmp == umem_firewall_arena) {
2562 		cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum);
2563 		cp->cache_mincolor = cp->cache_slabsize - chunksize;
2564 		cp->cache_maxcolor = cp->cache_mincolor;
2565 		cp->cache_flags |= UMF_HASH;
2566 		ASSERT(!(cp->cache_flags & UMF_BUFTAG));
2567 	} else if ((cflags & UMC_NOHASH) || (!(cflags & UMC_NOTOUCH) &&
2568 	    !(cp->cache_flags & UMF_AUDIT) &&
2569 	    chunksize < vmp->vm_quantum / UMEM_VOID_FRACTION)) {
2570 		cp->cache_slabsize = vmp->vm_quantum;
2571 		cp->cache_mincolor = 0;
2572 		cp->cache_maxcolor =
2573 		    (cp->cache_slabsize - sizeof (umem_slab_t)) % chunksize;
2574 
2575 		if (chunksize + sizeof (umem_slab_t) > cp->cache_slabsize) {
2576 			errno = EINVAL;
2577 			goto fail;
2578 		}
2579 		ASSERT(!(cp->cache_flags & UMF_AUDIT));
2580 	} else {
2581 		size_t chunks, bestfit, waste, slabsize;
2582 		size_t minwaste = LONG_MAX;
2583 
2584 		for (chunks = 1; chunks <= UMEM_VOID_FRACTION; chunks++) {
2585 			slabsize = P2ROUNDUP(chunksize * chunks,
2586 			    vmp->vm_quantum);
2587 			/*
2588 			 * check for overflow
2589 			 */
2590 			if ((slabsize / chunks) < chunksize) {
2591 				errno = ENOMEM;
2592 				goto fail;
2593 			}
2594 			chunks = slabsize / chunksize;
2595 			waste = (slabsize % chunksize) / chunks;
2596 			if (waste < minwaste) {
2597 				minwaste = waste;
2598 				bestfit = slabsize;
2599 			}
2600 		}
2601 		if (cflags & UMC_QCACHE)
2602 			bestfit = MAX(1 << highbit(3 * vmp->vm_qcache_max), 64);
2603 		cp->cache_slabsize = bestfit;
2604 		cp->cache_mincolor = 0;
2605 		cp->cache_maxcolor = bestfit % chunksize;
2606 		cp->cache_flags |= UMF_HASH;
2607 	}
2608 
2609 	if (cp->cache_flags & UMF_HASH) {
2610 		ASSERT(!(cflags & UMC_NOHASH));
2611 		cp->cache_bufctl_cache = (cp->cache_flags & UMF_AUDIT) ?
2612 		    umem_bufctl_audit_cache : umem_bufctl_cache;
2613 	}
2614 
2615 	if (cp->cache_maxcolor >= vmp->vm_quantum)
2616 		cp->cache_maxcolor = vmp->vm_quantum - 1;
2617 
2618 	cp->cache_color = cp->cache_mincolor;
2619 
2620 	/*
2621 	 * Initialize the rest of the slab layer.
2622 	 */
2623 	(void) mutex_init(&cp->cache_lock, USYNC_THREAD, NULL);
2624 
2625 	cp->cache_freelist = &cp->cache_nullslab;
2626 	cp->cache_nullslab.slab_cache = cp;
2627 	cp->cache_nullslab.slab_refcnt = -1;
2628 	cp->cache_nullslab.slab_next = &cp->cache_nullslab;
2629 	cp->cache_nullslab.slab_prev = &cp->cache_nullslab;
2630 
2631 	if (cp->cache_flags & UMF_HASH) {
2632 		cp->cache_hash_table = vmem_alloc(umem_hash_arena,
2633 		    UMEM_HASH_INITIAL * sizeof (void *), VM_NOSLEEP);
2634 		if (cp->cache_hash_table == NULL) {
2635 			errno = EAGAIN;
2636 			goto fail_lock;
2637 		}
2638 		bzero(cp->cache_hash_table,
2639 		    UMEM_HASH_INITIAL * sizeof (void *));
2640 		cp->cache_hash_mask = UMEM_HASH_INITIAL - 1;
2641 		cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1;
2642 	}
2643 
2644 	/*
2645 	 * Initialize the depot.
2646 	 */
2647 	(void) mutex_init(&cp->cache_depot_lock, USYNC_THREAD, NULL);
2648 
2649 	for (mtp = umem_magtype; chunksize <= mtp->mt_minbuf; mtp++)
2650 		continue;
2651 
2652 	cp->cache_magtype = mtp;
2653 
2654 	/*
2655 	 * Initialize the CPU layer.
2656 	 */
2657 	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++) {
2658 		umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
2659 		(void) mutex_init(&ccp->cc_lock, USYNC_THREAD, NULL);
2660 		ccp->cc_flags = cp->cache_flags;
2661 		ccp->cc_rounds = -1;
2662 		ccp->cc_prounds = -1;
2663 	}
2664 
2665 	/*
2666 	 * Add the cache to the global list.  This makes it visible
2667 	 * to umem_update(), so the cache must be ready for business.
2668 	 */
2669 	(void) mutex_lock(&umem_cache_lock);
2670 	cp->cache_next = cnext = &umem_null_cache;
2671 	cp->cache_prev = cprev = umem_null_cache.cache_prev;
2672 	cnext->cache_prev = cp;
2673 	cprev->cache_next = cp;
2674 	(void) mutex_unlock(&umem_cache_lock);
2675 
2676 	if (umem_ready == UMEM_READY)
2677 		umem_cache_magazine_enable(cp);
2678 
2679 	return (cp);
2680 
2681 fail_lock:
2682 	(void) mutex_destroy(&cp->cache_lock);
2683 fail:
2684 	vmem_xfree(umem_cache_arena, cp, csize);
2685 	return (NULL);
2686 }
2687 
2688 void
2689 umem_cache_destroy(umem_cache_t *cp)
2690 {
2691 	int cpu_seqid;
2692 
2693 	/*
2694 	 * Remove the cache from the global cache list so that no new updates
2695 	 * will be scheduled on its behalf, wait for any pending tasks to
2696 	 * complete, purge the cache, and then destroy it.
2697 	 */
2698 	(void) mutex_lock(&umem_cache_lock);
2699 	cp->cache_prev->cache_next = cp->cache_next;
2700 	cp->cache_next->cache_prev = cp->cache_prev;
2701 	cp->cache_prev = cp->cache_next = NULL;
2702 	(void) mutex_unlock(&umem_cache_lock);
2703 
2704 	umem_remove_updates(cp);
2705 
2706 	umem_cache_magazine_purge(cp);
2707 
2708 	(void) mutex_lock(&cp->cache_lock);
2709 	if (cp->cache_buftotal != 0)
2710 		log_message("umem_cache_destroy: '%s' (%p) not empty\n",
2711 		    cp->cache_name, (void *)cp);
2712 	cp->cache_reclaim = NULL;
2713 	/*
2714 	 * The cache is now dead.  There should be no further activity.
2715 	 * We enforce this by setting land mines in the constructor and
2716 	 * destructor routines that induce a segmentation fault if invoked.
2717 	 */
2718 	cp->cache_constructor = (umem_constructor_t *)1;
2719 	cp->cache_destructor = (umem_destructor_t *)2;
2720 	(void) mutex_unlock(&cp->cache_lock);
2721 
2722 	if (cp->cache_hash_table != NULL)
2723 		vmem_free(umem_hash_arena, cp->cache_hash_table,
2724 		    (cp->cache_hash_mask + 1) * sizeof (void *));
2725 
2726 	for (cpu_seqid = 0; cpu_seqid < umem_max_ncpus; cpu_seqid++)
2727 		(void) mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock);
2728 
2729 	(void) mutex_destroy(&cp->cache_depot_lock);
2730 	(void) mutex_destroy(&cp->cache_lock);
2731 
2732 	vmem_free(umem_cache_arena, cp, UMEM_CACHE_SIZE(umem_max_ncpus));
2733 }
2734 
2735 void
2736 umem_alloc_sizes_clear(void)
2737 {
2738 	int i;
2739 
2740 	umem_alloc_sizes[0] = UMEM_MAXBUF;
2741 	for (i = 1; i < NUM_ALLOC_SIZES; i++)
2742 		umem_alloc_sizes[i] = 0;
2743 }
2744 
2745 void
2746 umem_alloc_sizes_add(size_t size_arg)
2747 {
2748 	int i, j;
2749 	size_t size = size_arg;
2750 
2751 	if (size == 0) {
2752 		log_message("size_add: cannot add zero-sized cache\n",
2753 		    size, UMEM_MAXBUF);
2754 		return;
2755 	}
2756 
2757 	if (size > UMEM_MAXBUF) {
2758 		log_message("size_add: %ld > %d, cannot add\n", size,
2759 		    UMEM_MAXBUF);
2760 		return;
2761 	}
2762 
2763 	if (umem_alloc_sizes[NUM_ALLOC_SIZES - 1] != 0) {
2764 		log_message("size_add: no space in alloc_table for %d\n",
2765 		    size);
2766 		return;
2767 	}
2768 
2769 	if (P2PHASE(size, UMEM_ALIGN) != 0) {
2770 		size = P2ROUNDUP(size, UMEM_ALIGN);
2771 		log_message("size_add: rounding %d up to %d\n", size_arg,
2772 		    size);
2773 	}
2774 
2775 	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
2776 		int cur = umem_alloc_sizes[i];
2777 		if (cur == size) {
2778 			log_message("size_add: %ld already in table\n",
2779 			    size);
2780 			return;
2781 		}
2782 		if (cur > size)
2783 			break;
2784 	}
2785 
2786 	for (j = NUM_ALLOC_SIZES - 1; j > i; j--)
2787 		umem_alloc_sizes[j] = umem_alloc_sizes[j-1];
2788 	umem_alloc_sizes[i] = size;
2789 }
2790 
2791 void
2792 umem_alloc_sizes_remove(size_t size)
2793 {
2794 	int i;
2795 
2796 	if (size == UMEM_MAXBUF) {
2797 		log_message("size_remove: cannot remove %ld\n", size);
2798 		return;
2799 	}
2800 
2801 	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
2802 		int cur = umem_alloc_sizes[i];
2803 		if (cur == size)
2804 			break;
2805 		else if (cur > size || cur == 0) {
2806 			log_message("size_remove: %ld not found in table\n",
2807 			    size);
2808 			return;
2809 		}
2810 	}
2811 
2812 	for (; i + 1 < NUM_ALLOC_SIZES; i++)
2813 		umem_alloc_sizes[i] = umem_alloc_sizes[i+1];
2814 	umem_alloc_sizes[i] = 0;
2815 }
2816 
2817 static int
2818 umem_cache_init(void)
2819 {
2820 	int i;
2821 	size_t size, max_size;
2822 	umem_cache_t *cp;
2823 	umem_magtype_t *mtp;
2824 	char name[UMEM_CACHE_NAMELEN + 1];
2825 	umem_cache_t *umem_alloc_caches[NUM_ALLOC_SIZES];
2826 
2827 	for (i = 0; i < sizeof (umem_magtype) / sizeof (*mtp); i++) {
2828 		mtp = &umem_magtype[i];
2829 		(void) snprintf(name, sizeof (name), "umem_magazine_%d",
2830 		    mtp->mt_magsize);
2831 		mtp->mt_cache = umem_cache_create(name,
2832 		    (mtp->mt_magsize + 1) * sizeof (void *),
2833 		    mtp->mt_align, NULL, NULL, NULL, NULL,
2834 		    umem_internal_arena, UMC_NOHASH | UMC_INTERNAL);
2835 		if (mtp->mt_cache == NULL)
2836 			return (0);
2837 	}
2838 
2839 	umem_slab_cache = umem_cache_create("umem_slab_cache",
2840 	    sizeof (umem_slab_t), 0, NULL, NULL, NULL, NULL,
2841 	    umem_internal_arena, UMC_NOHASH | UMC_INTERNAL);
2842 
2843 	if (umem_slab_cache == NULL)
2844 		return (0);
2845 
2846 	umem_bufctl_cache = umem_cache_create("umem_bufctl_cache",
2847 	    sizeof (umem_bufctl_t), 0, NULL, NULL, NULL, NULL,
2848 	    umem_internal_arena, UMC_NOHASH | UMC_INTERNAL);
2849 
2850 	if (umem_bufctl_cache == NULL)
2851 		return (0);
2852 
2853 	/*
2854 	 * The size of the umem_bufctl_audit structure depends upon
2855 	 * umem_stack_depth.   See umem_impl.h for details on the size
2856 	 * restrictions.
2857 	 */
2858 
2859 	size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth);
2860 	max_size = UMEM_BUFCTL_AUDIT_MAX_SIZE;
2861 
2862 	if (size > max_size) {			/* too large -- truncate */
2863 		int max_frames = UMEM_MAX_STACK_DEPTH;
2864 
2865 		ASSERT(UMEM_BUFCTL_AUDIT_SIZE_DEPTH(max_frames) <= max_size);
2866 
2867 		umem_stack_depth = max_frames;
2868 		size = UMEM_BUFCTL_AUDIT_SIZE_DEPTH(umem_stack_depth);
2869 	}
2870 
2871 	umem_bufctl_audit_cache = umem_cache_create("umem_bufctl_audit_cache",
2872 	    size, 0, NULL, NULL, NULL, NULL, umem_internal_arena,
2873 	    UMC_NOHASH | UMC_INTERNAL);
2874 
2875 	if (umem_bufctl_audit_cache == NULL)
2876 		return (0);
2877 
2878 	if (vmem_backend & VMEM_BACKEND_MMAP)
2879 		umem_va_arena = vmem_create("umem_va",
2880 		    NULL, 0, pagesize,
2881 		    vmem_alloc, vmem_free, heap_arena,
2882 		    8 * pagesize, VM_NOSLEEP);
2883 	else
2884 		umem_va_arena = heap_arena;
2885 
2886 	if (umem_va_arena == NULL)
2887 		return (0);
2888 
2889 	umem_default_arena = vmem_create("umem_default",
2890 	    NULL, 0, pagesize,
2891 	    heap_alloc, heap_free, umem_va_arena,
2892 	    0, VM_NOSLEEP);
2893 
2894 	if (umem_default_arena == NULL)
2895 		return (0);
2896 
2897 	/*
2898 	 * make sure the umem_alloc table initializer is correct
2899 	 */
2900 	i = sizeof (umem_alloc_table) / sizeof (*umem_alloc_table);
2901 	ASSERT(umem_alloc_table[i - 1] == &umem_null_cache);
2902 
2903 	/*
2904 	 * Create the default caches to back umem_alloc()
2905 	 */
2906 	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
2907 		size_t cache_size = umem_alloc_sizes[i];
2908 		size_t align = 0;
2909 
2910 		if (cache_size == 0)
2911 			break;		/* 0 terminates the list */
2912 
2913 		/*
2914 		 * If they allocate a multiple of the coherency granularity,
2915 		 * they get a coherency-granularity-aligned address.
2916 		 */
2917 		if (IS_P2ALIGNED(cache_size, 64))
2918 			align = 64;
2919 		if (IS_P2ALIGNED(cache_size, pagesize))
2920 			align = pagesize;
2921 		(void) snprintf(name, sizeof (name), "umem_alloc_%lu",
2922 		    (long)cache_size);
2923 
2924 		cp = umem_cache_create(name, cache_size, align,
2925 		    NULL, NULL, NULL, NULL, NULL, UMC_INTERNAL);
2926 		if (cp == NULL)
2927 			return (0);
2928 
2929 		umem_alloc_caches[i] = cp;
2930 	}
2931 
2932 	/*
2933 	 * Initialization cannot fail at this point.  Make the caches
2934 	 * visible to umem_alloc() and friends.
2935 	 */
2936 	size = UMEM_ALIGN;
2937 	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
2938 		size_t cache_size = umem_alloc_sizes[i];
2939 
2940 		if (cache_size == 0)
2941 			break;		/* 0 terminates the list */
2942 
2943 		cp = umem_alloc_caches[i];
2944 
2945 		while (size <= cache_size) {
2946 			umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT] = cp;
2947 			size += UMEM_ALIGN;
2948 		}
2949 	}
2950 	ASSERT(size - UMEM_ALIGN == UMEM_MAXBUF);
2951 	return (1);
2952 }
2953 
2954 /*
2955  * umem_startup() is called early on, and must be called explicitly if we're
2956  * the standalone version.
2957  */
2958 #ifdef UMEM_STANDALONE
2959 void
2960 #else
2961 #pragma init(umem_startup)
2962 static void
2963 #endif
2964 umem_startup(caddr_t start, size_t len, size_t pagesize, caddr_t minstack,
2965     caddr_t maxstack)
2966 {
2967 #ifdef UMEM_STANDALONE
2968 	int idx;
2969 	/* Standalone doesn't fork */
2970 #else
2971 	umem_forkhandler_init(); /* register the fork handler */
2972 #endif
2973 
2974 #ifdef __lint
2975 	/* make lint happy */
2976 	minstack = maxstack;
2977 #endif
2978 
2979 #ifdef UMEM_STANDALONE
2980 	umem_ready = UMEM_READY_STARTUP;
2981 	umem_init_env_ready = 0;
2982 
2983 	umem_min_stack = minstack;
2984 	umem_max_stack = maxstack;
2985 
2986 	nofail_callback = NULL;
2987 	umem_slab_cache = NULL;
2988 	umem_bufctl_cache = NULL;
2989 	umem_bufctl_audit_cache = NULL;
2990 	heap_arena = NULL;
2991 	heap_alloc = NULL;
2992 	heap_free = NULL;
2993 	umem_internal_arena = NULL;
2994 	umem_cache_arena = NULL;
2995 	umem_hash_arena = NULL;
2996 	umem_log_arena = NULL;
2997 	umem_oversize_arena = NULL;
2998 	umem_va_arena = NULL;
2999 	umem_default_arena = NULL;
3000 	umem_firewall_va_arena = NULL;
3001 	umem_firewall_arena = NULL;
3002 	umem_memalign_arena = NULL;
3003 	umem_transaction_log = NULL;
3004 	umem_content_log = NULL;
3005 	umem_failure_log = NULL;
3006 	umem_slab_log = NULL;
3007 	umem_cpu_mask = 0;
3008 
3009 	umem_cpus = &umem_startup_cpu;
3010 	umem_startup_cpu.cpu_cache_offset = UMEM_CACHE_SIZE(0);
3011 	umem_startup_cpu.cpu_number = 0;
3012 
3013 	bcopy(&umem_null_cache_template, &umem_null_cache,
3014 	    sizeof (umem_cache_t));
3015 
3016 	for (idx = 0; idx < (UMEM_MAXBUF >> UMEM_ALIGN_SHIFT); idx++)
3017 		umem_alloc_table[idx] = &umem_null_cache;
3018 #endif
3019 
3020 	/*
3021 	 * Perform initialization specific to the way we've been compiled
3022 	 * (library or standalone)
3023 	 */
3024 	umem_type_init(start, len, pagesize);
3025 
3026 	vmem_startup();
3027 }
3028 
3029 int
3030 umem_init(void)
3031 {
3032 	size_t maxverify, minfirewall;
3033 	size_t size;
3034 	int idx;
3035 	umem_cpu_t *new_cpus;
3036 
3037 	vmem_t *memalign_arena, *oversize_arena;
3038 
3039 	if (thr_self() != umem_init_thr) {
3040 		/*
3041 		 * The usual case -- non-recursive invocation of umem_init().
3042 		 */
3043 		(void) mutex_lock(&umem_init_lock);
3044 		if (umem_ready != UMEM_READY_STARTUP) {
3045 			/*
3046 			 * someone else beat us to initializing umem.  Wait
3047 			 * for them to complete, then return.
3048 			 */
3049 			while (umem_ready == UMEM_READY_INITING) {
3050 				int cancel_state;
3051 
3052 				(void) pthread_setcancelstate(
3053 				    PTHREAD_CANCEL_DISABLE, &cancel_state);
3054 				(void) cond_wait(&umem_init_cv,
3055 				    &umem_init_lock);
3056 				(void) pthread_setcancelstate(
3057 				    cancel_state, NULL);
3058 			}
3059 			ASSERT(umem_ready == UMEM_READY ||
3060 			    umem_ready == UMEM_READY_INIT_FAILED);
3061 			(void) mutex_unlock(&umem_init_lock);
3062 			return (umem_ready == UMEM_READY);
3063 		}
3064 
3065 		ASSERT(umem_ready == UMEM_READY_STARTUP);
3066 		ASSERT(umem_init_env_ready == 0);
3067 
3068 		umem_ready = UMEM_READY_INITING;
3069 		umem_init_thr = thr_self();
3070 
3071 		(void) mutex_unlock(&umem_init_lock);
3072 		umem_setup_envvars(0);		/* can recurse -- see below */
3073 		if (umem_init_env_ready) {
3074 			/*
3075 			 * initialization was completed already
3076 			 */
3077 			ASSERT(umem_ready == UMEM_READY ||
3078 			    umem_ready == UMEM_READY_INIT_FAILED);
3079 			ASSERT(umem_init_thr == 0);
3080 			return (umem_ready == UMEM_READY);
3081 		}
3082 	} else if (!umem_init_env_ready) {
3083 		/*
3084 		 * The umem_setup_envvars() call (above) makes calls into
3085 		 * the dynamic linker and directly into user-supplied code.
3086 		 * Since we cannot know what that code will do, we could be
3087 		 * recursively invoked (by, say, a malloc() call in the code
3088 		 * itself, or in a (C++) _init section it causes to be fired).
3089 		 *
3090 		 * This code is where we end up if such recursion occurs.  We
3091 		 * first clean up any partial results in the envvar code, then
3092 		 * proceed to finish initialization processing in the recursive
3093 		 * call.  The original call will notice this, and return
3094 		 * immediately.
3095 		 */
3096 		umem_setup_envvars(1);		/* clean up any partial state */
3097 	} else {
3098 		umem_panic(
3099 		    "recursive allocation while initializing umem\n");
3100 	}
3101 	umem_init_env_ready = 1;
3102 
3103 	/*
3104 	 * From this point until we finish, recursion into umem_init() will
3105 	 * cause a umem_panic().
3106 	 */
3107 	maxverify = minfirewall = ULONG_MAX;
3108 
3109 	/* LINTED constant condition */
3110 	if (sizeof (umem_cpu_cache_t) != UMEM_CPU_CACHE_SIZE) {
3111 		umem_panic("sizeof (umem_cpu_cache_t) = %d, should be %d\n",
3112 		    sizeof (umem_cpu_cache_t), UMEM_CPU_CACHE_SIZE);
3113 	}
3114 
3115 	umem_max_ncpus = umem_get_max_ncpus();
3116 
3117 	/*
3118 	 * load tunables from environment
3119 	 */
3120 	umem_process_envvars();
3121 
3122 	if (issetugid())
3123 		umem_mtbf = 0;
3124 
3125 	/*
3126 	 * set up vmem
3127 	 */
3128 	if (!(umem_flags & UMF_AUDIT))
3129 		vmem_no_debug();
3130 
3131 	heap_arena = vmem_heap_arena(&heap_alloc, &heap_free);
3132 
3133 	pagesize = heap_arena->vm_quantum;
3134 
3135 	umem_internal_arena = vmem_create("umem_internal", NULL, 0, pagesize,
3136 	    heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP);
3137 
3138 	umem_default_arena = umem_internal_arena;
3139 
3140 	if (umem_internal_arena == NULL)
3141 		goto fail;
3142 
3143 	umem_cache_arena = vmem_create("umem_cache", NULL, 0, UMEM_ALIGN,
3144 	    vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP);
3145 
3146 	umem_hash_arena = vmem_create("umem_hash", NULL, 0, UMEM_ALIGN,
3147 	    vmem_alloc, vmem_free, umem_internal_arena, 0, VM_NOSLEEP);
3148 
3149 	umem_log_arena = vmem_create("umem_log", NULL, 0, UMEM_ALIGN,
3150 	    heap_alloc, heap_free, heap_arena, 0, VM_NOSLEEP);
3151 
3152 	umem_firewall_va_arena = vmem_create("umem_firewall_va",
3153 	    NULL, 0, pagesize,
3154 	    umem_firewall_va_alloc, umem_firewall_va_free, heap_arena,
3155 	    0, VM_NOSLEEP);
3156 
3157 	if (umem_cache_arena == NULL || umem_hash_arena == NULL ||
3158 	    umem_log_arena == NULL || umem_firewall_va_arena == NULL)
3159 		goto fail;
3160 
3161 	umem_firewall_arena = vmem_create("umem_firewall", NULL, 0, pagesize,
3162 	    heap_alloc, heap_free, umem_firewall_va_arena, 0,
3163 	    VM_NOSLEEP);
3164 
3165 	if (umem_firewall_arena == NULL)
3166 		goto fail;
3167 
3168 	oversize_arena = vmem_create("umem_oversize", NULL, 0, pagesize,
3169 	    heap_alloc, heap_free, minfirewall < ULONG_MAX ?
3170 	    umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP);
3171 
3172 	memalign_arena = vmem_create("umem_memalign", NULL, 0, UMEM_ALIGN,
3173 	    heap_alloc, heap_free, minfirewall < ULONG_MAX ?
3174 	    umem_firewall_va_arena : heap_arena, 0, VM_NOSLEEP);
3175 
3176 	if (oversize_arena == NULL || memalign_arena == NULL)
3177 		goto fail;
3178 
3179 	if (umem_max_ncpus > CPUHINT_MAX())
3180 		umem_max_ncpus = CPUHINT_MAX();
3181 
3182 	while ((umem_max_ncpus & (umem_max_ncpus - 1)) != 0)
3183 		umem_max_ncpus++;
3184 
3185 	if (umem_max_ncpus == 0)
3186 		umem_max_ncpus = 1;
3187 
3188 	size = umem_max_ncpus * sizeof (umem_cpu_t);
3189 	new_cpus = vmem_alloc(umem_internal_arena, size, VM_NOSLEEP);
3190 	if (new_cpus == NULL)
3191 		goto fail;
3192 
3193 	bzero(new_cpus, size);
3194 	for (idx = 0; idx < umem_max_ncpus; idx++) {
3195 		new_cpus[idx].cpu_number = idx;
3196 		new_cpus[idx].cpu_cache_offset = UMEM_CACHE_SIZE(idx);
3197 	}
3198 	umem_cpus = new_cpus;
3199 	umem_cpu_mask = (umem_max_ncpus - 1);
3200 
3201 	if (umem_maxverify == 0)
3202 		umem_maxverify = maxverify;
3203 
3204 	if (umem_minfirewall == 0)
3205 		umem_minfirewall = minfirewall;
3206 
3207 	/*
3208 	 * Set up updating and reaping
3209 	 */
3210 	umem_reap_next = gethrtime() + NANOSEC;
3211 
3212 #ifndef UMEM_STANDALONE
3213 	(void) gettimeofday(&umem_update_next, NULL);
3214 #endif
3215 
3216 	/*
3217 	 * Set up logging -- failure here is okay, since it will just disable
3218 	 * the logs
3219 	 */
3220 	if (umem_logging) {
3221 		umem_transaction_log = umem_log_init(umem_transaction_log_size);
3222 		umem_content_log = umem_log_init(umem_content_log_size);
3223 		umem_failure_log = umem_log_init(umem_failure_log_size);
3224 		umem_slab_log = umem_log_init(umem_slab_log_size);
3225 	}
3226 
3227 	/*
3228 	 * Set up caches -- if successful, initialization cannot fail, since
3229 	 * allocations from other threads can now succeed.
3230 	 */
3231 	if (umem_cache_init() == 0) {
3232 		log_message("unable to create initial caches\n");
3233 		goto fail;
3234 	}
3235 	umem_oversize_arena = oversize_arena;
3236 	umem_memalign_arena = memalign_arena;
3237 
3238 	umem_cache_applyall(umem_cache_magazine_enable);
3239 
3240 	/*
3241 	 * initialization done, ready to go
3242 	 */
3243 	(void) mutex_lock(&umem_init_lock);
3244 	umem_ready = UMEM_READY;
3245 	umem_init_thr = 0;
3246 	(void) cond_broadcast(&umem_init_cv);
3247 	(void) mutex_unlock(&umem_init_lock);
3248 	return (1);
3249 
3250 fail:
3251 	log_message("umem initialization failed\n");
3252 
3253 	(void) mutex_lock(&umem_init_lock);
3254 	umem_ready = UMEM_READY_INIT_FAILED;
3255 	umem_init_thr = 0;
3256 	(void) cond_broadcast(&umem_init_cv);
3257 	(void) mutex_unlock(&umem_init_lock);
3258 	return (0);
3259 }
3260