xref: /illumos-gate/usr/src/uts/common/os/kmem.c (revision 65a89a64c60f3061bbe2381edaacc81660af9a95)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * Kernel memory allocator, as described in the following two papers:
31  *
32  * Jeff Bonwick,
33  * The Slab Allocator: An Object-Caching Kernel Memory Allocator.
34  * Proceedings of the Summer 1994 Usenix Conference.
35  * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
36  *
37  * Jeff Bonwick and Jonathan Adams,
38  * Magazines and vmem: Extending the Slab Allocator to Many CPUs and
39  * Arbitrary Resources.
40  * Proceedings of the 2001 Usenix Conference.
41  * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
42  */
43 
44 #include <sys/kmem_impl.h>
45 #include <sys/vmem_impl.h>
46 #include <sys/param.h>
47 #include <sys/sysmacros.h>
48 #include <sys/vm.h>
49 #include <sys/proc.h>
50 #include <sys/tuneable.h>
51 #include <sys/systm.h>
52 #include <sys/cmn_err.h>
53 #include <sys/debug.h>
54 #include <sys/mutex.h>
55 #include <sys/bitmap.h>
56 #include <sys/atomic.h>
57 #include <sys/kobj.h>
58 #include <sys/disp.h>
59 #include <vm/seg_kmem.h>
60 #include <sys/log.h>
61 #include <sys/callb.h>
62 #include <sys/taskq.h>
63 #include <sys/modctl.h>
64 #include <sys/reboot.h>
65 #include <sys/id32.h>
66 #include <sys/zone.h>
67 
68 extern void streams_msg_init(void);
69 extern int segkp_fromheap;
70 extern void segkp_cache_free(void);
71 
72 struct kmem_cache_kstat {
73 	kstat_named_t	kmc_buf_size;
74 	kstat_named_t	kmc_align;
75 	kstat_named_t	kmc_chunk_size;
76 	kstat_named_t	kmc_slab_size;
77 	kstat_named_t	kmc_alloc;
78 	kstat_named_t	kmc_alloc_fail;
79 	kstat_named_t	kmc_free;
80 	kstat_named_t	kmc_depot_alloc;
81 	kstat_named_t	kmc_depot_free;
82 	kstat_named_t	kmc_depot_contention;
83 	kstat_named_t	kmc_slab_alloc;
84 	kstat_named_t	kmc_slab_free;
85 	kstat_named_t	kmc_buf_constructed;
86 	kstat_named_t	kmc_buf_avail;
87 	kstat_named_t	kmc_buf_inuse;
88 	kstat_named_t	kmc_buf_total;
89 	kstat_named_t	kmc_buf_max;
90 	kstat_named_t	kmc_slab_create;
91 	kstat_named_t	kmc_slab_destroy;
92 	kstat_named_t	kmc_vmem_source;
93 	kstat_named_t	kmc_hash_size;
94 	kstat_named_t	kmc_hash_lookup_depth;
95 	kstat_named_t	kmc_hash_rescale;
96 	kstat_named_t	kmc_full_magazines;
97 	kstat_named_t	kmc_empty_magazines;
98 	kstat_named_t	kmc_magazine_size;
99 } kmem_cache_kstat = {
100 	{ "buf_size",		KSTAT_DATA_UINT64 },
101 	{ "align",		KSTAT_DATA_UINT64 },
102 	{ "chunk_size",		KSTAT_DATA_UINT64 },
103 	{ "slab_size",		KSTAT_DATA_UINT64 },
104 	{ "alloc",		KSTAT_DATA_UINT64 },
105 	{ "alloc_fail",		KSTAT_DATA_UINT64 },
106 	{ "free",		KSTAT_DATA_UINT64 },
107 	{ "depot_alloc",	KSTAT_DATA_UINT64 },
108 	{ "depot_free",		KSTAT_DATA_UINT64 },
109 	{ "depot_contention",	KSTAT_DATA_UINT64 },
110 	{ "slab_alloc",		KSTAT_DATA_UINT64 },
111 	{ "slab_free",		KSTAT_DATA_UINT64 },
112 	{ "buf_constructed",	KSTAT_DATA_UINT64 },
113 	{ "buf_avail",		KSTAT_DATA_UINT64 },
114 	{ "buf_inuse",		KSTAT_DATA_UINT64 },
115 	{ "buf_total",		KSTAT_DATA_UINT64 },
116 	{ "buf_max",		KSTAT_DATA_UINT64 },
117 	{ "slab_create",	KSTAT_DATA_UINT64 },
118 	{ "slab_destroy",	KSTAT_DATA_UINT64 },
119 	{ "vmem_source",	KSTAT_DATA_UINT64 },
120 	{ "hash_size",		KSTAT_DATA_UINT64 },
121 	{ "hash_lookup_depth",	KSTAT_DATA_UINT64 },
122 	{ "hash_rescale",	KSTAT_DATA_UINT64 },
123 	{ "full_magazines",	KSTAT_DATA_UINT64 },
124 	{ "empty_magazines",	KSTAT_DATA_UINT64 },
125 	{ "magazine_size",	KSTAT_DATA_UINT64 },
126 };
127 
128 static kmutex_t kmem_cache_kstat_lock;
129 
130 /*
131  * The default set of caches to back kmem_alloc().
132  * These sizes should be reevaluated periodically.
133  *
134  * We want allocations that are multiples of the coherency granularity
135  * (64 bytes) to be satisfied from a cache which is a multiple of 64
136  * bytes, so that it will be 64-byte aligned.  For all multiples of 64,
137  * the next kmem_cache_size greater than or equal to it must be a
138  * multiple of 64.
139  */
140 static const int kmem_alloc_sizes[] = {
141 	1 * 8,
142 	2 * 8,
143 	3 * 8,
144 	4 * 8,		5 * 8,		6 * 8,		7 * 8,
145 	4 * 16,		5 * 16,		6 * 16,		7 * 16,
146 	4 * 32,		5 * 32,		6 * 32,		7 * 32,
147 	4 * 64,		5 * 64,		6 * 64,		7 * 64,
148 	4 * 128,	5 * 128,	6 * 128,	7 * 128,
149 	P2ALIGN(8192 / 7, 64),
150 	P2ALIGN(8192 / 6, 64),
151 	P2ALIGN(8192 / 5, 64),
152 	P2ALIGN(8192 / 4, 64),
153 	P2ALIGN(8192 / 3, 64),
154 	P2ALIGN(8192 / 2, 64),
155 	P2ALIGN(8192 / 1, 64),
156 	4096 * 3,
157 	8192 * 2,
158 };
159 
160 #define	KMEM_MAXBUF	16384
161 
162 static kmem_cache_t *kmem_alloc_table[KMEM_MAXBUF >> KMEM_ALIGN_SHIFT];
163 
164 static kmem_magtype_t kmem_magtype[] = {
165 	{ 1,	8,	3200,	65536	},
166 	{ 3,	16,	256,	32768	},
167 	{ 7,	32,	64,	16384	},
168 	{ 15,	64,	0,	8192	},
169 	{ 31,	64,	0,	4096	},
170 	{ 47,	64,	0,	2048	},
171 	{ 63,	64,	0,	1024	},
172 	{ 95,	64,	0,	512	},
173 	{ 143,	64,	0,	0	},
174 };
175 
176 static uint32_t kmem_reaping;
177 static uint32_t kmem_reaping_idspace;
178 
179 /*
180  * kmem tunables
181  */
182 clock_t kmem_reap_interval;	/* cache reaping rate [15 * HZ ticks] */
183 int kmem_depot_contention = 3;	/* max failed tryenters per real interval */
184 pgcnt_t kmem_reapahead = 0;	/* start reaping N pages before pageout */
185 int kmem_panic = 1;		/* whether to panic on error */
186 int kmem_logging = 1;		/* kmem_log_enter() override */
187 uint32_t kmem_mtbf = 0;		/* mean time between failures [default: off] */
188 size_t kmem_transaction_log_size; /* transaction log size [2% of memory] */
189 size_t kmem_content_log_size;	/* content log size [2% of memory] */
190 size_t kmem_failure_log_size;	/* failure log [4 pages per CPU] */
191 size_t kmem_slab_log_size;	/* slab create log [4 pages per CPU] */
192 size_t kmem_content_maxsave = 256; /* KMF_CONTENTS max bytes to log */
193 size_t kmem_lite_minsize = 0;	/* minimum buffer size for KMF_LITE */
194 size_t kmem_lite_maxalign = 1024; /* maximum buffer alignment for KMF_LITE */
195 int kmem_lite_pcs = 4;		/* number of PCs to store in KMF_LITE mode */
196 size_t kmem_maxverify;		/* maximum bytes to inspect in debug routines */
197 size_t kmem_minfirewall;	/* hardware-enforced redzone threshold */
198 
199 #ifdef DEBUG
200 int kmem_flags = KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | KMF_CONTENTS;
201 #else
202 int kmem_flags = 0;
203 #endif
204 int kmem_ready;
205 
206 static kmem_cache_t	*kmem_slab_cache;
207 static kmem_cache_t	*kmem_bufctl_cache;
208 static kmem_cache_t	*kmem_bufctl_audit_cache;
209 
210 static kmutex_t		kmem_cache_lock;	/* inter-cache linkage only */
211 kmem_cache_t		kmem_null_cache;
212 
213 static taskq_t		*kmem_taskq;
214 static kmutex_t		kmem_flags_lock;
215 static vmem_t		*kmem_metadata_arena;
216 static vmem_t		*kmem_msb_arena;	/* arena for metadata caches */
217 static vmem_t		*kmem_cache_arena;
218 static vmem_t		*kmem_hash_arena;
219 static vmem_t		*kmem_log_arena;
220 static vmem_t		*kmem_oversize_arena;
221 static vmem_t		*kmem_va_arena;
222 static vmem_t		*kmem_default_arena;
223 static vmem_t		*kmem_firewall_va_arena;
224 static vmem_t		*kmem_firewall_arena;
225 
226 kmem_log_header_t	*kmem_transaction_log;
227 kmem_log_header_t	*kmem_content_log;
228 kmem_log_header_t	*kmem_failure_log;
229 kmem_log_header_t	*kmem_slab_log;
230 
231 static int		kmem_lite_count; /* # of PCs in kmem_buftag_lite_t */
232 
233 #define	KMEM_BUFTAG_LITE_ENTER(bt, count, caller)			\
234 	if ((count) > 0) {						\
235 		pc_t *_s = ((kmem_buftag_lite_t *)(bt))->bt_history;	\
236 		pc_t *_e;						\
237 		/* memmove() the old entries down one notch */		\
238 		for (_e = &_s[(count) - 1]; _e > _s; _e--)		\
239 			*_e = *(_e - 1);				\
240 		*_s = (uintptr_t)(caller);				\
241 	}
242 
243 #define	KMERR_MODIFIED	0	/* buffer modified while on freelist */
244 #define	KMERR_REDZONE	1	/* redzone violation (write past end of buf) */
245 #define	KMERR_DUPFREE	2	/* freed a buffer twice */
246 #define	KMERR_BADADDR	3	/* freed a bad (unallocated) address */
247 #define	KMERR_BADBUFTAG	4	/* buftag corrupted */
248 #define	KMERR_BADBUFCTL	5	/* bufctl corrupted */
249 #define	KMERR_BADCACHE	6	/* freed a buffer to the wrong cache */
250 #define	KMERR_BADSIZE	7	/* alloc size != free size */
251 #define	KMERR_BADBASE	8	/* buffer base address wrong */
252 
253 struct {
254 	hrtime_t	kmp_timestamp;	/* timestamp of panic */
255 	int		kmp_error;	/* type of kmem error */
256 	void		*kmp_buffer;	/* buffer that induced panic */
257 	void		*kmp_realbuf;	/* real start address for buffer */
258 	kmem_cache_t	*kmp_cache;	/* buffer's cache according to client */
259 	kmem_cache_t	*kmp_realcache;	/* actual cache containing buffer */
260 	kmem_slab_t	*kmp_slab;	/* slab accoring to kmem_findslab() */
261 	kmem_bufctl_t	*kmp_bufctl;	/* bufctl */
262 } kmem_panic_info;
263 
264 
265 static void
266 copy_pattern(uint64_t pattern, void *buf_arg, size_t size)
267 {
268 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
269 	uint64_t *buf = buf_arg;
270 
271 	while (buf < bufend)
272 		*buf++ = pattern;
273 }
274 
275 static void *
276 verify_pattern(uint64_t pattern, void *buf_arg, size_t size)
277 {
278 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
279 	uint64_t *buf;
280 
281 	for (buf = buf_arg; buf < bufend; buf++)
282 		if (*buf != pattern)
283 			return (buf);
284 	return (NULL);
285 }
286 
287 static void *
288 verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size)
289 {
290 	uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
291 	uint64_t *buf;
292 
293 	for (buf = buf_arg; buf < bufend; buf++) {
294 		if (*buf != old) {
295 			copy_pattern(old, buf_arg,
296 				(char *)buf - (char *)buf_arg);
297 			return (buf);
298 		}
299 		*buf = new;
300 	}
301 
302 	return (NULL);
303 }
304 
305 static void
306 kmem_cache_applyall(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag)
307 {
308 	kmem_cache_t *cp;
309 
310 	mutex_enter(&kmem_cache_lock);
311 	for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache;
312 	    cp = cp->cache_next)
313 		if (tq != NULL)
314 			(void) taskq_dispatch(tq, (task_func_t *)func, cp,
315 			    tqflag);
316 		else
317 			func(cp);
318 	mutex_exit(&kmem_cache_lock);
319 }
320 
321 static void
322 kmem_cache_applyall_id(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag)
323 {
324 	kmem_cache_t *cp;
325 
326 	mutex_enter(&kmem_cache_lock);
327 	for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache;
328 	    cp = cp->cache_next) {
329 		if (!(cp->cache_cflags & KMC_IDENTIFIER))
330 			continue;
331 		if (tq != NULL)
332 			(void) taskq_dispatch(tq, (task_func_t *)func, cp,
333 			    tqflag);
334 		else
335 			func(cp);
336 	}
337 	mutex_exit(&kmem_cache_lock);
338 }
339 
340 /*
341  * Debugging support.  Given a buffer address, find its slab.
342  */
343 static kmem_slab_t *
344 kmem_findslab(kmem_cache_t *cp, void *buf)
345 {
346 	kmem_slab_t *sp;
347 
348 	mutex_enter(&cp->cache_lock);
349 	for (sp = cp->cache_nullslab.slab_next;
350 	    sp != &cp->cache_nullslab; sp = sp->slab_next) {
351 		if (KMEM_SLAB_MEMBER(sp, buf)) {
352 			mutex_exit(&cp->cache_lock);
353 			return (sp);
354 		}
355 	}
356 	mutex_exit(&cp->cache_lock);
357 
358 	return (NULL);
359 }
360 
361 static void
362 kmem_error(int error, kmem_cache_t *cparg, void *bufarg)
363 {
364 	kmem_buftag_t *btp = NULL;
365 	kmem_bufctl_t *bcp = NULL;
366 	kmem_cache_t *cp = cparg;
367 	kmem_slab_t *sp;
368 	uint64_t *off;
369 	void *buf = bufarg;
370 
371 	kmem_logging = 0;	/* stop logging when a bad thing happens */
372 
373 	kmem_panic_info.kmp_timestamp = gethrtime();
374 
375 	sp = kmem_findslab(cp, buf);
376 	if (sp == NULL) {
377 		for (cp = kmem_null_cache.cache_prev; cp != &kmem_null_cache;
378 		    cp = cp->cache_prev) {
379 			if ((sp = kmem_findslab(cp, buf)) != NULL)
380 				break;
381 		}
382 	}
383 
384 	if (sp == NULL) {
385 		cp = NULL;
386 		error = KMERR_BADADDR;
387 	} else {
388 		if (cp != cparg)
389 			error = KMERR_BADCACHE;
390 		else
391 			buf = (char *)bufarg - ((uintptr_t)bufarg -
392 			    (uintptr_t)sp->slab_base) % cp->cache_chunksize;
393 		if (buf != bufarg)
394 			error = KMERR_BADBASE;
395 		if (cp->cache_flags & KMF_BUFTAG)
396 			btp = KMEM_BUFTAG(cp, buf);
397 		if (cp->cache_flags & KMF_HASH) {
398 			mutex_enter(&cp->cache_lock);
399 			for (bcp = *KMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next)
400 				if (bcp->bc_addr == buf)
401 					break;
402 			mutex_exit(&cp->cache_lock);
403 			if (bcp == NULL && btp != NULL)
404 				bcp = btp->bt_bufctl;
405 			if (kmem_findslab(cp->cache_bufctl_cache, bcp) ==
406 			    NULL || P2PHASE((uintptr_t)bcp, KMEM_ALIGN) ||
407 			    bcp->bc_addr != buf) {
408 				error = KMERR_BADBUFCTL;
409 				bcp = NULL;
410 			}
411 		}
412 	}
413 
414 	kmem_panic_info.kmp_error = error;
415 	kmem_panic_info.kmp_buffer = bufarg;
416 	kmem_panic_info.kmp_realbuf = buf;
417 	kmem_panic_info.kmp_cache = cparg;
418 	kmem_panic_info.kmp_realcache = cp;
419 	kmem_panic_info.kmp_slab = sp;
420 	kmem_panic_info.kmp_bufctl = bcp;
421 
422 	printf("kernel memory allocator: ");
423 
424 	switch (error) {
425 
426 	case KMERR_MODIFIED:
427 		printf("buffer modified after being freed\n");
428 		off = verify_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify);
429 		if (off == NULL)	/* shouldn't happen */
430 			off = buf;
431 		printf("modification occurred at offset 0x%lx "
432 		    "(0x%llx replaced by 0x%llx)\n",
433 		    (uintptr_t)off - (uintptr_t)buf,
434 		    (longlong_t)KMEM_FREE_PATTERN, (longlong_t)*off);
435 		break;
436 
437 	case KMERR_REDZONE:
438 		printf("redzone violation: write past end of buffer\n");
439 		break;
440 
441 	case KMERR_BADADDR:
442 		printf("invalid free: buffer not in cache\n");
443 		break;
444 
445 	case KMERR_DUPFREE:
446 		printf("duplicate free: buffer freed twice\n");
447 		break;
448 
449 	case KMERR_BADBUFTAG:
450 		printf("boundary tag corrupted\n");
451 		printf("bcp ^ bxstat = %lx, should be %lx\n",
452 		    (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat,
453 		    KMEM_BUFTAG_FREE);
454 		break;
455 
456 	case KMERR_BADBUFCTL:
457 		printf("bufctl corrupted\n");
458 		break;
459 
460 	case KMERR_BADCACHE:
461 		printf("buffer freed to wrong cache\n");
462 		printf("buffer was allocated from %s,\n", cp->cache_name);
463 		printf("caller attempting free to %s.\n", cparg->cache_name);
464 		break;
465 
466 	case KMERR_BADSIZE:
467 		printf("bad free: free size (%u) != alloc size (%u)\n",
468 		    KMEM_SIZE_DECODE(((uint32_t *)btp)[0]),
469 		    KMEM_SIZE_DECODE(((uint32_t *)btp)[1]));
470 		break;
471 
472 	case KMERR_BADBASE:
473 		printf("bad free: free address (%p) != alloc address (%p)\n",
474 		    bufarg, buf);
475 		break;
476 	}
477 
478 	printf("buffer=%p  bufctl=%p  cache: %s\n",
479 	    bufarg, (void *)bcp, cparg->cache_name);
480 
481 	if (bcp != NULL && (cp->cache_flags & KMF_AUDIT) &&
482 	    error != KMERR_BADBUFCTL) {
483 		int d;
484 		timestruc_t ts;
485 		kmem_bufctl_audit_t *bcap = (kmem_bufctl_audit_t *)bcp;
486 
487 		hrt2ts(kmem_panic_info.kmp_timestamp - bcap->bc_timestamp, &ts);
488 		printf("previous transaction on buffer %p:\n", buf);
489 		printf("thread=%p  time=T-%ld.%09ld  slab=%p  cache: %s\n",
490 		    (void *)bcap->bc_thread, ts.tv_sec, ts.tv_nsec,
491 		    (void *)sp, cp->cache_name);
492 		for (d = 0; d < MIN(bcap->bc_depth, KMEM_STACK_DEPTH); d++) {
493 			ulong_t off;
494 			char *sym = kobj_getsymname(bcap->bc_stack[d], &off);
495 			printf("%s+%lx\n", sym ? sym : "?", off);
496 		}
497 	}
498 	if (kmem_panic > 0)
499 		panic("kernel heap corruption detected");
500 	if (kmem_panic == 0)
501 		debug_enter(NULL);
502 	kmem_logging = 1;	/* resume logging */
503 }
504 
505 static kmem_log_header_t *
506 kmem_log_init(size_t logsize)
507 {
508 	kmem_log_header_t *lhp;
509 	int nchunks = 4 * max_ncpus;
510 	size_t lhsize = (size_t)&((kmem_log_header_t *)0)->lh_cpu[max_ncpus];
511 	int i;
512 
513 	/*
514 	 * Make sure that lhp->lh_cpu[] is nicely aligned
515 	 * to prevent false sharing of cache lines.
516 	 */
517 	lhsize = P2ROUNDUP(lhsize, KMEM_ALIGN);
518 	lhp = vmem_xalloc(kmem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0,
519 	    NULL, NULL, VM_SLEEP);
520 	bzero(lhp, lhsize);
521 
522 	mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL);
523 	lhp->lh_nchunks = nchunks;
524 	lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE);
525 	lhp->lh_base = vmem_alloc(kmem_log_arena,
526 	    lhp->lh_chunksize * nchunks, VM_SLEEP);
527 	lhp->lh_free = vmem_alloc(kmem_log_arena,
528 	    nchunks * sizeof (int), VM_SLEEP);
529 	bzero(lhp->lh_base, lhp->lh_chunksize * nchunks);
530 
531 	for (i = 0; i < max_ncpus; i++) {
532 		kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i];
533 		mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL);
534 		clhp->clh_chunk = i;
535 	}
536 
537 	for (i = max_ncpus; i < nchunks; i++)
538 		lhp->lh_free[i] = i;
539 
540 	lhp->lh_head = max_ncpus;
541 	lhp->lh_tail = 0;
542 
543 	return (lhp);
544 }
545 
546 static void *
547 kmem_log_enter(kmem_log_header_t *lhp, void *data, size_t size)
548 {
549 	void *logspace;
550 	kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[CPU->cpu_seqid];
551 
552 	if (lhp == NULL || kmem_logging == 0 || panicstr)
553 		return (NULL);
554 
555 	mutex_enter(&clhp->clh_lock);
556 	clhp->clh_hits++;
557 	if (size > clhp->clh_avail) {
558 		mutex_enter(&lhp->lh_lock);
559 		lhp->lh_hits++;
560 		lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk;
561 		lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks;
562 		clhp->clh_chunk = lhp->lh_free[lhp->lh_head];
563 		lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks;
564 		clhp->clh_current = lhp->lh_base +
565 			clhp->clh_chunk * lhp->lh_chunksize;
566 		clhp->clh_avail = lhp->lh_chunksize;
567 		if (size > lhp->lh_chunksize)
568 			size = lhp->lh_chunksize;
569 		mutex_exit(&lhp->lh_lock);
570 	}
571 	logspace = clhp->clh_current;
572 	clhp->clh_current += size;
573 	clhp->clh_avail -= size;
574 	bcopy(data, logspace, size);
575 	mutex_exit(&clhp->clh_lock);
576 	return (logspace);
577 }
578 
579 #define	KMEM_AUDIT(lp, cp, bcp)						\
580 {									\
581 	kmem_bufctl_audit_t *_bcp = (kmem_bufctl_audit_t *)(bcp);	\
582 	_bcp->bc_timestamp = gethrtime();				\
583 	_bcp->bc_thread = curthread;					\
584 	_bcp->bc_depth = getpcstack(_bcp->bc_stack, KMEM_STACK_DEPTH);	\
585 	_bcp->bc_lastlog = kmem_log_enter((lp), _bcp, sizeof (*_bcp));	\
586 }
587 
588 static void
589 kmem_log_event(kmem_log_header_t *lp, kmem_cache_t *cp,
590 	kmem_slab_t *sp, void *addr)
591 {
592 	kmem_bufctl_audit_t bca;
593 
594 	bzero(&bca, sizeof (kmem_bufctl_audit_t));
595 	bca.bc_addr = addr;
596 	bca.bc_slab = sp;
597 	bca.bc_cache = cp;
598 	KMEM_AUDIT(lp, cp, &bca);
599 }
600 
601 /*
602  * Create a new slab for cache cp.
603  */
604 static kmem_slab_t *
605 kmem_slab_create(kmem_cache_t *cp, int kmflag)
606 {
607 	size_t slabsize = cp->cache_slabsize;
608 	size_t chunksize = cp->cache_chunksize;
609 	int cache_flags = cp->cache_flags;
610 	size_t color, chunks;
611 	char *buf, *slab;
612 	kmem_slab_t *sp;
613 	kmem_bufctl_t *bcp;
614 	vmem_t *vmp = cp->cache_arena;
615 
616 	color = cp->cache_color + cp->cache_align;
617 	if (color > cp->cache_maxcolor)
618 		color = cp->cache_mincolor;
619 	cp->cache_color = color;
620 
621 	slab = vmem_alloc(vmp, slabsize, kmflag & KM_VMFLAGS);
622 
623 	if (slab == NULL)
624 		goto vmem_alloc_failure;
625 
626 	ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0);
627 
628 	if (!(cp->cache_cflags & KMC_NOTOUCH))
629 		copy_pattern(KMEM_UNINITIALIZED_PATTERN, slab, slabsize);
630 
631 	if (cache_flags & KMF_HASH) {
632 		if ((sp = kmem_cache_alloc(kmem_slab_cache, kmflag)) == NULL)
633 			goto slab_alloc_failure;
634 		chunks = (slabsize - color) / chunksize;
635 	} else {
636 		sp = KMEM_SLAB(cp, slab);
637 		chunks = (slabsize - sizeof (kmem_slab_t) - color) / chunksize;
638 	}
639 
640 	sp->slab_cache	= cp;
641 	sp->slab_head	= NULL;
642 	sp->slab_refcnt	= 0;
643 	sp->slab_base	= buf = slab + color;
644 	sp->slab_chunks	= chunks;
645 
646 	ASSERT(chunks > 0);
647 	while (chunks-- != 0) {
648 		if (cache_flags & KMF_HASH) {
649 			bcp = kmem_cache_alloc(cp->cache_bufctl_cache, kmflag);
650 			if (bcp == NULL)
651 				goto bufctl_alloc_failure;
652 			if (cache_flags & KMF_AUDIT) {
653 				kmem_bufctl_audit_t *bcap =
654 				    (kmem_bufctl_audit_t *)bcp;
655 				bzero(bcap, sizeof (kmem_bufctl_audit_t));
656 				bcap->bc_cache = cp;
657 			}
658 			bcp->bc_addr = buf;
659 			bcp->bc_slab = sp;
660 		} else {
661 			bcp = KMEM_BUFCTL(cp, buf);
662 		}
663 		if (cache_flags & KMF_BUFTAG) {
664 			kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
665 			btp->bt_redzone = KMEM_REDZONE_PATTERN;
666 			btp->bt_bufctl = bcp;
667 			btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE;
668 			if (cache_flags & KMF_DEADBEEF) {
669 				copy_pattern(KMEM_FREE_PATTERN, buf,
670 				    cp->cache_verify);
671 			}
672 		}
673 		bcp->bc_next = sp->slab_head;
674 		sp->slab_head = bcp;
675 		buf += chunksize;
676 	}
677 
678 	kmem_log_event(kmem_slab_log, cp, sp, slab);
679 
680 	return (sp);
681 
682 bufctl_alloc_failure:
683 
684 	while ((bcp = sp->slab_head) != NULL) {
685 		sp->slab_head = bcp->bc_next;
686 		kmem_cache_free(cp->cache_bufctl_cache, bcp);
687 	}
688 	kmem_cache_free(kmem_slab_cache, sp);
689 
690 slab_alloc_failure:
691 
692 	vmem_free(vmp, slab, slabsize);
693 
694 vmem_alloc_failure:
695 
696 	kmem_log_event(kmem_failure_log, cp, NULL, NULL);
697 	atomic_add_64(&cp->cache_alloc_fail, 1);
698 
699 	return (NULL);
700 }
701 
702 /*
703  * Destroy a slab.
704  */
705 static void
706 kmem_slab_destroy(kmem_cache_t *cp, kmem_slab_t *sp)
707 {
708 	vmem_t *vmp = cp->cache_arena;
709 	void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum);
710 
711 	if (cp->cache_flags & KMF_HASH) {
712 		kmem_bufctl_t *bcp;
713 		while ((bcp = sp->slab_head) != NULL) {
714 			sp->slab_head = bcp->bc_next;
715 			kmem_cache_free(cp->cache_bufctl_cache, bcp);
716 		}
717 		kmem_cache_free(kmem_slab_cache, sp);
718 	}
719 	vmem_free(vmp, slab, cp->cache_slabsize);
720 }
721 
722 /*
723  * Allocate a raw (unconstructed) buffer from cp's slab layer.
724  */
725 static void *
726 kmem_slab_alloc(kmem_cache_t *cp, int kmflag)
727 {
728 	kmem_bufctl_t *bcp, **hash_bucket;
729 	kmem_slab_t *sp;
730 	void *buf;
731 
732 	mutex_enter(&cp->cache_lock);
733 	cp->cache_slab_alloc++;
734 	sp = cp->cache_freelist;
735 	ASSERT(sp->slab_cache == cp);
736 	if (sp->slab_head == NULL) {
737 		/*
738 		 * The freelist is empty.  Create a new slab.
739 		 */
740 		mutex_exit(&cp->cache_lock);
741 		if ((sp = kmem_slab_create(cp, kmflag)) == NULL)
742 			return (NULL);
743 		mutex_enter(&cp->cache_lock);
744 		cp->cache_slab_create++;
745 		if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax)
746 			cp->cache_bufmax = cp->cache_buftotal;
747 		sp->slab_next = cp->cache_freelist;
748 		sp->slab_prev = cp->cache_freelist->slab_prev;
749 		sp->slab_next->slab_prev = sp;
750 		sp->slab_prev->slab_next = sp;
751 		cp->cache_freelist = sp;
752 	}
753 
754 	sp->slab_refcnt++;
755 	ASSERT(sp->slab_refcnt <= sp->slab_chunks);
756 
757 	/*
758 	 * If we're taking the last buffer in the slab,
759 	 * remove the slab from the cache's freelist.
760 	 */
761 	bcp = sp->slab_head;
762 	if ((sp->slab_head = bcp->bc_next) == NULL) {
763 		cp->cache_freelist = sp->slab_next;
764 		ASSERT(sp->slab_refcnt == sp->slab_chunks);
765 	}
766 
767 	if (cp->cache_flags & KMF_HASH) {
768 		/*
769 		 * Add buffer to allocated-address hash table.
770 		 */
771 		buf = bcp->bc_addr;
772 		hash_bucket = KMEM_HASH(cp, buf);
773 		bcp->bc_next = *hash_bucket;
774 		*hash_bucket = bcp;
775 		if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) {
776 			KMEM_AUDIT(kmem_transaction_log, cp, bcp);
777 		}
778 	} else {
779 		buf = KMEM_BUF(cp, bcp);
780 	}
781 
782 	ASSERT(KMEM_SLAB_MEMBER(sp, buf));
783 
784 	mutex_exit(&cp->cache_lock);
785 
786 	return (buf);
787 }
788 
789 /*
790  * Free a raw (unconstructed) buffer to cp's slab layer.
791  */
792 static void
793 kmem_slab_free(kmem_cache_t *cp, void *buf)
794 {
795 	kmem_slab_t *sp;
796 	kmem_bufctl_t *bcp, **prev_bcpp;
797 
798 	ASSERT(buf != NULL);
799 
800 	mutex_enter(&cp->cache_lock);
801 	cp->cache_slab_free++;
802 
803 	if (cp->cache_flags & KMF_HASH) {
804 		/*
805 		 * Look up buffer in allocated-address hash table.
806 		 */
807 		prev_bcpp = KMEM_HASH(cp, buf);
808 		while ((bcp = *prev_bcpp) != NULL) {
809 			if (bcp->bc_addr == buf) {
810 				*prev_bcpp = bcp->bc_next;
811 				sp = bcp->bc_slab;
812 				break;
813 			}
814 			cp->cache_lookup_depth++;
815 			prev_bcpp = &bcp->bc_next;
816 		}
817 	} else {
818 		bcp = KMEM_BUFCTL(cp, buf);
819 		sp = KMEM_SLAB(cp, buf);
820 	}
821 
822 	if (bcp == NULL || sp->slab_cache != cp || !KMEM_SLAB_MEMBER(sp, buf)) {
823 		mutex_exit(&cp->cache_lock);
824 		kmem_error(KMERR_BADADDR, cp, buf);
825 		return;
826 	}
827 
828 	if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) {
829 		if (cp->cache_flags & KMF_CONTENTS)
830 			((kmem_bufctl_audit_t *)bcp)->bc_contents =
831 			    kmem_log_enter(kmem_content_log, buf,
832 				cp->cache_contents);
833 		KMEM_AUDIT(kmem_transaction_log, cp, bcp);
834 	}
835 
836 	/*
837 	 * If this slab isn't currently on the freelist, put it there.
838 	 */
839 	if (sp->slab_head == NULL) {
840 		ASSERT(sp->slab_refcnt == sp->slab_chunks);
841 		ASSERT(cp->cache_freelist != sp);
842 		sp->slab_next->slab_prev = sp->slab_prev;
843 		sp->slab_prev->slab_next = sp->slab_next;
844 		sp->slab_next = cp->cache_freelist;
845 		sp->slab_prev = cp->cache_freelist->slab_prev;
846 		sp->slab_next->slab_prev = sp;
847 		sp->slab_prev->slab_next = sp;
848 		cp->cache_freelist = sp;
849 	}
850 
851 	bcp->bc_next = sp->slab_head;
852 	sp->slab_head = bcp;
853 
854 	ASSERT(sp->slab_refcnt >= 1);
855 	if (--sp->slab_refcnt == 0) {
856 		/*
857 		 * There are no outstanding allocations from this slab,
858 		 * so we can reclaim the memory.
859 		 */
860 		sp->slab_next->slab_prev = sp->slab_prev;
861 		sp->slab_prev->slab_next = sp->slab_next;
862 		if (sp == cp->cache_freelist)
863 			cp->cache_freelist = sp->slab_next;
864 		cp->cache_slab_destroy++;
865 		cp->cache_buftotal -= sp->slab_chunks;
866 		mutex_exit(&cp->cache_lock);
867 		kmem_slab_destroy(cp, sp);
868 		return;
869 	}
870 	mutex_exit(&cp->cache_lock);
871 }
872 
873 static int
874 kmem_cache_alloc_debug(kmem_cache_t *cp, void *buf, int kmflag, int construct,
875     caddr_t caller)
876 {
877 	kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
878 	kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl;
879 	uint32_t mtbf;
880 
881 	if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) {
882 		kmem_error(KMERR_BADBUFTAG, cp, buf);
883 		return (-1);
884 	}
885 
886 	btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_ALLOC;
887 
888 	if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) {
889 		kmem_error(KMERR_BADBUFCTL, cp, buf);
890 		return (-1);
891 	}
892 
893 	if (cp->cache_flags & KMF_DEADBEEF) {
894 		if (!construct && (cp->cache_flags & KMF_LITE)) {
895 			if (*(uint64_t *)buf != KMEM_FREE_PATTERN) {
896 				kmem_error(KMERR_MODIFIED, cp, buf);
897 				return (-1);
898 			}
899 			if (cp->cache_constructor != NULL)
900 				*(uint64_t *)buf = btp->bt_redzone;
901 			else
902 				*(uint64_t *)buf = KMEM_UNINITIALIZED_PATTERN;
903 		} else {
904 			construct = 1;
905 			if (verify_and_copy_pattern(KMEM_FREE_PATTERN,
906 			    KMEM_UNINITIALIZED_PATTERN, buf,
907 			    cp->cache_verify)) {
908 				kmem_error(KMERR_MODIFIED, cp, buf);
909 				return (-1);
910 			}
911 		}
912 	}
913 	btp->bt_redzone = KMEM_REDZONE_PATTERN;
914 
915 	if ((mtbf = kmem_mtbf | cp->cache_mtbf) != 0 &&
916 	    gethrtime() % mtbf == 0 &&
917 	    (kmflag & (KM_NOSLEEP | KM_PANIC)) == KM_NOSLEEP) {
918 		kmem_log_event(kmem_failure_log, cp, NULL, NULL);
919 		if (!construct && cp->cache_destructor != NULL)
920 			cp->cache_destructor(buf, cp->cache_private);
921 	} else {
922 		mtbf = 0;
923 	}
924 
925 	if (mtbf || (construct && cp->cache_constructor != NULL &&
926 	    cp->cache_constructor(buf, cp->cache_private, kmflag) != 0)) {
927 		atomic_add_64(&cp->cache_alloc_fail, 1);
928 		btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE;
929 		if (cp->cache_flags & KMF_DEADBEEF)
930 			copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify);
931 		kmem_slab_free(cp, buf);
932 		return (-1);
933 	}
934 
935 	if (cp->cache_flags & KMF_AUDIT) {
936 		KMEM_AUDIT(kmem_transaction_log, cp, bcp);
937 	}
938 
939 	if ((cp->cache_flags & KMF_LITE) &&
940 	    !(cp->cache_cflags & KMC_KMEM_ALLOC)) {
941 		KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller);
942 	}
943 
944 	return (0);
945 }
946 
947 static int
948 kmem_cache_free_debug(kmem_cache_t *cp, void *buf, caddr_t caller)
949 {
950 	kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
951 	kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl;
952 	kmem_slab_t *sp;
953 
954 	if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_ALLOC)) {
955 		if (btp->bt_bxstat == ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) {
956 			kmem_error(KMERR_DUPFREE, cp, buf);
957 			return (-1);
958 		}
959 		sp = kmem_findslab(cp, buf);
960 		if (sp == NULL || sp->slab_cache != cp)
961 			kmem_error(KMERR_BADADDR, cp, buf);
962 		else
963 			kmem_error(KMERR_REDZONE, cp, buf);
964 		return (-1);
965 	}
966 
967 	btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE;
968 
969 	if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) {
970 		kmem_error(KMERR_BADBUFCTL, cp, buf);
971 		return (-1);
972 	}
973 
974 	if (btp->bt_redzone != KMEM_REDZONE_PATTERN) {
975 		kmem_error(KMERR_REDZONE, cp, buf);
976 		return (-1);
977 	}
978 
979 	if (cp->cache_flags & KMF_AUDIT) {
980 		if (cp->cache_flags & KMF_CONTENTS)
981 			bcp->bc_contents = kmem_log_enter(kmem_content_log,
982 			    buf, cp->cache_contents);
983 		KMEM_AUDIT(kmem_transaction_log, cp, bcp);
984 	}
985 
986 	if ((cp->cache_flags & KMF_LITE) &&
987 	    !(cp->cache_cflags & KMC_KMEM_ALLOC)) {
988 		KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller);
989 	}
990 
991 	if (cp->cache_flags & KMF_DEADBEEF) {
992 		if (cp->cache_flags & KMF_LITE)
993 			btp->bt_redzone = *(uint64_t *)buf;
994 		else if (cp->cache_destructor != NULL)
995 			cp->cache_destructor(buf, cp->cache_private);
996 
997 		copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify);
998 	}
999 
1000 	return (0);
1001 }
1002 
1003 /*
1004  * Free each object in magazine mp to cp's slab layer, and free mp itself.
1005  */
1006 static void
1007 kmem_magazine_destroy(kmem_cache_t *cp, kmem_magazine_t *mp, int nrounds)
1008 {
1009 	int round;
1010 
1011 	ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread));
1012 
1013 	for (round = 0; round < nrounds; round++) {
1014 		void *buf = mp->mag_round[round];
1015 
1016 		if (cp->cache_flags & KMF_DEADBEEF) {
1017 			if (verify_pattern(KMEM_FREE_PATTERN, buf,
1018 			    cp->cache_verify) != NULL) {
1019 				kmem_error(KMERR_MODIFIED, cp, buf);
1020 				continue;
1021 			}
1022 			if ((cp->cache_flags & KMF_LITE) &&
1023 			    cp->cache_destructor != NULL) {
1024 				kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
1025 				*(uint64_t *)buf = btp->bt_redzone;
1026 				cp->cache_destructor(buf, cp->cache_private);
1027 				*(uint64_t *)buf = KMEM_FREE_PATTERN;
1028 			}
1029 		} else if (cp->cache_destructor != NULL) {
1030 			cp->cache_destructor(buf, cp->cache_private);
1031 		}
1032 
1033 		kmem_slab_free(cp, buf);
1034 	}
1035 	ASSERT(KMEM_MAGAZINE_VALID(cp, mp));
1036 	kmem_cache_free(cp->cache_magtype->mt_cache, mp);
1037 }
1038 
1039 /*
1040  * Allocate a magazine from the depot.
1041  */
1042 static kmem_magazine_t *
1043 kmem_depot_alloc(kmem_cache_t *cp, kmem_maglist_t *mlp)
1044 {
1045 	kmem_magazine_t *mp;
1046 
1047 	/*
1048 	 * If we can't get the depot lock without contention,
1049 	 * update our contention count.  We use the depot
1050 	 * contention rate to determine whether we need to
1051 	 * increase the magazine size for better scalability.
1052 	 */
1053 	if (!mutex_tryenter(&cp->cache_depot_lock)) {
1054 		mutex_enter(&cp->cache_depot_lock);
1055 		cp->cache_depot_contention++;
1056 	}
1057 
1058 	if ((mp = mlp->ml_list) != NULL) {
1059 		ASSERT(KMEM_MAGAZINE_VALID(cp, mp));
1060 		mlp->ml_list = mp->mag_next;
1061 		if (--mlp->ml_total < mlp->ml_min)
1062 			mlp->ml_min = mlp->ml_total;
1063 		mlp->ml_alloc++;
1064 	}
1065 
1066 	mutex_exit(&cp->cache_depot_lock);
1067 
1068 	return (mp);
1069 }
1070 
1071 /*
1072  * Free a magazine to the depot.
1073  */
1074 static void
1075 kmem_depot_free(kmem_cache_t *cp, kmem_maglist_t *mlp, kmem_magazine_t *mp)
1076 {
1077 	mutex_enter(&cp->cache_depot_lock);
1078 	ASSERT(KMEM_MAGAZINE_VALID(cp, mp));
1079 	mp->mag_next = mlp->ml_list;
1080 	mlp->ml_list = mp;
1081 	mlp->ml_total++;
1082 	mutex_exit(&cp->cache_depot_lock);
1083 }
1084 
1085 /*
1086  * Update the working set statistics for cp's depot.
1087  */
1088 static void
1089 kmem_depot_ws_update(kmem_cache_t *cp)
1090 {
1091 	mutex_enter(&cp->cache_depot_lock);
1092 	cp->cache_full.ml_reaplimit = cp->cache_full.ml_min;
1093 	cp->cache_full.ml_min = cp->cache_full.ml_total;
1094 	cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min;
1095 	cp->cache_empty.ml_min = cp->cache_empty.ml_total;
1096 	mutex_exit(&cp->cache_depot_lock);
1097 }
1098 
1099 /*
1100  * Reap all magazines that have fallen out of the depot's working set.
1101  */
1102 static void
1103 kmem_depot_ws_reap(kmem_cache_t *cp)
1104 {
1105 	long reap;
1106 	kmem_magazine_t *mp;
1107 
1108 	ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread));
1109 
1110 	reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
1111 	while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL)
1112 		kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize);
1113 
1114 	reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min);
1115 	while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL)
1116 		kmem_magazine_destroy(cp, mp, 0);
1117 }
1118 
1119 static void
1120 kmem_cpu_reload(kmem_cpu_cache_t *ccp, kmem_magazine_t *mp, int rounds)
1121 {
1122 	ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) ||
1123 	    (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize));
1124 	ASSERT(ccp->cc_magsize > 0);
1125 
1126 	ccp->cc_ploaded = ccp->cc_loaded;
1127 	ccp->cc_prounds = ccp->cc_rounds;
1128 	ccp->cc_loaded = mp;
1129 	ccp->cc_rounds = rounds;
1130 }
1131 
1132 /*
1133  * Allocate a constructed object from cache cp.
1134  */
1135 void *
1136 kmem_cache_alloc(kmem_cache_t *cp, int kmflag)
1137 {
1138 	kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp);
1139 	kmem_magazine_t *fmp;
1140 	void *buf;
1141 
1142 	mutex_enter(&ccp->cc_lock);
1143 	for (;;) {
1144 		/*
1145 		 * If there's an object available in the current CPU's
1146 		 * loaded magazine, just take it and return.
1147 		 */
1148 		if (ccp->cc_rounds > 0) {
1149 			buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds];
1150 			ccp->cc_alloc++;
1151 			mutex_exit(&ccp->cc_lock);
1152 			if ((ccp->cc_flags & KMF_BUFTAG) &&
1153 			    kmem_cache_alloc_debug(cp, buf, kmflag, 0,
1154 			    caller()) == -1) {
1155 				if (kmflag & KM_NOSLEEP)
1156 					return (NULL);
1157 				mutex_enter(&ccp->cc_lock);
1158 				continue;
1159 			}
1160 			return (buf);
1161 		}
1162 
1163 		/*
1164 		 * The loaded magazine is empty.  If the previously loaded
1165 		 * magazine was full, exchange them and try again.
1166 		 */
1167 		if (ccp->cc_prounds > 0) {
1168 			kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
1169 			continue;
1170 		}
1171 
1172 		/*
1173 		 * If the magazine layer is disabled, break out now.
1174 		 */
1175 		if (ccp->cc_magsize == 0)
1176 			break;
1177 
1178 		/*
1179 		 * Try to get a full magazine from the depot.
1180 		 */
1181 		fmp = kmem_depot_alloc(cp, &cp->cache_full);
1182 		if (fmp != NULL) {
1183 			if (ccp->cc_ploaded != NULL)
1184 				kmem_depot_free(cp, &cp->cache_empty,
1185 				    ccp->cc_ploaded);
1186 			kmem_cpu_reload(ccp, fmp, ccp->cc_magsize);
1187 			continue;
1188 		}
1189 
1190 		/*
1191 		 * There are no full magazines in the depot,
1192 		 * so fall through to the slab layer.
1193 		 */
1194 		break;
1195 	}
1196 	mutex_exit(&ccp->cc_lock);
1197 
1198 	/*
1199 	 * We couldn't allocate a constructed object from the magazine layer,
1200 	 * so get a raw buffer from the slab layer and apply its constructor.
1201 	 */
1202 	buf = kmem_slab_alloc(cp, kmflag);
1203 
1204 	if (buf == NULL)
1205 		return (NULL);
1206 
1207 	if (cp->cache_flags & KMF_BUFTAG) {
1208 		/*
1209 		 * Make kmem_cache_alloc_debug() apply the constructor for us.
1210 		 */
1211 		if (kmem_cache_alloc_debug(cp, buf, kmflag, 1,
1212 		    caller()) == -1) {
1213 			if (kmflag & KM_NOSLEEP)
1214 				return (NULL);
1215 			/*
1216 			 * kmem_cache_alloc_debug() detected corruption
1217 			 * but didn't panic (kmem_panic <= 0).  Try again.
1218 			 */
1219 			return (kmem_cache_alloc(cp, kmflag));
1220 		}
1221 		return (buf);
1222 	}
1223 
1224 	if (cp->cache_constructor != NULL &&
1225 	    cp->cache_constructor(buf, cp->cache_private, kmflag) != 0) {
1226 		atomic_add_64(&cp->cache_alloc_fail, 1);
1227 		kmem_slab_free(cp, buf);
1228 		return (NULL);
1229 	}
1230 
1231 	return (buf);
1232 }
1233 
1234 /*
1235  * Free a constructed object to cache cp.
1236  */
1237 void
1238 kmem_cache_free(kmem_cache_t *cp, void *buf)
1239 {
1240 	kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp);
1241 	kmem_magazine_t *emp;
1242 	kmem_magtype_t *mtp;
1243 
1244 	if (ccp->cc_flags & KMF_BUFTAG)
1245 		if (kmem_cache_free_debug(cp, buf, caller()) == -1)
1246 			return;
1247 
1248 	mutex_enter(&ccp->cc_lock);
1249 	for (;;) {
1250 		/*
1251 		 * If there's a slot available in the current CPU's
1252 		 * loaded magazine, just put the object there and return.
1253 		 */
1254 		if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) {
1255 			ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf;
1256 			ccp->cc_free++;
1257 			mutex_exit(&ccp->cc_lock);
1258 			return;
1259 		}
1260 
1261 		/*
1262 		 * The loaded magazine is full.  If the previously loaded
1263 		 * magazine was empty, exchange them and try again.
1264 		 */
1265 		if (ccp->cc_prounds == 0) {
1266 			kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds);
1267 			continue;
1268 		}
1269 
1270 		/*
1271 		 * If the magazine layer is disabled, break out now.
1272 		 */
1273 		if (ccp->cc_magsize == 0)
1274 			break;
1275 
1276 		/*
1277 		 * Try to get an empty magazine from the depot.
1278 		 */
1279 		emp = kmem_depot_alloc(cp, &cp->cache_empty);
1280 		if (emp != NULL) {
1281 			if (ccp->cc_ploaded != NULL)
1282 				kmem_depot_free(cp, &cp->cache_full,
1283 				    ccp->cc_ploaded);
1284 			kmem_cpu_reload(ccp, emp, 0);
1285 			continue;
1286 		}
1287 
1288 		/*
1289 		 * There are no empty magazines in the depot,
1290 		 * so try to allocate a new one.  We must drop all locks
1291 		 * across kmem_cache_alloc() because lower layers may
1292 		 * attempt to allocate from this cache.
1293 		 */
1294 		mtp = cp->cache_magtype;
1295 		mutex_exit(&ccp->cc_lock);
1296 		emp = kmem_cache_alloc(mtp->mt_cache, KM_NOSLEEP);
1297 		mutex_enter(&ccp->cc_lock);
1298 
1299 		if (emp != NULL) {
1300 			/*
1301 			 * We successfully allocated an empty magazine.
1302 			 * However, we had to drop ccp->cc_lock to do it,
1303 			 * so the cache's magazine size may have changed.
1304 			 * If so, free the magazine and try again.
1305 			 */
1306 			if (ccp->cc_magsize != mtp->mt_magsize) {
1307 				mutex_exit(&ccp->cc_lock);
1308 				kmem_cache_free(mtp->mt_cache, emp);
1309 				mutex_enter(&ccp->cc_lock);
1310 				continue;
1311 			}
1312 
1313 			/*
1314 			 * We got a magazine of the right size.  Add it to
1315 			 * the depot and try the whole dance again.
1316 			 */
1317 			kmem_depot_free(cp, &cp->cache_empty, emp);
1318 			continue;
1319 		}
1320 
1321 		/*
1322 		 * We couldn't allocate an empty magazine,
1323 		 * so fall through to the slab layer.
1324 		 */
1325 		break;
1326 	}
1327 	mutex_exit(&ccp->cc_lock);
1328 
1329 	/*
1330 	 * We couldn't free our constructed object to the magazine layer,
1331 	 * so apply its destructor and free it to the slab layer.
1332 	 * Note that if KMF_DEADBEEF is in effect and KMF_LITE is not,
1333 	 * kmem_cache_free_debug() will have already applied the destructor.
1334 	 */
1335 	if ((cp->cache_flags & (KMF_DEADBEEF | KMF_LITE)) != KMF_DEADBEEF &&
1336 	    cp->cache_destructor != NULL) {
1337 		if (cp->cache_flags & KMF_DEADBEEF) {	/* KMF_LITE implied */
1338 			kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
1339 			*(uint64_t *)buf = btp->bt_redzone;
1340 			cp->cache_destructor(buf, cp->cache_private);
1341 			*(uint64_t *)buf = KMEM_FREE_PATTERN;
1342 		} else {
1343 			cp->cache_destructor(buf, cp->cache_private);
1344 		}
1345 	}
1346 
1347 	kmem_slab_free(cp, buf);
1348 }
1349 
1350 void *
1351 kmem_zalloc(size_t size, int kmflag)
1352 {
1353 	size_t index = (size - 1) >> KMEM_ALIGN_SHIFT;
1354 	void *buf;
1355 
1356 	if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) {
1357 		kmem_cache_t *cp = kmem_alloc_table[index];
1358 		buf = kmem_cache_alloc(cp, kmflag);
1359 		if (buf != NULL) {
1360 			if (cp->cache_flags & KMF_BUFTAG) {
1361 				kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
1362 				((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE;
1363 				((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size);
1364 
1365 				if (cp->cache_flags & KMF_LITE) {
1366 					KMEM_BUFTAG_LITE_ENTER(btp,
1367 					    kmem_lite_count, caller());
1368 				}
1369 			}
1370 			bzero(buf, size);
1371 		}
1372 	} else {
1373 		buf = kmem_alloc(size, kmflag);
1374 		if (buf != NULL)
1375 			bzero(buf, size);
1376 	}
1377 	return (buf);
1378 }
1379 
1380 void *
1381 kmem_alloc(size_t size, int kmflag)
1382 {
1383 	size_t index = (size - 1) >> KMEM_ALIGN_SHIFT;
1384 	void *buf;
1385 
1386 	if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) {
1387 		kmem_cache_t *cp = kmem_alloc_table[index];
1388 		buf = kmem_cache_alloc(cp, kmflag);
1389 		if ((cp->cache_flags & KMF_BUFTAG) && buf != NULL) {
1390 			kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
1391 			((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE;
1392 			((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size);
1393 
1394 			if (cp->cache_flags & KMF_LITE) {
1395 				KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count,
1396 				    caller());
1397 			}
1398 		}
1399 		return (buf);
1400 	}
1401 	if (size == 0)
1402 		return (NULL);
1403 	buf = vmem_alloc(kmem_oversize_arena, size, kmflag & KM_VMFLAGS);
1404 	if (buf == NULL)
1405 		kmem_log_event(kmem_failure_log, NULL, NULL, (void *)size);
1406 	return (buf);
1407 }
1408 
1409 void
1410 kmem_free(void *buf, size_t size)
1411 {
1412 	size_t index = (size - 1) >> KMEM_ALIGN_SHIFT;
1413 
1414 	if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) {
1415 		kmem_cache_t *cp = kmem_alloc_table[index];
1416 		if (cp->cache_flags & KMF_BUFTAG) {
1417 			kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf);
1418 			uint32_t *ip = (uint32_t *)btp;
1419 			if (ip[1] != KMEM_SIZE_ENCODE(size)) {
1420 				if (*(uint64_t *)buf == KMEM_FREE_PATTERN) {
1421 					kmem_error(KMERR_DUPFREE, cp, buf);
1422 					return;
1423 				}
1424 				if (KMEM_SIZE_VALID(ip[1])) {
1425 					ip[0] = KMEM_SIZE_ENCODE(size);
1426 					kmem_error(KMERR_BADSIZE, cp, buf);
1427 				} else {
1428 					kmem_error(KMERR_REDZONE, cp, buf);
1429 				}
1430 				return;
1431 			}
1432 			if (((uint8_t *)buf)[size] != KMEM_REDZONE_BYTE) {
1433 				kmem_error(KMERR_REDZONE, cp, buf);
1434 				return;
1435 			}
1436 			btp->bt_redzone = KMEM_REDZONE_PATTERN;
1437 			if (cp->cache_flags & KMF_LITE) {
1438 				KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count,
1439 				    caller());
1440 			}
1441 		}
1442 		kmem_cache_free(cp, buf);
1443 	} else {
1444 		if (buf == NULL && size == 0)
1445 			return;
1446 		vmem_free(kmem_oversize_arena, buf, size);
1447 	}
1448 }
1449 
1450 void *
1451 kmem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag)
1452 {
1453 	size_t realsize = size + vmp->vm_quantum;
1454 	void *addr;
1455 
1456 	/*
1457 	 * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding
1458 	 * vm_quantum will cause integer wraparound.  Check for this, and
1459 	 * blow off the firewall page in this case.  Note that such a
1460 	 * giant allocation (the entire kernel address space) can never
1461 	 * be satisfied, so it will either fail immediately (VM_NOSLEEP)
1462 	 * or sleep forever (VM_SLEEP).  Thus, there is no need for a
1463 	 * corresponding check in kmem_firewall_va_free().
1464 	 */
1465 	if (realsize < size)
1466 		realsize = size;
1467 
1468 	/*
1469 	 * While boot still owns resource management, make sure that this
1470 	 * redzone virtual address allocation is properly accounted for in
1471 	 * OBPs "virtual-memory" "available" lists because we're
1472 	 * effectively claiming them for a red zone.  If we don't do this,
1473 	 * the available lists become too fragmented and too large for the
1474 	 * current boot/kernel memory list interface.
1475 	 */
1476 	addr = vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT);
1477 
1478 	if (addr != NULL && kvseg.s_base == NULL && realsize != size)
1479 		(void) boot_virt_alloc((char *)addr + size, vmp->vm_quantum);
1480 
1481 	return (addr);
1482 }
1483 
1484 void
1485 kmem_firewall_va_free(vmem_t *vmp, void *addr, size_t size)
1486 {
1487 	ASSERT((kvseg.s_base == NULL ?
1488 	    va_to_pfn((char *)addr + size) :
1489 	    hat_getpfnum(kas.a_hat, (caddr_t)addr + size)) == PFN_INVALID);
1490 
1491 	vmem_free(vmp, addr, size + vmp->vm_quantum);
1492 }
1493 
1494 /*
1495  * Try to allocate at least `size' bytes of memory without sleeping or
1496  * panicking. Return actual allocated size in `asize'. If allocation failed,
1497  * try final allocation with sleep or panic allowed.
1498  */
1499 void *
1500 kmem_alloc_tryhard(size_t size, size_t *asize, int kmflag)
1501 {
1502 	void *p;
1503 
1504 	*asize = P2ROUNDUP(size, KMEM_ALIGN);
1505 	do {
1506 		p = kmem_alloc(*asize, (kmflag | KM_NOSLEEP) & ~KM_PANIC);
1507 		if (p != NULL)
1508 			return (p);
1509 		*asize += KMEM_ALIGN;
1510 	} while (*asize <= PAGESIZE);
1511 
1512 	*asize = P2ROUNDUP(size, KMEM_ALIGN);
1513 	return (kmem_alloc(*asize, kmflag));
1514 }
1515 
1516 /*
1517  * Reclaim all unused memory from a cache.
1518  */
1519 static void
1520 kmem_cache_reap(kmem_cache_t *cp)
1521 {
1522 	/*
1523 	 * Ask the cache's owner to free some memory if possible.
1524 	 * The idea is to handle things like the inode cache, which
1525 	 * typically sits on a bunch of memory that it doesn't truly
1526 	 * *need*.  Reclaim policy is entirely up to the owner; this
1527 	 * callback is just an advisory plea for help.
1528 	 */
1529 	if (cp->cache_reclaim != NULL)
1530 		cp->cache_reclaim(cp->cache_private);
1531 
1532 	kmem_depot_ws_reap(cp);
1533 }
1534 
1535 static void
1536 kmem_reap_timeout(void *flag_arg)
1537 {
1538 	uint32_t *flag = (uint32_t *)flag_arg;
1539 
1540 	ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace);
1541 	*flag = 0;
1542 }
1543 
1544 static void
1545 kmem_reap_done(void *flag)
1546 {
1547 	(void) timeout(kmem_reap_timeout, flag, kmem_reap_interval);
1548 }
1549 
1550 static void
1551 kmem_reap_start(void *flag)
1552 {
1553 	ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace);
1554 
1555 	if (flag == &kmem_reaping) {
1556 		kmem_cache_applyall(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP);
1557 		/*
1558 		 * if we have segkp under heap, reap segkp cache.
1559 		 */
1560 		if (segkp_fromheap)
1561 			segkp_cache_free();
1562 	}
1563 	else
1564 		kmem_cache_applyall_id(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP);
1565 
1566 	/*
1567 	 * We use taskq_dispatch() to schedule a timeout to clear
1568 	 * the flag so that kmem_reap() becomes self-throttling:
1569 	 * we won't reap again until the current reap completes *and*
1570 	 * at least kmem_reap_interval ticks have elapsed.
1571 	 */
1572 	if (!taskq_dispatch(kmem_taskq, kmem_reap_done, flag, TQ_NOSLEEP))
1573 		kmem_reap_done(flag);
1574 }
1575 
1576 static void
1577 kmem_reap_common(void *flag_arg)
1578 {
1579 	uint32_t *flag = (uint32_t *)flag_arg;
1580 
1581 	if (MUTEX_HELD(&kmem_cache_lock) || kmem_taskq == NULL ||
1582 	    cas32(flag, 0, 1) != 0)
1583 		return;
1584 
1585 	/*
1586 	 * It may not be kosher to do memory allocation when a reap is called
1587 	 * is called (for example, if vmem_populate() is in the call chain).
1588 	 * So we start the reap going with a TQ_NOALLOC dispatch.  If the
1589 	 * dispatch fails, we reset the flag, and the next reap will try again.
1590 	 */
1591 	if (!taskq_dispatch(kmem_taskq, kmem_reap_start, flag, TQ_NOALLOC))
1592 		*flag = 0;
1593 }
1594 
1595 /*
1596  * Reclaim all unused memory from all caches.  Called from the VM system
1597  * when memory gets tight.
1598  */
1599 void
1600 kmem_reap(void)
1601 {
1602 	kmem_reap_common(&kmem_reaping);
1603 }
1604 
1605 /*
1606  * Reclaim all unused memory from identifier arenas, called when a vmem
1607  * arena not back by memory is exhausted.  Since reaping memory-backed caches
1608  * cannot help with identifier exhaustion, we avoid both a large amount of
1609  * work and unwanted side-effects from reclaim callbacks.
1610  */
1611 void
1612 kmem_reap_idspace(void)
1613 {
1614 	kmem_reap_common(&kmem_reaping_idspace);
1615 }
1616 
1617 /*
1618  * Purge all magazines from a cache and set its magazine limit to zero.
1619  * All calls are serialized by the kmem_taskq lock, except for the final
1620  * call from kmem_cache_destroy().
1621  */
1622 static void
1623 kmem_cache_magazine_purge(kmem_cache_t *cp)
1624 {
1625 	kmem_cpu_cache_t *ccp;
1626 	kmem_magazine_t *mp, *pmp;
1627 	int rounds, prounds, cpu_seqid;
1628 
1629 	ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread));
1630 	ASSERT(MUTEX_NOT_HELD(&cp->cache_lock));
1631 
1632 	for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
1633 		ccp = &cp->cache_cpu[cpu_seqid];
1634 
1635 		mutex_enter(&ccp->cc_lock);
1636 		mp = ccp->cc_loaded;
1637 		pmp = ccp->cc_ploaded;
1638 		rounds = ccp->cc_rounds;
1639 		prounds = ccp->cc_prounds;
1640 		ccp->cc_loaded = NULL;
1641 		ccp->cc_ploaded = NULL;
1642 		ccp->cc_rounds = -1;
1643 		ccp->cc_prounds = -1;
1644 		ccp->cc_magsize = 0;
1645 		mutex_exit(&ccp->cc_lock);
1646 
1647 		if (mp)
1648 			kmem_magazine_destroy(cp, mp, rounds);
1649 		if (pmp)
1650 			kmem_magazine_destroy(cp, pmp, prounds);
1651 	}
1652 
1653 	/*
1654 	 * Updating the working set statistics twice in a row has the
1655 	 * effect of setting the working set size to zero, so everything
1656 	 * is eligible for reaping.
1657 	 */
1658 	kmem_depot_ws_update(cp);
1659 	kmem_depot_ws_update(cp);
1660 
1661 	kmem_depot_ws_reap(cp);
1662 }
1663 
1664 /*
1665  * Enable per-cpu magazines on a cache.
1666  */
1667 static void
1668 kmem_cache_magazine_enable(kmem_cache_t *cp)
1669 {
1670 	int cpu_seqid;
1671 
1672 	if (cp->cache_flags & KMF_NOMAGAZINE)
1673 		return;
1674 
1675 	for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
1676 		kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
1677 		mutex_enter(&ccp->cc_lock);
1678 		ccp->cc_magsize = cp->cache_magtype->mt_magsize;
1679 		mutex_exit(&ccp->cc_lock);
1680 	}
1681 
1682 }
1683 
1684 /*
1685  * Reap (almost) everything right now.  See kmem_cache_magazine_purge()
1686  * for explanation of the back-to-back kmem_depot_ws_update() calls.
1687  */
1688 void
1689 kmem_cache_reap_now(kmem_cache_t *cp)
1690 {
1691 	kmem_depot_ws_update(cp);
1692 	kmem_depot_ws_update(cp);
1693 
1694 	(void) taskq_dispatch(kmem_taskq,
1695 	    (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
1696 	taskq_wait(kmem_taskq);
1697 }
1698 
1699 /*
1700  * Recompute a cache's magazine size.  The trade-off is that larger magazines
1701  * provide a higher transfer rate with the depot, while smaller magazines
1702  * reduce memory consumption.  Magazine resizing is an expensive operation;
1703  * it should not be done frequently.
1704  *
1705  * Changes to the magazine size are serialized by the kmem_taskq lock.
1706  *
1707  * Note: at present this only grows the magazine size.  It might be useful
1708  * to allow shrinkage too.
1709  */
1710 static void
1711 kmem_cache_magazine_resize(kmem_cache_t *cp)
1712 {
1713 	kmem_magtype_t *mtp = cp->cache_magtype;
1714 
1715 	ASSERT(taskq_member(kmem_taskq, curthread));
1716 
1717 	if (cp->cache_chunksize < mtp->mt_maxbuf) {
1718 		kmem_cache_magazine_purge(cp);
1719 		mutex_enter(&cp->cache_depot_lock);
1720 		cp->cache_magtype = ++mtp;
1721 		cp->cache_depot_contention_prev =
1722 		    cp->cache_depot_contention + INT_MAX;
1723 		mutex_exit(&cp->cache_depot_lock);
1724 		kmem_cache_magazine_enable(cp);
1725 	}
1726 }
1727 
1728 /*
1729  * Rescale a cache's hash table, so that the table size is roughly the
1730  * cache size.  We want the average lookup time to be extremely small.
1731  */
1732 static void
1733 kmem_hash_rescale(kmem_cache_t *cp)
1734 {
1735 	kmem_bufctl_t **old_table, **new_table, *bcp;
1736 	size_t old_size, new_size, h;
1737 
1738 	ASSERT(taskq_member(kmem_taskq, curthread));
1739 
1740 	new_size = MAX(KMEM_HASH_INITIAL,
1741 	    1 << (highbit(3 * cp->cache_buftotal + 4) - 2));
1742 	old_size = cp->cache_hash_mask + 1;
1743 
1744 	if ((old_size >> 1) <= new_size && new_size <= (old_size << 1))
1745 		return;
1746 
1747 	new_table = vmem_alloc(kmem_hash_arena, new_size * sizeof (void *),
1748 	    VM_NOSLEEP);
1749 	if (new_table == NULL)
1750 		return;
1751 	bzero(new_table, new_size * sizeof (void *));
1752 
1753 	mutex_enter(&cp->cache_lock);
1754 
1755 	old_size = cp->cache_hash_mask + 1;
1756 	old_table = cp->cache_hash_table;
1757 
1758 	cp->cache_hash_mask = new_size - 1;
1759 	cp->cache_hash_table = new_table;
1760 	cp->cache_rescale++;
1761 
1762 	for (h = 0; h < old_size; h++) {
1763 		bcp = old_table[h];
1764 		while (bcp != NULL) {
1765 			void *addr = bcp->bc_addr;
1766 			kmem_bufctl_t *next_bcp = bcp->bc_next;
1767 			kmem_bufctl_t **hash_bucket = KMEM_HASH(cp, addr);
1768 			bcp->bc_next = *hash_bucket;
1769 			*hash_bucket = bcp;
1770 			bcp = next_bcp;
1771 		}
1772 	}
1773 
1774 	mutex_exit(&cp->cache_lock);
1775 
1776 	vmem_free(kmem_hash_arena, old_table, old_size * sizeof (void *));
1777 }
1778 
1779 /*
1780  * Perform periodic maintenance on a cache: hash rescaling,
1781  * depot working-set update, and magazine resizing.
1782  */
1783 static void
1784 kmem_cache_update(kmem_cache_t *cp)
1785 {
1786 	int need_hash_rescale = 0;
1787 	int need_magazine_resize = 0;
1788 
1789 	ASSERT(MUTEX_HELD(&kmem_cache_lock));
1790 
1791 	/*
1792 	 * If the cache has become much larger or smaller than its hash table,
1793 	 * fire off a request to rescale the hash table.
1794 	 */
1795 	mutex_enter(&cp->cache_lock);
1796 
1797 	if ((cp->cache_flags & KMF_HASH) &&
1798 	    (cp->cache_buftotal > (cp->cache_hash_mask << 1) ||
1799 	    (cp->cache_buftotal < (cp->cache_hash_mask >> 1) &&
1800 	    cp->cache_hash_mask > KMEM_HASH_INITIAL)))
1801 		need_hash_rescale = 1;
1802 
1803 	mutex_exit(&cp->cache_lock);
1804 
1805 	/*
1806 	 * Update the depot working set statistics.
1807 	 */
1808 	kmem_depot_ws_update(cp);
1809 
1810 	/*
1811 	 * If there's a lot of contention in the depot,
1812 	 * increase the magazine size.
1813 	 */
1814 	mutex_enter(&cp->cache_depot_lock);
1815 
1816 	if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf &&
1817 	    (int)(cp->cache_depot_contention -
1818 	    cp->cache_depot_contention_prev) > kmem_depot_contention)
1819 		need_magazine_resize = 1;
1820 
1821 	cp->cache_depot_contention_prev = cp->cache_depot_contention;
1822 
1823 	mutex_exit(&cp->cache_depot_lock);
1824 
1825 	if (need_hash_rescale)
1826 		(void) taskq_dispatch(kmem_taskq,
1827 		    (task_func_t *)kmem_hash_rescale, cp, TQ_NOSLEEP);
1828 
1829 	if (need_magazine_resize)
1830 		(void) taskq_dispatch(kmem_taskq,
1831 		    (task_func_t *)kmem_cache_magazine_resize, cp, TQ_NOSLEEP);
1832 }
1833 
1834 static void
1835 kmem_update_timeout(void *dummy)
1836 {
1837 	static void kmem_update(void *);
1838 
1839 	(void) timeout(kmem_update, dummy, kmem_reap_interval);
1840 }
1841 
1842 static void
1843 kmem_update(void *dummy)
1844 {
1845 	kmem_cache_applyall(kmem_cache_update, NULL, TQ_NOSLEEP);
1846 
1847 	/*
1848 	 * We use taskq_dispatch() to reschedule the timeout so that
1849 	 * kmem_update() becomes self-throttling: it won't schedule
1850 	 * new tasks until all previous tasks have completed.
1851 	 */
1852 	if (!taskq_dispatch(kmem_taskq, kmem_update_timeout, dummy, TQ_NOSLEEP))
1853 		kmem_update_timeout(NULL);
1854 }
1855 
1856 static int
1857 kmem_cache_kstat_update(kstat_t *ksp, int rw)
1858 {
1859 	struct kmem_cache_kstat *kmcp = &kmem_cache_kstat;
1860 	kmem_cache_t *cp = ksp->ks_private;
1861 	kmem_slab_t *sp;
1862 	uint64_t cpu_buf_avail;
1863 	uint64_t buf_avail = 0;
1864 	int cpu_seqid;
1865 
1866 	ASSERT(MUTEX_HELD(&kmem_cache_kstat_lock));
1867 
1868 	if (rw == KSTAT_WRITE)
1869 		return (EACCES);
1870 
1871 	mutex_enter(&cp->cache_lock);
1872 
1873 	kmcp->kmc_alloc_fail.value.ui64		= cp->cache_alloc_fail;
1874 	kmcp->kmc_alloc.value.ui64		= cp->cache_slab_alloc;
1875 	kmcp->kmc_free.value.ui64		= cp->cache_slab_free;
1876 	kmcp->kmc_slab_alloc.value.ui64		= cp->cache_slab_alloc;
1877 	kmcp->kmc_slab_free.value.ui64		= cp->cache_slab_free;
1878 
1879 	for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
1880 		kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
1881 
1882 		mutex_enter(&ccp->cc_lock);
1883 
1884 		cpu_buf_avail = 0;
1885 		if (ccp->cc_rounds > 0)
1886 			cpu_buf_avail += ccp->cc_rounds;
1887 		if (ccp->cc_prounds > 0)
1888 			cpu_buf_avail += ccp->cc_prounds;
1889 
1890 		kmcp->kmc_alloc.value.ui64	+= ccp->cc_alloc;
1891 		kmcp->kmc_free.value.ui64	+= ccp->cc_free;
1892 		buf_avail			+= cpu_buf_avail;
1893 
1894 		mutex_exit(&ccp->cc_lock);
1895 	}
1896 
1897 	mutex_enter(&cp->cache_depot_lock);
1898 
1899 	kmcp->kmc_depot_alloc.value.ui64	= cp->cache_full.ml_alloc;
1900 	kmcp->kmc_depot_free.value.ui64		= cp->cache_empty.ml_alloc;
1901 	kmcp->kmc_depot_contention.value.ui64	= cp->cache_depot_contention;
1902 	kmcp->kmc_full_magazines.value.ui64	= cp->cache_full.ml_total;
1903 	kmcp->kmc_empty_magazines.value.ui64	= cp->cache_empty.ml_total;
1904 	kmcp->kmc_magazine_size.value.ui64	=
1905 	    (cp->cache_flags & KMF_NOMAGAZINE) ?
1906 	    0 : cp->cache_magtype->mt_magsize;
1907 
1908 	kmcp->kmc_alloc.value.ui64		+= cp->cache_full.ml_alloc;
1909 	kmcp->kmc_free.value.ui64		+= cp->cache_empty.ml_alloc;
1910 	buf_avail += cp->cache_full.ml_total * cp->cache_magtype->mt_magsize;
1911 
1912 	mutex_exit(&cp->cache_depot_lock);
1913 
1914 	kmcp->kmc_buf_size.value.ui64	= cp->cache_bufsize;
1915 	kmcp->kmc_align.value.ui64	= cp->cache_align;
1916 	kmcp->kmc_chunk_size.value.ui64	= cp->cache_chunksize;
1917 	kmcp->kmc_slab_size.value.ui64	= cp->cache_slabsize;
1918 	kmcp->kmc_buf_constructed.value.ui64 = buf_avail;
1919 	for (sp = cp->cache_freelist; sp != &cp->cache_nullslab;
1920 	    sp = sp->slab_next)
1921 		buf_avail += sp->slab_chunks - sp->slab_refcnt;
1922 	kmcp->kmc_buf_avail.value.ui64	= buf_avail;
1923 	kmcp->kmc_buf_inuse.value.ui64	= cp->cache_buftotal - buf_avail;
1924 	kmcp->kmc_buf_total.value.ui64	= cp->cache_buftotal;
1925 	kmcp->kmc_buf_max.value.ui64	= cp->cache_bufmax;
1926 	kmcp->kmc_slab_create.value.ui64	= cp->cache_slab_create;
1927 	kmcp->kmc_slab_destroy.value.ui64	= cp->cache_slab_destroy;
1928 	kmcp->kmc_hash_size.value.ui64	= (cp->cache_flags & KMF_HASH) ?
1929 	    cp->cache_hash_mask + 1 : 0;
1930 	kmcp->kmc_hash_lookup_depth.value.ui64	= cp->cache_lookup_depth;
1931 	kmcp->kmc_hash_rescale.value.ui64	= cp->cache_rescale;
1932 	kmcp->kmc_vmem_source.value.ui64	= cp->cache_arena->vm_id;
1933 
1934 	mutex_exit(&cp->cache_lock);
1935 	return (0);
1936 }
1937 
1938 /*
1939  * Return a named statistic about a particular cache.
1940  * This shouldn't be called very often, so it's currently designed for
1941  * simplicity (leverages existing kstat support) rather than efficiency.
1942  */
1943 uint64_t
1944 kmem_cache_stat(kmem_cache_t *cp, char *name)
1945 {
1946 	int i;
1947 	kstat_t *ksp = cp->cache_kstat;
1948 	kstat_named_t *knp = (kstat_named_t *)&kmem_cache_kstat;
1949 	uint64_t value = 0;
1950 
1951 	if (ksp != NULL) {
1952 		mutex_enter(&kmem_cache_kstat_lock);
1953 		(void) kmem_cache_kstat_update(ksp, KSTAT_READ);
1954 		for (i = 0; i < ksp->ks_ndata; i++) {
1955 			if (strcmp(knp[i].name, name) == 0) {
1956 				value = knp[i].value.ui64;
1957 				break;
1958 			}
1959 		}
1960 		mutex_exit(&kmem_cache_kstat_lock);
1961 	}
1962 	return (value);
1963 }
1964 
1965 /*
1966  * Return an estimate of currently available kernel heap memory.
1967  * On 32-bit systems, physical memory may exceed virtual memory,
1968  * we just truncate the result at 1GB.
1969  */
1970 size_t
1971 kmem_avail(void)
1972 {
1973 	spgcnt_t rmem = availrmem - tune.t_minarmem;
1974 	spgcnt_t fmem = freemem - minfree;
1975 
1976 	return ((size_t)ptob(MIN(MAX(MIN(rmem, fmem), 0),
1977 	    1 << (30 - PAGESHIFT))));
1978 }
1979 
1980 /*
1981  * Return the maximum amount of memory that is (in theory) allocatable
1982  * from the heap. This may be used as an estimate only since there
1983  * is no guarentee this space will still be available when an allocation
1984  * request is made, nor that the space may be allocated in one big request
1985  * due to kernel heap fragmentation.
1986  */
1987 size_t
1988 kmem_maxavail(void)
1989 {
1990 	spgcnt_t pmem = availrmem - tune.t_minarmem;
1991 	spgcnt_t vmem = btop(vmem_size(heap_arena, VMEM_FREE));
1992 
1993 	return ((size_t)ptob(MAX(MIN(pmem, vmem), 0)));
1994 }
1995 
1996 /*
1997  * Indicate whether memory-intensive kmem debugging is enabled.
1998  */
1999 int
2000 kmem_debugging(void)
2001 {
2002 	return (kmem_flags & (KMF_AUDIT | KMF_REDZONE));
2003 }
2004 
2005 kmem_cache_t *
2006 kmem_cache_create(
2007 	char *name,		/* descriptive name for this cache */
2008 	size_t bufsize,		/* size of the objects it manages */
2009 	size_t align,		/* required object alignment */
2010 	int (*constructor)(void *, void *, int), /* object constructor */
2011 	void (*destructor)(void *, void *),	/* object destructor */
2012 	void (*reclaim)(void *), /* memory reclaim callback */
2013 	void *private,		/* pass-thru arg for constr/destr/reclaim */
2014 	vmem_t *vmp,		/* vmem source for slab allocation */
2015 	int cflags)		/* cache creation flags */
2016 {
2017 	int cpu_seqid;
2018 	size_t chunksize;
2019 	kmem_cache_t *cp, *cnext, *cprev;
2020 	kmem_magtype_t *mtp;
2021 	size_t csize = KMEM_CACHE_SIZE(max_ncpus);
2022 
2023 #ifdef	DEBUG
2024 	/*
2025 	 * Cache names should conform to the rules for valid C identifiers
2026 	 */
2027 	if (!strident_valid(name)) {
2028 		cmn_err(CE_CONT,
2029 		    "kmem_cache_create: '%s' is an invalid cache name\n"
2030 		    "cache names must conform to the rules for "
2031 		    "C identifiers\n", name);
2032 	}
2033 #endif	/* DEBUG */
2034 
2035 	if (vmp == NULL)
2036 		vmp = kmem_default_arena;
2037 
2038 	/*
2039 	 * If this kmem cache has an identifier vmem arena as its source, mark
2040 	 * it such to allow kmem_reap_idspace().
2041 	 */
2042 	ASSERT(!(cflags & KMC_IDENTIFIER));   /* consumer should not set this */
2043 	if (vmp->vm_cflags & VMC_IDENTIFIER)
2044 		cflags |= KMC_IDENTIFIER;
2045 
2046 	/*
2047 	 * Get a kmem_cache structure.  We arrange that cp->cache_cpu[]
2048 	 * is aligned on a KMEM_CPU_CACHE_SIZE boundary to prevent
2049 	 * false sharing of per-CPU data.
2050 	 */
2051 	cp = vmem_xalloc(kmem_cache_arena, csize, KMEM_CPU_CACHE_SIZE,
2052 	    P2NPHASE(csize, KMEM_CPU_CACHE_SIZE), 0, NULL, NULL, VM_SLEEP);
2053 	bzero(cp, csize);
2054 
2055 	if (align == 0)
2056 		align = KMEM_ALIGN;
2057 
2058 	/*
2059 	 * If we're not at least KMEM_ALIGN aligned, we can't use free
2060 	 * memory to hold bufctl information (because we can't safely
2061 	 * perform word loads and stores on it).
2062 	 */
2063 	if (align < KMEM_ALIGN)
2064 		cflags |= KMC_NOTOUCH;
2065 
2066 	if ((align & (align - 1)) != 0 || align > vmp->vm_quantum)
2067 		panic("kmem_cache_create: bad alignment %lu", align);
2068 
2069 	mutex_enter(&kmem_flags_lock);
2070 	if (kmem_flags & KMF_RANDOMIZE)
2071 		kmem_flags = (((kmem_flags | ~KMF_RANDOM) + 1) & KMF_RANDOM) |
2072 		    KMF_RANDOMIZE;
2073 	cp->cache_flags = (kmem_flags | cflags) & KMF_DEBUG;
2074 	mutex_exit(&kmem_flags_lock);
2075 
2076 	/*
2077 	 * Make sure all the various flags are reasonable.
2078 	 */
2079 	ASSERT(!(cflags & KMC_NOHASH) || !(cflags & KMC_NOTOUCH));
2080 
2081 	if (cp->cache_flags & KMF_LITE) {
2082 		if (bufsize >= kmem_lite_minsize &&
2083 		    align <= kmem_lite_maxalign &&
2084 		    P2PHASE(bufsize, kmem_lite_maxalign) != 0) {
2085 			cp->cache_flags |= KMF_BUFTAG;
2086 			cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL);
2087 		} else {
2088 			cp->cache_flags &= ~KMF_DEBUG;
2089 		}
2090 	}
2091 
2092 	if (cp->cache_flags & KMF_DEADBEEF)
2093 		cp->cache_flags |= KMF_REDZONE;
2094 
2095 	if ((cflags & KMC_QCACHE) && (cp->cache_flags & KMF_AUDIT))
2096 		cp->cache_flags |= KMF_NOMAGAZINE;
2097 
2098 	if (cflags & KMC_NODEBUG)
2099 		cp->cache_flags &= ~KMF_DEBUG;
2100 
2101 	if (cflags & KMC_NOTOUCH)
2102 		cp->cache_flags &= ~KMF_TOUCH;
2103 
2104 	if (cflags & KMC_NOHASH)
2105 		cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL);
2106 
2107 	if (cflags & KMC_NOMAGAZINE)
2108 		cp->cache_flags |= KMF_NOMAGAZINE;
2109 
2110 	if ((cp->cache_flags & KMF_AUDIT) && !(cflags & KMC_NOTOUCH))
2111 		cp->cache_flags |= KMF_REDZONE;
2112 
2113 	if (!(cp->cache_flags & KMF_AUDIT))
2114 		cp->cache_flags &= ~KMF_CONTENTS;
2115 
2116 	if ((cp->cache_flags & KMF_BUFTAG) && bufsize >= kmem_minfirewall &&
2117 	    !(cp->cache_flags & KMF_LITE) && !(cflags & KMC_NOHASH))
2118 		cp->cache_flags |= KMF_FIREWALL;
2119 
2120 	if (vmp != kmem_default_arena || kmem_firewall_arena == NULL)
2121 		cp->cache_flags &= ~KMF_FIREWALL;
2122 
2123 	if (cp->cache_flags & KMF_FIREWALL) {
2124 		cp->cache_flags &= ~KMF_BUFTAG;
2125 		cp->cache_flags |= KMF_NOMAGAZINE;
2126 		ASSERT(vmp == kmem_default_arena);
2127 		vmp = kmem_firewall_arena;
2128 	}
2129 
2130 	/*
2131 	 * Set cache properties.
2132 	 */
2133 	(void) strncpy(cp->cache_name, name, KMEM_CACHE_NAMELEN);
2134 	strident_canon(cp->cache_name, KMEM_CACHE_NAMELEN);
2135 	cp->cache_bufsize = bufsize;
2136 	cp->cache_align = align;
2137 	cp->cache_constructor = constructor;
2138 	cp->cache_destructor = destructor;
2139 	cp->cache_reclaim = reclaim;
2140 	cp->cache_private = private;
2141 	cp->cache_arena = vmp;
2142 	cp->cache_cflags = cflags;
2143 
2144 	/*
2145 	 * Determine the chunk size.
2146 	 */
2147 	chunksize = bufsize;
2148 
2149 	if (align >= KMEM_ALIGN) {
2150 		chunksize = P2ROUNDUP(chunksize, KMEM_ALIGN);
2151 		cp->cache_bufctl = chunksize - KMEM_ALIGN;
2152 	}
2153 
2154 	if (cp->cache_flags & KMF_BUFTAG) {
2155 		cp->cache_bufctl = chunksize;
2156 		cp->cache_buftag = chunksize;
2157 		if (cp->cache_flags & KMF_LITE)
2158 			chunksize += KMEM_BUFTAG_LITE_SIZE(kmem_lite_count);
2159 		else
2160 			chunksize += sizeof (kmem_buftag_t);
2161 	}
2162 
2163 	if (cp->cache_flags & KMF_DEADBEEF) {
2164 		cp->cache_verify = MIN(cp->cache_buftag, kmem_maxverify);
2165 		if (cp->cache_flags & KMF_LITE)
2166 			cp->cache_verify = sizeof (uint64_t);
2167 	}
2168 
2169 	cp->cache_contents = MIN(cp->cache_bufctl, kmem_content_maxsave);
2170 
2171 	cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align);
2172 
2173 	/*
2174 	 * Now that we know the chunk size, determine the optimal slab size.
2175 	 */
2176 	if (vmp == kmem_firewall_arena) {
2177 		cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum);
2178 		cp->cache_mincolor = cp->cache_slabsize - chunksize;
2179 		cp->cache_maxcolor = cp->cache_mincolor;
2180 		cp->cache_flags |= KMF_HASH;
2181 		ASSERT(!(cp->cache_flags & KMF_BUFTAG));
2182 	} else if ((cflags & KMC_NOHASH) || (!(cflags & KMC_NOTOUCH) &&
2183 	    !(cp->cache_flags & KMF_AUDIT) &&
2184 	    chunksize < vmp->vm_quantum / KMEM_VOID_FRACTION)) {
2185 		cp->cache_slabsize = vmp->vm_quantum;
2186 		cp->cache_mincolor = 0;
2187 		cp->cache_maxcolor =
2188 		    (cp->cache_slabsize - sizeof (kmem_slab_t)) % chunksize;
2189 		ASSERT(chunksize + sizeof (kmem_slab_t) <= cp->cache_slabsize);
2190 		ASSERT(!(cp->cache_flags & KMF_AUDIT));
2191 	} else {
2192 		size_t chunks, bestfit, waste, slabsize;
2193 		size_t minwaste = LONG_MAX;
2194 
2195 		for (chunks = 1; chunks <= KMEM_VOID_FRACTION; chunks++) {
2196 			slabsize = P2ROUNDUP(chunksize * chunks,
2197 			    vmp->vm_quantum);
2198 			chunks = slabsize / chunksize;
2199 			waste = (slabsize % chunksize) / chunks;
2200 			if (waste < minwaste) {
2201 				minwaste = waste;
2202 				bestfit = slabsize;
2203 			}
2204 		}
2205 		if (cflags & KMC_QCACHE)
2206 			bestfit = VMEM_QCACHE_SLABSIZE(vmp->vm_qcache_max);
2207 		cp->cache_slabsize = bestfit;
2208 		cp->cache_mincolor = 0;
2209 		cp->cache_maxcolor = bestfit % chunksize;
2210 		cp->cache_flags |= KMF_HASH;
2211 	}
2212 
2213 	if (cp->cache_flags & KMF_HASH) {
2214 		ASSERT(!(cflags & KMC_NOHASH));
2215 		cp->cache_bufctl_cache = (cp->cache_flags & KMF_AUDIT) ?
2216 		    kmem_bufctl_audit_cache : kmem_bufctl_cache;
2217 	}
2218 
2219 	if (cp->cache_maxcolor >= vmp->vm_quantum)
2220 		cp->cache_maxcolor = vmp->vm_quantum - 1;
2221 
2222 	cp->cache_color = cp->cache_mincolor;
2223 
2224 	/*
2225 	 * Initialize the rest of the slab layer.
2226 	 */
2227 	mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL);
2228 
2229 	cp->cache_freelist = &cp->cache_nullslab;
2230 	cp->cache_nullslab.slab_cache = cp;
2231 	cp->cache_nullslab.slab_refcnt = -1;
2232 	cp->cache_nullslab.slab_next = &cp->cache_nullslab;
2233 	cp->cache_nullslab.slab_prev = &cp->cache_nullslab;
2234 
2235 	if (cp->cache_flags & KMF_HASH) {
2236 		cp->cache_hash_table = vmem_alloc(kmem_hash_arena,
2237 		    KMEM_HASH_INITIAL * sizeof (void *), VM_SLEEP);
2238 		bzero(cp->cache_hash_table,
2239 		    KMEM_HASH_INITIAL * sizeof (void *));
2240 		cp->cache_hash_mask = KMEM_HASH_INITIAL - 1;
2241 		cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1;
2242 	}
2243 
2244 	/*
2245 	 * Initialize the depot.
2246 	 */
2247 	mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL);
2248 
2249 	for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++)
2250 		continue;
2251 
2252 	cp->cache_magtype = mtp;
2253 
2254 	/*
2255 	 * Initialize the CPU layer.
2256 	 */
2257 	for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
2258 		kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
2259 		mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL);
2260 		ccp->cc_flags = cp->cache_flags;
2261 		ccp->cc_rounds = -1;
2262 		ccp->cc_prounds = -1;
2263 	}
2264 
2265 	/*
2266 	 * Create the cache's kstats.
2267 	 */
2268 	if ((cp->cache_kstat = kstat_create("unix", 0, cp->cache_name,
2269 	    "kmem_cache", KSTAT_TYPE_NAMED,
2270 	    sizeof (kmem_cache_kstat) / sizeof (kstat_named_t),
2271 	    KSTAT_FLAG_VIRTUAL)) != NULL) {
2272 		cp->cache_kstat->ks_data = &kmem_cache_kstat;
2273 		cp->cache_kstat->ks_update = kmem_cache_kstat_update;
2274 		cp->cache_kstat->ks_private = cp;
2275 		cp->cache_kstat->ks_lock = &kmem_cache_kstat_lock;
2276 		kstat_install(cp->cache_kstat);
2277 	}
2278 
2279 	/*
2280 	 * Add the cache to the global list.  This makes it visible
2281 	 * to kmem_update(), so the cache must be ready for business.
2282 	 */
2283 	mutex_enter(&kmem_cache_lock);
2284 	cp->cache_next = cnext = &kmem_null_cache;
2285 	cp->cache_prev = cprev = kmem_null_cache.cache_prev;
2286 	cnext->cache_prev = cp;
2287 	cprev->cache_next = cp;
2288 	mutex_exit(&kmem_cache_lock);
2289 
2290 	if (kmem_ready)
2291 		kmem_cache_magazine_enable(cp);
2292 
2293 	return (cp);
2294 }
2295 
2296 void
2297 kmem_cache_destroy(kmem_cache_t *cp)
2298 {
2299 	int cpu_seqid;
2300 
2301 	/*
2302 	 * Remove the cache from the global cache list so that no one else
2303 	 * can schedule tasks on its behalf, wait for any pending tasks to
2304 	 * complete, purge the cache, and then destroy it.
2305 	 */
2306 	mutex_enter(&kmem_cache_lock);
2307 	cp->cache_prev->cache_next = cp->cache_next;
2308 	cp->cache_next->cache_prev = cp->cache_prev;
2309 	cp->cache_prev = cp->cache_next = NULL;
2310 	mutex_exit(&kmem_cache_lock);
2311 
2312 	if (kmem_taskq != NULL)
2313 		taskq_wait(kmem_taskq);
2314 
2315 	kmem_cache_magazine_purge(cp);
2316 
2317 	mutex_enter(&cp->cache_lock);
2318 	if (cp->cache_buftotal != 0)
2319 		cmn_err(CE_WARN, "kmem_cache_destroy: '%s' (%p) not empty",
2320 		    cp->cache_name, (void *)cp);
2321 	cp->cache_reclaim = NULL;
2322 	/*
2323 	 * The cache is now dead.  There should be no further activity.
2324 	 * We enforce this by setting land mines in the constructor and
2325 	 * destructor routines that induce a kernel text fault if invoked.
2326 	 */
2327 	cp->cache_constructor = (int (*)(void *, void *, int))1;
2328 	cp->cache_destructor = (void (*)(void *, void *))2;
2329 	mutex_exit(&cp->cache_lock);
2330 
2331 	kstat_delete(cp->cache_kstat);
2332 
2333 	if (cp->cache_hash_table != NULL)
2334 		vmem_free(kmem_hash_arena, cp->cache_hash_table,
2335 		    (cp->cache_hash_mask + 1) * sizeof (void *));
2336 
2337 	for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++)
2338 		mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock);
2339 
2340 	mutex_destroy(&cp->cache_depot_lock);
2341 	mutex_destroy(&cp->cache_lock);
2342 
2343 	vmem_free(kmem_cache_arena, cp, KMEM_CACHE_SIZE(max_ncpus));
2344 }
2345 
2346 /*ARGSUSED*/
2347 static int
2348 kmem_cpu_setup(cpu_setup_t what, int id, void *arg)
2349 {
2350 	ASSERT(MUTEX_HELD(&cpu_lock));
2351 	if (what == CPU_UNCONFIG) {
2352 		kmem_cache_applyall(kmem_cache_magazine_purge,
2353 		    kmem_taskq, TQ_SLEEP);
2354 		kmem_cache_applyall(kmem_cache_magazine_enable,
2355 		    kmem_taskq, TQ_SLEEP);
2356 	}
2357 	return (0);
2358 }
2359 
2360 static void
2361 kmem_cache_init(int pass, int use_large_pages)
2362 {
2363 	int i;
2364 	size_t size;
2365 	kmem_cache_t *cp;
2366 	kmem_magtype_t *mtp;
2367 	char name[KMEM_CACHE_NAMELEN + 1];
2368 
2369 	for (i = 0; i < sizeof (kmem_magtype) / sizeof (*mtp); i++) {
2370 		mtp = &kmem_magtype[i];
2371 		(void) sprintf(name, "kmem_magazine_%d", mtp->mt_magsize);
2372 		mtp->mt_cache = kmem_cache_create(name,
2373 		    (mtp->mt_magsize + 1) * sizeof (void *),
2374 		    mtp->mt_align, NULL, NULL, NULL, NULL,
2375 		    kmem_msb_arena, KMC_NOHASH);
2376 	}
2377 
2378 	kmem_slab_cache = kmem_cache_create("kmem_slab_cache",
2379 	    sizeof (kmem_slab_t), 0, NULL, NULL, NULL, NULL,
2380 	    kmem_msb_arena, KMC_NOHASH);
2381 
2382 	kmem_bufctl_cache = kmem_cache_create("kmem_bufctl_cache",
2383 	    sizeof (kmem_bufctl_t), 0, NULL, NULL, NULL, NULL,
2384 	    kmem_msb_arena, KMC_NOHASH);
2385 
2386 	kmem_bufctl_audit_cache = kmem_cache_create("kmem_bufctl_audit_cache",
2387 	    sizeof (kmem_bufctl_audit_t), 0, NULL, NULL, NULL, NULL,
2388 	    kmem_msb_arena, KMC_NOHASH);
2389 
2390 	if (pass == 2) {
2391 		kmem_va_arena = vmem_create("kmem_va",
2392 		    NULL, 0, PAGESIZE,
2393 		    vmem_alloc, vmem_free, heap_arena,
2394 		    8 * PAGESIZE, VM_SLEEP);
2395 
2396 		if (use_large_pages) {
2397 			kmem_default_arena = vmem_xcreate("kmem_default",
2398 			    NULL, 0, PAGESIZE,
2399 			    segkmem_alloc_lp, segkmem_free_lp, kmem_va_arena,
2400 			    0, VM_SLEEP);
2401 		} else {
2402 			kmem_default_arena = vmem_create("kmem_default",
2403 			    NULL, 0, PAGESIZE,
2404 			    segkmem_alloc, segkmem_free, kmem_va_arena,
2405 			    0, VM_SLEEP);
2406 		}
2407 	} else {
2408 		/*
2409 		 * During the first pass, the kmem_alloc_* caches
2410 		 * are treated as metadata.
2411 		 */
2412 		kmem_default_arena = kmem_msb_arena;
2413 	}
2414 
2415 	/*
2416 	 * Set up the default caches to back kmem_alloc()
2417 	 */
2418 	size = KMEM_ALIGN;
2419 	for (i = 0; i < sizeof (kmem_alloc_sizes) / sizeof (int); i++) {
2420 		size_t align = KMEM_ALIGN;
2421 		size_t cache_size = kmem_alloc_sizes[i];
2422 		/*
2423 		 * If they allocate a multiple of the coherency granularity,
2424 		 * they get a coherency-granularity-aligned address.
2425 		 */
2426 		if (IS_P2ALIGNED(cache_size, 64))
2427 			align = 64;
2428 		if (IS_P2ALIGNED(cache_size, PAGESIZE))
2429 			align = PAGESIZE;
2430 		(void) sprintf(name, "kmem_alloc_%lu", cache_size);
2431 		cp = kmem_cache_create(name, cache_size, align,
2432 		    NULL, NULL, NULL, NULL, NULL, KMC_KMEM_ALLOC);
2433 		while (size <= cache_size) {
2434 			kmem_alloc_table[(size - 1) >> KMEM_ALIGN_SHIFT] = cp;
2435 			size += KMEM_ALIGN;
2436 		}
2437 	}
2438 }
2439 
2440 void
2441 kmem_init(void)
2442 {
2443 	kmem_cache_t *cp;
2444 	int old_kmem_flags = kmem_flags;
2445 	int use_large_pages = 0;
2446 	size_t maxverify, minfirewall;
2447 
2448 	kstat_init();
2449 
2450 	/*
2451 	 * Small-memory systems (< 24 MB) can't handle kmem_flags overhead.
2452 	 */
2453 	if (physmem < btop(24 << 20) && !(old_kmem_flags & KMF_STICKY))
2454 		kmem_flags = 0;
2455 
2456 	/*
2457 	 * Don't do firewalled allocations if the heap is less than 1TB
2458 	 * (i.e. on a 32-bit kernel)
2459 	 * The resulting VM_NEXTFIT allocations would create too much
2460 	 * fragmentation in a small heap.
2461 	 */
2462 #if defined(_LP64)
2463 	maxverify = minfirewall = PAGESIZE / 2;
2464 #else
2465 	maxverify = minfirewall = ULONG_MAX;
2466 #endif
2467 
2468 	/* LINTED */
2469 	ASSERT(sizeof (kmem_cpu_cache_t) == KMEM_CPU_CACHE_SIZE);
2470 
2471 	kmem_null_cache.cache_next = &kmem_null_cache;
2472 	kmem_null_cache.cache_prev = &kmem_null_cache;
2473 
2474 	kmem_metadata_arena = vmem_create("kmem_metadata", NULL, 0, PAGESIZE,
2475 	    vmem_alloc, vmem_free, heap_arena, 8 * PAGESIZE,
2476 	    VM_SLEEP | VMC_NO_QCACHE);
2477 
2478 	kmem_msb_arena = vmem_create("kmem_msb", NULL, 0,
2479 	    PAGESIZE, segkmem_alloc, segkmem_free, kmem_metadata_arena, 0,
2480 	    VM_SLEEP);
2481 
2482 	kmem_cache_arena = vmem_create("kmem_cache", NULL, 0, KMEM_ALIGN,
2483 	    segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP);
2484 
2485 	kmem_hash_arena = vmem_create("kmem_hash", NULL, 0, KMEM_ALIGN,
2486 	    segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP);
2487 
2488 	kmem_log_arena = vmem_create("kmem_log", NULL, 0, KMEM_ALIGN,
2489 	    segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
2490 
2491 	kmem_firewall_va_arena = vmem_create("kmem_firewall_va",
2492 	    NULL, 0, PAGESIZE,
2493 	    kmem_firewall_va_alloc, kmem_firewall_va_free, heap_arena,
2494 	    0, VM_SLEEP);
2495 
2496 	kmem_firewall_arena = vmem_create("kmem_firewall", NULL, 0, PAGESIZE,
2497 	    segkmem_alloc, segkmem_free, kmem_firewall_va_arena, 0, VM_SLEEP);
2498 
2499 	/* temporary oversize arena for mod_read_system_file */
2500 	kmem_oversize_arena = vmem_create("kmem_oversize", NULL, 0, PAGESIZE,
2501 	    segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
2502 
2503 	kmem_null_cache.cache_next = &kmem_null_cache;
2504 	kmem_null_cache.cache_prev = &kmem_null_cache;
2505 
2506 	kmem_reap_interval = 15 * hz;
2507 
2508 	/*
2509 	 * Read /etc/system.  This is a chicken-and-egg problem because
2510 	 * kmem_flags may be set in /etc/system, but mod_read_system_file()
2511 	 * needs to use the allocator.  The simplest solution is to create
2512 	 * all the standard kmem caches, read /etc/system, destroy all the
2513 	 * caches we just created, and then create them all again in light
2514 	 * of the (possibly) new kmem_flags and other kmem tunables.
2515 	 */
2516 	kmem_cache_init(1, 0);
2517 
2518 	mod_read_system_file(boothowto & RB_ASKNAME);
2519 
2520 	while ((cp = kmem_null_cache.cache_prev) != &kmem_null_cache)
2521 		kmem_cache_destroy(cp);
2522 
2523 	vmem_destroy(kmem_oversize_arena);
2524 
2525 	if (old_kmem_flags & KMF_STICKY)
2526 		kmem_flags = old_kmem_flags;
2527 
2528 	if (!(kmem_flags & KMF_AUDIT))
2529 		vmem_seg_size = offsetof(vmem_seg_t, vs_thread);
2530 
2531 	if (kmem_maxverify == 0)
2532 		kmem_maxverify = maxverify;
2533 
2534 	if (kmem_minfirewall == 0)
2535 		kmem_minfirewall = minfirewall;
2536 
2537 	/*
2538 	 * give segkmem a chance to figure out if we are using large pages
2539 	 * for the kernel heap
2540 	 */
2541 	use_large_pages = segkmem_lpsetup();
2542 
2543 	/*
2544 	 * To protect against corruption, we keep the actual number of callers
2545 	 * KMF_LITE records seperate from the tunable.  We arbitrarily clamp
2546 	 * to 16, since the overhead for small buffers quickly gets out of
2547 	 * hand.
2548 	 *
2549 	 * The real limit would depend on the needs of the largest KMC_NOHASH
2550 	 * cache.
2551 	 */
2552 	kmem_lite_count = MIN(MAX(0, kmem_lite_pcs), 16);
2553 	kmem_lite_pcs = kmem_lite_count;
2554 
2555 	/*
2556 	 * Normally, we firewall oversized allocations when possible, but
2557 	 * if we are using large pages for kernel memory, and we don't have
2558 	 * any non-LITE debugging flags set, we want to allocate oversized
2559 	 * buffers from large pages, and so skip the firewalling.
2560 	 */
2561 	if (use_large_pages &&
2562 	    ((kmem_flags & KMF_LITE) || !(kmem_flags & KMF_DEBUG))) {
2563 		kmem_oversize_arena = vmem_xcreate("kmem_oversize", NULL, 0,
2564 		    PAGESIZE, segkmem_alloc_lp, segkmem_free_lp, heap_arena,
2565 		    0, VM_SLEEP);
2566 	} else {
2567 		kmem_oversize_arena = vmem_create("kmem_oversize",
2568 		    NULL, 0, PAGESIZE,
2569 		    segkmem_alloc, segkmem_free, kmem_minfirewall < ULONG_MAX?
2570 		    kmem_firewall_va_arena : heap_arena, 0, VM_SLEEP);
2571 	}
2572 
2573 	kmem_cache_init(2, use_large_pages);
2574 
2575 	if (kmem_flags & (KMF_AUDIT | KMF_RANDOMIZE)) {
2576 		if (kmem_transaction_log_size == 0)
2577 			kmem_transaction_log_size = kmem_maxavail() / 50;
2578 		kmem_transaction_log = kmem_log_init(kmem_transaction_log_size);
2579 	}
2580 
2581 	if (kmem_flags & (KMF_CONTENTS | KMF_RANDOMIZE)) {
2582 		if (kmem_content_log_size == 0)
2583 			kmem_content_log_size = kmem_maxavail() / 50;
2584 		kmem_content_log = kmem_log_init(kmem_content_log_size);
2585 	}
2586 
2587 	kmem_failure_log = kmem_log_init(kmem_failure_log_size);
2588 
2589 	kmem_slab_log = kmem_log_init(kmem_slab_log_size);
2590 
2591 	/*
2592 	 * Initialize STREAMS message caches so allocb() is available.
2593 	 * This allows us to initialize the logging framework (cmn_err(9F),
2594 	 * strlog(9F), etc) so we can start recording messages.
2595 	 */
2596 	streams_msg_init();
2597 	/*
2598 	 * Initialize the ZSD framework in Zones so modules loaded henceforth
2599 	 * can register their callbacks.
2600 	 */
2601 	zone_zsd_init();
2602 	log_init();
2603 	taskq_init();
2604 
2605 	kmem_cache_applyall(kmem_cache_magazine_enable, NULL, TQ_SLEEP);
2606 
2607 	kmem_ready = 1;
2608 
2609 	/*
2610 	 * Initialize the platform-specific aligned/DMA memory allocator.
2611 	 */
2612 	ka_init();
2613 
2614 	/*
2615 	 * Initialize 32-bit ID cache.
2616 	 */
2617 	id32_init();
2618 }
2619 
2620 void
2621 kmem_thread_init(void)
2622 {
2623 	kmem_taskq = taskq_create_instance("kmem_taskq", 0, 1, minclsyspri,
2624 	    300, INT_MAX, TASKQ_PREPOPULATE);
2625 }
2626 
2627 void
2628 kmem_mp_init(void)
2629 {
2630 	mutex_enter(&cpu_lock);
2631 	register_cpu_setup_func(kmem_cpu_setup, NULL);
2632 	mutex_exit(&cpu_lock);
2633 
2634 	kmem_update_timeout(NULL);
2635 }
2636