1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * Kernel memory allocator, as described in the following two papers: 31 * 32 * Jeff Bonwick, 33 * The Slab Allocator: An Object-Caching Kernel Memory Allocator. 34 * Proceedings of the Summer 1994 Usenix Conference. 35 * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf. 36 * 37 * Jeff Bonwick and Jonathan Adams, 38 * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 39 * Arbitrary Resources. 40 * Proceedings of the 2001 Usenix Conference. 41 * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf. 42 */ 43 44 #include <sys/kmem_impl.h> 45 #include <sys/vmem_impl.h> 46 #include <sys/param.h> 47 #include <sys/sysmacros.h> 48 #include <sys/vm.h> 49 #include <sys/proc.h> 50 #include <sys/tuneable.h> 51 #include <sys/systm.h> 52 #include <sys/cmn_err.h> 53 #include <sys/debug.h> 54 #include <sys/mutex.h> 55 #include <sys/bitmap.h> 56 #include <sys/atomic.h> 57 #include <sys/kobj.h> 58 #include <sys/disp.h> 59 #include <vm/seg_kmem.h> 60 #include <sys/log.h> 61 #include <sys/callb.h> 62 #include <sys/taskq.h> 63 #include <sys/modctl.h> 64 #include <sys/reboot.h> 65 #include <sys/id32.h> 66 #include <sys/zone.h> 67 68 extern void streams_msg_init(void); 69 extern int segkp_fromheap; 70 extern void segkp_cache_free(void); 71 72 struct kmem_cache_kstat { 73 kstat_named_t kmc_buf_size; 74 kstat_named_t kmc_align; 75 kstat_named_t kmc_chunk_size; 76 kstat_named_t kmc_slab_size; 77 kstat_named_t kmc_alloc; 78 kstat_named_t kmc_alloc_fail; 79 kstat_named_t kmc_free; 80 kstat_named_t kmc_depot_alloc; 81 kstat_named_t kmc_depot_free; 82 kstat_named_t kmc_depot_contention; 83 kstat_named_t kmc_slab_alloc; 84 kstat_named_t kmc_slab_free; 85 kstat_named_t kmc_buf_constructed; 86 kstat_named_t kmc_buf_avail; 87 kstat_named_t kmc_buf_inuse; 88 kstat_named_t kmc_buf_total; 89 kstat_named_t kmc_buf_max; 90 kstat_named_t kmc_slab_create; 91 kstat_named_t kmc_slab_destroy; 92 kstat_named_t kmc_vmem_source; 93 kstat_named_t kmc_hash_size; 94 kstat_named_t kmc_hash_lookup_depth; 95 kstat_named_t kmc_hash_rescale; 96 kstat_named_t kmc_full_magazines; 97 kstat_named_t kmc_empty_magazines; 98 kstat_named_t kmc_magazine_size; 99 } kmem_cache_kstat = { 100 { "buf_size", KSTAT_DATA_UINT64 }, 101 { "align", KSTAT_DATA_UINT64 }, 102 { "chunk_size", KSTAT_DATA_UINT64 }, 103 { "slab_size", KSTAT_DATA_UINT64 }, 104 { "alloc", KSTAT_DATA_UINT64 }, 105 { "alloc_fail", KSTAT_DATA_UINT64 }, 106 { "free", KSTAT_DATA_UINT64 }, 107 { "depot_alloc", KSTAT_DATA_UINT64 }, 108 { "depot_free", KSTAT_DATA_UINT64 }, 109 { "depot_contention", KSTAT_DATA_UINT64 }, 110 { "slab_alloc", KSTAT_DATA_UINT64 }, 111 { "slab_free", KSTAT_DATA_UINT64 }, 112 { "buf_constructed", KSTAT_DATA_UINT64 }, 113 { "buf_avail", KSTAT_DATA_UINT64 }, 114 { "buf_inuse", KSTAT_DATA_UINT64 }, 115 { "buf_total", KSTAT_DATA_UINT64 }, 116 { "buf_max", KSTAT_DATA_UINT64 }, 117 { "slab_create", KSTAT_DATA_UINT64 }, 118 { "slab_destroy", KSTAT_DATA_UINT64 }, 119 { "vmem_source", KSTAT_DATA_UINT64 }, 120 { "hash_size", KSTAT_DATA_UINT64 }, 121 { "hash_lookup_depth", KSTAT_DATA_UINT64 }, 122 { "hash_rescale", KSTAT_DATA_UINT64 }, 123 { "full_magazines", KSTAT_DATA_UINT64 }, 124 { "empty_magazines", KSTAT_DATA_UINT64 }, 125 { "magazine_size", KSTAT_DATA_UINT64 }, 126 }; 127 128 static kmutex_t kmem_cache_kstat_lock; 129 130 /* 131 * The default set of caches to back kmem_alloc(). 132 * These sizes should be reevaluated periodically. 133 * 134 * We want allocations that are multiples of the coherency granularity 135 * (64 bytes) to be satisfied from a cache which is a multiple of 64 136 * bytes, so that it will be 64-byte aligned. For all multiples of 64, 137 * the next kmem_cache_size greater than or equal to it must be a 138 * multiple of 64. 139 */ 140 static const int kmem_alloc_sizes[] = { 141 1 * 8, 142 2 * 8, 143 3 * 8, 144 4 * 8, 5 * 8, 6 * 8, 7 * 8, 145 4 * 16, 5 * 16, 6 * 16, 7 * 16, 146 4 * 32, 5 * 32, 6 * 32, 7 * 32, 147 4 * 64, 5 * 64, 6 * 64, 7 * 64, 148 4 * 128, 5 * 128, 6 * 128, 7 * 128, 149 P2ALIGN(8192 / 7, 64), 150 P2ALIGN(8192 / 6, 64), 151 P2ALIGN(8192 / 5, 64), 152 P2ALIGN(8192 / 4, 64), 153 P2ALIGN(8192 / 3, 64), 154 P2ALIGN(8192 / 2, 64), 155 P2ALIGN(8192 / 1, 64), 156 4096 * 3, 157 8192 * 2, 158 }; 159 160 #define KMEM_MAXBUF 16384 161 162 static kmem_cache_t *kmem_alloc_table[KMEM_MAXBUF >> KMEM_ALIGN_SHIFT]; 163 164 static kmem_magtype_t kmem_magtype[] = { 165 { 1, 8, 3200, 65536 }, 166 { 3, 16, 256, 32768 }, 167 { 7, 32, 64, 16384 }, 168 { 15, 64, 0, 8192 }, 169 { 31, 64, 0, 4096 }, 170 { 47, 64, 0, 2048 }, 171 { 63, 64, 0, 1024 }, 172 { 95, 64, 0, 512 }, 173 { 143, 64, 0, 0 }, 174 }; 175 176 static uint32_t kmem_reaping; 177 static uint32_t kmem_reaping_idspace; 178 179 /* 180 * kmem tunables 181 */ 182 clock_t kmem_reap_interval; /* cache reaping rate [15 * HZ ticks] */ 183 int kmem_depot_contention = 3; /* max failed tryenters per real interval */ 184 pgcnt_t kmem_reapahead = 0; /* start reaping N pages before pageout */ 185 int kmem_panic = 1; /* whether to panic on error */ 186 int kmem_logging = 1; /* kmem_log_enter() override */ 187 uint32_t kmem_mtbf = 0; /* mean time between failures [default: off] */ 188 size_t kmem_transaction_log_size; /* transaction log size [2% of memory] */ 189 size_t kmem_content_log_size; /* content log size [2% of memory] */ 190 size_t kmem_failure_log_size; /* failure log [4 pages per CPU] */ 191 size_t kmem_slab_log_size; /* slab create log [4 pages per CPU] */ 192 size_t kmem_content_maxsave = 256; /* KMF_CONTENTS max bytes to log */ 193 size_t kmem_lite_minsize = 0; /* minimum buffer size for KMF_LITE */ 194 size_t kmem_lite_maxalign = 1024; /* maximum buffer alignment for KMF_LITE */ 195 int kmem_lite_pcs = 4; /* number of PCs to store in KMF_LITE mode */ 196 size_t kmem_maxverify; /* maximum bytes to inspect in debug routines */ 197 size_t kmem_minfirewall; /* hardware-enforced redzone threshold */ 198 199 #ifdef DEBUG 200 int kmem_flags = KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | KMF_CONTENTS; 201 #else 202 int kmem_flags = 0; 203 #endif 204 int kmem_ready; 205 206 static kmem_cache_t *kmem_slab_cache; 207 static kmem_cache_t *kmem_bufctl_cache; 208 static kmem_cache_t *kmem_bufctl_audit_cache; 209 210 static kmutex_t kmem_cache_lock; /* inter-cache linkage only */ 211 kmem_cache_t kmem_null_cache; 212 213 static taskq_t *kmem_taskq; 214 static kmutex_t kmem_flags_lock; 215 static vmem_t *kmem_metadata_arena; 216 static vmem_t *kmem_msb_arena; /* arena for metadata caches */ 217 static vmem_t *kmem_cache_arena; 218 static vmem_t *kmem_hash_arena; 219 static vmem_t *kmem_log_arena; 220 static vmem_t *kmem_oversize_arena; 221 static vmem_t *kmem_va_arena; 222 static vmem_t *kmem_default_arena; 223 static vmem_t *kmem_firewall_va_arena; 224 static vmem_t *kmem_firewall_arena; 225 226 kmem_log_header_t *kmem_transaction_log; 227 kmem_log_header_t *kmem_content_log; 228 kmem_log_header_t *kmem_failure_log; 229 kmem_log_header_t *kmem_slab_log; 230 231 static int kmem_lite_count; /* # of PCs in kmem_buftag_lite_t */ 232 233 #define KMEM_BUFTAG_LITE_ENTER(bt, count, caller) \ 234 if ((count) > 0) { \ 235 pc_t *_s = ((kmem_buftag_lite_t *)(bt))->bt_history; \ 236 pc_t *_e; \ 237 /* memmove() the old entries down one notch */ \ 238 for (_e = &_s[(count) - 1]; _e > _s; _e--) \ 239 *_e = *(_e - 1); \ 240 *_s = (uintptr_t)(caller); \ 241 } 242 243 #define KMERR_MODIFIED 0 /* buffer modified while on freelist */ 244 #define KMERR_REDZONE 1 /* redzone violation (write past end of buf) */ 245 #define KMERR_DUPFREE 2 /* freed a buffer twice */ 246 #define KMERR_BADADDR 3 /* freed a bad (unallocated) address */ 247 #define KMERR_BADBUFTAG 4 /* buftag corrupted */ 248 #define KMERR_BADBUFCTL 5 /* bufctl corrupted */ 249 #define KMERR_BADCACHE 6 /* freed a buffer to the wrong cache */ 250 #define KMERR_BADSIZE 7 /* alloc size != free size */ 251 #define KMERR_BADBASE 8 /* buffer base address wrong */ 252 253 struct { 254 hrtime_t kmp_timestamp; /* timestamp of panic */ 255 int kmp_error; /* type of kmem error */ 256 void *kmp_buffer; /* buffer that induced panic */ 257 void *kmp_realbuf; /* real start address for buffer */ 258 kmem_cache_t *kmp_cache; /* buffer's cache according to client */ 259 kmem_cache_t *kmp_realcache; /* actual cache containing buffer */ 260 kmem_slab_t *kmp_slab; /* slab accoring to kmem_findslab() */ 261 kmem_bufctl_t *kmp_bufctl; /* bufctl */ 262 } kmem_panic_info; 263 264 265 static void 266 copy_pattern(uint64_t pattern, void *buf_arg, size_t size) 267 { 268 uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 269 uint64_t *buf = buf_arg; 270 271 while (buf < bufend) 272 *buf++ = pattern; 273 } 274 275 static void * 276 verify_pattern(uint64_t pattern, void *buf_arg, size_t size) 277 { 278 uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 279 uint64_t *buf; 280 281 for (buf = buf_arg; buf < bufend; buf++) 282 if (*buf != pattern) 283 return (buf); 284 return (NULL); 285 } 286 287 static void * 288 verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size) 289 { 290 uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 291 uint64_t *buf; 292 293 for (buf = buf_arg; buf < bufend; buf++) { 294 if (*buf != old) { 295 copy_pattern(old, buf_arg, 296 (char *)buf - (char *)buf_arg); 297 return (buf); 298 } 299 *buf = new; 300 } 301 302 return (NULL); 303 } 304 305 static void 306 kmem_cache_applyall(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 307 { 308 kmem_cache_t *cp; 309 310 mutex_enter(&kmem_cache_lock); 311 for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache; 312 cp = cp->cache_next) 313 if (tq != NULL) 314 (void) taskq_dispatch(tq, (task_func_t *)func, cp, 315 tqflag); 316 else 317 func(cp); 318 mutex_exit(&kmem_cache_lock); 319 } 320 321 static void 322 kmem_cache_applyall_id(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 323 { 324 kmem_cache_t *cp; 325 326 mutex_enter(&kmem_cache_lock); 327 for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache; 328 cp = cp->cache_next) { 329 if (!(cp->cache_cflags & KMC_IDENTIFIER)) 330 continue; 331 if (tq != NULL) 332 (void) taskq_dispatch(tq, (task_func_t *)func, cp, 333 tqflag); 334 else 335 func(cp); 336 } 337 mutex_exit(&kmem_cache_lock); 338 } 339 340 /* 341 * Debugging support. Given a buffer address, find its slab. 342 */ 343 static kmem_slab_t * 344 kmem_findslab(kmem_cache_t *cp, void *buf) 345 { 346 kmem_slab_t *sp; 347 348 mutex_enter(&cp->cache_lock); 349 for (sp = cp->cache_nullslab.slab_next; 350 sp != &cp->cache_nullslab; sp = sp->slab_next) { 351 if (KMEM_SLAB_MEMBER(sp, buf)) { 352 mutex_exit(&cp->cache_lock); 353 return (sp); 354 } 355 } 356 mutex_exit(&cp->cache_lock); 357 358 return (NULL); 359 } 360 361 static void 362 kmem_error(int error, kmem_cache_t *cparg, void *bufarg) 363 { 364 kmem_buftag_t *btp = NULL; 365 kmem_bufctl_t *bcp = NULL; 366 kmem_cache_t *cp = cparg; 367 kmem_slab_t *sp; 368 uint64_t *off; 369 void *buf = bufarg; 370 371 kmem_logging = 0; /* stop logging when a bad thing happens */ 372 373 kmem_panic_info.kmp_timestamp = gethrtime(); 374 375 sp = kmem_findslab(cp, buf); 376 if (sp == NULL) { 377 for (cp = kmem_null_cache.cache_prev; cp != &kmem_null_cache; 378 cp = cp->cache_prev) { 379 if ((sp = kmem_findslab(cp, buf)) != NULL) 380 break; 381 } 382 } 383 384 if (sp == NULL) { 385 cp = NULL; 386 error = KMERR_BADADDR; 387 } else { 388 if (cp != cparg) 389 error = KMERR_BADCACHE; 390 else 391 buf = (char *)bufarg - ((uintptr_t)bufarg - 392 (uintptr_t)sp->slab_base) % cp->cache_chunksize; 393 if (buf != bufarg) 394 error = KMERR_BADBASE; 395 if (cp->cache_flags & KMF_BUFTAG) 396 btp = KMEM_BUFTAG(cp, buf); 397 if (cp->cache_flags & KMF_HASH) { 398 mutex_enter(&cp->cache_lock); 399 for (bcp = *KMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) 400 if (bcp->bc_addr == buf) 401 break; 402 mutex_exit(&cp->cache_lock); 403 if (bcp == NULL && btp != NULL) 404 bcp = btp->bt_bufctl; 405 if (kmem_findslab(cp->cache_bufctl_cache, bcp) == 406 NULL || P2PHASE((uintptr_t)bcp, KMEM_ALIGN) || 407 bcp->bc_addr != buf) { 408 error = KMERR_BADBUFCTL; 409 bcp = NULL; 410 } 411 } 412 } 413 414 kmem_panic_info.kmp_error = error; 415 kmem_panic_info.kmp_buffer = bufarg; 416 kmem_panic_info.kmp_realbuf = buf; 417 kmem_panic_info.kmp_cache = cparg; 418 kmem_panic_info.kmp_realcache = cp; 419 kmem_panic_info.kmp_slab = sp; 420 kmem_panic_info.kmp_bufctl = bcp; 421 422 printf("kernel memory allocator: "); 423 424 switch (error) { 425 426 case KMERR_MODIFIED: 427 printf("buffer modified after being freed\n"); 428 off = verify_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 429 if (off == NULL) /* shouldn't happen */ 430 off = buf; 431 printf("modification occurred at offset 0x%lx " 432 "(0x%llx replaced by 0x%llx)\n", 433 (uintptr_t)off - (uintptr_t)buf, 434 (longlong_t)KMEM_FREE_PATTERN, (longlong_t)*off); 435 break; 436 437 case KMERR_REDZONE: 438 printf("redzone violation: write past end of buffer\n"); 439 break; 440 441 case KMERR_BADADDR: 442 printf("invalid free: buffer not in cache\n"); 443 break; 444 445 case KMERR_DUPFREE: 446 printf("duplicate free: buffer freed twice\n"); 447 break; 448 449 case KMERR_BADBUFTAG: 450 printf("boundary tag corrupted\n"); 451 printf("bcp ^ bxstat = %lx, should be %lx\n", 452 (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat, 453 KMEM_BUFTAG_FREE); 454 break; 455 456 case KMERR_BADBUFCTL: 457 printf("bufctl corrupted\n"); 458 break; 459 460 case KMERR_BADCACHE: 461 printf("buffer freed to wrong cache\n"); 462 printf("buffer was allocated from %s,\n", cp->cache_name); 463 printf("caller attempting free to %s.\n", cparg->cache_name); 464 break; 465 466 case KMERR_BADSIZE: 467 printf("bad free: free size (%u) != alloc size (%u)\n", 468 KMEM_SIZE_DECODE(((uint32_t *)btp)[0]), 469 KMEM_SIZE_DECODE(((uint32_t *)btp)[1])); 470 break; 471 472 case KMERR_BADBASE: 473 printf("bad free: free address (%p) != alloc address (%p)\n", 474 bufarg, buf); 475 break; 476 } 477 478 printf("buffer=%p bufctl=%p cache: %s\n", 479 bufarg, (void *)bcp, cparg->cache_name); 480 481 if (bcp != NULL && (cp->cache_flags & KMF_AUDIT) && 482 error != KMERR_BADBUFCTL) { 483 int d; 484 timestruc_t ts; 485 kmem_bufctl_audit_t *bcap = (kmem_bufctl_audit_t *)bcp; 486 487 hrt2ts(kmem_panic_info.kmp_timestamp - bcap->bc_timestamp, &ts); 488 printf("previous transaction on buffer %p:\n", buf); 489 printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n", 490 (void *)bcap->bc_thread, ts.tv_sec, ts.tv_nsec, 491 (void *)sp, cp->cache_name); 492 for (d = 0; d < MIN(bcap->bc_depth, KMEM_STACK_DEPTH); d++) { 493 ulong_t off; 494 char *sym = kobj_getsymname(bcap->bc_stack[d], &off); 495 printf("%s+%lx\n", sym ? sym : "?", off); 496 } 497 } 498 if (kmem_panic > 0) 499 panic("kernel heap corruption detected"); 500 if (kmem_panic == 0) 501 debug_enter(NULL); 502 kmem_logging = 1; /* resume logging */ 503 } 504 505 static kmem_log_header_t * 506 kmem_log_init(size_t logsize) 507 { 508 kmem_log_header_t *lhp; 509 int nchunks = 4 * max_ncpus; 510 size_t lhsize = (size_t)&((kmem_log_header_t *)0)->lh_cpu[max_ncpus]; 511 int i; 512 513 /* 514 * Make sure that lhp->lh_cpu[] is nicely aligned 515 * to prevent false sharing of cache lines. 516 */ 517 lhsize = P2ROUNDUP(lhsize, KMEM_ALIGN); 518 lhp = vmem_xalloc(kmem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0, 519 NULL, NULL, VM_SLEEP); 520 bzero(lhp, lhsize); 521 522 mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL); 523 lhp->lh_nchunks = nchunks; 524 lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE); 525 lhp->lh_base = vmem_alloc(kmem_log_arena, 526 lhp->lh_chunksize * nchunks, VM_SLEEP); 527 lhp->lh_free = vmem_alloc(kmem_log_arena, 528 nchunks * sizeof (int), VM_SLEEP); 529 bzero(lhp->lh_base, lhp->lh_chunksize * nchunks); 530 531 for (i = 0; i < max_ncpus; i++) { 532 kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; 533 mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL); 534 clhp->clh_chunk = i; 535 } 536 537 for (i = max_ncpus; i < nchunks; i++) 538 lhp->lh_free[i] = i; 539 540 lhp->lh_head = max_ncpus; 541 lhp->lh_tail = 0; 542 543 return (lhp); 544 } 545 546 static void * 547 kmem_log_enter(kmem_log_header_t *lhp, void *data, size_t size) 548 { 549 void *logspace; 550 kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[CPU->cpu_seqid]; 551 552 if (lhp == NULL || kmem_logging == 0 || panicstr) 553 return (NULL); 554 555 mutex_enter(&clhp->clh_lock); 556 clhp->clh_hits++; 557 if (size > clhp->clh_avail) { 558 mutex_enter(&lhp->lh_lock); 559 lhp->lh_hits++; 560 lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk; 561 lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks; 562 clhp->clh_chunk = lhp->lh_free[lhp->lh_head]; 563 lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks; 564 clhp->clh_current = lhp->lh_base + 565 clhp->clh_chunk * lhp->lh_chunksize; 566 clhp->clh_avail = lhp->lh_chunksize; 567 if (size > lhp->lh_chunksize) 568 size = lhp->lh_chunksize; 569 mutex_exit(&lhp->lh_lock); 570 } 571 logspace = clhp->clh_current; 572 clhp->clh_current += size; 573 clhp->clh_avail -= size; 574 bcopy(data, logspace, size); 575 mutex_exit(&clhp->clh_lock); 576 return (logspace); 577 } 578 579 #define KMEM_AUDIT(lp, cp, bcp) \ 580 { \ 581 kmem_bufctl_audit_t *_bcp = (kmem_bufctl_audit_t *)(bcp); \ 582 _bcp->bc_timestamp = gethrtime(); \ 583 _bcp->bc_thread = curthread; \ 584 _bcp->bc_depth = getpcstack(_bcp->bc_stack, KMEM_STACK_DEPTH); \ 585 _bcp->bc_lastlog = kmem_log_enter((lp), _bcp, sizeof (*_bcp)); \ 586 } 587 588 static void 589 kmem_log_event(kmem_log_header_t *lp, kmem_cache_t *cp, 590 kmem_slab_t *sp, void *addr) 591 { 592 kmem_bufctl_audit_t bca; 593 594 bzero(&bca, sizeof (kmem_bufctl_audit_t)); 595 bca.bc_addr = addr; 596 bca.bc_slab = sp; 597 bca.bc_cache = cp; 598 KMEM_AUDIT(lp, cp, &bca); 599 } 600 601 /* 602 * Create a new slab for cache cp. 603 */ 604 static kmem_slab_t * 605 kmem_slab_create(kmem_cache_t *cp, int kmflag) 606 { 607 size_t slabsize = cp->cache_slabsize; 608 size_t chunksize = cp->cache_chunksize; 609 int cache_flags = cp->cache_flags; 610 size_t color, chunks; 611 char *buf, *slab; 612 kmem_slab_t *sp; 613 kmem_bufctl_t *bcp; 614 vmem_t *vmp = cp->cache_arena; 615 616 color = cp->cache_color + cp->cache_align; 617 if (color > cp->cache_maxcolor) 618 color = cp->cache_mincolor; 619 cp->cache_color = color; 620 621 slab = vmem_alloc(vmp, slabsize, kmflag & KM_VMFLAGS); 622 623 if (slab == NULL) 624 goto vmem_alloc_failure; 625 626 ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0); 627 628 if (!(cp->cache_cflags & KMC_NOTOUCH)) 629 copy_pattern(KMEM_UNINITIALIZED_PATTERN, slab, slabsize); 630 631 if (cache_flags & KMF_HASH) { 632 if ((sp = kmem_cache_alloc(kmem_slab_cache, kmflag)) == NULL) 633 goto slab_alloc_failure; 634 chunks = (slabsize - color) / chunksize; 635 } else { 636 sp = KMEM_SLAB(cp, slab); 637 chunks = (slabsize - sizeof (kmem_slab_t) - color) / chunksize; 638 } 639 640 sp->slab_cache = cp; 641 sp->slab_head = NULL; 642 sp->slab_refcnt = 0; 643 sp->slab_base = buf = slab + color; 644 sp->slab_chunks = chunks; 645 646 ASSERT(chunks > 0); 647 while (chunks-- != 0) { 648 if (cache_flags & KMF_HASH) { 649 bcp = kmem_cache_alloc(cp->cache_bufctl_cache, kmflag); 650 if (bcp == NULL) 651 goto bufctl_alloc_failure; 652 if (cache_flags & KMF_AUDIT) { 653 kmem_bufctl_audit_t *bcap = 654 (kmem_bufctl_audit_t *)bcp; 655 bzero(bcap, sizeof (kmem_bufctl_audit_t)); 656 bcap->bc_cache = cp; 657 } 658 bcp->bc_addr = buf; 659 bcp->bc_slab = sp; 660 } else { 661 bcp = KMEM_BUFCTL(cp, buf); 662 } 663 if (cache_flags & KMF_BUFTAG) { 664 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 665 btp->bt_redzone = KMEM_REDZONE_PATTERN; 666 btp->bt_bufctl = bcp; 667 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 668 if (cache_flags & KMF_DEADBEEF) { 669 copy_pattern(KMEM_FREE_PATTERN, buf, 670 cp->cache_verify); 671 } 672 } 673 bcp->bc_next = sp->slab_head; 674 sp->slab_head = bcp; 675 buf += chunksize; 676 } 677 678 kmem_log_event(kmem_slab_log, cp, sp, slab); 679 680 return (sp); 681 682 bufctl_alloc_failure: 683 684 while ((bcp = sp->slab_head) != NULL) { 685 sp->slab_head = bcp->bc_next; 686 kmem_cache_free(cp->cache_bufctl_cache, bcp); 687 } 688 kmem_cache_free(kmem_slab_cache, sp); 689 690 slab_alloc_failure: 691 692 vmem_free(vmp, slab, slabsize); 693 694 vmem_alloc_failure: 695 696 kmem_log_event(kmem_failure_log, cp, NULL, NULL); 697 atomic_add_64(&cp->cache_alloc_fail, 1); 698 699 return (NULL); 700 } 701 702 /* 703 * Destroy a slab. 704 */ 705 static void 706 kmem_slab_destroy(kmem_cache_t *cp, kmem_slab_t *sp) 707 { 708 vmem_t *vmp = cp->cache_arena; 709 void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum); 710 711 if (cp->cache_flags & KMF_HASH) { 712 kmem_bufctl_t *bcp; 713 while ((bcp = sp->slab_head) != NULL) { 714 sp->slab_head = bcp->bc_next; 715 kmem_cache_free(cp->cache_bufctl_cache, bcp); 716 } 717 kmem_cache_free(kmem_slab_cache, sp); 718 } 719 vmem_free(vmp, slab, cp->cache_slabsize); 720 } 721 722 /* 723 * Allocate a raw (unconstructed) buffer from cp's slab layer. 724 */ 725 static void * 726 kmem_slab_alloc(kmem_cache_t *cp, int kmflag) 727 { 728 kmem_bufctl_t *bcp, **hash_bucket; 729 kmem_slab_t *sp; 730 void *buf; 731 732 mutex_enter(&cp->cache_lock); 733 cp->cache_slab_alloc++; 734 sp = cp->cache_freelist; 735 ASSERT(sp->slab_cache == cp); 736 if (sp->slab_head == NULL) { 737 /* 738 * The freelist is empty. Create a new slab. 739 */ 740 mutex_exit(&cp->cache_lock); 741 if ((sp = kmem_slab_create(cp, kmflag)) == NULL) 742 return (NULL); 743 mutex_enter(&cp->cache_lock); 744 cp->cache_slab_create++; 745 if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) 746 cp->cache_bufmax = cp->cache_buftotal; 747 sp->slab_next = cp->cache_freelist; 748 sp->slab_prev = cp->cache_freelist->slab_prev; 749 sp->slab_next->slab_prev = sp; 750 sp->slab_prev->slab_next = sp; 751 cp->cache_freelist = sp; 752 } 753 754 sp->slab_refcnt++; 755 ASSERT(sp->slab_refcnt <= sp->slab_chunks); 756 757 /* 758 * If we're taking the last buffer in the slab, 759 * remove the slab from the cache's freelist. 760 */ 761 bcp = sp->slab_head; 762 if ((sp->slab_head = bcp->bc_next) == NULL) { 763 cp->cache_freelist = sp->slab_next; 764 ASSERT(sp->slab_refcnt == sp->slab_chunks); 765 } 766 767 if (cp->cache_flags & KMF_HASH) { 768 /* 769 * Add buffer to allocated-address hash table. 770 */ 771 buf = bcp->bc_addr; 772 hash_bucket = KMEM_HASH(cp, buf); 773 bcp->bc_next = *hash_bucket; 774 *hash_bucket = bcp; 775 if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 776 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 777 } 778 } else { 779 buf = KMEM_BUF(cp, bcp); 780 } 781 782 ASSERT(KMEM_SLAB_MEMBER(sp, buf)); 783 784 mutex_exit(&cp->cache_lock); 785 786 return (buf); 787 } 788 789 /* 790 * Free a raw (unconstructed) buffer to cp's slab layer. 791 */ 792 static void 793 kmem_slab_free(kmem_cache_t *cp, void *buf) 794 { 795 kmem_slab_t *sp; 796 kmem_bufctl_t *bcp, **prev_bcpp; 797 798 ASSERT(buf != NULL); 799 800 mutex_enter(&cp->cache_lock); 801 cp->cache_slab_free++; 802 803 if (cp->cache_flags & KMF_HASH) { 804 /* 805 * Look up buffer in allocated-address hash table. 806 */ 807 prev_bcpp = KMEM_HASH(cp, buf); 808 while ((bcp = *prev_bcpp) != NULL) { 809 if (bcp->bc_addr == buf) { 810 *prev_bcpp = bcp->bc_next; 811 sp = bcp->bc_slab; 812 break; 813 } 814 cp->cache_lookup_depth++; 815 prev_bcpp = &bcp->bc_next; 816 } 817 } else { 818 bcp = KMEM_BUFCTL(cp, buf); 819 sp = KMEM_SLAB(cp, buf); 820 } 821 822 if (bcp == NULL || sp->slab_cache != cp || !KMEM_SLAB_MEMBER(sp, buf)) { 823 mutex_exit(&cp->cache_lock); 824 kmem_error(KMERR_BADADDR, cp, buf); 825 return; 826 } 827 828 if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 829 if (cp->cache_flags & KMF_CONTENTS) 830 ((kmem_bufctl_audit_t *)bcp)->bc_contents = 831 kmem_log_enter(kmem_content_log, buf, 832 cp->cache_contents); 833 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 834 } 835 836 /* 837 * If this slab isn't currently on the freelist, put it there. 838 */ 839 if (sp->slab_head == NULL) { 840 ASSERT(sp->slab_refcnt == sp->slab_chunks); 841 ASSERT(cp->cache_freelist != sp); 842 sp->slab_next->slab_prev = sp->slab_prev; 843 sp->slab_prev->slab_next = sp->slab_next; 844 sp->slab_next = cp->cache_freelist; 845 sp->slab_prev = cp->cache_freelist->slab_prev; 846 sp->slab_next->slab_prev = sp; 847 sp->slab_prev->slab_next = sp; 848 cp->cache_freelist = sp; 849 } 850 851 bcp->bc_next = sp->slab_head; 852 sp->slab_head = bcp; 853 854 ASSERT(sp->slab_refcnt >= 1); 855 if (--sp->slab_refcnt == 0) { 856 /* 857 * There are no outstanding allocations from this slab, 858 * so we can reclaim the memory. 859 */ 860 sp->slab_next->slab_prev = sp->slab_prev; 861 sp->slab_prev->slab_next = sp->slab_next; 862 if (sp == cp->cache_freelist) 863 cp->cache_freelist = sp->slab_next; 864 cp->cache_slab_destroy++; 865 cp->cache_buftotal -= sp->slab_chunks; 866 mutex_exit(&cp->cache_lock); 867 kmem_slab_destroy(cp, sp); 868 return; 869 } 870 mutex_exit(&cp->cache_lock); 871 } 872 873 static int 874 kmem_cache_alloc_debug(kmem_cache_t *cp, void *buf, int kmflag, int construct, 875 caddr_t caller) 876 { 877 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 878 kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 879 uint32_t mtbf; 880 881 if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 882 kmem_error(KMERR_BADBUFTAG, cp, buf); 883 return (-1); 884 } 885 886 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_ALLOC; 887 888 if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 889 kmem_error(KMERR_BADBUFCTL, cp, buf); 890 return (-1); 891 } 892 893 if (cp->cache_flags & KMF_DEADBEEF) { 894 if (!construct && (cp->cache_flags & KMF_LITE)) { 895 if (*(uint64_t *)buf != KMEM_FREE_PATTERN) { 896 kmem_error(KMERR_MODIFIED, cp, buf); 897 return (-1); 898 } 899 if (cp->cache_constructor != NULL) 900 *(uint64_t *)buf = btp->bt_redzone; 901 else 902 *(uint64_t *)buf = KMEM_UNINITIALIZED_PATTERN; 903 } else { 904 construct = 1; 905 if (verify_and_copy_pattern(KMEM_FREE_PATTERN, 906 KMEM_UNINITIALIZED_PATTERN, buf, 907 cp->cache_verify)) { 908 kmem_error(KMERR_MODIFIED, cp, buf); 909 return (-1); 910 } 911 } 912 } 913 btp->bt_redzone = KMEM_REDZONE_PATTERN; 914 915 if ((mtbf = kmem_mtbf | cp->cache_mtbf) != 0 && 916 gethrtime() % mtbf == 0 && 917 (kmflag & (KM_NOSLEEP | KM_PANIC)) == KM_NOSLEEP) { 918 kmem_log_event(kmem_failure_log, cp, NULL, NULL); 919 if (!construct && cp->cache_destructor != NULL) 920 cp->cache_destructor(buf, cp->cache_private); 921 } else { 922 mtbf = 0; 923 } 924 925 if (mtbf || (construct && cp->cache_constructor != NULL && 926 cp->cache_constructor(buf, cp->cache_private, kmflag) != 0)) { 927 atomic_add_64(&cp->cache_alloc_fail, 1); 928 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 929 if (cp->cache_flags & KMF_DEADBEEF) 930 copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 931 kmem_slab_free(cp, buf); 932 return (-1); 933 } 934 935 if (cp->cache_flags & KMF_AUDIT) { 936 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 937 } 938 939 if ((cp->cache_flags & KMF_LITE) && 940 !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 941 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 942 } 943 944 return (0); 945 } 946 947 static int 948 kmem_cache_free_debug(kmem_cache_t *cp, void *buf, caddr_t caller) 949 { 950 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 951 kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 952 kmem_slab_t *sp; 953 954 if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_ALLOC)) { 955 if (btp->bt_bxstat == ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 956 kmem_error(KMERR_DUPFREE, cp, buf); 957 return (-1); 958 } 959 sp = kmem_findslab(cp, buf); 960 if (sp == NULL || sp->slab_cache != cp) 961 kmem_error(KMERR_BADADDR, cp, buf); 962 else 963 kmem_error(KMERR_REDZONE, cp, buf); 964 return (-1); 965 } 966 967 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 968 969 if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 970 kmem_error(KMERR_BADBUFCTL, cp, buf); 971 return (-1); 972 } 973 974 if (btp->bt_redzone != KMEM_REDZONE_PATTERN) { 975 kmem_error(KMERR_REDZONE, cp, buf); 976 return (-1); 977 } 978 979 if (cp->cache_flags & KMF_AUDIT) { 980 if (cp->cache_flags & KMF_CONTENTS) 981 bcp->bc_contents = kmem_log_enter(kmem_content_log, 982 buf, cp->cache_contents); 983 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 984 } 985 986 if ((cp->cache_flags & KMF_LITE) && 987 !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 988 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 989 } 990 991 if (cp->cache_flags & KMF_DEADBEEF) { 992 if (cp->cache_flags & KMF_LITE) 993 btp->bt_redzone = *(uint64_t *)buf; 994 else if (cp->cache_destructor != NULL) 995 cp->cache_destructor(buf, cp->cache_private); 996 997 copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 998 } 999 1000 return (0); 1001 } 1002 1003 /* 1004 * Free each object in magazine mp to cp's slab layer, and free mp itself. 1005 */ 1006 static void 1007 kmem_magazine_destroy(kmem_cache_t *cp, kmem_magazine_t *mp, int nrounds) 1008 { 1009 int round; 1010 1011 ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 1012 1013 for (round = 0; round < nrounds; round++) { 1014 void *buf = mp->mag_round[round]; 1015 1016 if (cp->cache_flags & KMF_DEADBEEF) { 1017 if (verify_pattern(KMEM_FREE_PATTERN, buf, 1018 cp->cache_verify) != NULL) { 1019 kmem_error(KMERR_MODIFIED, cp, buf); 1020 continue; 1021 } 1022 if ((cp->cache_flags & KMF_LITE) && 1023 cp->cache_destructor != NULL) { 1024 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1025 *(uint64_t *)buf = btp->bt_redzone; 1026 cp->cache_destructor(buf, cp->cache_private); 1027 *(uint64_t *)buf = KMEM_FREE_PATTERN; 1028 } 1029 } else if (cp->cache_destructor != NULL) { 1030 cp->cache_destructor(buf, cp->cache_private); 1031 } 1032 1033 kmem_slab_free(cp, buf); 1034 } 1035 ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 1036 kmem_cache_free(cp->cache_magtype->mt_cache, mp); 1037 } 1038 1039 /* 1040 * Allocate a magazine from the depot. 1041 */ 1042 static kmem_magazine_t * 1043 kmem_depot_alloc(kmem_cache_t *cp, kmem_maglist_t *mlp) 1044 { 1045 kmem_magazine_t *mp; 1046 1047 /* 1048 * If we can't get the depot lock without contention, 1049 * update our contention count. We use the depot 1050 * contention rate to determine whether we need to 1051 * increase the magazine size for better scalability. 1052 */ 1053 if (!mutex_tryenter(&cp->cache_depot_lock)) { 1054 mutex_enter(&cp->cache_depot_lock); 1055 cp->cache_depot_contention++; 1056 } 1057 1058 if ((mp = mlp->ml_list) != NULL) { 1059 ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 1060 mlp->ml_list = mp->mag_next; 1061 if (--mlp->ml_total < mlp->ml_min) 1062 mlp->ml_min = mlp->ml_total; 1063 mlp->ml_alloc++; 1064 } 1065 1066 mutex_exit(&cp->cache_depot_lock); 1067 1068 return (mp); 1069 } 1070 1071 /* 1072 * Free a magazine to the depot. 1073 */ 1074 static void 1075 kmem_depot_free(kmem_cache_t *cp, kmem_maglist_t *mlp, kmem_magazine_t *mp) 1076 { 1077 mutex_enter(&cp->cache_depot_lock); 1078 ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 1079 mp->mag_next = mlp->ml_list; 1080 mlp->ml_list = mp; 1081 mlp->ml_total++; 1082 mutex_exit(&cp->cache_depot_lock); 1083 } 1084 1085 /* 1086 * Update the working set statistics for cp's depot. 1087 */ 1088 static void 1089 kmem_depot_ws_update(kmem_cache_t *cp) 1090 { 1091 mutex_enter(&cp->cache_depot_lock); 1092 cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; 1093 cp->cache_full.ml_min = cp->cache_full.ml_total; 1094 cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; 1095 cp->cache_empty.ml_min = cp->cache_empty.ml_total; 1096 mutex_exit(&cp->cache_depot_lock); 1097 } 1098 1099 /* 1100 * Reap all magazines that have fallen out of the depot's working set. 1101 */ 1102 static void 1103 kmem_depot_ws_reap(kmem_cache_t *cp) 1104 { 1105 long reap; 1106 kmem_magazine_t *mp; 1107 1108 ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 1109 1110 reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 1111 while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL) 1112 kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); 1113 1114 reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); 1115 while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL) 1116 kmem_magazine_destroy(cp, mp, 0); 1117 } 1118 1119 static void 1120 kmem_cpu_reload(kmem_cpu_cache_t *ccp, kmem_magazine_t *mp, int rounds) 1121 { 1122 ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) || 1123 (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize)); 1124 ASSERT(ccp->cc_magsize > 0); 1125 1126 ccp->cc_ploaded = ccp->cc_loaded; 1127 ccp->cc_prounds = ccp->cc_rounds; 1128 ccp->cc_loaded = mp; 1129 ccp->cc_rounds = rounds; 1130 } 1131 1132 /* 1133 * Allocate a constructed object from cache cp. 1134 */ 1135 void * 1136 kmem_cache_alloc(kmem_cache_t *cp, int kmflag) 1137 { 1138 kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 1139 kmem_magazine_t *fmp; 1140 void *buf; 1141 1142 mutex_enter(&ccp->cc_lock); 1143 for (;;) { 1144 /* 1145 * If there's an object available in the current CPU's 1146 * loaded magazine, just take it and return. 1147 */ 1148 if (ccp->cc_rounds > 0) { 1149 buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds]; 1150 ccp->cc_alloc++; 1151 mutex_exit(&ccp->cc_lock); 1152 if ((ccp->cc_flags & KMF_BUFTAG) && 1153 kmem_cache_alloc_debug(cp, buf, kmflag, 0, 1154 caller()) == -1) { 1155 if (kmflag & KM_NOSLEEP) 1156 return (NULL); 1157 mutex_enter(&ccp->cc_lock); 1158 continue; 1159 } 1160 return (buf); 1161 } 1162 1163 /* 1164 * The loaded magazine is empty. If the previously loaded 1165 * magazine was full, exchange them and try again. 1166 */ 1167 if (ccp->cc_prounds > 0) { 1168 kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 1169 continue; 1170 } 1171 1172 /* 1173 * If the magazine layer is disabled, break out now. 1174 */ 1175 if (ccp->cc_magsize == 0) 1176 break; 1177 1178 /* 1179 * Try to get a full magazine from the depot. 1180 */ 1181 fmp = kmem_depot_alloc(cp, &cp->cache_full); 1182 if (fmp != NULL) { 1183 if (ccp->cc_ploaded != NULL) 1184 kmem_depot_free(cp, &cp->cache_empty, 1185 ccp->cc_ploaded); 1186 kmem_cpu_reload(ccp, fmp, ccp->cc_magsize); 1187 continue; 1188 } 1189 1190 /* 1191 * There are no full magazines in the depot, 1192 * so fall through to the slab layer. 1193 */ 1194 break; 1195 } 1196 mutex_exit(&ccp->cc_lock); 1197 1198 /* 1199 * We couldn't allocate a constructed object from the magazine layer, 1200 * so get a raw buffer from the slab layer and apply its constructor. 1201 */ 1202 buf = kmem_slab_alloc(cp, kmflag); 1203 1204 if (buf == NULL) 1205 return (NULL); 1206 1207 if (cp->cache_flags & KMF_BUFTAG) { 1208 /* 1209 * Make kmem_cache_alloc_debug() apply the constructor for us. 1210 */ 1211 if (kmem_cache_alloc_debug(cp, buf, kmflag, 1, 1212 caller()) == -1) { 1213 if (kmflag & KM_NOSLEEP) 1214 return (NULL); 1215 /* 1216 * kmem_cache_alloc_debug() detected corruption 1217 * but didn't panic (kmem_panic <= 0). Try again. 1218 */ 1219 return (kmem_cache_alloc(cp, kmflag)); 1220 } 1221 return (buf); 1222 } 1223 1224 if (cp->cache_constructor != NULL && 1225 cp->cache_constructor(buf, cp->cache_private, kmflag) != 0) { 1226 atomic_add_64(&cp->cache_alloc_fail, 1); 1227 kmem_slab_free(cp, buf); 1228 return (NULL); 1229 } 1230 1231 return (buf); 1232 } 1233 1234 /* 1235 * Free a constructed object to cache cp. 1236 */ 1237 void 1238 kmem_cache_free(kmem_cache_t *cp, void *buf) 1239 { 1240 kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 1241 kmem_magazine_t *emp; 1242 kmem_magtype_t *mtp; 1243 1244 if (ccp->cc_flags & KMF_BUFTAG) 1245 if (kmem_cache_free_debug(cp, buf, caller()) == -1) 1246 return; 1247 1248 mutex_enter(&ccp->cc_lock); 1249 for (;;) { 1250 /* 1251 * If there's a slot available in the current CPU's 1252 * loaded magazine, just put the object there and return. 1253 */ 1254 if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 1255 ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf; 1256 ccp->cc_free++; 1257 mutex_exit(&ccp->cc_lock); 1258 return; 1259 } 1260 1261 /* 1262 * The loaded magazine is full. If the previously loaded 1263 * magazine was empty, exchange them and try again. 1264 */ 1265 if (ccp->cc_prounds == 0) { 1266 kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 1267 continue; 1268 } 1269 1270 /* 1271 * If the magazine layer is disabled, break out now. 1272 */ 1273 if (ccp->cc_magsize == 0) 1274 break; 1275 1276 /* 1277 * Try to get an empty magazine from the depot. 1278 */ 1279 emp = kmem_depot_alloc(cp, &cp->cache_empty); 1280 if (emp != NULL) { 1281 if (ccp->cc_ploaded != NULL) 1282 kmem_depot_free(cp, &cp->cache_full, 1283 ccp->cc_ploaded); 1284 kmem_cpu_reload(ccp, emp, 0); 1285 continue; 1286 } 1287 1288 /* 1289 * There are no empty magazines in the depot, 1290 * so try to allocate a new one. We must drop all locks 1291 * across kmem_cache_alloc() because lower layers may 1292 * attempt to allocate from this cache. 1293 */ 1294 mtp = cp->cache_magtype; 1295 mutex_exit(&ccp->cc_lock); 1296 emp = kmem_cache_alloc(mtp->mt_cache, KM_NOSLEEP); 1297 mutex_enter(&ccp->cc_lock); 1298 1299 if (emp != NULL) { 1300 /* 1301 * We successfully allocated an empty magazine. 1302 * However, we had to drop ccp->cc_lock to do it, 1303 * so the cache's magazine size may have changed. 1304 * If so, free the magazine and try again. 1305 */ 1306 if (ccp->cc_magsize != mtp->mt_magsize) { 1307 mutex_exit(&ccp->cc_lock); 1308 kmem_cache_free(mtp->mt_cache, emp); 1309 mutex_enter(&ccp->cc_lock); 1310 continue; 1311 } 1312 1313 /* 1314 * We got a magazine of the right size. Add it to 1315 * the depot and try the whole dance again. 1316 */ 1317 kmem_depot_free(cp, &cp->cache_empty, emp); 1318 continue; 1319 } 1320 1321 /* 1322 * We couldn't allocate an empty magazine, 1323 * so fall through to the slab layer. 1324 */ 1325 break; 1326 } 1327 mutex_exit(&ccp->cc_lock); 1328 1329 /* 1330 * We couldn't free our constructed object to the magazine layer, 1331 * so apply its destructor and free it to the slab layer. 1332 * Note that if KMF_DEADBEEF is in effect and KMF_LITE is not, 1333 * kmem_cache_free_debug() will have already applied the destructor. 1334 */ 1335 if ((cp->cache_flags & (KMF_DEADBEEF | KMF_LITE)) != KMF_DEADBEEF && 1336 cp->cache_destructor != NULL) { 1337 if (cp->cache_flags & KMF_DEADBEEF) { /* KMF_LITE implied */ 1338 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1339 *(uint64_t *)buf = btp->bt_redzone; 1340 cp->cache_destructor(buf, cp->cache_private); 1341 *(uint64_t *)buf = KMEM_FREE_PATTERN; 1342 } else { 1343 cp->cache_destructor(buf, cp->cache_private); 1344 } 1345 } 1346 1347 kmem_slab_free(cp, buf); 1348 } 1349 1350 void * 1351 kmem_zalloc(size_t size, int kmflag) 1352 { 1353 size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 1354 void *buf; 1355 1356 if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 1357 kmem_cache_t *cp = kmem_alloc_table[index]; 1358 buf = kmem_cache_alloc(cp, kmflag); 1359 if (buf != NULL) { 1360 if (cp->cache_flags & KMF_BUFTAG) { 1361 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1362 ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 1363 ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 1364 1365 if (cp->cache_flags & KMF_LITE) { 1366 KMEM_BUFTAG_LITE_ENTER(btp, 1367 kmem_lite_count, caller()); 1368 } 1369 } 1370 bzero(buf, size); 1371 } 1372 } else { 1373 buf = kmem_alloc(size, kmflag); 1374 if (buf != NULL) 1375 bzero(buf, size); 1376 } 1377 return (buf); 1378 } 1379 1380 void * 1381 kmem_alloc(size_t size, int kmflag) 1382 { 1383 size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 1384 void *buf; 1385 1386 if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 1387 kmem_cache_t *cp = kmem_alloc_table[index]; 1388 buf = kmem_cache_alloc(cp, kmflag); 1389 if ((cp->cache_flags & KMF_BUFTAG) && buf != NULL) { 1390 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1391 ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 1392 ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 1393 1394 if (cp->cache_flags & KMF_LITE) { 1395 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 1396 caller()); 1397 } 1398 } 1399 return (buf); 1400 } 1401 if (size == 0) 1402 return (NULL); 1403 buf = vmem_alloc(kmem_oversize_arena, size, kmflag & KM_VMFLAGS); 1404 if (buf == NULL) 1405 kmem_log_event(kmem_failure_log, NULL, NULL, (void *)size); 1406 return (buf); 1407 } 1408 1409 void 1410 kmem_free(void *buf, size_t size) 1411 { 1412 size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 1413 1414 if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 1415 kmem_cache_t *cp = kmem_alloc_table[index]; 1416 if (cp->cache_flags & KMF_BUFTAG) { 1417 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1418 uint32_t *ip = (uint32_t *)btp; 1419 if (ip[1] != KMEM_SIZE_ENCODE(size)) { 1420 if (*(uint64_t *)buf == KMEM_FREE_PATTERN) { 1421 kmem_error(KMERR_DUPFREE, cp, buf); 1422 return; 1423 } 1424 if (KMEM_SIZE_VALID(ip[1])) { 1425 ip[0] = KMEM_SIZE_ENCODE(size); 1426 kmem_error(KMERR_BADSIZE, cp, buf); 1427 } else { 1428 kmem_error(KMERR_REDZONE, cp, buf); 1429 } 1430 return; 1431 } 1432 if (((uint8_t *)buf)[size] != KMEM_REDZONE_BYTE) { 1433 kmem_error(KMERR_REDZONE, cp, buf); 1434 return; 1435 } 1436 btp->bt_redzone = KMEM_REDZONE_PATTERN; 1437 if (cp->cache_flags & KMF_LITE) { 1438 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 1439 caller()); 1440 } 1441 } 1442 kmem_cache_free(cp, buf); 1443 } else { 1444 if (buf == NULL && size == 0) 1445 return; 1446 vmem_free(kmem_oversize_arena, buf, size); 1447 } 1448 } 1449 1450 void * 1451 kmem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag) 1452 { 1453 size_t realsize = size + vmp->vm_quantum; 1454 void *addr; 1455 1456 /* 1457 * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding 1458 * vm_quantum will cause integer wraparound. Check for this, and 1459 * blow off the firewall page in this case. Note that such a 1460 * giant allocation (the entire kernel address space) can never 1461 * be satisfied, so it will either fail immediately (VM_NOSLEEP) 1462 * or sleep forever (VM_SLEEP). Thus, there is no need for a 1463 * corresponding check in kmem_firewall_va_free(). 1464 */ 1465 if (realsize < size) 1466 realsize = size; 1467 1468 /* 1469 * While boot still owns resource management, make sure that this 1470 * redzone virtual address allocation is properly accounted for in 1471 * OBPs "virtual-memory" "available" lists because we're 1472 * effectively claiming them for a red zone. If we don't do this, 1473 * the available lists become too fragmented and too large for the 1474 * current boot/kernel memory list interface. 1475 */ 1476 addr = vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT); 1477 1478 if (addr != NULL && kvseg.s_base == NULL && realsize != size) 1479 (void) boot_virt_alloc((char *)addr + size, vmp->vm_quantum); 1480 1481 return (addr); 1482 } 1483 1484 void 1485 kmem_firewall_va_free(vmem_t *vmp, void *addr, size_t size) 1486 { 1487 ASSERT((kvseg.s_base == NULL ? 1488 va_to_pfn((char *)addr + size) : 1489 hat_getpfnum(kas.a_hat, (caddr_t)addr + size)) == PFN_INVALID); 1490 1491 vmem_free(vmp, addr, size + vmp->vm_quantum); 1492 } 1493 1494 /* 1495 * Try to allocate at least `size' bytes of memory without sleeping or 1496 * panicking. Return actual allocated size in `asize'. If allocation failed, 1497 * try final allocation with sleep or panic allowed. 1498 */ 1499 void * 1500 kmem_alloc_tryhard(size_t size, size_t *asize, int kmflag) 1501 { 1502 void *p; 1503 1504 *asize = P2ROUNDUP(size, KMEM_ALIGN); 1505 do { 1506 p = kmem_alloc(*asize, (kmflag | KM_NOSLEEP) & ~KM_PANIC); 1507 if (p != NULL) 1508 return (p); 1509 *asize += KMEM_ALIGN; 1510 } while (*asize <= PAGESIZE); 1511 1512 *asize = P2ROUNDUP(size, KMEM_ALIGN); 1513 return (kmem_alloc(*asize, kmflag)); 1514 } 1515 1516 /* 1517 * Reclaim all unused memory from a cache. 1518 */ 1519 static void 1520 kmem_cache_reap(kmem_cache_t *cp) 1521 { 1522 /* 1523 * Ask the cache's owner to free some memory if possible. 1524 * The idea is to handle things like the inode cache, which 1525 * typically sits on a bunch of memory that it doesn't truly 1526 * *need*. Reclaim policy is entirely up to the owner; this 1527 * callback is just an advisory plea for help. 1528 */ 1529 if (cp->cache_reclaim != NULL) 1530 cp->cache_reclaim(cp->cache_private); 1531 1532 kmem_depot_ws_reap(cp); 1533 } 1534 1535 static void 1536 kmem_reap_timeout(void *flag_arg) 1537 { 1538 uint32_t *flag = (uint32_t *)flag_arg; 1539 1540 ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 1541 *flag = 0; 1542 } 1543 1544 static void 1545 kmem_reap_done(void *flag) 1546 { 1547 (void) timeout(kmem_reap_timeout, flag, kmem_reap_interval); 1548 } 1549 1550 static void 1551 kmem_reap_start(void *flag) 1552 { 1553 ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 1554 1555 if (flag == &kmem_reaping) { 1556 kmem_cache_applyall(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 1557 /* 1558 * if we have segkp under heap, reap segkp cache. 1559 */ 1560 if (segkp_fromheap) 1561 segkp_cache_free(); 1562 } 1563 else 1564 kmem_cache_applyall_id(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 1565 1566 /* 1567 * We use taskq_dispatch() to schedule a timeout to clear 1568 * the flag so that kmem_reap() becomes self-throttling: 1569 * we won't reap again until the current reap completes *and* 1570 * at least kmem_reap_interval ticks have elapsed. 1571 */ 1572 if (!taskq_dispatch(kmem_taskq, kmem_reap_done, flag, TQ_NOSLEEP)) 1573 kmem_reap_done(flag); 1574 } 1575 1576 static void 1577 kmem_reap_common(void *flag_arg) 1578 { 1579 uint32_t *flag = (uint32_t *)flag_arg; 1580 1581 if (MUTEX_HELD(&kmem_cache_lock) || kmem_taskq == NULL || 1582 cas32(flag, 0, 1) != 0) 1583 return; 1584 1585 /* 1586 * It may not be kosher to do memory allocation when a reap is called 1587 * is called (for example, if vmem_populate() is in the call chain). 1588 * So we start the reap going with a TQ_NOALLOC dispatch. If the 1589 * dispatch fails, we reset the flag, and the next reap will try again. 1590 */ 1591 if (!taskq_dispatch(kmem_taskq, kmem_reap_start, flag, TQ_NOALLOC)) 1592 *flag = 0; 1593 } 1594 1595 /* 1596 * Reclaim all unused memory from all caches. Called from the VM system 1597 * when memory gets tight. 1598 */ 1599 void 1600 kmem_reap(void) 1601 { 1602 kmem_reap_common(&kmem_reaping); 1603 } 1604 1605 /* 1606 * Reclaim all unused memory from identifier arenas, called when a vmem 1607 * arena not back by memory is exhausted. Since reaping memory-backed caches 1608 * cannot help with identifier exhaustion, we avoid both a large amount of 1609 * work and unwanted side-effects from reclaim callbacks. 1610 */ 1611 void 1612 kmem_reap_idspace(void) 1613 { 1614 kmem_reap_common(&kmem_reaping_idspace); 1615 } 1616 1617 /* 1618 * Purge all magazines from a cache and set its magazine limit to zero. 1619 * All calls are serialized by the kmem_taskq lock, except for the final 1620 * call from kmem_cache_destroy(). 1621 */ 1622 static void 1623 kmem_cache_magazine_purge(kmem_cache_t *cp) 1624 { 1625 kmem_cpu_cache_t *ccp; 1626 kmem_magazine_t *mp, *pmp; 1627 int rounds, prounds, cpu_seqid; 1628 1629 ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 1630 ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 1631 1632 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 1633 ccp = &cp->cache_cpu[cpu_seqid]; 1634 1635 mutex_enter(&ccp->cc_lock); 1636 mp = ccp->cc_loaded; 1637 pmp = ccp->cc_ploaded; 1638 rounds = ccp->cc_rounds; 1639 prounds = ccp->cc_prounds; 1640 ccp->cc_loaded = NULL; 1641 ccp->cc_ploaded = NULL; 1642 ccp->cc_rounds = -1; 1643 ccp->cc_prounds = -1; 1644 ccp->cc_magsize = 0; 1645 mutex_exit(&ccp->cc_lock); 1646 1647 if (mp) 1648 kmem_magazine_destroy(cp, mp, rounds); 1649 if (pmp) 1650 kmem_magazine_destroy(cp, pmp, prounds); 1651 } 1652 1653 /* 1654 * Updating the working set statistics twice in a row has the 1655 * effect of setting the working set size to zero, so everything 1656 * is eligible for reaping. 1657 */ 1658 kmem_depot_ws_update(cp); 1659 kmem_depot_ws_update(cp); 1660 1661 kmem_depot_ws_reap(cp); 1662 } 1663 1664 /* 1665 * Enable per-cpu magazines on a cache. 1666 */ 1667 static void 1668 kmem_cache_magazine_enable(kmem_cache_t *cp) 1669 { 1670 int cpu_seqid; 1671 1672 if (cp->cache_flags & KMF_NOMAGAZINE) 1673 return; 1674 1675 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 1676 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 1677 mutex_enter(&ccp->cc_lock); 1678 ccp->cc_magsize = cp->cache_magtype->mt_magsize; 1679 mutex_exit(&ccp->cc_lock); 1680 } 1681 1682 } 1683 1684 /* 1685 * Reap (almost) everything right now. See kmem_cache_magazine_purge() 1686 * for explanation of the back-to-back kmem_depot_ws_update() calls. 1687 */ 1688 void 1689 kmem_cache_reap_now(kmem_cache_t *cp) 1690 { 1691 kmem_depot_ws_update(cp); 1692 kmem_depot_ws_update(cp); 1693 1694 (void) taskq_dispatch(kmem_taskq, 1695 (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP); 1696 taskq_wait(kmem_taskq); 1697 } 1698 1699 /* 1700 * Recompute a cache's magazine size. The trade-off is that larger magazines 1701 * provide a higher transfer rate with the depot, while smaller magazines 1702 * reduce memory consumption. Magazine resizing is an expensive operation; 1703 * it should not be done frequently. 1704 * 1705 * Changes to the magazine size are serialized by the kmem_taskq lock. 1706 * 1707 * Note: at present this only grows the magazine size. It might be useful 1708 * to allow shrinkage too. 1709 */ 1710 static void 1711 kmem_cache_magazine_resize(kmem_cache_t *cp) 1712 { 1713 kmem_magtype_t *mtp = cp->cache_magtype; 1714 1715 ASSERT(taskq_member(kmem_taskq, curthread)); 1716 1717 if (cp->cache_chunksize < mtp->mt_maxbuf) { 1718 kmem_cache_magazine_purge(cp); 1719 mutex_enter(&cp->cache_depot_lock); 1720 cp->cache_magtype = ++mtp; 1721 cp->cache_depot_contention_prev = 1722 cp->cache_depot_contention + INT_MAX; 1723 mutex_exit(&cp->cache_depot_lock); 1724 kmem_cache_magazine_enable(cp); 1725 } 1726 } 1727 1728 /* 1729 * Rescale a cache's hash table, so that the table size is roughly the 1730 * cache size. We want the average lookup time to be extremely small. 1731 */ 1732 static void 1733 kmem_hash_rescale(kmem_cache_t *cp) 1734 { 1735 kmem_bufctl_t **old_table, **new_table, *bcp; 1736 size_t old_size, new_size, h; 1737 1738 ASSERT(taskq_member(kmem_taskq, curthread)); 1739 1740 new_size = MAX(KMEM_HASH_INITIAL, 1741 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); 1742 old_size = cp->cache_hash_mask + 1; 1743 1744 if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 1745 return; 1746 1747 new_table = vmem_alloc(kmem_hash_arena, new_size * sizeof (void *), 1748 VM_NOSLEEP); 1749 if (new_table == NULL) 1750 return; 1751 bzero(new_table, new_size * sizeof (void *)); 1752 1753 mutex_enter(&cp->cache_lock); 1754 1755 old_size = cp->cache_hash_mask + 1; 1756 old_table = cp->cache_hash_table; 1757 1758 cp->cache_hash_mask = new_size - 1; 1759 cp->cache_hash_table = new_table; 1760 cp->cache_rescale++; 1761 1762 for (h = 0; h < old_size; h++) { 1763 bcp = old_table[h]; 1764 while (bcp != NULL) { 1765 void *addr = bcp->bc_addr; 1766 kmem_bufctl_t *next_bcp = bcp->bc_next; 1767 kmem_bufctl_t **hash_bucket = KMEM_HASH(cp, addr); 1768 bcp->bc_next = *hash_bucket; 1769 *hash_bucket = bcp; 1770 bcp = next_bcp; 1771 } 1772 } 1773 1774 mutex_exit(&cp->cache_lock); 1775 1776 vmem_free(kmem_hash_arena, old_table, old_size * sizeof (void *)); 1777 } 1778 1779 /* 1780 * Perform periodic maintenance on a cache: hash rescaling, 1781 * depot working-set update, and magazine resizing. 1782 */ 1783 static void 1784 kmem_cache_update(kmem_cache_t *cp) 1785 { 1786 int need_hash_rescale = 0; 1787 int need_magazine_resize = 0; 1788 1789 ASSERT(MUTEX_HELD(&kmem_cache_lock)); 1790 1791 /* 1792 * If the cache has become much larger or smaller than its hash table, 1793 * fire off a request to rescale the hash table. 1794 */ 1795 mutex_enter(&cp->cache_lock); 1796 1797 if ((cp->cache_flags & KMF_HASH) && 1798 (cp->cache_buftotal > (cp->cache_hash_mask << 1) || 1799 (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && 1800 cp->cache_hash_mask > KMEM_HASH_INITIAL))) 1801 need_hash_rescale = 1; 1802 1803 mutex_exit(&cp->cache_lock); 1804 1805 /* 1806 * Update the depot working set statistics. 1807 */ 1808 kmem_depot_ws_update(cp); 1809 1810 /* 1811 * If there's a lot of contention in the depot, 1812 * increase the magazine size. 1813 */ 1814 mutex_enter(&cp->cache_depot_lock); 1815 1816 if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && 1817 (int)(cp->cache_depot_contention - 1818 cp->cache_depot_contention_prev) > kmem_depot_contention) 1819 need_magazine_resize = 1; 1820 1821 cp->cache_depot_contention_prev = cp->cache_depot_contention; 1822 1823 mutex_exit(&cp->cache_depot_lock); 1824 1825 if (need_hash_rescale) 1826 (void) taskq_dispatch(kmem_taskq, 1827 (task_func_t *)kmem_hash_rescale, cp, TQ_NOSLEEP); 1828 1829 if (need_magazine_resize) 1830 (void) taskq_dispatch(kmem_taskq, 1831 (task_func_t *)kmem_cache_magazine_resize, cp, TQ_NOSLEEP); 1832 } 1833 1834 static void 1835 kmem_update_timeout(void *dummy) 1836 { 1837 static void kmem_update(void *); 1838 1839 (void) timeout(kmem_update, dummy, kmem_reap_interval); 1840 } 1841 1842 static void 1843 kmem_update(void *dummy) 1844 { 1845 kmem_cache_applyall(kmem_cache_update, NULL, TQ_NOSLEEP); 1846 1847 /* 1848 * We use taskq_dispatch() to reschedule the timeout so that 1849 * kmem_update() becomes self-throttling: it won't schedule 1850 * new tasks until all previous tasks have completed. 1851 */ 1852 if (!taskq_dispatch(kmem_taskq, kmem_update_timeout, dummy, TQ_NOSLEEP)) 1853 kmem_update_timeout(NULL); 1854 } 1855 1856 static int 1857 kmem_cache_kstat_update(kstat_t *ksp, int rw) 1858 { 1859 struct kmem_cache_kstat *kmcp = &kmem_cache_kstat; 1860 kmem_cache_t *cp = ksp->ks_private; 1861 kmem_slab_t *sp; 1862 uint64_t cpu_buf_avail; 1863 uint64_t buf_avail = 0; 1864 int cpu_seqid; 1865 1866 ASSERT(MUTEX_HELD(&kmem_cache_kstat_lock)); 1867 1868 if (rw == KSTAT_WRITE) 1869 return (EACCES); 1870 1871 mutex_enter(&cp->cache_lock); 1872 1873 kmcp->kmc_alloc_fail.value.ui64 = cp->cache_alloc_fail; 1874 kmcp->kmc_alloc.value.ui64 = cp->cache_slab_alloc; 1875 kmcp->kmc_free.value.ui64 = cp->cache_slab_free; 1876 kmcp->kmc_slab_alloc.value.ui64 = cp->cache_slab_alloc; 1877 kmcp->kmc_slab_free.value.ui64 = cp->cache_slab_free; 1878 1879 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 1880 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 1881 1882 mutex_enter(&ccp->cc_lock); 1883 1884 cpu_buf_avail = 0; 1885 if (ccp->cc_rounds > 0) 1886 cpu_buf_avail += ccp->cc_rounds; 1887 if (ccp->cc_prounds > 0) 1888 cpu_buf_avail += ccp->cc_prounds; 1889 1890 kmcp->kmc_alloc.value.ui64 += ccp->cc_alloc; 1891 kmcp->kmc_free.value.ui64 += ccp->cc_free; 1892 buf_avail += cpu_buf_avail; 1893 1894 mutex_exit(&ccp->cc_lock); 1895 } 1896 1897 mutex_enter(&cp->cache_depot_lock); 1898 1899 kmcp->kmc_depot_alloc.value.ui64 = cp->cache_full.ml_alloc; 1900 kmcp->kmc_depot_free.value.ui64 = cp->cache_empty.ml_alloc; 1901 kmcp->kmc_depot_contention.value.ui64 = cp->cache_depot_contention; 1902 kmcp->kmc_full_magazines.value.ui64 = cp->cache_full.ml_total; 1903 kmcp->kmc_empty_magazines.value.ui64 = cp->cache_empty.ml_total; 1904 kmcp->kmc_magazine_size.value.ui64 = 1905 (cp->cache_flags & KMF_NOMAGAZINE) ? 1906 0 : cp->cache_magtype->mt_magsize; 1907 1908 kmcp->kmc_alloc.value.ui64 += cp->cache_full.ml_alloc; 1909 kmcp->kmc_free.value.ui64 += cp->cache_empty.ml_alloc; 1910 buf_avail += cp->cache_full.ml_total * cp->cache_magtype->mt_magsize; 1911 1912 mutex_exit(&cp->cache_depot_lock); 1913 1914 kmcp->kmc_buf_size.value.ui64 = cp->cache_bufsize; 1915 kmcp->kmc_align.value.ui64 = cp->cache_align; 1916 kmcp->kmc_chunk_size.value.ui64 = cp->cache_chunksize; 1917 kmcp->kmc_slab_size.value.ui64 = cp->cache_slabsize; 1918 kmcp->kmc_buf_constructed.value.ui64 = buf_avail; 1919 for (sp = cp->cache_freelist; sp != &cp->cache_nullslab; 1920 sp = sp->slab_next) 1921 buf_avail += sp->slab_chunks - sp->slab_refcnt; 1922 kmcp->kmc_buf_avail.value.ui64 = buf_avail; 1923 kmcp->kmc_buf_inuse.value.ui64 = cp->cache_buftotal - buf_avail; 1924 kmcp->kmc_buf_total.value.ui64 = cp->cache_buftotal; 1925 kmcp->kmc_buf_max.value.ui64 = cp->cache_bufmax; 1926 kmcp->kmc_slab_create.value.ui64 = cp->cache_slab_create; 1927 kmcp->kmc_slab_destroy.value.ui64 = cp->cache_slab_destroy; 1928 kmcp->kmc_hash_size.value.ui64 = (cp->cache_flags & KMF_HASH) ? 1929 cp->cache_hash_mask + 1 : 0; 1930 kmcp->kmc_hash_lookup_depth.value.ui64 = cp->cache_lookup_depth; 1931 kmcp->kmc_hash_rescale.value.ui64 = cp->cache_rescale; 1932 kmcp->kmc_vmem_source.value.ui64 = cp->cache_arena->vm_id; 1933 1934 mutex_exit(&cp->cache_lock); 1935 return (0); 1936 } 1937 1938 /* 1939 * Return a named statistic about a particular cache. 1940 * This shouldn't be called very often, so it's currently designed for 1941 * simplicity (leverages existing kstat support) rather than efficiency. 1942 */ 1943 uint64_t 1944 kmem_cache_stat(kmem_cache_t *cp, char *name) 1945 { 1946 int i; 1947 kstat_t *ksp = cp->cache_kstat; 1948 kstat_named_t *knp = (kstat_named_t *)&kmem_cache_kstat; 1949 uint64_t value = 0; 1950 1951 if (ksp != NULL) { 1952 mutex_enter(&kmem_cache_kstat_lock); 1953 (void) kmem_cache_kstat_update(ksp, KSTAT_READ); 1954 for (i = 0; i < ksp->ks_ndata; i++) { 1955 if (strcmp(knp[i].name, name) == 0) { 1956 value = knp[i].value.ui64; 1957 break; 1958 } 1959 } 1960 mutex_exit(&kmem_cache_kstat_lock); 1961 } 1962 return (value); 1963 } 1964 1965 /* 1966 * Return an estimate of currently available kernel heap memory. 1967 * On 32-bit systems, physical memory may exceed virtual memory, 1968 * we just truncate the result at 1GB. 1969 */ 1970 size_t 1971 kmem_avail(void) 1972 { 1973 spgcnt_t rmem = availrmem - tune.t_minarmem; 1974 spgcnt_t fmem = freemem - minfree; 1975 1976 return ((size_t)ptob(MIN(MAX(MIN(rmem, fmem), 0), 1977 1 << (30 - PAGESHIFT)))); 1978 } 1979 1980 /* 1981 * Return the maximum amount of memory that is (in theory) allocatable 1982 * from the heap. This may be used as an estimate only since there 1983 * is no guarentee this space will still be available when an allocation 1984 * request is made, nor that the space may be allocated in one big request 1985 * due to kernel heap fragmentation. 1986 */ 1987 size_t 1988 kmem_maxavail(void) 1989 { 1990 spgcnt_t pmem = availrmem - tune.t_minarmem; 1991 spgcnt_t vmem = btop(vmem_size(heap_arena, VMEM_FREE)); 1992 1993 return ((size_t)ptob(MAX(MIN(pmem, vmem), 0))); 1994 } 1995 1996 /* 1997 * Indicate whether memory-intensive kmem debugging is enabled. 1998 */ 1999 int 2000 kmem_debugging(void) 2001 { 2002 return (kmem_flags & (KMF_AUDIT | KMF_REDZONE)); 2003 } 2004 2005 kmem_cache_t * 2006 kmem_cache_create( 2007 char *name, /* descriptive name for this cache */ 2008 size_t bufsize, /* size of the objects it manages */ 2009 size_t align, /* required object alignment */ 2010 int (*constructor)(void *, void *, int), /* object constructor */ 2011 void (*destructor)(void *, void *), /* object destructor */ 2012 void (*reclaim)(void *), /* memory reclaim callback */ 2013 void *private, /* pass-thru arg for constr/destr/reclaim */ 2014 vmem_t *vmp, /* vmem source for slab allocation */ 2015 int cflags) /* cache creation flags */ 2016 { 2017 int cpu_seqid; 2018 size_t chunksize; 2019 kmem_cache_t *cp, *cnext, *cprev; 2020 kmem_magtype_t *mtp; 2021 size_t csize = KMEM_CACHE_SIZE(max_ncpus); 2022 2023 #ifdef DEBUG 2024 /* 2025 * Cache names should conform to the rules for valid C identifiers 2026 */ 2027 if (!strident_valid(name)) { 2028 cmn_err(CE_CONT, 2029 "kmem_cache_create: '%s' is an invalid cache name\n" 2030 "cache names must conform to the rules for " 2031 "C identifiers\n", name); 2032 } 2033 #endif /* DEBUG */ 2034 2035 if (vmp == NULL) 2036 vmp = kmem_default_arena; 2037 2038 /* 2039 * If this kmem cache has an identifier vmem arena as its source, mark 2040 * it such to allow kmem_reap_idspace(). 2041 */ 2042 ASSERT(!(cflags & KMC_IDENTIFIER)); /* consumer should not set this */ 2043 if (vmp->vm_cflags & VMC_IDENTIFIER) 2044 cflags |= KMC_IDENTIFIER; 2045 2046 /* 2047 * Get a kmem_cache structure. We arrange that cp->cache_cpu[] 2048 * is aligned on a KMEM_CPU_CACHE_SIZE boundary to prevent 2049 * false sharing of per-CPU data. 2050 */ 2051 cp = vmem_xalloc(kmem_cache_arena, csize, KMEM_CPU_CACHE_SIZE, 2052 P2NPHASE(csize, KMEM_CPU_CACHE_SIZE), 0, NULL, NULL, VM_SLEEP); 2053 bzero(cp, csize); 2054 2055 if (align == 0) 2056 align = KMEM_ALIGN; 2057 2058 /* 2059 * If we're not at least KMEM_ALIGN aligned, we can't use free 2060 * memory to hold bufctl information (because we can't safely 2061 * perform word loads and stores on it). 2062 */ 2063 if (align < KMEM_ALIGN) 2064 cflags |= KMC_NOTOUCH; 2065 2066 if ((align & (align - 1)) != 0 || align > vmp->vm_quantum) 2067 panic("kmem_cache_create: bad alignment %lu", align); 2068 2069 mutex_enter(&kmem_flags_lock); 2070 if (kmem_flags & KMF_RANDOMIZE) 2071 kmem_flags = (((kmem_flags | ~KMF_RANDOM) + 1) & KMF_RANDOM) | 2072 KMF_RANDOMIZE; 2073 cp->cache_flags = (kmem_flags | cflags) & KMF_DEBUG; 2074 mutex_exit(&kmem_flags_lock); 2075 2076 /* 2077 * Make sure all the various flags are reasonable. 2078 */ 2079 ASSERT(!(cflags & KMC_NOHASH) || !(cflags & KMC_NOTOUCH)); 2080 2081 if (cp->cache_flags & KMF_LITE) { 2082 if (bufsize >= kmem_lite_minsize && 2083 align <= kmem_lite_maxalign && 2084 P2PHASE(bufsize, kmem_lite_maxalign) != 0) { 2085 cp->cache_flags |= KMF_BUFTAG; 2086 cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 2087 } else { 2088 cp->cache_flags &= ~KMF_DEBUG; 2089 } 2090 } 2091 2092 if (cp->cache_flags & KMF_DEADBEEF) 2093 cp->cache_flags |= KMF_REDZONE; 2094 2095 if ((cflags & KMC_QCACHE) && (cp->cache_flags & KMF_AUDIT)) 2096 cp->cache_flags |= KMF_NOMAGAZINE; 2097 2098 if (cflags & KMC_NODEBUG) 2099 cp->cache_flags &= ~KMF_DEBUG; 2100 2101 if (cflags & KMC_NOTOUCH) 2102 cp->cache_flags &= ~KMF_TOUCH; 2103 2104 if (cflags & KMC_NOHASH) 2105 cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 2106 2107 if (cflags & KMC_NOMAGAZINE) 2108 cp->cache_flags |= KMF_NOMAGAZINE; 2109 2110 if ((cp->cache_flags & KMF_AUDIT) && !(cflags & KMC_NOTOUCH)) 2111 cp->cache_flags |= KMF_REDZONE; 2112 2113 if (!(cp->cache_flags & KMF_AUDIT)) 2114 cp->cache_flags &= ~KMF_CONTENTS; 2115 2116 if ((cp->cache_flags & KMF_BUFTAG) && bufsize >= kmem_minfirewall && 2117 !(cp->cache_flags & KMF_LITE) && !(cflags & KMC_NOHASH)) 2118 cp->cache_flags |= KMF_FIREWALL; 2119 2120 if (vmp != kmem_default_arena || kmem_firewall_arena == NULL) 2121 cp->cache_flags &= ~KMF_FIREWALL; 2122 2123 if (cp->cache_flags & KMF_FIREWALL) { 2124 cp->cache_flags &= ~KMF_BUFTAG; 2125 cp->cache_flags |= KMF_NOMAGAZINE; 2126 ASSERT(vmp == kmem_default_arena); 2127 vmp = kmem_firewall_arena; 2128 } 2129 2130 /* 2131 * Set cache properties. 2132 */ 2133 (void) strncpy(cp->cache_name, name, KMEM_CACHE_NAMELEN); 2134 strident_canon(cp->cache_name, KMEM_CACHE_NAMELEN); 2135 cp->cache_bufsize = bufsize; 2136 cp->cache_align = align; 2137 cp->cache_constructor = constructor; 2138 cp->cache_destructor = destructor; 2139 cp->cache_reclaim = reclaim; 2140 cp->cache_private = private; 2141 cp->cache_arena = vmp; 2142 cp->cache_cflags = cflags; 2143 2144 /* 2145 * Determine the chunk size. 2146 */ 2147 chunksize = bufsize; 2148 2149 if (align >= KMEM_ALIGN) { 2150 chunksize = P2ROUNDUP(chunksize, KMEM_ALIGN); 2151 cp->cache_bufctl = chunksize - KMEM_ALIGN; 2152 } 2153 2154 if (cp->cache_flags & KMF_BUFTAG) { 2155 cp->cache_bufctl = chunksize; 2156 cp->cache_buftag = chunksize; 2157 if (cp->cache_flags & KMF_LITE) 2158 chunksize += KMEM_BUFTAG_LITE_SIZE(kmem_lite_count); 2159 else 2160 chunksize += sizeof (kmem_buftag_t); 2161 } 2162 2163 if (cp->cache_flags & KMF_DEADBEEF) { 2164 cp->cache_verify = MIN(cp->cache_buftag, kmem_maxverify); 2165 if (cp->cache_flags & KMF_LITE) 2166 cp->cache_verify = sizeof (uint64_t); 2167 } 2168 2169 cp->cache_contents = MIN(cp->cache_bufctl, kmem_content_maxsave); 2170 2171 cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); 2172 2173 /* 2174 * Now that we know the chunk size, determine the optimal slab size. 2175 */ 2176 if (vmp == kmem_firewall_arena) { 2177 cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); 2178 cp->cache_mincolor = cp->cache_slabsize - chunksize; 2179 cp->cache_maxcolor = cp->cache_mincolor; 2180 cp->cache_flags |= KMF_HASH; 2181 ASSERT(!(cp->cache_flags & KMF_BUFTAG)); 2182 } else if ((cflags & KMC_NOHASH) || (!(cflags & KMC_NOTOUCH) && 2183 !(cp->cache_flags & KMF_AUDIT) && 2184 chunksize < vmp->vm_quantum / KMEM_VOID_FRACTION)) { 2185 cp->cache_slabsize = vmp->vm_quantum; 2186 cp->cache_mincolor = 0; 2187 cp->cache_maxcolor = 2188 (cp->cache_slabsize - sizeof (kmem_slab_t)) % chunksize; 2189 ASSERT(chunksize + sizeof (kmem_slab_t) <= cp->cache_slabsize); 2190 ASSERT(!(cp->cache_flags & KMF_AUDIT)); 2191 } else { 2192 size_t chunks, bestfit, waste, slabsize; 2193 size_t minwaste = LONG_MAX; 2194 2195 for (chunks = 1; chunks <= KMEM_VOID_FRACTION; chunks++) { 2196 slabsize = P2ROUNDUP(chunksize * chunks, 2197 vmp->vm_quantum); 2198 chunks = slabsize / chunksize; 2199 waste = (slabsize % chunksize) / chunks; 2200 if (waste < minwaste) { 2201 minwaste = waste; 2202 bestfit = slabsize; 2203 } 2204 } 2205 if (cflags & KMC_QCACHE) 2206 bestfit = VMEM_QCACHE_SLABSIZE(vmp->vm_qcache_max); 2207 cp->cache_slabsize = bestfit; 2208 cp->cache_mincolor = 0; 2209 cp->cache_maxcolor = bestfit % chunksize; 2210 cp->cache_flags |= KMF_HASH; 2211 } 2212 2213 if (cp->cache_flags & KMF_HASH) { 2214 ASSERT(!(cflags & KMC_NOHASH)); 2215 cp->cache_bufctl_cache = (cp->cache_flags & KMF_AUDIT) ? 2216 kmem_bufctl_audit_cache : kmem_bufctl_cache; 2217 } 2218 2219 if (cp->cache_maxcolor >= vmp->vm_quantum) 2220 cp->cache_maxcolor = vmp->vm_quantum - 1; 2221 2222 cp->cache_color = cp->cache_mincolor; 2223 2224 /* 2225 * Initialize the rest of the slab layer. 2226 */ 2227 mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL); 2228 2229 cp->cache_freelist = &cp->cache_nullslab; 2230 cp->cache_nullslab.slab_cache = cp; 2231 cp->cache_nullslab.slab_refcnt = -1; 2232 cp->cache_nullslab.slab_next = &cp->cache_nullslab; 2233 cp->cache_nullslab.slab_prev = &cp->cache_nullslab; 2234 2235 if (cp->cache_flags & KMF_HASH) { 2236 cp->cache_hash_table = vmem_alloc(kmem_hash_arena, 2237 KMEM_HASH_INITIAL * sizeof (void *), VM_SLEEP); 2238 bzero(cp->cache_hash_table, 2239 KMEM_HASH_INITIAL * sizeof (void *)); 2240 cp->cache_hash_mask = KMEM_HASH_INITIAL - 1; 2241 cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; 2242 } 2243 2244 /* 2245 * Initialize the depot. 2246 */ 2247 mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL); 2248 2249 for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++) 2250 continue; 2251 2252 cp->cache_magtype = mtp; 2253 2254 /* 2255 * Initialize the CPU layer. 2256 */ 2257 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 2258 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 2259 mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL); 2260 ccp->cc_flags = cp->cache_flags; 2261 ccp->cc_rounds = -1; 2262 ccp->cc_prounds = -1; 2263 } 2264 2265 /* 2266 * Create the cache's kstats. 2267 */ 2268 if ((cp->cache_kstat = kstat_create("unix", 0, cp->cache_name, 2269 "kmem_cache", KSTAT_TYPE_NAMED, 2270 sizeof (kmem_cache_kstat) / sizeof (kstat_named_t), 2271 KSTAT_FLAG_VIRTUAL)) != NULL) { 2272 cp->cache_kstat->ks_data = &kmem_cache_kstat; 2273 cp->cache_kstat->ks_update = kmem_cache_kstat_update; 2274 cp->cache_kstat->ks_private = cp; 2275 cp->cache_kstat->ks_lock = &kmem_cache_kstat_lock; 2276 kstat_install(cp->cache_kstat); 2277 } 2278 2279 /* 2280 * Add the cache to the global list. This makes it visible 2281 * to kmem_update(), so the cache must be ready for business. 2282 */ 2283 mutex_enter(&kmem_cache_lock); 2284 cp->cache_next = cnext = &kmem_null_cache; 2285 cp->cache_prev = cprev = kmem_null_cache.cache_prev; 2286 cnext->cache_prev = cp; 2287 cprev->cache_next = cp; 2288 mutex_exit(&kmem_cache_lock); 2289 2290 if (kmem_ready) 2291 kmem_cache_magazine_enable(cp); 2292 2293 return (cp); 2294 } 2295 2296 void 2297 kmem_cache_destroy(kmem_cache_t *cp) 2298 { 2299 int cpu_seqid; 2300 2301 /* 2302 * Remove the cache from the global cache list so that no one else 2303 * can schedule tasks on its behalf, wait for any pending tasks to 2304 * complete, purge the cache, and then destroy it. 2305 */ 2306 mutex_enter(&kmem_cache_lock); 2307 cp->cache_prev->cache_next = cp->cache_next; 2308 cp->cache_next->cache_prev = cp->cache_prev; 2309 cp->cache_prev = cp->cache_next = NULL; 2310 mutex_exit(&kmem_cache_lock); 2311 2312 if (kmem_taskq != NULL) 2313 taskq_wait(kmem_taskq); 2314 2315 kmem_cache_magazine_purge(cp); 2316 2317 mutex_enter(&cp->cache_lock); 2318 if (cp->cache_buftotal != 0) 2319 cmn_err(CE_WARN, "kmem_cache_destroy: '%s' (%p) not empty", 2320 cp->cache_name, (void *)cp); 2321 cp->cache_reclaim = NULL; 2322 /* 2323 * The cache is now dead. There should be no further activity. 2324 * We enforce this by setting land mines in the constructor and 2325 * destructor routines that induce a kernel text fault if invoked. 2326 */ 2327 cp->cache_constructor = (int (*)(void *, void *, int))1; 2328 cp->cache_destructor = (void (*)(void *, void *))2; 2329 mutex_exit(&cp->cache_lock); 2330 2331 kstat_delete(cp->cache_kstat); 2332 2333 if (cp->cache_hash_table != NULL) 2334 vmem_free(kmem_hash_arena, cp->cache_hash_table, 2335 (cp->cache_hash_mask + 1) * sizeof (void *)); 2336 2337 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) 2338 mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); 2339 2340 mutex_destroy(&cp->cache_depot_lock); 2341 mutex_destroy(&cp->cache_lock); 2342 2343 vmem_free(kmem_cache_arena, cp, KMEM_CACHE_SIZE(max_ncpus)); 2344 } 2345 2346 /*ARGSUSED*/ 2347 static int 2348 kmem_cpu_setup(cpu_setup_t what, int id, void *arg) 2349 { 2350 ASSERT(MUTEX_HELD(&cpu_lock)); 2351 if (what == CPU_UNCONFIG) { 2352 kmem_cache_applyall(kmem_cache_magazine_purge, 2353 kmem_taskq, TQ_SLEEP); 2354 kmem_cache_applyall(kmem_cache_magazine_enable, 2355 kmem_taskq, TQ_SLEEP); 2356 } 2357 return (0); 2358 } 2359 2360 static void 2361 kmem_cache_init(int pass, int use_large_pages) 2362 { 2363 int i; 2364 size_t size; 2365 kmem_cache_t *cp; 2366 kmem_magtype_t *mtp; 2367 char name[KMEM_CACHE_NAMELEN + 1]; 2368 2369 for (i = 0; i < sizeof (kmem_magtype) / sizeof (*mtp); i++) { 2370 mtp = &kmem_magtype[i]; 2371 (void) sprintf(name, "kmem_magazine_%d", mtp->mt_magsize); 2372 mtp->mt_cache = kmem_cache_create(name, 2373 (mtp->mt_magsize + 1) * sizeof (void *), 2374 mtp->mt_align, NULL, NULL, NULL, NULL, 2375 kmem_msb_arena, KMC_NOHASH); 2376 } 2377 2378 kmem_slab_cache = kmem_cache_create("kmem_slab_cache", 2379 sizeof (kmem_slab_t), 0, NULL, NULL, NULL, NULL, 2380 kmem_msb_arena, KMC_NOHASH); 2381 2382 kmem_bufctl_cache = kmem_cache_create("kmem_bufctl_cache", 2383 sizeof (kmem_bufctl_t), 0, NULL, NULL, NULL, NULL, 2384 kmem_msb_arena, KMC_NOHASH); 2385 2386 kmem_bufctl_audit_cache = kmem_cache_create("kmem_bufctl_audit_cache", 2387 sizeof (kmem_bufctl_audit_t), 0, NULL, NULL, NULL, NULL, 2388 kmem_msb_arena, KMC_NOHASH); 2389 2390 if (pass == 2) { 2391 kmem_va_arena = vmem_create("kmem_va", 2392 NULL, 0, PAGESIZE, 2393 vmem_alloc, vmem_free, heap_arena, 2394 8 * PAGESIZE, VM_SLEEP); 2395 2396 if (use_large_pages) { 2397 kmem_default_arena = vmem_xcreate("kmem_default", 2398 NULL, 0, PAGESIZE, 2399 segkmem_alloc_lp, segkmem_free_lp, kmem_va_arena, 2400 0, VM_SLEEP); 2401 } else { 2402 kmem_default_arena = vmem_create("kmem_default", 2403 NULL, 0, PAGESIZE, 2404 segkmem_alloc, segkmem_free, kmem_va_arena, 2405 0, VM_SLEEP); 2406 } 2407 } else { 2408 /* 2409 * During the first pass, the kmem_alloc_* caches 2410 * are treated as metadata. 2411 */ 2412 kmem_default_arena = kmem_msb_arena; 2413 } 2414 2415 /* 2416 * Set up the default caches to back kmem_alloc() 2417 */ 2418 size = KMEM_ALIGN; 2419 for (i = 0; i < sizeof (kmem_alloc_sizes) / sizeof (int); i++) { 2420 size_t align = KMEM_ALIGN; 2421 size_t cache_size = kmem_alloc_sizes[i]; 2422 /* 2423 * If they allocate a multiple of the coherency granularity, 2424 * they get a coherency-granularity-aligned address. 2425 */ 2426 if (IS_P2ALIGNED(cache_size, 64)) 2427 align = 64; 2428 if (IS_P2ALIGNED(cache_size, PAGESIZE)) 2429 align = PAGESIZE; 2430 (void) sprintf(name, "kmem_alloc_%lu", cache_size); 2431 cp = kmem_cache_create(name, cache_size, align, 2432 NULL, NULL, NULL, NULL, NULL, KMC_KMEM_ALLOC); 2433 while (size <= cache_size) { 2434 kmem_alloc_table[(size - 1) >> KMEM_ALIGN_SHIFT] = cp; 2435 size += KMEM_ALIGN; 2436 } 2437 } 2438 } 2439 2440 void 2441 kmem_init(void) 2442 { 2443 kmem_cache_t *cp; 2444 int old_kmem_flags = kmem_flags; 2445 int use_large_pages = 0; 2446 size_t maxverify, minfirewall; 2447 2448 kstat_init(); 2449 2450 /* 2451 * Small-memory systems (< 24 MB) can't handle kmem_flags overhead. 2452 */ 2453 if (physmem < btop(24 << 20) && !(old_kmem_flags & KMF_STICKY)) 2454 kmem_flags = 0; 2455 2456 /* 2457 * Don't do firewalled allocations if the heap is less than 1TB 2458 * (i.e. on a 32-bit kernel) 2459 * The resulting VM_NEXTFIT allocations would create too much 2460 * fragmentation in a small heap. 2461 */ 2462 #if defined(_LP64) 2463 maxverify = minfirewall = PAGESIZE / 2; 2464 #else 2465 maxverify = minfirewall = ULONG_MAX; 2466 #endif 2467 2468 /* LINTED */ 2469 ASSERT(sizeof (kmem_cpu_cache_t) == KMEM_CPU_CACHE_SIZE); 2470 2471 kmem_null_cache.cache_next = &kmem_null_cache; 2472 kmem_null_cache.cache_prev = &kmem_null_cache; 2473 2474 kmem_metadata_arena = vmem_create("kmem_metadata", NULL, 0, PAGESIZE, 2475 vmem_alloc, vmem_free, heap_arena, 8 * PAGESIZE, 2476 VM_SLEEP | VMC_NO_QCACHE); 2477 2478 kmem_msb_arena = vmem_create("kmem_msb", NULL, 0, 2479 PAGESIZE, segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, 2480 VM_SLEEP); 2481 2482 kmem_cache_arena = vmem_create("kmem_cache", NULL, 0, KMEM_ALIGN, 2483 segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 2484 2485 kmem_hash_arena = vmem_create("kmem_hash", NULL, 0, KMEM_ALIGN, 2486 segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 2487 2488 kmem_log_arena = vmem_create("kmem_log", NULL, 0, KMEM_ALIGN, 2489 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 2490 2491 kmem_firewall_va_arena = vmem_create("kmem_firewall_va", 2492 NULL, 0, PAGESIZE, 2493 kmem_firewall_va_alloc, kmem_firewall_va_free, heap_arena, 2494 0, VM_SLEEP); 2495 2496 kmem_firewall_arena = vmem_create("kmem_firewall", NULL, 0, PAGESIZE, 2497 segkmem_alloc, segkmem_free, kmem_firewall_va_arena, 0, VM_SLEEP); 2498 2499 /* temporary oversize arena for mod_read_system_file */ 2500 kmem_oversize_arena = vmem_create("kmem_oversize", NULL, 0, PAGESIZE, 2501 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 2502 2503 kmem_null_cache.cache_next = &kmem_null_cache; 2504 kmem_null_cache.cache_prev = &kmem_null_cache; 2505 2506 kmem_reap_interval = 15 * hz; 2507 2508 /* 2509 * Read /etc/system. This is a chicken-and-egg problem because 2510 * kmem_flags may be set in /etc/system, but mod_read_system_file() 2511 * needs to use the allocator. The simplest solution is to create 2512 * all the standard kmem caches, read /etc/system, destroy all the 2513 * caches we just created, and then create them all again in light 2514 * of the (possibly) new kmem_flags and other kmem tunables. 2515 */ 2516 kmem_cache_init(1, 0); 2517 2518 mod_read_system_file(boothowto & RB_ASKNAME); 2519 2520 while ((cp = kmem_null_cache.cache_prev) != &kmem_null_cache) 2521 kmem_cache_destroy(cp); 2522 2523 vmem_destroy(kmem_oversize_arena); 2524 2525 if (old_kmem_flags & KMF_STICKY) 2526 kmem_flags = old_kmem_flags; 2527 2528 if (!(kmem_flags & KMF_AUDIT)) 2529 vmem_seg_size = offsetof(vmem_seg_t, vs_thread); 2530 2531 if (kmem_maxverify == 0) 2532 kmem_maxverify = maxverify; 2533 2534 if (kmem_minfirewall == 0) 2535 kmem_minfirewall = minfirewall; 2536 2537 /* 2538 * give segkmem a chance to figure out if we are using large pages 2539 * for the kernel heap 2540 */ 2541 use_large_pages = segkmem_lpsetup(); 2542 2543 /* 2544 * To protect against corruption, we keep the actual number of callers 2545 * KMF_LITE records seperate from the tunable. We arbitrarily clamp 2546 * to 16, since the overhead for small buffers quickly gets out of 2547 * hand. 2548 * 2549 * The real limit would depend on the needs of the largest KMC_NOHASH 2550 * cache. 2551 */ 2552 kmem_lite_count = MIN(MAX(0, kmem_lite_pcs), 16); 2553 kmem_lite_pcs = kmem_lite_count; 2554 2555 /* 2556 * Normally, we firewall oversized allocations when possible, but 2557 * if we are using large pages for kernel memory, and we don't have 2558 * any non-LITE debugging flags set, we want to allocate oversized 2559 * buffers from large pages, and so skip the firewalling. 2560 */ 2561 if (use_large_pages && 2562 ((kmem_flags & KMF_LITE) || !(kmem_flags & KMF_DEBUG))) { 2563 kmem_oversize_arena = vmem_xcreate("kmem_oversize", NULL, 0, 2564 PAGESIZE, segkmem_alloc_lp, segkmem_free_lp, heap_arena, 2565 0, VM_SLEEP); 2566 } else { 2567 kmem_oversize_arena = vmem_create("kmem_oversize", 2568 NULL, 0, PAGESIZE, 2569 segkmem_alloc, segkmem_free, kmem_minfirewall < ULONG_MAX? 2570 kmem_firewall_va_arena : heap_arena, 0, VM_SLEEP); 2571 } 2572 2573 kmem_cache_init(2, use_large_pages); 2574 2575 if (kmem_flags & (KMF_AUDIT | KMF_RANDOMIZE)) { 2576 if (kmem_transaction_log_size == 0) 2577 kmem_transaction_log_size = kmem_maxavail() / 50; 2578 kmem_transaction_log = kmem_log_init(kmem_transaction_log_size); 2579 } 2580 2581 if (kmem_flags & (KMF_CONTENTS | KMF_RANDOMIZE)) { 2582 if (kmem_content_log_size == 0) 2583 kmem_content_log_size = kmem_maxavail() / 50; 2584 kmem_content_log = kmem_log_init(kmem_content_log_size); 2585 } 2586 2587 kmem_failure_log = kmem_log_init(kmem_failure_log_size); 2588 2589 kmem_slab_log = kmem_log_init(kmem_slab_log_size); 2590 2591 /* 2592 * Initialize STREAMS message caches so allocb() is available. 2593 * This allows us to initialize the logging framework (cmn_err(9F), 2594 * strlog(9F), etc) so we can start recording messages. 2595 */ 2596 streams_msg_init(); 2597 /* 2598 * Initialize the ZSD framework in Zones so modules loaded henceforth 2599 * can register their callbacks. 2600 */ 2601 zone_zsd_init(); 2602 log_init(); 2603 taskq_init(); 2604 2605 kmem_cache_applyall(kmem_cache_magazine_enable, NULL, TQ_SLEEP); 2606 2607 kmem_ready = 1; 2608 2609 /* 2610 * Initialize the platform-specific aligned/DMA memory allocator. 2611 */ 2612 ka_init(); 2613 2614 /* 2615 * Initialize 32-bit ID cache. 2616 */ 2617 id32_init(); 2618 } 2619 2620 void 2621 kmem_thread_init(void) 2622 { 2623 kmem_taskq = taskq_create_instance("kmem_taskq", 0, 1, minclsyspri, 2624 300, INT_MAX, TASKQ_PREPOPULATE); 2625 } 2626 2627 void 2628 kmem_mp_init(void) 2629 { 2630 mutex_enter(&cpu_lock); 2631 register_cpu_setup_func(kmem_cpu_setup, NULL); 2632 mutex_exit(&cpu_lock); 2633 2634 kmem_update_timeout(NULL); 2635 } 2636