1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * Kernel memory allocator, as described in the following two papers: 30 * 31 * Jeff Bonwick, 32 * The Slab Allocator: An Object-Caching Kernel Memory Allocator. 33 * Proceedings of the Summer 1994 Usenix Conference. 34 * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf. 35 * 36 * Jeff Bonwick and Jonathan Adams, 37 * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 38 * Arbitrary Resources. 39 * Proceedings of the 2001 Usenix Conference. 40 * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf. 41 */ 42 43 #include <sys/kmem_impl.h> 44 #include <sys/vmem_impl.h> 45 #include <sys/param.h> 46 #include <sys/sysmacros.h> 47 #include <sys/vm.h> 48 #include <sys/proc.h> 49 #include <sys/tuneable.h> 50 #include <sys/systm.h> 51 #include <sys/cmn_err.h> 52 #include <sys/debug.h> 53 #include <sys/mutex.h> 54 #include <sys/bitmap.h> 55 #include <sys/atomic.h> 56 #include <sys/kobj.h> 57 #include <sys/disp.h> 58 #include <vm/seg_kmem.h> 59 #include <sys/log.h> 60 #include <sys/callb.h> 61 #include <sys/taskq.h> 62 #include <sys/modctl.h> 63 #include <sys/reboot.h> 64 #include <sys/id32.h> 65 #include <sys/zone.h> 66 #include <sys/netstack.h> 67 68 extern void streams_msg_init(void); 69 extern int segkp_fromheap; 70 extern void segkp_cache_free(void); 71 72 struct kmem_cache_kstat { 73 kstat_named_t kmc_buf_size; 74 kstat_named_t kmc_align; 75 kstat_named_t kmc_chunk_size; 76 kstat_named_t kmc_slab_size; 77 kstat_named_t kmc_alloc; 78 kstat_named_t kmc_alloc_fail; 79 kstat_named_t kmc_free; 80 kstat_named_t kmc_depot_alloc; 81 kstat_named_t kmc_depot_free; 82 kstat_named_t kmc_depot_contention; 83 kstat_named_t kmc_slab_alloc; 84 kstat_named_t kmc_slab_free; 85 kstat_named_t kmc_buf_constructed; 86 kstat_named_t kmc_buf_avail; 87 kstat_named_t kmc_buf_inuse; 88 kstat_named_t kmc_buf_total; 89 kstat_named_t kmc_buf_max; 90 kstat_named_t kmc_slab_create; 91 kstat_named_t kmc_slab_destroy; 92 kstat_named_t kmc_vmem_source; 93 kstat_named_t kmc_hash_size; 94 kstat_named_t kmc_hash_lookup_depth; 95 kstat_named_t kmc_hash_rescale; 96 kstat_named_t kmc_full_magazines; 97 kstat_named_t kmc_empty_magazines; 98 kstat_named_t kmc_magazine_size; 99 } kmem_cache_kstat = { 100 { "buf_size", KSTAT_DATA_UINT64 }, 101 { "align", KSTAT_DATA_UINT64 }, 102 { "chunk_size", KSTAT_DATA_UINT64 }, 103 { "slab_size", KSTAT_DATA_UINT64 }, 104 { "alloc", KSTAT_DATA_UINT64 }, 105 { "alloc_fail", KSTAT_DATA_UINT64 }, 106 { "free", KSTAT_DATA_UINT64 }, 107 { "depot_alloc", KSTAT_DATA_UINT64 }, 108 { "depot_free", KSTAT_DATA_UINT64 }, 109 { "depot_contention", KSTAT_DATA_UINT64 }, 110 { "slab_alloc", KSTAT_DATA_UINT64 }, 111 { "slab_free", KSTAT_DATA_UINT64 }, 112 { "buf_constructed", KSTAT_DATA_UINT64 }, 113 { "buf_avail", KSTAT_DATA_UINT64 }, 114 { "buf_inuse", KSTAT_DATA_UINT64 }, 115 { "buf_total", KSTAT_DATA_UINT64 }, 116 { "buf_max", KSTAT_DATA_UINT64 }, 117 { "slab_create", KSTAT_DATA_UINT64 }, 118 { "slab_destroy", KSTAT_DATA_UINT64 }, 119 { "vmem_source", KSTAT_DATA_UINT64 }, 120 { "hash_size", KSTAT_DATA_UINT64 }, 121 { "hash_lookup_depth", KSTAT_DATA_UINT64 }, 122 { "hash_rescale", KSTAT_DATA_UINT64 }, 123 { "full_magazines", KSTAT_DATA_UINT64 }, 124 { "empty_magazines", KSTAT_DATA_UINT64 }, 125 { "magazine_size", KSTAT_DATA_UINT64 }, 126 }; 127 128 static kmutex_t kmem_cache_kstat_lock; 129 130 /* 131 * The default set of caches to back kmem_alloc(). 132 * These sizes should be reevaluated periodically. 133 * 134 * We want allocations that are multiples of the coherency granularity 135 * (64 bytes) to be satisfied from a cache which is a multiple of 64 136 * bytes, so that it will be 64-byte aligned. For all multiples of 64, 137 * the next kmem_cache_size greater than or equal to it must be a 138 * multiple of 64. 139 */ 140 static const int kmem_alloc_sizes[] = { 141 1 * 8, 142 2 * 8, 143 3 * 8, 144 4 * 8, 5 * 8, 6 * 8, 7 * 8, 145 4 * 16, 5 * 16, 6 * 16, 7 * 16, 146 4 * 32, 5 * 32, 6 * 32, 7 * 32, 147 4 * 64, 5 * 64, 6 * 64, 7 * 64, 148 4 * 128, 5 * 128, 6 * 128, 7 * 128, 149 P2ALIGN(8192 / 7, 64), 150 P2ALIGN(8192 / 6, 64), 151 P2ALIGN(8192 / 5, 64), 152 P2ALIGN(8192 / 4, 64), 153 P2ALIGN(8192 / 3, 64), 154 P2ALIGN(8192 / 2, 64), 155 P2ALIGN(8192 / 1, 64), 156 4096 * 3, 157 8192 * 2, 158 8192 * 3, 159 8192 * 4, 160 }; 161 162 #define KMEM_MAXBUF 32768 163 164 static kmem_cache_t *kmem_alloc_table[KMEM_MAXBUF >> KMEM_ALIGN_SHIFT]; 165 166 static kmem_magtype_t kmem_magtype[] = { 167 { 1, 8, 3200, 65536 }, 168 { 3, 16, 256, 32768 }, 169 { 7, 32, 64, 16384 }, 170 { 15, 64, 0, 8192 }, 171 { 31, 64, 0, 4096 }, 172 { 47, 64, 0, 2048 }, 173 { 63, 64, 0, 1024 }, 174 { 95, 64, 0, 512 }, 175 { 143, 64, 0, 0 }, 176 }; 177 178 static uint32_t kmem_reaping; 179 static uint32_t kmem_reaping_idspace; 180 181 /* 182 * kmem tunables 183 */ 184 clock_t kmem_reap_interval; /* cache reaping rate [15 * HZ ticks] */ 185 int kmem_depot_contention = 3; /* max failed tryenters per real interval */ 186 pgcnt_t kmem_reapahead = 0; /* start reaping N pages before pageout */ 187 int kmem_panic = 1; /* whether to panic on error */ 188 int kmem_logging = 1; /* kmem_log_enter() override */ 189 uint32_t kmem_mtbf = 0; /* mean time between failures [default: off] */ 190 size_t kmem_transaction_log_size; /* transaction log size [2% of memory] */ 191 size_t kmem_content_log_size; /* content log size [2% of memory] */ 192 size_t kmem_failure_log_size; /* failure log [4 pages per CPU] */ 193 size_t kmem_slab_log_size; /* slab create log [4 pages per CPU] */ 194 size_t kmem_content_maxsave = 256; /* KMF_CONTENTS max bytes to log */ 195 size_t kmem_lite_minsize = 0; /* minimum buffer size for KMF_LITE */ 196 size_t kmem_lite_maxalign = 1024; /* maximum buffer alignment for KMF_LITE */ 197 int kmem_lite_pcs = 4; /* number of PCs to store in KMF_LITE mode */ 198 size_t kmem_maxverify; /* maximum bytes to inspect in debug routines */ 199 size_t kmem_minfirewall; /* hardware-enforced redzone threshold */ 200 201 #ifdef DEBUG 202 int kmem_flags = KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | KMF_CONTENTS; 203 #else 204 int kmem_flags = 0; 205 #endif 206 int kmem_ready; 207 208 static kmem_cache_t *kmem_slab_cache; 209 static kmem_cache_t *kmem_bufctl_cache; 210 static kmem_cache_t *kmem_bufctl_audit_cache; 211 212 static kmutex_t kmem_cache_lock; /* inter-cache linkage only */ 213 kmem_cache_t kmem_null_cache; 214 215 static taskq_t *kmem_taskq; 216 static kmutex_t kmem_flags_lock; 217 static vmem_t *kmem_metadata_arena; 218 static vmem_t *kmem_msb_arena; /* arena for metadata caches */ 219 static vmem_t *kmem_cache_arena; 220 static vmem_t *kmem_hash_arena; 221 static vmem_t *kmem_log_arena; 222 static vmem_t *kmem_oversize_arena; 223 static vmem_t *kmem_va_arena; 224 static vmem_t *kmem_default_arena; 225 static vmem_t *kmem_firewall_va_arena; 226 static vmem_t *kmem_firewall_arena; 227 228 kmem_log_header_t *kmem_transaction_log; 229 kmem_log_header_t *kmem_content_log; 230 kmem_log_header_t *kmem_failure_log; 231 kmem_log_header_t *kmem_slab_log; 232 233 static int kmem_lite_count; /* # of PCs in kmem_buftag_lite_t */ 234 235 #define KMEM_BUFTAG_LITE_ENTER(bt, count, caller) \ 236 if ((count) > 0) { \ 237 pc_t *_s = ((kmem_buftag_lite_t *)(bt))->bt_history; \ 238 pc_t *_e; \ 239 /* memmove() the old entries down one notch */ \ 240 for (_e = &_s[(count) - 1]; _e > _s; _e--) \ 241 *_e = *(_e - 1); \ 242 *_s = (uintptr_t)(caller); \ 243 } 244 245 #define KMERR_MODIFIED 0 /* buffer modified while on freelist */ 246 #define KMERR_REDZONE 1 /* redzone violation (write past end of buf) */ 247 #define KMERR_DUPFREE 2 /* freed a buffer twice */ 248 #define KMERR_BADADDR 3 /* freed a bad (unallocated) address */ 249 #define KMERR_BADBUFTAG 4 /* buftag corrupted */ 250 #define KMERR_BADBUFCTL 5 /* bufctl corrupted */ 251 #define KMERR_BADCACHE 6 /* freed a buffer to the wrong cache */ 252 #define KMERR_BADSIZE 7 /* alloc size != free size */ 253 #define KMERR_BADBASE 8 /* buffer base address wrong */ 254 255 struct { 256 hrtime_t kmp_timestamp; /* timestamp of panic */ 257 int kmp_error; /* type of kmem error */ 258 void *kmp_buffer; /* buffer that induced panic */ 259 void *kmp_realbuf; /* real start address for buffer */ 260 kmem_cache_t *kmp_cache; /* buffer's cache according to client */ 261 kmem_cache_t *kmp_realcache; /* actual cache containing buffer */ 262 kmem_slab_t *kmp_slab; /* slab accoring to kmem_findslab() */ 263 kmem_bufctl_t *kmp_bufctl; /* bufctl */ 264 } kmem_panic_info; 265 266 267 static void 268 copy_pattern(uint64_t pattern, void *buf_arg, size_t size) 269 { 270 uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 271 uint64_t *buf = buf_arg; 272 273 while (buf < bufend) 274 *buf++ = pattern; 275 } 276 277 static void * 278 verify_pattern(uint64_t pattern, void *buf_arg, size_t size) 279 { 280 uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 281 uint64_t *buf; 282 283 for (buf = buf_arg; buf < bufend; buf++) 284 if (*buf != pattern) 285 return (buf); 286 return (NULL); 287 } 288 289 static void * 290 verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size) 291 { 292 uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 293 uint64_t *buf; 294 295 for (buf = buf_arg; buf < bufend; buf++) { 296 if (*buf != old) { 297 copy_pattern(old, buf_arg, 298 (char *)buf - (char *)buf_arg); 299 return (buf); 300 } 301 *buf = new; 302 } 303 304 return (NULL); 305 } 306 307 static void 308 kmem_cache_applyall(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 309 { 310 kmem_cache_t *cp; 311 312 mutex_enter(&kmem_cache_lock); 313 for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache; 314 cp = cp->cache_next) 315 if (tq != NULL) 316 (void) taskq_dispatch(tq, (task_func_t *)func, cp, 317 tqflag); 318 else 319 func(cp); 320 mutex_exit(&kmem_cache_lock); 321 } 322 323 static void 324 kmem_cache_applyall_id(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 325 { 326 kmem_cache_t *cp; 327 328 mutex_enter(&kmem_cache_lock); 329 for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache; 330 cp = cp->cache_next) { 331 if (!(cp->cache_cflags & KMC_IDENTIFIER)) 332 continue; 333 if (tq != NULL) 334 (void) taskq_dispatch(tq, (task_func_t *)func, cp, 335 tqflag); 336 else 337 func(cp); 338 } 339 mutex_exit(&kmem_cache_lock); 340 } 341 342 /* 343 * Debugging support. Given a buffer address, find its slab. 344 */ 345 static kmem_slab_t * 346 kmem_findslab(kmem_cache_t *cp, void *buf) 347 { 348 kmem_slab_t *sp; 349 350 mutex_enter(&cp->cache_lock); 351 for (sp = cp->cache_nullslab.slab_next; 352 sp != &cp->cache_nullslab; sp = sp->slab_next) { 353 if (KMEM_SLAB_MEMBER(sp, buf)) { 354 mutex_exit(&cp->cache_lock); 355 return (sp); 356 } 357 } 358 mutex_exit(&cp->cache_lock); 359 360 return (NULL); 361 } 362 363 static void 364 kmem_error(int error, kmem_cache_t *cparg, void *bufarg) 365 { 366 kmem_buftag_t *btp = NULL; 367 kmem_bufctl_t *bcp = NULL; 368 kmem_cache_t *cp = cparg; 369 kmem_slab_t *sp; 370 uint64_t *off; 371 void *buf = bufarg; 372 373 kmem_logging = 0; /* stop logging when a bad thing happens */ 374 375 kmem_panic_info.kmp_timestamp = gethrtime(); 376 377 sp = kmem_findslab(cp, buf); 378 if (sp == NULL) { 379 for (cp = kmem_null_cache.cache_prev; cp != &kmem_null_cache; 380 cp = cp->cache_prev) { 381 if ((sp = kmem_findslab(cp, buf)) != NULL) 382 break; 383 } 384 } 385 386 if (sp == NULL) { 387 cp = NULL; 388 error = KMERR_BADADDR; 389 } else { 390 if (cp != cparg) 391 error = KMERR_BADCACHE; 392 else 393 buf = (char *)bufarg - ((uintptr_t)bufarg - 394 (uintptr_t)sp->slab_base) % cp->cache_chunksize; 395 if (buf != bufarg) 396 error = KMERR_BADBASE; 397 if (cp->cache_flags & KMF_BUFTAG) 398 btp = KMEM_BUFTAG(cp, buf); 399 if (cp->cache_flags & KMF_HASH) { 400 mutex_enter(&cp->cache_lock); 401 for (bcp = *KMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) 402 if (bcp->bc_addr == buf) 403 break; 404 mutex_exit(&cp->cache_lock); 405 if (bcp == NULL && btp != NULL) 406 bcp = btp->bt_bufctl; 407 if (kmem_findslab(cp->cache_bufctl_cache, bcp) == 408 NULL || P2PHASE((uintptr_t)bcp, KMEM_ALIGN) || 409 bcp->bc_addr != buf) { 410 error = KMERR_BADBUFCTL; 411 bcp = NULL; 412 } 413 } 414 } 415 416 kmem_panic_info.kmp_error = error; 417 kmem_panic_info.kmp_buffer = bufarg; 418 kmem_panic_info.kmp_realbuf = buf; 419 kmem_panic_info.kmp_cache = cparg; 420 kmem_panic_info.kmp_realcache = cp; 421 kmem_panic_info.kmp_slab = sp; 422 kmem_panic_info.kmp_bufctl = bcp; 423 424 printf("kernel memory allocator: "); 425 426 switch (error) { 427 428 case KMERR_MODIFIED: 429 printf("buffer modified after being freed\n"); 430 off = verify_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 431 if (off == NULL) /* shouldn't happen */ 432 off = buf; 433 printf("modification occurred at offset 0x%lx " 434 "(0x%llx replaced by 0x%llx)\n", 435 (uintptr_t)off - (uintptr_t)buf, 436 (longlong_t)KMEM_FREE_PATTERN, (longlong_t)*off); 437 break; 438 439 case KMERR_REDZONE: 440 printf("redzone violation: write past end of buffer\n"); 441 break; 442 443 case KMERR_BADADDR: 444 printf("invalid free: buffer not in cache\n"); 445 break; 446 447 case KMERR_DUPFREE: 448 printf("duplicate free: buffer freed twice\n"); 449 break; 450 451 case KMERR_BADBUFTAG: 452 printf("boundary tag corrupted\n"); 453 printf("bcp ^ bxstat = %lx, should be %lx\n", 454 (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat, 455 KMEM_BUFTAG_FREE); 456 break; 457 458 case KMERR_BADBUFCTL: 459 printf("bufctl corrupted\n"); 460 break; 461 462 case KMERR_BADCACHE: 463 printf("buffer freed to wrong cache\n"); 464 printf("buffer was allocated from %s,\n", cp->cache_name); 465 printf("caller attempting free to %s.\n", cparg->cache_name); 466 break; 467 468 case KMERR_BADSIZE: 469 printf("bad free: free size (%u) != alloc size (%u)\n", 470 KMEM_SIZE_DECODE(((uint32_t *)btp)[0]), 471 KMEM_SIZE_DECODE(((uint32_t *)btp)[1])); 472 break; 473 474 case KMERR_BADBASE: 475 printf("bad free: free address (%p) != alloc address (%p)\n", 476 bufarg, buf); 477 break; 478 } 479 480 printf("buffer=%p bufctl=%p cache: %s\n", 481 bufarg, (void *)bcp, cparg->cache_name); 482 483 if (bcp != NULL && (cp->cache_flags & KMF_AUDIT) && 484 error != KMERR_BADBUFCTL) { 485 int d; 486 timestruc_t ts; 487 kmem_bufctl_audit_t *bcap = (kmem_bufctl_audit_t *)bcp; 488 489 hrt2ts(kmem_panic_info.kmp_timestamp - bcap->bc_timestamp, &ts); 490 printf("previous transaction on buffer %p:\n", buf); 491 printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n", 492 (void *)bcap->bc_thread, ts.tv_sec, ts.tv_nsec, 493 (void *)sp, cp->cache_name); 494 for (d = 0; d < MIN(bcap->bc_depth, KMEM_STACK_DEPTH); d++) { 495 ulong_t off; 496 char *sym = kobj_getsymname(bcap->bc_stack[d], &off); 497 printf("%s+%lx\n", sym ? sym : "?", off); 498 } 499 } 500 if (kmem_panic > 0) 501 panic("kernel heap corruption detected"); 502 if (kmem_panic == 0) 503 debug_enter(NULL); 504 kmem_logging = 1; /* resume logging */ 505 } 506 507 static kmem_log_header_t * 508 kmem_log_init(size_t logsize) 509 { 510 kmem_log_header_t *lhp; 511 int nchunks = 4 * max_ncpus; 512 size_t lhsize = (size_t)&((kmem_log_header_t *)0)->lh_cpu[max_ncpus]; 513 int i; 514 515 /* 516 * Make sure that lhp->lh_cpu[] is nicely aligned 517 * to prevent false sharing of cache lines. 518 */ 519 lhsize = P2ROUNDUP(lhsize, KMEM_ALIGN); 520 lhp = vmem_xalloc(kmem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0, 521 NULL, NULL, VM_SLEEP); 522 bzero(lhp, lhsize); 523 524 mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL); 525 lhp->lh_nchunks = nchunks; 526 lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE); 527 lhp->lh_base = vmem_alloc(kmem_log_arena, 528 lhp->lh_chunksize * nchunks, VM_SLEEP); 529 lhp->lh_free = vmem_alloc(kmem_log_arena, 530 nchunks * sizeof (int), VM_SLEEP); 531 bzero(lhp->lh_base, lhp->lh_chunksize * nchunks); 532 533 for (i = 0; i < max_ncpus; i++) { 534 kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; 535 mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL); 536 clhp->clh_chunk = i; 537 } 538 539 for (i = max_ncpus; i < nchunks; i++) 540 lhp->lh_free[i] = i; 541 542 lhp->lh_head = max_ncpus; 543 lhp->lh_tail = 0; 544 545 return (lhp); 546 } 547 548 static void * 549 kmem_log_enter(kmem_log_header_t *lhp, void *data, size_t size) 550 { 551 void *logspace; 552 kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[CPU->cpu_seqid]; 553 554 if (lhp == NULL || kmem_logging == 0 || panicstr) 555 return (NULL); 556 557 mutex_enter(&clhp->clh_lock); 558 clhp->clh_hits++; 559 if (size > clhp->clh_avail) { 560 mutex_enter(&lhp->lh_lock); 561 lhp->lh_hits++; 562 lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk; 563 lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks; 564 clhp->clh_chunk = lhp->lh_free[lhp->lh_head]; 565 lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks; 566 clhp->clh_current = lhp->lh_base + 567 clhp->clh_chunk * lhp->lh_chunksize; 568 clhp->clh_avail = lhp->lh_chunksize; 569 if (size > lhp->lh_chunksize) 570 size = lhp->lh_chunksize; 571 mutex_exit(&lhp->lh_lock); 572 } 573 logspace = clhp->clh_current; 574 clhp->clh_current += size; 575 clhp->clh_avail -= size; 576 bcopy(data, logspace, size); 577 mutex_exit(&clhp->clh_lock); 578 return (logspace); 579 } 580 581 #define KMEM_AUDIT(lp, cp, bcp) \ 582 { \ 583 kmem_bufctl_audit_t *_bcp = (kmem_bufctl_audit_t *)(bcp); \ 584 _bcp->bc_timestamp = gethrtime(); \ 585 _bcp->bc_thread = curthread; \ 586 _bcp->bc_depth = getpcstack(_bcp->bc_stack, KMEM_STACK_DEPTH); \ 587 _bcp->bc_lastlog = kmem_log_enter((lp), _bcp, sizeof (*_bcp)); \ 588 } 589 590 static void 591 kmem_log_event(kmem_log_header_t *lp, kmem_cache_t *cp, 592 kmem_slab_t *sp, void *addr) 593 { 594 kmem_bufctl_audit_t bca; 595 596 bzero(&bca, sizeof (kmem_bufctl_audit_t)); 597 bca.bc_addr = addr; 598 bca.bc_slab = sp; 599 bca.bc_cache = cp; 600 KMEM_AUDIT(lp, cp, &bca); 601 } 602 603 /* 604 * Create a new slab for cache cp. 605 */ 606 static kmem_slab_t * 607 kmem_slab_create(kmem_cache_t *cp, int kmflag) 608 { 609 size_t slabsize = cp->cache_slabsize; 610 size_t chunksize = cp->cache_chunksize; 611 int cache_flags = cp->cache_flags; 612 size_t color, chunks; 613 char *buf, *slab; 614 kmem_slab_t *sp; 615 kmem_bufctl_t *bcp; 616 vmem_t *vmp = cp->cache_arena; 617 618 color = cp->cache_color + cp->cache_align; 619 if (color > cp->cache_maxcolor) 620 color = cp->cache_mincolor; 621 cp->cache_color = color; 622 623 slab = vmem_alloc(vmp, slabsize, kmflag & KM_VMFLAGS); 624 625 if (slab == NULL) 626 goto vmem_alloc_failure; 627 628 ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0); 629 630 if (!(cp->cache_cflags & KMC_NOTOUCH)) 631 copy_pattern(KMEM_UNINITIALIZED_PATTERN, slab, slabsize); 632 633 if (cache_flags & KMF_HASH) { 634 if ((sp = kmem_cache_alloc(kmem_slab_cache, kmflag)) == NULL) 635 goto slab_alloc_failure; 636 chunks = (slabsize - color) / chunksize; 637 } else { 638 sp = KMEM_SLAB(cp, slab); 639 chunks = (slabsize - sizeof (kmem_slab_t) - color) / chunksize; 640 } 641 642 sp->slab_cache = cp; 643 sp->slab_head = NULL; 644 sp->slab_refcnt = 0; 645 sp->slab_base = buf = slab + color; 646 sp->slab_chunks = chunks; 647 648 ASSERT(chunks > 0); 649 while (chunks-- != 0) { 650 if (cache_flags & KMF_HASH) { 651 bcp = kmem_cache_alloc(cp->cache_bufctl_cache, kmflag); 652 if (bcp == NULL) 653 goto bufctl_alloc_failure; 654 if (cache_flags & KMF_AUDIT) { 655 kmem_bufctl_audit_t *bcap = 656 (kmem_bufctl_audit_t *)bcp; 657 bzero(bcap, sizeof (kmem_bufctl_audit_t)); 658 bcap->bc_cache = cp; 659 } 660 bcp->bc_addr = buf; 661 bcp->bc_slab = sp; 662 } else { 663 bcp = KMEM_BUFCTL(cp, buf); 664 } 665 if (cache_flags & KMF_BUFTAG) { 666 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 667 btp->bt_redzone = KMEM_REDZONE_PATTERN; 668 btp->bt_bufctl = bcp; 669 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 670 if (cache_flags & KMF_DEADBEEF) { 671 copy_pattern(KMEM_FREE_PATTERN, buf, 672 cp->cache_verify); 673 } 674 } 675 bcp->bc_next = sp->slab_head; 676 sp->slab_head = bcp; 677 buf += chunksize; 678 } 679 680 kmem_log_event(kmem_slab_log, cp, sp, slab); 681 682 return (sp); 683 684 bufctl_alloc_failure: 685 686 while ((bcp = sp->slab_head) != NULL) { 687 sp->slab_head = bcp->bc_next; 688 kmem_cache_free(cp->cache_bufctl_cache, bcp); 689 } 690 kmem_cache_free(kmem_slab_cache, sp); 691 692 slab_alloc_failure: 693 694 vmem_free(vmp, slab, slabsize); 695 696 vmem_alloc_failure: 697 698 kmem_log_event(kmem_failure_log, cp, NULL, NULL); 699 atomic_add_64(&cp->cache_alloc_fail, 1); 700 701 return (NULL); 702 } 703 704 /* 705 * Destroy a slab. 706 */ 707 static void 708 kmem_slab_destroy(kmem_cache_t *cp, kmem_slab_t *sp) 709 { 710 vmem_t *vmp = cp->cache_arena; 711 void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum); 712 713 if (cp->cache_flags & KMF_HASH) { 714 kmem_bufctl_t *bcp; 715 while ((bcp = sp->slab_head) != NULL) { 716 sp->slab_head = bcp->bc_next; 717 kmem_cache_free(cp->cache_bufctl_cache, bcp); 718 } 719 kmem_cache_free(kmem_slab_cache, sp); 720 } 721 vmem_free(vmp, slab, cp->cache_slabsize); 722 } 723 724 /* 725 * Allocate a raw (unconstructed) buffer from cp's slab layer. 726 */ 727 static void * 728 kmem_slab_alloc(kmem_cache_t *cp, int kmflag) 729 { 730 kmem_bufctl_t *bcp, **hash_bucket; 731 kmem_slab_t *sp; 732 void *buf; 733 734 mutex_enter(&cp->cache_lock); 735 cp->cache_slab_alloc++; 736 sp = cp->cache_freelist; 737 ASSERT(sp->slab_cache == cp); 738 if (sp->slab_head == NULL) { 739 ASSERT(cp->cache_bufslab == 0); 740 741 /* 742 * The freelist is empty. Create a new slab. 743 */ 744 mutex_exit(&cp->cache_lock); 745 if ((sp = kmem_slab_create(cp, kmflag)) == NULL) 746 return (NULL); 747 mutex_enter(&cp->cache_lock); 748 cp->cache_slab_create++; 749 if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) 750 cp->cache_bufmax = cp->cache_buftotal; 751 cp->cache_bufslab += sp->slab_chunks; 752 sp->slab_next = cp->cache_freelist; 753 sp->slab_prev = cp->cache_freelist->slab_prev; 754 sp->slab_next->slab_prev = sp; 755 sp->slab_prev->slab_next = sp; 756 cp->cache_freelist = sp; 757 } 758 759 cp->cache_bufslab--; 760 sp->slab_refcnt++; 761 ASSERT(sp->slab_refcnt <= sp->slab_chunks); 762 763 /* 764 * If we're taking the last buffer in the slab, 765 * remove the slab from the cache's freelist. 766 */ 767 bcp = sp->slab_head; 768 if ((sp->slab_head = bcp->bc_next) == NULL) { 769 cp->cache_freelist = sp->slab_next; 770 ASSERT(sp->slab_refcnt == sp->slab_chunks); 771 } 772 773 if (cp->cache_flags & KMF_HASH) { 774 /* 775 * Add buffer to allocated-address hash table. 776 */ 777 buf = bcp->bc_addr; 778 hash_bucket = KMEM_HASH(cp, buf); 779 bcp->bc_next = *hash_bucket; 780 *hash_bucket = bcp; 781 if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 782 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 783 } 784 } else { 785 buf = KMEM_BUF(cp, bcp); 786 } 787 788 ASSERT(KMEM_SLAB_MEMBER(sp, buf)); 789 790 mutex_exit(&cp->cache_lock); 791 792 return (buf); 793 } 794 795 /* 796 * Free a raw (unconstructed) buffer to cp's slab layer. 797 */ 798 static void 799 kmem_slab_free(kmem_cache_t *cp, void *buf) 800 { 801 kmem_slab_t *sp; 802 kmem_bufctl_t *bcp, **prev_bcpp; 803 804 ASSERT(buf != NULL); 805 806 mutex_enter(&cp->cache_lock); 807 cp->cache_slab_free++; 808 809 if (cp->cache_flags & KMF_HASH) { 810 /* 811 * Look up buffer in allocated-address hash table. 812 */ 813 prev_bcpp = KMEM_HASH(cp, buf); 814 while ((bcp = *prev_bcpp) != NULL) { 815 if (bcp->bc_addr == buf) { 816 *prev_bcpp = bcp->bc_next; 817 sp = bcp->bc_slab; 818 break; 819 } 820 cp->cache_lookup_depth++; 821 prev_bcpp = &bcp->bc_next; 822 } 823 } else { 824 bcp = KMEM_BUFCTL(cp, buf); 825 sp = KMEM_SLAB(cp, buf); 826 } 827 828 if (bcp == NULL || sp->slab_cache != cp || !KMEM_SLAB_MEMBER(sp, buf)) { 829 mutex_exit(&cp->cache_lock); 830 kmem_error(KMERR_BADADDR, cp, buf); 831 return; 832 } 833 834 if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 835 if (cp->cache_flags & KMF_CONTENTS) 836 ((kmem_bufctl_audit_t *)bcp)->bc_contents = 837 kmem_log_enter(kmem_content_log, buf, 838 cp->cache_contents); 839 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 840 } 841 842 /* 843 * If this slab isn't currently on the freelist, put it there. 844 */ 845 if (sp->slab_head == NULL) { 846 ASSERT(sp->slab_refcnt == sp->slab_chunks); 847 ASSERT(cp->cache_freelist != sp); 848 sp->slab_next->slab_prev = sp->slab_prev; 849 sp->slab_prev->slab_next = sp->slab_next; 850 sp->slab_next = cp->cache_freelist; 851 sp->slab_prev = cp->cache_freelist->slab_prev; 852 sp->slab_next->slab_prev = sp; 853 sp->slab_prev->slab_next = sp; 854 cp->cache_freelist = sp; 855 } 856 857 bcp->bc_next = sp->slab_head; 858 sp->slab_head = bcp; 859 860 cp->cache_bufslab++; 861 ASSERT(sp->slab_refcnt >= 1); 862 if (--sp->slab_refcnt == 0) { 863 /* 864 * There are no outstanding allocations from this slab, 865 * so we can reclaim the memory. 866 */ 867 sp->slab_next->slab_prev = sp->slab_prev; 868 sp->slab_prev->slab_next = sp->slab_next; 869 if (sp == cp->cache_freelist) 870 cp->cache_freelist = sp->slab_next; 871 cp->cache_slab_destroy++; 872 cp->cache_buftotal -= sp->slab_chunks; 873 cp->cache_bufslab -= sp->slab_chunks; 874 mutex_exit(&cp->cache_lock); 875 kmem_slab_destroy(cp, sp); 876 return; 877 } 878 mutex_exit(&cp->cache_lock); 879 } 880 881 static int 882 kmem_cache_alloc_debug(kmem_cache_t *cp, void *buf, int kmflag, int construct, 883 caddr_t caller) 884 { 885 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 886 kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 887 uint32_t mtbf; 888 889 if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 890 kmem_error(KMERR_BADBUFTAG, cp, buf); 891 return (-1); 892 } 893 894 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_ALLOC; 895 896 if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 897 kmem_error(KMERR_BADBUFCTL, cp, buf); 898 return (-1); 899 } 900 901 if (cp->cache_flags & KMF_DEADBEEF) { 902 if (!construct && (cp->cache_flags & KMF_LITE)) { 903 if (*(uint64_t *)buf != KMEM_FREE_PATTERN) { 904 kmem_error(KMERR_MODIFIED, cp, buf); 905 return (-1); 906 } 907 if (cp->cache_constructor != NULL) 908 *(uint64_t *)buf = btp->bt_redzone; 909 else 910 *(uint64_t *)buf = KMEM_UNINITIALIZED_PATTERN; 911 } else { 912 construct = 1; 913 if (verify_and_copy_pattern(KMEM_FREE_PATTERN, 914 KMEM_UNINITIALIZED_PATTERN, buf, 915 cp->cache_verify)) { 916 kmem_error(KMERR_MODIFIED, cp, buf); 917 return (-1); 918 } 919 } 920 } 921 btp->bt_redzone = KMEM_REDZONE_PATTERN; 922 923 if ((mtbf = kmem_mtbf | cp->cache_mtbf) != 0 && 924 gethrtime() % mtbf == 0 && 925 (kmflag & (KM_NOSLEEP | KM_PANIC)) == KM_NOSLEEP) { 926 kmem_log_event(kmem_failure_log, cp, NULL, NULL); 927 if (!construct && cp->cache_destructor != NULL) 928 cp->cache_destructor(buf, cp->cache_private); 929 } else { 930 mtbf = 0; 931 } 932 933 if (mtbf || (construct && cp->cache_constructor != NULL && 934 cp->cache_constructor(buf, cp->cache_private, kmflag) != 0)) { 935 atomic_add_64(&cp->cache_alloc_fail, 1); 936 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 937 if (cp->cache_flags & KMF_DEADBEEF) 938 copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 939 kmem_slab_free(cp, buf); 940 return (-1); 941 } 942 943 if (cp->cache_flags & KMF_AUDIT) { 944 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 945 } 946 947 if ((cp->cache_flags & KMF_LITE) && 948 !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 949 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 950 } 951 952 return (0); 953 } 954 955 static int 956 kmem_cache_free_debug(kmem_cache_t *cp, void *buf, caddr_t caller) 957 { 958 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 959 kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 960 kmem_slab_t *sp; 961 962 if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_ALLOC)) { 963 if (btp->bt_bxstat == ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 964 kmem_error(KMERR_DUPFREE, cp, buf); 965 return (-1); 966 } 967 sp = kmem_findslab(cp, buf); 968 if (sp == NULL || sp->slab_cache != cp) 969 kmem_error(KMERR_BADADDR, cp, buf); 970 else 971 kmem_error(KMERR_REDZONE, cp, buf); 972 return (-1); 973 } 974 975 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 976 977 if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 978 kmem_error(KMERR_BADBUFCTL, cp, buf); 979 return (-1); 980 } 981 982 if (btp->bt_redzone != KMEM_REDZONE_PATTERN) { 983 kmem_error(KMERR_REDZONE, cp, buf); 984 return (-1); 985 } 986 987 if (cp->cache_flags & KMF_AUDIT) { 988 if (cp->cache_flags & KMF_CONTENTS) 989 bcp->bc_contents = kmem_log_enter(kmem_content_log, 990 buf, cp->cache_contents); 991 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 992 } 993 994 if ((cp->cache_flags & KMF_LITE) && 995 !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 996 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 997 } 998 999 if (cp->cache_flags & KMF_DEADBEEF) { 1000 if (cp->cache_flags & KMF_LITE) 1001 btp->bt_redzone = *(uint64_t *)buf; 1002 else if (cp->cache_destructor != NULL) 1003 cp->cache_destructor(buf, cp->cache_private); 1004 1005 copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 1006 } 1007 1008 return (0); 1009 } 1010 1011 /* 1012 * Free each object in magazine mp to cp's slab layer, and free mp itself. 1013 */ 1014 static void 1015 kmem_magazine_destroy(kmem_cache_t *cp, kmem_magazine_t *mp, int nrounds) 1016 { 1017 int round; 1018 1019 ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 1020 1021 for (round = 0; round < nrounds; round++) { 1022 void *buf = mp->mag_round[round]; 1023 1024 if (cp->cache_flags & KMF_DEADBEEF) { 1025 if (verify_pattern(KMEM_FREE_PATTERN, buf, 1026 cp->cache_verify) != NULL) { 1027 kmem_error(KMERR_MODIFIED, cp, buf); 1028 continue; 1029 } 1030 if ((cp->cache_flags & KMF_LITE) && 1031 cp->cache_destructor != NULL) { 1032 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1033 *(uint64_t *)buf = btp->bt_redzone; 1034 cp->cache_destructor(buf, cp->cache_private); 1035 *(uint64_t *)buf = KMEM_FREE_PATTERN; 1036 } 1037 } else if (cp->cache_destructor != NULL) { 1038 cp->cache_destructor(buf, cp->cache_private); 1039 } 1040 1041 kmem_slab_free(cp, buf); 1042 } 1043 ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 1044 kmem_cache_free(cp->cache_magtype->mt_cache, mp); 1045 } 1046 1047 /* 1048 * Allocate a magazine from the depot. 1049 */ 1050 static kmem_magazine_t * 1051 kmem_depot_alloc(kmem_cache_t *cp, kmem_maglist_t *mlp) 1052 { 1053 kmem_magazine_t *mp; 1054 1055 /* 1056 * If we can't get the depot lock without contention, 1057 * update our contention count. We use the depot 1058 * contention rate to determine whether we need to 1059 * increase the magazine size for better scalability. 1060 */ 1061 if (!mutex_tryenter(&cp->cache_depot_lock)) { 1062 mutex_enter(&cp->cache_depot_lock); 1063 cp->cache_depot_contention++; 1064 } 1065 1066 if ((mp = mlp->ml_list) != NULL) { 1067 ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 1068 mlp->ml_list = mp->mag_next; 1069 if (--mlp->ml_total < mlp->ml_min) 1070 mlp->ml_min = mlp->ml_total; 1071 mlp->ml_alloc++; 1072 } 1073 1074 mutex_exit(&cp->cache_depot_lock); 1075 1076 return (mp); 1077 } 1078 1079 /* 1080 * Free a magazine to the depot. 1081 */ 1082 static void 1083 kmem_depot_free(kmem_cache_t *cp, kmem_maglist_t *mlp, kmem_magazine_t *mp) 1084 { 1085 mutex_enter(&cp->cache_depot_lock); 1086 ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 1087 mp->mag_next = mlp->ml_list; 1088 mlp->ml_list = mp; 1089 mlp->ml_total++; 1090 mutex_exit(&cp->cache_depot_lock); 1091 } 1092 1093 /* 1094 * Update the working set statistics for cp's depot. 1095 */ 1096 static void 1097 kmem_depot_ws_update(kmem_cache_t *cp) 1098 { 1099 mutex_enter(&cp->cache_depot_lock); 1100 cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; 1101 cp->cache_full.ml_min = cp->cache_full.ml_total; 1102 cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; 1103 cp->cache_empty.ml_min = cp->cache_empty.ml_total; 1104 mutex_exit(&cp->cache_depot_lock); 1105 } 1106 1107 /* 1108 * Reap all magazines that have fallen out of the depot's working set. 1109 */ 1110 static void 1111 kmem_depot_ws_reap(kmem_cache_t *cp) 1112 { 1113 long reap; 1114 kmem_magazine_t *mp; 1115 1116 ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 1117 1118 reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 1119 while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL) 1120 kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); 1121 1122 reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); 1123 while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL) 1124 kmem_magazine_destroy(cp, mp, 0); 1125 } 1126 1127 static void 1128 kmem_cpu_reload(kmem_cpu_cache_t *ccp, kmem_magazine_t *mp, int rounds) 1129 { 1130 ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) || 1131 (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize)); 1132 ASSERT(ccp->cc_magsize > 0); 1133 1134 ccp->cc_ploaded = ccp->cc_loaded; 1135 ccp->cc_prounds = ccp->cc_rounds; 1136 ccp->cc_loaded = mp; 1137 ccp->cc_rounds = rounds; 1138 } 1139 1140 /* 1141 * Allocate a constructed object from cache cp. 1142 */ 1143 void * 1144 kmem_cache_alloc(kmem_cache_t *cp, int kmflag) 1145 { 1146 kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 1147 kmem_magazine_t *fmp; 1148 void *buf; 1149 1150 mutex_enter(&ccp->cc_lock); 1151 for (;;) { 1152 /* 1153 * If there's an object available in the current CPU's 1154 * loaded magazine, just take it and return. 1155 */ 1156 if (ccp->cc_rounds > 0) { 1157 buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds]; 1158 ccp->cc_alloc++; 1159 mutex_exit(&ccp->cc_lock); 1160 if ((ccp->cc_flags & KMF_BUFTAG) && 1161 kmem_cache_alloc_debug(cp, buf, kmflag, 0, 1162 caller()) == -1) { 1163 if (kmflag & KM_NOSLEEP) 1164 return (NULL); 1165 mutex_enter(&ccp->cc_lock); 1166 continue; 1167 } 1168 return (buf); 1169 } 1170 1171 /* 1172 * The loaded magazine is empty. If the previously loaded 1173 * magazine was full, exchange them and try again. 1174 */ 1175 if (ccp->cc_prounds > 0) { 1176 kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 1177 continue; 1178 } 1179 1180 /* 1181 * If the magazine layer is disabled, break out now. 1182 */ 1183 if (ccp->cc_magsize == 0) 1184 break; 1185 1186 /* 1187 * Try to get a full magazine from the depot. 1188 */ 1189 fmp = kmem_depot_alloc(cp, &cp->cache_full); 1190 if (fmp != NULL) { 1191 if (ccp->cc_ploaded != NULL) 1192 kmem_depot_free(cp, &cp->cache_empty, 1193 ccp->cc_ploaded); 1194 kmem_cpu_reload(ccp, fmp, ccp->cc_magsize); 1195 continue; 1196 } 1197 1198 /* 1199 * There are no full magazines in the depot, 1200 * so fall through to the slab layer. 1201 */ 1202 break; 1203 } 1204 mutex_exit(&ccp->cc_lock); 1205 1206 /* 1207 * We couldn't allocate a constructed object from the magazine layer, 1208 * so get a raw buffer from the slab layer and apply its constructor. 1209 */ 1210 buf = kmem_slab_alloc(cp, kmflag); 1211 1212 if (buf == NULL) 1213 return (NULL); 1214 1215 if (cp->cache_flags & KMF_BUFTAG) { 1216 /* 1217 * Make kmem_cache_alloc_debug() apply the constructor for us. 1218 */ 1219 if (kmem_cache_alloc_debug(cp, buf, kmflag, 1, 1220 caller()) == -1) { 1221 if (kmflag & KM_NOSLEEP) 1222 return (NULL); 1223 /* 1224 * kmem_cache_alloc_debug() detected corruption 1225 * but didn't panic (kmem_panic <= 0). Try again. 1226 */ 1227 return (kmem_cache_alloc(cp, kmflag)); 1228 } 1229 return (buf); 1230 } 1231 1232 if (cp->cache_constructor != NULL && 1233 cp->cache_constructor(buf, cp->cache_private, kmflag) != 0) { 1234 atomic_add_64(&cp->cache_alloc_fail, 1); 1235 kmem_slab_free(cp, buf); 1236 return (NULL); 1237 } 1238 1239 return (buf); 1240 } 1241 1242 /* 1243 * Free a constructed object to cache cp. 1244 */ 1245 void 1246 kmem_cache_free(kmem_cache_t *cp, void *buf) 1247 { 1248 kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 1249 kmem_magazine_t *emp; 1250 kmem_magtype_t *mtp; 1251 1252 if (ccp->cc_flags & KMF_BUFTAG) 1253 if (kmem_cache_free_debug(cp, buf, caller()) == -1) 1254 return; 1255 1256 mutex_enter(&ccp->cc_lock); 1257 for (;;) { 1258 /* 1259 * If there's a slot available in the current CPU's 1260 * loaded magazine, just put the object there and return. 1261 */ 1262 if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 1263 ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf; 1264 ccp->cc_free++; 1265 mutex_exit(&ccp->cc_lock); 1266 return; 1267 } 1268 1269 /* 1270 * The loaded magazine is full. If the previously loaded 1271 * magazine was empty, exchange them and try again. 1272 */ 1273 if (ccp->cc_prounds == 0) { 1274 kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 1275 continue; 1276 } 1277 1278 /* 1279 * If the magazine layer is disabled, break out now. 1280 */ 1281 if (ccp->cc_magsize == 0) 1282 break; 1283 1284 /* 1285 * Try to get an empty magazine from the depot. 1286 */ 1287 emp = kmem_depot_alloc(cp, &cp->cache_empty); 1288 if (emp != NULL) { 1289 if (ccp->cc_ploaded != NULL) 1290 kmem_depot_free(cp, &cp->cache_full, 1291 ccp->cc_ploaded); 1292 kmem_cpu_reload(ccp, emp, 0); 1293 continue; 1294 } 1295 1296 /* 1297 * There are no empty magazines in the depot, 1298 * so try to allocate a new one. We must drop all locks 1299 * across kmem_cache_alloc() because lower layers may 1300 * attempt to allocate from this cache. 1301 */ 1302 mtp = cp->cache_magtype; 1303 mutex_exit(&ccp->cc_lock); 1304 emp = kmem_cache_alloc(mtp->mt_cache, KM_NOSLEEP); 1305 mutex_enter(&ccp->cc_lock); 1306 1307 if (emp != NULL) { 1308 /* 1309 * We successfully allocated an empty magazine. 1310 * However, we had to drop ccp->cc_lock to do it, 1311 * so the cache's magazine size may have changed. 1312 * If so, free the magazine and try again. 1313 */ 1314 if (ccp->cc_magsize != mtp->mt_magsize) { 1315 mutex_exit(&ccp->cc_lock); 1316 kmem_cache_free(mtp->mt_cache, emp); 1317 mutex_enter(&ccp->cc_lock); 1318 continue; 1319 } 1320 1321 /* 1322 * We got a magazine of the right size. Add it to 1323 * the depot and try the whole dance again. 1324 */ 1325 kmem_depot_free(cp, &cp->cache_empty, emp); 1326 continue; 1327 } 1328 1329 /* 1330 * We couldn't allocate an empty magazine, 1331 * so fall through to the slab layer. 1332 */ 1333 break; 1334 } 1335 mutex_exit(&ccp->cc_lock); 1336 1337 /* 1338 * We couldn't free our constructed object to the magazine layer, 1339 * so apply its destructor and free it to the slab layer. 1340 * Note that if KMF_DEADBEEF is in effect and KMF_LITE is not, 1341 * kmem_cache_free_debug() will have already applied the destructor. 1342 */ 1343 if ((cp->cache_flags & (KMF_DEADBEEF | KMF_LITE)) != KMF_DEADBEEF && 1344 cp->cache_destructor != NULL) { 1345 if (cp->cache_flags & KMF_DEADBEEF) { /* KMF_LITE implied */ 1346 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1347 *(uint64_t *)buf = btp->bt_redzone; 1348 cp->cache_destructor(buf, cp->cache_private); 1349 *(uint64_t *)buf = KMEM_FREE_PATTERN; 1350 } else { 1351 cp->cache_destructor(buf, cp->cache_private); 1352 } 1353 } 1354 1355 kmem_slab_free(cp, buf); 1356 } 1357 1358 void * 1359 kmem_zalloc(size_t size, int kmflag) 1360 { 1361 size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 1362 void *buf; 1363 1364 if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 1365 kmem_cache_t *cp = kmem_alloc_table[index]; 1366 buf = kmem_cache_alloc(cp, kmflag); 1367 if (buf != NULL) { 1368 if (cp->cache_flags & KMF_BUFTAG) { 1369 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1370 ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 1371 ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 1372 1373 if (cp->cache_flags & KMF_LITE) { 1374 KMEM_BUFTAG_LITE_ENTER(btp, 1375 kmem_lite_count, caller()); 1376 } 1377 } 1378 bzero(buf, size); 1379 } 1380 } else { 1381 buf = kmem_alloc(size, kmflag); 1382 if (buf != NULL) 1383 bzero(buf, size); 1384 } 1385 return (buf); 1386 } 1387 1388 void * 1389 kmem_alloc(size_t size, int kmflag) 1390 { 1391 size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 1392 void *buf; 1393 1394 if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 1395 kmem_cache_t *cp = kmem_alloc_table[index]; 1396 buf = kmem_cache_alloc(cp, kmflag); 1397 if ((cp->cache_flags & KMF_BUFTAG) && buf != NULL) { 1398 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1399 ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 1400 ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 1401 1402 if (cp->cache_flags & KMF_LITE) { 1403 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 1404 caller()); 1405 } 1406 } 1407 return (buf); 1408 } 1409 if (size == 0) 1410 return (NULL); 1411 buf = vmem_alloc(kmem_oversize_arena, size, kmflag & KM_VMFLAGS); 1412 if (buf == NULL) 1413 kmem_log_event(kmem_failure_log, NULL, NULL, (void *)size); 1414 return (buf); 1415 } 1416 1417 void 1418 kmem_free(void *buf, size_t size) 1419 { 1420 size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 1421 1422 if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 1423 kmem_cache_t *cp = kmem_alloc_table[index]; 1424 if (cp->cache_flags & KMF_BUFTAG) { 1425 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1426 uint32_t *ip = (uint32_t *)btp; 1427 if (ip[1] != KMEM_SIZE_ENCODE(size)) { 1428 if (*(uint64_t *)buf == KMEM_FREE_PATTERN) { 1429 kmem_error(KMERR_DUPFREE, cp, buf); 1430 return; 1431 } 1432 if (KMEM_SIZE_VALID(ip[1])) { 1433 ip[0] = KMEM_SIZE_ENCODE(size); 1434 kmem_error(KMERR_BADSIZE, cp, buf); 1435 } else { 1436 kmem_error(KMERR_REDZONE, cp, buf); 1437 } 1438 return; 1439 } 1440 if (((uint8_t *)buf)[size] != KMEM_REDZONE_BYTE) { 1441 kmem_error(KMERR_REDZONE, cp, buf); 1442 return; 1443 } 1444 btp->bt_redzone = KMEM_REDZONE_PATTERN; 1445 if (cp->cache_flags & KMF_LITE) { 1446 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 1447 caller()); 1448 } 1449 } 1450 kmem_cache_free(cp, buf); 1451 } else { 1452 if (buf == NULL && size == 0) 1453 return; 1454 vmem_free(kmem_oversize_arena, buf, size); 1455 } 1456 } 1457 1458 void * 1459 kmem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag) 1460 { 1461 size_t realsize = size + vmp->vm_quantum; 1462 void *addr; 1463 1464 /* 1465 * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding 1466 * vm_quantum will cause integer wraparound. Check for this, and 1467 * blow off the firewall page in this case. Note that such a 1468 * giant allocation (the entire kernel address space) can never 1469 * be satisfied, so it will either fail immediately (VM_NOSLEEP) 1470 * or sleep forever (VM_SLEEP). Thus, there is no need for a 1471 * corresponding check in kmem_firewall_va_free(). 1472 */ 1473 if (realsize < size) 1474 realsize = size; 1475 1476 /* 1477 * While boot still owns resource management, make sure that this 1478 * redzone virtual address allocation is properly accounted for in 1479 * OBPs "virtual-memory" "available" lists because we're 1480 * effectively claiming them for a red zone. If we don't do this, 1481 * the available lists become too fragmented and too large for the 1482 * current boot/kernel memory list interface. 1483 */ 1484 addr = vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT); 1485 1486 if (addr != NULL && kvseg.s_base == NULL && realsize != size) 1487 (void) boot_virt_alloc((char *)addr + size, vmp->vm_quantum); 1488 1489 return (addr); 1490 } 1491 1492 void 1493 kmem_firewall_va_free(vmem_t *vmp, void *addr, size_t size) 1494 { 1495 ASSERT((kvseg.s_base == NULL ? 1496 va_to_pfn((char *)addr + size) : 1497 hat_getpfnum(kas.a_hat, (caddr_t)addr + size)) == PFN_INVALID); 1498 1499 vmem_free(vmp, addr, size + vmp->vm_quantum); 1500 } 1501 1502 /* 1503 * Try to allocate at least `size' bytes of memory without sleeping or 1504 * panicking. Return actual allocated size in `asize'. If allocation failed, 1505 * try final allocation with sleep or panic allowed. 1506 */ 1507 void * 1508 kmem_alloc_tryhard(size_t size, size_t *asize, int kmflag) 1509 { 1510 void *p; 1511 1512 *asize = P2ROUNDUP(size, KMEM_ALIGN); 1513 do { 1514 p = kmem_alloc(*asize, (kmflag | KM_NOSLEEP) & ~KM_PANIC); 1515 if (p != NULL) 1516 return (p); 1517 *asize += KMEM_ALIGN; 1518 } while (*asize <= PAGESIZE); 1519 1520 *asize = P2ROUNDUP(size, KMEM_ALIGN); 1521 return (kmem_alloc(*asize, kmflag)); 1522 } 1523 1524 /* 1525 * Reclaim all unused memory from a cache. 1526 */ 1527 static void 1528 kmem_cache_reap(kmem_cache_t *cp) 1529 { 1530 /* 1531 * Ask the cache's owner to free some memory if possible. 1532 * The idea is to handle things like the inode cache, which 1533 * typically sits on a bunch of memory that it doesn't truly 1534 * *need*. Reclaim policy is entirely up to the owner; this 1535 * callback is just an advisory plea for help. 1536 */ 1537 if (cp->cache_reclaim != NULL) 1538 cp->cache_reclaim(cp->cache_private); 1539 1540 kmem_depot_ws_reap(cp); 1541 } 1542 1543 static void 1544 kmem_reap_timeout(void *flag_arg) 1545 { 1546 uint32_t *flag = (uint32_t *)flag_arg; 1547 1548 ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 1549 *flag = 0; 1550 } 1551 1552 static void 1553 kmem_reap_done(void *flag) 1554 { 1555 (void) timeout(kmem_reap_timeout, flag, kmem_reap_interval); 1556 } 1557 1558 static void 1559 kmem_reap_start(void *flag) 1560 { 1561 ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 1562 1563 if (flag == &kmem_reaping) { 1564 kmem_cache_applyall(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 1565 /* 1566 * if we have segkp under heap, reap segkp cache. 1567 */ 1568 if (segkp_fromheap) 1569 segkp_cache_free(); 1570 } 1571 else 1572 kmem_cache_applyall_id(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 1573 1574 /* 1575 * We use taskq_dispatch() to schedule a timeout to clear 1576 * the flag so that kmem_reap() becomes self-throttling: 1577 * we won't reap again until the current reap completes *and* 1578 * at least kmem_reap_interval ticks have elapsed. 1579 */ 1580 if (!taskq_dispatch(kmem_taskq, kmem_reap_done, flag, TQ_NOSLEEP)) 1581 kmem_reap_done(flag); 1582 } 1583 1584 static void 1585 kmem_reap_common(void *flag_arg) 1586 { 1587 uint32_t *flag = (uint32_t *)flag_arg; 1588 1589 if (MUTEX_HELD(&kmem_cache_lock) || kmem_taskq == NULL || 1590 cas32(flag, 0, 1) != 0) 1591 return; 1592 1593 /* 1594 * It may not be kosher to do memory allocation when a reap is called 1595 * is called (for example, if vmem_populate() is in the call chain). 1596 * So we start the reap going with a TQ_NOALLOC dispatch. If the 1597 * dispatch fails, we reset the flag, and the next reap will try again. 1598 */ 1599 if (!taskq_dispatch(kmem_taskq, kmem_reap_start, flag, TQ_NOALLOC)) 1600 *flag = 0; 1601 } 1602 1603 /* 1604 * Reclaim all unused memory from all caches. Called from the VM system 1605 * when memory gets tight. 1606 */ 1607 void 1608 kmem_reap(void) 1609 { 1610 kmem_reap_common(&kmem_reaping); 1611 } 1612 1613 /* 1614 * Reclaim all unused memory from identifier arenas, called when a vmem 1615 * arena not back by memory is exhausted. Since reaping memory-backed caches 1616 * cannot help with identifier exhaustion, we avoid both a large amount of 1617 * work and unwanted side-effects from reclaim callbacks. 1618 */ 1619 void 1620 kmem_reap_idspace(void) 1621 { 1622 kmem_reap_common(&kmem_reaping_idspace); 1623 } 1624 1625 /* 1626 * Purge all magazines from a cache and set its magazine limit to zero. 1627 * All calls are serialized by the kmem_taskq lock, except for the final 1628 * call from kmem_cache_destroy(). 1629 */ 1630 static void 1631 kmem_cache_magazine_purge(kmem_cache_t *cp) 1632 { 1633 kmem_cpu_cache_t *ccp; 1634 kmem_magazine_t *mp, *pmp; 1635 int rounds, prounds, cpu_seqid; 1636 1637 ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 1638 ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 1639 1640 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 1641 ccp = &cp->cache_cpu[cpu_seqid]; 1642 1643 mutex_enter(&ccp->cc_lock); 1644 mp = ccp->cc_loaded; 1645 pmp = ccp->cc_ploaded; 1646 rounds = ccp->cc_rounds; 1647 prounds = ccp->cc_prounds; 1648 ccp->cc_loaded = NULL; 1649 ccp->cc_ploaded = NULL; 1650 ccp->cc_rounds = -1; 1651 ccp->cc_prounds = -1; 1652 ccp->cc_magsize = 0; 1653 mutex_exit(&ccp->cc_lock); 1654 1655 if (mp) 1656 kmem_magazine_destroy(cp, mp, rounds); 1657 if (pmp) 1658 kmem_magazine_destroy(cp, pmp, prounds); 1659 } 1660 1661 /* 1662 * Updating the working set statistics twice in a row has the 1663 * effect of setting the working set size to zero, so everything 1664 * is eligible for reaping. 1665 */ 1666 kmem_depot_ws_update(cp); 1667 kmem_depot_ws_update(cp); 1668 1669 kmem_depot_ws_reap(cp); 1670 } 1671 1672 /* 1673 * Enable per-cpu magazines on a cache. 1674 */ 1675 static void 1676 kmem_cache_magazine_enable(kmem_cache_t *cp) 1677 { 1678 int cpu_seqid; 1679 1680 if (cp->cache_flags & KMF_NOMAGAZINE) 1681 return; 1682 1683 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 1684 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 1685 mutex_enter(&ccp->cc_lock); 1686 ccp->cc_magsize = cp->cache_magtype->mt_magsize; 1687 mutex_exit(&ccp->cc_lock); 1688 } 1689 1690 } 1691 1692 /* 1693 * Reap (almost) everything right now. See kmem_cache_magazine_purge() 1694 * for explanation of the back-to-back kmem_depot_ws_update() calls. 1695 */ 1696 void 1697 kmem_cache_reap_now(kmem_cache_t *cp) 1698 { 1699 kmem_depot_ws_update(cp); 1700 kmem_depot_ws_update(cp); 1701 1702 (void) taskq_dispatch(kmem_taskq, 1703 (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP); 1704 taskq_wait(kmem_taskq); 1705 } 1706 1707 /* 1708 * Recompute a cache's magazine size. The trade-off is that larger magazines 1709 * provide a higher transfer rate with the depot, while smaller magazines 1710 * reduce memory consumption. Magazine resizing is an expensive operation; 1711 * it should not be done frequently. 1712 * 1713 * Changes to the magazine size are serialized by the kmem_taskq lock. 1714 * 1715 * Note: at present this only grows the magazine size. It might be useful 1716 * to allow shrinkage too. 1717 */ 1718 static void 1719 kmem_cache_magazine_resize(kmem_cache_t *cp) 1720 { 1721 kmem_magtype_t *mtp = cp->cache_magtype; 1722 1723 ASSERT(taskq_member(kmem_taskq, curthread)); 1724 1725 if (cp->cache_chunksize < mtp->mt_maxbuf) { 1726 kmem_cache_magazine_purge(cp); 1727 mutex_enter(&cp->cache_depot_lock); 1728 cp->cache_magtype = ++mtp; 1729 cp->cache_depot_contention_prev = 1730 cp->cache_depot_contention + INT_MAX; 1731 mutex_exit(&cp->cache_depot_lock); 1732 kmem_cache_magazine_enable(cp); 1733 } 1734 } 1735 1736 /* 1737 * Rescale a cache's hash table, so that the table size is roughly the 1738 * cache size. We want the average lookup time to be extremely small. 1739 */ 1740 static void 1741 kmem_hash_rescale(kmem_cache_t *cp) 1742 { 1743 kmem_bufctl_t **old_table, **new_table, *bcp; 1744 size_t old_size, new_size, h; 1745 1746 ASSERT(taskq_member(kmem_taskq, curthread)); 1747 1748 new_size = MAX(KMEM_HASH_INITIAL, 1749 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); 1750 old_size = cp->cache_hash_mask + 1; 1751 1752 if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 1753 return; 1754 1755 new_table = vmem_alloc(kmem_hash_arena, new_size * sizeof (void *), 1756 VM_NOSLEEP); 1757 if (new_table == NULL) 1758 return; 1759 bzero(new_table, new_size * sizeof (void *)); 1760 1761 mutex_enter(&cp->cache_lock); 1762 1763 old_size = cp->cache_hash_mask + 1; 1764 old_table = cp->cache_hash_table; 1765 1766 cp->cache_hash_mask = new_size - 1; 1767 cp->cache_hash_table = new_table; 1768 cp->cache_rescale++; 1769 1770 for (h = 0; h < old_size; h++) { 1771 bcp = old_table[h]; 1772 while (bcp != NULL) { 1773 void *addr = bcp->bc_addr; 1774 kmem_bufctl_t *next_bcp = bcp->bc_next; 1775 kmem_bufctl_t **hash_bucket = KMEM_HASH(cp, addr); 1776 bcp->bc_next = *hash_bucket; 1777 *hash_bucket = bcp; 1778 bcp = next_bcp; 1779 } 1780 } 1781 1782 mutex_exit(&cp->cache_lock); 1783 1784 vmem_free(kmem_hash_arena, old_table, old_size * sizeof (void *)); 1785 } 1786 1787 /* 1788 * Perform periodic maintenance on a cache: hash rescaling, 1789 * depot working-set update, and magazine resizing. 1790 */ 1791 static void 1792 kmem_cache_update(kmem_cache_t *cp) 1793 { 1794 int need_hash_rescale = 0; 1795 int need_magazine_resize = 0; 1796 1797 ASSERT(MUTEX_HELD(&kmem_cache_lock)); 1798 1799 /* 1800 * If the cache has become much larger or smaller than its hash table, 1801 * fire off a request to rescale the hash table. 1802 */ 1803 mutex_enter(&cp->cache_lock); 1804 1805 if ((cp->cache_flags & KMF_HASH) && 1806 (cp->cache_buftotal > (cp->cache_hash_mask << 1) || 1807 (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && 1808 cp->cache_hash_mask > KMEM_HASH_INITIAL))) 1809 need_hash_rescale = 1; 1810 1811 mutex_exit(&cp->cache_lock); 1812 1813 /* 1814 * Update the depot working set statistics. 1815 */ 1816 kmem_depot_ws_update(cp); 1817 1818 /* 1819 * If there's a lot of contention in the depot, 1820 * increase the magazine size. 1821 */ 1822 mutex_enter(&cp->cache_depot_lock); 1823 1824 if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && 1825 (int)(cp->cache_depot_contention - 1826 cp->cache_depot_contention_prev) > kmem_depot_contention) 1827 need_magazine_resize = 1; 1828 1829 cp->cache_depot_contention_prev = cp->cache_depot_contention; 1830 1831 mutex_exit(&cp->cache_depot_lock); 1832 1833 if (need_hash_rescale) 1834 (void) taskq_dispatch(kmem_taskq, 1835 (task_func_t *)kmem_hash_rescale, cp, TQ_NOSLEEP); 1836 1837 if (need_magazine_resize) 1838 (void) taskq_dispatch(kmem_taskq, 1839 (task_func_t *)kmem_cache_magazine_resize, cp, TQ_NOSLEEP); 1840 } 1841 1842 static void 1843 kmem_update_timeout(void *dummy) 1844 { 1845 static void kmem_update(void *); 1846 1847 (void) timeout(kmem_update, dummy, kmem_reap_interval); 1848 } 1849 1850 static void 1851 kmem_update(void *dummy) 1852 { 1853 kmem_cache_applyall(kmem_cache_update, NULL, TQ_NOSLEEP); 1854 1855 /* 1856 * We use taskq_dispatch() to reschedule the timeout so that 1857 * kmem_update() becomes self-throttling: it won't schedule 1858 * new tasks until all previous tasks have completed. 1859 */ 1860 if (!taskq_dispatch(kmem_taskq, kmem_update_timeout, dummy, TQ_NOSLEEP)) 1861 kmem_update_timeout(NULL); 1862 } 1863 1864 static int 1865 kmem_cache_kstat_update(kstat_t *ksp, int rw) 1866 { 1867 struct kmem_cache_kstat *kmcp = &kmem_cache_kstat; 1868 kmem_cache_t *cp = ksp->ks_private; 1869 uint64_t cpu_buf_avail; 1870 uint64_t buf_avail = 0; 1871 int cpu_seqid; 1872 1873 ASSERT(MUTEX_HELD(&kmem_cache_kstat_lock)); 1874 1875 if (rw == KSTAT_WRITE) 1876 return (EACCES); 1877 1878 mutex_enter(&cp->cache_lock); 1879 1880 kmcp->kmc_alloc_fail.value.ui64 = cp->cache_alloc_fail; 1881 kmcp->kmc_alloc.value.ui64 = cp->cache_slab_alloc; 1882 kmcp->kmc_free.value.ui64 = cp->cache_slab_free; 1883 kmcp->kmc_slab_alloc.value.ui64 = cp->cache_slab_alloc; 1884 kmcp->kmc_slab_free.value.ui64 = cp->cache_slab_free; 1885 1886 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 1887 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 1888 1889 mutex_enter(&ccp->cc_lock); 1890 1891 cpu_buf_avail = 0; 1892 if (ccp->cc_rounds > 0) 1893 cpu_buf_avail += ccp->cc_rounds; 1894 if (ccp->cc_prounds > 0) 1895 cpu_buf_avail += ccp->cc_prounds; 1896 1897 kmcp->kmc_alloc.value.ui64 += ccp->cc_alloc; 1898 kmcp->kmc_free.value.ui64 += ccp->cc_free; 1899 buf_avail += cpu_buf_avail; 1900 1901 mutex_exit(&ccp->cc_lock); 1902 } 1903 1904 mutex_enter(&cp->cache_depot_lock); 1905 1906 kmcp->kmc_depot_alloc.value.ui64 = cp->cache_full.ml_alloc; 1907 kmcp->kmc_depot_free.value.ui64 = cp->cache_empty.ml_alloc; 1908 kmcp->kmc_depot_contention.value.ui64 = cp->cache_depot_contention; 1909 kmcp->kmc_full_magazines.value.ui64 = cp->cache_full.ml_total; 1910 kmcp->kmc_empty_magazines.value.ui64 = cp->cache_empty.ml_total; 1911 kmcp->kmc_magazine_size.value.ui64 = 1912 (cp->cache_flags & KMF_NOMAGAZINE) ? 1913 0 : cp->cache_magtype->mt_magsize; 1914 1915 kmcp->kmc_alloc.value.ui64 += cp->cache_full.ml_alloc; 1916 kmcp->kmc_free.value.ui64 += cp->cache_empty.ml_alloc; 1917 buf_avail += cp->cache_full.ml_total * cp->cache_magtype->mt_magsize; 1918 1919 mutex_exit(&cp->cache_depot_lock); 1920 1921 kmcp->kmc_buf_size.value.ui64 = cp->cache_bufsize; 1922 kmcp->kmc_align.value.ui64 = cp->cache_align; 1923 kmcp->kmc_chunk_size.value.ui64 = cp->cache_chunksize; 1924 kmcp->kmc_slab_size.value.ui64 = cp->cache_slabsize; 1925 kmcp->kmc_buf_constructed.value.ui64 = buf_avail; 1926 buf_avail += cp->cache_bufslab; 1927 kmcp->kmc_buf_avail.value.ui64 = buf_avail; 1928 kmcp->kmc_buf_inuse.value.ui64 = cp->cache_buftotal - buf_avail; 1929 kmcp->kmc_buf_total.value.ui64 = cp->cache_buftotal; 1930 kmcp->kmc_buf_max.value.ui64 = cp->cache_bufmax; 1931 kmcp->kmc_slab_create.value.ui64 = cp->cache_slab_create; 1932 kmcp->kmc_slab_destroy.value.ui64 = cp->cache_slab_destroy; 1933 kmcp->kmc_hash_size.value.ui64 = (cp->cache_flags & KMF_HASH) ? 1934 cp->cache_hash_mask + 1 : 0; 1935 kmcp->kmc_hash_lookup_depth.value.ui64 = cp->cache_lookup_depth; 1936 kmcp->kmc_hash_rescale.value.ui64 = cp->cache_rescale; 1937 kmcp->kmc_vmem_source.value.ui64 = cp->cache_arena->vm_id; 1938 1939 mutex_exit(&cp->cache_lock); 1940 return (0); 1941 } 1942 1943 /* 1944 * Return a named statistic about a particular cache. 1945 * This shouldn't be called very often, so it's currently designed for 1946 * simplicity (leverages existing kstat support) rather than efficiency. 1947 */ 1948 uint64_t 1949 kmem_cache_stat(kmem_cache_t *cp, char *name) 1950 { 1951 int i; 1952 kstat_t *ksp = cp->cache_kstat; 1953 kstat_named_t *knp = (kstat_named_t *)&kmem_cache_kstat; 1954 uint64_t value = 0; 1955 1956 if (ksp != NULL) { 1957 mutex_enter(&kmem_cache_kstat_lock); 1958 (void) kmem_cache_kstat_update(ksp, KSTAT_READ); 1959 for (i = 0; i < ksp->ks_ndata; i++) { 1960 if (strcmp(knp[i].name, name) == 0) { 1961 value = knp[i].value.ui64; 1962 break; 1963 } 1964 } 1965 mutex_exit(&kmem_cache_kstat_lock); 1966 } 1967 return (value); 1968 } 1969 1970 /* 1971 * Return an estimate of currently available kernel heap memory. 1972 * On 32-bit systems, physical memory may exceed virtual memory, 1973 * we just truncate the result at 1GB. 1974 */ 1975 size_t 1976 kmem_avail(void) 1977 { 1978 spgcnt_t rmem = availrmem - tune.t_minarmem; 1979 spgcnt_t fmem = freemem - minfree; 1980 1981 return ((size_t)ptob(MIN(MAX(MIN(rmem, fmem), 0), 1982 1 << (30 - PAGESHIFT)))); 1983 } 1984 1985 /* 1986 * Return the maximum amount of memory that is (in theory) allocatable 1987 * from the heap. This may be used as an estimate only since there 1988 * is no guarentee this space will still be available when an allocation 1989 * request is made, nor that the space may be allocated in one big request 1990 * due to kernel heap fragmentation. 1991 */ 1992 size_t 1993 kmem_maxavail(void) 1994 { 1995 spgcnt_t pmem = availrmem - tune.t_minarmem; 1996 spgcnt_t vmem = btop(vmem_size(heap_arena, VMEM_FREE)); 1997 1998 return ((size_t)ptob(MAX(MIN(pmem, vmem), 0))); 1999 } 2000 2001 /* 2002 * Indicate whether memory-intensive kmem debugging is enabled. 2003 */ 2004 int 2005 kmem_debugging(void) 2006 { 2007 return (kmem_flags & (KMF_AUDIT | KMF_REDZONE)); 2008 } 2009 2010 kmem_cache_t * 2011 kmem_cache_create( 2012 char *name, /* descriptive name for this cache */ 2013 size_t bufsize, /* size of the objects it manages */ 2014 size_t align, /* required object alignment */ 2015 int (*constructor)(void *, void *, int), /* object constructor */ 2016 void (*destructor)(void *, void *), /* object destructor */ 2017 void (*reclaim)(void *), /* memory reclaim callback */ 2018 void *private, /* pass-thru arg for constr/destr/reclaim */ 2019 vmem_t *vmp, /* vmem source for slab allocation */ 2020 int cflags) /* cache creation flags */ 2021 { 2022 int cpu_seqid; 2023 size_t chunksize; 2024 kmem_cache_t *cp, *cnext, *cprev; 2025 kmem_magtype_t *mtp; 2026 size_t csize = KMEM_CACHE_SIZE(max_ncpus); 2027 2028 #ifdef DEBUG 2029 /* 2030 * Cache names should conform to the rules for valid C identifiers 2031 */ 2032 if (!strident_valid(name)) { 2033 cmn_err(CE_CONT, 2034 "kmem_cache_create: '%s' is an invalid cache name\n" 2035 "cache names must conform to the rules for " 2036 "C identifiers\n", name); 2037 } 2038 #endif /* DEBUG */ 2039 2040 if (vmp == NULL) 2041 vmp = kmem_default_arena; 2042 2043 /* 2044 * If this kmem cache has an identifier vmem arena as its source, mark 2045 * it such to allow kmem_reap_idspace(). 2046 */ 2047 ASSERT(!(cflags & KMC_IDENTIFIER)); /* consumer should not set this */ 2048 if (vmp->vm_cflags & VMC_IDENTIFIER) 2049 cflags |= KMC_IDENTIFIER; 2050 2051 /* 2052 * Get a kmem_cache structure. We arrange that cp->cache_cpu[] 2053 * is aligned on a KMEM_CPU_CACHE_SIZE boundary to prevent 2054 * false sharing of per-CPU data. 2055 */ 2056 cp = vmem_xalloc(kmem_cache_arena, csize, KMEM_CPU_CACHE_SIZE, 2057 P2NPHASE(csize, KMEM_CPU_CACHE_SIZE), 0, NULL, NULL, VM_SLEEP); 2058 bzero(cp, csize); 2059 2060 if (align == 0) 2061 align = KMEM_ALIGN; 2062 2063 /* 2064 * If we're not at least KMEM_ALIGN aligned, we can't use free 2065 * memory to hold bufctl information (because we can't safely 2066 * perform word loads and stores on it). 2067 */ 2068 if (align < KMEM_ALIGN) 2069 cflags |= KMC_NOTOUCH; 2070 2071 if ((align & (align - 1)) != 0 || align > vmp->vm_quantum) 2072 panic("kmem_cache_create: bad alignment %lu", align); 2073 2074 mutex_enter(&kmem_flags_lock); 2075 if (kmem_flags & KMF_RANDOMIZE) 2076 kmem_flags = (((kmem_flags | ~KMF_RANDOM) + 1) & KMF_RANDOM) | 2077 KMF_RANDOMIZE; 2078 cp->cache_flags = (kmem_flags | cflags) & KMF_DEBUG; 2079 mutex_exit(&kmem_flags_lock); 2080 2081 /* 2082 * Make sure all the various flags are reasonable. 2083 */ 2084 ASSERT(!(cflags & KMC_NOHASH) || !(cflags & KMC_NOTOUCH)); 2085 2086 if (cp->cache_flags & KMF_LITE) { 2087 if (bufsize >= kmem_lite_minsize && 2088 align <= kmem_lite_maxalign && 2089 P2PHASE(bufsize, kmem_lite_maxalign) != 0) { 2090 cp->cache_flags |= KMF_BUFTAG; 2091 cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 2092 } else { 2093 cp->cache_flags &= ~KMF_DEBUG; 2094 } 2095 } 2096 2097 if (cp->cache_flags & KMF_DEADBEEF) 2098 cp->cache_flags |= KMF_REDZONE; 2099 2100 if ((cflags & KMC_QCACHE) && (cp->cache_flags & KMF_AUDIT)) 2101 cp->cache_flags |= KMF_NOMAGAZINE; 2102 2103 if (cflags & KMC_NODEBUG) 2104 cp->cache_flags &= ~KMF_DEBUG; 2105 2106 if (cflags & KMC_NOTOUCH) 2107 cp->cache_flags &= ~KMF_TOUCH; 2108 2109 if (cflags & KMC_NOHASH) 2110 cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 2111 2112 if (cflags & KMC_NOMAGAZINE) 2113 cp->cache_flags |= KMF_NOMAGAZINE; 2114 2115 if ((cp->cache_flags & KMF_AUDIT) && !(cflags & KMC_NOTOUCH)) 2116 cp->cache_flags |= KMF_REDZONE; 2117 2118 if (!(cp->cache_flags & KMF_AUDIT)) 2119 cp->cache_flags &= ~KMF_CONTENTS; 2120 2121 if ((cp->cache_flags & KMF_BUFTAG) && bufsize >= kmem_minfirewall && 2122 !(cp->cache_flags & KMF_LITE) && !(cflags & KMC_NOHASH)) 2123 cp->cache_flags |= KMF_FIREWALL; 2124 2125 if (vmp != kmem_default_arena || kmem_firewall_arena == NULL) 2126 cp->cache_flags &= ~KMF_FIREWALL; 2127 2128 if (cp->cache_flags & KMF_FIREWALL) { 2129 cp->cache_flags &= ~KMF_BUFTAG; 2130 cp->cache_flags |= KMF_NOMAGAZINE; 2131 ASSERT(vmp == kmem_default_arena); 2132 vmp = kmem_firewall_arena; 2133 } 2134 2135 /* 2136 * Set cache properties. 2137 */ 2138 (void) strncpy(cp->cache_name, name, KMEM_CACHE_NAMELEN); 2139 strident_canon(cp->cache_name, KMEM_CACHE_NAMELEN); 2140 cp->cache_bufsize = bufsize; 2141 cp->cache_align = align; 2142 cp->cache_constructor = constructor; 2143 cp->cache_destructor = destructor; 2144 cp->cache_reclaim = reclaim; 2145 cp->cache_private = private; 2146 cp->cache_arena = vmp; 2147 cp->cache_cflags = cflags; 2148 2149 /* 2150 * Determine the chunk size. 2151 */ 2152 chunksize = bufsize; 2153 2154 if (align >= KMEM_ALIGN) { 2155 chunksize = P2ROUNDUP(chunksize, KMEM_ALIGN); 2156 cp->cache_bufctl = chunksize - KMEM_ALIGN; 2157 } 2158 2159 if (cp->cache_flags & KMF_BUFTAG) { 2160 cp->cache_bufctl = chunksize; 2161 cp->cache_buftag = chunksize; 2162 if (cp->cache_flags & KMF_LITE) 2163 chunksize += KMEM_BUFTAG_LITE_SIZE(kmem_lite_count); 2164 else 2165 chunksize += sizeof (kmem_buftag_t); 2166 } 2167 2168 if (cp->cache_flags & KMF_DEADBEEF) { 2169 cp->cache_verify = MIN(cp->cache_buftag, kmem_maxverify); 2170 if (cp->cache_flags & KMF_LITE) 2171 cp->cache_verify = sizeof (uint64_t); 2172 } 2173 2174 cp->cache_contents = MIN(cp->cache_bufctl, kmem_content_maxsave); 2175 2176 cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); 2177 2178 /* 2179 * Now that we know the chunk size, determine the optimal slab size. 2180 */ 2181 if (vmp == kmem_firewall_arena) { 2182 cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); 2183 cp->cache_mincolor = cp->cache_slabsize - chunksize; 2184 cp->cache_maxcolor = cp->cache_mincolor; 2185 cp->cache_flags |= KMF_HASH; 2186 ASSERT(!(cp->cache_flags & KMF_BUFTAG)); 2187 } else if ((cflags & KMC_NOHASH) || (!(cflags & KMC_NOTOUCH) && 2188 !(cp->cache_flags & KMF_AUDIT) && 2189 chunksize < vmp->vm_quantum / KMEM_VOID_FRACTION)) { 2190 cp->cache_slabsize = vmp->vm_quantum; 2191 cp->cache_mincolor = 0; 2192 cp->cache_maxcolor = 2193 (cp->cache_slabsize - sizeof (kmem_slab_t)) % chunksize; 2194 ASSERT(chunksize + sizeof (kmem_slab_t) <= cp->cache_slabsize); 2195 ASSERT(!(cp->cache_flags & KMF_AUDIT)); 2196 } else { 2197 size_t chunks, bestfit, waste, slabsize; 2198 size_t minwaste = LONG_MAX; 2199 2200 for (chunks = 1; chunks <= KMEM_VOID_FRACTION; chunks++) { 2201 slabsize = P2ROUNDUP(chunksize * chunks, 2202 vmp->vm_quantum); 2203 chunks = slabsize / chunksize; 2204 waste = (slabsize % chunksize) / chunks; 2205 if (waste < minwaste) { 2206 minwaste = waste; 2207 bestfit = slabsize; 2208 } 2209 } 2210 if (cflags & KMC_QCACHE) 2211 bestfit = VMEM_QCACHE_SLABSIZE(vmp->vm_qcache_max); 2212 cp->cache_slabsize = bestfit; 2213 cp->cache_mincolor = 0; 2214 cp->cache_maxcolor = bestfit % chunksize; 2215 cp->cache_flags |= KMF_HASH; 2216 } 2217 2218 if (cp->cache_flags & KMF_HASH) { 2219 ASSERT(!(cflags & KMC_NOHASH)); 2220 cp->cache_bufctl_cache = (cp->cache_flags & KMF_AUDIT) ? 2221 kmem_bufctl_audit_cache : kmem_bufctl_cache; 2222 } 2223 2224 if (cp->cache_maxcolor >= vmp->vm_quantum) 2225 cp->cache_maxcolor = vmp->vm_quantum - 1; 2226 2227 cp->cache_color = cp->cache_mincolor; 2228 2229 /* 2230 * Initialize the rest of the slab layer. 2231 */ 2232 mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL); 2233 2234 cp->cache_freelist = &cp->cache_nullslab; 2235 cp->cache_nullslab.slab_cache = cp; 2236 cp->cache_nullslab.slab_refcnt = -1; 2237 cp->cache_nullslab.slab_next = &cp->cache_nullslab; 2238 cp->cache_nullslab.slab_prev = &cp->cache_nullslab; 2239 2240 if (cp->cache_flags & KMF_HASH) { 2241 cp->cache_hash_table = vmem_alloc(kmem_hash_arena, 2242 KMEM_HASH_INITIAL * sizeof (void *), VM_SLEEP); 2243 bzero(cp->cache_hash_table, 2244 KMEM_HASH_INITIAL * sizeof (void *)); 2245 cp->cache_hash_mask = KMEM_HASH_INITIAL - 1; 2246 cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; 2247 } 2248 2249 /* 2250 * Initialize the depot. 2251 */ 2252 mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL); 2253 2254 for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++) 2255 continue; 2256 2257 cp->cache_magtype = mtp; 2258 2259 /* 2260 * Initialize the CPU layer. 2261 */ 2262 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 2263 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 2264 mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL); 2265 ccp->cc_flags = cp->cache_flags; 2266 ccp->cc_rounds = -1; 2267 ccp->cc_prounds = -1; 2268 } 2269 2270 /* 2271 * Create the cache's kstats. 2272 */ 2273 if ((cp->cache_kstat = kstat_create("unix", 0, cp->cache_name, 2274 "kmem_cache", KSTAT_TYPE_NAMED, 2275 sizeof (kmem_cache_kstat) / sizeof (kstat_named_t), 2276 KSTAT_FLAG_VIRTUAL)) != NULL) { 2277 cp->cache_kstat->ks_data = &kmem_cache_kstat; 2278 cp->cache_kstat->ks_update = kmem_cache_kstat_update; 2279 cp->cache_kstat->ks_private = cp; 2280 cp->cache_kstat->ks_lock = &kmem_cache_kstat_lock; 2281 kstat_install(cp->cache_kstat); 2282 } 2283 2284 /* 2285 * Add the cache to the global list. This makes it visible 2286 * to kmem_update(), so the cache must be ready for business. 2287 */ 2288 mutex_enter(&kmem_cache_lock); 2289 cp->cache_next = cnext = &kmem_null_cache; 2290 cp->cache_prev = cprev = kmem_null_cache.cache_prev; 2291 cnext->cache_prev = cp; 2292 cprev->cache_next = cp; 2293 mutex_exit(&kmem_cache_lock); 2294 2295 if (kmem_ready) 2296 kmem_cache_magazine_enable(cp); 2297 2298 return (cp); 2299 } 2300 2301 void 2302 kmem_cache_destroy(kmem_cache_t *cp) 2303 { 2304 int cpu_seqid; 2305 2306 /* 2307 * Remove the cache from the global cache list so that no one else 2308 * can schedule tasks on its behalf, wait for any pending tasks to 2309 * complete, purge the cache, and then destroy it. 2310 */ 2311 mutex_enter(&kmem_cache_lock); 2312 cp->cache_prev->cache_next = cp->cache_next; 2313 cp->cache_next->cache_prev = cp->cache_prev; 2314 cp->cache_prev = cp->cache_next = NULL; 2315 mutex_exit(&kmem_cache_lock); 2316 2317 if (kmem_taskq != NULL) 2318 taskq_wait(kmem_taskq); 2319 2320 kmem_cache_magazine_purge(cp); 2321 2322 mutex_enter(&cp->cache_lock); 2323 if (cp->cache_buftotal != 0) 2324 cmn_err(CE_WARN, "kmem_cache_destroy: '%s' (%p) not empty", 2325 cp->cache_name, (void *)cp); 2326 cp->cache_reclaim = NULL; 2327 /* 2328 * The cache is now dead. There should be no further activity. 2329 * We enforce this by setting land mines in the constructor and 2330 * destructor routines that induce a kernel text fault if invoked. 2331 */ 2332 cp->cache_constructor = (int (*)(void *, void *, int))1; 2333 cp->cache_destructor = (void (*)(void *, void *))2; 2334 mutex_exit(&cp->cache_lock); 2335 2336 kstat_delete(cp->cache_kstat); 2337 2338 if (cp->cache_hash_table != NULL) 2339 vmem_free(kmem_hash_arena, cp->cache_hash_table, 2340 (cp->cache_hash_mask + 1) * sizeof (void *)); 2341 2342 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) 2343 mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); 2344 2345 mutex_destroy(&cp->cache_depot_lock); 2346 mutex_destroy(&cp->cache_lock); 2347 2348 vmem_free(kmem_cache_arena, cp, KMEM_CACHE_SIZE(max_ncpus)); 2349 } 2350 2351 /*ARGSUSED*/ 2352 static int 2353 kmem_cpu_setup(cpu_setup_t what, int id, void *arg) 2354 { 2355 ASSERT(MUTEX_HELD(&cpu_lock)); 2356 if (what == CPU_UNCONFIG) { 2357 kmem_cache_applyall(kmem_cache_magazine_purge, 2358 kmem_taskq, TQ_SLEEP); 2359 kmem_cache_applyall(kmem_cache_magazine_enable, 2360 kmem_taskq, TQ_SLEEP); 2361 } 2362 return (0); 2363 } 2364 2365 static void 2366 kmem_cache_init(int pass, int use_large_pages) 2367 { 2368 int i; 2369 size_t size; 2370 kmem_cache_t *cp; 2371 kmem_magtype_t *mtp; 2372 char name[KMEM_CACHE_NAMELEN + 1]; 2373 2374 for (i = 0; i < sizeof (kmem_magtype) / sizeof (*mtp); i++) { 2375 mtp = &kmem_magtype[i]; 2376 (void) sprintf(name, "kmem_magazine_%d", mtp->mt_magsize); 2377 mtp->mt_cache = kmem_cache_create(name, 2378 (mtp->mt_magsize + 1) * sizeof (void *), 2379 mtp->mt_align, NULL, NULL, NULL, NULL, 2380 kmem_msb_arena, KMC_NOHASH); 2381 } 2382 2383 kmem_slab_cache = kmem_cache_create("kmem_slab_cache", 2384 sizeof (kmem_slab_t), 0, NULL, NULL, NULL, NULL, 2385 kmem_msb_arena, KMC_NOHASH); 2386 2387 kmem_bufctl_cache = kmem_cache_create("kmem_bufctl_cache", 2388 sizeof (kmem_bufctl_t), 0, NULL, NULL, NULL, NULL, 2389 kmem_msb_arena, KMC_NOHASH); 2390 2391 kmem_bufctl_audit_cache = kmem_cache_create("kmem_bufctl_audit_cache", 2392 sizeof (kmem_bufctl_audit_t), 0, NULL, NULL, NULL, NULL, 2393 kmem_msb_arena, KMC_NOHASH); 2394 2395 if (pass == 2) { 2396 kmem_va_arena = vmem_create("kmem_va", 2397 NULL, 0, PAGESIZE, 2398 vmem_alloc, vmem_free, heap_arena, 2399 8 * PAGESIZE, VM_SLEEP); 2400 2401 if (use_large_pages) { 2402 kmem_default_arena = vmem_xcreate("kmem_default", 2403 NULL, 0, PAGESIZE, 2404 segkmem_alloc_lp, segkmem_free_lp, kmem_va_arena, 2405 0, VM_SLEEP); 2406 } else { 2407 kmem_default_arena = vmem_create("kmem_default", 2408 NULL, 0, PAGESIZE, 2409 segkmem_alloc, segkmem_free, kmem_va_arena, 2410 0, VM_SLEEP); 2411 } 2412 } else { 2413 /* 2414 * During the first pass, the kmem_alloc_* caches 2415 * are treated as metadata. 2416 */ 2417 kmem_default_arena = kmem_msb_arena; 2418 } 2419 2420 /* 2421 * Set up the default caches to back kmem_alloc() 2422 */ 2423 size = KMEM_ALIGN; 2424 for (i = 0; i < sizeof (kmem_alloc_sizes) / sizeof (int); i++) { 2425 size_t align = KMEM_ALIGN; 2426 size_t cache_size = kmem_alloc_sizes[i]; 2427 /* 2428 * If they allocate a multiple of the coherency granularity, 2429 * they get a coherency-granularity-aligned address. 2430 */ 2431 if (IS_P2ALIGNED(cache_size, 64)) 2432 align = 64; 2433 if (IS_P2ALIGNED(cache_size, PAGESIZE)) 2434 align = PAGESIZE; 2435 (void) sprintf(name, "kmem_alloc_%lu", cache_size); 2436 cp = kmem_cache_create(name, cache_size, align, 2437 NULL, NULL, NULL, NULL, NULL, KMC_KMEM_ALLOC); 2438 while (size <= cache_size) { 2439 kmem_alloc_table[(size - 1) >> KMEM_ALIGN_SHIFT] = cp; 2440 size += KMEM_ALIGN; 2441 } 2442 } 2443 } 2444 2445 void 2446 kmem_init(void) 2447 { 2448 kmem_cache_t *cp; 2449 int old_kmem_flags = kmem_flags; 2450 int use_large_pages = 0; 2451 size_t maxverify, minfirewall; 2452 2453 kstat_init(); 2454 2455 /* 2456 * Small-memory systems (< 24 MB) can't handle kmem_flags overhead. 2457 */ 2458 if (physmem < btop(24 << 20) && !(old_kmem_flags & KMF_STICKY)) 2459 kmem_flags = 0; 2460 2461 /* 2462 * Don't do firewalled allocations if the heap is less than 1TB 2463 * (i.e. on a 32-bit kernel) 2464 * The resulting VM_NEXTFIT allocations would create too much 2465 * fragmentation in a small heap. 2466 */ 2467 #if defined(_LP64) 2468 maxverify = minfirewall = PAGESIZE / 2; 2469 #else 2470 maxverify = minfirewall = ULONG_MAX; 2471 #endif 2472 2473 /* LINTED */ 2474 ASSERT(sizeof (kmem_cpu_cache_t) == KMEM_CPU_CACHE_SIZE); 2475 2476 kmem_null_cache.cache_next = &kmem_null_cache; 2477 kmem_null_cache.cache_prev = &kmem_null_cache; 2478 2479 kmem_metadata_arena = vmem_create("kmem_metadata", NULL, 0, PAGESIZE, 2480 vmem_alloc, vmem_free, heap_arena, 8 * PAGESIZE, 2481 VM_SLEEP | VMC_NO_QCACHE); 2482 2483 kmem_msb_arena = vmem_create("kmem_msb", NULL, 0, 2484 PAGESIZE, segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, 2485 VM_SLEEP); 2486 2487 kmem_cache_arena = vmem_create("kmem_cache", NULL, 0, KMEM_ALIGN, 2488 segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 2489 2490 kmem_hash_arena = vmem_create("kmem_hash", NULL, 0, KMEM_ALIGN, 2491 segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 2492 2493 kmem_log_arena = vmem_create("kmem_log", NULL, 0, KMEM_ALIGN, 2494 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 2495 2496 kmem_firewall_va_arena = vmem_create("kmem_firewall_va", 2497 NULL, 0, PAGESIZE, 2498 kmem_firewall_va_alloc, kmem_firewall_va_free, heap_arena, 2499 0, VM_SLEEP); 2500 2501 kmem_firewall_arena = vmem_create("kmem_firewall", NULL, 0, PAGESIZE, 2502 segkmem_alloc, segkmem_free, kmem_firewall_va_arena, 0, VM_SLEEP); 2503 2504 /* temporary oversize arena for mod_read_system_file */ 2505 kmem_oversize_arena = vmem_create("kmem_oversize", NULL, 0, PAGESIZE, 2506 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 2507 2508 kmem_null_cache.cache_next = &kmem_null_cache; 2509 kmem_null_cache.cache_prev = &kmem_null_cache; 2510 2511 kmem_reap_interval = 15 * hz; 2512 2513 /* 2514 * Read /etc/system. This is a chicken-and-egg problem because 2515 * kmem_flags may be set in /etc/system, but mod_read_system_file() 2516 * needs to use the allocator. The simplest solution is to create 2517 * all the standard kmem caches, read /etc/system, destroy all the 2518 * caches we just created, and then create them all again in light 2519 * of the (possibly) new kmem_flags and other kmem tunables. 2520 */ 2521 kmem_cache_init(1, 0); 2522 2523 mod_read_system_file(boothowto & RB_ASKNAME); 2524 2525 while ((cp = kmem_null_cache.cache_prev) != &kmem_null_cache) 2526 kmem_cache_destroy(cp); 2527 2528 vmem_destroy(kmem_oversize_arena); 2529 2530 if (old_kmem_flags & KMF_STICKY) 2531 kmem_flags = old_kmem_flags; 2532 2533 if (!(kmem_flags & KMF_AUDIT)) 2534 vmem_seg_size = offsetof(vmem_seg_t, vs_thread); 2535 2536 if (kmem_maxverify == 0) 2537 kmem_maxverify = maxverify; 2538 2539 if (kmem_minfirewall == 0) 2540 kmem_minfirewall = minfirewall; 2541 2542 /* 2543 * give segkmem a chance to figure out if we are using large pages 2544 * for the kernel heap 2545 */ 2546 use_large_pages = segkmem_lpsetup(); 2547 2548 /* 2549 * To protect against corruption, we keep the actual number of callers 2550 * KMF_LITE records seperate from the tunable. We arbitrarily clamp 2551 * to 16, since the overhead for small buffers quickly gets out of 2552 * hand. 2553 * 2554 * The real limit would depend on the needs of the largest KMC_NOHASH 2555 * cache. 2556 */ 2557 kmem_lite_count = MIN(MAX(0, kmem_lite_pcs), 16); 2558 kmem_lite_pcs = kmem_lite_count; 2559 2560 /* 2561 * Normally, we firewall oversized allocations when possible, but 2562 * if we are using large pages for kernel memory, and we don't have 2563 * any non-LITE debugging flags set, we want to allocate oversized 2564 * buffers from large pages, and so skip the firewalling. 2565 */ 2566 if (use_large_pages && 2567 ((kmem_flags & KMF_LITE) || !(kmem_flags & KMF_DEBUG))) { 2568 kmem_oversize_arena = vmem_xcreate("kmem_oversize", NULL, 0, 2569 PAGESIZE, segkmem_alloc_lp, segkmem_free_lp, heap_arena, 2570 0, VM_SLEEP); 2571 } else { 2572 kmem_oversize_arena = vmem_create("kmem_oversize", 2573 NULL, 0, PAGESIZE, 2574 segkmem_alloc, segkmem_free, kmem_minfirewall < ULONG_MAX? 2575 kmem_firewall_va_arena : heap_arena, 0, VM_SLEEP); 2576 } 2577 2578 kmem_cache_init(2, use_large_pages); 2579 2580 if (kmem_flags & (KMF_AUDIT | KMF_RANDOMIZE)) { 2581 if (kmem_transaction_log_size == 0) 2582 kmem_transaction_log_size = kmem_maxavail() / 50; 2583 kmem_transaction_log = kmem_log_init(kmem_transaction_log_size); 2584 } 2585 2586 if (kmem_flags & (KMF_CONTENTS | KMF_RANDOMIZE)) { 2587 if (kmem_content_log_size == 0) 2588 kmem_content_log_size = kmem_maxavail() / 50; 2589 kmem_content_log = kmem_log_init(kmem_content_log_size); 2590 } 2591 2592 kmem_failure_log = kmem_log_init(kmem_failure_log_size); 2593 2594 kmem_slab_log = kmem_log_init(kmem_slab_log_size); 2595 2596 /* 2597 * Initialize STREAMS message caches so allocb() is available. 2598 * This allows us to initialize the logging framework (cmn_err(9F), 2599 * strlog(9F), etc) so we can start recording messages. 2600 */ 2601 streams_msg_init(); 2602 2603 /* 2604 * Initialize the ZSD framework in Zones so modules loaded henceforth 2605 * can register their callbacks. 2606 */ 2607 zone_zsd_init(); 2608 2609 log_init(); 2610 taskq_init(); 2611 2612 /* 2613 * Warn about invalid or dangerous values of kmem_flags. 2614 * Always warn about unsupported values. 2615 */ 2616 if (((kmem_flags & ~(KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | 2617 KMF_CONTENTS | KMF_LITE)) != 0) || 2618 ((kmem_flags & KMF_LITE) && kmem_flags != KMF_LITE)) 2619 cmn_err(CE_WARN, "kmem_flags set to unsupported value 0x%x. " 2620 "See the Solaris Tunable Parameters Reference Manual.", 2621 kmem_flags); 2622 2623 #ifdef DEBUG 2624 if ((kmem_flags & KMF_DEBUG) == 0) 2625 cmn_err(CE_NOTE, "kmem debugging disabled."); 2626 #else 2627 /* 2628 * For non-debug kernels, the only "normal" flags are 0, KMF_LITE, 2629 * KMF_REDZONE, and KMF_CONTENTS (the last because it is only enabled 2630 * if KMF_AUDIT is set). We should warn the user about the performance 2631 * penalty of KMF_AUDIT or KMF_DEADBEEF if they are set and KMF_LITE 2632 * isn't set (since that disables AUDIT). 2633 */ 2634 if (!(kmem_flags & KMF_LITE) && 2635 (kmem_flags & (KMF_AUDIT | KMF_DEADBEEF)) != 0) 2636 cmn_err(CE_WARN, "High-overhead kmem debugging features " 2637 "enabled (kmem_flags = 0x%x). Performance degradation " 2638 "and large memory overhead possible. See the Solaris " 2639 "Tunable Parameters Reference Manual.", kmem_flags); 2640 #endif /* not DEBUG */ 2641 2642 kmem_cache_applyall(kmem_cache_magazine_enable, NULL, TQ_SLEEP); 2643 2644 kmem_ready = 1; 2645 2646 /* 2647 * Initialize the platform-specific aligned/DMA memory allocator. 2648 */ 2649 ka_init(); 2650 2651 /* 2652 * Initialize 32-bit ID cache. 2653 */ 2654 id32_init(); 2655 2656 /* 2657 * Initialize the networking stack so modules loaded can 2658 * register their callbacks. 2659 */ 2660 netstack_init(); 2661 } 2662 2663 void 2664 kmem_thread_init(void) 2665 { 2666 kmem_taskq = taskq_create_instance("kmem_taskq", 0, 1, minclsyspri, 2667 300, INT_MAX, TASKQ_PREPOPULATE); 2668 } 2669 2670 void 2671 kmem_mp_init(void) 2672 { 2673 mutex_enter(&cpu_lock); 2674 register_cpu_setup_func(kmem_cpu_setup, NULL); 2675 mutex_exit(&cpu_lock); 2676 2677 kmem_update_timeout(NULL); 2678 } 2679