1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * Kernel memory allocator, as described in the following two papers: 30 * 31 * Jeff Bonwick, 32 * The Slab Allocator: An Object-Caching Kernel Memory Allocator. 33 * Proceedings of the Summer 1994 Usenix Conference. 34 * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf. 35 * 36 * Jeff Bonwick and Jonathan Adams, 37 * Magazines and vmem: Extending the Slab Allocator to Many CPUs and 38 * Arbitrary Resources. 39 * Proceedings of the 2001 Usenix Conference. 40 * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf. 41 */ 42 43 #include <sys/kmem_impl.h> 44 #include <sys/vmem_impl.h> 45 #include <sys/param.h> 46 #include <sys/sysmacros.h> 47 #include <sys/vm.h> 48 #include <sys/proc.h> 49 #include <sys/tuneable.h> 50 #include <sys/systm.h> 51 #include <sys/cmn_err.h> 52 #include <sys/debug.h> 53 #include <sys/mutex.h> 54 #include <sys/bitmap.h> 55 #include <sys/atomic.h> 56 #include <sys/kobj.h> 57 #include <sys/disp.h> 58 #include <vm/seg_kmem.h> 59 #include <sys/log.h> 60 #include <sys/callb.h> 61 #include <sys/taskq.h> 62 #include <sys/modctl.h> 63 #include <sys/reboot.h> 64 #include <sys/id32.h> 65 #include <sys/zone.h> 66 67 extern void streams_msg_init(void); 68 extern int segkp_fromheap; 69 extern void segkp_cache_free(void); 70 71 struct kmem_cache_kstat { 72 kstat_named_t kmc_buf_size; 73 kstat_named_t kmc_align; 74 kstat_named_t kmc_chunk_size; 75 kstat_named_t kmc_slab_size; 76 kstat_named_t kmc_alloc; 77 kstat_named_t kmc_alloc_fail; 78 kstat_named_t kmc_free; 79 kstat_named_t kmc_depot_alloc; 80 kstat_named_t kmc_depot_free; 81 kstat_named_t kmc_depot_contention; 82 kstat_named_t kmc_slab_alloc; 83 kstat_named_t kmc_slab_free; 84 kstat_named_t kmc_buf_constructed; 85 kstat_named_t kmc_buf_avail; 86 kstat_named_t kmc_buf_inuse; 87 kstat_named_t kmc_buf_total; 88 kstat_named_t kmc_buf_max; 89 kstat_named_t kmc_slab_create; 90 kstat_named_t kmc_slab_destroy; 91 kstat_named_t kmc_vmem_source; 92 kstat_named_t kmc_hash_size; 93 kstat_named_t kmc_hash_lookup_depth; 94 kstat_named_t kmc_hash_rescale; 95 kstat_named_t kmc_full_magazines; 96 kstat_named_t kmc_empty_magazines; 97 kstat_named_t kmc_magazine_size; 98 } kmem_cache_kstat = { 99 { "buf_size", KSTAT_DATA_UINT64 }, 100 { "align", KSTAT_DATA_UINT64 }, 101 { "chunk_size", KSTAT_DATA_UINT64 }, 102 { "slab_size", KSTAT_DATA_UINT64 }, 103 { "alloc", KSTAT_DATA_UINT64 }, 104 { "alloc_fail", KSTAT_DATA_UINT64 }, 105 { "free", KSTAT_DATA_UINT64 }, 106 { "depot_alloc", KSTAT_DATA_UINT64 }, 107 { "depot_free", KSTAT_DATA_UINT64 }, 108 { "depot_contention", KSTAT_DATA_UINT64 }, 109 { "slab_alloc", KSTAT_DATA_UINT64 }, 110 { "slab_free", KSTAT_DATA_UINT64 }, 111 { "buf_constructed", KSTAT_DATA_UINT64 }, 112 { "buf_avail", KSTAT_DATA_UINT64 }, 113 { "buf_inuse", KSTAT_DATA_UINT64 }, 114 { "buf_total", KSTAT_DATA_UINT64 }, 115 { "buf_max", KSTAT_DATA_UINT64 }, 116 { "slab_create", KSTAT_DATA_UINT64 }, 117 { "slab_destroy", KSTAT_DATA_UINT64 }, 118 { "vmem_source", KSTAT_DATA_UINT64 }, 119 { "hash_size", KSTAT_DATA_UINT64 }, 120 { "hash_lookup_depth", KSTAT_DATA_UINT64 }, 121 { "hash_rescale", KSTAT_DATA_UINT64 }, 122 { "full_magazines", KSTAT_DATA_UINT64 }, 123 { "empty_magazines", KSTAT_DATA_UINT64 }, 124 { "magazine_size", KSTAT_DATA_UINT64 }, 125 }; 126 127 static kmutex_t kmem_cache_kstat_lock; 128 129 /* 130 * The default set of caches to back kmem_alloc(). 131 * These sizes should be reevaluated periodically. 132 * 133 * We want allocations that are multiples of the coherency granularity 134 * (64 bytes) to be satisfied from a cache which is a multiple of 64 135 * bytes, so that it will be 64-byte aligned. For all multiples of 64, 136 * the next kmem_cache_size greater than or equal to it must be a 137 * multiple of 64. 138 */ 139 static const int kmem_alloc_sizes[] = { 140 1 * 8, 141 2 * 8, 142 3 * 8, 143 4 * 8, 5 * 8, 6 * 8, 7 * 8, 144 4 * 16, 5 * 16, 6 * 16, 7 * 16, 145 4 * 32, 5 * 32, 6 * 32, 7 * 32, 146 4 * 64, 5 * 64, 6 * 64, 7 * 64, 147 4 * 128, 5 * 128, 6 * 128, 7 * 128, 148 P2ALIGN(8192 / 7, 64), 149 P2ALIGN(8192 / 6, 64), 150 P2ALIGN(8192 / 5, 64), 151 P2ALIGN(8192 / 4, 64), 152 P2ALIGN(8192 / 3, 64), 153 P2ALIGN(8192 / 2, 64), 154 P2ALIGN(8192 / 1, 64), 155 4096 * 3, 156 8192 * 2, 157 }; 158 159 #define KMEM_MAXBUF 16384 160 161 static kmem_cache_t *kmem_alloc_table[KMEM_MAXBUF >> KMEM_ALIGN_SHIFT]; 162 163 static kmem_magtype_t kmem_magtype[] = { 164 { 1, 8, 3200, 65536 }, 165 { 3, 16, 256, 32768 }, 166 { 7, 32, 64, 16384 }, 167 { 15, 64, 0, 8192 }, 168 { 31, 64, 0, 4096 }, 169 { 47, 64, 0, 2048 }, 170 { 63, 64, 0, 1024 }, 171 { 95, 64, 0, 512 }, 172 { 143, 64, 0, 0 }, 173 }; 174 175 static uint32_t kmem_reaping; 176 static uint32_t kmem_reaping_idspace; 177 178 /* 179 * kmem tunables 180 */ 181 clock_t kmem_reap_interval; /* cache reaping rate [15 * HZ ticks] */ 182 int kmem_depot_contention = 3; /* max failed tryenters per real interval */ 183 pgcnt_t kmem_reapahead = 0; /* start reaping N pages before pageout */ 184 int kmem_panic = 1; /* whether to panic on error */ 185 int kmem_logging = 1; /* kmem_log_enter() override */ 186 uint32_t kmem_mtbf = 0; /* mean time between failures [default: off] */ 187 size_t kmem_transaction_log_size; /* transaction log size [2% of memory] */ 188 size_t kmem_content_log_size; /* content log size [2% of memory] */ 189 size_t kmem_failure_log_size; /* failure log [4 pages per CPU] */ 190 size_t kmem_slab_log_size; /* slab create log [4 pages per CPU] */ 191 size_t kmem_content_maxsave = 256; /* KMF_CONTENTS max bytes to log */ 192 size_t kmem_lite_minsize = 0; /* minimum buffer size for KMF_LITE */ 193 size_t kmem_lite_maxalign = 1024; /* maximum buffer alignment for KMF_LITE */ 194 int kmem_lite_pcs = 4; /* number of PCs to store in KMF_LITE mode */ 195 size_t kmem_maxverify; /* maximum bytes to inspect in debug routines */ 196 size_t kmem_minfirewall; /* hardware-enforced redzone threshold */ 197 198 #ifdef DEBUG 199 int kmem_flags = KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | KMF_CONTENTS; 200 #else 201 int kmem_flags = 0; 202 #endif 203 int kmem_ready; 204 205 static kmem_cache_t *kmem_slab_cache; 206 static kmem_cache_t *kmem_bufctl_cache; 207 static kmem_cache_t *kmem_bufctl_audit_cache; 208 209 static kmutex_t kmem_cache_lock; /* inter-cache linkage only */ 210 kmem_cache_t kmem_null_cache; 211 212 static taskq_t *kmem_taskq; 213 static kmutex_t kmem_flags_lock; 214 static vmem_t *kmem_metadata_arena; 215 static vmem_t *kmem_msb_arena; /* arena for metadata caches */ 216 static vmem_t *kmem_cache_arena; 217 static vmem_t *kmem_hash_arena; 218 static vmem_t *kmem_log_arena; 219 static vmem_t *kmem_oversize_arena; 220 static vmem_t *kmem_va_arena; 221 static vmem_t *kmem_default_arena; 222 static vmem_t *kmem_firewall_va_arena; 223 static vmem_t *kmem_firewall_arena; 224 225 kmem_log_header_t *kmem_transaction_log; 226 kmem_log_header_t *kmem_content_log; 227 kmem_log_header_t *kmem_failure_log; 228 kmem_log_header_t *kmem_slab_log; 229 230 static int kmem_lite_count; /* # of PCs in kmem_buftag_lite_t */ 231 232 #define KMEM_BUFTAG_LITE_ENTER(bt, count, caller) \ 233 if ((count) > 0) { \ 234 pc_t *_s = ((kmem_buftag_lite_t *)(bt))->bt_history; \ 235 pc_t *_e; \ 236 /* memmove() the old entries down one notch */ \ 237 for (_e = &_s[(count) - 1]; _e > _s; _e--) \ 238 *_e = *(_e - 1); \ 239 *_s = (uintptr_t)(caller); \ 240 } 241 242 #define KMERR_MODIFIED 0 /* buffer modified while on freelist */ 243 #define KMERR_REDZONE 1 /* redzone violation (write past end of buf) */ 244 #define KMERR_DUPFREE 2 /* freed a buffer twice */ 245 #define KMERR_BADADDR 3 /* freed a bad (unallocated) address */ 246 #define KMERR_BADBUFTAG 4 /* buftag corrupted */ 247 #define KMERR_BADBUFCTL 5 /* bufctl corrupted */ 248 #define KMERR_BADCACHE 6 /* freed a buffer to the wrong cache */ 249 #define KMERR_BADSIZE 7 /* alloc size != free size */ 250 #define KMERR_BADBASE 8 /* buffer base address wrong */ 251 252 struct { 253 hrtime_t kmp_timestamp; /* timestamp of panic */ 254 int kmp_error; /* type of kmem error */ 255 void *kmp_buffer; /* buffer that induced panic */ 256 void *kmp_realbuf; /* real start address for buffer */ 257 kmem_cache_t *kmp_cache; /* buffer's cache according to client */ 258 kmem_cache_t *kmp_realcache; /* actual cache containing buffer */ 259 kmem_slab_t *kmp_slab; /* slab accoring to kmem_findslab() */ 260 kmem_bufctl_t *kmp_bufctl; /* bufctl */ 261 } kmem_panic_info; 262 263 264 static void 265 copy_pattern(uint64_t pattern, void *buf_arg, size_t size) 266 { 267 uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 268 uint64_t *buf = buf_arg; 269 270 while (buf < bufend) 271 *buf++ = pattern; 272 } 273 274 static void * 275 verify_pattern(uint64_t pattern, void *buf_arg, size_t size) 276 { 277 uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 278 uint64_t *buf; 279 280 for (buf = buf_arg; buf < bufend; buf++) 281 if (*buf != pattern) 282 return (buf); 283 return (NULL); 284 } 285 286 static void * 287 verify_and_copy_pattern(uint64_t old, uint64_t new, void *buf_arg, size_t size) 288 { 289 uint64_t *bufend = (uint64_t *)((char *)buf_arg + size); 290 uint64_t *buf; 291 292 for (buf = buf_arg; buf < bufend; buf++) { 293 if (*buf != old) { 294 copy_pattern(old, buf_arg, 295 (char *)buf - (char *)buf_arg); 296 return (buf); 297 } 298 *buf = new; 299 } 300 301 return (NULL); 302 } 303 304 static void 305 kmem_cache_applyall(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 306 { 307 kmem_cache_t *cp; 308 309 mutex_enter(&kmem_cache_lock); 310 for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache; 311 cp = cp->cache_next) 312 if (tq != NULL) 313 (void) taskq_dispatch(tq, (task_func_t *)func, cp, 314 tqflag); 315 else 316 func(cp); 317 mutex_exit(&kmem_cache_lock); 318 } 319 320 static void 321 kmem_cache_applyall_id(void (*func)(kmem_cache_t *), taskq_t *tq, int tqflag) 322 { 323 kmem_cache_t *cp; 324 325 mutex_enter(&kmem_cache_lock); 326 for (cp = kmem_null_cache.cache_next; cp != &kmem_null_cache; 327 cp = cp->cache_next) { 328 if (!(cp->cache_cflags & KMC_IDENTIFIER)) 329 continue; 330 if (tq != NULL) 331 (void) taskq_dispatch(tq, (task_func_t *)func, cp, 332 tqflag); 333 else 334 func(cp); 335 } 336 mutex_exit(&kmem_cache_lock); 337 } 338 339 /* 340 * Debugging support. Given a buffer address, find its slab. 341 */ 342 static kmem_slab_t * 343 kmem_findslab(kmem_cache_t *cp, void *buf) 344 { 345 kmem_slab_t *sp; 346 347 mutex_enter(&cp->cache_lock); 348 for (sp = cp->cache_nullslab.slab_next; 349 sp != &cp->cache_nullslab; sp = sp->slab_next) { 350 if (KMEM_SLAB_MEMBER(sp, buf)) { 351 mutex_exit(&cp->cache_lock); 352 return (sp); 353 } 354 } 355 mutex_exit(&cp->cache_lock); 356 357 return (NULL); 358 } 359 360 static void 361 kmem_error(int error, kmem_cache_t *cparg, void *bufarg) 362 { 363 kmem_buftag_t *btp = NULL; 364 kmem_bufctl_t *bcp = NULL; 365 kmem_cache_t *cp = cparg; 366 kmem_slab_t *sp; 367 uint64_t *off; 368 void *buf = bufarg; 369 370 kmem_logging = 0; /* stop logging when a bad thing happens */ 371 372 kmem_panic_info.kmp_timestamp = gethrtime(); 373 374 sp = kmem_findslab(cp, buf); 375 if (sp == NULL) { 376 for (cp = kmem_null_cache.cache_prev; cp != &kmem_null_cache; 377 cp = cp->cache_prev) { 378 if ((sp = kmem_findslab(cp, buf)) != NULL) 379 break; 380 } 381 } 382 383 if (sp == NULL) { 384 cp = NULL; 385 error = KMERR_BADADDR; 386 } else { 387 if (cp != cparg) 388 error = KMERR_BADCACHE; 389 else 390 buf = (char *)bufarg - ((uintptr_t)bufarg - 391 (uintptr_t)sp->slab_base) % cp->cache_chunksize; 392 if (buf != bufarg) 393 error = KMERR_BADBASE; 394 if (cp->cache_flags & KMF_BUFTAG) 395 btp = KMEM_BUFTAG(cp, buf); 396 if (cp->cache_flags & KMF_HASH) { 397 mutex_enter(&cp->cache_lock); 398 for (bcp = *KMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) 399 if (bcp->bc_addr == buf) 400 break; 401 mutex_exit(&cp->cache_lock); 402 if (bcp == NULL && btp != NULL) 403 bcp = btp->bt_bufctl; 404 if (kmem_findslab(cp->cache_bufctl_cache, bcp) == 405 NULL || P2PHASE((uintptr_t)bcp, KMEM_ALIGN) || 406 bcp->bc_addr != buf) { 407 error = KMERR_BADBUFCTL; 408 bcp = NULL; 409 } 410 } 411 } 412 413 kmem_panic_info.kmp_error = error; 414 kmem_panic_info.kmp_buffer = bufarg; 415 kmem_panic_info.kmp_realbuf = buf; 416 kmem_panic_info.kmp_cache = cparg; 417 kmem_panic_info.kmp_realcache = cp; 418 kmem_panic_info.kmp_slab = sp; 419 kmem_panic_info.kmp_bufctl = bcp; 420 421 printf("kernel memory allocator: "); 422 423 switch (error) { 424 425 case KMERR_MODIFIED: 426 printf("buffer modified after being freed\n"); 427 off = verify_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 428 if (off == NULL) /* shouldn't happen */ 429 off = buf; 430 printf("modification occurred at offset 0x%lx " 431 "(0x%llx replaced by 0x%llx)\n", 432 (uintptr_t)off - (uintptr_t)buf, 433 (longlong_t)KMEM_FREE_PATTERN, (longlong_t)*off); 434 break; 435 436 case KMERR_REDZONE: 437 printf("redzone violation: write past end of buffer\n"); 438 break; 439 440 case KMERR_BADADDR: 441 printf("invalid free: buffer not in cache\n"); 442 break; 443 444 case KMERR_DUPFREE: 445 printf("duplicate free: buffer freed twice\n"); 446 break; 447 448 case KMERR_BADBUFTAG: 449 printf("boundary tag corrupted\n"); 450 printf("bcp ^ bxstat = %lx, should be %lx\n", 451 (intptr_t)btp->bt_bufctl ^ btp->bt_bxstat, 452 KMEM_BUFTAG_FREE); 453 break; 454 455 case KMERR_BADBUFCTL: 456 printf("bufctl corrupted\n"); 457 break; 458 459 case KMERR_BADCACHE: 460 printf("buffer freed to wrong cache\n"); 461 printf("buffer was allocated from %s,\n", cp->cache_name); 462 printf("caller attempting free to %s.\n", cparg->cache_name); 463 break; 464 465 case KMERR_BADSIZE: 466 printf("bad free: free size (%u) != alloc size (%u)\n", 467 KMEM_SIZE_DECODE(((uint32_t *)btp)[0]), 468 KMEM_SIZE_DECODE(((uint32_t *)btp)[1])); 469 break; 470 471 case KMERR_BADBASE: 472 printf("bad free: free address (%p) != alloc address (%p)\n", 473 bufarg, buf); 474 break; 475 } 476 477 printf("buffer=%p bufctl=%p cache: %s\n", 478 bufarg, (void *)bcp, cparg->cache_name); 479 480 if (bcp != NULL && (cp->cache_flags & KMF_AUDIT) && 481 error != KMERR_BADBUFCTL) { 482 int d; 483 timestruc_t ts; 484 kmem_bufctl_audit_t *bcap = (kmem_bufctl_audit_t *)bcp; 485 486 hrt2ts(kmem_panic_info.kmp_timestamp - bcap->bc_timestamp, &ts); 487 printf("previous transaction on buffer %p:\n", buf); 488 printf("thread=%p time=T-%ld.%09ld slab=%p cache: %s\n", 489 (void *)bcap->bc_thread, ts.tv_sec, ts.tv_nsec, 490 (void *)sp, cp->cache_name); 491 for (d = 0; d < MIN(bcap->bc_depth, KMEM_STACK_DEPTH); d++) { 492 ulong_t off; 493 char *sym = kobj_getsymname(bcap->bc_stack[d], &off); 494 printf("%s+%lx\n", sym ? sym : "?", off); 495 } 496 } 497 if (kmem_panic > 0) 498 panic("kernel heap corruption detected"); 499 if (kmem_panic == 0) 500 debug_enter(NULL); 501 kmem_logging = 1; /* resume logging */ 502 } 503 504 static kmem_log_header_t * 505 kmem_log_init(size_t logsize) 506 { 507 kmem_log_header_t *lhp; 508 int nchunks = 4 * max_ncpus; 509 size_t lhsize = (size_t)&((kmem_log_header_t *)0)->lh_cpu[max_ncpus]; 510 int i; 511 512 /* 513 * Make sure that lhp->lh_cpu[] is nicely aligned 514 * to prevent false sharing of cache lines. 515 */ 516 lhsize = P2ROUNDUP(lhsize, KMEM_ALIGN); 517 lhp = vmem_xalloc(kmem_log_arena, lhsize, 64, P2NPHASE(lhsize, 64), 0, 518 NULL, NULL, VM_SLEEP); 519 bzero(lhp, lhsize); 520 521 mutex_init(&lhp->lh_lock, NULL, MUTEX_DEFAULT, NULL); 522 lhp->lh_nchunks = nchunks; 523 lhp->lh_chunksize = P2ROUNDUP(logsize / nchunks + 1, PAGESIZE); 524 lhp->lh_base = vmem_alloc(kmem_log_arena, 525 lhp->lh_chunksize * nchunks, VM_SLEEP); 526 lhp->lh_free = vmem_alloc(kmem_log_arena, 527 nchunks * sizeof (int), VM_SLEEP); 528 bzero(lhp->lh_base, lhp->lh_chunksize * nchunks); 529 530 for (i = 0; i < max_ncpus; i++) { 531 kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[i]; 532 mutex_init(&clhp->clh_lock, NULL, MUTEX_DEFAULT, NULL); 533 clhp->clh_chunk = i; 534 } 535 536 for (i = max_ncpus; i < nchunks; i++) 537 lhp->lh_free[i] = i; 538 539 lhp->lh_head = max_ncpus; 540 lhp->lh_tail = 0; 541 542 return (lhp); 543 } 544 545 static void * 546 kmem_log_enter(kmem_log_header_t *lhp, void *data, size_t size) 547 { 548 void *logspace; 549 kmem_cpu_log_header_t *clhp = &lhp->lh_cpu[CPU->cpu_seqid]; 550 551 if (lhp == NULL || kmem_logging == 0 || panicstr) 552 return (NULL); 553 554 mutex_enter(&clhp->clh_lock); 555 clhp->clh_hits++; 556 if (size > clhp->clh_avail) { 557 mutex_enter(&lhp->lh_lock); 558 lhp->lh_hits++; 559 lhp->lh_free[lhp->lh_tail] = clhp->clh_chunk; 560 lhp->lh_tail = (lhp->lh_tail + 1) % lhp->lh_nchunks; 561 clhp->clh_chunk = lhp->lh_free[lhp->lh_head]; 562 lhp->lh_head = (lhp->lh_head + 1) % lhp->lh_nchunks; 563 clhp->clh_current = lhp->lh_base + 564 clhp->clh_chunk * lhp->lh_chunksize; 565 clhp->clh_avail = lhp->lh_chunksize; 566 if (size > lhp->lh_chunksize) 567 size = lhp->lh_chunksize; 568 mutex_exit(&lhp->lh_lock); 569 } 570 logspace = clhp->clh_current; 571 clhp->clh_current += size; 572 clhp->clh_avail -= size; 573 bcopy(data, logspace, size); 574 mutex_exit(&clhp->clh_lock); 575 return (logspace); 576 } 577 578 #define KMEM_AUDIT(lp, cp, bcp) \ 579 { \ 580 kmem_bufctl_audit_t *_bcp = (kmem_bufctl_audit_t *)(bcp); \ 581 _bcp->bc_timestamp = gethrtime(); \ 582 _bcp->bc_thread = curthread; \ 583 _bcp->bc_depth = getpcstack(_bcp->bc_stack, KMEM_STACK_DEPTH); \ 584 _bcp->bc_lastlog = kmem_log_enter((lp), _bcp, sizeof (*_bcp)); \ 585 } 586 587 static void 588 kmem_log_event(kmem_log_header_t *lp, kmem_cache_t *cp, 589 kmem_slab_t *sp, void *addr) 590 { 591 kmem_bufctl_audit_t bca; 592 593 bzero(&bca, sizeof (kmem_bufctl_audit_t)); 594 bca.bc_addr = addr; 595 bca.bc_slab = sp; 596 bca.bc_cache = cp; 597 KMEM_AUDIT(lp, cp, &bca); 598 } 599 600 /* 601 * Create a new slab for cache cp. 602 */ 603 static kmem_slab_t * 604 kmem_slab_create(kmem_cache_t *cp, int kmflag) 605 { 606 size_t slabsize = cp->cache_slabsize; 607 size_t chunksize = cp->cache_chunksize; 608 int cache_flags = cp->cache_flags; 609 size_t color, chunks; 610 char *buf, *slab; 611 kmem_slab_t *sp; 612 kmem_bufctl_t *bcp; 613 vmem_t *vmp = cp->cache_arena; 614 615 color = cp->cache_color + cp->cache_align; 616 if (color > cp->cache_maxcolor) 617 color = cp->cache_mincolor; 618 cp->cache_color = color; 619 620 slab = vmem_alloc(vmp, slabsize, kmflag & KM_VMFLAGS); 621 622 if (slab == NULL) 623 goto vmem_alloc_failure; 624 625 ASSERT(P2PHASE((uintptr_t)slab, vmp->vm_quantum) == 0); 626 627 if (!(cp->cache_cflags & KMC_NOTOUCH)) 628 copy_pattern(KMEM_UNINITIALIZED_PATTERN, slab, slabsize); 629 630 if (cache_flags & KMF_HASH) { 631 if ((sp = kmem_cache_alloc(kmem_slab_cache, kmflag)) == NULL) 632 goto slab_alloc_failure; 633 chunks = (slabsize - color) / chunksize; 634 } else { 635 sp = KMEM_SLAB(cp, slab); 636 chunks = (slabsize - sizeof (kmem_slab_t) - color) / chunksize; 637 } 638 639 sp->slab_cache = cp; 640 sp->slab_head = NULL; 641 sp->slab_refcnt = 0; 642 sp->slab_base = buf = slab + color; 643 sp->slab_chunks = chunks; 644 645 ASSERT(chunks > 0); 646 while (chunks-- != 0) { 647 if (cache_flags & KMF_HASH) { 648 bcp = kmem_cache_alloc(cp->cache_bufctl_cache, kmflag); 649 if (bcp == NULL) 650 goto bufctl_alloc_failure; 651 if (cache_flags & KMF_AUDIT) { 652 kmem_bufctl_audit_t *bcap = 653 (kmem_bufctl_audit_t *)bcp; 654 bzero(bcap, sizeof (kmem_bufctl_audit_t)); 655 bcap->bc_cache = cp; 656 } 657 bcp->bc_addr = buf; 658 bcp->bc_slab = sp; 659 } else { 660 bcp = KMEM_BUFCTL(cp, buf); 661 } 662 if (cache_flags & KMF_BUFTAG) { 663 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 664 btp->bt_redzone = KMEM_REDZONE_PATTERN; 665 btp->bt_bufctl = bcp; 666 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 667 if (cache_flags & KMF_DEADBEEF) { 668 copy_pattern(KMEM_FREE_PATTERN, buf, 669 cp->cache_verify); 670 } 671 } 672 bcp->bc_next = sp->slab_head; 673 sp->slab_head = bcp; 674 buf += chunksize; 675 } 676 677 kmem_log_event(kmem_slab_log, cp, sp, slab); 678 679 return (sp); 680 681 bufctl_alloc_failure: 682 683 while ((bcp = sp->slab_head) != NULL) { 684 sp->slab_head = bcp->bc_next; 685 kmem_cache_free(cp->cache_bufctl_cache, bcp); 686 } 687 kmem_cache_free(kmem_slab_cache, sp); 688 689 slab_alloc_failure: 690 691 vmem_free(vmp, slab, slabsize); 692 693 vmem_alloc_failure: 694 695 kmem_log_event(kmem_failure_log, cp, NULL, NULL); 696 atomic_add_64(&cp->cache_alloc_fail, 1); 697 698 return (NULL); 699 } 700 701 /* 702 * Destroy a slab. 703 */ 704 static void 705 kmem_slab_destroy(kmem_cache_t *cp, kmem_slab_t *sp) 706 { 707 vmem_t *vmp = cp->cache_arena; 708 void *slab = (void *)P2ALIGN((uintptr_t)sp->slab_base, vmp->vm_quantum); 709 710 if (cp->cache_flags & KMF_HASH) { 711 kmem_bufctl_t *bcp; 712 while ((bcp = sp->slab_head) != NULL) { 713 sp->slab_head = bcp->bc_next; 714 kmem_cache_free(cp->cache_bufctl_cache, bcp); 715 } 716 kmem_cache_free(kmem_slab_cache, sp); 717 } 718 vmem_free(vmp, slab, cp->cache_slabsize); 719 } 720 721 /* 722 * Allocate a raw (unconstructed) buffer from cp's slab layer. 723 */ 724 static void * 725 kmem_slab_alloc(kmem_cache_t *cp, int kmflag) 726 { 727 kmem_bufctl_t *bcp, **hash_bucket; 728 kmem_slab_t *sp; 729 void *buf; 730 731 mutex_enter(&cp->cache_lock); 732 cp->cache_slab_alloc++; 733 sp = cp->cache_freelist; 734 ASSERT(sp->slab_cache == cp); 735 if (sp->slab_head == NULL) { 736 /* 737 * The freelist is empty. Create a new slab. 738 */ 739 mutex_exit(&cp->cache_lock); 740 if ((sp = kmem_slab_create(cp, kmflag)) == NULL) 741 return (NULL); 742 mutex_enter(&cp->cache_lock); 743 cp->cache_slab_create++; 744 if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) 745 cp->cache_bufmax = cp->cache_buftotal; 746 sp->slab_next = cp->cache_freelist; 747 sp->slab_prev = cp->cache_freelist->slab_prev; 748 sp->slab_next->slab_prev = sp; 749 sp->slab_prev->slab_next = sp; 750 cp->cache_freelist = sp; 751 } 752 753 sp->slab_refcnt++; 754 ASSERT(sp->slab_refcnt <= sp->slab_chunks); 755 756 /* 757 * If we're taking the last buffer in the slab, 758 * remove the slab from the cache's freelist. 759 */ 760 bcp = sp->slab_head; 761 if ((sp->slab_head = bcp->bc_next) == NULL) { 762 cp->cache_freelist = sp->slab_next; 763 ASSERT(sp->slab_refcnt == sp->slab_chunks); 764 } 765 766 if (cp->cache_flags & KMF_HASH) { 767 /* 768 * Add buffer to allocated-address hash table. 769 */ 770 buf = bcp->bc_addr; 771 hash_bucket = KMEM_HASH(cp, buf); 772 bcp->bc_next = *hash_bucket; 773 *hash_bucket = bcp; 774 if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 775 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 776 } 777 } else { 778 buf = KMEM_BUF(cp, bcp); 779 } 780 781 ASSERT(KMEM_SLAB_MEMBER(sp, buf)); 782 783 mutex_exit(&cp->cache_lock); 784 785 return (buf); 786 } 787 788 /* 789 * Free a raw (unconstructed) buffer to cp's slab layer. 790 */ 791 static void 792 kmem_slab_free(kmem_cache_t *cp, void *buf) 793 { 794 kmem_slab_t *sp; 795 kmem_bufctl_t *bcp, **prev_bcpp; 796 797 ASSERT(buf != NULL); 798 799 mutex_enter(&cp->cache_lock); 800 cp->cache_slab_free++; 801 802 if (cp->cache_flags & KMF_HASH) { 803 /* 804 * Look up buffer in allocated-address hash table. 805 */ 806 prev_bcpp = KMEM_HASH(cp, buf); 807 while ((bcp = *prev_bcpp) != NULL) { 808 if (bcp->bc_addr == buf) { 809 *prev_bcpp = bcp->bc_next; 810 sp = bcp->bc_slab; 811 break; 812 } 813 cp->cache_lookup_depth++; 814 prev_bcpp = &bcp->bc_next; 815 } 816 } else { 817 bcp = KMEM_BUFCTL(cp, buf); 818 sp = KMEM_SLAB(cp, buf); 819 } 820 821 if (bcp == NULL || sp->slab_cache != cp || !KMEM_SLAB_MEMBER(sp, buf)) { 822 mutex_exit(&cp->cache_lock); 823 kmem_error(KMERR_BADADDR, cp, buf); 824 return; 825 } 826 827 if ((cp->cache_flags & (KMF_AUDIT | KMF_BUFTAG)) == KMF_AUDIT) { 828 if (cp->cache_flags & KMF_CONTENTS) 829 ((kmem_bufctl_audit_t *)bcp)->bc_contents = 830 kmem_log_enter(kmem_content_log, buf, 831 cp->cache_contents); 832 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 833 } 834 835 /* 836 * If this slab isn't currently on the freelist, put it there. 837 */ 838 if (sp->slab_head == NULL) { 839 ASSERT(sp->slab_refcnt == sp->slab_chunks); 840 ASSERT(cp->cache_freelist != sp); 841 sp->slab_next->slab_prev = sp->slab_prev; 842 sp->slab_prev->slab_next = sp->slab_next; 843 sp->slab_next = cp->cache_freelist; 844 sp->slab_prev = cp->cache_freelist->slab_prev; 845 sp->slab_next->slab_prev = sp; 846 sp->slab_prev->slab_next = sp; 847 cp->cache_freelist = sp; 848 } 849 850 bcp->bc_next = sp->slab_head; 851 sp->slab_head = bcp; 852 853 ASSERT(sp->slab_refcnt >= 1); 854 if (--sp->slab_refcnt == 0) { 855 /* 856 * There are no outstanding allocations from this slab, 857 * so we can reclaim the memory. 858 */ 859 sp->slab_next->slab_prev = sp->slab_prev; 860 sp->slab_prev->slab_next = sp->slab_next; 861 if (sp == cp->cache_freelist) 862 cp->cache_freelist = sp->slab_next; 863 cp->cache_slab_destroy++; 864 cp->cache_buftotal -= sp->slab_chunks; 865 mutex_exit(&cp->cache_lock); 866 kmem_slab_destroy(cp, sp); 867 return; 868 } 869 mutex_exit(&cp->cache_lock); 870 } 871 872 static int 873 kmem_cache_alloc_debug(kmem_cache_t *cp, void *buf, int kmflag, int construct, 874 caddr_t caller) 875 { 876 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 877 kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 878 uint32_t mtbf; 879 880 if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 881 kmem_error(KMERR_BADBUFTAG, cp, buf); 882 return (-1); 883 } 884 885 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_ALLOC; 886 887 if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 888 kmem_error(KMERR_BADBUFCTL, cp, buf); 889 return (-1); 890 } 891 892 if (cp->cache_flags & KMF_DEADBEEF) { 893 if (!construct && (cp->cache_flags & KMF_LITE)) { 894 if (*(uint64_t *)buf != KMEM_FREE_PATTERN) { 895 kmem_error(KMERR_MODIFIED, cp, buf); 896 return (-1); 897 } 898 if (cp->cache_constructor != NULL) 899 *(uint64_t *)buf = btp->bt_redzone; 900 else 901 *(uint64_t *)buf = KMEM_UNINITIALIZED_PATTERN; 902 } else { 903 construct = 1; 904 if (verify_and_copy_pattern(KMEM_FREE_PATTERN, 905 KMEM_UNINITIALIZED_PATTERN, buf, 906 cp->cache_verify)) { 907 kmem_error(KMERR_MODIFIED, cp, buf); 908 return (-1); 909 } 910 } 911 } 912 btp->bt_redzone = KMEM_REDZONE_PATTERN; 913 914 if ((mtbf = kmem_mtbf | cp->cache_mtbf) != 0 && 915 gethrtime() % mtbf == 0 && 916 (kmflag & (KM_NOSLEEP | KM_PANIC)) == KM_NOSLEEP) { 917 kmem_log_event(kmem_failure_log, cp, NULL, NULL); 918 if (!construct && cp->cache_destructor != NULL) 919 cp->cache_destructor(buf, cp->cache_private); 920 } else { 921 mtbf = 0; 922 } 923 924 if (mtbf || (construct && cp->cache_constructor != NULL && 925 cp->cache_constructor(buf, cp->cache_private, kmflag) != 0)) { 926 atomic_add_64(&cp->cache_alloc_fail, 1); 927 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 928 if (cp->cache_flags & KMF_DEADBEEF) 929 copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 930 kmem_slab_free(cp, buf); 931 return (-1); 932 } 933 934 if (cp->cache_flags & KMF_AUDIT) { 935 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 936 } 937 938 if ((cp->cache_flags & KMF_LITE) && 939 !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 940 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 941 } 942 943 return (0); 944 } 945 946 static int 947 kmem_cache_free_debug(kmem_cache_t *cp, void *buf, caddr_t caller) 948 { 949 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 950 kmem_bufctl_audit_t *bcp = (kmem_bufctl_audit_t *)btp->bt_bufctl; 951 kmem_slab_t *sp; 952 953 if (btp->bt_bxstat != ((intptr_t)bcp ^ KMEM_BUFTAG_ALLOC)) { 954 if (btp->bt_bxstat == ((intptr_t)bcp ^ KMEM_BUFTAG_FREE)) { 955 kmem_error(KMERR_DUPFREE, cp, buf); 956 return (-1); 957 } 958 sp = kmem_findslab(cp, buf); 959 if (sp == NULL || sp->slab_cache != cp) 960 kmem_error(KMERR_BADADDR, cp, buf); 961 else 962 kmem_error(KMERR_REDZONE, cp, buf); 963 return (-1); 964 } 965 966 btp->bt_bxstat = (intptr_t)bcp ^ KMEM_BUFTAG_FREE; 967 968 if ((cp->cache_flags & KMF_HASH) && bcp->bc_addr != buf) { 969 kmem_error(KMERR_BADBUFCTL, cp, buf); 970 return (-1); 971 } 972 973 if (btp->bt_redzone != KMEM_REDZONE_PATTERN) { 974 kmem_error(KMERR_REDZONE, cp, buf); 975 return (-1); 976 } 977 978 if (cp->cache_flags & KMF_AUDIT) { 979 if (cp->cache_flags & KMF_CONTENTS) 980 bcp->bc_contents = kmem_log_enter(kmem_content_log, 981 buf, cp->cache_contents); 982 KMEM_AUDIT(kmem_transaction_log, cp, bcp); 983 } 984 985 if ((cp->cache_flags & KMF_LITE) && 986 !(cp->cache_cflags & KMC_KMEM_ALLOC)) { 987 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, caller); 988 } 989 990 if (cp->cache_flags & KMF_DEADBEEF) { 991 if (cp->cache_flags & KMF_LITE) 992 btp->bt_redzone = *(uint64_t *)buf; 993 else if (cp->cache_destructor != NULL) 994 cp->cache_destructor(buf, cp->cache_private); 995 996 copy_pattern(KMEM_FREE_PATTERN, buf, cp->cache_verify); 997 } 998 999 return (0); 1000 } 1001 1002 /* 1003 * Free each object in magazine mp to cp's slab layer, and free mp itself. 1004 */ 1005 static void 1006 kmem_magazine_destroy(kmem_cache_t *cp, kmem_magazine_t *mp, int nrounds) 1007 { 1008 int round; 1009 1010 ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 1011 1012 for (round = 0; round < nrounds; round++) { 1013 void *buf = mp->mag_round[round]; 1014 1015 if (cp->cache_flags & KMF_DEADBEEF) { 1016 if (verify_pattern(KMEM_FREE_PATTERN, buf, 1017 cp->cache_verify) != NULL) { 1018 kmem_error(KMERR_MODIFIED, cp, buf); 1019 continue; 1020 } 1021 if ((cp->cache_flags & KMF_LITE) && 1022 cp->cache_destructor != NULL) { 1023 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1024 *(uint64_t *)buf = btp->bt_redzone; 1025 cp->cache_destructor(buf, cp->cache_private); 1026 *(uint64_t *)buf = KMEM_FREE_PATTERN; 1027 } 1028 } else if (cp->cache_destructor != NULL) { 1029 cp->cache_destructor(buf, cp->cache_private); 1030 } 1031 1032 kmem_slab_free(cp, buf); 1033 } 1034 ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 1035 kmem_cache_free(cp->cache_magtype->mt_cache, mp); 1036 } 1037 1038 /* 1039 * Allocate a magazine from the depot. 1040 */ 1041 static kmem_magazine_t * 1042 kmem_depot_alloc(kmem_cache_t *cp, kmem_maglist_t *mlp) 1043 { 1044 kmem_magazine_t *mp; 1045 1046 /* 1047 * If we can't get the depot lock without contention, 1048 * update our contention count. We use the depot 1049 * contention rate to determine whether we need to 1050 * increase the magazine size for better scalability. 1051 */ 1052 if (!mutex_tryenter(&cp->cache_depot_lock)) { 1053 mutex_enter(&cp->cache_depot_lock); 1054 cp->cache_depot_contention++; 1055 } 1056 1057 if ((mp = mlp->ml_list) != NULL) { 1058 ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 1059 mlp->ml_list = mp->mag_next; 1060 if (--mlp->ml_total < mlp->ml_min) 1061 mlp->ml_min = mlp->ml_total; 1062 mlp->ml_alloc++; 1063 } 1064 1065 mutex_exit(&cp->cache_depot_lock); 1066 1067 return (mp); 1068 } 1069 1070 /* 1071 * Free a magazine to the depot. 1072 */ 1073 static void 1074 kmem_depot_free(kmem_cache_t *cp, kmem_maglist_t *mlp, kmem_magazine_t *mp) 1075 { 1076 mutex_enter(&cp->cache_depot_lock); 1077 ASSERT(KMEM_MAGAZINE_VALID(cp, mp)); 1078 mp->mag_next = mlp->ml_list; 1079 mlp->ml_list = mp; 1080 mlp->ml_total++; 1081 mutex_exit(&cp->cache_depot_lock); 1082 } 1083 1084 /* 1085 * Update the working set statistics for cp's depot. 1086 */ 1087 static void 1088 kmem_depot_ws_update(kmem_cache_t *cp) 1089 { 1090 mutex_enter(&cp->cache_depot_lock); 1091 cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; 1092 cp->cache_full.ml_min = cp->cache_full.ml_total; 1093 cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; 1094 cp->cache_empty.ml_min = cp->cache_empty.ml_total; 1095 mutex_exit(&cp->cache_depot_lock); 1096 } 1097 1098 /* 1099 * Reap all magazines that have fallen out of the depot's working set. 1100 */ 1101 static void 1102 kmem_depot_ws_reap(kmem_cache_t *cp) 1103 { 1104 long reap; 1105 kmem_magazine_t *mp; 1106 1107 ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 1108 1109 reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); 1110 while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL) 1111 kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); 1112 1113 reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); 1114 while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL) 1115 kmem_magazine_destroy(cp, mp, 0); 1116 } 1117 1118 static void 1119 kmem_cpu_reload(kmem_cpu_cache_t *ccp, kmem_magazine_t *mp, int rounds) 1120 { 1121 ASSERT((ccp->cc_loaded == NULL && ccp->cc_rounds == -1) || 1122 (ccp->cc_loaded && ccp->cc_rounds + rounds == ccp->cc_magsize)); 1123 ASSERT(ccp->cc_magsize > 0); 1124 1125 ccp->cc_ploaded = ccp->cc_loaded; 1126 ccp->cc_prounds = ccp->cc_rounds; 1127 ccp->cc_loaded = mp; 1128 ccp->cc_rounds = rounds; 1129 } 1130 1131 /* 1132 * Allocate a constructed object from cache cp. 1133 */ 1134 void * 1135 kmem_cache_alloc(kmem_cache_t *cp, int kmflag) 1136 { 1137 kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 1138 kmem_magazine_t *fmp; 1139 void *buf; 1140 1141 mutex_enter(&ccp->cc_lock); 1142 for (;;) { 1143 /* 1144 * If there's an object available in the current CPU's 1145 * loaded magazine, just take it and return. 1146 */ 1147 if (ccp->cc_rounds > 0) { 1148 buf = ccp->cc_loaded->mag_round[--ccp->cc_rounds]; 1149 ccp->cc_alloc++; 1150 mutex_exit(&ccp->cc_lock); 1151 if ((ccp->cc_flags & KMF_BUFTAG) && 1152 kmem_cache_alloc_debug(cp, buf, kmflag, 0, 1153 caller()) == -1) { 1154 if (kmflag & KM_NOSLEEP) 1155 return (NULL); 1156 mutex_enter(&ccp->cc_lock); 1157 continue; 1158 } 1159 return (buf); 1160 } 1161 1162 /* 1163 * The loaded magazine is empty. If the previously loaded 1164 * magazine was full, exchange them and try again. 1165 */ 1166 if (ccp->cc_prounds > 0) { 1167 kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 1168 continue; 1169 } 1170 1171 /* 1172 * If the magazine layer is disabled, break out now. 1173 */ 1174 if (ccp->cc_magsize == 0) 1175 break; 1176 1177 /* 1178 * Try to get a full magazine from the depot. 1179 */ 1180 fmp = kmem_depot_alloc(cp, &cp->cache_full); 1181 if (fmp != NULL) { 1182 if (ccp->cc_ploaded != NULL) 1183 kmem_depot_free(cp, &cp->cache_empty, 1184 ccp->cc_ploaded); 1185 kmem_cpu_reload(ccp, fmp, ccp->cc_magsize); 1186 continue; 1187 } 1188 1189 /* 1190 * There are no full magazines in the depot, 1191 * so fall through to the slab layer. 1192 */ 1193 break; 1194 } 1195 mutex_exit(&ccp->cc_lock); 1196 1197 /* 1198 * We couldn't allocate a constructed object from the magazine layer, 1199 * so get a raw buffer from the slab layer and apply its constructor. 1200 */ 1201 buf = kmem_slab_alloc(cp, kmflag); 1202 1203 if (buf == NULL) 1204 return (NULL); 1205 1206 if (cp->cache_flags & KMF_BUFTAG) { 1207 /* 1208 * Make kmem_cache_alloc_debug() apply the constructor for us. 1209 */ 1210 if (kmem_cache_alloc_debug(cp, buf, kmflag, 1, 1211 caller()) == -1) { 1212 if (kmflag & KM_NOSLEEP) 1213 return (NULL); 1214 /* 1215 * kmem_cache_alloc_debug() detected corruption 1216 * but didn't panic (kmem_panic <= 0). Try again. 1217 */ 1218 return (kmem_cache_alloc(cp, kmflag)); 1219 } 1220 return (buf); 1221 } 1222 1223 if (cp->cache_constructor != NULL && 1224 cp->cache_constructor(buf, cp->cache_private, kmflag) != 0) { 1225 atomic_add_64(&cp->cache_alloc_fail, 1); 1226 kmem_slab_free(cp, buf); 1227 return (NULL); 1228 } 1229 1230 return (buf); 1231 } 1232 1233 /* 1234 * Free a constructed object to cache cp. 1235 */ 1236 void 1237 kmem_cache_free(kmem_cache_t *cp, void *buf) 1238 { 1239 kmem_cpu_cache_t *ccp = KMEM_CPU_CACHE(cp); 1240 kmem_magazine_t *emp; 1241 kmem_magtype_t *mtp; 1242 1243 if (ccp->cc_flags & KMF_BUFTAG) 1244 if (kmem_cache_free_debug(cp, buf, caller()) == -1) 1245 return; 1246 1247 mutex_enter(&ccp->cc_lock); 1248 for (;;) { 1249 /* 1250 * If there's a slot available in the current CPU's 1251 * loaded magazine, just put the object there and return. 1252 */ 1253 if ((uint_t)ccp->cc_rounds < ccp->cc_magsize) { 1254 ccp->cc_loaded->mag_round[ccp->cc_rounds++] = buf; 1255 ccp->cc_free++; 1256 mutex_exit(&ccp->cc_lock); 1257 return; 1258 } 1259 1260 /* 1261 * The loaded magazine is full. If the previously loaded 1262 * magazine was empty, exchange them and try again. 1263 */ 1264 if (ccp->cc_prounds == 0) { 1265 kmem_cpu_reload(ccp, ccp->cc_ploaded, ccp->cc_prounds); 1266 continue; 1267 } 1268 1269 /* 1270 * If the magazine layer is disabled, break out now. 1271 */ 1272 if (ccp->cc_magsize == 0) 1273 break; 1274 1275 /* 1276 * Try to get an empty magazine from the depot. 1277 */ 1278 emp = kmem_depot_alloc(cp, &cp->cache_empty); 1279 if (emp != NULL) { 1280 if (ccp->cc_ploaded != NULL) 1281 kmem_depot_free(cp, &cp->cache_full, 1282 ccp->cc_ploaded); 1283 kmem_cpu_reload(ccp, emp, 0); 1284 continue; 1285 } 1286 1287 /* 1288 * There are no empty magazines in the depot, 1289 * so try to allocate a new one. We must drop all locks 1290 * across kmem_cache_alloc() because lower layers may 1291 * attempt to allocate from this cache. 1292 */ 1293 mtp = cp->cache_magtype; 1294 mutex_exit(&ccp->cc_lock); 1295 emp = kmem_cache_alloc(mtp->mt_cache, KM_NOSLEEP); 1296 mutex_enter(&ccp->cc_lock); 1297 1298 if (emp != NULL) { 1299 /* 1300 * We successfully allocated an empty magazine. 1301 * However, we had to drop ccp->cc_lock to do it, 1302 * so the cache's magazine size may have changed. 1303 * If so, free the magazine and try again. 1304 */ 1305 if (ccp->cc_magsize != mtp->mt_magsize) { 1306 mutex_exit(&ccp->cc_lock); 1307 kmem_cache_free(mtp->mt_cache, emp); 1308 mutex_enter(&ccp->cc_lock); 1309 continue; 1310 } 1311 1312 /* 1313 * We got a magazine of the right size. Add it to 1314 * the depot and try the whole dance again. 1315 */ 1316 kmem_depot_free(cp, &cp->cache_empty, emp); 1317 continue; 1318 } 1319 1320 /* 1321 * We couldn't allocate an empty magazine, 1322 * so fall through to the slab layer. 1323 */ 1324 break; 1325 } 1326 mutex_exit(&ccp->cc_lock); 1327 1328 /* 1329 * We couldn't free our constructed object to the magazine layer, 1330 * so apply its destructor and free it to the slab layer. 1331 * Note that if KMF_DEADBEEF is in effect and KMF_LITE is not, 1332 * kmem_cache_free_debug() will have already applied the destructor. 1333 */ 1334 if ((cp->cache_flags & (KMF_DEADBEEF | KMF_LITE)) != KMF_DEADBEEF && 1335 cp->cache_destructor != NULL) { 1336 if (cp->cache_flags & KMF_DEADBEEF) { /* KMF_LITE implied */ 1337 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1338 *(uint64_t *)buf = btp->bt_redzone; 1339 cp->cache_destructor(buf, cp->cache_private); 1340 *(uint64_t *)buf = KMEM_FREE_PATTERN; 1341 } else { 1342 cp->cache_destructor(buf, cp->cache_private); 1343 } 1344 } 1345 1346 kmem_slab_free(cp, buf); 1347 } 1348 1349 void * 1350 kmem_zalloc(size_t size, int kmflag) 1351 { 1352 size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 1353 void *buf; 1354 1355 if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 1356 kmem_cache_t *cp = kmem_alloc_table[index]; 1357 buf = kmem_cache_alloc(cp, kmflag); 1358 if (buf != NULL) { 1359 if (cp->cache_flags & KMF_BUFTAG) { 1360 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1361 ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 1362 ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 1363 1364 if (cp->cache_flags & KMF_LITE) { 1365 KMEM_BUFTAG_LITE_ENTER(btp, 1366 kmem_lite_count, caller()); 1367 } 1368 } 1369 bzero(buf, size); 1370 } 1371 } else { 1372 buf = kmem_alloc(size, kmflag); 1373 if (buf != NULL) 1374 bzero(buf, size); 1375 } 1376 return (buf); 1377 } 1378 1379 void * 1380 kmem_alloc(size_t size, int kmflag) 1381 { 1382 size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 1383 void *buf; 1384 1385 if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 1386 kmem_cache_t *cp = kmem_alloc_table[index]; 1387 buf = kmem_cache_alloc(cp, kmflag); 1388 if ((cp->cache_flags & KMF_BUFTAG) && buf != NULL) { 1389 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1390 ((uint8_t *)buf)[size] = KMEM_REDZONE_BYTE; 1391 ((uint32_t *)btp)[1] = KMEM_SIZE_ENCODE(size); 1392 1393 if (cp->cache_flags & KMF_LITE) { 1394 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 1395 caller()); 1396 } 1397 } 1398 return (buf); 1399 } 1400 if (size == 0) 1401 return (NULL); 1402 buf = vmem_alloc(kmem_oversize_arena, size, kmflag & KM_VMFLAGS); 1403 if (buf == NULL) 1404 kmem_log_event(kmem_failure_log, NULL, NULL, (void *)size); 1405 return (buf); 1406 } 1407 1408 void 1409 kmem_free(void *buf, size_t size) 1410 { 1411 size_t index = (size - 1) >> KMEM_ALIGN_SHIFT; 1412 1413 if (index < KMEM_MAXBUF >> KMEM_ALIGN_SHIFT) { 1414 kmem_cache_t *cp = kmem_alloc_table[index]; 1415 if (cp->cache_flags & KMF_BUFTAG) { 1416 kmem_buftag_t *btp = KMEM_BUFTAG(cp, buf); 1417 uint32_t *ip = (uint32_t *)btp; 1418 if (ip[1] != KMEM_SIZE_ENCODE(size)) { 1419 if (*(uint64_t *)buf == KMEM_FREE_PATTERN) { 1420 kmem_error(KMERR_DUPFREE, cp, buf); 1421 return; 1422 } 1423 if (KMEM_SIZE_VALID(ip[1])) { 1424 ip[0] = KMEM_SIZE_ENCODE(size); 1425 kmem_error(KMERR_BADSIZE, cp, buf); 1426 } else { 1427 kmem_error(KMERR_REDZONE, cp, buf); 1428 } 1429 return; 1430 } 1431 if (((uint8_t *)buf)[size] != KMEM_REDZONE_BYTE) { 1432 kmem_error(KMERR_REDZONE, cp, buf); 1433 return; 1434 } 1435 btp->bt_redzone = KMEM_REDZONE_PATTERN; 1436 if (cp->cache_flags & KMF_LITE) { 1437 KMEM_BUFTAG_LITE_ENTER(btp, kmem_lite_count, 1438 caller()); 1439 } 1440 } 1441 kmem_cache_free(cp, buf); 1442 } else { 1443 if (buf == NULL && size == 0) 1444 return; 1445 vmem_free(kmem_oversize_arena, buf, size); 1446 } 1447 } 1448 1449 void * 1450 kmem_firewall_va_alloc(vmem_t *vmp, size_t size, int vmflag) 1451 { 1452 size_t realsize = size + vmp->vm_quantum; 1453 void *addr; 1454 1455 /* 1456 * Annoying edge case: if 'size' is just shy of ULONG_MAX, adding 1457 * vm_quantum will cause integer wraparound. Check for this, and 1458 * blow off the firewall page in this case. Note that such a 1459 * giant allocation (the entire kernel address space) can never 1460 * be satisfied, so it will either fail immediately (VM_NOSLEEP) 1461 * or sleep forever (VM_SLEEP). Thus, there is no need for a 1462 * corresponding check in kmem_firewall_va_free(). 1463 */ 1464 if (realsize < size) 1465 realsize = size; 1466 1467 /* 1468 * While boot still owns resource management, make sure that this 1469 * redzone virtual address allocation is properly accounted for in 1470 * OBPs "virtual-memory" "available" lists because we're 1471 * effectively claiming them for a red zone. If we don't do this, 1472 * the available lists become too fragmented and too large for the 1473 * current boot/kernel memory list interface. 1474 */ 1475 addr = vmem_alloc(vmp, realsize, vmflag | VM_NEXTFIT); 1476 1477 if (addr != NULL && kvseg.s_base == NULL && realsize != size) 1478 (void) boot_virt_alloc((char *)addr + size, vmp->vm_quantum); 1479 1480 return (addr); 1481 } 1482 1483 void 1484 kmem_firewall_va_free(vmem_t *vmp, void *addr, size_t size) 1485 { 1486 ASSERT((kvseg.s_base == NULL ? 1487 va_to_pfn((char *)addr + size) : 1488 hat_getpfnum(kas.a_hat, (caddr_t)addr + size)) == PFN_INVALID); 1489 1490 vmem_free(vmp, addr, size + vmp->vm_quantum); 1491 } 1492 1493 /* 1494 * Try to allocate at least `size' bytes of memory without sleeping or 1495 * panicking. Return actual allocated size in `asize'. If allocation failed, 1496 * try final allocation with sleep or panic allowed. 1497 */ 1498 void * 1499 kmem_alloc_tryhard(size_t size, size_t *asize, int kmflag) 1500 { 1501 void *p; 1502 1503 *asize = P2ROUNDUP(size, KMEM_ALIGN); 1504 do { 1505 p = kmem_alloc(*asize, (kmflag | KM_NOSLEEP) & ~KM_PANIC); 1506 if (p != NULL) 1507 return (p); 1508 *asize += KMEM_ALIGN; 1509 } while (*asize <= PAGESIZE); 1510 1511 *asize = P2ROUNDUP(size, KMEM_ALIGN); 1512 return (kmem_alloc(*asize, kmflag)); 1513 } 1514 1515 /* 1516 * Reclaim all unused memory from a cache. 1517 */ 1518 static void 1519 kmem_cache_reap(kmem_cache_t *cp) 1520 { 1521 /* 1522 * Ask the cache's owner to free some memory if possible. 1523 * The idea is to handle things like the inode cache, which 1524 * typically sits on a bunch of memory that it doesn't truly 1525 * *need*. Reclaim policy is entirely up to the owner; this 1526 * callback is just an advisory plea for help. 1527 */ 1528 if (cp->cache_reclaim != NULL) 1529 cp->cache_reclaim(cp->cache_private); 1530 1531 kmem_depot_ws_reap(cp); 1532 } 1533 1534 static void 1535 kmem_reap_timeout(void *flag_arg) 1536 { 1537 uint32_t *flag = (uint32_t *)flag_arg; 1538 1539 ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 1540 *flag = 0; 1541 } 1542 1543 static void 1544 kmem_reap_done(void *flag) 1545 { 1546 (void) timeout(kmem_reap_timeout, flag, kmem_reap_interval); 1547 } 1548 1549 static void 1550 kmem_reap_start(void *flag) 1551 { 1552 ASSERT(flag == &kmem_reaping || flag == &kmem_reaping_idspace); 1553 1554 if (flag == &kmem_reaping) { 1555 kmem_cache_applyall(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 1556 /* 1557 * if we have segkp under heap, reap segkp cache. 1558 */ 1559 if (segkp_fromheap) 1560 segkp_cache_free(); 1561 } 1562 else 1563 kmem_cache_applyall_id(kmem_cache_reap, kmem_taskq, TQ_NOSLEEP); 1564 1565 /* 1566 * We use taskq_dispatch() to schedule a timeout to clear 1567 * the flag so that kmem_reap() becomes self-throttling: 1568 * we won't reap again until the current reap completes *and* 1569 * at least kmem_reap_interval ticks have elapsed. 1570 */ 1571 if (!taskq_dispatch(kmem_taskq, kmem_reap_done, flag, TQ_NOSLEEP)) 1572 kmem_reap_done(flag); 1573 } 1574 1575 static void 1576 kmem_reap_common(void *flag_arg) 1577 { 1578 uint32_t *flag = (uint32_t *)flag_arg; 1579 1580 if (MUTEX_HELD(&kmem_cache_lock) || kmem_taskq == NULL || 1581 cas32(flag, 0, 1) != 0) 1582 return; 1583 1584 /* 1585 * It may not be kosher to do memory allocation when a reap is called 1586 * is called (for example, if vmem_populate() is in the call chain). 1587 * So we start the reap going with a TQ_NOALLOC dispatch. If the 1588 * dispatch fails, we reset the flag, and the next reap will try again. 1589 */ 1590 if (!taskq_dispatch(kmem_taskq, kmem_reap_start, flag, TQ_NOALLOC)) 1591 *flag = 0; 1592 } 1593 1594 /* 1595 * Reclaim all unused memory from all caches. Called from the VM system 1596 * when memory gets tight. 1597 */ 1598 void 1599 kmem_reap(void) 1600 { 1601 kmem_reap_common(&kmem_reaping); 1602 } 1603 1604 /* 1605 * Reclaim all unused memory from identifier arenas, called when a vmem 1606 * arena not back by memory is exhausted. Since reaping memory-backed caches 1607 * cannot help with identifier exhaustion, we avoid both a large amount of 1608 * work and unwanted side-effects from reclaim callbacks. 1609 */ 1610 void 1611 kmem_reap_idspace(void) 1612 { 1613 kmem_reap_common(&kmem_reaping_idspace); 1614 } 1615 1616 /* 1617 * Purge all magazines from a cache and set its magazine limit to zero. 1618 * All calls are serialized by the kmem_taskq lock, except for the final 1619 * call from kmem_cache_destroy(). 1620 */ 1621 static void 1622 kmem_cache_magazine_purge(kmem_cache_t *cp) 1623 { 1624 kmem_cpu_cache_t *ccp; 1625 kmem_magazine_t *mp, *pmp; 1626 int rounds, prounds, cpu_seqid; 1627 1628 ASSERT(cp->cache_next == NULL || taskq_member(kmem_taskq, curthread)); 1629 ASSERT(MUTEX_NOT_HELD(&cp->cache_lock)); 1630 1631 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 1632 ccp = &cp->cache_cpu[cpu_seqid]; 1633 1634 mutex_enter(&ccp->cc_lock); 1635 mp = ccp->cc_loaded; 1636 pmp = ccp->cc_ploaded; 1637 rounds = ccp->cc_rounds; 1638 prounds = ccp->cc_prounds; 1639 ccp->cc_loaded = NULL; 1640 ccp->cc_ploaded = NULL; 1641 ccp->cc_rounds = -1; 1642 ccp->cc_prounds = -1; 1643 ccp->cc_magsize = 0; 1644 mutex_exit(&ccp->cc_lock); 1645 1646 if (mp) 1647 kmem_magazine_destroy(cp, mp, rounds); 1648 if (pmp) 1649 kmem_magazine_destroy(cp, pmp, prounds); 1650 } 1651 1652 /* 1653 * Updating the working set statistics twice in a row has the 1654 * effect of setting the working set size to zero, so everything 1655 * is eligible for reaping. 1656 */ 1657 kmem_depot_ws_update(cp); 1658 kmem_depot_ws_update(cp); 1659 1660 kmem_depot_ws_reap(cp); 1661 } 1662 1663 /* 1664 * Enable per-cpu magazines on a cache. 1665 */ 1666 static void 1667 kmem_cache_magazine_enable(kmem_cache_t *cp) 1668 { 1669 int cpu_seqid; 1670 1671 if (cp->cache_flags & KMF_NOMAGAZINE) 1672 return; 1673 1674 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 1675 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 1676 mutex_enter(&ccp->cc_lock); 1677 ccp->cc_magsize = cp->cache_magtype->mt_magsize; 1678 mutex_exit(&ccp->cc_lock); 1679 } 1680 1681 } 1682 1683 /* 1684 * Reap (almost) everything right now. See kmem_cache_magazine_purge() 1685 * for explanation of the back-to-back kmem_depot_ws_update() calls. 1686 */ 1687 void 1688 kmem_cache_reap_now(kmem_cache_t *cp) 1689 { 1690 kmem_depot_ws_update(cp); 1691 kmem_depot_ws_update(cp); 1692 1693 (void) taskq_dispatch(kmem_taskq, 1694 (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP); 1695 taskq_wait(kmem_taskq); 1696 } 1697 1698 /* 1699 * Recompute a cache's magazine size. The trade-off is that larger magazines 1700 * provide a higher transfer rate with the depot, while smaller magazines 1701 * reduce memory consumption. Magazine resizing is an expensive operation; 1702 * it should not be done frequently. 1703 * 1704 * Changes to the magazine size are serialized by the kmem_taskq lock. 1705 * 1706 * Note: at present this only grows the magazine size. It might be useful 1707 * to allow shrinkage too. 1708 */ 1709 static void 1710 kmem_cache_magazine_resize(kmem_cache_t *cp) 1711 { 1712 kmem_magtype_t *mtp = cp->cache_magtype; 1713 1714 ASSERT(taskq_member(kmem_taskq, curthread)); 1715 1716 if (cp->cache_chunksize < mtp->mt_maxbuf) { 1717 kmem_cache_magazine_purge(cp); 1718 mutex_enter(&cp->cache_depot_lock); 1719 cp->cache_magtype = ++mtp; 1720 cp->cache_depot_contention_prev = 1721 cp->cache_depot_contention + INT_MAX; 1722 mutex_exit(&cp->cache_depot_lock); 1723 kmem_cache_magazine_enable(cp); 1724 } 1725 } 1726 1727 /* 1728 * Rescale a cache's hash table, so that the table size is roughly the 1729 * cache size. We want the average lookup time to be extremely small. 1730 */ 1731 static void 1732 kmem_hash_rescale(kmem_cache_t *cp) 1733 { 1734 kmem_bufctl_t **old_table, **new_table, *bcp; 1735 size_t old_size, new_size, h; 1736 1737 ASSERT(taskq_member(kmem_taskq, curthread)); 1738 1739 new_size = MAX(KMEM_HASH_INITIAL, 1740 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); 1741 old_size = cp->cache_hash_mask + 1; 1742 1743 if ((old_size >> 1) <= new_size && new_size <= (old_size << 1)) 1744 return; 1745 1746 new_table = vmem_alloc(kmem_hash_arena, new_size * sizeof (void *), 1747 VM_NOSLEEP); 1748 if (new_table == NULL) 1749 return; 1750 bzero(new_table, new_size * sizeof (void *)); 1751 1752 mutex_enter(&cp->cache_lock); 1753 1754 old_size = cp->cache_hash_mask + 1; 1755 old_table = cp->cache_hash_table; 1756 1757 cp->cache_hash_mask = new_size - 1; 1758 cp->cache_hash_table = new_table; 1759 cp->cache_rescale++; 1760 1761 for (h = 0; h < old_size; h++) { 1762 bcp = old_table[h]; 1763 while (bcp != NULL) { 1764 void *addr = bcp->bc_addr; 1765 kmem_bufctl_t *next_bcp = bcp->bc_next; 1766 kmem_bufctl_t **hash_bucket = KMEM_HASH(cp, addr); 1767 bcp->bc_next = *hash_bucket; 1768 *hash_bucket = bcp; 1769 bcp = next_bcp; 1770 } 1771 } 1772 1773 mutex_exit(&cp->cache_lock); 1774 1775 vmem_free(kmem_hash_arena, old_table, old_size * sizeof (void *)); 1776 } 1777 1778 /* 1779 * Perform periodic maintenance on a cache: hash rescaling, 1780 * depot working-set update, and magazine resizing. 1781 */ 1782 static void 1783 kmem_cache_update(kmem_cache_t *cp) 1784 { 1785 int need_hash_rescale = 0; 1786 int need_magazine_resize = 0; 1787 1788 ASSERT(MUTEX_HELD(&kmem_cache_lock)); 1789 1790 /* 1791 * If the cache has become much larger or smaller than its hash table, 1792 * fire off a request to rescale the hash table. 1793 */ 1794 mutex_enter(&cp->cache_lock); 1795 1796 if ((cp->cache_flags & KMF_HASH) && 1797 (cp->cache_buftotal > (cp->cache_hash_mask << 1) || 1798 (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && 1799 cp->cache_hash_mask > KMEM_HASH_INITIAL))) 1800 need_hash_rescale = 1; 1801 1802 mutex_exit(&cp->cache_lock); 1803 1804 /* 1805 * Update the depot working set statistics. 1806 */ 1807 kmem_depot_ws_update(cp); 1808 1809 /* 1810 * If there's a lot of contention in the depot, 1811 * increase the magazine size. 1812 */ 1813 mutex_enter(&cp->cache_depot_lock); 1814 1815 if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && 1816 (int)(cp->cache_depot_contention - 1817 cp->cache_depot_contention_prev) > kmem_depot_contention) 1818 need_magazine_resize = 1; 1819 1820 cp->cache_depot_contention_prev = cp->cache_depot_contention; 1821 1822 mutex_exit(&cp->cache_depot_lock); 1823 1824 if (need_hash_rescale) 1825 (void) taskq_dispatch(kmem_taskq, 1826 (task_func_t *)kmem_hash_rescale, cp, TQ_NOSLEEP); 1827 1828 if (need_magazine_resize) 1829 (void) taskq_dispatch(kmem_taskq, 1830 (task_func_t *)kmem_cache_magazine_resize, cp, TQ_NOSLEEP); 1831 } 1832 1833 static void 1834 kmem_update_timeout(void *dummy) 1835 { 1836 static void kmem_update(void *); 1837 1838 (void) timeout(kmem_update, dummy, kmem_reap_interval); 1839 } 1840 1841 static void 1842 kmem_update(void *dummy) 1843 { 1844 kmem_cache_applyall(kmem_cache_update, NULL, TQ_NOSLEEP); 1845 1846 /* 1847 * We use taskq_dispatch() to reschedule the timeout so that 1848 * kmem_update() becomes self-throttling: it won't schedule 1849 * new tasks until all previous tasks have completed. 1850 */ 1851 if (!taskq_dispatch(kmem_taskq, kmem_update_timeout, dummy, TQ_NOSLEEP)) 1852 kmem_update_timeout(NULL); 1853 } 1854 1855 static int 1856 kmem_cache_kstat_update(kstat_t *ksp, int rw) 1857 { 1858 struct kmem_cache_kstat *kmcp = &kmem_cache_kstat; 1859 kmem_cache_t *cp = ksp->ks_private; 1860 kmem_slab_t *sp; 1861 uint64_t cpu_buf_avail; 1862 uint64_t buf_avail = 0; 1863 int cpu_seqid; 1864 1865 ASSERT(MUTEX_HELD(&kmem_cache_kstat_lock)); 1866 1867 if (rw == KSTAT_WRITE) 1868 return (EACCES); 1869 1870 mutex_enter(&cp->cache_lock); 1871 1872 kmcp->kmc_alloc_fail.value.ui64 = cp->cache_alloc_fail; 1873 kmcp->kmc_alloc.value.ui64 = cp->cache_slab_alloc; 1874 kmcp->kmc_free.value.ui64 = cp->cache_slab_free; 1875 kmcp->kmc_slab_alloc.value.ui64 = cp->cache_slab_alloc; 1876 kmcp->kmc_slab_free.value.ui64 = cp->cache_slab_free; 1877 1878 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 1879 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 1880 1881 mutex_enter(&ccp->cc_lock); 1882 1883 cpu_buf_avail = 0; 1884 if (ccp->cc_rounds > 0) 1885 cpu_buf_avail += ccp->cc_rounds; 1886 if (ccp->cc_prounds > 0) 1887 cpu_buf_avail += ccp->cc_prounds; 1888 1889 kmcp->kmc_alloc.value.ui64 += ccp->cc_alloc; 1890 kmcp->kmc_free.value.ui64 += ccp->cc_free; 1891 buf_avail += cpu_buf_avail; 1892 1893 mutex_exit(&ccp->cc_lock); 1894 } 1895 1896 mutex_enter(&cp->cache_depot_lock); 1897 1898 kmcp->kmc_depot_alloc.value.ui64 = cp->cache_full.ml_alloc; 1899 kmcp->kmc_depot_free.value.ui64 = cp->cache_empty.ml_alloc; 1900 kmcp->kmc_depot_contention.value.ui64 = cp->cache_depot_contention; 1901 kmcp->kmc_full_magazines.value.ui64 = cp->cache_full.ml_total; 1902 kmcp->kmc_empty_magazines.value.ui64 = cp->cache_empty.ml_total; 1903 kmcp->kmc_magazine_size.value.ui64 = 1904 (cp->cache_flags & KMF_NOMAGAZINE) ? 1905 0 : cp->cache_magtype->mt_magsize; 1906 1907 kmcp->kmc_alloc.value.ui64 += cp->cache_full.ml_alloc; 1908 kmcp->kmc_free.value.ui64 += cp->cache_empty.ml_alloc; 1909 buf_avail += cp->cache_full.ml_total * cp->cache_magtype->mt_magsize; 1910 1911 mutex_exit(&cp->cache_depot_lock); 1912 1913 kmcp->kmc_buf_size.value.ui64 = cp->cache_bufsize; 1914 kmcp->kmc_align.value.ui64 = cp->cache_align; 1915 kmcp->kmc_chunk_size.value.ui64 = cp->cache_chunksize; 1916 kmcp->kmc_slab_size.value.ui64 = cp->cache_slabsize; 1917 kmcp->kmc_buf_constructed.value.ui64 = buf_avail; 1918 for (sp = cp->cache_freelist; sp != &cp->cache_nullslab; 1919 sp = sp->slab_next) 1920 buf_avail += sp->slab_chunks - sp->slab_refcnt; 1921 kmcp->kmc_buf_avail.value.ui64 = buf_avail; 1922 kmcp->kmc_buf_inuse.value.ui64 = cp->cache_buftotal - buf_avail; 1923 kmcp->kmc_buf_total.value.ui64 = cp->cache_buftotal; 1924 kmcp->kmc_buf_max.value.ui64 = cp->cache_bufmax; 1925 kmcp->kmc_slab_create.value.ui64 = cp->cache_slab_create; 1926 kmcp->kmc_slab_destroy.value.ui64 = cp->cache_slab_destroy; 1927 kmcp->kmc_hash_size.value.ui64 = (cp->cache_flags & KMF_HASH) ? 1928 cp->cache_hash_mask + 1 : 0; 1929 kmcp->kmc_hash_lookup_depth.value.ui64 = cp->cache_lookup_depth; 1930 kmcp->kmc_hash_rescale.value.ui64 = cp->cache_rescale; 1931 kmcp->kmc_vmem_source.value.ui64 = cp->cache_arena->vm_id; 1932 1933 mutex_exit(&cp->cache_lock); 1934 return (0); 1935 } 1936 1937 /* 1938 * Return a named statistic about a particular cache. 1939 * This shouldn't be called very often, so it's currently designed for 1940 * simplicity (leverages existing kstat support) rather than efficiency. 1941 */ 1942 uint64_t 1943 kmem_cache_stat(kmem_cache_t *cp, char *name) 1944 { 1945 int i; 1946 kstat_t *ksp = cp->cache_kstat; 1947 kstat_named_t *knp = (kstat_named_t *)&kmem_cache_kstat; 1948 uint64_t value = 0; 1949 1950 if (ksp != NULL) { 1951 mutex_enter(&kmem_cache_kstat_lock); 1952 (void) kmem_cache_kstat_update(ksp, KSTAT_READ); 1953 for (i = 0; i < ksp->ks_ndata; i++) { 1954 if (strcmp(knp[i].name, name) == 0) { 1955 value = knp[i].value.ui64; 1956 break; 1957 } 1958 } 1959 mutex_exit(&kmem_cache_kstat_lock); 1960 } 1961 return (value); 1962 } 1963 1964 /* 1965 * Return an estimate of currently available kernel heap memory. 1966 * On 32-bit systems, physical memory may exceed virtual memory, 1967 * we just truncate the result at 1GB. 1968 */ 1969 size_t 1970 kmem_avail(void) 1971 { 1972 spgcnt_t rmem = availrmem - tune.t_minarmem; 1973 spgcnt_t fmem = freemem - minfree; 1974 1975 return ((size_t)ptob(MIN(MAX(MIN(rmem, fmem), 0), 1976 1 << (30 - PAGESHIFT)))); 1977 } 1978 1979 /* 1980 * Return the maximum amount of memory that is (in theory) allocatable 1981 * from the heap. This may be used as an estimate only since there 1982 * is no guarentee this space will still be available when an allocation 1983 * request is made, nor that the space may be allocated in one big request 1984 * due to kernel heap fragmentation. 1985 */ 1986 size_t 1987 kmem_maxavail(void) 1988 { 1989 spgcnt_t pmem = availrmem - tune.t_minarmem; 1990 spgcnt_t vmem = btop(vmem_size(heap_arena, VMEM_FREE)); 1991 1992 return ((size_t)ptob(MAX(MIN(pmem, vmem), 0))); 1993 } 1994 1995 /* 1996 * Indicate whether memory-intensive kmem debugging is enabled. 1997 */ 1998 int 1999 kmem_debugging(void) 2000 { 2001 return (kmem_flags & (KMF_AUDIT | KMF_REDZONE)); 2002 } 2003 2004 kmem_cache_t * 2005 kmem_cache_create( 2006 char *name, /* descriptive name for this cache */ 2007 size_t bufsize, /* size of the objects it manages */ 2008 size_t align, /* required object alignment */ 2009 int (*constructor)(void *, void *, int), /* object constructor */ 2010 void (*destructor)(void *, void *), /* object destructor */ 2011 void (*reclaim)(void *), /* memory reclaim callback */ 2012 void *private, /* pass-thru arg for constr/destr/reclaim */ 2013 vmem_t *vmp, /* vmem source for slab allocation */ 2014 int cflags) /* cache creation flags */ 2015 { 2016 int cpu_seqid; 2017 size_t chunksize; 2018 kmem_cache_t *cp, *cnext, *cprev; 2019 kmem_magtype_t *mtp; 2020 size_t csize = KMEM_CACHE_SIZE(max_ncpus); 2021 2022 #ifdef DEBUG 2023 /* 2024 * Cache names should conform to the rules for valid C identifiers 2025 */ 2026 if (!strident_valid(name)) { 2027 cmn_err(CE_CONT, 2028 "kmem_cache_create: '%s' is an invalid cache name\n" 2029 "cache names must conform to the rules for " 2030 "C identifiers\n", name); 2031 } 2032 #endif /* DEBUG */ 2033 2034 if (vmp == NULL) 2035 vmp = kmem_default_arena; 2036 2037 /* 2038 * If this kmem cache has an identifier vmem arena as its source, mark 2039 * it such to allow kmem_reap_idspace(). 2040 */ 2041 ASSERT(!(cflags & KMC_IDENTIFIER)); /* consumer should not set this */ 2042 if (vmp->vm_cflags & VMC_IDENTIFIER) 2043 cflags |= KMC_IDENTIFIER; 2044 2045 /* 2046 * Get a kmem_cache structure. We arrange that cp->cache_cpu[] 2047 * is aligned on a KMEM_CPU_CACHE_SIZE boundary to prevent 2048 * false sharing of per-CPU data. 2049 */ 2050 cp = vmem_xalloc(kmem_cache_arena, csize, KMEM_CPU_CACHE_SIZE, 2051 P2NPHASE(csize, KMEM_CPU_CACHE_SIZE), 0, NULL, NULL, VM_SLEEP); 2052 bzero(cp, csize); 2053 2054 if (align == 0) 2055 align = KMEM_ALIGN; 2056 2057 /* 2058 * If we're not at least KMEM_ALIGN aligned, we can't use free 2059 * memory to hold bufctl information (because we can't safely 2060 * perform word loads and stores on it). 2061 */ 2062 if (align < KMEM_ALIGN) 2063 cflags |= KMC_NOTOUCH; 2064 2065 if ((align & (align - 1)) != 0 || align > vmp->vm_quantum) 2066 panic("kmem_cache_create: bad alignment %lu", align); 2067 2068 mutex_enter(&kmem_flags_lock); 2069 if (kmem_flags & KMF_RANDOMIZE) 2070 kmem_flags = (((kmem_flags | ~KMF_RANDOM) + 1) & KMF_RANDOM) | 2071 KMF_RANDOMIZE; 2072 cp->cache_flags = (kmem_flags | cflags) & KMF_DEBUG; 2073 mutex_exit(&kmem_flags_lock); 2074 2075 /* 2076 * Make sure all the various flags are reasonable. 2077 */ 2078 ASSERT(!(cflags & KMC_NOHASH) || !(cflags & KMC_NOTOUCH)); 2079 2080 if (cp->cache_flags & KMF_LITE) { 2081 if (bufsize >= kmem_lite_minsize && 2082 align <= kmem_lite_maxalign && 2083 P2PHASE(bufsize, kmem_lite_maxalign) != 0) { 2084 cp->cache_flags |= KMF_BUFTAG; 2085 cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 2086 } else { 2087 cp->cache_flags &= ~KMF_DEBUG; 2088 } 2089 } 2090 2091 if (cp->cache_flags & KMF_DEADBEEF) 2092 cp->cache_flags |= KMF_REDZONE; 2093 2094 if ((cflags & KMC_QCACHE) && (cp->cache_flags & KMF_AUDIT)) 2095 cp->cache_flags |= KMF_NOMAGAZINE; 2096 2097 if (cflags & KMC_NODEBUG) 2098 cp->cache_flags &= ~KMF_DEBUG; 2099 2100 if (cflags & KMC_NOTOUCH) 2101 cp->cache_flags &= ~KMF_TOUCH; 2102 2103 if (cflags & KMC_NOHASH) 2104 cp->cache_flags &= ~(KMF_AUDIT | KMF_FIREWALL); 2105 2106 if (cflags & KMC_NOMAGAZINE) 2107 cp->cache_flags |= KMF_NOMAGAZINE; 2108 2109 if ((cp->cache_flags & KMF_AUDIT) && !(cflags & KMC_NOTOUCH)) 2110 cp->cache_flags |= KMF_REDZONE; 2111 2112 if (!(cp->cache_flags & KMF_AUDIT)) 2113 cp->cache_flags &= ~KMF_CONTENTS; 2114 2115 if ((cp->cache_flags & KMF_BUFTAG) && bufsize >= kmem_minfirewall && 2116 !(cp->cache_flags & KMF_LITE) && !(cflags & KMC_NOHASH)) 2117 cp->cache_flags |= KMF_FIREWALL; 2118 2119 if (vmp != kmem_default_arena || kmem_firewall_arena == NULL) 2120 cp->cache_flags &= ~KMF_FIREWALL; 2121 2122 if (cp->cache_flags & KMF_FIREWALL) { 2123 cp->cache_flags &= ~KMF_BUFTAG; 2124 cp->cache_flags |= KMF_NOMAGAZINE; 2125 ASSERT(vmp == kmem_default_arena); 2126 vmp = kmem_firewall_arena; 2127 } 2128 2129 /* 2130 * Set cache properties. 2131 */ 2132 (void) strncpy(cp->cache_name, name, KMEM_CACHE_NAMELEN); 2133 strident_canon(cp->cache_name, KMEM_CACHE_NAMELEN); 2134 cp->cache_bufsize = bufsize; 2135 cp->cache_align = align; 2136 cp->cache_constructor = constructor; 2137 cp->cache_destructor = destructor; 2138 cp->cache_reclaim = reclaim; 2139 cp->cache_private = private; 2140 cp->cache_arena = vmp; 2141 cp->cache_cflags = cflags; 2142 2143 /* 2144 * Determine the chunk size. 2145 */ 2146 chunksize = bufsize; 2147 2148 if (align >= KMEM_ALIGN) { 2149 chunksize = P2ROUNDUP(chunksize, KMEM_ALIGN); 2150 cp->cache_bufctl = chunksize - KMEM_ALIGN; 2151 } 2152 2153 if (cp->cache_flags & KMF_BUFTAG) { 2154 cp->cache_bufctl = chunksize; 2155 cp->cache_buftag = chunksize; 2156 if (cp->cache_flags & KMF_LITE) 2157 chunksize += KMEM_BUFTAG_LITE_SIZE(kmem_lite_count); 2158 else 2159 chunksize += sizeof (kmem_buftag_t); 2160 } 2161 2162 if (cp->cache_flags & KMF_DEADBEEF) { 2163 cp->cache_verify = MIN(cp->cache_buftag, kmem_maxverify); 2164 if (cp->cache_flags & KMF_LITE) 2165 cp->cache_verify = sizeof (uint64_t); 2166 } 2167 2168 cp->cache_contents = MIN(cp->cache_bufctl, kmem_content_maxsave); 2169 2170 cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); 2171 2172 /* 2173 * Now that we know the chunk size, determine the optimal slab size. 2174 */ 2175 if (vmp == kmem_firewall_arena) { 2176 cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); 2177 cp->cache_mincolor = cp->cache_slabsize - chunksize; 2178 cp->cache_maxcolor = cp->cache_mincolor; 2179 cp->cache_flags |= KMF_HASH; 2180 ASSERT(!(cp->cache_flags & KMF_BUFTAG)); 2181 } else if ((cflags & KMC_NOHASH) || (!(cflags & KMC_NOTOUCH) && 2182 !(cp->cache_flags & KMF_AUDIT) && 2183 chunksize < vmp->vm_quantum / KMEM_VOID_FRACTION)) { 2184 cp->cache_slabsize = vmp->vm_quantum; 2185 cp->cache_mincolor = 0; 2186 cp->cache_maxcolor = 2187 (cp->cache_slabsize - sizeof (kmem_slab_t)) % chunksize; 2188 ASSERT(chunksize + sizeof (kmem_slab_t) <= cp->cache_slabsize); 2189 ASSERT(!(cp->cache_flags & KMF_AUDIT)); 2190 } else { 2191 size_t chunks, bestfit, waste, slabsize; 2192 size_t minwaste = LONG_MAX; 2193 2194 for (chunks = 1; chunks <= KMEM_VOID_FRACTION; chunks++) { 2195 slabsize = P2ROUNDUP(chunksize * chunks, 2196 vmp->vm_quantum); 2197 chunks = slabsize / chunksize; 2198 waste = (slabsize % chunksize) / chunks; 2199 if (waste < minwaste) { 2200 minwaste = waste; 2201 bestfit = slabsize; 2202 } 2203 } 2204 if (cflags & KMC_QCACHE) 2205 bestfit = VMEM_QCACHE_SLABSIZE(vmp->vm_qcache_max); 2206 cp->cache_slabsize = bestfit; 2207 cp->cache_mincolor = 0; 2208 cp->cache_maxcolor = bestfit % chunksize; 2209 cp->cache_flags |= KMF_HASH; 2210 } 2211 2212 if (cp->cache_flags & KMF_HASH) { 2213 ASSERT(!(cflags & KMC_NOHASH)); 2214 cp->cache_bufctl_cache = (cp->cache_flags & KMF_AUDIT) ? 2215 kmem_bufctl_audit_cache : kmem_bufctl_cache; 2216 } 2217 2218 if (cp->cache_maxcolor >= vmp->vm_quantum) 2219 cp->cache_maxcolor = vmp->vm_quantum - 1; 2220 2221 cp->cache_color = cp->cache_mincolor; 2222 2223 /* 2224 * Initialize the rest of the slab layer. 2225 */ 2226 mutex_init(&cp->cache_lock, NULL, MUTEX_DEFAULT, NULL); 2227 2228 cp->cache_freelist = &cp->cache_nullslab; 2229 cp->cache_nullslab.slab_cache = cp; 2230 cp->cache_nullslab.slab_refcnt = -1; 2231 cp->cache_nullslab.slab_next = &cp->cache_nullslab; 2232 cp->cache_nullslab.slab_prev = &cp->cache_nullslab; 2233 2234 if (cp->cache_flags & KMF_HASH) { 2235 cp->cache_hash_table = vmem_alloc(kmem_hash_arena, 2236 KMEM_HASH_INITIAL * sizeof (void *), VM_SLEEP); 2237 bzero(cp->cache_hash_table, 2238 KMEM_HASH_INITIAL * sizeof (void *)); 2239 cp->cache_hash_mask = KMEM_HASH_INITIAL - 1; 2240 cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; 2241 } 2242 2243 /* 2244 * Initialize the depot. 2245 */ 2246 mutex_init(&cp->cache_depot_lock, NULL, MUTEX_DEFAULT, NULL); 2247 2248 for (mtp = kmem_magtype; chunksize <= mtp->mt_minbuf; mtp++) 2249 continue; 2250 2251 cp->cache_magtype = mtp; 2252 2253 /* 2254 * Initialize the CPU layer. 2255 */ 2256 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) { 2257 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; 2258 mutex_init(&ccp->cc_lock, NULL, MUTEX_DEFAULT, NULL); 2259 ccp->cc_flags = cp->cache_flags; 2260 ccp->cc_rounds = -1; 2261 ccp->cc_prounds = -1; 2262 } 2263 2264 /* 2265 * Create the cache's kstats. 2266 */ 2267 if ((cp->cache_kstat = kstat_create("unix", 0, cp->cache_name, 2268 "kmem_cache", KSTAT_TYPE_NAMED, 2269 sizeof (kmem_cache_kstat) / sizeof (kstat_named_t), 2270 KSTAT_FLAG_VIRTUAL)) != NULL) { 2271 cp->cache_kstat->ks_data = &kmem_cache_kstat; 2272 cp->cache_kstat->ks_update = kmem_cache_kstat_update; 2273 cp->cache_kstat->ks_private = cp; 2274 cp->cache_kstat->ks_lock = &kmem_cache_kstat_lock; 2275 kstat_install(cp->cache_kstat); 2276 } 2277 2278 /* 2279 * Add the cache to the global list. This makes it visible 2280 * to kmem_update(), so the cache must be ready for business. 2281 */ 2282 mutex_enter(&kmem_cache_lock); 2283 cp->cache_next = cnext = &kmem_null_cache; 2284 cp->cache_prev = cprev = kmem_null_cache.cache_prev; 2285 cnext->cache_prev = cp; 2286 cprev->cache_next = cp; 2287 mutex_exit(&kmem_cache_lock); 2288 2289 if (kmem_ready) 2290 kmem_cache_magazine_enable(cp); 2291 2292 return (cp); 2293 } 2294 2295 void 2296 kmem_cache_destroy(kmem_cache_t *cp) 2297 { 2298 int cpu_seqid; 2299 2300 /* 2301 * Remove the cache from the global cache list so that no one else 2302 * can schedule tasks on its behalf, wait for any pending tasks to 2303 * complete, purge the cache, and then destroy it. 2304 */ 2305 mutex_enter(&kmem_cache_lock); 2306 cp->cache_prev->cache_next = cp->cache_next; 2307 cp->cache_next->cache_prev = cp->cache_prev; 2308 cp->cache_prev = cp->cache_next = NULL; 2309 mutex_exit(&kmem_cache_lock); 2310 2311 if (kmem_taskq != NULL) 2312 taskq_wait(kmem_taskq); 2313 2314 kmem_cache_magazine_purge(cp); 2315 2316 mutex_enter(&cp->cache_lock); 2317 if (cp->cache_buftotal != 0) 2318 cmn_err(CE_WARN, "kmem_cache_destroy: '%s' (%p) not empty", 2319 cp->cache_name, (void *)cp); 2320 cp->cache_reclaim = NULL; 2321 /* 2322 * The cache is now dead. There should be no further activity. 2323 * We enforce this by setting land mines in the constructor and 2324 * destructor routines that induce a kernel text fault if invoked. 2325 */ 2326 cp->cache_constructor = (int (*)(void *, void *, int))1; 2327 cp->cache_destructor = (void (*)(void *, void *))2; 2328 mutex_exit(&cp->cache_lock); 2329 2330 kstat_delete(cp->cache_kstat); 2331 2332 if (cp->cache_hash_table != NULL) 2333 vmem_free(kmem_hash_arena, cp->cache_hash_table, 2334 (cp->cache_hash_mask + 1) * sizeof (void *)); 2335 2336 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) 2337 mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); 2338 2339 mutex_destroy(&cp->cache_depot_lock); 2340 mutex_destroy(&cp->cache_lock); 2341 2342 vmem_free(kmem_cache_arena, cp, KMEM_CACHE_SIZE(max_ncpus)); 2343 } 2344 2345 /*ARGSUSED*/ 2346 static int 2347 kmem_cpu_setup(cpu_setup_t what, int id, void *arg) 2348 { 2349 ASSERT(MUTEX_HELD(&cpu_lock)); 2350 if (what == CPU_UNCONFIG) { 2351 kmem_cache_applyall(kmem_cache_magazine_purge, 2352 kmem_taskq, TQ_SLEEP); 2353 kmem_cache_applyall(kmem_cache_magazine_enable, 2354 kmem_taskq, TQ_SLEEP); 2355 } 2356 return (0); 2357 } 2358 2359 static void 2360 kmem_cache_init(int pass, int use_large_pages) 2361 { 2362 int i; 2363 size_t size; 2364 kmem_cache_t *cp; 2365 kmem_magtype_t *mtp; 2366 char name[KMEM_CACHE_NAMELEN + 1]; 2367 2368 for (i = 0; i < sizeof (kmem_magtype) / sizeof (*mtp); i++) { 2369 mtp = &kmem_magtype[i]; 2370 (void) sprintf(name, "kmem_magazine_%d", mtp->mt_magsize); 2371 mtp->mt_cache = kmem_cache_create(name, 2372 (mtp->mt_magsize + 1) * sizeof (void *), 2373 mtp->mt_align, NULL, NULL, NULL, NULL, 2374 kmem_msb_arena, KMC_NOHASH); 2375 } 2376 2377 kmem_slab_cache = kmem_cache_create("kmem_slab_cache", 2378 sizeof (kmem_slab_t), 0, NULL, NULL, NULL, NULL, 2379 kmem_msb_arena, KMC_NOHASH); 2380 2381 kmem_bufctl_cache = kmem_cache_create("kmem_bufctl_cache", 2382 sizeof (kmem_bufctl_t), 0, NULL, NULL, NULL, NULL, 2383 kmem_msb_arena, KMC_NOHASH); 2384 2385 kmem_bufctl_audit_cache = kmem_cache_create("kmem_bufctl_audit_cache", 2386 sizeof (kmem_bufctl_audit_t), 0, NULL, NULL, NULL, NULL, 2387 kmem_msb_arena, KMC_NOHASH); 2388 2389 if (pass == 2) { 2390 kmem_va_arena = vmem_create("kmem_va", 2391 NULL, 0, PAGESIZE, 2392 vmem_alloc, vmem_free, heap_arena, 2393 8 * PAGESIZE, VM_SLEEP); 2394 2395 if (use_large_pages) { 2396 kmem_default_arena = vmem_xcreate("kmem_default", 2397 NULL, 0, PAGESIZE, 2398 segkmem_alloc_lp, segkmem_free_lp, kmem_va_arena, 2399 0, VM_SLEEP); 2400 } else { 2401 kmem_default_arena = vmem_create("kmem_default", 2402 NULL, 0, PAGESIZE, 2403 segkmem_alloc, segkmem_free, kmem_va_arena, 2404 0, VM_SLEEP); 2405 } 2406 } else { 2407 /* 2408 * During the first pass, the kmem_alloc_* caches 2409 * are treated as metadata. 2410 */ 2411 kmem_default_arena = kmem_msb_arena; 2412 } 2413 2414 /* 2415 * Set up the default caches to back kmem_alloc() 2416 */ 2417 size = KMEM_ALIGN; 2418 for (i = 0; i < sizeof (kmem_alloc_sizes) / sizeof (int); i++) { 2419 size_t align = KMEM_ALIGN; 2420 size_t cache_size = kmem_alloc_sizes[i]; 2421 /* 2422 * If they allocate a multiple of the coherency granularity, 2423 * they get a coherency-granularity-aligned address. 2424 */ 2425 if (IS_P2ALIGNED(cache_size, 64)) 2426 align = 64; 2427 if (IS_P2ALIGNED(cache_size, PAGESIZE)) 2428 align = PAGESIZE; 2429 (void) sprintf(name, "kmem_alloc_%lu", cache_size); 2430 cp = kmem_cache_create(name, cache_size, align, 2431 NULL, NULL, NULL, NULL, NULL, KMC_KMEM_ALLOC); 2432 while (size <= cache_size) { 2433 kmem_alloc_table[(size - 1) >> KMEM_ALIGN_SHIFT] = cp; 2434 size += KMEM_ALIGN; 2435 } 2436 } 2437 } 2438 2439 void 2440 kmem_init(void) 2441 { 2442 kmem_cache_t *cp; 2443 int old_kmem_flags = kmem_flags; 2444 int use_large_pages = 0; 2445 size_t maxverify, minfirewall; 2446 2447 kstat_init(); 2448 2449 /* 2450 * Small-memory systems (< 24 MB) can't handle kmem_flags overhead. 2451 */ 2452 if (physmem < btop(24 << 20) && !(old_kmem_flags & KMF_STICKY)) 2453 kmem_flags = 0; 2454 2455 /* 2456 * Don't do firewalled allocations if the heap is less than 1TB 2457 * (i.e. on a 32-bit kernel) 2458 * The resulting VM_NEXTFIT allocations would create too much 2459 * fragmentation in a small heap. 2460 */ 2461 #if defined(_LP64) 2462 maxverify = minfirewall = PAGESIZE / 2; 2463 #else 2464 maxverify = minfirewall = ULONG_MAX; 2465 #endif 2466 2467 /* LINTED */ 2468 ASSERT(sizeof (kmem_cpu_cache_t) == KMEM_CPU_CACHE_SIZE); 2469 2470 kmem_null_cache.cache_next = &kmem_null_cache; 2471 kmem_null_cache.cache_prev = &kmem_null_cache; 2472 2473 kmem_metadata_arena = vmem_create("kmem_metadata", NULL, 0, PAGESIZE, 2474 vmem_alloc, vmem_free, heap_arena, 8 * PAGESIZE, 2475 VM_SLEEP | VMC_NO_QCACHE); 2476 2477 kmem_msb_arena = vmem_create("kmem_msb", NULL, 0, 2478 PAGESIZE, segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, 2479 VM_SLEEP); 2480 2481 kmem_cache_arena = vmem_create("kmem_cache", NULL, 0, KMEM_ALIGN, 2482 segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 2483 2484 kmem_hash_arena = vmem_create("kmem_hash", NULL, 0, KMEM_ALIGN, 2485 segkmem_alloc, segkmem_free, kmem_metadata_arena, 0, VM_SLEEP); 2486 2487 kmem_log_arena = vmem_create("kmem_log", NULL, 0, KMEM_ALIGN, 2488 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 2489 2490 kmem_firewall_va_arena = vmem_create("kmem_firewall_va", 2491 NULL, 0, PAGESIZE, 2492 kmem_firewall_va_alloc, kmem_firewall_va_free, heap_arena, 2493 0, VM_SLEEP); 2494 2495 kmem_firewall_arena = vmem_create("kmem_firewall", NULL, 0, PAGESIZE, 2496 segkmem_alloc, segkmem_free, kmem_firewall_va_arena, 0, VM_SLEEP); 2497 2498 /* temporary oversize arena for mod_read_system_file */ 2499 kmem_oversize_arena = vmem_create("kmem_oversize", NULL, 0, PAGESIZE, 2500 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP); 2501 2502 kmem_null_cache.cache_next = &kmem_null_cache; 2503 kmem_null_cache.cache_prev = &kmem_null_cache; 2504 2505 kmem_reap_interval = 15 * hz; 2506 2507 /* 2508 * Read /etc/system. This is a chicken-and-egg problem because 2509 * kmem_flags may be set in /etc/system, but mod_read_system_file() 2510 * needs to use the allocator. The simplest solution is to create 2511 * all the standard kmem caches, read /etc/system, destroy all the 2512 * caches we just created, and then create them all again in light 2513 * of the (possibly) new kmem_flags and other kmem tunables. 2514 */ 2515 kmem_cache_init(1, 0); 2516 2517 mod_read_system_file(boothowto & RB_ASKNAME); 2518 2519 while ((cp = kmem_null_cache.cache_prev) != &kmem_null_cache) 2520 kmem_cache_destroy(cp); 2521 2522 vmem_destroy(kmem_oversize_arena); 2523 2524 if (old_kmem_flags & KMF_STICKY) 2525 kmem_flags = old_kmem_flags; 2526 2527 if (!(kmem_flags & KMF_AUDIT)) 2528 vmem_seg_size = offsetof(vmem_seg_t, vs_thread); 2529 2530 if (kmem_maxverify == 0) 2531 kmem_maxverify = maxverify; 2532 2533 if (kmem_minfirewall == 0) 2534 kmem_minfirewall = minfirewall; 2535 2536 /* 2537 * give segkmem a chance to figure out if we are using large pages 2538 * for the kernel heap 2539 */ 2540 use_large_pages = segkmem_lpsetup(); 2541 2542 /* 2543 * To protect against corruption, we keep the actual number of callers 2544 * KMF_LITE records seperate from the tunable. We arbitrarily clamp 2545 * to 16, since the overhead for small buffers quickly gets out of 2546 * hand. 2547 * 2548 * The real limit would depend on the needs of the largest KMC_NOHASH 2549 * cache. 2550 */ 2551 kmem_lite_count = MIN(MAX(0, kmem_lite_pcs), 16); 2552 kmem_lite_pcs = kmem_lite_count; 2553 2554 /* 2555 * Normally, we firewall oversized allocations when possible, but 2556 * if we are using large pages for kernel memory, and we don't have 2557 * any non-LITE debugging flags set, we want to allocate oversized 2558 * buffers from large pages, and so skip the firewalling. 2559 */ 2560 if (use_large_pages && 2561 ((kmem_flags & KMF_LITE) || !(kmem_flags & KMF_DEBUG))) { 2562 kmem_oversize_arena = vmem_xcreate("kmem_oversize", NULL, 0, 2563 PAGESIZE, segkmem_alloc_lp, segkmem_free_lp, heap_arena, 2564 0, VM_SLEEP); 2565 } else { 2566 kmem_oversize_arena = vmem_create("kmem_oversize", 2567 NULL, 0, PAGESIZE, 2568 segkmem_alloc, segkmem_free, kmem_minfirewall < ULONG_MAX? 2569 kmem_firewall_va_arena : heap_arena, 0, VM_SLEEP); 2570 } 2571 2572 kmem_cache_init(2, use_large_pages); 2573 2574 if (kmem_flags & (KMF_AUDIT | KMF_RANDOMIZE)) { 2575 if (kmem_transaction_log_size == 0) 2576 kmem_transaction_log_size = kmem_maxavail() / 50; 2577 kmem_transaction_log = kmem_log_init(kmem_transaction_log_size); 2578 } 2579 2580 if (kmem_flags & (KMF_CONTENTS | KMF_RANDOMIZE)) { 2581 if (kmem_content_log_size == 0) 2582 kmem_content_log_size = kmem_maxavail() / 50; 2583 kmem_content_log = kmem_log_init(kmem_content_log_size); 2584 } 2585 2586 kmem_failure_log = kmem_log_init(kmem_failure_log_size); 2587 2588 kmem_slab_log = kmem_log_init(kmem_slab_log_size); 2589 2590 /* 2591 * Initialize STREAMS message caches so allocb() is available. 2592 * This allows us to initialize the logging framework (cmn_err(9F), 2593 * strlog(9F), etc) so we can start recording messages. 2594 */ 2595 streams_msg_init(); 2596 2597 /* 2598 * Initialize the ZSD framework in Zones so modules loaded henceforth 2599 * can register their callbacks. 2600 */ 2601 zone_zsd_init(); 2602 log_init(); 2603 taskq_init(); 2604 2605 /* 2606 * Warn about invalid or dangerous values of kmem_flags. 2607 * Always warn about unsupported values. 2608 */ 2609 if (((kmem_flags & ~(KMF_AUDIT | KMF_DEADBEEF | KMF_REDZONE | 2610 KMF_CONTENTS | KMF_LITE)) != 0) || 2611 ((kmem_flags & KMF_LITE) && kmem_flags != KMF_LITE)) 2612 cmn_err(CE_WARN, "kmem_flags set to unsupported value 0x%x. " 2613 "See the Solaris Tunable Parameters Reference Manual.", 2614 kmem_flags); 2615 2616 #ifdef DEBUG 2617 if ((kmem_flags & KMF_DEBUG) == 0) 2618 cmn_err(CE_NOTE, "kmem debugging disabled."); 2619 #else 2620 /* 2621 * For non-debug kernels, the only "normal" flags are 0, KMF_LITE, 2622 * KMF_REDZONE, and KMF_CONTENTS (the last because it is only enabled 2623 * if KMF_AUDIT is set). We should warn the user about the performance 2624 * penalty of KMF_AUDIT or KMF_DEADBEEF if they are set and KMF_LITE 2625 * isn't set (since that disables AUDIT). 2626 */ 2627 if (!(kmem_flags & KMF_LITE) && 2628 (kmem_flags & (KMF_AUDIT | KMF_DEADBEEF)) != 0) 2629 cmn_err(CE_WARN, "High-overhead kmem debugging features " 2630 "enabled (kmem_flags = 0x%x). Performance degradation " 2631 "and large memory overhead possible. See the Solaris " 2632 "Tunable Parameters Reference Manual.", kmem_flags); 2633 #endif /* not DEBUG */ 2634 2635 kmem_cache_applyall(kmem_cache_magazine_enable, NULL, TQ_SLEEP); 2636 2637 kmem_ready = 1; 2638 2639 /* 2640 * Initialize the platform-specific aligned/DMA memory allocator. 2641 */ 2642 ka_init(); 2643 2644 /* 2645 * Initialize 32-bit ID cache. 2646 */ 2647 id32_init(); 2648 } 2649 2650 void 2651 kmem_thread_init(void) 2652 { 2653 kmem_taskq = taskq_create_instance("kmem_taskq", 0, 1, minclsyspri, 2654 300, INT_MAX, TASKQ_PREPOPULATE); 2655 } 2656 2657 void 2658 kmem_mp_init(void) 2659 { 2660 mutex_enter(&cpu_lock); 2661 register_cpu_setup_func(kmem_cpu_setup, NULL); 2662 mutex_exit(&cpu_lock); 2663 2664 kmem_update_timeout(NULL); 2665 } 2666