1 /*- 2 * Copyright (c) 1987, 1991, 1993 3 * The Regents of the University of California. 4 * Copyright (c) 2005 Robert N. M. Watson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 4. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * @(#)kern_malloc.c 8.3 (Berkeley) 1/4/94 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_vm.h" 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/kdb.h> 42 #include <sys/kernel.h> 43 #include <sys/lock.h> 44 #include <sys/malloc.h> 45 #include <sys/mbuf.h> 46 #include <sys/mutex.h> 47 #include <sys/vmmeter.h> 48 #include <sys/proc.h> 49 #include <sys/sbuf.h> 50 #include <sys/sysctl.h> 51 #include <sys/time.h> 52 53 #include <vm/vm.h> 54 #include <vm/pmap.h> 55 #include <vm/vm_param.h> 56 #include <vm/vm_kern.h> 57 #include <vm/vm_extern.h> 58 #include <vm/vm_map.h> 59 #include <vm/vm_page.h> 60 #include <vm/uma.h> 61 #include <vm/uma_int.h> 62 #include <vm/uma_dbg.h> 63 64 #ifdef DEBUG_MEMGUARD 65 #include <vm/memguard.h> 66 #endif 67 68 #if defined(INVARIANTS) && defined(__i386__) 69 #include <machine/cpu.h> 70 #endif 71 72 /* 73 * When realloc() is called, if the new size is sufficiently smaller than 74 * the old size, realloc() will allocate a new, smaller block to avoid 75 * wasting memory. 'Sufficiently smaller' is defined as: newsize <= 76 * oldsize / 2^n, where REALLOC_FRACTION defines the value of 'n'. 77 */ 78 #ifndef REALLOC_FRACTION 79 #define REALLOC_FRACTION 1 /* new block if <= half the size */ 80 #endif 81 82 MALLOC_DEFINE(M_CACHE, "cache", "Various Dynamically allocated caches"); 83 MALLOC_DEFINE(M_DEVBUF, "devbuf", "device driver memory"); 84 MALLOC_DEFINE(M_TEMP, "temp", "misc temporary data buffers"); 85 86 MALLOC_DEFINE(M_IP6OPT, "ip6opt", "IPv6 options"); 87 MALLOC_DEFINE(M_IP6NDP, "ip6ndp", "IPv6 Neighbor Discovery"); 88 89 static void kmeminit(void *); 90 SYSINIT(kmem, SI_SUB_KMEM, SI_ORDER_FIRST, kmeminit, NULL) 91 92 static MALLOC_DEFINE(M_FREE, "free", "should be on free list"); 93 94 static struct malloc_type *kmemstatistics; 95 static char *kmembase; 96 static char *kmemlimit; 97 static int kmemcount; 98 99 #define KMEM_ZSHIFT 4 100 #define KMEM_ZBASE 16 101 #define KMEM_ZMASK (KMEM_ZBASE - 1) 102 103 #define KMEM_ZMAX PAGE_SIZE 104 #define KMEM_ZSIZE (KMEM_ZMAX >> KMEM_ZSHIFT) 105 static u_int8_t kmemsize[KMEM_ZSIZE + 1]; 106 107 /* These won't be powers of two for long */ 108 struct { 109 int kz_size; 110 char *kz_name; 111 uma_zone_t kz_zone; 112 } kmemzones[] = { 113 {16, "16", NULL}, 114 {32, "32", NULL}, 115 {64, "64", NULL}, 116 {128, "128", NULL}, 117 {256, "256", NULL}, 118 {512, "512", NULL}, 119 {1024, "1024", NULL}, 120 {2048, "2048", NULL}, 121 {4096, "4096", NULL}, 122 #if PAGE_SIZE > 4096 123 {8192, "8192", NULL}, 124 #if PAGE_SIZE > 8192 125 {16384, "16384", NULL}, 126 #if PAGE_SIZE > 16384 127 {32768, "32768", NULL}, 128 #if PAGE_SIZE > 32768 129 {65536, "65536", NULL}, 130 #if PAGE_SIZE > 65536 131 #error "Unsupported PAGE_SIZE" 132 #endif /* 65536 */ 133 #endif /* 32768 */ 134 #endif /* 16384 */ 135 #endif /* 8192 */ 136 #endif /* 4096 */ 137 {0, NULL}, 138 }; 139 140 static uma_zone_t mt_zone; 141 142 #ifdef DEBUG_MEMGUARD 143 u_int vm_memguard_divisor; 144 SYSCTL_UINT(_vm, OID_AUTO, memguard_divisor, CTLFLAG_RD, &vm_memguard_divisor, 145 0, "(kmem_size/memguard_divisor) == memguard submap size"); 146 #endif 147 148 u_int vm_kmem_size; 149 SYSCTL_UINT(_vm, OID_AUTO, kmem_size, CTLFLAG_RD, &vm_kmem_size, 0, 150 "Size of kernel memory"); 151 152 u_int vm_kmem_size_max; 153 SYSCTL_UINT(_vm, OID_AUTO, kmem_size_max, CTLFLAG_RD, &vm_kmem_size_max, 0, 154 "Maximum size of kernel memory"); 155 156 u_int vm_kmem_size_scale; 157 SYSCTL_UINT(_vm, OID_AUTO, kmem_size_scale, CTLFLAG_RD, &vm_kmem_size_scale, 0, 158 "Scale factor for kernel memory size"); 159 160 /* 161 * The malloc_mtx protects the kmemstatistics linked list. 162 */ 163 164 struct mtx malloc_mtx; 165 166 #ifdef MALLOC_PROFILE 167 uint64_t krequests[KMEM_ZSIZE + 1]; 168 169 static int sysctl_kern_mprof(SYSCTL_HANDLER_ARGS); 170 #endif 171 172 static int sysctl_kern_malloc(SYSCTL_HANDLER_ARGS); 173 static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS); 174 175 /* time_uptime of last malloc(9) failure */ 176 static time_t t_malloc_fail; 177 178 #ifdef MALLOC_MAKE_FAILURES 179 /* 180 * Causes malloc failures every (n) mallocs with M_NOWAIT. If set to 0, 181 * doesn't cause failures. 182 */ 183 SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0, 184 "Kernel malloc debugging options"); 185 186 static int malloc_failure_rate; 187 static int malloc_nowait_count; 188 static int malloc_failure_count; 189 SYSCTL_INT(_debug_malloc, OID_AUTO, failure_rate, CTLFLAG_RW, 190 &malloc_failure_rate, 0, "Every (n) mallocs with M_NOWAIT will fail"); 191 TUNABLE_INT("debug.malloc.failure_rate", &malloc_failure_rate); 192 SYSCTL_INT(_debug_malloc, OID_AUTO, failure_count, CTLFLAG_RD, 193 &malloc_failure_count, 0, "Number of imposed M_NOWAIT malloc failures"); 194 #endif 195 196 int 197 malloc_last_fail(void) 198 { 199 200 return (time_uptime - t_malloc_fail); 201 } 202 203 /* 204 * Add this to the informational malloc_type bucket. 205 */ 206 static void 207 malloc_type_zone_allocated(struct malloc_type *mtp, unsigned long size, 208 int zindx) 209 { 210 struct malloc_type_internal *mtip; 211 struct malloc_type_stats *mtsp; 212 213 critical_enter(); 214 mtip = mtp->ks_handle; 215 mtsp = &mtip->mti_stats[curcpu]; 216 if (size > 0) { 217 mtsp->mts_memalloced += size; 218 mtsp->mts_numallocs++; 219 } 220 if (zindx != -1) 221 mtsp->mts_size |= 1 << zindx; 222 critical_exit(); 223 } 224 225 void 226 malloc_type_allocated(struct malloc_type *mtp, unsigned long size) 227 { 228 229 if (size > 0) 230 malloc_type_zone_allocated(mtp, size, -1); 231 } 232 233 /* 234 * Remove this allocation from the informational malloc_type bucket. 235 */ 236 void 237 malloc_type_freed(struct malloc_type *mtp, unsigned long size) 238 { 239 struct malloc_type_internal *mtip; 240 struct malloc_type_stats *mtsp; 241 242 critical_enter(); 243 mtip = mtp->ks_handle; 244 mtsp = &mtip->mti_stats[curcpu]; 245 mtsp->mts_memfreed += size; 246 mtsp->mts_numfrees++; 247 critical_exit(); 248 } 249 250 /* 251 * malloc: 252 * 253 * Allocate a block of memory. 254 * 255 * If M_NOWAIT is set, this routine will not block and return NULL if 256 * the allocation fails. 257 */ 258 void * 259 malloc(unsigned long size, struct malloc_type *mtp, int flags) 260 { 261 int indx; 262 caddr_t va; 263 uma_zone_t zone; 264 uma_keg_t keg; 265 #ifdef DIAGNOSTIC 266 unsigned long osize = size; 267 #endif 268 269 #ifdef INVARIANTS 270 /* 271 * Check that exactly one of M_WAITOK or M_NOWAIT is specified. 272 */ 273 indx = flags & (M_WAITOK | M_NOWAIT); 274 if (indx != M_NOWAIT && indx != M_WAITOK) { 275 static struct timeval lasterr; 276 static int curerr, once; 277 if (once == 0 && ppsratecheck(&lasterr, &curerr, 1)) { 278 printf("Bad malloc flags: %x\n", indx); 279 kdb_backtrace(); 280 flags |= M_WAITOK; 281 once++; 282 } 283 } 284 #endif 285 #if 0 286 if (size == 0) 287 kdb_enter("zero size malloc"); 288 #endif 289 #ifdef MALLOC_MAKE_FAILURES 290 if ((flags & M_NOWAIT) && (malloc_failure_rate != 0)) { 291 atomic_add_int(&malloc_nowait_count, 1); 292 if ((malloc_nowait_count % malloc_failure_rate) == 0) { 293 atomic_add_int(&malloc_failure_count, 1); 294 t_malloc_fail = time_uptime; 295 return (NULL); 296 } 297 } 298 #endif 299 if (flags & M_WAITOK) 300 KASSERT(curthread->td_intr_nesting_level == 0, 301 ("malloc(M_WAITOK) in interrupt context")); 302 303 #ifdef DEBUG_MEMGUARD 304 /* XXX CHANGEME! */ 305 if (mtp == M_SUBPROC) 306 return memguard_alloc(size, flags); 307 #endif 308 309 if (size <= KMEM_ZMAX) { 310 if (size & KMEM_ZMASK) 311 size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; 312 indx = kmemsize[size >> KMEM_ZSHIFT]; 313 zone = kmemzones[indx].kz_zone; 314 keg = zone->uz_keg; 315 #ifdef MALLOC_PROFILE 316 krequests[size >> KMEM_ZSHIFT]++; 317 #endif 318 va = uma_zalloc(zone, flags); 319 if (va != NULL) 320 size = keg->uk_size; 321 malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); 322 } else { 323 size = roundup(size, PAGE_SIZE); 324 zone = NULL; 325 keg = NULL; 326 va = uma_large_malloc(size, flags); 327 malloc_type_allocated(mtp, va == NULL ? 0 : size); 328 } 329 if (flags & M_WAITOK) 330 KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL")); 331 else if (va == NULL) 332 t_malloc_fail = time_uptime; 333 #ifdef DIAGNOSTIC 334 if (va != NULL && !(flags & M_ZERO)) { 335 memset(va, 0x70, osize); 336 } 337 #endif 338 return ((void *) va); 339 } 340 341 /* 342 * free: 343 * 344 * Free a block of memory allocated by malloc. 345 * 346 * This routine may not block. 347 */ 348 void 349 free(void *addr, struct malloc_type *mtp) 350 { 351 uma_slab_t slab; 352 u_long size; 353 354 /* free(NULL, ...) does nothing */ 355 if (addr == NULL) 356 return; 357 358 #ifdef DEBUG_MEMGUARD 359 /* XXX CHANGEME! */ 360 if (mtp == M_SUBPROC) { 361 memguard_free(addr); 362 return; 363 } 364 #endif 365 366 size = 0; 367 368 slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK)); 369 370 if (slab == NULL) 371 panic("free: address %p(%p) has not been allocated.\n", 372 addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); 373 374 375 if (!(slab->us_flags & UMA_SLAB_MALLOC)) { 376 #ifdef INVARIANTS 377 struct malloc_type **mtpp = addr; 378 #endif 379 size = slab->us_keg->uk_size; 380 #ifdef INVARIANTS 381 /* 382 * Cache a pointer to the malloc_type that most recently freed 383 * this memory here. This way we know who is most likely to 384 * have stepped on it later. 385 * 386 * This code assumes that size is a multiple of 8 bytes for 387 * 64 bit machines 388 */ 389 mtpp = (struct malloc_type **) 390 ((unsigned long)mtpp & ~UMA_ALIGN_PTR); 391 mtpp += (size - sizeof(struct malloc_type *)) / 392 sizeof(struct malloc_type *); 393 *mtpp = mtp; 394 #endif 395 uma_zfree_arg(LIST_FIRST(&slab->us_keg->uk_zones), addr, slab); 396 } else { 397 size = slab->us_size; 398 uma_large_free(slab); 399 } 400 malloc_type_freed(mtp, size); 401 } 402 403 /* 404 * realloc: change the size of a memory block 405 */ 406 void * 407 realloc(void *addr, unsigned long size, struct malloc_type *mtp, int flags) 408 { 409 uma_slab_t slab; 410 unsigned long alloc; 411 void *newaddr; 412 413 /* realloc(NULL, ...) is equivalent to malloc(...) */ 414 if (addr == NULL) 415 return (malloc(size, mtp, flags)); 416 417 /* 418 * XXX: Should report free of old memory and alloc of new memory to 419 * per-CPU stats. 420 */ 421 422 #ifdef DEBUG_MEMGUARD 423 /* XXX: CHANGEME! */ 424 if (mtp == M_SUBPROC) { 425 slab = NULL; 426 alloc = size; 427 } else { 428 #endif 429 430 slab = vtoslab((vm_offset_t)addr & ~(UMA_SLAB_MASK)); 431 432 /* Sanity check */ 433 KASSERT(slab != NULL, 434 ("realloc: address %p out of range", (void *)addr)); 435 436 /* Get the size of the original block */ 437 if (slab->us_keg) 438 alloc = slab->us_keg->uk_size; 439 else 440 alloc = slab->us_size; 441 442 /* Reuse the original block if appropriate */ 443 if (size <= alloc 444 && (size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE)) 445 return (addr); 446 447 #ifdef DEBUG_MEMGUARD 448 } 449 #endif 450 451 /* Allocate a new, bigger (or smaller) block */ 452 if ((newaddr = malloc(size, mtp, flags)) == NULL) 453 return (NULL); 454 455 /* Copy over original contents */ 456 bcopy(addr, newaddr, min(size, alloc)); 457 free(addr, mtp); 458 return (newaddr); 459 } 460 461 /* 462 * reallocf: same as realloc() but free memory on failure. 463 */ 464 void * 465 reallocf(void *addr, unsigned long size, struct malloc_type *mtp, int flags) 466 { 467 void *mem; 468 469 if ((mem = realloc(addr, size, mtp, flags)) == NULL) 470 free(addr, mtp); 471 return (mem); 472 } 473 474 /* 475 * Initialize the kernel memory allocator 476 */ 477 /* ARGSUSED*/ 478 static void 479 kmeminit(void *dummy) 480 { 481 u_int8_t indx; 482 u_long mem_size; 483 int i; 484 485 mtx_init(&malloc_mtx, "malloc", NULL, MTX_DEF); 486 487 /* 488 * Try to auto-tune the kernel memory size, so that it is 489 * more applicable for a wider range of machine sizes. 490 * On an X86, a VM_KMEM_SIZE_SCALE value of 4 is good, while 491 * a VM_KMEM_SIZE of 12MB is a fair compromise. The 492 * VM_KMEM_SIZE_MAX is dependent on the maximum KVA space 493 * available, and on an X86 with a total KVA space of 256MB, 494 * try to keep VM_KMEM_SIZE_MAX at 80MB or below. 495 * 496 * Note that the kmem_map is also used by the zone allocator, 497 * so make sure that there is enough space. 498 */ 499 vm_kmem_size = VM_KMEM_SIZE + nmbclusters * PAGE_SIZE; 500 mem_size = cnt.v_page_count; 501 502 #if defined(VM_KMEM_SIZE_SCALE) 503 vm_kmem_size_scale = VM_KMEM_SIZE_SCALE; 504 #endif 505 TUNABLE_INT_FETCH("vm.kmem_size_scale", &vm_kmem_size_scale); 506 if (vm_kmem_size_scale > 0 && 507 (mem_size / vm_kmem_size_scale) > (vm_kmem_size / PAGE_SIZE)) 508 vm_kmem_size = (mem_size / vm_kmem_size_scale) * PAGE_SIZE; 509 510 #if defined(VM_KMEM_SIZE_MAX) 511 vm_kmem_size_max = VM_KMEM_SIZE_MAX; 512 #endif 513 TUNABLE_INT_FETCH("vm.kmem_size_max", &vm_kmem_size_max); 514 if (vm_kmem_size_max > 0 && vm_kmem_size >= vm_kmem_size_max) 515 vm_kmem_size = vm_kmem_size_max; 516 517 /* Allow final override from the kernel environment */ 518 #ifndef BURN_BRIDGES 519 if (TUNABLE_INT_FETCH("kern.vm.kmem.size", &vm_kmem_size) != 0) 520 printf("kern.vm.kmem.size is now called vm.kmem_size!\n"); 521 #endif 522 TUNABLE_INT_FETCH("vm.kmem_size", &vm_kmem_size); 523 524 /* 525 * Limit kmem virtual size to twice the physical memory. 526 * This allows for kmem map sparseness, but limits the size 527 * to something sane. Be careful to not overflow the 32bit 528 * ints while doing the check. 529 */ 530 if (((vm_kmem_size / 2) / PAGE_SIZE) > cnt.v_page_count) 531 vm_kmem_size = 2 * cnt.v_page_count * PAGE_SIZE; 532 533 /* 534 * Tune settings based on the kernel map's size at this time. 535 */ 536 init_param3(vm_kmem_size / PAGE_SIZE); 537 538 kmem_map = kmem_suballoc(kernel_map, (vm_offset_t *)&kmembase, 539 (vm_offset_t *)&kmemlimit, vm_kmem_size); 540 kmem_map->system_map = 1; 541 542 #ifdef DEBUG_MEMGUARD 543 /* 544 * Initialize MemGuard if support compiled in. MemGuard is a 545 * replacement allocator used for detecting tamper-after-free 546 * scenarios as they occur. It is only used for debugging. 547 */ 548 vm_memguard_divisor = 10; 549 TUNABLE_INT_FETCH("vm.memguard_divisor", &vm_memguard_divisor); 550 551 /* Pick a conservative value if provided value sucks. */ 552 if ((vm_memguard_divisor <= 0) || 553 ((vm_kmem_size / vm_memguard_divisor) == 0)) 554 vm_memguard_divisor = 10; 555 memguard_init(kmem_map, vm_kmem_size / vm_memguard_divisor); 556 #endif 557 558 uma_startup2(); 559 560 mt_zone = uma_zcreate("mt_zone", sizeof(struct malloc_type_internal), 561 #ifdef INVARIANTS 562 mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini, 563 #else 564 NULL, NULL, NULL, NULL, 565 #endif 566 UMA_ALIGN_PTR, UMA_ZONE_MALLOC); 567 for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) { 568 int size = kmemzones[indx].kz_size; 569 char *name = kmemzones[indx].kz_name; 570 571 kmemzones[indx].kz_zone = uma_zcreate(name, size, 572 #ifdef INVARIANTS 573 mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini, 574 #else 575 NULL, NULL, NULL, NULL, 576 #endif 577 UMA_ALIGN_PTR, UMA_ZONE_MALLOC); 578 579 for (;i <= size; i+= KMEM_ZBASE) 580 kmemsize[i >> KMEM_ZSHIFT] = indx; 581 582 } 583 } 584 585 void 586 malloc_init(void *data) 587 { 588 struct malloc_type_internal *mtip; 589 struct malloc_type *mtp; 590 591 KASSERT(cnt.v_page_count != 0, ("malloc_register before vm_init")); 592 593 mtp = data; 594 mtip = uma_zalloc(mt_zone, M_WAITOK | M_ZERO); 595 mtp->ks_handle = mtip; 596 597 mtx_lock(&malloc_mtx); 598 mtp->ks_next = kmemstatistics; 599 kmemstatistics = mtp; 600 kmemcount++; 601 mtx_unlock(&malloc_mtx); 602 } 603 604 void 605 malloc_uninit(void *data) 606 { 607 struct malloc_type_internal *mtip; 608 struct malloc_type *mtp, *temp; 609 610 mtp = data; 611 KASSERT(mtp->ks_handle != NULL, ("malloc_deregister: cookie NULL")); 612 mtx_lock(&malloc_mtx); 613 mtip = mtp->ks_handle; 614 mtp->ks_handle = NULL; 615 if (mtp != kmemstatistics) { 616 for (temp = kmemstatistics; temp != NULL; 617 temp = temp->ks_next) { 618 if (temp->ks_next == mtp) 619 temp->ks_next = mtp->ks_next; 620 } 621 } else 622 kmemstatistics = mtp->ks_next; 623 kmemcount--; 624 mtx_unlock(&malloc_mtx); 625 uma_zfree(mt_zone, mtip); 626 } 627 628 static int 629 sysctl_kern_malloc(SYSCTL_HANDLER_ARGS) 630 { 631 struct malloc_type_stats mts_local, *mtsp; 632 struct malloc_type_internal *mtip; 633 struct malloc_type *mtp; 634 struct sbuf sbuf; 635 long temp_allocs, temp_bytes; 636 int linesize = 128; 637 int bufsize; 638 int first; 639 int error; 640 char *buf; 641 int cnt; 642 int i; 643 644 cnt = 0; 645 646 /* Guess at how much room is needed. */ 647 mtx_lock(&malloc_mtx); 648 cnt = kmemcount; 649 mtx_unlock(&malloc_mtx); 650 651 bufsize = linesize * (cnt + 1); 652 buf = malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO); 653 sbuf_new(&sbuf, buf, bufsize, SBUF_FIXEDLEN); 654 655 mtx_lock(&malloc_mtx); 656 sbuf_printf(&sbuf, 657 "\n Type InUse MemUse HighUse Requests Size(s)\n"); 658 for (mtp = kmemstatistics; cnt != 0 && mtp != NULL; 659 mtp = mtp->ks_next, cnt--) { 660 mtip = mtp->ks_handle; 661 bzero(&mts_local, sizeof(mts_local)); 662 for (i = 0; i < MAXCPU; i++) { 663 mtsp = &mtip->mti_stats[i]; 664 mts_local.mts_memalloced += mtsp->mts_memalloced; 665 mts_local.mts_memfreed += mtsp->mts_memfreed; 666 mts_local.mts_numallocs += mtsp->mts_numallocs; 667 mts_local.mts_numfrees += mtsp->mts_numfrees; 668 mts_local.mts_size |= mtsp->mts_size; 669 } 670 if (mts_local.mts_numallocs == 0) 671 continue; 672 673 /* 674 * Due to races in per-CPU statistics gather, it's possible to 675 * get a slightly negative number here. If we do, approximate 676 * with 0. 677 */ 678 if (mts_local.mts_numallocs > mts_local.mts_numfrees) 679 temp_allocs = mts_local.mts_numallocs - 680 mts_local.mts_numfrees; 681 else 682 temp_allocs = 0; 683 684 /* 685 * Ditto for bytes allocated. 686 */ 687 if (mts_local.mts_memalloced > mts_local.mts_memfreed) 688 temp_bytes = mts_local.mts_memalloced - 689 mts_local.mts_memfreed; 690 else 691 temp_bytes = 0; 692 693 /* 694 * High-waterwark is no longer easily available, so we just 695 * print '-' for that column. 696 */ 697 sbuf_printf(&sbuf, "%13s%6lu%6luK -%9llu", 698 mtp->ks_shortdesc, 699 temp_allocs, 700 (temp_bytes + 1023) / 1024, 701 (unsigned long long)mts_local.mts_numallocs); 702 703 first = 1; 704 for (i = 0; i < sizeof(kmemzones) / sizeof(kmemzones[0]) - 1; 705 i++) { 706 if (mts_local.mts_size & (1 << i)) { 707 if (first) 708 sbuf_printf(&sbuf, " "); 709 else 710 sbuf_printf(&sbuf, ","); 711 sbuf_printf(&sbuf, "%s", 712 kmemzones[i].kz_name); 713 first = 0; 714 } 715 } 716 sbuf_printf(&sbuf, "\n"); 717 } 718 sbuf_finish(&sbuf); 719 mtx_unlock(&malloc_mtx); 720 721 error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf)); 722 723 sbuf_delete(&sbuf); 724 free(buf, M_TEMP); 725 return (error); 726 } 727 728 SYSCTL_OID(_kern, OID_AUTO, malloc, CTLTYPE_STRING|CTLFLAG_RD, 729 NULL, 0, sysctl_kern_malloc, "A", "Malloc Stats"); 730 731 static int 732 sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS) 733 { 734 struct malloc_type_stream_header mtsh; 735 struct malloc_type_internal *mtip; 736 struct malloc_type_header mth; 737 struct malloc_type *mtp; 738 int buflen, count, error, i; 739 struct sbuf sbuf; 740 char *buffer; 741 742 mtx_lock(&malloc_mtx); 743 restart: 744 mtx_assert(&malloc_mtx, MA_OWNED); 745 count = kmemcount; 746 mtx_unlock(&malloc_mtx); 747 buflen = sizeof(mtsh) + count * (sizeof(mth) + 748 sizeof(struct malloc_type_stats) * MAXCPU) + 1; 749 buffer = malloc(buflen, M_TEMP, M_WAITOK | M_ZERO); 750 mtx_lock(&malloc_mtx); 751 if (count < kmemcount) { 752 free(buffer, M_TEMP); 753 goto restart; 754 } 755 756 sbuf_new(&sbuf, buffer, buflen, SBUF_FIXEDLEN); 757 758 /* 759 * Insert stream header. 760 */ 761 bzero(&mtsh, sizeof(mtsh)); 762 mtsh.mtsh_version = MALLOC_TYPE_STREAM_VERSION; 763 mtsh.mtsh_maxcpus = MAXCPU; 764 mtsh.mtsh_count = kmemcount; 765 if (sbuf_bcat(&sbuf, &mtsh, sizeof(mtsh)) < 0) { 766 mtx_unlock(&malloc_mtx); 767 error = ENOMEM; 768 goto out; 769 } 770 771 /* 772 * Insert alternating sequence of type headers and type statistics. 773 */ 774 for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) { 775 mtip = (struct malloc_type_internal *)mtp->ks_handle; 776 777 /* 778 * Insert type header. 779 */ 780 bzero(&mth, sizeof(mth)); 781 strlcpy(mth.mth_name, mtp->ks_shortdesc, MALLOC_MAX_NAME); 782 if (sbuf_bcat(&sbuf, &mth, sizeof(mth)) < 0) { 783 mtx_unlock(&malloc_mtx); 784 error = ENOMEM; 785 goto out; 786 } 787 788 /* 789 * Insert type statistics for each CPU. 790 */ 791 for (i = 0; i < MAXCPU; i++) { 792 if (sbuf_bcat(&sbuf, &mtip->mti_stats[i], 793 sizeof(mtip->mti_stats[i])) < 0) { 794 mtx_unlock(&malloc_mtx); 795 error = ENOMEM; 796 goto out; 797 } 798 } 799 } 800 mtx_unlock(&malloc_mtx); 801 sbuf_finish(&sbuf); 802 error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf)); 803 out: 804 sbuf_delete(&sbuf); 805 free(buffer, M_TEMP); 806 return (error); 807 } 808 809 SYSCTL_PROC(_kern, OID_AUTO, malloc_stats, CTLFLAG_RD|CTLTYPE_STRUCT, 810 0, 0, sysctl_kern_malloc_stats, "s,malloc_type_ustats", 811 "Return malloc types"); 812 813 SYSCTL_INT(_kern, OID_AUTO, malloc_count, CTLFLAG_RD, &kmemcount, 0, 814 "Count of kernel malloc types"); 815 816 #ifdef MALLOC_PROFILE 817 818 static int 819 sysctl_kern_mprof(SYSCTL_HANDLER_ARGS) 820 { 821 int linesize = 64; 822 struct sbuf sbuf; 823 uint64_t count; 824 uint64_t waste; 825 uint64_t mem; 826 int bufsize; 827 int error; 828 char *buf; 829 int rsize; 830 int size; 831 int i; 832 833 bufsize = linesize * (KMEM_ZSIZE + 1); 834 bufsize += 128; /* For the stats line */ 835 bufsize += 128; /* For the banner line */ 836 waste = 0; 837 mem = 0; 838 839 buf = malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO); 840 sbuf_new(&sbuf, buf, bufsize, SBUF_FIXEDLEN); 841 sbuf_printf(&sbuf, 842 "\n Size Requests Real Size\n"); 843 for (i = 0; i < KMEM_ZSIZE; i++) { 844 size = i << KMEM_ZSHIFT; 845 rsize = kmemzones[kmemsize[i]].kz_size; 846 count = (long long unsigned)krequests[i]; 847 848 sbuf_printf(&sbuf, "%6d%28llu%11d\n", size, 849 (unsigned long long)count, rsize); 850 851 if ((rsize * count) > (size * count)) 852 waste += (rsize * count) - (size * count); 853 mem += (rsize * count); 854 } 855 sbuf_printf(&sbuf, 856 "\nTotal memory used:\t%30llu\nTotal Memory wasted:\t%30llu\n", 857 (unsigned long long)mem, (unsigned long long)waste); 858 sbuf_finish(&sbuf); 859 860 error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf)); 861 862 sbuf_delete(&sbuf); 863 free(buf, M_TEMP); 864 return (error); 865 } 866 867 SYSCTL_OID(_kern, OID_AUTO, mprof, CTLTYPE_STRING|CTLFLAG_RD, 868 NULL, 0, sysctl_kern_mprof, "A", "Malloc Profiling"); 869 #endif /* MALLOC_PROFILE */ 870