1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 malloc_tsd_data(, arenas, arena_t *, NULL) 8 malloc_tsd_data(, thread_allocated, thread_allocated_t, 9 THREAD_ALLOCATED_INITIALIZER) 10 11 /* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */ 12 const char *__malloc_options_1_0 = NULL; 13 __sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); 14 15 /* Runtime configuration options. */ 16 const char *je_malloc_conf; 17 bool opt_abort = 18 #ifdef JEMALLOC_DEBUG 19 true 20 #else 21 false 22 #endif 23 ; 24 bool opt_junk = 25 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 26 true 27 #else 28 false 29 #endif 30 ; 31 size_t opt_quarantine = ZU(0); 32 bool opt_redzone = false; 33 bool opt_utrace = false; 34 bool opt_valgrind = false; 35 bool opt_xmalloc = false; 36 bool opt_zero = false; 37 size_t opt_narenas = 0; 38 39 unsigned ncpus; 40 41 malloc_mutex_t arenas_lock; 42 arena_t **arenas; 43 unsigned narenas_total; 44 unsigned narenas_auto; 45 46 /* Set to true once the allocator has been initialized. */ 47 static bool malloc_initialized = false; 48 49 #ifdef JEMALLOC_THREADED_INIT 50 /* Used to let the initializing thread recursively allocate. */ 51 # define NO_INITIALIZER ((unsigned long)0) 52 # define INITIALIZER pthread_self() 53 # define IS_INITIALIZER (malloc_initializer == pthread_self()) 54 static pthread_t malloc_initializer = NO_INITIALIZER; 55 #else 56 # define NO_INITIALIZER false 57 # define INITIALIZER true 58 # define IS_INITIALIZER malloc_initializer 59 static bool malloc_initializer = NO_INITIALIZER; 60 #endif 61 62 /* Used to avoid initialization races. */ 63 #ifdef _WIN32 64 static malloc_mutex_t init_lock; 65 66 JEMALLOC_ATTR(constructor) 67 static void WINAPI 68 _init_init_lock(void) 69 { 70 71 malloc_mutex_init(&init_lock); 72 } 73 74 #ifdef _MSC_VER 75 # pragma section(".CRT$XCU", read) 76 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 77 static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 78 #endif 79 80 #else 81 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 82 #endif 83 84 typedef struct { 85 void *p; /* Input pointer (as in realloc(p, s)). */ 86 size_t s; /* Request size. */ 87 void *r; /* Result pointer. */ 88 } malloc_utrace_t; 89 90 #ifdef JEMALLOC_UTRACE 91 # define UTRACE(a, b, c) do { \ 92 if (opt_utrace) { \ 93 int utrace_serrno = errno; \ 94 malloc_utrace_t ut; \ 95 ut.p = (a); \ 96 ut.s = (b); \ 97 ut.r = (c); \ 98 utrace(&ut, sizeof(ut)); \ 99 errno = utrace_serrno; \ 100 } \ 101 } while (0) 102 #else 103 # define UTRACE(a, b, c) 104 #endif 105 106 /******************************************************************************/ 107 /* 108 * Function prototypes for static functions that are referenced prior to 109 * definition. 110 */ 111 112 static bool malloc_init_hard(void); 113 114 /******************************************************************************/ 115 /* 116 * Begin miscellaneous support functions. 117 */ 118 119 /* Create a new arena and insert it into the arenas array at index ind. */ 120 arena_t * 121 arenas_extend(unsigned ind) 122 { 123 arena_t *ret; 124 125 ret = (arena_t *)base_alloc(sizeof(arena_t)); 126 if (ret != NULL && arena_new(ret, ind) == false) { 127 arenas[ind] = ret; 128 return (ret); 129 } 130 /* Only reached if there is an OOM error. */ 131 132 /* 133 * OOM here is quite inconvenient to propagate, since dealing with it 134 * would require a check for failure in the fast path. Instead, punt 135 * by using arenas[0]. In practice, this is an extremely unlikely 136 * failure. 137 */ 138 malloc_write("<jemalloc>: Error initializing arena\n"); 139 if (opt_abort) 140 abort(); 141 142 return (arenas[0]); 143 } 144 145 /* Slow path, called only by choose_arena(). */ 146 arena_t * 147 choose_arena_hard(void) 148 { 149 arena_t *ret; 150 151 if (narenas_auto > 1) { 152 unsigned i, choose, first_null; 153 154 choose = 0; 155 first_null = narenas_auto; 156 malloc_mutex_lock(&arenas_lock); 157 assert(arenas[0] != NULL); 158 for (i = 1; i < narenas_auto; i++) { 159 if (arenas[i] != NULL) { 160 /* 161 * Choose the first arena that has the lowest 162 * number of threads assigned to it. 163 */ 164 if (arenas[i]->nthreads < 165 arenas[choose]->nthreads) 166 choose = i; 167 } else if (first_null == narenas_auto) { 168 /* 169 * Record the index of the first uninitialized 170 * arena, in case all extant arenas are in use. 171 * 172 * NB: It is possible for there to be 173 * discontinuities in terms of initialized 174 * versus uninitialized arenas, due to the 175 * "thread.arena" mallctl. 176 */ 177 first_null = i; 178 } 179 } 180 181 if (arenas[choose]->nthreads == 0 182 || first_null == narenas_auto) { 183 /* 184 * Use an unloaded arena, or the least loaded arena if 185 * all arenas are already initialized. 186 */ 187 ret = arenas[choose]; 188 } else { 189 /* Initialize a new arena. */ 190 ret = arenas_extend(first_null); 191 } 192 ret->nthreads++; 193 malloc_mutex_unlock(&arenas_lock); 194 } else { 195 ret = arenas[0]; 196 malloc_mutex_lock(&arenas_lock); 197 ret->nthreads++; 198 malloc_mutex_unlock(&arenas_lock); 199 } 200 201 arenas_tsd_set(&ret); 202 203 return (ret); 204 } 205 206 static void 207 stats_print_atexit(void) 208 { 209 210 if (config_tcache && config_stats) { 211 unsigned narenas, i; 212 213 /* 214 * Merge stats from extant threads. This is racy, since 215 * individual threads do not lock when recording tcache stats 216 * events. As a consequence, the final stats may be slightly 217 * out of date by the time they are reported, if other threads 218 * continue to allocate. 219 */ 220 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 221 arena_t *arena = arenas[i]; 222 if (arena != NULL) { 223 tcache_t *tcache; 224 225 /* 226 * tcache_stats_merge() locks bins, so if any 227 * code is introduced that acquires both arena 228 * and bin locks in the opposite order, 229 * deadlocks may result. 230 */ 231 malloc_mutex_lock(&arena->lock); 232 ql_foreach(tcache, &arena->tcache_ql, link) { 233 tcache_stats_merge(tcache, arena); 234 } 235 malloc_mutex_unlock(&arena->lock); 236 } 237 } 238 } 239 je_malloc_stats_print(NULL, NULL, NULL); 240 } 241 242 /* 243 * End miscellaneous support functions. 244 */ 245 /******************************************************************************/ 246 /* 247 * Begin initialization functions. 248 */ 249 250 static unsigned 251 malloc_ncpus(void) 252 { 253 long result; 254 255 #ifdef _WIN32 256 SYSTEM_INFO si; 257 GetSystemInfo(&si); 258 result = si.dwNumberOfProcessors; 259 #else 260 result = sysconf(_SC_NPROCESSORS_ONLN); 261 #endif 262 return ((result == -1) ? 1 : (unsigned)result); 263 } 264 265 void 266 arenas_cleanup(void *arg) 267 { 268 arena_t *arena = *(arena_t **)arg; 269 270 malloc_mutex_lock(&arenas_lock); 271 arena->nthreads--; 272 malloc_mutex_unlock(&arenas_lock); 273 } 274 275 JEMALLOC_ALWAYS_INLINE_C void 276 malloc_thread_init(void) 277 { 278 279 /* 280 * TSD initialization can't be safely done as a side effect of 281 * deallocation, because it is possible for a thread to do nothing but 282 * deallocate its TLS data via free(), in which case writing to TLS 283 * would cause write-after-free memory corruption. The quarantine 284 * facility *only* gets used as a side effect of deallocation, so make 285 * a best effort attempt at initializing its TSD by hooking all 286 * allocation events. 287 */ 288 if (config_fill && opt_quarantine) 289 quarantine_alloc_hook(); 290 } 291 292 JEMALLOC_ALWAYS_INLINE_C bool 293 malloc_init(void) 294 { 295 296 if (malloc_initialized == false && malloc_init_hard()) 297 return (true); 298 malloc_thread_init(); 299 300 return (false); 301 } 302 303 static bool 304 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 305 char const **v_p, size_t *vlen_p) 306 { 307 bool accept; 308 const char *opts = *opts_p; 309 310 *k_p = opts; 311 312 for (accept = false; accept == false;) { 313 switch (*opts) { 314 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 315 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 316 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 317 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 318 case 'Y': case 'Z': 319 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 320 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 321 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 322 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 323 case 'y': case 'z': 324 case '0': case '1': case '2': case '3': case '4': case '5': 325 case '6': case '7': case '8': case '9': 326 case '_': 327 opts++; 328 break; 329 case ':': 330 opts++; 331 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 332 *v_p = opts; 333 accept = true; 334 break; 335 case '\0': 336 if (opts != *opts_p) { 337 malloc_write("<jemalloc>: Conf string ends " 338 "with key\n"); 339 } 340 return (true); 341 default: 342 malloc_write("<jemalloc>: Malformed conf string\n"); 343 return (true); 344 } 345 } 346 347 for (accept = false; accept == false;) { 348 switch (*opts) { 349 case ',': 350 opts++; 351 /* 352 * Look ahead one character here, because the next time 353 * this function is called, it will assume that end of 354 * input has been cleanly reached if no input remains, 355 * but we have optimistically already consumed the 356 * comma if one exists. 357 */ 358 if (*opts == '\0') { 359 malloc_write("<jemalloc>: Conf string ends " 360 "with comma\n"); 361 } 362 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 363 accept = true; 364 break; 365 case '\0': 366 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 367 accept = true; 368 break; 369 default: 370 opts++; 371 break; 372 } 373 } 374 375 *opts_p = opts; 376 return (false); 377 } 378 379 static void 380 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 381 size_t vlen) 382 { 383 384 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 385 (int)vlen, v); 386 } 387 388 static void 389 malloc_conf_init(void) 390 { 391 unsigned i; 392 char buf[PATH_MAX + 1]; 393 const char *opts, *k, *v; 394 size_t klen, vlen; 395 396 /* 397 * Automatically configure valgrind before processing options. The 398 * valgrind option remains in jemalloc 3.x for compatibility reasons. 399 */ 400 if (config_valgrind) { 401 opt_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; 402 if (config_fill && opt_valgrind) { 403 opt_junk = false; 404 assert(opt_zero == false); 405 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 406 opt_redzone = true; 407 } 408 if (config_tcache && opt_valgrind) 409 opt_tcache = false; 410 } 411 412 for (i = 0; i < 3; i++) { 413 /* Get runtime configuration. */ 414 switch (i) { 415 case 0: 416 if (je_malloc_conf != NULL) { 417 /* 418 * Use options that were compiled into the 419 * program. 420 */ 421 opts = je_malloc_conf; 422 } else { 423 /* No configuration specified. */ 424 buf[0] = '\0'; 425 opts = buf; 426 } 427 break; 428 case 1: { 429 int linklen = 0; 430 #ifndef _WIN32 431 int saved_errno = errno; 432 const char *linkname = 433 # ifdef JEMALLOC_PREFIX 434 "/etc/"JEMALLOC_PREFIX"malloc.conf" 435 # else 436 "/etc/malloc.conf" 437 # endif 438 ; 439 440 /* 441 * Try to use the contents of the "/etc/malloc.conf" 442 * symbolic link's name. 443 */ 444 linklen = readlink(linkname, buf, sizeof(buf) - 1); 445 if (linklen == -1) { 446 /* No configuration specified. */ 447 linklen = 0; 448 /* restore errno */ 449 set_errno(saved_errno); 450 } 451 #endif 452 buf[linklen] = '\0'; 453 opts = buf; 454 break; 455 } case 2: { 456 const char *envname = 457 #ifdef JEMALLOC_PREFIX 458 JEMALLOC_CPREFIX"MALLOC_CONF" 459 #else 460 "MALLOC_CONF" 461 #endif 462 ; 463 464 if (issetugid() == 0 && (opts = getenv(envname)) != 465 NULL) { 466 /* 467 * Do nothing; opts is already initialized to 468 * the value of the MALLOC_CONF environment 469 * variable. 470 */ 471 } else { 472 /* No configuration specified. */ 473 buf[0] = '\0'; 474 opts = buf; 475 } 476 break; 477 } default: 478 not_reached(); 479 buf[0] = '\0'; 480 opts = buf; 481 } 482 483 while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, 484 &vlen) == false) { 485 #define CONF_HANDLE_BOOL(o, n) \ 486 if (sizeof(n)-1 == klen && strncmp(n, k, \ 487 klen) == 0) { \ 488 if (strncmp("true", v, vlen) == 0 && \ 489 vlen == sizeof("true")-1) \ 490 o = true; \ 491 else if (strncmp("false", v, vlen) == \ 492 0 && vlen == sizeof("false")-1) \ 493 o = false; \ 494 else { \ 495 malloc_conf_error( \ 496 "Invalid conf value", \ 497 k, klen, v, vlen); \ 498 } \ 499 continue; \ 500 } 501 #define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ 502 if (sizeof(n)-1 == klen && strncmp(n, k, \ 503 klen) == 0) { \ 504 uintmax_t um; \ 505 char *end; \ 506 \ 507 set_errno(0); \ 508 um = malloc_strtoumax(v, &end, 0); \ 509 if (get_errno() != 0 || (uintptr_t)end -\ 510 (uintptr_t)v != vlen) { \ 511 malloc_conf_error( \ 512 "Invalid conf value", \ 513 k, klen, v, vlen); \ 514 } else if (clip) { \ 515 if (min != 0 && um < min) \ 516 o = min; \ 517 else if (um > max) \ 518 o = max; \ 519 else \ 520 o = um; \ 521 } else { \ 522 if ((min != 0 && um < min) || \ 523 um > max) { \ 524 malloc_conf_error( \ 525 "Out-of-range " \ 526 "conf value", \ 527 k, klen, v, vlen); \ 528 } else \ 529 o = um; \ 530 } \ 531 continue; \ 532 } 533 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 534 if (sizeof(n)-1 == klen && strncmp(n, k, \ 535 klen) == 0) { \ 536 long l; \ 537 char *end; \ 538 \ 539 set_errno(0); \ 540 l = strtol(v, &end, 0); \ 541 if (get_errno() != 0 || (uintptr_t)end -\ 542 (uintptr_t)v != vlen) { \ 543 malloc_conf_error( \ 544 "Invalid conf value", \ 545 k, klen, v, vlen); \ 546 } else if (l < (ssize_t)min || l > \ 547 (ssize_t)max) { \ 548 malloc_conf_error( \ 549 "Out-of-range conf value", \ 550 k, klen, v, vlen); \ 551 } else \ 552 o = l; \ 553 continue; \ 554 } 555 #define CONF_HANDLE_CHAR_P(o, n, d) \ 556 if (sizeof(n)-1 == klen && strncmp(n, k, \ 557 klen) == 0) { \ 558 size_t cpylen = (vlen <= \ 559 sizeof(o)-1) ? vlen : \ 560 sizeof(o)-1; \ 561 strncpy(o, v, cpylen); \ 562 o[cpylen] = '\0'; \ 563 continue; \ 564 } 565 566 CONF_HANDLE_BOOL(opt_abort, "abort") 567 /* 568 * Chunks always require at least one header page, plus 569 * one data page in the absence of redzones, or three 570 * pages in the presence of redzones. In order to 571 * simplify options processing, fix the limit based on 572 * config_fill. 573 */ 574 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 575 (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1, 576 true) 577 if (strncmp("dss", k, klen) == 0) { 578 int i; 579 bool match = false; 580 for (i = 0; i < dss_prec_limit; i++) { 581 if (strncmp(dss_prec_names[i], v, vlen) 582 == 0) { 583 if (chunk_dss_prec_set(i)) { 584 malloc_conf_error( 585 "Error setting dss", 586 k, klen, v, vlen); 587 } else { 588 opt_dss = 589 dss_prec_names[i]; 590 match = true; 591 break; 592 } 593 } 594 } 595 if (match == false) { 596 malloc_conf_error("Invalid conf value", 597 k, klen, v, vlen); 598 } 599 continue; 600 } 601 CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, 602 SIZE_T_MAX, false) 603 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 604 -1, (sizeof(size_t) << 3) - 1) 605 CONF_HANDLE_BOOL(opt_stats_print, "stats_print") 606 if (config_fill) { 607 CONF_HANDLE_BOOL(opt_junk, "junk") 608 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 609 0, SIZE_T_MAX, false) 610 CONF_HANDLE_BOOL(opt_redzone, "redzone") 611 CONF_HANDLE_BOOL(opt_zero, "zero") 612 } 613 if (config_utrace) { 614 CONF_HANDLE_BOOL(opt_utrace, "utrace") 615 } 616 if (config_valgrind) { 617 CONF_HANDLE_BOOL(opt_valgrind, "valgrind") 618 } 619 if (config_xmalloc) { 620 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") 621 } 622 if (config_tcache) { 623 CONF_HANDLE_BOOL(opt_tcache, "tcache") 624 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 625 "lg_tcache_max", -1, 626 (sizeof(size_t) << 3) - 1) 627 } 628 if (config_prof) { 629 CONF_HANDLE_BOOL(opt_prof, "prof") 630 CONF_HANDLE_CHAR_P(opt_prof_prefix, 631 "prof_prefix", "jeprof") 632 CONF_HANDLE_BOOL(opt_prof_active, "prof_active") 633 CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, 634 "lg_prof_sample", 0, 635 (sizeof(uint64_t) << 3) - 1) 636 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") 637 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 638 "lg_prof_interval", -1, 639 (sizeof(uint64_t) << 3) - 1) 640 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") 641 CONF_HANDLE_BOOL(opt_prof_final, "prof_final") 642 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") 643 } 644 malloc_conf_error("Invalid conf pair", k, klen, v, 645 vlen); 646 #undef CONF_HANDLE_BOOL 647 #undef CONF_HANDLE_SIZE_T 648 #undef CONF_HANDLE_SSIZE_T 649 #undef CONF_HANDLE_CHAR_P 650 } 651 } 652 } 653 654 static bool 655 malloc_init_hard(void) 656 { 657 arena_t *init_arenas[1]; 658 659 malloc_mutex_lock(&init_lock); 660 if (malloc_initialized || IS_INITIALIZER) { 661 /* 662 * Another thread initialized the allocator before this one 663 * acquired init_lock, or this thread is the initializing 664 * thread, and it is recursively allocating. 665 */ 666 malloc_mutex_unlock(&init_lock); 667 return (false); 668 } 669 #ifdef JEMALLOC_THREADED_INIT 670 if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) { 671 /* Busy-wait until the initializing thread completes. */ 672 do { 673 malloc_mutex_unlock(&init_lock); 674 CPU_SPINWAIT; 675 malloc_mutex_lock(&init_lock); 676 } while (malloc_initialized == false); 677 malloc_mutex_unlock(&init_lock); 678 return (false); 679 } 680 #endif 681 malloc_initializer = INITIALIZER; 682 683 malloc_tsd_boot(); 684 if (config_prof) 685 prof_boot0(); 686 687 malloc_conf_init(); 688 689 if (opt_stats_print) { 690 /* Print statistics at exit. */ 691 if (atexit(stats_print_atexit) != 0) { 692 malloc_write("<jemalloc>: Error in atexit()\n"); 693 if (opt_abort) 694 abort(); 695 } 696 } 697 698 if (base_boot()) { 699 malloc_mutex_unlock(&init_lock); 700 return (true); 701 } 702 703 if (chunk_boot()) { 704 malloc_mutex_unlock(&init_lock); 705 return (true); 706 } 707 708 if (ctl_boot()) { 709 malloc_mutex_unlock(&init_lock); 710 return (true); 711 } 712 713 if (config_prof) 714 prof_boot1(); 715 716 arena_boot(); 717 718 if (config_tcache && tcache_boot0()) { 719 malloc_mutex_unlock(&init_lock); 720 return (true); 721 } 722 723 if (huge_boot()) { 724 malloc_mutex_unlock(&init_lock); 725 return (true); 726 } 727 728 if (malloc_mutex_init(&arenas_lock)) { 729 malloc_mutex_unlock(&init_lock); 730 return (true); 731 } 732 733 /* 734 * Create enough scaffolding to allow recursive allocation in 735 * malloc_ncpus(). 736 */ 737 narenas_total = narenas_auto = 1; 738 arenas = init_arenas; 739 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 740 741 /* 742 * Initialize one arena here. The rest are lazily created in 743 * choose_arena_hard(). 744 */ 745 arenas_extend(0); 746 if (arenas[0] == NULL) { 747 malloc_mutex_unlock(&init_lock); 748 return (true); 749 } 750 751 /* Initialize allocation counters before any allocations can occur. */ 752 if (config_stats && thread_allocated_tsd_boot()) { 753 malloc_mutex_unlock(&init_lock); 754 return (true); 755 } 756 757 if (arenas_tsd_boot()) { 758 malloc_mutex_unlock(&init_lock); 759 return (true); 760 } 761 762 if (config_tcache && tcache_boot1()) { 763 malloc_mutex_unlock(&init_lock); 764 return (true); 765 } 766 767 if (config_fill && quarantine_boot()) { 768 malloc_mutex_unlock(&init_lock); 769 return (true); 770 } 771 772 if (config_prof && prof_boot2()) { 773 malloc_mutex_unlock(&init_lock); 774 return (true); 775 } 776 777 malloc_mutex_unlock(&init_lock); 778 /**********************************************************************/ 779 /* Recursive allocation may follow. */ 780 781 ncpus = malloc_ncpus(); 782 783 #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ 784 && !defined(_WIN32)) 785 /* LinuxThreads's pthread_atfork() allocates. */ 786 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 787 jemalloc_postfork_child) != 0) { 788 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 789 if (opt_abort) 790 abort(); 791 } 792 #endif 793 794 /* Done recursively allocating. */ 795 /**********************************************************************/ 796 malloc_mutex_lock(&init_lock); 797 798 if (mutex_boot()) { 799 malloc_mutex_unlock(&init_lock); 800 return (true); 801 } 802 803 if (opt_narenas == 0) { 804 /* 805 * For SMP systems, create more than one arena per CPU by 806 * default. 807 */ 808 if (ncpus > 1) 809 opt_narenas = ncpus << 2; 810 else 811 opt_narenas = 1; 812 } 813 narenas_auto = opt_narenas; 814 /* 815 * Make sure that the arenas array can be allocated. In practice, this 816 * limit is enough to allow the allocator to function, but the ctl 817 * machinery will fail to allocate memory at far lower limits. 818 */ 819 if (narenas_auto > chunksize / sizeof(arena_t *)) { 820 narenas_auto = chunksize / sizeof(arena_t *); 821 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 822 narenas_auto); 823 } 824 narenas_total = narenas_auto; 825 826 /* Allocate and initialize arenas. */ 827 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total); 828 if (arenas == NULL) { 829 malloc_mutex_unlock(&init_lock); 830 return (true); 831 } 832 /* 833 * Zero the array. In practice, this should always be pre-zeroed, 834 * since it was just mmap()ed, but let's be sure. 835 */ 836 memset(arenas, 0, sizeof(arena_t *) * narenas_total); 837 /* Copy the pointer to the one arena that was already initialized. */ 838 arenas[0] = init_arenas[0]; 839 840 malloc_initialized = true; 841 malloc_mutex_unlock(&init_lock); 842 843 return (false); 844 } 845 846 /* 847 * End initialization functions. 848 */ 849 /******************************************************************************/ 850 /* 851 * Begin malloc(3)-compatible functions. 852 */ 853 854 static void * 855 imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) 856 { 857 void *p; 858 859 if (cnt == NULL) 860 return (NULL); 861 if (prof_promote && usize <= SMALL_MAXCLASS) { 862 p = imalloc(SMALL_MAXCLASS+1); 863 if (p == NULL) 864 return (NULL); 865 arena_prof_promoted(p, usize); 866 } else 867 p = imalloc(usize); 868 869 return (p); 870 } 871 872 JEMALLOC_ALWAYS_INLINE_C void * 873 imalloc_prof(size_t usize, prof_thr_cnt_t *cnt) 874 { 875 void *p; 876 877 if ((uintptr_t)cnt != (uintptr_t)1U) 878 p = imalloc_prof_sample(usize, cnt); 879 else 880 p = imalloc(usize); 881 if (p == NULL) 882 return (NULL); 883 prof_malloc(p, usize, cnt); 884 885 return (p); 886 } 887 888 /* 889 * MALLOC_BODY() is a macro rather than a function because its contents are in 890 * the fast path, but inlining would cause reliability issues when determining 891 * how many frames to discard from heap profiling backtraces. 892 */ 893 #define MALLOC_BODY(ret, size, usize) do { \ 894 if (malloc_init()) \ 895 ret = NULL; \ 896 else { \ 897 if (config_prof && opt_prof) { \ 898 prof_thr_cnt_t *cnt; \ 899 \ 900 usize = s2u(size); \ 901 /* \ 902 * Call PROF_ALLOC_PREP() here rather than in \ 903 * imalloc_prof() so that imalloc_prof() can be \ 904 * inlined without introducing uncertainty \ 905 * about the number of backtrace frames to \ 906 * ignore. imalloc_prof() is in the fast path \ 907 * when heap profiling is enabled, so inlining \ 908 * is critical to performance. (For \ 909 * consistency all callers of PROF_ALLOC_PREP() \ 910 * are structured similarly, even though e.g. \ 911 * realloc() isn't called enough for inlining \ 912 * to be critical.) \ 913 */ \ 914 PROF_ALLOC_PREP(1, usize, cnt); \ 915 ret = imalloc_prof(usize, cnt); \ 916 } else { \ 917 if (config_stats || (config_valgrind && \ 918 opt_valgrind)) \ 919 usize = s2u(size); \ 920 ret = imalloc(size); \ 921 } \ 922 } \ 923 } while (0) 924 925 void * 926 je_malloc(size_t size) 927 { 928 void *ret; 929 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 930 931 if (size == 0) 932 size = 1; 933 934 MALLOC_BODY(ret, size, usize); 935 936 if (ret == NULL) { 937 if (config_xmalloc && opt_xmalloc) { 938 malloc_write("<jemalloc>: Error in malloc(): " 939 "out of memory\n"); 940 abort(); 941 } 942 set_errno(ENOMEM); 943 } 944 if (config_stats && ret != NULL) { 945 assert(usize == isalloc(ret, config_prof)); 946 thread_allocated_tsd_get()->allocated += usize; 947 } 948 UTRACE(0, size, ret); 949 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); 950 return (ret); 951 } 952 953 static void * 954 imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) 955 { 956 void *p; 957 958 if (cnt == NULL) 959 return (NULL); 960 if (prof_promote && usize <= SMALL_MAXCLASS) { 961 assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0); 962 p = ipalloc(sa2u(SMALL_MAXCLASS+1, alignment), alignment, 963 false); 964 if (p == NULL) 965 return (NULL); 966 arena_prof_promoted(p, usize); 967 } else 968 p = ipalloc(usize, alignment, false); 969 970 return (p); 971 } 972 973 JEMALLOC_ALWAYS_INLINE_C void * 974 imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) 975 { 976 void *p; 977 978 if ((uintptr_t)cnt != (uintptr_t)1U) 979 p = imemalign_prof_sample(alignment, usize, cnt); 980 else 981 p = ipalloc(usize, alignment, false); 982 if (p == NULL) 983 return (NULL); 984 prof_malloc(p, usize, cnt); 985 986 return (p); 987 } 988 989 JEMALLOC_ATTR(nonnull(1)) 990 #ifdef JEMALLOC_PROF 991 /* 992 * Avoid any uncertainty as to how many backtrace frames to ignore in 993 * PROF_ALLOC_PREP(). 994 */ 995 JEMALLOC_NOINLINE 996 #endif 997 static int 998 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) 999 { 1000 int ret; 1001 size_t usize; 1002 void *result; 1003 1004 assert(min_alignment != 0); 1005 1006 if (malloc_init()) { 1007 result = NULL; 1008 goto label_oom; 1009 } else { 1010 if (size == 0) 1011 size = 1; 1012 1013 /* Make sure that alignment is a large enough power of 2. */ 1014 if (((alignment - 1) & alignment) != 0 1015 || (alignment < min_alignment)) { 1016 if (config_xmalloc && opt_xmalloc) { 1017 malloc_write("<jemalloc>: Error allocating " 1018 "aligned memory: invalid alignment\n"); 1019 abort(); 1020 } 1021 result = NULL; 1022 ret = EINVAL; 1023 goto label_return; 1024 } 1025 1026 usize = sa2u(size, alignment); 1027 if (usize == 0) { 1028 result = NULL; 1029 goto label_oom; 1030 } 1031 1032 if (config_prof && opt_prof) { 1033 prof_thr_cnt_t *cnt; 1034 1035 PROF_ALLOC_PREP(2, usize, cnt); 1036 result = imemalign_prof(alignment, usize, cnt); 1037 } else 1038 result = ipalloc(usize, alignment, false); 1039 if (result == NULL) 1040 goto label_oom; 1041 } 1042 1043 *memptr = result; 1044 ret = 0; 1045 label_return: 1046 if (config_stats && result != NULL) { 1047 assert(usize == isalloc(result, config_prof)); 1048 thread_allocated_tsd_get()->allocated += usize; 1049 } 1050 UTRACE(0, size, result); 1051 return (ret); 1052 label_oom: 1053 assert(result == NULL); 1054 if (config_xmalloc && opt_xmalloc) { 1055 malloc_write("<jemalloc>: Error allocating aligned memory: " 1056 "out of memory\n"); 1057 abort(); 1058 } 1059 ret = ENOMEM; 1060 goto label_return; 1061 } 1062 1063 int 1064 je_posix_memalign(void **memptr, size_t alignment, size_t size) 1065 { 1066 int ret = imemalign(memptr, alignment, size, sizeof(void *)); 1067 JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, 1068 config_prof), false); 1069 return (ret); 1070 } 1071 1072 void * 1073 je_aligned_alloc(size_t alignment, size_t size) 1074 { 1075 void *ret; 1076 int err; 1077 1078 if ((err = imemalign(&ret, alignment, size, 1)) != 0) { 1079 ret = NULL; 1080 set_errno(err); 1081 } 1082 JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), 1083 false); 1084 return (ret); 1085 } 1086 1087 static void * 1088 icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) 1089 { 1090 void *p; 1091 1092 if (cnt == NULL) 1093 return (NULL); 1094 if (prof_promote && usize <= SMALL_MAXCLASS) { 1095 p = icalloc(SMALL_MAXCLASS+1); 1096 if (p == NULL) 1097 return (NULL); 1098 arena_prof_promoted(p, usize); 1099 } else 1100 p = icalloc(usize); 1101 1102 return (p); 1103 } 1104 1105 JEMALLOC_ALWAYS_INLINE_C void * 1106 icalloc_prof(size_t usize, prof_thr_cnt_t *cnt) 1107 { 1108 void *p; 1109 1110 if ((uintptr_t)cnt != (uintptr_t)1U) 1111 p = icalloc_prof_sample(usize, cnt); 1112 else 1113 p = icalloc(usize); 1114 if (p == NULL) 1115 return (NULL); 1116 prof_malloc(p, usize, cnt); 1117 1118 return (p); 1119 } 1120 1121 void * 1122 je_calloc(size_t num, size_t size) 1123 { 1124 void *ret; 1125 size_t num_size; 1126 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1127 1128 if (malloc_init()) { 1129 num_size = 0; 1130 ret = NULL; 1131 goto label_return; 1132 } 1133 1134 num_size = num * size; 1135 if (num_size == 0) { 1136 if (num == 0 || size == 0) 1137 num_size = 1; 1138 else { 1139 ret = NULL; 1140 goto label_return; 1141 } 1142 /* 1143 * Try to avoid division here. We know that it isn't possible to 1144 * overflow during multiplication if neither operand uses any of the 1145 * most significant half of the bits in a size_t. 1146 */ 1147 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) 1148 && (num_size / size != num)) { 1149 /* size_t overflow. */ 1150 ret = NULL; 1151 goto label_return; 1152 } 1153 1154 if (config_prof && opt_prof) { 1155 prof_thr_cnt_t *cnt; 1156 1157 usize = s2u(num_size); 1158 PROF_ALLOC_PREP(1, usize, cnt); 1159 ret = icalloc_prof(usize, cnt); 1160 } else { 1161 if (config_stats || (config_valgrind && opt_valgrind)) 1162 usize = s2u(num_size); 1163 ret = icalloc(num_size); 1164 } 1165 1166 label_return: 1167 if (ret == NULL) { 1168 if (config_xmalloc && opt_xmalloc) { 1169 malloc_write("<jemalloc>: Error in calloc(): out of " 1170 "memory\n"); 1171 abort(); 1172 } 1173 set_errno(ENOMEM); 1174 } 1175 if (config_stats && ret != NULL) { 1176 assert(usize == isalloc(ret, config_prof)); 1177 thread_allocated_tsd_get()->allocated += usize; 1178 } 1179 UTRACE(0, num_size, ret); 1180 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); 1181 return (ret); 1182 } 1183 1184 static void * 1185 irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt) 1186 { 1187 void *p; 1188 1189 if (cnt == NULL) 1190 return (NULL); 1191 if (prof_promote && usize <= SMALL_MAXCLASS) { 1192 p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false); 1193 if (p == NULL) 1194 return (NULL); 1195 arena_prof_promoted(p, usize); 1196 } else 1197 p = iralloc(oldptr, usize, 0, 0, false); 1198 1199 return (p); 1200 } 1201 1202 JEMALLOC_ALWAYS_INLINE_C void * 1203 irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt) 1204 { 1205 void *p; 1206 prof_ctx_t *old_ctx; 1207 1208 old_ctx = prof_ctx_get(oldptr); 1209 if ((uintptr_t)cnt != (uintptr_t)1U) 1210 p = irealloc_prof_sample(oldptr, usize, cnt); 1211 else 1212 p = iralloc(oldptr, usize, 0, 0, false); 1213 if (p == NULL) 1214 return (NULL); 1215 prof_realloc(p, usize, cnt, old_usize, old_ctx); 1216 1217 return (p); 1218 } 1219 1220 JEMALLOC_INLINE_C void 1221 ifree(void *ptr) 1222 { 1223 size_t usize; 1224 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1225 1226 assert(ptr != NULL); 1227 assert(malloc_initialized || IS_INITIALIZER); 1228 1229 if (config_prof && opt_prof) { 1230 usize = isalloc(ptr, config_prof); 1231 prof_free(ptr, usize); 1232 } else if (config_stats || config_valgrind) 1233 usize = isalloc(ptr, config_prof); 1234 if (config_stats) 1235 thread_allocated_tsd_get()->deallocated += usize; 1236 if (config_valgrind && opt_valgrind) 1237 rzsize = p2rz(ptr); 1238 iqalloc(ptr); 1239 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1240 } 1241 1242 void * 1243 je_realloc(void *ptr, size_t size) 1244 { 1245 void *ret; 1246 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1247 size_t old_usize = 0; 1248 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1249 1250 if (size == 0) { 1251 if (ptr != NULL) { 1252 /* realloc(ptr, 0) is equivalent to free(ptr). */ 1253 UTRACE(ptr, 0, 0); 1254 ifree(ptr); 1255 return (NULL); 1256 } 1257 size = 1; 1258 } 1259 1260 if (ptr != NULL) { 1261 assert(malloc_initialized || IS_INITIALIZER); 1262 malloc_thread_init(); 1263 1264 if ((config_prof && opt_prof) || config_stats || 1265 (config_valgrind && opt_valgrind)) 1266 old_usize = isalloc(ptr, config_prof); 1267 if (config_valgrind && opt_valgrind) 1268 old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); 1269 1270 if (config_prof && opt_prof) { 1271 prof_thr_cnt_t *cnt; 1272 1273 usize = s2u(size); 1274 PROF_ALLOC_PREP(1, usize, cnt); 1275 ret = irealloc_prof(ptr, old_usize, usize, cnt); 1276 } else { 1277 if (config_stats || (config_valgrind && opt_valgrind)) 1278 usize = s2u(size); 1279 ret = iralloc(ptr, size, 0, 0, false); 1280 } 1281 } else { 1282 /* realloc(NULL, size) is equivalent to malloc(size). */ 1283 MALLOC_BODY(ret, size, usize); 1284 } 1285 1286 if (ret == NULL) { 1287 if (config_xmalloc && opt_xmalloc) { 1288 malloc_write("<jemalloc>: Error in realloc(): " 1289 "out of memory\n"); 1290 abort(); 1291 } 1292 set_errno(ENOMEM); 1293 } 1294 if (config_stats && ret != NULL) { 1295 thread_allocated_t *ta; 1296 assert(usize == isalloc(ret, config_prof)); 1297 ta = thread_allocated_tsd_get(); 1298 ta->allocated += usize; 1299 ta->deallocated += old_usize; 1300 } 1301 UTRACE(ptr, size, ret); 1302 JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_usize, old_rzsize, 1303 false); 1304 return (ret); 1305 } 1306 1307 void 1308 je_free(void *ptr) 1309 { 1310 1311 UTRACE(ptr, 0, 0); 1312 if (ptr != NULL) 1313 ifree(ptr); 1314 } 1315 1316 /* 1317 * End malloc(3)-compatible functions. 1318 */ 1319 /******************************************************************************/ 1320 /* 1321 * Begin non-standard override functions. 1322 */ 1323 1324 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 1325 void * 1326 je_memalign(size_t alignment, size_t size) 1327 { 1328 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1329 imemalign(&ret, alignment, size, 1); 1330 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1331 return (ret); 1332 } 1333 #endif 1334 1335 #ifdef JEMALLOC_OVERRIDE_VALLOC 1336 void * 1337 je_valloc(size_t size) 1338 { 1339 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1340 imemalign(&ret, PAGE, size, 1); 1341 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1342 return (ret); 1343 } 1344 #endif 1345 1346 /* 1347 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 1348 * #define je_malloc malloc 1349 */ 1350 #define malloc_is_malloc 1 1351 #define is_malloc_(a) malloc_is_ ## a 1352 #define is_malloc(a) is_malloc_(a) 1353 1354 #if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__)) 1355 /* 1356 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 1357 * to inconsistently reference libc's malloc(3)-compatible functions 1358 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 1359 * 1360 * These definitions interpose hooks in glibc. The functions are actually 1361 * passed an extra argument for the caller return address, which will be 1362 * ignored. 1363 */ 1364 JEMALLOC_EXPORT void (* __free_hook)(void *ptr) = je_free; 1365 JEMALLOC_EXPORT void *(* __malloc_hook)(size_t size) = je_malloc; 1366 JEMALLOC_EXPORT void *(* __realloc_hook)(void *ptr, size_t size) = je_realloc; 1367 JEMALLOC_EXPORT void *(* __memalign_hook)(size_t alignment, size_t size) = 1368 je_memalign; 1369 #endif 1370 1371 /* 1372 * End non-standard override functions. 1373 */ 1374 /******************************************************************************/ 1375 /* 1376 * Begin non-standard functions. 1377 */ 1378 1379 JEMALLOC_ALWAYS_INLINE_C void * 1380 imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, 1381 arena_t *arena) 1382 { 1383 1384 assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, 1385 alignment))); 1386 1387 if (alignment != 0) 1388 return (ipalloct(usize, alignment, zero, try_tcache, arena)); 1389 else if (zero) 1390 return (icalloct(usize, try_tcache, arena)); 1391 else 1392 return (imalloct(usize, try_tcache, arena)); 1393 } 1394 1395 static void * 1396 imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache, 1397 arena_t *arena, prof_thr_cnt_t *cnt) 1398 { 1399 void *p; 1400 1401 if (cnt == NULL) 1402 return (NULL); 1403 if (prof_promote && usize <= SMALL_MAXCLASS) { 1404 size_t usize_promoted = (alignment == 0) ? 1405 s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment); 1406 assert(usize_promoted != 0); 1407 p = imallocx(usize_promoted, alignment, zero, try_tcache, 1408 arena); 1409 if (p == NULL) 1410 return (NULL); 1411 arena_prof_promoted(p, usize); 1412 } else 1413 p = imallocx(usize, alignment, zero, try_tcache, arena); 1414 1415 return (p); 1416 } 1417 1418 JEMALLOC_ALWAYS_INLINE_C void * 1419 imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache, 1420 arena_t *arena, prof_thr_cnt_t *cnt) 1421 { 1422 void *p; 1423 1424 if ((uintptr_t)cnt != (uintptr_t)1U) { 1425 p = imallocx_prof_sample(usize, alignment, zero, try_tcache, 1426 arena, cnt); 1427 } else 1428 p = imallocx(usize, alignment, zero, try_tcache, arena); 1429 if (p == NULL) 1430 return (NULL); 1431 prof_malloc(p, usize, cnt); 1432 1433 return (p); 1434 } 1435 1436 void * 1437 je_mallocx(size_t size, int flags) 1438 { 1439 void *p; 1440 size_t usize; 1441 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1442 & (SIZE_T_MAX-1)); 1443 bool zero = flags & MALLOCX_ZERO; 1444 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1445 arena_t *arena; 1446 bool try_tcache; 1447 1448 assert(size != 0); 1449 1450 if (malloc_init()) 1451 goto label_oom; 1452 1453 if (arena_ind != UINT_MAX) { 1454 arena = arenas[arena_ind]; 1455 try_tcache = false; 1456 } else { 1457 arena = NULL; 1458 try_tcache = true; 1459 } 1460 1461 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1462 assert(usize != 0); 1463 1464 if (config_prof && opt_prof) { 1465 prof_thr_cnt_t *cnt; 1466 1467 PROF_ALLOC_PREP(1, usize, cnt); 1468 p = imallocx_prof(usize, alignment, zero, try_tcache, arena, 1469 cnt); 1470 } else 1471 p = imallocx(usize, alignment, zero, try_tcache, arena); 1472 if (p == NULL) 1473 goto label_oom; 1474 1475 if (config_stats) { 1476 assert(usize == isalloc(p, config_prof)); 1477 thread_allocated_tsd_get()->allocated += usize; 1478 } 1479 UTRACE(0, size, p); 1480 JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); 1481 return (p); 1482 label_oom: 1483 if (config_xmalloc && opt_xmalloc) { 1484 malloc_write("<jemalloc>: Error in mallocx(): out of memory\n"); 1485 abort(); 1486 } 1487 UTRACE(0, size, 0); 1488 return (NULL); 1489 } 1490 1491 static void * 1492 irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, 1493 bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena, 1494 prof_thr_cnt_t *cnt) 1495 { 1496 void *p; 1497 1498 if (cnt == NULL) 1499 return (NULL); 1500 if (prof_promote && usize <= SMALL_MAXCLASS) { 1501 p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= 1502 size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero, 1503 try_tcache_alloc, try_tcache_dalloc, arena); 1504 if (p == NULL) 1505 return (NULL); 1506 arena_prof_promoted(p, usize); 1507 } else { 1508 p = iralloct(oldptr, size, 0, alignment, zero, 1509 try_tcache_alloc, try_tcache_dalloc, arena); 1510 } 1511 1512 return (p); 1513 } 1514 1515 JEMALLOC_ALWAYS_INLINE_C void * 1516 irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, 1517 size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, 1518 arena_t *arena, prof_thr_cnt_t *cnt) 1519 { 1520 void *p; 1521 prof_ctx_t *old_ctx; 1522 1523 old_ctx = prof_ctx_get(oldptr); 1524 if ((uintptr_t)cnt != (uintptr_t)1U) 1525 p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, 1526 try_tcache_alloc, try_tcache_dalloc, arena, cnt); 1527 else { 1528 p = iralloct(oldptr, size, 0, alignment, zero, 1529 try_tcache_alloc, try_tcache_dalloc, arena); 1530 } 1531 if (p == NULL) 1532 return (NULL); 1533 1534 if (p == oldptr && alignment != 0) { 1535 /* 1536 * The allocation did not move, so it is possible that the size 1537 * class is smaller than would guarantee the requested 1538 * alignment, and that the alignment constraint was 1539 * serendipitously satisfied. Additionally, old_usize may not 1540 * be the same as the current usize because of in-place large 1541 * reallocation. Therefore, query the actual value of usize. 1542 */ 1543 *usize = isalloc(p, config_prof); 1544 } 1545 prof_realloc(p, *usize, cnt, old_usize, old_ctx); 1546 1547 return (p); 1548 } 1549 1550 void * 1551 je_rallocx(void *ptr, size_t size, int flags) 1552 { 1553 void *p; 1554 size_t usize, old_usize; 1555 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1556 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1557 & (SIZE_T_MAX-1)); 1558 bool zero = flags & MALLOCX_ZERO; 1559 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1560 bool try_tcache_alloc, try_tcache_dalloc; 1561 arena_t *arena; 1562 1563 assert(ptr != NULL); 1564 assert(size != 0); 1565 assert(malloc_initialized || IS_INITIALIZER); 1566 malloc_thread_init(); 1567 1568 if (arena_ind != UINT_MAX) { 1569 arena_chunk_t *chunk; 1570 try_tcache_alloc = false; 1571 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1572 try_tcache_dalloc = (chunk == ptr || chunk->arena != 1573 arenas[arena_ind]); 1574 arena = arenas[arena_ind]; 1575 } else { 1576 try_tcache_alloc = true; 1577 try_tcache_dalloc = true; 1578 arena = NULL; 1579 } 1580 1581 if ((config_prof && opt_prof) || config_stats || 1582 (config_valgrind && opt_valgrind)) 1583 old_usize = isalloc(ptr, config_prof); 1584 if (config_valgrind && opt_valgrind) 1585 old_rzsize = u2rz(old_usize); 1586 1587 if (config_prof && opt_prof) { 1588 prof_thr_cnt_t *cnt; 1589 1590 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1591 assert(usize != 0); 1592 PROF_ALLOC_PREP(1, usize, cnt); 1593 p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero, 1594 try_tcache_alloc, try_tcache_dalloc, arena, cnt); 1595 if (p == NULL) 1596 goto label_oom; 1597 } else { 1598 p = iralloct(ptr, size, 0, alignment, zero, try_tcache_alloc, 1599 try_tcache_dalloc, arena); 1600 if (p == NULL) 1601 goto label_oom; 1602 if (config_stats || (config_valgrind && opt_valgrind)) 1603 usize = isalloc(p, config_prof); 1604 } 1605 1606 if (config_stats) { 1607 thread_allocated_t *ta; 1608 ta = thread_allocated_tsd_get(); 1609 ta->allocated += usize; 1610 ta->deallocated += old_usize; 1611 } 1612 UTRACE(ptr, size, p); 1613 JEMALLOC_VALGRIND_REALLOC(p, usize, ptr, old_usize, old_rzsize, zero); 1614 return (p); 1615 label_oom: 1616 if (config_xmalloc && opt_xmalloc) { 1617 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 1618 abort(); 1619 } 1620 UTRACE(ptr, size, 0); 1621 return (NULL); 1622 } 1623 1624 JEMALLOC_ALWAYS_INLINE_C size_t 1625 ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, 1626 size_t alignment, bool zero, arena_t *arena) 1627 { 1628 size_t usize; 1629 1630 if (ixalloc(ptr, size, extra, alignment, zero)) 1631 return (old_usize); 1632 usize = isalloc(ptr, config_prof); 1633 1634 return (usize); 1635 } 1636 1637 static size_t 1638 ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, 1639 size_t alignment, size_t max_usize, bool zero, arena_t *arena, 1640 prof_thr_cnt_t *cnt) 1641 { 1642 size_t usize; 1643 1644 if (cnt == NULL) 1645 return (old_usize); 1646 /* Use minimum usize to determine whether promotion may happen. */ 1647 if (prof_promote && ((alignment == 0) ? s2u(size) : sa2u(size, 1648 alignment)) <= SMALL_MAXCLASS) { 1649 if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= 1650 size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), 1651 alignment, zero)) 1652 return (old_usize); 1653 usize = isalloc(ptr, config_prof); 1654 if (max_usize < PAGE) 1655 arena_prof_promoted(ptr, usize); 1656 } else { 1657 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 1658 zero, arena); 1659 } 1660 1661 return (usize); 1662 } 1663 1664 JEMALLOC_ALWAYS_INLINE_C size_t 1665 ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, 1666 size_t alignment, size_t max_usize, bool zero, arena_t *arena, 1667 prof_thr_cnt_t *cnt) 1668 { 1669 size_t usize; 1670 prof_ctx_t *old_ctx; 1671 1672 old_ctx = prof_ctx_get(ptr); 1673 if ((uintptr_t)cnt != (uintptr_t)1U) { 1674 usize = ixallocx_prof_sample(ptr, old_usize, size, extra, 1675 alignment, zero, max_usize, arena, cnt); 1676 } else { 1677 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 1678 zero, arena); 1679 } 1680 if (usize == old_usize) 1681 return (usize); 1682 prof_realloc(ptr, usize, cnt, old_usize, old_ctx); 1683 1684 return (usize); 1685 } 1686 1687 size_t 1688 je_xallocx(void *ptr, size_t size, size_t extra, int flags) 1689 { 1690 size_t usize, old_usize; 1691 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1692 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1693 & (SIZE_T_MAX-1)); 1694 bool zero = flags & MALLOCX_ZERO; 1695 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1696 arena_t *arena; 1697 1698 assert(ptr != NULL); 1699 assert(size != 0); 1700 assert(SIZE_T_MAX - size >= extra); 1701 assert(malloc_initialized || IS_INITIALIZER); 1702 malloc_thread_init(); 1703 1704 if (arena_ind != UINT_MAX) 1705 arena = arenas[arena_ind]; 1706 else 1707 arena = NULL; 1708 1709 old_usize = isalloc(ptr, config_prof); 1710 if (config_valgrind && opt_valgrind) 1711 old_rzsize = u2rz(old_usize); 1712 1713 if (config_prof && opt_prof) { 1714 prof_thr_cnt_t *cnt; 1715 /* 1716 * usize isn't knowable before ixalloc() returns when extra is 1717 * non-zero. Therefore, compute its maximum possible value and 1718 * use that in PROF_ALLOC_PREP() to decide whether to capture a 1719 * backtrace. prof_realloc() will use the actual usize to 1720 * decide whether to sample. 1721 */ 1722 size_t max_usize = (alignment == 0) ? s2u(size+extra) : 1723 sa2u(size+extra, alignment); 1724 PROF_ALLOC_PREP(1, max_usize, cnt); 1725 usize = ixallocx_prof(ptr, old_usize, size, extra, alignment, 1726 max_usize, zero, arena, cnt); 1727 } else { 1728 usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, 1729 zero, arena); 1730 } 1731 if (usize == old_usize) 1732 goto label_not_resized; 1733 1734 if (config_stats) { 1735 thread_allocated_t *ta; 1736 ta = thread_allocated_tsd_get(); 1737 ta->allocated += usize; 1738 ta->deallocated += old_usize; 1739 } 1740 JEMALLOC_VALGRIND_REALLOC(ptr, usize, ptr, old_usize, old_rzsize, zero); 1741 label_not_resized: 1742 UTRACE(ptr, size, ptr); 1743 return (usize); 1744 } 1745 1746 size_t 1747 je_sallocx(const void *ptr, int flags) 1748 { 1749 size_t usize; 1750 1751 assert(malloc_initialized || IS_INITIALIZER); 1752 malloc_thread_init(); 1753 1754 if (config_ivsalloc) 1755 usize = ivsalloc(ptr, config_prof); 1756 else { 1757 assert(ptr != NULL); 1758 usize = isalloc(ptr, config_prof); 1759 } 1760 1761 return (usize); 1762 } 1763 1764 void 1765 je_dallocx(void *ptr, int flags) 1766 { 1767 size_t usize; 1768 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1769 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1770 bool try_tcache; 1771 1772 assert(ptr != NULL); 1773 assert(malloc_initialized || IS_INITIALIZER); 1774 1775 if (arena_ind != UINT_MAX) { 1776 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1777 try_tcache = (chunk == ptr || chunk->arena != 1778 arenas[arena_ind]); 1779 } else 1780 try_tcache = true; 1781 1782 UTRACE(ptr, 0, 0); 1783 if (config_stats || config_valgrind) 1784 usize = isalloc(ptr, config_prof); 1785 if (config_prof && opt_prof) { 1786 if (config_stats == false && config_valgrind == false) 1787 usize = isalloc(ptr, config_prof); 1788 prof_free(ptr, usize); 1789 } 1790 if (config_stats) 1791 thread_allocated_tsd_get()->deallocated += usize; 1792 if (config_valgrind && opt_valgrind) 1793 rzsize = p2rz(ptr); 1794 iqalloct(ptr, try_tcache); 1795 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1796 } 1797 1798 size_t 1799 je_nallocx(size_t size, int flags) 1800 { 1801 size_t usize; 1802 size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) 1803 & (SIZE_T_MAX-1)); 1804 1805 assert(size != 0); 1806 1807 if (malloc_init()) 1808 return (0); 1809 1810 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1811 assert(usize != 0); 1812 return (usize); 1813 } 1814 1815 int 1816 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 1817 size_t newlen) 1818 { 1819 1820 if (malloc_init()) 1821 return (EAGAIN); 1822 1823 return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 1824 } 1825 1826 int 1827 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 1828 { 1829 1830 if (malloc_init()) 1831 return (EAGAIN); 1832 1833 return (ctl_nametomib(name, mibp, miblenp)); 1834 } 1835 1836 int 1837 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1838 void *newp, size_t newlen) 1839 { 1840 1841 if (malloc_init()) 1842 return (EAGAIN); 1843 1844 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 1845 } 1846 1847 void 1848 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 1849 const char *opts) 1850 { 1851 1852 stats_print(write_cb, cbopaque, opts); 1853 } 1854 1855 size_t 1856 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 1857 { 1858 size_t ret; 1859 1860 assert(malloc_initialized || IS_INITIALIZER); 1861 malloc_thread_init(); 1862 1863 if (config_ivsalloc) 1864 ret = ivsalloc(ptr, config_prof); 1865 else 1866 ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; 1867 1868 return (ret); 1869 } 1870 1871 /* 1872 * End non-standard functions. 1873 */ 1874 /******************************************************************************/ 1875 /* 1876 * Begin experimental functions. 1877 */ 1878 #ifdef JEMALLOC_EXPERIMENTAL 1879 1880 int 1881 je_allocm(void **ptr, size_t *rsize, size_t size, int flags) 1882 { 1883 void *p; 1884 1885 assert(ptr != NULL); 1886 1887 p = je_mallocx(size, flags); 1888 if (p == NULL) 1889 return (ALLOCM_ERR_OOM); 1890 if (rsize != NULL) 1891 *rsize = isalloc(p, config_prof); 1892 *ptr = p; 1893 return (ALLOCM_SUCCESS); 1894 } 1895 1896 int 1897 je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) 1898 { 1899 int ret; 1900 bool no_move = flags & ALLOCM_NO_MOVE; 1901 1902 assert(ptr != NULL); 1903 assert(*ptr != NULL); 1904 assert(size != 0); 1905 assert(SIZE_T_MAX - size >= extra); 1906 1907 if (no_move) { 1908 size_t usize = je_xallocx(*ptr, size, extra, flags); 1909 ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; 1910 if (rsize != NULL) 1911 *rsize = usize; 1912 } else { 1913 void *p = je_rallocx(*ptr, size+extra, flags); 1914 if (p != NULL) { 1915 *ptr = p; 1916 ret = ALLOCM_SUCCESS; 1917 } else 1918 ret = ALLOCM_ERR_OOM; 1919 if (rsize != NULL) 1920 *rsize = isalloc(*ptr, config_prof); 1921 } 1922 return (ret); 1923 } 1924 1925 int 1926 je_sallocm(const void *ptr, size_t *rsize, int flags) 1927 { 1928 1929 assert(rsize != NULL); 1930 *rsize = je_sallocx(ptr, flags); 1931 return (ALLOCM_SUCCESS); 1932 } 1933 1934 int 1935 je_dallocm(void *ptr, int flags) 1936 { 1937 1938 je_dallocx(ptr, flags); 1939 return (ALLOCM_SUCCESS); 1940 } 1941 1942 int 1943 je_nallocm(size_t *rsize, size_t size, int flags) 1944 { 1945 size_t usize; 1946 1947 usize = je_nallocx(size, flags); 1948 if (usize == 0) 1949 return (ALLOCM_ERR_OOM); 1950 if (rsize != NULL) 1951 *rsize = usize; 1952 return (ALLOCM_SUCCESS); 1953 } 1954 1955 #endif 1956 /* 1957 * End experimental functions. 1958 */ 1959 /******************************************************************************/ 1960 /* 1961 * The following functions are used by threading libraries for protection of 1962 * malloc during fork(). 1963 */ 1964 1965 /* 1966 * If an application creates a thread before doing any allocation in the main 1967 * thread, then calls fork(2) in the main thread followed by memory allocation 1968 * in the child process, a race can occur that results in deadlock within the 1969 * child: the main thread may have forked while the created thread had 1970 * partially initialized the allocator. Ordinarily jemalloc prevents 1971 * fork/malloc races via the following functions it registers during 1972 * initialization using pthread_atfork(), but of course that does no good if 1973 * the allocator isn't fully initialized at fork time. The following library 1974 * constructor is a partial solution to this problem. It may still possible to 1975 * trigger the deadlock described above, but doing so would involve forking via 1976 * a library constructor that runs before jemalloc's runs. 1977 */ 1978 JEMALLOC_ATTR(constructor) 1979 static void 1980 jemalloc_constructor(void) 1981 { 1982 1983 malloc_init(); 1984 } 1985 1986 #ifndef JEMALLOC_MUTEX_INIT_CB 1987 void 1988 jemalloc_prefork(void) 1989 #else 1990 JEMALLOC_EXPORT void 1991 _malloc_prefork(void) 1992 #endif 1993 { 1994 unsigned i; 1995 1996 #ifdef JEMALLOC_MUTEX_INIT_CB 1997 if (malloc_initialized == false) 1998 return; 1999 #endif 2000 assert(malloc_initialized); 2001 2002 /* Acquire all mutexes in a safe order. */ 2003 ctl_prefork(); 2004 prof_prefork(); 2005 malloc_mutex_prefork(&arenas_lock); 2006 for (i = 0; i < narenas_total; i++) { 2007 if (arenas[i] != NULL) 2008 arena_prefork(arenas[i]); 2009 } 2010 chunk_prefork(); 2011 base_prefork(); 2012 huge_prefork(); 2013 } 2014 2015 #ifndef JEMALLOC_MUTEX_INIT_CB 2016 void 2017 jemalloc_postfork_parent(void) 2018 #else 2019 JEMALLOC_EXPORT void 2020 _malloc_postfork(void) 2021 #endif 2022 { 2023 unsigned i; 2024 2025 #ifdef JEMALLOC_MUTEX_INIT_CB 2026 if (malloc_initialized == false) 2027 return; 2028 #endif 2029 assert(malloc_initialized); 2030 2031 /* Release all mutexes, now that fork() has completed. */ 2032 huge_postfork_parent(); 2033 base_postfork_parent(); 2034 chunk_postfork_parent(); 2035 for (i = 0; i < narenas_total; i++) { 2036 if (arenas[i] != NULL) 2037 arena_postfork_parent(arenas[i]); 2038 } 2039 malloc_mutex_postfork_parent(&arenas_lock); 2040 prof_postfork_parent(); 2041 ctl_postfork_parent(); 2042 } 2043 2044 void 2045 jemalloc_postfork_child(void) 2046 { 2047 unsigned i; 2048 2049 assert(malloc_initialized); 2050 2051 /* Release all mutexes, now that fork() has completed. */ 2052 huge_postfork_child(); 2053 base_postfork_child(); 2054 chunk_postfork_child(); 2055 for (i = 0; i < narenas_total; i++) { 2056 if (arenas[i] != NULL) 2057 arena_postfork_child(arenas[i]); 2058 } 2059 malloc_mutex_postfork_child(&arenas_lock); 2060 prof_postfork_child(); 2061 ctl_postfork_child(); 2062 } 2063 2064 void 2065 _malloc_first_thread(void) 2066 { 2067 2068 (void)malloc_mutex_first_thread(); 2069 } 2070 2071 /******************************************************************************/ 2072 /* 2073 * The following functions are used for TLS allocation/deallocation in static 2074 * binaries on FreeBSD. The primary difference between these and i[mcd]alloc() 2075 * is that these avoid accessing TLS variables. 2076 */ 2077 2078 static void * 2079 a0alloc(size_t size, bool zero) 2080 { 2081 2082 if (malloc_init()) 2083 return (NULL); 2084 2085 if (size == 0) 2086 size = 1; 2087 2088 if (size <= arena_maxclass) 2089 return (arena_malloc(arenas[0], size, zero, false)); 2090 else 2091 return (huge_malloc(size, zero, huge_dss_prec_get(arenas[0]))); 2092 } 2093 2094 void * 2095 a0malloc(size_t size) 2096 { 2097 2098 return (a0alloc(size, false)); 2099 } 2100 2101 void * 2102 a0calloc(size_t num, size_t size) 2103 { 2104 2105 return (a0alloc(num * size, true)); 2106 } 2107 2108 void 2109 a0free(void *ptr) 2110 { 2111 arena_chunk_t *chunk; 2112 2113 if (ptr == NULL) 2114 return; 2115 2116 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2117 if (chunk != ptr) 2118 arena_dalloc(chunk->arena, chunk, ptr, false); 2119 else 2120 huge_dalloc(ptr, true); 2121 } 2122 2123 /******************************************************************************/ 2124