1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 malloc_tsd_data(, arenas, arena_t *, NULL) 8 malloc_tsd_data(, thread_allocated, thread_allocated_t, 9 THREAD_ALLOCATED_INITIALIZER) 10 11 /* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */ 12 const char *__malloc_options_1_0 = NULL; 13 __sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); 14 15 /* Runtime configuration options. */ 16 const char *je_malloc_conf; 17 bool opt_abort = 18 #ifdef JEMALLOC_DEBUG 19 true 20 #else 21 false 22 #endif 23 ; 24 bool opt_junk = 25 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 26 true 27 #else 28 false 29 #endif 30 ; 31 size_t opt_quarantine = ZU(0); 32 bool opt_redzone = false; 33 bool opt_utrace = false; 34 bool opt_valgrind = false; 35 bool opt_xmalloc = false; 36 bool opt_zero = false; 37 size_t opt_narenas = 0; 38 39 unsigned ncpus; 40 41 malloc_mutex_t arenas_lock; 42 arena_t **arenas; 43 unsigned narenas_total; 44 unsigned narenas_auto; 45 46 /* Set to true once the allocator has been initialized. */ 47 static bool malloc_initialized = false; 48 49 #ifdef JEMALLOC_THREADED_INIT 50 /* Used to let the initializing thread recursively allocate. */ 51 # define NO_INITIALIZER ((unsigned long)0) 52 # define INITIALIZER pthread_self() 53 # define IS_INITIALIZER (malloc_initializer == pthread_self()) 54 static pthread_t malloc_initializer = NO_INITIALIZER; 55 #else 56 # define NO_INITIALIZER false 57 # define INITIALIZER true 58 # define IS_INITIALIZER malloc_initializer 59 static bool malloc_initializer = NO_INITIALIZER; 60 #endif 61 62 /* Used to avoid initialization races. */ 63 #ifdef _WIN32 64 static malloc_mutex_t init_lock; 65 66 JEMALLOC_ATTR(constructor) 67 static void WINAPI 68 _init_init_lock(void) 69 { 70 71 malloc_mutex_init(&init_lock); 72 } 73 74 #ifdef _MSC_VER 75 # pragma section(".CRT$XCU", read) 76 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 77 static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 78 #endif 79 80 #else 81 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 82 #endif 83 84 typedef struct { 85 void *p; /* Input pointer (as in realloc(p, s)). */ 86 size_t s; /* Request size. */ 87 void *r; /* Result pointer. */ 88 } malloc_utrace_t; 89 90 #ifdef JEMALLOC_UTRACE 91 # define UTRACE(a, b, c) do { \ 92 if (opt_utrace) { \ 93 int utrace_serrno = errno; \ 94 malloc_utrace_t ut; \ 95 ut.p = (a); \ 96 ut.s = (b); \ 97 ut.r = (c); \ 98 utrace(&ut, sizeof(ut)); \ 99 errno = utrace_serrno; \ 100 } \ 101 } while (0) 102 #else 103 # define UTRACE(a, b, c) 104 #endif 105 106 /******************************************************************************/ 107 /* Function prototypes for non-inline static functions. */ 108 109 static void stats_print_atexit(void); 110 static unsigned malloc_ncpus(void); 111 static bool malloc_conf_next(char const **opts_p, char const **k_p, 112 size_t *klen_p, char const **v_p, size_t *vlen_p); 113 static void malloc_conf_error(const char *msg, const char *k, size_t klen, 114 const char *v, size_t vlen); 115 static void malloc_conf_init(void); 116 static bool malloc_init_hard(void); 117 static int imemalign(void **memptr, size_t alignment, size_t size, 118 size_t min_alignment); 119 120 /******************************************************************************/ 121 /* 122 * Begin miscellaneous support functions. 123 */ 124 125 /* Create a new arena and insert it into the arenas array at index ind. */ 126 arena_t * 127 arenas_extend(unsigned ind) 128 { 129 arena_t *ret; 130 131 ret = (arena_t *)base_alloc(sizeof(arena_t)); 132 if (ret != NULL && arena_new(ret, ind) == false) { 133 arenas[ind] = ret; 134 return (ret); 135 } 136 /* Only reached if there is an OOM error. */ 137 138 /* 139 * OOM here is quite inconvenient to propagate, since dealing with it 140 * would require a check for failure in the fast path. Instead, punt 141 * by using arenas[0]. In practice, this is an extremely unlikely 142 * failure. 143 */ 144 malloc_write("<jemalloc>: Error initializing arena\n"); 145 if (opt_abort) 146 abort(); 147 148 return (arenas[0]); 149 } 150 151 /* Slow path, called only by choose_arena(). */ 152 arena_t * 153 choose_arena_hard(void) 154 { 155 arena_t *ret; 156 157 if (narenas_auto > 1) { 158 unsigned i, choose, first_null; 159 160 choose = 0; 161 first_null = narenas_auto; 162 malloc_mutex_lock(&arenas_lock); 163 assert(arenas[0] != NULL); 164 for (i = 1; i < narenas_auto; i++) { 165 if (arenas[i] != NULL) { 166 /* 167 * Choose the first arena that has the lowest 168 * number of threads assigned to it. 169 */ 170 if (arenas[i]->nthreads < 171 arenas[choose]->nthreads) 172 choose = i; 173 } else if (first_null == narenas_auto) { 174 /* 175 * Record the index of the first uninitialized 176 * arena, in case all extant arenas are in use. 177 * 178 * NB: It is possible for there to be 179 * discontinuities in terms of initialized 180 * versus uninitialized arenas, due to the 181 * "thread.arena" mallctl. 182 */ 183 first_null = i; 184 } 185 } 186 187 if (arenas[choose]->nthreads == 0 188 || first_null == narenas_auto) { 189 /* 190 * Use an unloaded arena, or the least loaded arena if 191 * all arenas are already initialized. 192 */ 193 ret = arenas[choose]; 194 } else { 195 /* Initialize a new arena. */ 196 ret = arenas_extend(first_null); 197 } 198 ret->nthreads++; 199 malloc_mutex_unlock(&arenas_lock); 200 } else { 201 ret = arenas[0]; 202 malloc_mutex_lock(&arenas_lock); 203 ret->nthreads++; 204 malloc_mutex_unlock(&arenas_lock); 205 } 206 207 arenas_tsd_set(&ret); 208 209 return (ret); 210 } 211 212 static void 213 stats_print_atexit(void) 214 { 215 216 if (config_tcache && config_stats) { 217 unsigned narenas, i; 218 219 /* 220 * Merge stats from extant threads. This is racy, since 221 * individual threads do not lock when recording tcache stats 222 * events. As a consequence, the final stats may be slightly 223 * out of date by the time they are reported, if other threads 224 * continue to allocate. 225 */ 226 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 227 arena_t *arena = arenas[i]; 228 if (arena != NULL) { 229 tcache_t *tcache; 230 231 /* 232 * tcache_stats_merge() locks bins, so if any 233 * code is introduced that acquires both arena 234 * and bin locks in the opposite order, 235 * deadlocks may result. 236 */ 237 malloc_mutex_lock(&arena->lock); 238 ql_foreach(tcache, &arena->tcache_ql, link) { 239 tcache_stats_merge(tcache, arena); 240 } 241 malloc_mutex_unlock(&arena->lock); 242 } 243 } 244 } 245 je_malloc_stats_print(NULL, NULL, NULL); 246 } 247 248 /* 249 * End miscellaneous support functions. 250 */ 251 /******************************************************************************/ 252 /* 253 * Begin initialization functions. 254 */ 255 256 static unsigned 257 malloc_ncpus(void) 258 { 259 unsigned ret; 260 long result; 261 262 #ifdef _WIN32 263 SYSTEM_INFO si; 264 GetSystemInfo(&si); 265 result = si.dwNumberOfProcessors; 266 #else 267 result = sysconf(_SC_NPROCESSORS_ONLN); 268 #endif 269 if (result == -1) { 270 /* Error. */ 271 ret = 1; 272 } else { 273 ret = (unsigned)result; 274 } 275 276 return (ret); 277 } 278 279 void 280 arenas_cleanup(void *arg) 281 { 282 arena_t *arena = *(arena_t **)arg; 283 284 malloc_mutex_lock(&arenas_lock); 285 arena->nthreads--; 286 malloc_mutex_unlock(&arenas_lock); 287 } 288 289 static JEMALLOC_ATTR(always_inline) bool 290 malloc_init(void) 291 { 292 293 if (malloc_initialized == false) 294 return (malloc_init_hard()); 295 296 return (false); 297 } 298 299 static bool 300 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 301 char const **v_p, size_t *vlen_p) 302 { 303 bool accept; 304 const char *opts = *opts_p; 305 306 *k_p = opts; 307 308 for (accept = false; accept == false;) { 309 switch (*opts) { 310 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 311 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 312 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 313 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 314 case 'Y': case 'Z': 315 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 316 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 317 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 318 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 319 case 'y': case 'z': 320 case '0': case '1': case '2': case '3': case '4': case '5': 321 case '6': case '7': case '8': case '9': 322 case '_': 323 opts++; 324 break; 325 case ':': 326 opts++; 327 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 328 *v_p = opts; 329 accept = true; 330 break; 331 case '\0': 332 if (opts != *opts_p) { 333 malloc_write("<jemalloc>: Conf string ends " 334 "with key\n"); 335 } 336 return (true); 337 default: 338 malloc_write("<jemalloc>: Malformed conf string\n"); 339 return (true); 340 } 341 } 342 343 for (accept = false; accept == false;) { 344 switch (*opts) { 345 case ',': 346 opts++; 347 /* 348 * Look ahead one character here, because the next time 349 * this function is called, it will assume that end of 350 * input has been cleanly reached if no input remains, 351 * but we have optimistically already consumed the 352 * comma if one exists. 353 */ 354 if (*opts == '\0') { 355 malloc_write("<jemalloc>: Conf string ends " 356 "with comma\n"); 357 } 358 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 359 accept = true; 360 break; 361 case '\0': 362 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 363 accept = true; 364 break; 365 default: 366 opts++; 367 break; 368 } 369 } 370 371 *opts_p = opts; 372 return (false); 373 } 374 375 static void 376 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 377 size_t vlen) 378 { 379 380 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 381 (int)vlen, v); 382 } 383 384 static void 385 malloc_conf_init(void) 386 { 387 unsigned i; 388 char buf[PATH_MAX + 1]; 389 const char *opts, *k, *v; 390 size_t klen, vlen; 391 392 /* 393 * Automatically configure valgrind before processing options. The 394 * valgrind option remains in jemalloc 3.x for compatibility reasons. 395 */ 396 if (config_valgrind) { 397 opt_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; 398 if (config_fill && opt_valgrind) { 399 opt_junk = false; 400 assert(opt_zero == false); 401 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 402 opt_redzone = true; 403 } 404 if (config_tcache && opt_valgrind) 405 opt_tcache = false; 406 } 407 408 for (i = 0; i < 3; i++) { 409 /* Get runtime configuration. */ 410 switch (i) { 411 case 0: 412 if (je_malloc_conf != NULL) { 413 /* 414 * Use options that were compiled into the 415 * program. 416 */ 417 opts = je_malloc_conf; 418 } else { 419 /* No configuration specified. */ 420 buf[0] = '\0'; 421 opts = buf; 422 } 423 break; 424 case 1: { 425 #ifndef _WIN32 426 int linklen; 427 const char *linkname = 428 # ifdef JEMALLOC_PREFIX 429 "/etc/"JEMALLOC_PREFIX"malloc.conf" 430 # else 431 "/etc/malloc.conf" 432 # endif 433 ; 434 435 if ((linklen = readlink(linkname, buf, 436 sizeof(buf) - 1)) != -1) { 437 /* 438 * Use the contents of the "/etc/malloc.conf" 439 * symbolic link's name. 440 */ 441 buf[linklen] = '\0'; 442 opts = buf; 443 } else 444 #endif 445 { 446 /* No configuration specified. */ 447 buf[0] = '\0'; 448 opts = buf; 449 } 450 break; 451 } case 2: { 452 const char *envname = 453 #ifdef JEMALLOC_PREFIX 454 JEMALLOC_CPREFIX"MALLOC_CONF" 455 #else 456 "MALLOC_CONF" 457 #endif 458 ; 459 460 if (issetugid() == 0 && (opts = getenv(envname)) != 461 NULL) { 462 /* 463 * Do nothing; opts is already initialized to 464 * the value of the MALLOC_CONF environment 465 * variable. 466 */ 467 } else { 468 /* No configuration specified. */ 469 buf[0] = '\0'; 470 opts = buf; 471 } 472 break; 473 } default: 474 /* NOTREACHED */ 475 assert(false); 476 buf[0] = '\0'; 477 opts = buf; 478 } 479 480 while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, 481 &vlen) == false) { 482 #define CONF_HANDLE_BOOL(o, n) \ 483 if (sizeof(n)-1 == klen && strncmp(n, k, \ 484 klen) == 0) { \ 485 if (strncmp("true", v, vlen) == 0 && \ 486 vlen == sizeof("true")-1) \ 487 o = true; \ 488 else if (strncmp("false", v, vlen) == \ 489 0 && vlen == sizeof("false")-1) \ 490 o = false; \ 491 else { \ 492 malloc_conf_error( \ 493 "Invalid conf value", \ 494 k, klen, v, vlen); \ 495 } \ 496 continue; \ 497 } 498 #define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ 499 if (sizeof(n)-1 == klen && strncmp(n, k, \ 500 klen) == 0) { \ 501 uintmax_t um; \ 502 char *end; \ 503 \ 504 set_errno(0); \ 505 um = malloc_strtoumax(v, &end, 0); \ 506 if (get_errno() != 0 || (uintptr_t)end -\ 507 (uintptr_t)v != vlen) { \ 508 malloc_conf_error( \ 509 "Invalid conf value", \ 510 k, klen, v, vlen); \ 511 } else if (clip) { \ 512 if (um < min) \ 513 o = min; \ 514 else if (um > max) \ 515 o = max; \ 516 else \ 517 o = um; \ 518 } else { \ 519 if (um < min || um > max) { \ 520 malloc_conf_error( \ 521 "Out-of-range " \ 522 "conf value", \ 523 k, klen, v, vlen); \ 524 } else \ 525 o = um; \ 526 } \ 527 continue; \ 528 } 529 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 530 if (sizeof(n)-1 == klen && strncmp(n, k, \ 531 klen) == 0) { \ 532 long l; \ 533 char *end; \ 534 \ 535 set_errno(0); \ 536 l = strtol(v, &end, 0); \ 537 if (get_errno() != 0 || (uintptr_t)end -\ 538 (uintptr_t)v != vlen) { \ 539 malloc_conf_error( \ 540 "Invalid conf value", \ 541 k, klen, v, vlen); \ 542 } else if (l < (ssize_t)min || l > \ 543 (ssize_t)max) { \ 544 malloc_conf_error( \ 545 "Out-of-range conf value", \ 546 k, klen, v, vlen); \ 547 } else \ 548 o = l; \ 549 continue; \ 550 } 551 #define CONF_HANDLE_CHAR_P(o, n, d) \ 552 if (sizeof(n)-1 == klen && strncmp(n, k, \ 553 klen) == 0) { \ 554 size_t cpylen = (vlen <= \ 555 sizeof(o)-1) ? vlen : \ 556 sizeof(o)-1; \ 557 strncpy(o, v, cpylen); \ 558 o[cpylen] = '\0'; \ 559 continue; \ 560 } 561 562 CONF_HANDLE_BOOL(opt_abort, "abort") 563 /* 564 * Chunks always require at least one header page, plus 565 * one data page in the absence of redzones, or three 566 * pages in the presence of redzones. In order to 567 * simplify options processing, fix the limit based on 568 * config_fill. 569 */ 570 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 571 (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1, 572 true) 573 if (strncmp("dss", k, klen) == 0) { 574 int i; 575 bool match = false; 576 for (i = 0; i < dss_prec_limit; i++) { 577 if (strncmp(dss_prec_names[i], v, vlen) 578 == 0) { 579 if (chunk_dss_prec_set(i)) { 580 malloc_conf_error( 581 "Error setting dss", 582 k, klen, v, vlen); 583 } else { 584 opt_dss = 585 dss_prec_names[i]; 586 match = true; 587 break; 588 } 589 } 590 } 591 if (match == false) { 592 malloc_conf_error("Invalid conf value", 593 k, klen, v, vlen); 594 } 595 continue; 596 } 597 CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, 598 SIZE_T_MAX, false) 599 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 600 -1, (sizeof(size_t) << 3) - 1) 601 CONF_HANDLE_BOOL(opt_stats_print, "stats_print") 602 if (config_fill) { 603 CONF_HANDLE_BOOL(opt_junk, "junk") 604 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 605 0, SIZE_T_MAX, false) 606 CONF_HANDLE_BOOL(opt_redzone, "redzone") 607 CONF_HANDLE_BOOL(opt_zero, "zero") 608 } 609 if (config_utrace) { 610 CONF_HANDLE_BOOL(opt_utrace, "utrace") 611 } 612 if (config_valgrind) { 613 CONF_HANDLE_BOOL(opt_valgrind, "valgrind") 614 } 615 if (config_xmalloc) { 616 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") 617 } 618 if (config_tcache) { 619 CONF_HANDLE_BOOL(opt_tcache, "tcache") 620 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 621 "lg_tcache_max", -1, 622 (sizeof(size_t) << 3) - 1) 623 } 624 if (config_prof) { 625 CONF_HANDLE_BOOL(opt_prof, "prof") 626 CONF_HANDLE_CHAR_P(opt_prof_prefix, 627 "prof_prefix", "jeprof") 628 CONF_HANDLE_BOOL(opt_prof_active, "prof_active") 629 CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, 630 "lg_prof_sample", 0, 631 (sizeof(uint64_t) << 3) - 1) 632 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") 633 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 634 "lg_prof_interval", -1, 635 (sizeof(uint64_t) << 3) - 1) 636 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") 637 CONF_HANDLE_BOOL(opt_prof_final, "prof_final") 638 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") 639 } 640 malloc_conf_error("Invalid conf pair", k, klen, v, 641 vlen); 642 #undef CONF_HANDLE_BOOL 643 #undef CONF_HANDLE_SIZE_T 644 #undef CONF_HANDLE_SSIZE_T 645 #undef CONF_HANDLE_CHAR_P 646 } 647 } 648 } 649 650 static bool 651 malloc_init_hard(void) 652 { 653 arena_t *init_arenas[1]; 654 655 malloc_mutex_lock(&init_lock); 656 if (malloc_initialized || IS_INITIALIZER) { 657 /* 658 * Another thread initialized the allocator before this one 659 * acquired init_lock, or this thread is the initializing 660 * thread, and it is recursively allocating. 661 */ 662 malloc_mutex_unlock(&init_lock); 663 return (false); 664 } 665 #ifdef JEMALLOC_THREADED_INIT 666 if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) { 667 /* Busy-wait until the initializing thread completes. */ 668 do { 669 malloc_mutex_unlock(&init_lock); 670 CPU_SPINWAIT; 671 malloc_mutex_lock(&init_lock); 672 } while (malloc_initialized == false); 673 malloc_mutex_unlock(&init_lock); 674 return (false); 675 } 676 #endif 677 malloc_initializer = INITIALIZER; 678 679 malloc_tsd_boot(); 680 if (config_prof) 681 prof_boot0(); 682 683 malloc_conf_init(); 684 685 #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ 686 && !defined(_WIN32)) 687 /* Register fork handlers. */ 688 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 689 jemalloc_postfork_child) != 0) { 690 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 691 if (opt_abort) 692 abort(); 693 } 694 #endif 695 696 if (opt_stats_print) { 697 /* Print statistics at exit. */ 698 if (atexit(stats_print_atexit) != 0) { 699 malloc_write("<jemalloc>: Error in atexit()\n"); 700 if (opt_abort) 701 abort(); 702 } 703 } 704 705 if (base_boot()) { 706 malloc_mutex_unlock(&init_lock); 707 return (true); 708 } 709 710 if (chunk_boot()) { 711 malloc_mutex_unlock(&init_lock); 712 return (true); 713 } 714 715 if (ctl_boot()) { 716 malloc_mutex_unlock(&init_lock); 717 return (true); 718 } 719 720 if (config_prof) 721 prof_boot1(); 722 723 arena_boot(); 724 725 if (config_tcache && tcache_boot0()) { 726 malloc_mutex_unlock(&init_lock); 727 return (true); 728 } 729 730 if (huge_boot()) { 731 malloc_mutex_unlock(&init_lock); 732 return (true); 733 } 734 735 if (malloc_mutex_init(&arenas_lock)) 736 return (true); 737 738 /* 739 * Create enough scaffolding to allow recursive allocation in 740 * malloc_ncpus(). 741 */ 742 narenas_total = narenas_auto = 1; 743 arenas = init_arenas; 744 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 745 746 /* 747 * Initialize one arena here. The rest are lazily created in 748 * choose_arena_hard(). 749 */ 750 arenas_extend(0); 751 if (arenas[0] == NULL) { 752 malloc_mutex_unlock(&init_lock); 753 return (true); 754 } 755 756 /* Initialize allocation counters before any allocations can occur. */ 757 if (config_stats && thread_allocated_tsd_boot()) { 758 malloc_mutex_unlock(&init_lock); 759 return (true); 760 } 761 762 if (arenas_tsd_boot()) { 763 malloc_mutex_unlock(&init_lock); 764 return (true); 765 } 766 767 if (config_tcache && tcache_boot1()) { 768 malloc_mutex_unlock(&init_lock); 769 return (true); 770 } 771 772 if (config_fill && quarantine_boot()) { 773 malloc_mutex_unlock(&init_lock); 774 return (true); 775 } 776 777 if (config_prof && prof_boot2()) { 778 malloc_mutex_unlock(&init_lock); 779 return (true); 780 } 781 782 /* Get number of CPUs. */ 783 malloc_mutex_unlock(&init_lock); 784 ncpus = malloc_ncpus(); 785 malloc_mutex_lock(&init_lock); 786 787 if (mutex_boot()) { 788 malloc_mutex_unlock(&init_lock); 789 return (true); 790 } 791 792 if (opt_narenas == 0) { 793 /* 794 * For SMP systems, create more than one arena per CPU by 795 * default. 796 */ 797 if (ncpus > 1) 798 opt_narenas = ncpus << 2; 799 else 800 opt_narenas = 1; 801 } 802 narenas_auto = opt_narenas; 803 /* 804 * Make sure that the arenas array can be allocated. In practice, this 805 * limit is enough to allow the allocator to function, but the ctl 806 * machinery will fail to allocate memory at far lower limits. 807 */ 808 if (narenas_auto > chunksize / sizeof(arena_t *)) { 809 narenas_auto = chunksize / sizeof(arena_t *); 810 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 811 narenas_auto); 812 } 813 narenas_total = narenas_auto; 814 815 /* Allocate and initialize arenas. */ 816 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total); 817 if (arenas == NULL) { 818 malloc_mutex_unlock(&init_lock); 819 return (true); 820 } 821 /* 822 * Zero the array. In practice, this should always be pre-zeroed, 823 * since it was just mmap()ed, but let's be sure. 824 */ 825 memset(arenas, 0, sizeof(arena_t *) * narenas_total); 826 /* Copy the pointer to the one arena that was already initialized. */ 827 arenas[0] = init_arenas[0]; 828 829 malloc_initialized = true; 830 malloc_mutex_unlock(&init_lock); 831 return (false); 832 } 833 834 /* 835 * End initialization functions. 836 */ 837 /******************************************************************************/ 838 /* 839 * Begin malloc(3)-compatible functions. 840 */ 841 842 void * 843 je_malloc(size_t size) 844 { 845 void *ret; 846 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 847 prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 848 849 if (malloc_init()) { 850 ret = NULL; 851 goto label_oom; 852 } 853 854 if (size == 0) 855 size = 1; 856 857 if (config_prof && opt_prof) { 858 usize = s2u(size); 859 PROF_ALLOC_PREP(1, usize, cnt); 860 if (cnt == NULL) { 861 ret = NULL; 862 goto label_oom; 863 } 864 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= 865 SMALL_MAXCLASS) { 866 ret = imalloc(SMALL_MAXCLASS+1); 867 if (ret != NULL) 868 arena_prof_promoted(ret, usize); 869 } else 870 ret = imalloc(size); 871 } else { 872 if (config_stats || (config_valgrind && opt_valgrind)) 873 usize = s2u(size); 874 ret = imalloc(size); 875 } 876 877 label_oom: 878 if (ret == NULL) { 879 if (config_xmalloc && opt_xmalloc) { 880 malloc_write("<jemalloc>: Error in malloc(): " 881 "out of memory\n"); 882 abort(); 883 } 884 set_errno(ENOMEM); 885 } 886 if (config_prof && opt_prof && ret != NULL) 887 prof_malloc(ret, usize, cnt); 888 if (config_stats && ret != NULL) { 889 assert(usize == isalloc(ret, config_prof)); 890 thread_allocated_tsd_get()->allocated += usize; 891 } 892 UTRACE(0, size, ret); 893 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); 894 return (ret); 895 } 896 897 JEMALLOC_ATTR(nonnull(1)) 898 #ifdef JEMALLOC_PROF 899 /* 900 * Avoid any uncertainty as to how many backtrace frames to ignore in 901 * PROF_ALLOC_PREP(). 902 */ 903 JEMALLOC_NOINLINE 904 #endif 905 static int 906 imemalign(void **memptr, size_t alignment, size_t size, 907 size_t min_alignment) 908 { 909 int ret; 910 size_t usize; 911 void *result; 912 prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 913 914 assert(min_alignment != 0); 915 916 if (malloc_init()) 917 result = NULL; 918 else { 919 if (size == 0) 920 size = 1; 921 922 /* Make sure that alignment is a large enough power of 2. */ 923 if (((alignment - 1) & alignment) != 0 924 || (alignment < min_alignment)) { 925 if (config_xmalloc && opt_xmalloc) { 926 malloc_write("<jemalloc>: Error allocating " 927 "aligned memory: invalid alignment\n"); 928 abort(); 929 } 930 result = NULL; 931 ret = EINVAL; 932 goto label_return; 933 } 934 935 usize = sa2u(size, alignment); 936 if (usize == 0) { 937 result = NULL; 938 ret = ENOMEM; 939 goto label_return; 940 } 941 942 if (config_prof && opt_prof) { 943 PROF_ALLOC_PREP(2, usize, cnt); 944 if (cnt == NULL) { 945 result = NULL; 946 ret = EINVAL; 947 } else { 948 if (prof_promote && (uintptr_t)cnt != 949 (uintptr_t)1U && usize <= SMALL_MAXCLASS) { 950 assert(sa2u(SMALL_MAXCLASS+1, 951 alignment) != 0); 952 result = ipalloc(sa2u(SMALL_MAXCLASS+1, 953 alignment), alignment, false); 954 if (result != NULL) { 955 arena_prof_promoted(result, 956 usize); 957 } 958 } else { 959 result = ipalloc(usize, alignment, 960 false); 961 } 962 } 963 } else 964 result = ipalloc(usize, alignment, false); 965 } 966 967 if (result == NULL) { 968 if (config_xmalloc && opt_xmalloc) { 969 malloc_write("<jemalloc>: Error allocating aligned " 970 "memory: out of memory\n"); 971 abort(); 972 } 973 ret = ENOMEM; 974 goto label_return; 975 } 976 977 *memptr = result; 978 ret = 0; 979 980 label_return: 981 if (config_stats && result != NULL) { 982 assert(usize == isalloc(result, config_prof)); 983 thread_allocated_tsd_get()->allocated += usize; 984 } 985 if (config_prof && opt_prof && result != NULL) 986 prof_malloc(result, usize, cnt); 987 UTRACE(0, size, result); 988 return (ret); 989 } 990 991 int 992 je_posix_memalign(void **memptr, size_t alignment, size_t size) 993 { 994 int ret = imemalign(memptr, alignment, size, sizeof(void *)); 995 JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, 996 config_prof), false); 997 return (ret); 998 } 999 1000 void * 1001 je_aligned_alloc(size_t alignment, size_t size) 1002 { 1003 void *ret; 1004 int err; 1005 1006 if ((err = imemalign(&ret, alignment, size, 1)) != 0) { 1007 ret = NULL; 1008 set_errno(err); 1009 } 1010 JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), 1011 false); 1012 return (ret); 1013 } 1014 1015 void * 1016 je_calloc(size_t num, size_t size) 1017 { 1018 void *ret; 1019 size_t num_size; 1020 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1021 prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 1022 1023 if (malloc_init()) { 1024 num_size = 0; 1025 ret = NULL; 1026 goto label_return; 1027 } 1028 1029 num_size = num * size; 1030 if (num_size == 0) { 1031 if (num == 0 || size == 0) 1032 num_size = 1; 1033 else { 1034 ret = NULL; 1035 goto label_return; 1036 } 1037 /* 1038 * Try to avoid division here. We know that it isn't possible to 1039 * overflow during multiplication if neither operand uses any of the 1040 * most significant half of the bits in a size_t. 1041 */ 1042 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) 1043 && (num_size / size != num)) { 1044 /* size_t overflow. */ 1045 ret = NULL; 1046 goto label_return; 1047 } 1048 1049 if (config_prof && opt_prof) { 1050 usize = s2u(num_size); 1051 PROF_ALLOC_PREP(1, usize, cnt); 1052 if (cnt == NULL) { 1053 ret = NULL; 1054 goto label_return; 1055 } 1056 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize 1057 <= SMALL_MAXCLASS) { 1058 ret = icalloc(SMALL_MAXCLASS+1); 1059 if (ret != NULL) 1060 arena_prof_promoted(ret, usize); 1061 } else 1062 ret = icalloc(num_size); 1063 } else { 1064 if (config_stats || (config_valgrind && opt_valgrind)) 1065 usize = s2u(num_size); 1066 ret = icalloc(num_size); 1067 } 1068 1069 label_return: 1070 if (ret == NULL) { 1071 if (config_xmalloc && opt_xmalloc) { 1072 malloc_write("<jemalloc>: Error in calloc(): out of " 1073 "memory\n"); 1074 abort(); 1075 } 1076 set_errno(ENOMEM); 1077 } 1078 1079 if (config_prof && opt_prof && ret != NULL) 1080 prof_malloc(ret, usize, cnt); 1081 if (config_stats && ret != NULL) { 1082 assert(usize == isalloc(ret, config_prof)); 1083 thread_allocated_tsd_get()->allocated += usize; 1084 } 1085 UTRACE(0, num_size, ret); 1086 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); 1087 return (ret); 1088 } 1089 1090 void * 1091 je_realloc(void *ptr, size_t size) 1092 { 1093 void *ret; 1094 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1095 size_t old_size = 0; 1096 size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1097 prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); 1098 prof_ctx_t *old_ctx JEMALLOC_CC_SILENCE_INIT(NULL); 1099 1100 if (size == 0) { 1101 if (ptr != NULL) { 1102 /* realloc(ptr, 0) is equivalent to free(p). */ 1103 if (config_prof) { 1104 old_size = isalloc(ptr, true); 1105 if (config_valgrind && opt_valgrind) 1106 old_rzsize = p2rz(ptr); 1107 } else if (config_stats) { 1108 old_size = isalloc(ptr, false); 1109 if (config_valgrind && opt_valgrind) 1110 old_rzsize = u2rz(old_size); 1111 } else if (config_valgrind && opt_valgrind) { 1112 old_size = isalloc(ptr, false); 1113 old_rzsize = u2rz(old_size); 1114 } 1115 if (config_prof && opt_prof) { 1116 old_ctx = prof_ctx_get(ptr); 1117 cnt = NULL; 1118 } 1119 iqalloc(ptr); 1120 ret = NULL; 1121 goto label_return; 1122 } else 1123 size = 1; 1124 } 1125 1126 if (ptr != NULL) { 1127 assert(malloc_initialized || IS_INITIALIZER); 1128 1129 if (config_prof) { 1130 old_size = isalloc(ptr, true); 1131 if (config_valgrind && opt_valgrind) 1132 old_rzsize = p2rz(ptr); 1133 } else if (config_stats) { 1134 old_size = isalloc(ptr, false); 1135 if (config_valgrind && opt_valgrind) 1136 old_rzsize = u2rz(old_size); 1137 } else if (config_valgrind && opt_valgrind) { 1138 old_size = isalloc(ptr, false); 1139 old_rzsize = u2rz(old_size); 1140 } 1141 if (config_prof && opt_prof) { 1142 usize = s2u(size); 1143 old_ctx = prof_ctx_get(ptr); 1144 PROF_ALLOC_PREP(1, usize, cnt); 1145 if (cnt == NULL) { 1146 old_ctx = NULL; 1147 ret = NULL; 1148 goto label_oom; 1149 } 1150 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && 1151 usize <= SMALL_MAXCLASS) { 1152 ret = iralloc(ptr, SMALL_MAXCLASS+1, 0, 0, 1153 false, false); 1154 if (ret != NULL) 1155 arena_prof_promoted(ret, usize); 1156 else 1157 old_ctx = NULL; 1158 } else { 1159 ret = iralloc(ptr, size, 0, 0, false, false); 1160 if (ret == NULL) 1161 old_ctx = NULL; 1162 } 1163 } else { 1164 if (config_stats || (config_valgrind && opt_valgrind)) 1165 usize = s2u(size); 1166 ret = iralloc(ptr, size, 0, 0, false, false); 1167 } 1168 1169 label_oom: 1170 if (ret == NULL) { 1171 if (config_xmalloc && opt_xmalloc) { 1172 malloc_write("<jemalloc>: Error in realloc(): " 1173 "out of memory\n"); 1174 abort(); 1175 } 1176 set_errno(ENOMEM); 1177 } 1178 } else { 1179 /* realloc(NULL, size) is equivalent to malloc(size). */ 1180 if (config_prof && opt_prof) 1181 old_ctx = NULL; 1182 if (malloc_init()) { 1183 if (config_prof && opt_prof) 1184 cnt = NULL; 1185 ret = NULL; 1186 } else { 1187 if (config_prof && opt_prof) { 1188 usize = s2u(size); 1189 PROF_ALLOC_PREP(1, usize, cnt); 1190 if (cnt == NULL) 1191 ret = NULL; 1192 else { 1193 if (prof_promote && (uintptr_t)cnt != 1194 (uintptr_t)1U && usize <= 1195 SMALL_MAXCLASS) { 1196 ret = imalloc(SMALL_MAXCLASS+1); 1197 if (ret != NULL) { 1198 arena_prof_promoted(ret, 1199 usize); 1200 } 1201 } else 1202 ret = imalloc(size); 1203 } 1204 } else { 1205 if (config_stats || (config_valgrind && 1206 opt_valgrind)) 1207 usize = s2u(size); 1208 ret = imalloc(size); 1209 } 1210 } 1211 1212 if (ret == NULL) { 1213 if (config_xmalloc && opt_xmalloc) { 1214 malloc_write("<jemalloc>: Error in realloc(): " 1215 "out of memory\n"); 1216 abort(); 1217 } 1218 set_errno(ENOMEM); 1219 } 1220 } 1221 1222 label_return: 1223 if (config_prof && opt_prof) 1224 prof_realloc(ret, usize, cnt, old_size, old_ctx); 1225 if (config_stats && ret != NULL) { 1226 thread_allocated_t *ta; 1227 assert(usize == isalloc(ret, config_prof)); 1228 ta = thread_allocated_tsd_get(); 1229 ta->allocated += usize; 1230 ta->deallocated += old_size; 1231 } 1232 UTRACE(ptr, size, ret); 1233 JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_size, old_rzsize, false); 1234 return (ret); 1235 } 1236 1237 void 1238 je_free(void *ptr) 1239 { 1240 1241 UTRACE(ptr, 0, 0); 1242 if (ptr != NULL) { 1243 size_t usize; 1244 size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1245 1246 assert(malloc_initialized || IS_INITIALIZER); 1247 1248 if (config_prof && opt_prof) { 1249 usize = isalloc(ptr, config_prof); 1250 prof_free(ptr, usize); 1251 } else if (config_stats || config_valgrind) 1252 usize = isalloc(ptr, config_prof); 1253 if (config_stats) 1254 thread_allocated_tsd_get()->deallocated += usize; 1255 if (config_valgrind && opt_valgrind) 1256 rzsize = p2rz(ptr); 1257 iqalloc(ptr); 1258 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1259 } 1260 } 1261 1262 /* 1263 * End malloc(3)-compatible functions. 1264 */ 1265 /******************************************************************************/ 1266 /* 1267 * Begin non-standard override functions. 1268 */ 1269 1270 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 1271 void * 1272 je_memalign(size_t alignment, size_t size) 1273 { 1274 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1275 imemalign(&ret, alignment, size, 1); 1276 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1277 return (ret); 1278 } 1279 #endif 1280 1281 #ifdef JEMALLOC_OVERRIDE_VALLOC 1282 void * 1283 je_valloc(size_t size) 1284 { 1285 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1286 imemalign(&ret, PAGE, size, 1); 1287 JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); 1288 return (ret); 1289 } 1290 #endif 1291 1292 /* 1293 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 1294 * #define je_malloc malloc 1295 */ 1296 #define malloc_is_malloc 1 1297 #define is_malloc_(a) malloc_is_ ## a 1298 #define is_malloc(a) is_malloc_(a) 1299 1300 #if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__)) 1301 /* 1302 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 1303 * to inconsistently reference libc's malloc(3)-compatible functions 1304 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 1305 * 1306 * These definitions interpose hooks in glibc. The functions are actually 1307 * passed an extra argument for the caller return address, which will be 1308 * ignored. 1309 */ 1310 JEMALLOC_EXPORT void (* __free_hook)(void *ptr) = je_free; 1311 JEMALLOC_EXPORT void *(* __malloc_hook)(size_t size) = je_malloc; 1312 JEMALLOC_EXPORT void *(* __realloc_hook)(void *ptr, size_t size) = je_realloc; 1313 JEMALLOC_EXPORT void *(* __memalign_hook)(size_t alignment, size_t size) = 1314 je_memalign; 1315 #endif 1316 1317 /* 1318 * End non-standard override functions. 1319 */ 1320 /******************************************************************************/ 1321 /* 1322 * Begin non-standard functions. 1323 */ 1324 1325 size_t 1326 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 1327 { 1328 size_t ret; 1329 1330 assert(malloc_initialized || IS_INITIALIZER); 1331 1332 if (config_ivsalloc) 1333 ret = ivsalloc(ptr, config_prof); 1334 else 1335 ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; 1336 1337 return (ret); 1338 } 1339 1340 void 1341 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 1342 const char *opts) 1343 { 1344 1345 stats_print(write_cb, cbopaque, opts); 1346 } 1347 1348 int 1349 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 1350 size_t newlen) 1351 { 1352 1353 if (malloc_init()) 1354 return (EAGAIN); 1355 1356 return (ctl_byname(name, oldp, oldlenp, newp, newlen)); 1357 } 1358 1359 int 1360 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 1361 { 1362 1363 if (malloc_init()) 1364 return (EAGAIN); 1365 1366 return (ctl_nametomib(name, mibp, miblenp)); 1367 } 1368 1369 int 1370 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1371 void *newp, size_t newlen) 1372 { 1373 1374 if (malloc_init()) 1375 return (EAGAIN); 1376 1377 return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); 1378 } 1379 1380 /* 1381 * End non-standard functions. 1382 */ 1383 /******************************************************************************/ 1384 /* 1385 * Begin experimental functions. 1386 */ 1387 #ifdef JEMALLOC_EXPERIMENTAL 1388 1389 static JEMALLOC_ATTR(always_inline) void * 1390 iallocm(size_t usize, size_t alignment, bool zero, bool try_tcache, 1391 arena_t *arena) 1392 { 1393 1394 assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, 1395 alignment))); 1396 1397 if (alignment != 0) 1398 return (ipallocx(usize, alignment, zero, try_tcache, arena)); 1399 else if (zero) 1400 return (icallocx(usize, try_tcache, arena)); 1401 else 1402 return (imallocx(usize, try_tcache, arena)); 1403 } 1404 1405 int 1406 je_allocm(void **ptr, size_t *rsize, size_t size, int flags) 1407 { 1408 void *p; 1409 size_t usize; 1410 size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1411 & (SIZE_T_MAX-1)); 1412 bool zero = flags & ALLOCM_ZERO; 1413 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1414 arena_t *arena; 1415 bool try_tcache; 1416 1417 assert(ptr != NULL); 1418 assert(size != 0); 1419 1420 if (malloc_init()) 1421 goto label_oom; 1422 1423 if (arena_ind != UINT_MAX) { 1424 arena = arenas[arena_ind]; 1425 try_tcache = false; 1426 } else { 1427 arena = NULL; 1428 try_tcache = true; 1429 } 1430 1431 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1432 if (usize == 0) 1433 goto label_oom; 1434 1435 if (config_prof && opt_prof) { 1436 prof_thr_cnt_t *cnt; 1437 1438 PROF_ALLOC_PREP(1, usize, cnt); 1439 if (cnt == NULL) 1440 goto label_oom; 1441 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= 1442 SMALL_MAXCLASS) { 1443 size_t usize_promoted = (alignment == 0) ? 1444 s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, 1445 alignment); 1446 assert(usize_promoted != 0); 1447 p = iallocm(usize_promoted, alignment, zero, 1448 try_tcache, arena); 1449 if (p == NULL) 1450 goto label_oom; 1451 arena_prof_promoted(p, usize); 1452 } else { 1453 p = iallocm(usize, alignment, zero, try_tcache, arena); 1454 if (p == NULL) 1455 goto label_oom; 1456 } 1457 prof_malloc(p, usize, cnt); 1458 } else { 1459 p = iallocm(usize, alignment, zero, try_tcache, arena); 1460 if (p == NULL) 1461 goto label_oom; 1462 } 1463 if (rsize != NULL) 1464 *rsize = usize; 1465 1466 *ptr = p; 1467 if (config_stats) { 1468 assert(usize == isalloc(p, config_prof)); 1469 thread_allocated_tsd_get()->allocated += usize; 1470 } 1471 UTRACE(0, size, p); 1472 JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); 1473 return (ALLOCM_SUCCESS); 1474 label_oom: 1475 if (config_xmalloc && opt_xmalloc) { 1476 malloc_write("<jemalloc>: Error in allocm(): " 1477 "out of memory\n"); 1478 abort(); 1479 } 1480 *ptr = NULL; 1481 UTRACE(0, size, 0); 1482 return (ALLOCM_ERR_OOM); 1483 } 1484 1485 int 1486 je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) 1487 { 1488 void *p, *q; 1489 size_t usize; 1490 size_t old_size; 1491 size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1492 size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1493 & (SIZE_T_MAX-1)); 1494 bool zero = flags & ALLOCM_ZERO; 1495 bool no_move = flags & ALLOCM_NO_MOVE; 1496 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1497 bool try_tcache_alloc, try_tcache_dalloc; 1498 arena_t *arena; 1499 1500 assert(ptr != NULL); 1501 assert(*ptr != NULL); 1502 assert(size != 0); 1503 assert(SIZE_T_MAX - size >= extra); 1504 assert(malloc_initialized || IS_INITIALIZER); 1505 1506 if (arena_ind != UINT_MAX) { 1507 arena_chunk_t *chunk; 1508 try_tcache_alloc = true; 1509 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(*ptr); 1510 try_tcache_dalloc = (chunk == *ptr || chunk->arena != 1511 arenas[arena_ind]); 1512 arena = arenas[arena_ind]; 1513 } else { 1514 try_tcache_alloc = true; 1515 try_tcache_dalloc = true; 1516 arena = NULL; 1517 } 1518 1519 p = *ptr; 1520 if (config_prof && opt_prof) { 1521 prof_thr_cnt_t *cnt; 1522 1523 /* 1524 * usize isn't knowable before iralloc() returns when extra is 1525 * non-zero. Therefore, compute its maximum possible value and 1526 * use that in PROF_ALLOC_PREP() to decide whether to capture a 1527 * backtrace. prof_realloc() will use the actual usize to 1528 * decide whether to sample. 1529 */ 1530 size_t max_usize = (alignment == 0) ? s2u(size+extra) : 1531 sa2u(size+extra, alignment); 1532 prof_ctx_t *old_ctx = prof_ctx_get(p); 1533 old_size = isalloc(p, true); 1534 if (config_valgrind && opt_valgrind) 1535 old_rzsize = p2rz(p); 1536 PROF_ALLOC_PREP(1, max_usize, cnt); 1537 if (cnt == NULL) 1538 goto label_oom; 1539 /* 1540 * Use minimum usize to determine whether promotion may happen. 1541 */ 1542 if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U 1543 && ((alignment == 0) ? s2u(size) : sa2u(size, alignment)) 1544 <= SMALL_MAXCLASS) { 1545 q = irallocx(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= 1546 size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), 1547 alignment, zero, no_move, try_tcache_alloc, 1548 try_tcache_dalloc, arena); 1549 if (q == NULL) 1550 goto label_err; 1551 if (max_usize < PAGE) { 1552 usize = max_usize; 1553 arena_prof_promoted(q, usize); 1554 } else 1555 usize = isalloc(q, config_prof); 1556 } else { 1557 q = irallocx(p, size, extra, alignment, zero, no_move, 1558 try_tcache_alloc, try_tcache_dalloc, arena); 1559 if (q == NULL) 1560 goto label_err; 1561 usize = isalloc(q, config_prof); 1562 } 1563 prof_realloc(q, usize, cnt, old_size, old_ctx); 1564 if (rsize != NULL) 1565 *rsize = usize; 1566 } else { 1567 if (config_stats) { 1568 old_size = isalloc(p, false); 1569 if (config_valgrind && opt_valgrind) 1570 old_rzsize = u2rz(old_size); 1571 } else if (config_valgrind && opt_valgrind) { 1572 old_size = isalloc(p, false); 1573 old_rzsize = u2rz(old_size); 1574 } 1575 q = irallocx(p, size, extra, alignment, zero, no_move, 1576 try_tcache_alloc, try_tcache_dalloc, arena); 1577 if (q == NULL) 1578 goto label_err; 1579 if (config_stats) 1580 usize = isalloc(q, config_prof); 1581 if (rsize != NULL) { 1582 if (config_stats == false) 1583 usize = isalloc(q, config_prof); 1584 *rsize = usize; 1585 } 1586 } 1587 1588 *ptr = q; 1589 if (config_stats) { 1590 thread_allocated_t *ta; 1591 ta = thread_allocated_tsd_get(); 1592 ta->allocated += usize; 1593 ta->deallocated += old_size; 1594 } 1595 UTRACE(p, size, q); 1596 JEMALLOC_VALGRIND_REALLOC(q, usize, p, old_size, old_rzsize, zero); 1597 return (ALLOCM_SUCCESS); 1598 label_err: 1599 if (no_move) { 1600 UTRACE(p, size, q); 1601 return (ALLOCM_ERR_NOT_MOVED); 1602 } 1603 label_oom: 1604 if (config_xmalloc && opt_xmalloc) { 1605 malloc_write("<jemalloc>: Error in rallocm(): " 1606 "out of memory\n"); 1607 abort(); 1608 } 1609 UTRACE(p, size, 0); 1610 return (ALLOCM_ERR_OOM); 1611 } 1612 1613 int 1614 je_sallocm(const void *ptr, size_t *rsize, int flags) 1615 { 1616 size_t sz; 1617 1618 assert(malloc_initialized || IS_INITIALIZER); 1619 1620 if (config_ivsalloc) 1621 sz = ivsalloc(ptr, config_prof); 1622 else { 1623 assert(ptr != NULL); 1624 sz = isalloc(ptr, config_prof); 1625 } 1626 assert(rsize != NULL); 1627 *rsize = sz; 1628 1629 return (ALLOCM_SUCCESS); 1630 } 1631 1632 int 1633 je_dallocm(void *ptr, int flags) 1634 { 1635 size_t usize; 1636 size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1637 unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; 1638 bool try_tcache; 1639 1640 assert(ptr != NULL); 1641 assert(malloc_initialized || IS_INITIALIZER); 1642 1643 if (arena_ind != UINT_MAX) { 1644 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1645 try_tcache = (chunk == ptr || chunk->arena != 1646 arenas[arena_ind]); 1647 } else 1648 try_tcache = true; 1649 1650 UTRACE(ptr, 0, 0); 1651 if (config_stats || config_valgrind) 1652 usize = isalloc(ptr, config_prof); 1653 if (config_prof && opt_prof) { 1654 if (config_stats == false && config_valgrind == false) 1655 usize = isalloc(ptr, config_prof); 1656 prof_free(ptr, usize); 1657 } 1658 if (config_stats) 1659 thread_allocated_tsd_get()->deallocated += usize; 1660 if (config_valgrind && opt_valgrind) 1661 rzsize = p2rz(ptr); 1662 iqallocx(ptr, try_tcache); 1663 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1664 1665 return (ALLOCM_SUCCESS); 1666 } 1667 1668 int 1669 je_nallocm(size_t *rsize, size_t size, int flags) 1670 { 1671 size_t usize; 1672 size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) 1673 & (SIZE_T_MAX-1)); 1674 1675 assert(size != 0); 1676 1677 if (malloc_init()) 1678 return (ALLOCM_ERR_OOM); 1679 1680 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 1681 if (usize == 0) 1682 return (ALLOCM_ERR_OOM); 1683 1684 if (rsize != NULL) 1685 *rsize = usize; 1686 return (ALLOCM_SUCCESS); 1687 } 1688 1689 #endif 1690 /* 1691 * End experimental functions. 1692 */ 1693 /******************************************************************************/ 1694 /* 1695 * The following functions are used by threading libraries for protection of 1696 * malloc during fork(). 1697 */ 1698 1699 /* 1700 * If an application creates a thread before doing any allocation in the main 1701 * thread, then calls fork(2) in the main thread followed by memory allocation 1702 * in the child process, a race can occur that results in deadlock within the 1703 * child: the main thread may have forked while the created thread had 1704 * partially initialized the allocator. Ordinarily jemalloc prevents 1705 * fork/malloc races via the following functions it registers during 1706 * initialization using pthread_atfork(), but of course that does no good if 1707 * the allocator isn't fully initialized at fork time. The following library 1708 * constructor is a partial solution to this problem. It may still possible to 1709 * trigger the deadlock described above, but doing so would involve forking via 1710 * a library constructor that runs before jemalloc's runs. 1711 */ 1712 JEMALLOC_ATTR(constructor) 1713 static void 1714 jemalloc_constructor(void) 1715 { 1716 1717 malloc_init(); 1718 } 1719 1720 #ifndef JEMALLOC_MUTEX_INIT_CB 1721 void 1722 jemalloc_prefork(void) 1723 #else 1724 JEMALLOC_EXPORT void 1725 _malloc_prefork(void) 1726 #endif 1727 { 1728 unsigned i; 1729 1730 #ifdef JEMALLOC_MUTEX_INIT_CB 1731 if (malloc_initialized == false) 1732 return; 1733 #endif 1734 assert(malloc_initialized); 1735 1736 /* Acquire all mutexes in a safe order. */ 1737 ctl_prefork(); 1738 malloc_mutex_prefork(&arenas_lock); 1739 for (i = 0; i < narenas_total; i++) { 1740 if (arenas[i] != NULL) 1741 arena_prefork(arenas[i]); 1742 } 1743 prof_prefork(); 1744 chunk_prefork(); 1745 base_prefork(); 1746 huge_prefork(); 1747 } 1748 1749 #ifndef JEMALLOC_MUTEX_INIT_CB 1750 void 1751 jemalloc_postfork_parent(void) 1752 #else 1753 JEMALLOC_EXPORT void 1754 _malloc_postfork(void) 1755 #endif 1756 { 1757 unsigned i; 1758 1759 #ifdef JEMALLOC_MUTEX_INIT_CB 1760 if (malloc_initialized == false) 1761 return; 1762 #endif 1763 assert(malloc_initialized); 1764 1765 /* Release all mutexes, now that fork() has completed. */ 1766 huge_postfork_parent(); 1767 base_postfork_parent(); 1768 chunk_postfork_parent(); 1769 prof_postfork_parent(); 1770 for (i = 0; i < narenas_total; i++) { 1771 if (arenas[i] != NULL) 1772 arena_postfork_parent(arenas[i]); 1773 } 1774 malloc_mutex_postfork_parent(&arenas_lock); 1775 ctl_postfork_parent(); 1776 } 1777 1778 void 1779 jemalloc_postfork_child(void) 1780 { 1781 unsigned i; 1782 1783 assert(malloc_initialized); 1784 1785 /* Release all mutexes, now that fork() has completed. */ 1786 huge_postfork_child(); 1787 base_postfork_child(); 1788 chunk_postfork_child(); 1789 prof_postfork_child(); 1790 for (i = 0; i < narenas_total; i++) { 1791 if (arenas[i] != NULL) 1792 arena_postfork_child(arenas[i]); 1793 } 1794 malloc_mutex_postfork_child(&arenas_lock); 1795 ctl_postfork_child(); 1796 } 1797 1798 /******************************************************************************/ 1799 /* 1800 * The following functions are used for TLS allocation/deallocation in static 1801 * binaries on FreeBSD. The primary difference between these and i[mcd]alloc() 1802 * is that these avoid accessing TLS variables. 1803 */ 1804 1805 static void * 1806 a0alloc(size_t size, bool zero) 1807 { 1808 1809 if (malloc_init()) 1810 return (NULL); 1811 1812 if (size == 0) 1813 size = 1; 1814 1815 if (size <= arena_maxclass) 1816 return (arena_malloc(arenas[0], size, zero, false)); 1817 else 1818 return (huge_malloc(size, zero)); 1819 } 1820 1821 void * 1822 a0malloc(size_t size) 1823 { 1824 1825 return (a0alloc(size, false)); 1826 } 1827 1828 void * 1829 a0calloc(size_t num, size_t size) 1830 { 1831 1832 return (a0alloc(num * size, true)); 1833 } 1834 1835 void 1836 a0free(void *ptr) 1837 { 1838 arena_chunk_t *chunk; 1839 1840 if (ptr == NULL) 1841 return; 1842 1843 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1844 if (chunk != ptr) 1845 arena_dalloc(chunk->arena, chunk, ptr, false); 1846 else 1847 huge_dalloc(ptr, true); 1848 } 1849 1850 /******************************************************************************/ 1851