1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_preamble.h" 3 #include "jemalloc/internal/jemalloc_internal_includes.h" 4 5 #include "jemalloc/internal/assert.h" 6 #include "jemalloc/internal/atomic.h" 7 #include "jemalloc/internal/ctl.h" 8 #include "jemalloc/internal/extent_dss.h" 9 #include "jemalloc/internal/extent_mmap.h" 10 #include "jemalloc/internal/hook.h" 11 #include "jemalloc/internal/jemalloc_internal_types.h" 12 #include "jemalloc/internal/log.h" 13 #include "jemalloc/internal/malloc_io.h" 14 #include "jemalloc/internal/mutex.h" 15 #include "jemalloc/internal/rtree.h" 16 #include "jemalloc/internal/safety_check.h" 17 #include "jemalloc/internal/sc.h" 18 #include "jemalloc/internal/spin.h" 19 #include "jemalloc/internal/sz.h" 20 #include "jemalloc/internal/ticker.h" 21 #include "jemalloc/internal/util.h" 22 23 /******************************************************************************/ 24 /* Data. */ 25 26 /* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */ 27 const char *__malloc_options_1_0 = NULL; 28 __sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); 29 30 /* Runtime configuration options. */ 31 const char *je_malloc_conf 32 #ifndef _WIN32 33 JEMALLOC_ATTR(weak) 34 #endif 35 ; 36 bool opt_abort = 37 #ifdef JEMALLOC_DEBUG 38 true 39 #else 40 false 41 #endif 42 ; 43 bool opt_abort_conf = 44 #ifdef JEMALLOC_DEBUG 45 true 46 #else 47 false 48 #endif 49 ; 50 /* Intentionally default off, even with debug builds. */ 51 bool opt_confirm_conf = false; 52 const char *opt_junk = 53 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 54 "true" 55 #else 56 "false" 57 #endif 58 ; 59 bool opt_junk_alloc = 60 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 61 true 62 #else 63 false 64 #endif 65 ; 66 bool opt_junk_free = 67 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 68 true 69 #else 70 false 71 #endif 72 ; 73 74 bool opt_utrace = false; 75 bool opt_xmalloc = false; 76 bool opt_zero = false; 77 unsigned opt_narenas = 0; 78 79 unsigned ncpus; 80 81 /* Protects arenas initialization. */ 82 malloc_mutex_t arenas_lock; 83 /* 84 * Arenas that are used to service external requests. Not all elements of the 85 * arenas array are necessarily used; arenas are created lazily as needed. 86 * 87 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and 88 * arenas. arenas[narenas_auto..narenas_total) are only used if the application 89 * takes some action to create them and allocate from them. 90 * 91 * Points to an arena_t. 92 */ 93 JEMALLOC_ALIGNED(CACHELINE) 94 atomic_p_t arenas[MALLOCX_ARENA_LIMIT]; 95 static atomic_u_t narenas_total; /* Use narenas_total_*(). */ 96 /* Below three are read-only after initialization. */ 97 static arena_t *a0; /* arenas[0]. */ 98 unsigned narenas_auto; 99 unsigned manual_arena_base; 100 101 typedef enum { 102 malloc_init_uninitialized = 3, 103 malloc_init_a0_initialized = 2, 104 malloc_init_recursible = 1, 105 malloc_init_initialized = 0 /* Common case --> jnz. */ 106 } malloc_init_t; 107 static malloc_init_t malloc_init_state = malloc_init_uninitialized; 108 109 /* False should be the common case. Set to true to trigger initialization. */ 110 bool malloc_slow = true; 111 112 /* When malloc_slow is true, set the corresponding bits for sanity check. */ 113 enum { 114 flag_opt_junk_alloc = (1U), 115 flag_opt_junk_free = (1U << 1), 116 flag_opt_zero = (1U << 2), 117 flag_opt_utrace = (1U << 3), 118 flag_opt_xmalloc = (1U << 4) 119 }; 120 static uint8_t malloc_slow_flags; 121 122 #ifdef JEMALLOC_THREADED_INIT 123 /* Used to let the initializing thread recursively allocate. */ 124 # define NO_INITIALIZER ((unsigned long)0) 125 # define INITIALIZER pthread_self() 126 # define IS_INITIALIZER (malloc_initializer == pthread_self()) 127 static pthread_t malloc_initializer = NO_INITIALIZER; 128 #else 129 # define NO_INITIALIZER false 130 # define INITIALIZER true 131 # define IS_INITIALIZER malloc_initializer 132 static bool malloc_initializer = NO_INITIALIZER; 133 #endif 134 135 /* Used to avoid initialization races. */ 136 #ifdef _WIN32 137 #if _WIN32_WINNT >= 0x0600 138 static malloc_mutex_t init_lock = SRWLOCK_INIT; 139 #else 140 static malloc_mutex_t init_lock; 141 static bool init_lock_initialized = false; 142 143 JEMALLOC_ATTR(constructor) 144 static void WINAPI 145 _init_init_lock(void) { 146 /* 147 * If another constructor in the same binary is using mallctl to e.g. 148 * set up extent hooks, it may end up running before this one, and 149 * malloc_init_hard will crash trying to lock the uninitialized lock. So 150 * we force an initialization of the lock in malloc_init_hard as well. 151 * We don't try to care about atomicity of the accessed to the 152 * init_lock_initialized boolean, since it really only matters early in 153 * the process creation, before any separate thread normally starts 154 * doing anything. 155 */ 156 if (!init_lock_initialized) { 157 malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT, 158 malloc_mutex_rank_exclusive); 159 } 160 init_lock_initialized = true; 161 } 162 163 #ifdef _MSC_VER 164 # pragma section(".CRT$XCU", read) 165 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 166 static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 167 #endif 168 #endif 169 #else 170 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 171 #endif 172 173 typedef struct { 174 void *p; /* Input pointer (as in realloc(p, s)). */ 175 size_t s; /* Request size. */ 176 void *r; /* Result pointer. */ 177 } malloc_utrace_t; 178 179 #ifdef JEMALLOC_UTRACE 180 # define UTRACE(a, b, c) do { \ 181 if (unlikely(opt_utrace)) { \ 182 int utrace_serrno = errno; \ 183 malloc_utrace_t ut; \ 184 ut.p = (a); \ 185 ut.s = (b); \ 186 ut.r = (c); \ 187 utrace(&ut, sizeof(ut)); \ 188 errno = utrace_serrno; \ 189 } \ 190 } while (0) 191 #else 192 # define UTRACE(a, b, c) 193 #endif 194 195 /* Whether encountered any invalid config options. */ 196 static bool had_conf_error = false; 197 198 /******************************************************************************/ 199 /* 200 * Function prototypes for static functions that are referenced prior to 201 * definition. 202 */ 203 204 static bool malloc_init_hard_a0(void); 205 static bool malloc_init_hard(void); 206 207 /******************************************************************************/ 208 /* 209 * Begin miscellaneous support functions. 210 */ 211 212 bool 213 malloc_initialized(void) { 214 return (malloc_init_state == malloc_init_initialized); 215 } 216 217 JEMALLOC_ALWAYS_INLINE bool 218 malloc_init_a0(void) { 219 if (unlikely(malloc_init_state == malloc_init_uninitialized)) { 220 return malloc_init_hard_a0(); 221 } 222 return false; 223 } 224 225 JEMALLOC_ALWAYS_INLINE bool 226 malloc_init(void) { 227 if (unlikely(!malloc_initialized()) && malloc_init_hard()) { 228 return true; 229 } 230 return false; 231 } 232 233 /* 234 * The a0*() functions are used instead of i{d,}alloc() in situations that 235 * cannot tolerate TLS variable access. 236 */ 237 238 static void * 239 a0ialloc(size_t size, bool zero, bool is_internal) { 240 if (unlikely(malloc_init_a0())) { 241 return NULL; 242 } 243 244 return iallocztm(TSDN_NULL, size, sz_size2index(size), zero, NULL, 245 is_internal, arena_get(TSDN_NULL, 0, true), true); 246 } 247 248 static void 249 a0idalloc(void *ptr, bool is_internal) { 250 idalloctm(TSDN_NULL, ptr, NULL, NULL, is_internal, true); 251 } 252 253 void * 254 a0malloc(size_t size) { 255 return a0ialloc(size, false, true); 256 } 257 258 void 259 a0dalloc(void *ptr) { 260 a0idalloc(ptr, true); 261 } 262 263 /* 264 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive 265 * situations that cannot tolerate TLS variable access (TLS allocation and very 266 * early internal data structure initialization). 267 */ 268 269 void * 270 bootstrap_malloc(size_t size) { 271 if (unlikely(size == 0)) { 272 size = 1; 273 } 274 275 return a0ialloc(size, false, false); 276 } 277 278 void * 279 bootstrap_calloc(size_t num, size_t size) { 280 size_t num_size; 281 282 num_size = num * size; 283 if (unlikely(num_size == 0)) { 284 assert(num == 0 || size == 0); 285 num_size = 1; 286 } 287 288 return a0ialloc(num_size, true, false); 289 } 290 291 void 292 bootstrap_free(void *ptr) { 293 if (unlikely(ptr == NULL)) { 294 return; 295 } 296 297 a0idalloc(ptr, false); 298 } 299 300 void 301 arena_set(unsigned ind, arena_t *arena) { 302 atomic_store_p(&arenas[ind], arena, ATOMIC_RELEASE); 303 } 304 305 static void 306 narenas_total_set(unsigned narenas) { 307 atomic_store_u(&narenas_total, narenas, ATOMIC_RELEASE); 308 } 309 310 static void 311 narenas_total_inc(void) { 312 atomic_fetch_add_u(&narenas_total, 1, ATOMIC_RELEASE); 313 } 314 315 unsigned 316 narenas_total_get(void) { 317 return atomic_load_u(&narenas_total, ATOMIC_ACQUIRE); 318 } 319 320 /* Create a new arena and insert it into the arenas array at index ind. */ 321 static arena_t * 322 arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { 323 arena_t *arena; 324 325 assert(ind <= narenas_total_get()); 326 if (ind >= MALLOCX_ARENA_LIMIT) { 327 return NULL; 328 } 329 if (ind == narenas_total_get()) { 330 narenas_total_inc(); 331 } 332 333 /* 334 * Another thread may have already initialized arenas[ind] if it's an 335 * auto arena. 336 */ 337 arena = arena_get(tsdn, ind, false); 338 if (arena != NULL) { 339 assert(arena_is_auto(arena)); 340 return arena; 341 } 342 343 /* Actually initialize the arena. */ 344 arena = arena_new(tsdn, ind, extent_hooks); 345 346 return arena; 347 } 348 349 static void 350 arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) { 351 if (ind == 0) { 352 return; 353 } 354 /* 355 * Avoid creating a new background thread just for the huge arena, which 356 * purges eagerly by default. 357 */ 358 if (have_background_thread && !arena_is_huge(ind)) { 359 if (background_thread_create(tsdn_tsd(tsdn), ind)) { 360 malloc_printf("<jemalloc>: error in background thread " 361 "creation for arena %u. Abort.\n", ind); 362 abort(); 363 } 364 } 365 } 366 367 arena_t * 368 arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { 369 arena_t *arena; 370 371 malloc_mutex_lock(tsdn, &arenas_lock); 372 arena = arena_init_locked(tsdn, ind, extent_hooks); 373 malloc_mutex_unlock(tsdn, &arenas_lock); 374 375 arena_new_create_background_thread(tsdn, ind); 376 377 return arena; 378 } 379 380 static void 381 arena_bind(tsd_t *tsd, unsigned ind, bool internal) { 382 arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false); 383 arena_nthreads_inc(arena, internal); 384 385 if (internal) { 386 tsd_iarena_set(tsd, arena); 387 } else { 388 tsd_arena_set(tsd, arena); 389 unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1, 390 ATOMIC_RELAXED); 391 tsd_binshards_t *bins = tsd_binshardsp_get(tsd); 392 for (unsigned i = 0; i < SC_NBINS; i++) { 393 assert(bin_infos[i].n_shards > 0 && 394 bin_infos[i].n_shards <= BIN_SHARDS_MAX); 395 bins->binshard[i] = shard % bin_infos[i].n_shards; 396 } 397 } 398 } 399 400 void 401 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) { 402 arena_t *oldarena, *newarena; 403 404 oldarena = arena_get(tsd_tsdn(tsd), oldind, false); 405 newarena = arena_get(tsd_tsdn(tsd), newind, false); 406 arena_nthreads_dec(oldarena, false); 407 arena_nthreads_inc(newarena, false); 408 tsd_arena_set(tsd, newarena); 409 } 410 411 static void 412 arena_unbind(tsd_t *tsd, unsigned ind, bool internal) { 413 arena_t *arena; 414 415 arena = arena_get(tsd_tsdn(tsd), ind, false); 416 arena_nthreads_dec(arena, internal); 417 418 if (internal) { 419 tsd_iarena_set(tsd, NULL); 420 } else { 421 tsd_arena_set(tsd, NULL); 422 } 423 } 424 425 arena_tdata_t * 426 arena_tdata_get_hard(tsd_t *tsd, unsigned ind) { 427 arena_tdata_t *tdata, *arenas_tdata_old; 428 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); 429 unsigned narenas_tdata_old, i; 430 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd); 431 unsigned narenas_actual = narenas_total_get(); 432 433 /* 434 * Dissociate old tdata array (and set up for deallocation upon return) 435 * if it's too small. 436 */ 437 if (arenas_tdata != NULL && narenas_tdata < narenas_actual) { 438 arenas_tdata_old = arenas_tdata; 439 narenas_tdata_old = narenas_tdata; 440 arenas_tdata = NULL; 441 narenas_tdata = 0; 442 tsd_arenas_tdata_set(tsd, arenas_tdata); 443 tsd_narenas_tdata_set(tsd, narenas_tdata); 444 } else { 445 arenas_tdata_old = NULL; 446 narenas_tdata_old = 0; 447 } 448 449 /* Allocate tdata array if it's missing. */ 450 if (arenas_tdata == NULL) { 451 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd); 452 narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1; 453 454 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) { 455 *arenas_tdata_bypassp = true; 456 arenas_tdata = (arena_tdata_t *)a0malloc( 457 sizeof(arena_tdata_t) * narenas_tdata); 458 *arenas_tdata_bypassp = false; 459 } 460 if (arenas_tdata == NULL) { 461 tdata = NULL; 462 goto label_return; 463 } 464 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp); 465 tsd_arenas_tdata_set(tsd, arenas_tdata); 466 tsd_narenas_tdata_set(tsd, narenas_tdata); 467 } 468 469 /* 470 * Copy to tdata array. It's possible that the actual number of arenas 471 * has increased since narenas_total_get() was called above, but that 472 * causes no correctness issues unless two threads concurrently execute 473 * the arenas.create mallctl, which we trust mallctl synchronization to 474 * prevent. 475 */ 476 477 /* Copy/initialize tickers. */ 478 for (i = 0; i < narenas_actual; i++) { 479 if (i < narenas_tdata_old) { 480 ticker_copy(&arenas_tdata[i].decay_ticker, 481 &arenas_tdata_old[i].decay_ticker); 482 } else { 483 ticker_init(&arenas_tdata[i].decay_ticker, 484 DECAY_NTICKS_PER_UPDATE); 485 } 486 } 487 if (narenas_tdata > narenas_actual) { 488 memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t) 489 * (narenas_tdata - narenas_actual)); 490 } 491 492 /* Read the refreshed tdata array. */ 493 tdata = &arenas_tdata[ind]; 494 label_return: 495 if (arenas_tdata_old != NULL) { 496 a0dalloc(arenas_tdata_old); 497 } 498 return tdata; 499 } 500 501 /* Slow path, called only by arena_choose(). */ 502 arena_t * 503 arena_choose_hard(tsd_t *tsd, bool internal) { 504 arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL); 505 506 if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) { 507 unsigned choose = percpu_arena_choose(); 508 ret = arena_get(tsd_tsdn(tsd), choose, true); 509 assert(ret != NULL); 510 arena_bind(tsd, arena_ind_get(ret), false); 511 arena_bind(tsd, arena_ind_get(ret), true); 512 513 return ret; 514 } 515 516 if (narenas_auto > 1) { 517 unsigned i, j, choose[2], first_null; 518 bool is_new_arena[2]; 519 520 /* 521 * Determine binding for both non-internal and internal 522 * allocation. 523 * 524 * choose[0]: For application allocation. 525 * choose[1]: For internal metadata allocation. 526 */ 527 528 for (j = 0; j < 2; j++) { 529 choose[j] = 0; 530 is_new_arena[j] = false; 531 } 532 533 first_null = narenas_auto; 534 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock); 535 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL); 536 for (i = 1; i < narenas_auto; i++) { 537 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) { 538 /* 539 * Choose the first arena that has the lowest 540 * number of threads assigned to it. 541 */ 542 for (j = 0; j < 2; j++) { 543 if (arena_nthreads_get(arena_get( 544 tsd_tsdn(tsd), i, false), !!j) < 545 arena_nthreads_get(arena_get( 546 tsd_tsdn(tsd), choose[j], false), 547 !!j)) { 548 choose[j] = i; 549 } 550 } 551 } else if (first_null == narenas_auto) { 552 /* 553 * Record the index of the first uninitialized 554 * arena, in case all extant arenas are in use. 555 * 556 * NB: It is possible for there to be 557 * discontinuities in terms of initialized 558 * versus uninitialized arenas, due to the 559 * "thread.arena" mallctl. 560 */ 561 first_null = i; 562 } 563 } 564 565 for (j = 0; j < 2; j++) { 566 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd), 567 choose[j], false), !!j) == 0 || first_null == 568 narenas_auto) { 569 /* 570 * Use an unloaded arena, or the least loaded 571 * arena if all arenas are already initialized. 572 */ 573 if (!!j == internal) { 574 ret = arena_get(tsd_tsdn(tsd), 575 choose[j], false); 576 } 577 } else { 578 arena_t *arena; 579 580 /* Initialize a new arena. */ 581 choose[j] = first_null; 582 arena = arena_init_locked(tsd_tsdn(tsd), 583 choose[j], 584 (extent_hooks_t *)&extent_hooks_default); 585 if (arena == NULL) { 586 malloc_mutex_unlock(tsd_tsdn(tsd), 587 &arenas_lock); 588 return NULL; 589 } 590 is_new_arena[j] = true; 591 if (!!j == internal) { 592 ret = arena; 593 } 594 } 595 arena_bind(tsd, choose[j], !!j); 596 } 597 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock); 598 599 for (j = 0; j < 2; j++) { 600 if (is_new_arena[j]) { 601 assert(choose[j] > 0); 602 arena_new_create_background_thread( 603 tsd_tsdn(tsd), choose[j]); 604 } 605 } 606 607 } else { 608 ret = arena_get(tsd_tsdn(tsd), 0, false); 609 arena_bind(tsd, 0, false); 610 arena_bind(tsd, 0, true); 611 } 612 613 return ret; 614 } 615 616 void 617 iarena_cleanup(tsd_t *tsd) { 618 arena_t *iarena; 619 620 iarena = tsd_iarena_get(tsd); 621 if (iarena != NULL) { 622 arena_unbind(tsd, arena_ind_get(iarena), true); 623 } 624 } 625 626 void 627 arena_cleanup(tsd_t *tsd) { 628 arena_t *arena; 629 630 arena = tsd_arena_get(tsd); 631 if (arena != NULL) { 632 arena_unbind(tsd, arena_ind_get(arena), false); 633 } 634 } 635 636 void 637 arenas_tdata_cleanup(tsd_t *tsd) { 638 arena_tdata_t *arenas_tdata; 639 640 /* Prevent tsd->arenas_tdata from being (re)created. */ 641 *tsd_arenas_tdata_bypassp_get(tsd) = true; 642 643 arenas_tdata = tsd_arenas_tdata_get(tsd); 644 if (arenas_tdata != NULL) { 645 tsd_arenas_tdata_set(tsd, NULL); 646 a0dalloc(arenas_tdata); 647 } 648 } 649 650 static void 651 stats_print_atexit(void) { 652 if (config_stats) { 653 tsdn_t *tsdn; 654 unsigned narenas, i; 655 656 tsdn = tsdn_fetch(); 657 658 /* 659 * Merge stats from extant threads. This is racy, since 660 * individual threads do not lock when recording tcache stats 661 * events. As a consequence, the final stats may be slightly 662 * out of date by the time they are reported, if other threads 663 * continue to allocate. 664 */ 665 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 666 arena_t *arena = arena_get(tsdn, i, false); 667 if (arena != NULL) { 668 tcache_t *tcache; 669 670 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); 671 ql_foreach(tcache, &arena->tcache_ql, link) { 672 tcache_stats_merge(tsdn, tcache, arena); 673 } 674 malloc_mutex_unlock(tsdn, 675 &arena->tcache_ql_mtx); 676 } 677 } 678 } 679 je_malloc_stats_print(NULL, NULL, opt_stats_print_opts); 680 } 681 682 /* 683 * Ensure that we don't hold any locks upon entry to or exit from allocator 684 * code (in a "broad" sense that doesn't count a reentrant allocation as an 685 * entrance or exit). 686 */ 687 JEMALLOC_ALWAYS_INLINE void 688 check_entry_exit_locking(tsdn_t *tsdn) { 689 if (!config_debug) { 690 return; 691 } 692 if (tsdn_null(tsdn)) { 693 return; 694 } 695 tsd_t *tsd = tsdn_tsd(tsdn); 696 /* 697 * It's possible we hold locks at entry/exit if we're in a nested 698 * allocation. 699 */ 700 int8_t reentrancy_level = tsd_reentrancy_level_get(tsd); 701 if (reentrancy_level != 0) { 702 return; 703 } 704 witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); 705 } 706 707 /* 708 * End miscellaneous support functions. 709 */ 710 /******************************************************************************/ 711 /* 712 * Begin initialization functions. 713 */ 714 715 static char * 716 jemalloc_secure_getenv(const char *name) { 717 #ifdef JEMALLOC_HAVE_SECURE_GETENV 718 return secure_getenv(name); 719 #else 720 # ifdef JEMALLOC_HAVE_ISSETUGID 721 if (issetugid() != 0) { 722 return NULL; 723 } 724 # endif 725 return getenv(name); 726 #endif 727 } 728 729 static unsigned 730 malloc_ncpus(void) { 731 long result; 732 733 #ifdef _WIN32 734 SYSTEM_INFO si; 735 GetSystemInfo(&si); 736 result = si.dwNumberOfProcessors; 737 #elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT) 738 /* 739 * glibc >= 2.6 has the CPU_COUNT macro. 740 * 741 * glibc's sysconf() uses isspace(). glibc allocates for the first time 742 * *before* setting up the isspace tables. Therefore we need a 743 * different method to get the number of CPUs. 744 */ 745 { 746 cpu_set_t set; 747 748 pthread_getaffinity_np(pthread_self(), sizeof(set), &set); 749 result = CPU_COUNT(&set); 750 } 751 #else 752 result = sysconf(_SC_NPROCESSORS_ONLN); 753 #endif 754 return ((result == -1) ? 1 : (unsigned)result); 755 } 756 757 static void 758 init_opt_stats_print_opts(const char *v, size_t vlen) { 759 size_t opts_len = strlen(opt_stats_print_opts); 760 assert(opts_len <= stats_print_tot_num_options); 761 762 for (size_t i = 0; i < vlen; i++) { 763 switch (v[i]) { 764 #define OPTION(o, v, d, s) case o: break; 765 STATS_PRINT_OPTIONS 766 #undef OPTION 767 default: continue; 768 } 769 770 if (strchr(opt_stats_print_opts, v[i]) != NULL) { 771 /* Ignore repeated. */ 772 continue; 773 } 774 775 opt_stats_print_opts[opts_len++] = v[i]; 776 opt_stats_print_opts[opts_len] = '\0'; 777 assert(opts_len <= stats_print_tot_num_options); 778 } 779 assert(opts_len == strlen(opt_stats_print_opts)); 780 } 781 782 /* Reads the next size pair in a multi-sized option. */ 783 static bool 784 malloc_conf_multi_sizes_next(const char **slab_size_segment_cur, 785 size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) { 786 const char *cur = *slab_size_segment_cur; 787 char *end; 788 uintmax_t um; 789 790 set_errno(0); 791 792 /* First number, then '-' */ 793 um = malloc_strtoumax(cur, &end, 0); 794 if (get_errno() != 0 || *end != '-') { 795 return true; 796 } 797 *slab_start = (size_t)um; 798 cur = end + 1; 799 800 /* Second number, then ':' */ 801 um = malloc_strtoumax(cur, &end, 0); 802 if (get_errno() != 0 || *end != ':') { 803 return true; 804 } 805 *slab_end = (size_t)um; 806 cur = end + 1; 807 808 /* Last number */ 809 um = malloc_strtoumax(cur, &end, 0); 810 if (get_errno() != 0) { 811 return true; 812 } 813 *new_size = (size_t)um; 814 815 /* Consume the separator if there is one. */ 816 if (*end == '|') { 817 end++; 818 } 819 820 *vlen_left -= end - *slab_size_segment_cur; 821 *slab_size_segment_cur = end; 822 823 return false; 824 } 825 826 static bool 827 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 828 char const **v_p, size_t *vlen_p) { 829 bool accept; 830 const char *opts = *opts_p; 831 832 *k_p = opts; 833 834 for (accept = false; !accept;) { 835 switch (*opts) { 836 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 837 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 838 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 839 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 840 case 'Y': case 'Z': 841 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 842 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 843 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 844 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 845 case 'y': case 'z': 846 case '0': case '1': case '2': case '3': case '4': case '5': 847 case '6': case '7': case '8': case '9': 848 case '_': 849 opts++; 850 break; 851 case ':': 852 opts++; 853 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 854 *v_p = opts; 855 accept = true; 856 break; 857 case '\0': 858 if (opts != *opts_p) { 859 malloc_write("<jemalloc>: Conf string ends " 860 "with key\n"); 861 } 862 return true; 863 default: 864 malloc_write("<jemalloc>: Malformed conf string\n"); 865 return true; 866 } 867 } 868 869 for (accept = false; !accept;) { 870 switch (*opts) { 871 case ',': 872 opts++; 873 /* 874 * Look ahead one character here, because the next time 875 * this function is called, it will assume that end of 876 * input has been cleanly reached if no input remains, 877 * but we have optimistically already consumed the 878 * comma if one exists. 879 */ 880 if (*opts == '\0') { 881 malloc_write("<jemalloc>: Conf string ends " 882 "with comma\n"); 883 } 884 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 885 accept = true; 886 break; 887 case '\0': 888 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 889 accept = true; 890 break; 891 default: 892 opts++; 893 break; 894 } 895 } 896 897 *opts_p = opts; 898 return false; 899 } 900 901 static void 902 malloc_abort_invalid_conf(void) { 903 assert(opt_abort_conf); 904 malloc_printf("<jemalloc>: Abort (abort_conf:true) on invalid conf " 905 "value (see above).\n"); 906 abort(); 907 } 908 909 static void 910 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 911 size_t vlen) { 912 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 913 (int)vlen, v); 914 /* If abort_conf is set, error out after processing all options. */ 915 const char *experimental = "experimental_"; 916 if (strncmp(k, experimental, strlen(experimental)) == 0) { 917 /* However, tolerate experimental features. */ 918 return; 919 } 920 had_conf_error = true; 921 } 922 923 static void 924 malloc_slow_flag_init(void) { 925 /* 926 * Combine the runtime options into malloc_slow for fast path. Called 927 * after processing all the options. 928 */ 929 malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0) 930 | (opt_junk_free ? flag_opt_junk_free : 0) 931 | (opt_zero ? flag_opt_zero : 0) 932 | (opt_utrace ? flag_opt_utrace : 0) 933 | (opt_xmalloc ? flag_opt_xmalloc : 0); 934 935 malloc_slow = (malloc_slow_flags != 0); 936 } 937 938 /* Number of sources for initializing malloc_conf */ 939 #define MALLOC_CONF_NSOURCES 4 940 941 static const char * 942 obtain_malloc_conf(unsigned which_source, char buf[PATH_MAX + 1]) { 943 if (config_debug) { 944 static unsigned read_source = 0; 945 /* 946 * Each source should only be read once, to minimize # of 947 * syscalls on init. 948 */ 949 assert(read_source++ == which_source); 950 } 951 assert(which_source < MALLOC_CONF_NSOURCES); 952 953 const char *ret; 954 switch (which_source) { 955 case 0: 956 ret = config_malloc_conf; 957 break; 958 case 1: 959 if (je_malloc_conf != NULL) { 960 /* Use options that were compiled into the program. */ 961 ret = je_malloc_conf; 962 } else { 963 /* No configuration specified. */ 964 ret = NULL; 965 } 966 break; 967 case 2: { 968 ssize_t linklen = 0; 969 #ifndef _WIN32 970 int saved_errno = errno; 971 const char *linkname = 972 # ifdef JEMALLOC_PREFIX 973 "/etc/"JEMALLOC_PREFIX"malloc.conf" 974 # else 975 "/etc/malloc.conf" 976 # endif 977 ; 978 979 /* 980 * Try to use the contents of the "/etc/malloc.conf" symbolic 981 * link's name. 982 */ 983 #ifndef JEMALLOC_READLINKAT 984 linklen = readlink(linkname, buf, PATH_MAX); 985 #else 986 linklen = readlinkat(AT_FDCWD, linkname, buf, PATH_MAX); 987 #endif 988 if (linklen == -1) { 989 /* No configuration specified. */ 990 linklen = 0; 991 /* Restore errno. */ 992 set_errno(saved_errno); 993 } 994 #endif 995 buf[linklen] = '\0'; 996 ret = buf; 997 break; 998 } case 3: { 999 const char *envname = 1000 #ifdef JEMALLOC_PREFIX 1001 JEMALLOC_CPREFIX"MALLOC_CONF" 1002 #else 1003 "MALLOC_CONF" 1004 #endif 1005 ; 1006 1007 if ((ret = jemalloc_secure_getenv(envname)) != NULL) { 1008 /* 1009 * Do nothing; opts is already initialized to the value 1010 * of the MALLOC_CONF environment variable. 1011 */ 1012 } else { 1013 /* No configuration specified. */ 1014 ret = NULL; 1015 } 1016 break; 1017 } default: 1018 not_reached(); 1019 ret = NULL; 1020 } 1021 return ret; 1022 } 1023 1024 static void 1025 malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], 1026 bool initial_call, const char *opts_cache[MALLOC_CONF_NSOURCES], 1027 char buf[PATH_MAX + 1]) { 1028 static const char *opts_explain[MALLOC_CONF_NSOURCES] = { 1029 "string specified via --with-malloc-conf", 1030 "string pointed to by the global variable malloc_conf", 1031 "\"name\" of the file referenced by the symbolic link named " 1032 "/etc/malloc.conf", 1033 "value of the environment variable MALLOC_CONF" 1034 }; 1035 unsigned i; 1036 const char *opts, *k, *v; 1037 size_t klen, vlen; 1038 1039 for (i = 0; i < MALLOC_CONF_NSOURCES; i++) { 1040 /* Get runtime configuration. */ 1041 if (initial_call) { 1042 opts_cache[i] = obtain_malloc_conf(i, buf); 1043 } 1044 opts = opts_cache[i]; 1045 if (!initial_call && opt_confirm_conf) { 1046 malloc_printf( 1047 "<jemalloc>: malloc_conf #%u (%s): \"%s\"\n", 1048 i + 1, opts_explain[i], opts != NULL ? opts : ""); 1049 } 1050 if (opts == NULL) { 1051 continue; 1052 } 1053 1054 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, 1055 &vlen)) { 1056 1057 #define CONF_ERROR(msg, k, klen, v, vlen) \ 1058 if (!initial_call) { \ 1059 malloc_conf_error( \ 1060 msg, k, klen, v, vlen); \ 1061 cur_opt_valid = false; \ 1062 } 1063 #define CONF_CONTINUE { \ 1064 if (!initial_call && opt_confirm_conf \ 1065 && cur_opt_valid) { \ 1066 malloc_printf("<jemalloc>: -- " \ 1067 "Set conf value: %.*s:%.*s" \ 1068 "\n", (int)klen, k, \ 1069 (int)vlen, v); \ 1070 } \ 1071 continue; \ 1072 } 1073 #define CONF_MATCH(n) \ 1074 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) 1075 #define CONF_MATCH_VALUE(n) \ 1076 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0) 1077 #define CONF_HANDLE_BOOL(o, n) \ 1078 if (CONF_MATCH(n)) { \ 1079 if (CONF_MATCH_VALUE("true")) { \ 1080 o = true; \ 1081 } else if (CONF_MATCH_VALUE("false")) { \ 1082 o = false; \ 1083 } else { \ 1084 CONF_ERROR("Invalid conf value",\ 1085 k, klen, v, vlen); \ 1086 } \ 1087 CONF_CONTINUE; \ 1088 } 1089 /* 1090 * One of the CONF_MIN macros below expands, in one of the use points, 1091 * to "unsigned integer < 0", which is always false, triggering the 1092 * GCC -Wtype-limits warning, which we disable here and re-enable below. 1093 */ 1094 JEMALLOC_DIAGNOSTIC_PUSH 1095 JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS 1096 1097 #define CONF_DONT_CHECK_MIN(um, min) false 1098 #define CONF_CHECK_MIN(um, min) ((um) < (min)) 1099 #define CONF_DONT_CHECK_MAX(um, max) false 1100 #define CONF_CHECK_MAX(um, max) ((um) > (max)) 1101 #define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \ 1102 if (CONF_MATCH(n)) { \ 1103 uintmax_t um; \ 1104 char *end; \ 1105 \ 1106 set_errno(0); \ 1107 um = malloc_strtoumax(v, &end, 0); \ 1108 if (get_errno() != 0 || (uintptr_t)end -\ 1109 (uintptr_t)v != vlen) { \ 1110 CONF_ERROR("Invalid conf value",\ 1111 k, klen, v, vlen); \ 1112 } else if (clip) { \ 1113 if (check_min(um, (t)(min))) { \ 1114 o = (t)(min); \ 1115 } else if ( \ 1116 check_max(um, (t)(max))) { \ 1117 o = (t)(max); \ 1118 } else { \ 1119 o = (t)um; \ 1120 } \ 1121 } else { \ 1122 if (check_min(um, (t)(min)) || \ 1123 check_max(um, (t)(max))) { \ 1124 CONF_ERROR( \ 1125 "Out-of-range " \ 1126 "conf value", \ 1127 k, klen, v, vlen); \ 1128 } else { \ 1129 o = (t)um; \ 1130 } \ 1131 } \ 1132 CONF_CONTINUE; \ 1133 } 1134 #define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \ 1135 clip) \ 1136 CONF_HANDLE_T_U(unsigned, o, n, min, max, \ 1137 check_min, check_max, clip) 1138 #define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \ 1139 CONF_HANDLE_T_U(size_t, o, n, min, max, \ 1140 check_min, check_max, clip) 1141 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 1142 if (CONF_MATCH(n)) { \ 1143 long l; \ 1144 char *end; \ 1145 \ 1146 set_errno(0); \ 1147 l = strtol(v, &end, 0); \ 1148 if (get_errno() != 0 || (uintptr_t)end -\ 1149 (uintptr_t)v != vlen) { \ 1150 CONF_ERROR("Invalid conf value",\ 1151 k, klen, v, vlen); \ 1152 } else if (l < (ssize_t)(min) || l > \ 1153 (ssize_t)(max)) { \ 1154 CONF_ERROR( \ 1155 "Out-of-range conf value", \ 1156 k, klen, v, vlen); \ 1157 } else { \ 1158 o = l; \ 1159 } \ 1160 CONF_CONTINUE; \ 1161 } 1162 #define CONF_HANDLE_CHAR_P(o, n, d) \ 1163 if (CONF_MATCH(n)) { \ 1164 size_t cpylen = (vlen <= \ 1165 sizeof(o)-1) ? vlen : \ 1166 sizeof(o)-1; \ 1167 strncpy(o, v, cpylen); \ 1168 o[cpylen] = '\0'; \ 1169 CONF_CONTINUE; \ 1170 } 1171 1172 bool cur_opt_valid = true; 1173 1174 CONF_HANDLE_BOOL(opt_confirm_conf, "confirm_conf") 1175 if (initial_call) { 1176 continue; 1177 } 1178 1179 CONF_HANDLE_BOOL(opt_abort, "abort") 1180 CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf") 1181 if (strncmp("metadata_thp", k, klen) == 0) { 1182 int i; 1183 bool match = false; 1184 for (i = 0; i < metadata_thp_mode_limit; i++) { 1185 if (strncmp(metadata_thp_mode_names[i], 1186 v, vlen) == 0) { 1187 opt_metadata_thp = i; 1188 match = true; 1189 break; 1190 } 1191 } 1192 if (!match) { 1193 CONF_ERROR("Invalid conf value", 1194 k, klen, v, vlen); 1195 } 1196 CONF_CONTINUE; 1197 } 1198 CONF_HANDLE_BOOL(opt_retain, "retain") 1199 if (strncmp("dss", k, klen) == 0) { 1200 int i; 1201 bool match = false; 1202 for (i = 0; i < dss_prec_limit; i++) { 1203 if (strncmp(dss_prec_names[i], v, vlen) 1204 == 0) { 1205 if (extent_dss_prec_set(i)) { 1206 CONF_ERROR( 1207 "Error setting dss", 1208 k, klen, v, vlen); 1209 } else { 1210 opt_dss = 1211 dss_prec_names[i]; 1212 match = true; 1213 break; 1214 } 1215 } 1216 } 1217 if (!match) { 1218 CONF_ERROR("Invalid conf value", 1219 k, klen, v, vlen); 1220 } 1221 CONF_CONTINUE; 1222 } 1223 CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, 1224 UINT_MAX, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, 1225 false) 1226 if (CONF_MATCH("bin_shards")) { 1227 const char *bin_shards_segment_cur = v; 1228 size_t vlen_left = vlen; 1229 do { 1230 size_t size_start; 1231 size_t size_end; 1232 size_t nshards; 1233 bool err = malloc_conf_multi_sizes_next( 1234 &bin_shards_segment_cur, &vlen_left, 1235 &size_start, &size_end, &nshards); 1236 if (err || bin_update_shard_size( 1237 bin_shard_sizes, size_start, 1238 size_end, nshards)) { 1239 CONF_ERROR( 1240 "Invalid settings for " 1241 "bin_shards", k, klen, v, 1242 vlen); 1243 break; 1244 } 1245 } while (vlen_left > 0); 1246 CONF_CONTINUE; 1247 } 1248 CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms, 1249 "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < 1250 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : 1251 SSIZE_MAX); 1252 CONF_HANDLE_SSIZE_T(opt_muzzy_decay_ms, 1253 "muzzy_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < 1254 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : 1255 SSIZE_MAX); 1256 CONF_HANDLE_BOOL(opt_stats_print, "stats_print") 1257 if (CONF_MATCH("stats_print_opts")) { 1258 init_opt_stats_print_opts(v, vlen); 1259 CONF_CONTINUE; 1260 } 1261 if (config_fill) { 1262 if (CONF_MATCH("junk")) { 1263 if (CONF_MATCH_VALUE("true")) { 1264 opt_junk = "true"; 1265 opt_junk_alloc = opt_junk_free = 1266 true; 1267 } else if (CONF_MATCH_VALUE("false")) { 1268 opt_junk = "false"; 1269 opt_junk_alloc = opt_junk_free = 1270 false; 1271 } else if (CONF_MATCH_VALUE("alloc")) { 1272 opt_junk = "alloc"; 1273 opt_junk_alloc = true; 1274 opt_junk_free = false; 1275 } else if (CONF_MATCH_VALUE("free")) { 1276 opt_junk = "free"; 1277 opt_junk_alloc = false; 1278 opt_junk_free = true; 1279 } else { 1280 CONF_ERROR( 1281 "Invalid conf value", 1282 k, klen, v, vlen); 1283 } 1284 CONF_CONTINUE; 1285 } 1286 CONF_HANDLE_BOOL(opt_zero, "zero") 1287 } 1288 if (config_utrace) { 1289 CONF_HANDLE_BOOL(opt_utrace, "utrace") 1290 } 1291 if (config_xmalloc) { 1292 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") 1293 } 1294 CONF_HANDLE_BOOL(opt_tcache, "tcache") 1295 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max", 1296 -1, (sizeof(size_t) << 3) - 1) 1297 1298 /* 1299 * The runtime option of oversize_threshold remains 1300 * undocumented. It may be tweaked in the next major 1301 * release (6.0). The default value 8M is rather 1302 * conservative / safe. Tuning it further down may 1303 * improve fragmentation a bit more, but may also cause 1304 * contention on the huge arena. 1305 */ 1306 CONF_HANDLE_SIZE_T(opt_oversize_threshold, 1307 "oversize_threshold", 0, SC_LARGE_MAXCLASS, 1308 CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, false) 1309 CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit, 1310 "lg_extent_max_active_fit", 0, 1311 (sizeof(size_t) << 3), CONF_DONT_CHECK_MIN, 1312 CONF_CHECK_MAX, false) 1313 1314 if (strncmp("percpu_arena", k, klen) == 0) { 1315 bool match = false; 1316 for (int i = percpu_arena_mode_names_base; i < 1317 percpu_arena_mode_names_limit; i++) { 1318 if (strncmp(percpu_arena_mode_names[i], 1319 v, vlen) == 0) { 1320 if (!have_percpu_arena) { 1321 CONF_ERROR( 1322 "No getcpu support", 1323 k, klen, v, vlen); 1324 } 1325 opt_percpu_arena = i; 1326 match = true; 1327 break; 1328 } 1329 } 1330 if (!match) { 1331 CONF_ERROR("Invalid conf value", 1332 k, klen, v, vlen); 1333 } 1334 CONF_CONTINUE; 1335 } 1336 CONF_HANDLE_BOOL(opt_background_thread, 1337 "background_thread"); 1338 CONF_HANDLE_SIZE_T(opt_max_background_threads, 1339 "max_background_threads", 1, 1340 opt_max_background_threads, 1341 CONF_CHECK_MIN, CONF_CHECK_MAX, 1342 true); 1343 if (CONF_MATCH("slab_sizes")) { 1344 bool err; 1345 const char *slab_size_segment_cur = v; 1346 size_t vlen_left = vlen; 1347 do { 1348 size_t slab_start; 1349 size_t slab_end; 1350 size_t pgs; 1351 err = malloc_conf_multi_sizes_next( 1352 &slab_size_segment_cur, 1353 &vlen_left, &slab_start, &slab_end, 1354 &pgs); 1355 if (!err) { 1356 sc_data_update_slab_size( 1357 sc_data, slab_start, 1358 slab_end, (int)pgs); 1359 } else { 1360 CONF_ERROR("Invalid settings " 1361 "for slab_sizes", 1362 k, klen, v, vlen); 1363 } 1364 } while (!err && vlen_left > 0); 1365 CONF_CONTINUE; 1366 } 1367 if (config_prof) { 1368 CONF_HANDLE_BOOL(opt_prof, "prof") 1369 CONF_HANDLE_CHAR_P(opt_prof_prefix, 1370 "prof_prefix", "jeprof") 1371 CONF_HANDLE_BOOL(opt_prof_active, "prof_active") 1372 CONF_HANDLE_BOOL(opt_prof_thread_active_init, 1373 "prof_thread_active_init") 1374 CONF_HANDLE_SIZE_T(opt_lg_prof_sample, 1375 "lg_prof_sample", 0, (sizeof(uint64_t) << 3) 1376 - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, 1377 true) 1378 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") 1379 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 1380 "lg_prof_interval", -1, 1381 (sizeof(uint64_t) << 3) - 1) 1382 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") 1383 CONF_HANDLE_BOOL(opt_prof_final, "prof_final") 1384 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") 1385 CONF_HANDLE_BOOL(opt_prof_log, "prof_log") 1386 } 1387 if (config_log) { 1388 if (CONF_MATCH("log")) { 1389 size_t cpylen = ( 1390 vlen <= sizeof(log_var_names) ? 1391 vlen : sizeof(log_var_names) - 1); 1392 strncpy(log_var_names, v, cpylen); 1393 log_var_names[cpylen] = '\0'; 1394 CONF_CONTINUE; 1395 } 1396 } 1397 if (CONF_MATCH("thp")) { 1398 bool match = false; 1399 for (int i = 0; i < thp_mode_names_limit; i++) { 1400 if (strncmp(thp_mode_names[i],v, vlen) 1401 == 0) { 1402 if (!have_madvise_huge) { 1403 CONF_ERROR( 1404 "No THP support", 1405 k, klen, v, vlen); 1406 } 1407 opt_thp = i; 1408 match = true; 1409 break; 1410 } 1411 } 1412 if (!match) { 1413 CONF_ERROR("Invalid conf value", 1414 k, klen, v, vlen); 1415 } 1416 CONF_CONTINUE; 1417 } 1418 CONF_ERROR("Invalid conf pair", k, klen, v, vlen); 1419 #undef CONF_ERROR 1420 #undef CONF_CONTINUE 1421 #undef CONF_MATCH 1422 #undef CONF_MATCH_VALUE 1423 #undef CONF_HANDLE_BOOL 1424 #undef CONF_DONT_CHECK_MIN 1425 #undef CONF_CHECK_MIN 1426 #undef CONF_DONT_CHECK_MAX 1427 #undef CONF_CHECK_MAX 1428 #undef CONF_HANDLE_T_U 1429 #undef CONF_HANDLE_UNSIGNED 1430 #undef CONF_HANDLE_SIZE_T 1431 #undef CONF_HANDLE_SSIZE_T 1432 #undef CONF_HANDLE_CHAR_P 1433 /* Re-enable diagnostic "-Wtype-limits" */ 1434 JEMALLOC_DIAGNOSTIC_POP 1435 } 1436 if (opt_abort_conf && had_conf_error) { 1437 malloc_abort_invalid_conf(); 1438 } 1439 } 1440 atomic_store_b(&log_init_done, true, ATOMIC_RELEASE); 1441 } 1442 1443 static void 1444 malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { 1445 const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL}; 1446 char buf[PATH_MAX + 1]; 1447 1448 /* The first call only set the confirm_conf option and opts_cache */ 1449 malloc_conf_init_helper(NULL, NULL, true, opts_cache, buf); 1450 malloc_conf_init_helper(sc_data, bin_shard_sizes, false, opts_cache, 1451 NULL); 1452 } 1453 1454 #undef MALLOC_CONF_NSOURCES 1455 1456 static bool 1457 malloc_init_hard_needed(void) { 1458 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == 1459 malloc_init_recursible)) { 1460 /* 1461 * Another thread initialized the allocator before this one 1462 * acquired init_lock, or this thread is the initializing 1463 * thread, and it is recursively allocating. 1464 */ 1465 return false; 1466 } 1467 #ifdef JEMALLOC_THREADED_INIT 1468 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) { 1469 /* Busy-wait until the initializing thread completes. */ 1470 spin_t spinner = SPIN_INITIALIZER; 1471 do { 1472 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1473 spin_adaptive(&spinner); 1474 malloc_mutex_lock(TSDN_NULL, &init_lock); 1475 } while (!malloc_initialized()); 1476 return false; 1477 } 1478 #endif 1479 return true; 1480 } 1481 1482 static bool 1483 malloc_init_hard_a0_locked() { 1484 malloc_initializer = INITIALIZER; 1485 1486 JEMALLOC_DIAGNOSTIC_PUSH 1487 JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS 1488 sc_data_t sc_data = {0}; 1489 JEMALLOC_DIAGNOSTIC_POP 1490 1491 /* 1492 * Ordering here is somewhat tricky; we need sc_boot() first, since that 1493 * determines what the size classes will be, and then 1494 * malloc_conf_init(), since any slab size tweaking will need to be done 1495 * before sz_boot and bin_boot, which assume that the values they read 1496 * out of sc_data_global are final. 1497 */ 1498 sc_boot(&sc_data); 1499 unsigned bin_shard_sizes[SC_NBINS]; 1500 bin_shard_sizes_boot(bin_shard_sizes); 1501 /* 1502 * prof_boot0 only initializes opt_prof_prefix. We need to do it before 1503 * we parse malloc_conf options, in case malloc_conf parsing overwrites 1504 * it. 1505 */ 1506 if (config_prof) { 1507 prof_boot0(); 1508 } 1509 malloc_conf_init(&sc_data, bin_shard_sizes); 1510 sz_boot(&sc_data); 1511 bin_boot(&sc_data, bin_shard_sizes); 1512 1513 if (opt_stats_print) { 1514 /* Print statistics at exit. */ 1515 if (atexit(stats_print_atexit) != 0) { 1516 malloc_write("<jemalloc>: Error in atexit()\n"); 1517 if (opt_abort) { 1518 abort(); 1519 } 1520 } 1521 } 1522 if (pages_boot()) { 1523 return true; 1524 } 1525 if (base_boot(TSDN_NULL)) { 1526 return true; 1527 } 1528 if (extent_boot()) { 1529 return true; 1530 } 1531 if (ctl_boot()) { 1532 return true; 1533 } 1534 if (config_prof) { 1535 prof_boot1(); 1536 } 1537 arena_boot(&sc_data); 1538 if (tcache_boot(TSDN_NULL)) { 1539 return true; 1540 } 1541 if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS, 1542 malloc_mutex_rank_exclusive)) { 1543 return true; 1544 } 1545 hook_boot(); 1546 /* 1547 * Create enough scaffolding to allow recursive allocation in 1548 * malloc_ncpus(). 1549 */ 1550 narenas_auto = 1; 1551 manual_arena_base = narenas_auto + 1; 1552 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 1553 /* 1554 * Initialize one arena here. The rest are lazily created in 1555 * arena_choose_hard(). 1556 */ 1557 if (arena_init(TSDN_NULL, 0, (extent_hooks_t *)&extent_hooks_default) 1558 == NULL) { 1559 return true; 1560 } 1561 a0 = arena_get(TSDN_NULL, 0, false); 1562 malloc_init_state = malloc_init_a0_initialized; 1563 1564 return false; 1565 } 1566 1567 static bool 1568 malloc_init_hard_a0(void) { 1569 bool ret; 1570 1571 malloc_mutex_lock(TSDN_NULL, &init_lock); 1572 ret = malloc_init_hard_a0_locked(); 1573 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1574 return ret; 1575 } 1576 1577 /* Initialize data structures which may trigger recursive allocation. */ 1578 static bool 1579 malloc_init_hard_recursible(void) { 1580 malloc_init_state = malloc_init_recursible; 1581 1582 ncpus = malloc_ncpus(); 1583 1584 #if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \ 1585 && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \ 1586 !defined(__native_client__)) 1587 /* LinuxThreads' pthread_atfork() allocates. */ 1588 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 1589 jemalloc_postfork_child) != 0) { 1590 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 1591 if (opt_abort) { 1592 abort(); 1593 } 1594 return true; 1595 } 1596 #endif 1597 1598 if (background_thread_boot0()) { 1599 return true; 1600 } 1601 1602 return false; 1603 } 1604 1605 static unsigned 1606 malloc_narenas_default(void) { 1607 assert(ncpus > 0); 1608 /* 1609 * For SMP systems, create more than one arena per CPU by 1610 * default. 1611 */ 1612 if (ncpus > 1) { 1613 return ncpus << 2; 1614 } else { 1615 return 1; 1616 } 1617 } 1618 1619 static percpu_arena_mode_t 1620 percpu_arena_as_initialized(percpu_arena_mode_t mode) { 1621 assert(!malloc_initialized()); 1622 assert(mode <= percpu_arena_disabled); 1623 1624 if (mode != percpu_arena_disabled) { 1625 mode += percpu_arena_mode_enabled_base; 1626 } 1627 1628 return mode; 1629 } 1630 1631 static bool 1632 malloc_init_narenas(void) { 1633 assert(ncpus > 0); 1634 1635 if (opt_percpu_arena != percpu_arena_disabled) { 1636 if (!have_percpu_arena || malloc_getcpu() < 0) { 1637 opt_percpu_arena = percpu_arena_disabled; 1638 malloc_printf("<jemalloc>: perCPU arena getcpu() not " 1639 "available. Setting narenas to %u.\n", opt_narenas ? 1640 opt_narenas : malloc_narenas_default()); 1641 if (opt_abort) { 1642 abort(); 1643 } 1644 } else { 1645 if (ncpus >= MALLOCX_ARENA_LIMIT) { 1646 malloc_printf("<jemalloc>: narenas w/ percpu" 1647 "arena beyond limit (%d)\n", ncpus); 1648 if (opt_abort) { 1649 abort(); 1650 } 1651 return true; 1652 } 1653 /* NB: opt_percpu_arena isn't fully initialized yet. */ 1654 if (percpu_arena_as_initialized(opt_percpu_arena) == 1655 per_phycpu_arena && ncpus % 2 != 0) { 1656 malloc_printf("<jemalloc>: invalid " 1657 "configuration -- per physical CPU arena " 1658 "with odd number (%u) of CPUs (no hyper " 1659 "threading?).\n", ncpus); 1660 if (opt_abort) 1661 abort(); 1662 } 1663 unsigned n = percpu_arena_ind_limit( 1664 percpu_arena_as_initialized(opt_percpu_arena)); 1665 if (opt_narenas < n) { 1666 /* 1667 * If narenas is specified with percpu_arena 1668 * enabled, actual narenas is set as the greater 1669 * of the two. percpu_arena_choose will be free 1670 * to use any of the arenas based on CPU 1671 * id. This is conservative (at a small cost) 1672 * but ensures correctness. 1673 * 1674 * If for some reason the ncpus determined at 1675 * boot is not the actual number (e.g. because 1676 * of affinity setting from numactl), reserving 1677 * narenas this way provides a workaround for 1678 * percpu_arena. 1679 */ 1680 opt_narenas = n; 1681 } 1682 } 1683 } 1684 if (opt_narenas == 0) { 1685 opt_narenas = malloc_narenas_default(); 1686 } 1687 assert(opt_narenas > 0); 1688 1689 narenas_auto = opt_narenas; 1690 /* 1691 * Limit the number of arenas to the indexing range of MALLOCX_ARENA(). 1692 */ 1693 if (narenas_auto >= MALLOCX_ARENA_LIMIT) { 1694 narenas_auto = MALLOCX_ARENA_LIMIT - 1; 1695 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 1696 narenas_auto); 1697 } 1698 narenas_total_set(narenas_auto); 1699 if (arena_init_huge()) { 1700 narenas_total_inc(); 1701 } 1702 manual_arena_base = narenas_total_get(); 1703 1704 return false; 1705 } 1706 1707 static void 1708 malloc_init_percpu(void) { 1709 opt_percpu_arena = percpu_arena_as_initialized(opt_percpu_arena); 1710 } 1711 1712 static bool 1713 malloc_init_hard_finish(void) { 1714 if (malloc_mutex_boot()) { 1715 return true; 1716 } 1717 1718 malloc_init_state = malloc_init_initialized; 1719 malloc_slow_flag_init(); 1720 1721 return false; 1722 } 1723 1724 static void 1725 malloc_init_hard_cleanup(tsdn_t *tsdn, bool reentrancy_set) { 1726 malloc_mutex_assert_owner(tsdn, &init_lock); 1727 malloc_mutex_unlock(tsdn, &init_lock); 1728 if (reentrancy_set) { 1729 assert(!tsdn_null(tsdn)); 1730 tsd_t *tsd = tsdn_tsd(tsdn); 1731 assert(tsd_reentrancy_level_get(tsd) > 0); 1732 post_reentrancy(tsd); 1733 } 1734 } 1735 1736 static bool 1737 malloc_init_hard(void) { 1738 tsd_t *tsd; 1739 1740 #if defined(_WIN32) && _WIN32_WINNT < 0x0600 1741 _init_init_lock(); 1742 #endif 1743 malloc_mutex_lock(TSDN_NULL, &init_lock); 1744 1745 #define UNLOCK_RETURN(tsdn, ret, reentrancy) \ 1746 malloc_init_hard_cleanup(tsdn, reentrancy); \ 1747 return ret; 1748 1749 if (!malloc_init_hard_needed()) { 1750 UNLOCK_RETURN(TSDN_NULL, false, false) 1751 } 1752 1753 if (malloc_init_state != malloc_init_a0_initialized && 1754 malloc_init_hard_a0_locked()) { 1755 UNLOCK_RETURN(TSDN_NULL, true, false) 1756 } 1757 1758 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1759 /* Recursive allocation relies on functional tsd. */ 1760 tsd = malloc_tsd_boot0(); 1761 if (tsd == NULL) { 1762 return true; 1763 } 1764 if (malloc_init_hard_recursible()) { 1765 return true; 1766 } 1767 1768 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock); 1769 /* Set reentrancy level to 1 during init. */ 1770 pre_reentrancy(tsd, NULL); 1771 /* Initialize narenas before prof_boot2 (for allocation). */ 1772 if (malloc_init_narenas() || background_thread_boot1(tsd_tsdn(tsd))) { 1773 UNLOCK_RETURN(tsd_tsdn(tsd), true, true) 1774 } 1775 if (config_prof && prof_boot2(tsd)) { 1776 UNLOCK_RETURN(tsd_tsdn(tsd), true, true) 1777 } 1778 1779 malloc_init_percpu(); 1780 1781 if (malloc_init_hard_finish()) { 1782 UNLOCK_RETURN(tsd_tsdn(tsd), true, true) 1783 } 1784 post_reentrancy(tsd); 1785 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1786 1787 witness_assert_lockless(witness_tsd_tsdn( 1788 tsd_witness_tsdp_get_unsafe(tsd))); 1789 malloc_tsd_boot1(); 1790 /* Update TSD after tsd_boot1. */ 1791 tsd = tsd_fetch(); 1792 if (opt_background_thread) { 1793 assert(have_background_thread); 1794 /* 1795 * Need to finish init & unlock first before creating background 1796 * threads (pthread_create depends on malloc). ctl_init (which 1797 * sets isthreaded) needs to be called without holding any lock. 1798 */ 1799 background_thread_ctl_init(tsd_tsdn(tsd)); 1800 if (background_thread_create(tsd, 0)) { 1801 return true; 1802 } 1803 } 1804 #undef UNLOCK_RETURN 1805 return false; 1806 } 1807 1808 /* 1809 * End initialization functions. 1810 */ 1811 /******************************************************************************/ 1812 /* 1813 * Begin allocation-path internal functions and data structures. 1814 */ 1815 1816 /* 1817 * Settings determined by the documented behavior of the allocation functions. 1818 */ 1819 typedef struct static_opts_s static_opts_t; 1820 struct static_opts_s { 1821 /* Whether or not allocation size may overflow. */ 1822 bool may_overflow; 1823 1824 /* 1825 * Whether or not allocations (with alignment) of size 0 should be 1826 * treated as size 1. 1827 */ 1828 bool bump_empty_aligned_alloc; 1829 /* 1830 * Whether to assert that allocations are not of size 0 (after any 1831 * bumping). 1832 */ 1833 bool assert_nonempty_alloc; 1834 1835 /* 1836 * Whether or not to modify the 'result' argument to malloc in case of 1837 * error. 1838 */ 1839 bool null_out_result_on_error; 1840 /* Whether to set errno when we encounter an error condition. */ 1841 bool set_errno_on_error; 1842 1843 /* 1844 * The minimum valid alignment for functions requesting aligned storage. 1845 */ 1846 size_t min_alignment; 1847 1848 /* The error string to use if we oom. */ 1849 const char *oom_string; 1850 /* The error string to use if the passed-in alignment is invalid. */ 1851 const char *invalid_alignment_string; 1852 1853 /* 1854 * False if we're configured to skip some time-consuming operations. 1855 * 1856 * This isn't really a malloc "behavior", but it acts as a useful 1857 * summary of several other static (or at least, static after program 1858 * initialization) options. 1859 */ 1860 bool slow; 1861 /* 1862 * Return size. 1863 */ 1864 bool usize; 1865 }; 1866 1867 JEMALLOC_ALWAYS_INLINE void 1868 static_opts_init(static_opts_t *static_opts) { 1869 static_opts->may_overflow = false; 1870 static_opts->bump_empty_aligned_alloc = false; 1871 static_opts->assert_nonempty_alloc = false; 1872 static_opts->null_out_result_on_error = false; 1873 static_opts->set_errno_on_error = false; 1874 static_opts->min_alignment = 0; 1875 static_opts->oom_string = ""; 1876 static_opts->invalid_alignment_string = ""; 1877 static_opts->slow = false; 1878 static_opts->usize = false; 1879 } 1880 1881 /* 1882 * These correspond to the macros in jemalloc/jemalloc_macros.h. Broadly, we 1883 * should have one constant here per magic value there. Note however that the 1884 * representations need not be related. 1885 */ 1886 #define TCACHE_IND_NONE ((unsigned)-1) 1887 #define TCACHE_IND_AUTOMATIC ((unsigned)-2) 1888 #define ARENA_IND_AUTOMATIC ((unsigned)-1) 1889 1890 typedef struct dynamic_opts_s dynamic_opts_t; 1891 struct dynamic_opts_s { 1892 void **result; 1893 size_t usize; 1894 size_t num_items; 1895 size_t item_size; 1896 size_t alignment; 1897 bool zero; 1898 unsigned tcache_ind; 1899 unsigned arena_ind; 1900 }; 1901 1902 JEMALLOC_ALWAYS_INLINE void 1903 dynamic_opts_init(dynamic_opts_t *dynamic_opts) { 1904 dynamic_opts->result = NULL; 1905 dynamic_opts->usize = 0; 1906 dynamic_opts->num_items = 0; 1907 dynamic_opts->item_size = 0; 1908 dynamic_opts->alignment = 0; 1909 dynamic_opts->zero = false; 1910 dynamic_opts->tcache_ind = TCACHE_IND_AUTOMATIC; 1911 dynamic_opts->arena_ind = ARENA_IND_AUTOMATIC; 1912 } 1913 1914 /* ind is ignored if dopts->alignment > 0. */ 1915 JEMALLOC_ALWAYS_INLINE void * 1916 imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, 1917 size_t size, size_t usize, szind_t ind) { 1918 tcache_t *tcache; 1919 arena_t *arena; 1920 1921 /* Fill in the tcache. */ 1922 if (dopts->tcache_ind == TCACHE_IND_AUTOMATIC) { 1923 if (likely(!sopts->slow)) { 1924 /* Getting tcache ptr unconditionally. */ 1925 tcache = tsd_tcachep_get(tsd); 1926 assert(tcache == tcache_get(tsd)); 1927 } else { 1928 tcache = tcache_get(tsd); 1929 } 1930 } else if (dopts->tcache_ind == TCACHE_IND_NONE) { 1931 tcache = NULL; 1932 } else { 1933 tcache = tcaches_get(tsd, dopts->tcache_ind); 1934 } 1935 1936 /* Fill in the arena. */ 1937 if (dopts->arena_ind == ARENA_IND_AUTOMATIC) { 1938 /* 1939 * In case of automatic arena management, we defer arena 1940 * computation until as late as we can, hoping to fill the 1941 * allocation out of the tcache. 1942 */ 1943 arena = NULL; 1944 } else { 1945 arena = arena_get(tsd_tsdn(tsd), dopts->arena_ind, true); 1946 } 1947 1948 if (unlikely(dopts->alignment != 0)) { 1949 return ipalloct(tsd_tsdn(tsd), usize, dopts->alignment, 1950 dopts->zero, tcache, arena); 1951 } 1952 1953 return iallocztm(tsd_tsdn(tsd), size, ind, dopts->zero, tcache, false, 1954 arena, sopts->slow); 1955 } 1956 1957 JEMALLOC_ALWAYS_INLINE void * 1958 imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, 1959 size_t usize, szind_t ind) { 1960 void *ret; 1961 1962 /* 1963 * For small allocations, sampling bumps the usize. If so, we allocate 1964 * from the ind_large bucket. 1965 */ 1966 szind_t ind_large; 1967 size_t bumped_usize = usize; 1968 1969 if (usize <= SC_SMALL_MAXCLASS) { 1970 assert(((dopts->alignment == 0) ? 1971 sz_s2u(SC_LARGE_MINCLASS) : 1972 sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment)) 1973 == SC_LARGE_MINCLASS); 1974 ind_large = sz_size2index(SC_LARGE_MINCLASS); 1975 bumped_usize = sz_s2u(SC_LARGE_MINCLASS); 1976 ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize, 1977 bumped_usize, ind_large); 1978 if (unlikely(ret == NULL)) { 1979 return NULL; 1980 } 1981 arena_prof_promote(tsd_tsdn(tsd), ret, usize); 1982 } else { 1983 ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind); 1984 } 1985 1986 return ret; 1987 } 1988 1989 /* 1990 * Returns true if the allocation will overflow, and false otherwise. Sets 1991 * *size to the product either way. 1992 */ 1993 JEMALLOC_ALWAYS_INLINE bool 1994 compute_size_with_overflow(bool may_overflow, dynamic_opts_t *dopts, 1995 size_t *size) { 1996 /* 1997 * This function is just num_items * item_size, except that we may have 1998 * to check for overflow. 1999 */ 2000 2001 if (!may_overflow) { 2002 assert(dopts->num_items == 1); 2003 *size = dopts->item_size; 2004 return false; 2005 } 2006 2007 /* A size_t with its high-half bits all set to 1. */ 2008 static const size_t high_bits = SIZE_T_MAX << (sizeof(size_t) * 8 / 2); 2009 2010 *size = dopts->item_size * dopts->num_items; 2011 2012 if (unlikely(*size == 0)) { 2013 return (dopts->num_items != 0 && dopts->item_size != 0); 2014 } 2015 2016 /* 2017 * We got a non-zero size, but we don't know if we overflowed to get 2018 * there. To avoid having to do a divide, we'll be clever and note that 2019 * if both A and B can be represented in N/2 bits, then their product 2020 * can be represented in N bits (without the possibility of overflow). 2021 */ 2022 if (likely((high_bits & (dopts->num_items | dopts->item_size)) == 0)) { 2023 return false; 2024 } 2025 if (likely(*size / dopts->item_size == dopts->num_items)) { 2026 return false; 2027 } 2028 return true; 2029 } 2030 2031 JEMALLOC_ALWAYS_INLINE int 2032 imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { 2033 /* Where the actual allocated memory will live. */ 2034 void *allocation = NULL; 2035 /* Filled in by compute_size_with_overflow below. */ 2036 size_t size = 0; 2037 /* 2038 * For unaligned allocations, we need only ind. For aligned 2039 * allocations, or in case of stats or profiling we need usize. 2040 * 2041 * These are actually dead stores, in that their values are reset before 2042 * any branch on their value is taken. Sometimes though, it's 2043 * convenient to pass them as arguments before this point. To avoid 2044 * undefined behavior then, we initialize them with dummy stores. 2045 */ 2046 szind_t ind = 0; 2047 size_t usize = 0; 2048 2049 /* Reentrancy is only checked on slow path. */ 2050 int8_t reentrancy_level; 2051 2052 /* Compute the amount of memory the user wants. */ 2053 if (unlikely(compute_size_with_overflow(sopts->may_overflow, dopts, 2054 &size))) { 2055 goto label_oom; 2056 } 2057 2058 if (unlikely(dopts->alignment < sopts->min_alignment 2059 || (dopts->alignment & (dopts->alignment - 1)) != 0)) { 2060 goto label_invalid_alignment; 2061 } 2062 2063 /* This is the beginning of the "core" algorithm. */ 2064 2065 if (dopts->alignment == 0) { 2066 ind = sz_size2index(size); 2067 if (unlikely(ind >= SC_NSIZES)) { 2068 goto label_oom; 2069 } 2070 if (config_stats || (config_prof && opt_prof) || sopts->usize) { 2071 usize = sz_index2size(ind); 2072 dopts->usize = usize; 2073 assert(usize > 0 && usize 2074 <= SC_LARGE_MAXCLASS); 2075 } 2076 } else { 2077 if (sopts->bump_empty_aligned_alloc) { 2078 if (unlikely(size == 0)) { 2079 size = 1; 2080 } 2081 } 2082 usize = sz_sa2u(size, dopts->alignment); 2083 dopts->usize = usize; 2084 if (unlikely(usize == 0 2085 || usize > SC_LARGE_MAXCLASS)) { 2086 goto label_oom; 2087 } 2088 } 2089 /* Validate the user input. */ 2090 if (sopts->assert_nonempty_alloc) { 2091 assert (size != 0); 2092 } 2093 2094 check_entry_exit_locking(tsd_tsdn(tsd)); 2095 2096 /* 2097 * If we need to handle reentrancy, we can do it out of a 2098 * known-initialized arena (i.e. arena 0). 2099 */ 2100 reentrancy_level = tsd_reentrancy_level_get(tsd); 2101 if (sopts->slow && unlikely(reentrancy_level > 0)) { 2102 /* 2103 * We should never specify particular arenas or tcaches from 2104 * within our internal allocations. 2105 */ 2106 assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC || 2107 dopts->tcache_ind == TCACHE_IND_NONE); 2108 assert(dopts->arena_ind == ARENA_IND_AUTOMATIC); 2109 dopts->tcache_ind = TCACHE_IND_NONE; 2110 /* We know that arena 0 has already been initialized. */ 2111 dopts->arena_ind = 0; 2112 } 2113 2114 /* If profiling is on, get our profiling context. */ 2115 if (config_prof && opt_prof) { 2116 /* 2117 * Note that if we're going down this path, usize must have been 2118 * initialized in the previous if statement. 2119 */ 2120 prof_tctx_t *tctx = prof_alloc_prep( 2121 tsd, usize, prof_active_get_unlocked(), true); 2122 2123 alloc_ctx_t alloc_ctx; 2124 if (likely((uintptr_t)tctx == (uintptr_t)1U)) { 2125 alloc_ctx.slab = (usize 2126 <= SC_SMALL_MAXCLASS); 2127 allocation = imalloc_no_sample( 2128 sopts, dopts, tsd, usize, usize, ind); 2129 } else if ((uintptr_t)tctx > (uintptr_t)1U) { 2130 /* 2131 * Note that ind might still be 0 here. This is fine; 2132 * imalloc_sample ignores ind if dopts->alignment > 0. 2133 */ 2134 allocation = imalloc_sample( 2135 sopts, dopts, tsd, usize, ind); 2136 alloc_ctx.slab = false; 2137 } else { 2138 allocation = NULL; 2139 } 2140 2141 if (unlikely(allocation == NULL)) { 2142 prof_alloc_rollback(tsd, tctx, true); 2143 goto label_oom; 2144 } 2145 prof_malloc(tsd_tsdn(tsd), allocation, usize, &alloc_ctx, tctx); 2146 } else { 2147 /* 2148 * If dopts->alignment > 0, then ind is still 0, but usize was 2149 * computed in the previous if statement. Down the positive 2150 * alignment path, imalloc_no_sample ignores ind and size 2151 * (relying only on usize). 2152 */ 2153 allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize, 2154 ind); 2155 if (unlikely(allocation == NULL)) { 2156 goto label_oom; 2157 } 2158 } 2159 2160 /* 2161 * Allocation has been done at this point. We still have some 2162 * post-allocation work to do though. 2163 */ 2164 assert(dopts->alignment == 0 2165 || ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0)); 2166 2167 if (config_stats) { 2168 assert(usize == isalloc(tsd_tsdn(tsd), allocation)); 2169 *tsd_thread_allocatedp_get(tsd) += usize; 2170 } 2171 2172 if (sopts->slow) { 2173 UTRACE(0, size, allocation); 2174 } 2175 2176 /* Success! */ 2177 check_entry_exit_locking(tsd_tsdn(tsd)); 2178 *dopts->result = allocation; 2179 return 0; 2180 2181 label_oom: 2182 if (unlikely(sopts->slow) && config_xmalloc && unlikely(opt_xmalloc)) { 2183 malloc_write(sopts->oom_string); 2184 abort(); 2185 } 2186 2187 if (sopts->slow) { 2188 UTRACE(NULL, size, NULL); 2189 } 2190 2191 check_entry_exit_locking(tsd_tsdn(tsd)); 2192 2193 if (sopts->set_errno_on_error) { 2194 set_errno(ENOMEM); 2195 } 2196 2197 if (sopts->null_out_result_on_error) { 2198 *dopts->result = NULL; 2199 } 2200 2201 return ENOMEM; 2202 2203 /* 2204 * This label is only jumped to by one goto; we move it out of line 2205 * anyways to avoid obscuring the non-error paths, and for symmetry with 2206 * the oom case. 2207 */ 2208 label_invalid_alignment: 2209 if (config_xmalloc && unlikely(opt_xmalloc)) { 2210 malloc_write(sopts->invalid_alignment_string); 2211 abort(); 2212 } 2213 2214 if (sopts->set_errno_on_error) { 2215 set_errno(EINVAL); 2216 } 2217 2218 if (sopts->slow) { 2219 UTRACE(NULL, size, NULL); 2220 } 2221 2222 check_entry_exit_locking(tsd_tsdn(tsd)); 2223 2224 if (sopts->null_out_result_on_error) { 2225 *dopts->result = NULL; 2226 } 2227 2228 return EINVAL; 2229 } 2230 2231 JEMALLOC_ALWAYS_INLINE bool 2232 imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) { 2233 if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) { 2234 if (config_xmalloc && unlikely(opt_xmalloc)) { 2235 malloc_write(sopts->oom_string); 2236 abort(); 2237 } 2238 UTRACE(NULL, dopts->num_items * dopts->item_size, NULL); 2239 set_errno(ENOMEM); 2240 *dopts->result = NULL; 2241 2242 return false; 2243 } 2244 2245 return true; 2246 } 2247 2248 /* Returns the errno-style error code of the allocation. */ 2249 JEMALLOC_ALWAYS_INLINE int 2250 imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { 2251 if (tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) { 2252 return ENOMEM; 2253 } 2254 2255 /* We always need the tsd. Let's grab it right away. */ 2256 tsd_t *tsd = tsd_fetch(); 2257 assert(tsd); 2258 if (likely(tsd_fast(tsd))) { 2259 /* Fast and common path. */ 2260 tsd_assert_fast(tsd); 2261 sopts->slow = false; 2262 return imalloc_body(sopts, dopts, tsd); 2263 } else { 2264 if (!tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) { 2265 return ENOMEM; 2266 } 2267 2268 sopts->slow = true; 2269 return imalloc_body(sopts, dopts, tsd); 2270 } 2271 } 2272 2273 JEMALLOC_NOINLINE 2274 void * 2275 malloc_default(size_t size) { 2276 void *ret; 2277 static_opts_t sopts; 2278 dynamic_opts_t dopts; 2279 2280 LOG("core.malloc.entry", "size: %zu", size); 2281 2282 static_opts_init(&sopts); 2283 dynamic_opts_init(&dopts); 2284 2285 sopts.null_out_result_on_error = true; 2286 sopts.set_errno_on_error = true; 2287 sopts.oom_string = "<jemalloc>: Error in malloc(): out of memory\n"; 2288 2289 dopts.result = &ret; 2290 dopts.num_items = 1; 2291 dopts.item_size = size; 2292 2293 imalloc(&sopts, &dopts); 2294 /* 2295 * Note that this branch gets optimized away -- it immediately follows 2296 * the check on tsd_fast that sets sopts.slow. 2297 */ 2298 if (sopts.slow) { 2299 uintptr_t args[3] = {size}; 2300 hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args); 2301 } 2302 2303 LOG("core.malloc.exit", "result: %p", ret); 2304 2305 return ret; 2306 } 2307 2308 /******************************************************************************/ 2309 /* 2310 * Begin malloc(3)-compatible functions. 2311 */ 2312 2313 /* 2314 * malloc() fastpath. 2315 * 2316 * Fastpath assumes size <= SC_LOOKUP_MAXCLASS, and that we hit 2317 * tcache. If either of these is false, we tail-call to the slowpath, 2318 * malloc_default(). Tail-calling is used to avoid any caller-saved 2319 * registers. 2320 * 2321 * fastpath supports ticker and profiling, both of which will also 2322 * tail-call to the slowpath if they fire. 2323 */ 2324 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2325 void JEMALLOC_NOTHROW * 2326 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 2327 je_malloc(size_t size) { 2328 LOG("core.malloc.entry", "size: %zu", size); 2329 2330 if (tsd_get_allocates() && unlikely(!malloc_initialized())) { 2331 return malloc_default(size); 2332 } 2333 2334 tsd_t *tsd = tsd_get(false); 2335 if (unlikely(!tsd || !tsd_fast(tsd) || (size > SC_LOOKUP_MAXCLASS))) { 2336 return malloc_default(size); 2337 } 2338 2339 tcache_t *tcache = tsd_tcachep_get(tsd); 2340 2341 if (unlikely(ticker_trytick(&tcache->gc_ticker))) { 2342 return malloc_default(size); 2343 } 2344 2345 szind_t ind = sz_size2index_lookup(size); 2346 size_t usize; 2347 if (config_stats || config_prof) { 2348 usize = sz_index2size(ind); 2349 } 2350 /* Fast path relies on size being a bin. I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS */ 2351 assert(ind < SC_NBINS); 2352 assert(size <= SC_SMALL_MAXCLASS); 2353 2354 if (config_prof) { 2355 int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd); 2356 bytes_until_sample -= usize; 2357 tsd_bytes_until_sample_set(tsd, bytes_until_sample); 2358 2359 if (unlikely(bytes_until_sample < 0)) { 2360 /* 2361 * Avoid a prof_active check on the fastpath. 2362 * If prof_active is false, set bytes_until_sample to 2363 * a large value. If prof_active is set to true, 2364 * bytes_until_sample will be reset. 2365 */ 2366 if (!prof_active) { 2367 tsd_bytes_until_sample_set(tsd, SSIZE_MAX); 2368 } 2369 return malloc_default(size); 2370 } 2371 } 2372 2373 cache_bin_t *bin = tcache_small_bin_get(tcache, ind); 2374 bool tcache_success; 2375 void* ret = cache_bin_alloc_easy(bin, &tcache_success); 2376 2377 if (tcache_success) { 2378 if (config_stats) { 2379 *tsd_thread_allocatedp_get(tsd) += usize; 2380 bin->tstats.nrequests++; 2381 } 2382 if (config_prof) { 2383 tcache->prof_accumbytes += usize; 2384 } 2385 2386 LOG("core.malloc.exit", "result: %p", ret); 2387 2388 /* Fastpath success */ 2389 return ret; 2390 } 2391 2392 return malloc_default(size); 2393 } 2394 2395 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2396 JEMALLOC_ATTR(nonnull(1)) 2397 je_posix_memalign(void **memptr, size_t alignment, size_t size) { 2398 int ret; 2399 static_opts_t sopts; 2400 dynamic_opts_t dopts; 2401 2402 LOG("core.posix_memalign.entry", "mem ptr: %p, alignment: %zu, " 2403 "size: %zu", memptr, alignment, size); 2404 2405 static_opts_init(&sopts); 2406 dynamic_opts_init(&dopts); 2407 2408 sopts.bump_empty_aligned_alloc = true; 2409 sopts.min_alignment = sizeof(void *); 2410 sopts.oom_string = 2411 "<jemalloc>: Error allocating aligned memory: out of memory\n"; 2412 sopts.invalid_alignment_string = 2413 "<jemalloc>: Error allocating aligned memory: invalid alignment\n"; 2414 2415 dopts.result = memptr; 2416 dopts.num_items = 1; 2417 dopts.item_size = size; 2418 dopts.alignment = alignment; 2419 2420 ret = imalloc(&sopts, &dopts); 2421 if (sopts.slow) { 2422 uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment, 2423 (uintptr_t)size}; 2424 hook_invoke_alloc(hook_alloc_posix_memalign, *memptr, 2425 (uintptr_t)ret, args); 2426 } 2427 2428 LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret, 2429 *memptr); 2430 2431 return ret; 2432 } 2433 2434 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2435 void JEMALLOC_NOTHROW * 2436 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2) 2437 je_aligned_alloc(size_t alignment, size_t size) { 2438 void *ret; 2439 2440 static_opts_t sopts; 2441 dynamic_opts_t dopts; 2442 2443 LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n", 2444 alignment, size); 2445 2446 static_opts_init(&sopts); 2447 dynamic_opts_init(&dopts); 2448 2449 sopts.bump_empty_aligned_alloc = true; 2450 sopts.null_out_result_on_error = true; 2451 sopts.set_errno_on_error = true; 2452 sopts.min_alignment = 1; 2453 sopts.oom_string = 2454 "<jemalloc>: Error allocating aligned memory: out of memory\n"; 2455 sopts.invalid_alignment_string = 2456 "<jemalloc>: Error allocating aligned memory: invalid alignment\n"; 2457 2458 dopts.result = &ret; 2459 dopts.num_items = 1; 2460 dopts.item_size = size; 2461 dopts.alignment = alignment; 2462 2463 imalloc(&sopts, &dopts); 2464 if (sopts.slow) { 2465 uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size}; 2466 hook_invoke_alloc(hook_alloc_aligned_alloc, ret, 2467 (uintptr_t)ret, args); 2468 } 2469 2470 LOG("core.aligned_alloc.exit", "result: %p", ret); 2471 2472 return ret; 2473 } 2474 2475 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2476 void JEMALLOC_NOTHROW * 2477 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2) 2478 je_calloc(size_t num, size_t size) { 2479 void *ret; 2480 static_opts_t sopts; 2481 dynamic_opts_t dopts; 2482 2483 LOG("core.calloc.entry", "num: %zu, size: %zu\n", num, size); 2484 2485 static_opts_init(&sopts); 2486 dynamic_opts_init(&dopts); 2487 2488 sopts.may_overflow = true; 2489 sopts.null_out_result_on_error = true; 2490 sopts.set_errno_on_error = true; 2491 sopts.oom_string = "<jemalloc>: Error in calloc(): out of memory\n"; 2492 2493 dopts.result = &ret; 2494 dopts.num_items = num; 2495 dopts.item_size = size; 2496 dopts.zero = true; 2497 2498 imalloc(&sopts, &dopts); 2499 if (sopts.slow) { 2500 uintptr_t args[3] = {(uintptr_t)num, (uintptr_t)size}; 2501 hook_invoke_alloc(hook_alloc_calloc, ret, (uintptr_t)ret, args); 2502 } 2503 2504 LOG("core.calloc.exit", "result: %p", ret); 2505 2506 return ret; 2507 } 2508 2509 static void * 2510 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, 2511 prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) { 2512 void *p; 2513 2514 if (tctx == NULL) { 2515 return NULL; 2516 } 2517 if (usize <= SC_SMALL_MAXCLASS) { 2518 p = iralloc(tsd, old_ptr, old_usize, 2519 SC_LARGE_MINCLASS, 0, false, hook_args); 2520 if (p == NULL) { 2521 return NULL; 2522 } 2523 arena_prof_promote(tsd_tsdn(tsd), p, usize); 2524 } else { 2525 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false, 2526 hook_args); 2527 } 2528 2529 return p; 2530 } 2531 2532 JEMALLOC_ALWAYS_INLINE void * 2533 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, 2534 alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { 2535 void *p; 2536 bool prof_active; 2537 prof_tctx_t *old_tctx, *tctx; 2538 2539 prof_active = prof_active_get_unlocked(); 2540 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx); 2541 tctx = prof_alloc_prep(tsd, usize, prof_active, true); 2542 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2543 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx, 2544 hook_args); 2545 } else { 2546 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false, 2547 hook_args); 2548 } 2549 if (unlikely(p == NULL)) { 2550 prof_alloc_rollback(tsd, tctx, true); 2551 return NULL; 2552 } 2553 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize, 2554 old_tctx); 2555 2556 return p; 2557 } 2558 2559 JEMALLOC_ALWAYS_INLINE void 2560 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { 2561 if (!slow_path) { 2562 tsd_assert_fast(tsd); 2563 } 2564 check_entry_exit_locking(tsd_tsdn(tsd)); 2565 if (tsd_reentrancy_level_get(tsd) != 0) { 2566 assert(slow_path); 2567 } 2568 2569 assert(ptr != NULL); 2570 assert(malloc_initialized() || IS_INITIALIZER); 2571 2572 alloc_ctx_t alloc_ctx; 2573 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2574 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, 2575 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); 2576 assert(alloc_ctx.szind != SC_NSIZES); 2577 2578 size_t usize; 2579 if (config_prof && opt_prof) { 2580 usize = sz_index2size(alloc_ctx.szind); 2581 prof_free(tsd, ptr, usize, &alloc_ctx); 2582 } else if (config_stats) { 2583 usize = sz_index2size(alloc_ctx.szind); 2584 } 2585 if (config_stats) { 2586 *tsd_thread_deallocatedp_get(tsd) += usize; 2587 } 2588 2589 if (likely(!slow_path)) { 2590 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, 2591 false); 2592 } else { 2593 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, 2594 true); 2595 } 2596 } 2597 2598 JEMALLOC_ALWAYS_INLINE void 2599 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { 2600 if (!slow_path) { 2601 tsd_assert_fast(tsd); 2602 } 2603 check_entry_exit_locking(tsd_tsdn(tsd)); 2604 if (tsd_reentrancy_level_get(tsd) != 0) { 2605 assert(slow_path); 2606 } 2607 2608 assert(ptr != NULL); 2609 assert(malloc_initialized() || IS_INITIALIZER); 2610 2611 alloc_ctx_t alloc_ctx, *ctx; 2612 if (!config_cache_oblivious && ((uintptr_t)ptr & PAGE_MASK) != 0) { 2613 /* 2614 * When cache_oblivious is disabled and ptr is not page aligned, 2615 * the allocation was not sampled -- usize can be used to 2616 * determine szind directly. 2617 */ 2618 alloc_ctx.szind = sz_size2index(usize); 2619 alloc_ctx.slab = true; 2620 ctx = &alloc_ctx; 2621 if (config_debug) { 2622 alloc_ctx_t dbg_ctx; 2623 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2624 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, 2625 rtree_ctx, (uintptr_t)ptr, true, &dbg_ctx.szind, 2626 &dbg_ctx.slab); 2627 assert(dbg_ctx.szind == alloc_ctx.szind); 2628 assert(dbg_ctx.slab == alloc_ctx.slab); 2629 } 2630 } else if (config_prof && opt_prof) { 2631 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2632 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, 2633 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); 2634 assert(alloc_ctx.szind == sz_size2index(usize)); 2635 ctx = &alloc_ctx; 2636 } else { 2637 ctx = NULL; 2638 } 2639 2640 if (config_prof && opt_prof) { 2641 prof_free(tsd, ptr, usize, ctx); 2642 } 2643 if (config_stats) { 2644 *tsd_thread_deallocatedp_get(tsd) += usize; 2645 } 2646 2647 if (likely(!slow_path)) { 2648 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false); 2649 } else { 2650 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true); 2651 } 2652 } 2653 2654 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2655 void JEMALLOC_NOTHROW * 2656 JEMALLOC_ALLOC_SIZE(2) 2657 je_realloc(void *ptr, size_t arg_size) { 2658 void *ret; 2659 tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL); 2660 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 2661 size_t old_usize = 0; 2662 size_t size = arg_size; 2663 2664 LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size); 2665 2666 if (unlikely(size == 0)) { 2667 size = 1; 2668 } 2669 2670 if (likely(ptr != NULL)) { 2671 assert(malloc_initialized() || IS_INITIALIZER); 2672 tsd_t *tsd = tsd_fetch(); 2673 2674 check_entry_exit_locking(tsd_tsdn(tsd)); 2675 2676 2677 hook_ralloc_args_t hook_args = {true, {(uintptr_t)ptr, 2678 (uintptr_t)arg_size, 0, 0}}; 2679 2680 alloc_ctx_t alloc_ctx; 2681 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2682 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, 2683 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); 2684 assert(alloc_ctx.szind != SC_NSIZES); 2685 old_usize = sz_index2size(alloc_ctx.szind); 2686 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); 2687 if (config_prof && opt_prof) { 2688 usize = sz_s2u(size); 2689 if (unlikely(usize == 0 2690 || usize > SC_LARGE_MAXCLASS)) { 2691 ret = NULL; 2692 } else { 2693 ret = irealloc_prof(tsd, ptr, old_usize, usize, 2694 &alloc_ctx, &hook_args); 2695 } 2696 } else { 2697 if (config_stats) { 2698 usize = sz_s2u(size); 2699 } 2700 ret = iralloc(tsd, ptr, old_usize, size, 0, false, 2701 &hook_args); 2702 } 2703 tsdn = tsd_tsdn(tsd); 2704 } else { 2705 /* realloc(NULL, size) is equivalent to malloc(size). */ 2706 static_opts_t sopts; 2707 dynamic_opts_t dopts; 2708 2709 static_opts_init(&sopts); 2710 dynamic_opts_init(&dopts); 2711 2712 sopts.null_out_result_on_error = true; 2713 sopts.set_errno_on_error = true; 2714 sopts.oom_string = 2715 "<jemalloc>: Error in realloc(): out of memory\n"; 2716 2717 dopts.result = &ret; 2718 dopts.num_items = 1; 2719 dopts.item_size = size; 2720 2721 imalloc(&sopts, &dopts); 2722 if (sopts.slow) { 2723 uintptr_t args[3] = {(uintptr_t)ptr, arg_size}; 2724 hook_invoke_alloc(hook_alloc_realloc, ret, 2725 (uintptr_t)ret, args); 2726 } 2727 2728 return ret; 2729 } 2730 2731 if (unlikely(ret == NULL)) { 2732 if (config_xmalloc && unlikely(opt_xmalloc)) { 2733 malloc_write("<jemalloc>: Error in realloc(): " 2734 "out of memory\n"); 2735 abort(); 2736 } 2737 set_errno(ENOMEM); 2738 } 2739 if (config_stats && likely(ret != NULL)) { 2740 tsd_t *tsd; 2741 2742 assert(usize == isalloc(tsdn, ret)); 2743 tsd = tsdn_tsd(tsdn); 2744 *tsd_thread_allocatedp_get(tsd) += usize; 2745 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2746 } 2747 UTRACE(ptr, size, ret); 2748 check_entry_exit_locking(tsdn); 2749 2750 LOG("core.realloc.exit", "result: %p", ret); 2751 return ret; 2752 } 2753 2754 JEMALLOC_NOINLINE 2755 void 2756 free_default(void *ptr) { 2757 UTRACE(ptr, 0, 0); 2758 if (likely(ptr != NULL)) { 2759 /* 2760 * We avoid setting up tsd fully (e.g. tcache, arena binding) 2761 * based on only free() calls -- other activities trigger the 2762 * minimal to full transition. This is because free() may 2763 * happen during thread shutdown after tls deallocation: if a 2764 * thread never had any malloc activities until then, a 2765 * fully-setup tsd won't be destructed properly. 2766 */ 2767 tsd_t *tsd = tsd_fetch_min(); 2768 check_entry_exit_locking(tsd_tsdn(tsd)); 2769 2770 tcache_t *tcache; 2771 if (likely(tsd_fast(tsd))) { 2772 tsd_assert_fast(tsd); 2773 /* Unconditionally get tcache ptr on fast path. */ 2774 tcache = tsd_tcachep_get(tsd); 2775 ifree(tsd, ptr, tcache, false); 2776 } else { 2777 if (likely(tsd_reentrancy_level_get(tsd) == 0)) { 2778 tcache = tcache_get(tsd); 2779 } else { 2780 tcache = NULL; 2781 } 2782 uintptr_t args_raw[3] = {(uintptr_t)ptr}; 2783 hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw); 2784 ifree(tsd, ptr, tcache, true); 2785 } 2786 check_entry_exit_locking(tsd_tsdn(tsd)); 2787 } 2788 } 2789 2790 JEMALLOC_ALWAYS_INLINE 2791 bool free_fastpath(void *ptr, size_t size, bool size_hint) { 2792 tsd_t *tsd = tsd_get(false); 2793 if (unlikely(!tsd || !tsd_fast(tsd))) { 2794 return false; 2795 } 2796 2797 tcache_t *tcache = tsd_tcachep_get(tsd); 2798 2799 alloc_ctx_t alloc_ctx; 2800 /* 2801 * If !config_cache_oblivious, we can check PAGE alignment to 2802 * detect sampled objects. Otherwise addresses are 2803 * randomized, and we have to look it up in the rtree anyway. 2804 * See also isfree(). 2805 */ 2806 if (!size_hint || config_cache_oblivious) { 2807 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 2808 bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), &extents_rtree, 2809 rtree_ctx, (uintptr_t)ptr, 2810 &alloc_ctx.szind, &alloc_ctx.slab); 2811 2812 /* Note: profiled objects will have alloc_ctx.slab set */ 2813 if (!res || !alloc_ctx.slab) { 2814 return false; 2815 } 2816 assert(alloc_ctx.szind != SC_NSIZES); 2817 } else { 2818 /* 2819 * Check for both sizes that are too large, and for sampled objects. 2820 * Sampled objects are always page-aligned. The sampled object check 2821 * will also check for null ptr. 2822 */ 2823 if (size > SC_LOOKUP_MAXCLASS || (((uintptr_t)ptr & PAGE_MASK) == 0)) { 2824 return false; 2825 } 2826 alloc_ctx.szind = sz_size2index_lookup(size); 2827 } 2828 2829 if (unlikely(ticker_trytick(&tcache->gc_ticker))) { 2830 return false; 2831 } 2832 2833 cache_bin_t *bin = tcache_small_bin_get(tcache, alloc_ctx.szind); 2834 cache_bin_info_t *bin_info = &tcache_bin_info[alloc_ctx.szind]; 2835 if (!cache_bin_dalloc_easy(bin, bin_info, ptr)) { 2836 return false; 2837 } 2838 2839 if (config_stats) { 2840 size_t usize = sz_index2size(alloc_ctx.szind); 2841 *tsd_thread_deallocatedp_get(tsd) += usize; 2842 } 2843 2844 return true; 2845 } 2846 2847 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2848 je_free(void *ptr) { 2849 LOG("core.free.entry", "ptr: %p", ptr); 2850 2851 if (!free_fastpath(ptr, 0, false)) { 2852 free_default(ptr); 2853 } 2854 2855 LOG("core.free.exit", ""); 2856 } 2857 2858 /* 2859 * End malloc(3)-compatible functions. 2860 */ 2861 /******************************************************************************/ 2862 /* 2863 * Begin non-standard override functions. 2864 */ 2865 2866 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 2867 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2868 void JEMALLOC_NOTHROW * 2869 JEMALLOC_ATTR(malloc) 2870 je_memalign(size_t alignment, size_t size) { 2871 void *ret; 2872 static_opts_t sopts; 2873 dynamic_opts_t dopts; 2874 2875 LOG("core.memalign.entry", "alignment: %zu, size: %zu\n", alignment, 2876 size); 2877 2878 static_opts_init(&sopts); 2879 dynamic_opts_init(&dopts); 2880 2881 sopts.min_alignment = 1; 2882 sopts.oom_string = 2883 "<jemalloc>: Error allocating aligned memory: out of memory\n"; 2884 sopts.invalid_alignment_string = 2885 "<jemalloc>: Error allocating aligned memory: invalid alignment\n"; 2886 sopts.null_out_result_on_error = true; 2887 2888 dopts.result = &ret; 2889 dopts.num_items = 1; 2890 dopts.item_size = size; 2891 dopts.alignment = alignment; 2892 2893 imalloc(&sopts, &dopts); 2894 if (sopts.slow) { 2895 uintptr_t args[3] = {alignment, size}; 2896 hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret, 2897 args); 2898 } 2899 2900 LOG("core.memalign.exit", "result: %p", ret); 2901 return ret; 2902 } 2903 #endif 2904 2905 #ifdef JEMALLOC_OVERRIDE_VALLOC 2906 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2907 void JEMALLOC_NOTHROW * 2908 JEMALLOC_ATTR(malloc) 2909 je_valloc(size_t size) { 2910 void *ret; 2911 2912 static_opts_t sopts; 2913 dynamic_opts_t dopts; 2914 2915 LOG("core.valloc.entry", "size: %zu\n", size); 2916 2917 static_opts_init(&sopts); 2918 dynamic_opts_init(&dopts); 2919 2920 sopts.null_out_result_on_error = true; 2921 sopts.min_alignment = PAGE; 2922 sopts.oom_string = 2923 "<jemalloc>: Error allocating aligned memory: out of memory\n"; 2924 sopts.invalid_alignment_string = 2925 "<jemalloc>: Error allocating aligned memory: invalid alignment\n"; 2926 2927 dopts.result = &ret; 2928 dopts.num_items = 1; 2929 dopts.item_size = size; 2930 dopts.alignment = PAGE; 2931 2932 imalloc(&sopts, &dopts); 2933 if (sopts.slow) { 2934 uintptr_t args[3] = {size}; 2935 hook_invoke_alloc(hook_alloc_valloc, ret, (uintptr_t)ret, args); 2936 } 2937 2938 LOG("core.valloc.exit", "result: %p\n", ret); 2939 return ret; 2940 } 2941 #endif 2942 2943 #if defined(JEMALLOC_IS_MALLOC) && defined(JEMALLOC_GLIBC_MALLOC_HOOK) 2944 /* 2945 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 2946 * to inconsistently reference libc's malloc(3)-compatible functions 2947 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 2948 * 2949 * These definitions interpose hooks in glibc. The functions are actually 2950 * passed an extra argument for the caller return address, which will be 2951 * ignored. 2952 */ 2953 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; 2954 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; 2955 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; 2956 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK 2957 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = 2958 je_memalign; 2959 # endif 2960 2961 # ifdef CPU_COUNT 2962 /* 2963 * To enable static linking with glibc, the libc specific malloc interface must 2964 * be implemented also, so none of glibc's malloc.o functions are added to the 2965 * link. 2966 */ 2967 # define ALIAS(je_fn) __attribute__((alias (#je_fn), used)) 2968 /* To force macro expansion of je_ prefix before stringification. */ 2969 # define PREALIAS(je_fn) ALIAS(je_fn) 2970 # ifdef JEMALLOC_OVERRIDE___LIBC_CALLOC 2971 void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc); 2972 # endif 2973 # ifdef JEMALLOC_OVERRIDE___LIBC_FREE 2974 void __libc_free(void* ptr) PREALIAS(je_free); 2975 # endif 2976 # ifdef JEMALLOC_OVERRIDE___LIBC_MALLOC 2977 void *__libc_malloc(size_t size) PREALIAS(je_malloc); 2978 # endif 2979 # ifdef JEMALLOC_OVERRIDE___LIBC_MEMALIGN 2980 void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign); 2981 # endif 2982 # ifdef JEMALLOC_OVERRIDE___LIBC_REALLOC 2983 void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc); 2984 # endif 2985 # ifdef JEMALLOC_OVERRIDE___LIBC_VALLOC 2986 void *__libc_valloc(size_t size) PREALIAS(je_valloc); 2987 # endif 2988 # ifdef JEMALLOC_OVERRIDE___POSIX_MEMALIGN 2989 int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign); 2990 # endif 2991 # undef PREALIAS 2992 # undef ALIAS 2993 # endif 2994 #endif 2995 2996 /* 2997 * End non-standard override functions. 2998 */ 2999 /******************************************************************************/ 3000 /* 3001 * Begin non-standard functions. 3002 */ 3003 3004 #ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API 3005 3006 #define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y 3007 #define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \ 3008 JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) 3009 3010 typedef struct { 3011 void *ptr; 3012 size_t size; 3013 } smallocx_return_t; 3014 3015 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 3016 smallocx_return_t JEMALLOC_NOTHROW 3017 /* 3018 * The attribute JEMALLOC_ATTR(malloc) cannot be used due to: 3019 * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488 3020 */ 3021 JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT) 3022 (size_t size, int flags) { 3023 /* 3024 * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be 3025 * used here because it makes writing beyond the `size` 3026 * of the `ptr` undefined behavior, but the objective 3027 * of this function is to allow writing beyond `size` 3028 * up to `smallocx_return_t::size`. 3029 */ 3030 smallocx_return_t ret; 3031 static_opts_t sopts; 3032 dynamic_opts_t dopts; 3033 3034 LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags); 3035 3036 static_opts_init(&sopts); 3037 dynamic_opts_init(&dopts); 3038 3039 sopts.assert_nonempty_alloc = true; 3040 sopts.null_out_result_on_error = true; 3041 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n"; 3042 sopts.usize = true; 3043 3044 dopts.result = &ret.ptr; 3045 dopts.num_items = 1; 3046 dopts.item_size = size; 3047 if (unlikely(flags != 0)) { 3048 if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) { 3049 dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); 3050 } 3051 3052 dopts.zero = MALLOCX_ZERO_GET(flags); 3053 3054 if ((flags & MALLOCX_TCACHE_MASK) != 0) { 3055 if ((flags & MALLOCX_TCACHE_MASK) 3056 == MALLOCX_TCACHE_NONE) { 3057 dopts.tcache_ind = TCACHE_IND_NONE; 3058 } else { 3059 dopts.tcache_ind = MALLOCX_TCACHE_GET(flags); 3060 } 3061 } else { 3062 dopts.tcache_ind = TCACHE_IND_AUTOMATIC; 3063 } 3064 3065 if ((flags & MALLOCX_ARENA_MASK) != 0) 3066 dopts.arena_ind = MALLOCX_ARENA_GET(flags); 3067 } 3068 3069 imalloc(&sopts, &dopts); 3070 assert(dopts.usize == je_nallocx(size, flags)); 3071 ret.size = dopts.usize; 3072 3073 LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size); 3074 return ret; 3075 } 3076 #undef JEMALLOC_SMALLOCX_CONCAT_HELPER 3077 #undef JEMALLOC_SMALLOCX_CONCAT_HELPER2 3078 #endif 3079 3080 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 3081 void JEMALLOC_NOTHROW * 3082 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 3083 je_mallocx(size_t size, int flags) { 3084 void *ret; 3085 static_opts_t sopts; 3086 dynamic_opts_t dopts; 3087 3088 LOG("core.mallocx.entry", "size: %zu, flags: %d", size, flags); 3089 3090 static_opts_init(&sopts); 3091 dynamic_opts_init(&dopts); 3092 3093 sopts.assert_nonempty_alloc = true; 3094 sopts.null_out_result_on_error = true; 3095 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n"; 3096 3097 dopts.result = &ret; 3098 dopts.num_items = 1; 3099 dopts.item_size = size; 3100 if (unlikely(flags != 0)) { 3101 if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) { 3102 dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); 3103 } 3104 3105 dopts.zero = MALLOCX_ZERO_GET(flags); 3106 3107 if ((flags & MALLOCX_TCACHE_MASK) != 0) { 3108 if ((flags & MALLOCX_TCACHE_MASK) 3109 == MALLOCX_TCACHE_NONE) { 3110 dopts.tcache_ind = TCACHE_IND_NONE; 3111 } else { 3112 dopts.tcache_ind = MALLOCX_TCACHE_GET(flags); 3113 } 3114 } else { 3115 dopts.tcache_ind = TCACHE_IND_AUTOMATIC; 3116 } 3117 3118 if ((flags & MALLOCX_ARENA_MASK) != 0) 3119 dopts.arena_ind = MALLOCX_ARENA_GET(flags); 3120 } 3121 3122 imalloc(&sopts, &dopts); 3123 if (sopts.slow) { 3124 uintptr_t args[3] = {size, flags}; 3125 hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret, 3126 args); 3127 } 3128 3129 LOG("core.mallocx.exit", "result: %p", ret); 3130 return ret; 3131 } 3132 3133 static void * 3134 irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, 3135 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, 3136 prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) { 3137 void *p; 3138 3139 if (tctx == NULL) { 3140 return NULL; 3141 } 3142 if (usize <= SC_SMALL_MAXCLASS) { 3143 p = iralloct(tsdn, old_ptr, old_usize, 3144 SC_LARGE_MINCLASS, alignment, zero, tcache, 3145 arena, hook_args); 3146 if (p == NULL) { 3147 return NULL; 3148 } 3149 arena_prof_promote(tsdn, p, usize); 3150 } else { 3151 p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero, 3152 tcache, arena, hook_args); 3153 } 3154 3155 return p; 3156 } 3157 3158 JEMALLOC_ALWAYS_INLINE void * 3159 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, 3160 size_t alignment, size_t *usize, bool zero, tcache_t *tcache, 3161 arena_t *arena, alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { 3162 void *p; 3163 bool prof_active; 3164 prof_tctx_t *old_tctx, *tctx; 3165 3166 prof_active = prof_active_get_unlocked(); 3167 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx); 3168 tctx = prof_alloc_prep(tsd, *usize, prof_active, false); 3169 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 3170 p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize, 3171 *usize, alignment, zero, tcache, arena, tctx, hook_args); 3172 } else { 3173 p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment, 3174 zero, tcache, arena, hook_args); 3175 } 3176 if (unlikely(p == NULL)) { 3177 prof_alloc_rollback(tsd, tctx, false); 3178 return NULL; 3179 } 3180 3181 if (p == old_ptr && alignment != 0) { 3182 /* 3183 * The allocation did not move, so it is possible that the size 3184 * class is smaller than would guarantee the requested 3185 * alignment, and that the alignment constraint was 3186 * serendipitously satisfied. Additionally, old_usize may not 3187 * be the same as the current usize because of in-place large 3188 * reallocation. Therefore, query the actual value of usize. 3189 */ 3190 *usize = isalloc(tsd_tsdn(tsd), p); 3191 } 3192 prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr, 3193 old_usize, old_tctx); 3194 3195 return p; 3196 } 3197 3198 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 3199 void JEMALLOC_NOTHROW * 3200 JEMALLOC_ALLOC_SIZE(2) 3201 je_rallocx(void *ptr, size_t size, int flags) { 3202 void *p; 3203 tsd_t *tsd; 3204 size_t usize; 3205 size_t old_usize; 3206 size_t alignment = MALLOCX_ALIGN_GET(flags); 3207 bool zero = flags & MALLOCX_ZERO; 3208 arena_t *arena; 3209 tcache_t *tcache; 3210 3211 LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, 3212 size, flags); 3213 3214 3215 assert(ptr != NULL); 3216 assert(size != 0); 3217 assert(malloc_initialized() || IS_INITIALIZER); 3218 tsd = tsd_fetch(); 3219 check_entry_exit_locking(tsd_tsdn(tsd)); 3220 3221 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 3222 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 3223 arena = arena_get(tsd_tsdn(tsd), arena_ind, true); 3224 if (unlikely(arena == NULL)) { 3225 goto label_oom; 3226 } 3227 } else { 3228 arena = NULL; 3229 } 3230 3231 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 3232 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) { 3233 tcache = NULL; 3234 } else { 3235 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 3236 } 3237 } else { 3238 tcache = tcache_get(tsd); 3239 } 3240 3241 alloc_ctx_t alloc_ctx; 3242 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 3243 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, 3244 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); 3245 assert(alloc_ctx.szind != SC_NSIZES); 3246 old_usize = sz_index2size(alloc_ctx.szind); 3247 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); 3248 3249 hook_ralloc_args_t hook_args = {false, {(uintptr_t)ptr, size, flags, 3250 0}}; 3251 if (config_prof && opt_prof) { 3252 usize = (alignment == 0) ? 3253 sz_s2u(size) : sz_sa2u(size, alignment); 3254 if (unlikely(usize == 0 3255 || usize > SC_LARGE_MAXCLASS)) { 3256 goto label_oom; 3257 } 3258 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, 3259 zero, tcache, arena, &alloc_ctx, &hook_args); 3260 if (unlikely(p == NULL)) { 3261 goto label_oom; 3262 } 3263 } else { 3264 p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment, 3265 zero, tcache, arena, &hook_args); 3266 if (unlikely(p == NULL)) { 3267 goto label_oom; 3268 } 3269 if (config_stats) { 3270 usize = isalloc(tsd_tsdn(tsd), p); 3271 } 3272 } 3273 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 3274 3275 if (config_stats) { 3276 *tsd_thread_allocatedp_get(tsd) += usize; 3277 *tsd_thread_deallocatedp_get(tsd) += old_usize; 3278 } 3279 UTRACE(ptr, size, p); 3280 check_entry_exit_locking(tsd_tsdn(tsd)); 3281 3282 LOG("core.rallocx.exit", "result: %p", p); 3283 return p; 3284 label_oom: 3285 if (config_xmalloc && unlikely(opt_xmalloc)) { 3286 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 3287 abort(); 3288 } 3289 UTRACE(ptr, size, 0); 3290 check_entry_exit_locking(tsd_tsdn(tsd)); 3291 3292 LOG("core.rallocx.exit", "result: %p", NULL); 3293 return NULL; 3294 } 3295 3296 JEMALLOC_ALWAYS_INLINE size_t 3297 ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, 3298 size_t extra, size_t alignment, bool zero) { 3299 size_t newsize; 3300 3301 if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero, 3302 &newsize)) { 3303 return old_usize; 3304 } 3305 3306 return newsize; 3307 } 3308 3309 static size_t 3310 ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, 3311 size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) { 3312 size_t usize; 3313 3314 if (tctx == NULL) { 3315 return old_usize; 3316 } 3317 usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment, 3318 zero); 3319 3320 return usize; 3321 } 3322 3323 JEMALLOC_ALWAYS_INLINE size_t 3324 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, 3325 size_t extra, size_t alignment, bool zero, alloc_ctx_t *alloc_ctx) { 3326 size_t usize_max, usize; 3327 bool prof_active; 3328 prof_tctx_t *old_tctx, *tctx; 3329 3330 prof_active = prof_active_get_unlocked(); 3331 old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr, alloc_ctx); 3332 /* 3333 * usize isn't knowable before ixalloc() returns when extra is non-zero. 3334 * Therefore, compute its maximum possible value and use that in 3335 * prof_alloc_prep() to decide whether to capture a backtrace. 3336 * prof_realloc() will use the actual usize to decide whether to sample. 3337 */ 3338 if (alignment == 0) { 3339 usize_max = sz_s2u(size+extra); 3340 assert(usize_max > 0 3341 && usize_max <= SC_LARGE_MAXCLASS); 3342 } else { 3343 usize_max = sz_sa2u(size+extra, alignment); 3344 if (unlikely(usize_max == 0 3345 || usize_max > SC_LARGE_MAXCLASS)) { 3346 /* 3347 * usize_max is out of range, and chances are that 3348 * allocation will fail, but use the maximum possible 3349 * value and carry on with prof_alloc_prep(), just in 3350 * case allocation succeeds. 3351 */ 3352 usize_max = SC_LARGE_MAXCLASS; 3353 } 3354 } 3355 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); 3356 3357 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 3358 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize, 3359 size, extra, alignment, zero, tctx); 3360 } else { 3361 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, 3362 extra, alignment, zero); 3363 } 3364 if (usize == old_usize) { 3365 prof_alloc_rollback(tsd, tctx, false); 3366 return usize; 3367 } 3368 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize, 3369 old_tctx); 3370 3371 return usize; 3372 } 3373 3374 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 3375 je_xallocx(void *ptr, size_t size, size_t extra, int flags) { 3376 tsd_t *tsd; 3377 size_t usize, old_usize; 3378 size_t alignment = MALLOCX_ALIGN_GET(flags); 3379 bool zero = flags & MALLOCX_ZERO; 3380 3381 LOG("core.xallocx.entry", "ptr: %p, size: %zu, extra: %zu, " 3382 "flags: %d", ptr, size, extra, flags); 3383 3384 assert(ptr != NULL); 3385 assert(size != 0); 3386 assert(SIZE_T_MAX - size >= extra); 3387 assert(malloc_initialized() || IS_INITIALIZER); 3388 tsd = tsd_fetch(); 3389 check_entry_exit_locking(tsd_tsdn(tsd)); 3390 3391 alloc_ctx_t alloc_ctx; 3392 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); 3393 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, 3394 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); 3395 assert(alloc_ctx.szind != SC_NSIZES); 3396 old_usize = sz_index2size(alloc_ctx.szind); 3397 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); 3398 /* 3399 * The API explicitly absolves itself of protecting against (size + 3400 * extra) numerical overflow, but we may need to clamp extra to avoid 3401 * exceeding SC_LARGE_MAXCLASS. 3402 * 3403 * Ordinarily, size limit checking is handled deeper down, but here we 3404 * have to check as part of (size + extra) clamping, since we need the 3405 * clamped value in the above helper functions. 3406 */ 3407 if (unlikely(size > SC_LARGE_MAXCLASS)) { 3408 usize = old_usize; 3409 goto label_not_resized; 3410 } 3411 if (unlikely(SC_LARGE_MAXCLASS - size < extra)) { 3412 extra = SC_LARGE_MAXCLASS - size; 3413 } 3414 3415 if (config_prof && opt_prof) { 3416 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, 3417 alignment, zero, &alloc_ctx); 3418 } else { 3419 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, 3420 extra, alignment, zero); 3421 } 3422 if (unlikely(usize == old_usize)) { 3423 goto label_not_resized; 3424 } 3425 3426 if (config_stats) { 3427 *tsd_thread_allocatedp_get(tsd) += usize; 3428 *tsd_thread_deallocatedp_get(tsd) += old_usize; 3429 } 3430 label_not_resized: 3431 if (unlikely(!tsd_fast(tsd))) { 3432 uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags}; 3433 hook_invoke_expand(hook_expand_xallocx, ptr, old_usize, 3434 usize, (uintptr_t)usize, args); 3435 } 3436 3437 UTRACE(ptr, size, ptr); 3438 check_entry_exit_locking(tsd_tsdn(tsd)); 3439 3440 LOG("core.xallocx.exit", "result: %zu", usize); 3441 return usize; 3442 } 3443 3444 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 3445 JEMALLOC_ATTR(pure) 3446 je_sallocx(const void *ptr, int flags) { 3447 size_t usize; 3448 tsdn_t *tsdn; 3449 3450 LOG("core.sallocx.entry", "ptr: %p, flags: %d", ptr, flags); 3451 3452 assert(malloc_initialized() || IS_INITIALIZER); 3453 assert(ptr != NULL); 3454 3455 tsdn = tsdn_fetch(); 3456 check_entry_exit_locking(tsdn); 3457 3458 if (config_debug || force_ivsalloc) { 3459 usize = ivsalloc(tsdn, ptr); 3460 assert(force_ivsalloc || usize != 0); 3461 } else { 3462 usize = isalloc(tsdn, ptr); 3463 } 3464 3465 check_entry_exit_locking(tsdn); 3466 3467 LOG("core.sallocx.exit", "result: %zu", usize); 3468 return usize; 3469 } 3470 3471 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 3472 je_dallocx(void *ptr, int flags) { 3473 LOG("core.dallocx.entry", "ptr: %p, flags: %d", ptr, flags); 3474 3475 assert(ptr != NULL); 3476 assert(malloc_initialized() || IS_INITIALIZER); 3477 3478 tsd_t *tsd = tsd_fetch(); 3479 bool fast = tsd_fast(tsd); 3480 check_entry_exit_locking(tsd_tsdn(tsd)); 3481 3482 tcache_t *tcache; 3483 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 3484 /* Not allowed to be reentrant and specify a custom tcache. */ 3485 assert(tsd_reentrancy_level_get(tsd) == 0); 3486 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) { 3487 tcache = NULL; 3488 } else { 3489 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 3490 } 3491 } else { 3492 if (likely(fast)) { 3493 tcache = tsd_tcachep_get(tsd); 3494 assert(tcache == tcache_get(tsd)); 3495 } else { 3496 if (likely(tsd_reentrancy_level_get(tsd) == 0)) { 3497 tcache = tcache_get(tsd); 3498 } else { 3499 tcache = NULL; 3500 } 3501 } 3502 } 3503 3504 UTRACE(ptr, 0, 0); 3505 if (likely(fast)) { 3506 tsd_assert_fast(tsd); 3507 ifree(tsd, ptr, tcache, false); 3508 } else { 3509 uintptr_t args_raw[3] = {(uintptr_t)ptr, flags}; 3510 hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw); 3511 ifree(tsd, ptr, tcache, true); 3512 } 3513 check_entry_exit_locking(tsd_tsdn(tsd)); 3514 3515 LOG("core.dallocx.exit", ""); 3516 } 3517 3518 JEMALLOC_ALWAYS_INLINE size_t 3519 inallocx(tsdn_t *tsdn, size_t size, int flags) { 3520 check_entry_exit_locking(tsdn); 3521 3522 size_t usize; 3523 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) { 3524 usize = sz_s2u(size); 3525 } else { 3526 usize = sz_sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); 3527 } 3528 check_entry_exit_locking(tsdn); 3529 return usize; 3530 } 3531 3532 JEMALLOC_NOINLINE void 3533 sdallocx_default(void *ptr, size_t size, int flags) { 3534 assert(ptr != NULL); 3535 assert(malloc_initialized() || IS_INITIALIZER); 3536 3537 tsd_t *tsd = tsd_fetch(); 3538 bool fast = tsd_fast(tsd); 3539 size_t usize = inallocx(tsd_tsdn(tsd), size, flags); 3540 assert(usize == isalloc(tsd_tsdn(tsd), ptr)); 3541 check_entry_exit_locking(tsd_tsdn(tsd)); 3542 3543 tcache_t *tcache; 3544 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 3545 /* Not allowed to be reentrant and specify a custom tcache. */ 3546 assert(tsd_reentrancy_level_get(tsd) == 0); 3547 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) { 3548 tcache = NULL; 3549 } else { 3550 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 3551 } 3552 } else { 3553 if (likely(fast)) { 3554 tcache = tsd_tcachep_get(tsd); 3555 assert(tcache == tcache_get(tsd)); 3556 } else { 3557 if (likely(tsd_reentrancy_level_get(tsd) == 0)) { 3558 tcache = tcache_get(tsd); 3559 } else { 3560 tcache = NULL; 3561 } 3562 } 3563 } 3564 3565 UTRACE(ptr, 0, 0); 3566 if (likely(fast)) { 3567 tsd_assert_fast(tsd); 3568 isfree(tsd, ptr, usize, tcache, false); 3569 } else { 3570 uintptr_t args_raw[3] = {(uintptr_t)ptr, size, flags}; 3571 hook_invoke_dalloc(hook_dalloc_sdallocx, ptr, args_raw); 3572 isfree(tsd, ptr, usize, tcache, true); 3573 } 3574 check_entry_exit_locking(tsd_tsdn(tsd)); 3575 3576 } 3577 3578 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 3579 je_sdallocx(void *ptr, size_t size, int flags) { 3580 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, 3581 size, flags); 3582 3583 if (flags !=0 || !free_fastpath(ptr, size, true)) { 3584 sdallocx_default(ptr, size, flags); 3585 } 3586 3587 LOG("core.sdallocx.exit", ""); 3588 } 3589 3590 void JEMALLOC_NOTHROW 3591 je_sdallocx_noflags(void *ptr, size_t size) { 3592 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: 0", ptr, 3593 size); 3594 3595 if (!free_fastpath(ptr, size, true)) { 3596 sdallocx_default(ptr, size, 0); 3597 } 3598 3599 LOG("core.sdallocx.exit", ""); 3600 } 3601 3602 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 3603 JEMALLOC_ATTR(pure) 3604 je_nallocx(size_t size, int flags) { 3605 size_t usize; 3606 tsdn_t *tsdn; 3607 3608 assert(size != 0); 3609 3610 if (unlikely(malloc_init())) { 3611 LOG("core.nallocx.exit", "result: %zu", ZU(0)); 3612 return 0; 3613 } 3614 3615 tsdn = tsdn_fetch(); 3616 check_entry_exit_locking(tsdn); 3617 3618 usize = inallocx(tsdn, size, flags); 3619 if (unlikely(usize > SC_LARGE_MAXCLASS)) { 3620 LOG("core.nallocx.exit", "result: %zu", ZU(0)); 3621 return 0; 3622 } 3623 3624 check_entry_exit_locking(tsdn); 3625 LOG("core.nallocx.exit", "result: %zu", usize); 3626 return usize; 3627 } 3628 3629 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 3630 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 3631 size_t newlen) { 3632 int ret; 3633 tsd_t *tsd; 3634 3635 LOG("core.mallctl.entry", "name: %s", name); 3636 3637 if (unlikely(malloc_init())) { 3638 LOG("core.mallctl.exit", "result: %d", EAGAIN); 3639 return EAGAIN; 3640 } 3641 3642 tsd = tsd_fetch(); 3643 check_entry_exit_locking(tsd_tsdn(tsd)); 3644 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); 3645 check_entry_exit_locking(tsd_tsdn(tsd)); 3646 3647 LOG("core.mallctl.exit", "result: %d", ret); 3648 return ret; 3649 } 3650 3651 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 3652 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) { 3653 int ret; 3654 3655 LOG("core.mallctlnametomib.entry", "name: %s", name); 3656 3657 if (unlikely(malloc_init())) { 3658 LOG("core.mallctlnametomib.exit", "result: %d", EAGAIN); 3659 return EAGAIN; 3660 } 3661 3662 tsd_t *tsd = tsd_fetch(); 3663 check_entry_exit_locking(tsd_tsdn(tsd)); 3664 ret = ctl_nametomib(tsd, name, mibp, miblenp); 3665 check_entry_exit_locking(tsd_tsdn(tsd)); 3666 3667 LOG("core.mallctlnametomib.exit", "result: %d", ret); 3668 return ret; 3669 } 3670 3671 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 3672 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 3673 void *newp, size_t newlen) { 3674 int ret; 3675 tsd_t *tsd; 3676 3677 LOG("core.mallctlbymib.entry", ""); 3678 3679 if (unlikely(malloc_init())) { 3680 LOG("core.mallctlbymib.exit", "result: %d", EAGAIN); 3681 return EAGAIN; 3682 } 3683 3684 tsd = tsd_fetch(); 3685 check_entry_exit_locking(tsd_tsdn(tsd)); 3686 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); 3687 check_entry_exit_locking(tsd_tsdn(tsd)); 3688 LOG("core.mallctlbymib.exit", "result: %d", ret); 3689 return ret; 3690 } 3691 3692 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 3693 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 3694 const char *opts) { 3695 tsdn_t *tsdn; 3696 3697 LOG("core.malloc_stats_print.entry", ""); 3698 3699 tsdn = tsdn_fetch(); 3700 check_entry_exit_locking(tsdn); 3701 stats_print(write_cb, cbopaque, opts); 3702 check_entry_exit_locking(tsdn); 3703 LOG("core.malloc_stats_print.exit", ""); 3704 } 3705 3706 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 3707 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { 3708 size_t ret; 3709 tsdn_t *tsdn; 3710 3711 LOG("core.malloc_usable_size.entry", "ptr: %p", ptr); 3712 3713 assert(malloc_initialized() || IS_INITIALIZER); 3714 3715 tsdn = tsdn_fetch(); 3716 check_entry_exit_locking(tsdn); 3717 3718 if (unlikely(ptr == NULL)) { 3719 ret = 0; 3720 } else { 3721 if (config_debug || force_ivsalloc) { 3722 ret = ivsalloc(tsdn, ptr); 3723 assert(force_ivsalloc || ret != 0); 3724 } else { 3725 ret = isalloc(tsdn, ptr); 3726 } 3727 } 3728 3729 check_entry_exit_locking(tsdn); 3730 LOG("core.malloc_usable_size.exit", "result: %zu", ret); 3731 return ret; 3732 } 3733 3734 /* 3735 * End non-standard functions. 3736 */ 3737 /******************************************************************************/ 3738 /* 3739 * Begin compatibility functions. 3740 */ 3741 3742 #define ALLOCM_LG_ALIGN(la) (la) 3743 #define ALLOCM_ALIGN(a) (ffsl(a)-1) 3744 #define ALLOCM_ZERO ((int)0x40) 3745 #define ALLOCM_NO_MOVE ((int)0x80) 3746 3747 #define ALLOCM_SUCCESS 0 3748 #define ALLOCM_ERR_OOM 1 3749 #define ALLOCM_ERR_NOT_MOVED 2 3750 3751 int 3752 je_allocm(void **ptr, size_t *rsize, size_t size, int flags) { 3753 assert(ptr != NULL); 3754 3755 void *p = je_mallocx(size, flags); 3756 if (p == NULL) { 3757 return (ALLOCM_ERR_OOM); 3758 } 3759 if (rsize != NULL) { 3760 *rsize = isalloc(tsdn_fetch(), p); 3761 } 3762 *ptr = p; 3763 return ALLOCM_SUCCESS; 3764 } 3765 3766 int 3767 je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) { 3768 assert(ptr != NULL); 3769 assert(*ptr != NULL); 3770 assert(size != 0); 3771 assert(SIZE_T_MAX - size >= extra); 3772 3773 int ret; 3774 bool no_move = flags & ALLOCM_NO_MOVE; 3775 3776 if (no_move) { 3777 size_t usize = je_xallocx(*ptr, size, extra, flags); 3778 ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; 3779 if (rsize != NULL) { 3780 *rsize = usize; 3781 } 3782 } else { 3783 void *p = je_rallocx(*ptr, size+extra, flags); 3784 if (p != NULL) { 3785 *ptr = p; 3786 ret = ALLOCM_SUCCESS; 3787 } else { 3788 ret = ALLOCM_ERR_OOM; 3789 } 3790 if (rsize != NULL) { 3791 *rsize = isalloc(tsdn_fetch(), *ptr); 3792 } 3793 } 3794 return ret; 3795 } 3796 3797 int 3798 je_sallocm(const void *ptr, size_t *rsize, int flags) { 3799 assert(rsize != NULL); 3800 *rsize = je_sallocx(ptr, flags); 3801 return ALLOCM_SUCCESS; 3802 } 3803 3804 int 3805 je_dallocm(void *ptr, int flags) { 3806 je_dallocx(ptr, flags); 3807 return ALLOCM_SUCCESS; 3808 } 3809 3810 int 3811 je_nallocm(size_t *rsize, size_t size, int flags) { 3812 size_t usize = je_nallocx(size, flags); 3813 if (usize == 0) { 3814 return ALLOCM_ERR_OOM; 3815 } 3816 if (rsize != NULL) { 3817 *rsize = usize; 3818 } 3819 return ALLOCM_SUCCESS; 3820 } 3821 3822 #undef ALLOCM_LG_ALIGN 3823 #undef ALLOCM_ALIGN 3824 #undef ALLOCM_ZERO 3825 #undef ALLOCM_NO_MOVE 3826 3827 #undef ALLOCM_SUCCESS 3828 #undef ALLOCM_ERR_OOM 3829 #undef ALLOCM_ERR_NOT_MOVED 3830 3831 /* 3832 * End compatibility functions. 3833 */ 3834 /******************************************************************************/ 3835 /* 3836 * The following functions are used by threading libraries for protection of 3837 * malloc during fork(). 3838 */ 3839 3840 /* 3841 * If an application creates a thread before doing any allocation in the main 3842 * thread, then calls fork(2) in the main thread followed by memory allocation 3843 * in the child process, a race can occur that results in deadlock within the 3844 * child: the main thread may have forked while the created thread had 3845 * partially initialized the allocator. Ordinarily jemalloc prevents 3846 * fork/malloc races via the following functions it registers during 3847 * initialization using pthread_atfork(), but of course that does no good if 3848 * the allocator isn't fully initialized at fork time. The following library 3849 * constructor is a partial solution to this problem. It may still be possible 3850 * to trigger the deadlock described above, but doing so would involve forking 3851 * via a library constructor that runs before jemalloc's runs. 3852 */ 3853 #ifndef JEMALLOC_JET 3854 JEMALLOC_ATTR(constructor) 3855 static void 3856 jemalloc_constructor(void) { 3857 malloc_init(); 3858 } 3859 #endif 3860 3861 #ifndef JEMALLOC_MUTEX_INIT_CB 3862 void 3863 jemalloc_prefork(void) 3864 #else 3865 JEMALLOC_EXPORT void 3866 _malloc_prefork(void) 3867 #endif 3868 { 3869 tsd_t *tsd; 3870 unsigned i, j, narenas; 3871 arena_t *arena; 3872 3873 #ifdef JEMALLOC_MUTEX_INIT_CB 3874 if (!malloc_initialized()) { 3875 return; 3876 } 3877 #endif 3878 assert(malloc_initialized()); 3879 3880 tsd = tsd_fetch(); 3881 3882 narenas = narenas_total_get(); 3883 3884 witness_prefork(tsd_witness_tsdp_get(tsd)); 3885 /* Acquire all mutexes in a safe order. */ 3886 ctl_prefork(tsd_tsdn(tsd)); 3887 tcache_prefork(tsd_tsdn(tsd)); 3888 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock); 3889 if (have_background_thread) { 3890 background_thread_prefork0(tsd_tsdn(tsd)); 3891 } 3892 prof_prefork0(tsd_tsdn(tsd)); 3893 if (have_background_thread) { 3894 background_thread_prefork1(tsd_tsdn(tsd)); 3895 } 3896 /* Break arena prefork into stages to preserve lock order. */ 3897 for (i = 0; i < 8; i++) { 3898 for (j = 0; j < narenas; j++) { 3899 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) != 3900 NULL) { 3901 switch (i) { 3902 case 0: 3903 arena_prefork0(tsd_tsdn(tsd), arena); 3904 break; 3905 case 1: 3906 arena_prefork1(tsd_tsdn(tsd), arena); 3907 break; 3908 case 2: 3909 arena_prefork2(tsd_tsdn(tsd), arena); 3910 break; 3911 case 3: 3912 arena_prefork3(tsd_tsdn(tsd), arena); 3913 break; 3914 case 4: 3915 arena_prefork4(tsd_tsdn(tsd), arena); 3916 break; 3917 case 5: 3918 arena_prefork5(tsd_tsdn(tsd), arena); 3919 break; 3920 case 6: 3921 arena_prefork6(tsd_tsdn(tsd), arena); 3922 break; 3923 case 7: 3924 arena_prefork7(tsd_tsdn(tsd), arena); 3925 break; 3926 default: not_reached(); 3927 } 3928 } 3929 } 3930 } 3931 prof_prefork1(tsd_tsdn(tsd)); 3932 tsd_prefork(tsd); 3933 } 3934 3935 #ifndef JEMALLOC_MUTEX_INIT_CB 3936 void 3937 jemalloc_postfork_parent(void) 3938 #else 3939 JEMALLOC_EXPORT void 3940 _malloc_postfork(void) 3941 #endif 3942 { 3943 tsd_t *tsd; 3944 unsigned i, narenas; 3945 3946 #ifdef JEMALLOC_MUTEX_INIT_CB 3947 if (!malloc_initialized()) { 3948 return; 3949 } 3950 #endif 3951 assert(malloc_initialized()); 3952 3953 tsd = tsd_fetch(); 3954 3955 tsd_postfork_parent(tsd); 3956 3957 witness_postfork_parent(tsd_witness_tsdp_get(tsd)); 3958 /* Release all mutexes, now that fork() has completed. */ 3959 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 3960 arena_t *arena; 3961 3962 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) { 3963 arena_postfork_parent(tsd_tsdn(tsd), arena); 3964 } 3965 } 3966 prof_postfork_parent(tsd_tsdn(tsd)); 3967 if (have_background_thread) { 3968 background_thread_postfork_parent(tsd_tsdn(tsd)); 3969 } 3970 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock); 3971 tcache_postfork_parent(tsd_tsdn(tsd)); 3972 ctl_postfork_parent(tsd_tsdn(tsd)); 3973 } 3974 3975 void 3976 jemalloc_postfork_child(void) { 3977 tsd_t *tsd; 3978 unsigned i, narenas; 3979 3980 assert(malloc_initialized()); 3981 3982 tsd = tsd_fetch(); 3983 3984 tsd_postfork_child(tsd); 3985 3986 witness_postfork_child(tsd_witness_tsdp_get(tsd)); 3987 /* Release all mutexes, now that fork() has completed. */ 3988 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 3989 arena_t *arena; 3990 3991 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) { 3992 arena_postfork_child(tsd_tsdn(tsd), arena); 3993 } 3994 } 3995 prof_postfork_child(tsd_tsdn(tsd)); 3996 if (have_background_thread) { 3997 background_thread_postfork_child(tsd_tsdn(tsd)); 3998 } 3999 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock); 4000 tcache_postfork_child(tsd_tsdn(tsd)); 4001 ctl_postfork_child(tsd_tsdn(tsd)); 4002 } 4003 4004 void 4005 _malloc_first_thread(void) 4006 { 4007 4008 (void)malloc_mutex_first_thread(); 4009 } 4010 4011 /******************************************************************************/ 4012