1 #define JEMALLOC_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 /* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */ 8 const char *__malloc_options_1_0 = NULL; 9 __sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); 10 11 /* Runtime configuration options. */ 12 const char *je_malloc_conf JEMALLOC_ATTR(weak); 13 bool opt_abort = 14 #ifdef JEMALLOC_DEBUG 15 true 16 #else 17 false 18 #endif 19 ; 20 const char *opt_junk = 21 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 22 "true" 23 #else 24 "false" 25 #endif 26 ; 27 bool opt_junk_alloc = 28 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 29 true 30 #else 31 false 32 #endif 33 ; 34 bool opt_junk_free = 35 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) 36 true 37 #else 38 false 39 #endif 40 ; 41 42 size_t opt_quarantine = ZU(0); 43 bool opt_redzone = false; 44 bool opt_utrace = false; 45 bool opt_xmalloc = false; 46 bool opt_zero = false; 47 unsigned opt_narenas = 0; 48 49 /* Initialized to true if the process is running inside Valgrind. */ 50 bool in_valgrind; 51 52 unsigned ncpus; 53 54 /* Protects arenas initialization. */ 55 static malloc_mutex_t arenas_lock; 56 /* 57 * Arenas that are used to service external requests. Not all elements of the 58 * arenas array are necessarily used; arenas are created lazily as needed. 59 * 60 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and 61 * arenas. arenas[narenas_auto..narenas_total) are only used if the application 62 * takes some action to create them and allocate from them. 63 */ 64 arena_t **arenas; 65 static unsigned narenas_total; /* Use narenas_total_*(). */ 66 static arena_t *a0; /* arenas[0]; read-only after initialization. */ 67 unsigned narenas_auto; /* Read-only after initialization. */ 68 69 typedef enum { 70 malloc_init_uninitialized = 3, 71 malloc_init_a0_initialized = 2, 72 malloc_init_recursible = 1, 73 malloc_init_initialized = 0 /* Common case --> jnz. */ 74 } malloc_init_t; 75 static malloc_init_t malloc_init_state = malloc_init_uninitialized; 76 77 /* False should be the common case. Set to true to trigger initialization. */ 78 static bool malloc_slow = true; 79 80 /* When malloc_slow is true, set the corresponding bits for sanity check. */ 81 enum { 82 flag_opt_junk_alloc = (1U), 83 flag_opt_junk_free = (1U << 1), 84 flag_opt_quarantine = (1U << 2), 85 flag_opt_zero = (1U << 3), 86 flag_opt_utrace = (1U << 4), 87 flag_in_valgrind = (1U << 5), 88 flag_opt_xmalloc = (1U << 6) 89 }; 90 static uint8_t malloc_slow_flags; 91 92 /* Last entry for overflow detection only. */ 93 JEMALLOC_ALIGNED(CACHELINE) 94 const size_t index2size_tab[NSIZES+1] = { 95 #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 96 ((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)), 97 SIZE_CLASSES 98 #undef SC 99 ZU(0) 100 }; 101 102 JEMALLOC_ALIGNED(CACHELINE) 103 const uint8_t size2index_tab[] = { 104 #if LG_TINY_MIN == 0 105 #warning "Dangerous LG_TINY_MIN" 106 #define S2B_0(i) i, 107 #elif LG_TINY_MIN == 1 108 #warning "Dangerous LG_TINY_MIN" 109 #define S2B_1(i) i, 110 #elif LG_TINY_MIN == 2 111 #warning "Dangerous LG_TINY_MIN" 112 #define S2B_2(i) i, 113 #elif LG_TINY_MIN == 3 114 #define S2B_3(i) i, 115 #elif LG_TINY_MIN == 4 116 #define S2B_4(i) i, 117 #elif LG_TINY_MIN == 5 118 #define S2B_5(i) i, 119 #elif LG_TINY_MIN == 6 120 #define S2B_6(i) i, 121 #elif LG_TINY_MIN == 7 122 #define S2B_7(i) i, 123 #elif LG_TINY_MIN == 8 124 #define S2B_8(i) i, 125 #elif LG_TINY_MIN == 9 126 #define S2B_9(i) i, 127 #elif LG_TINY_MIN == 10 128 #define S2B_10(i) i, 129 #elif LG_TINY_MIN == 11 130 #define S2B_11(i) i, 131 #else 132 #error "Unsupported LG_TINY_MIN" 133 #endif 134 #if LG_TINY_MIN < 1 135 #define S2B_1(i) S2B_0(i) S2B_0(i) 136 #endif 137 #if LG_TINY_MIN < 2 138 #define S2B_2(i) S2B_1(i) S2B_1(i) 139 #endif 140 #if LG_TINY_MIN < 3 141 #define S2B_3(i) S2B_2(i) S2B_2(i) 142 #endif 143 #if LG_TINY_MIN < 4 144 #define S2B_4(i) S2B_3(i) S2B_3(i) 145 #endif 146 #if LG_TINY_MIN < 5 147 #define S2B_5(i) S2B_4(i) S2B_4(i) 148 #endif 149 #if LG_TINY_MIN < 6 150 #define S2B_6(i) S2B_5(i) S2B_5(i) 151 #endif 152 #if LG_TINY_MIN < 7 153 #define S2B_7(i) S2B_6(i) S2B_6(i) 154 #endif 155 #if LG_TINY_MIN < 8 156 #define S2B_8(i) S2B_7(i) S2B_7(i) 157 #endif 158 #if LG_TINY_MIN < 9 159 #define S2B_9(i) S2B_8(i) S2B_8(i) 160 #endif 161 #if LG_TINY_MIN < 10 162 #define S2B_10(i) S2B_9(i) S2B_9(i) 163 #endif 164 #if LG_TINY_MIN < 11 165 #define S2B_11(i) S2B_10(i) S2B_10(i) 166 #endif 167 #define S2B_no(i) 168 #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 169 S2B_##lg_delta_lookup(index) 170 SIZE_CLASSES 171 #undef S2B_3 172 #undef S2B_4 173 #undef S2B_5 174 #undef S2B_6 175 #undef S2B_7 176 #undef S2B_8 177 #undef S2B_9 178 #undef S2B_10 179 #undef S2B_11 180 #undef S2B_no 181 #undef SC 182 }; 183 184 #ifdef JEMALLOC_THREADED_INIT 185 /* Used to let the initializing thread recursively allocate. */ 186 # define NO_INITIALIZER ((unsigned long)0) 187 # define INITIALIZER pthread_self() 188 # define IS_INITIALIZER (malloc_initializer == pthread_self()) 189 static pthread_t malloc_initializer = NO_INITIALIZER; 190 #else 191 # define NO_INITIALIZER false 192 # define INITIALIZER true 193 # define IS_INITIALIZER malloc_initializer 194 static bool malloc_initializer = NO_INITIALIZER; 195 #endif 196 197 /* Used to avoid initialization races. */ 198 #ifdef _WIN32 199 #if _WIN32_WINNT >= 0x0600 200 static malloc_mutex_t init_lock = SRWLOCK_INIT; 201 #else 202 static malloc_mutex_t init_lock; 203 static bool init_lock_initialized = false; 204 205 JEMALLOC_ATTR(constructor) 206 static void WINAPI 207 _init_init_lock(void) 208 { 209 210 /* If another constructor in the same binary is using mallctl to 211 * e.g. setup chunk hooks, it may end up running before this one, 212 * and malloc_init_hard will crash trying to lock the uninitialized 213 * lock. So we force an initialization of the lock in 214 * malloc_init_hard as well. We don't try to care about atomicity 215 * of the accessed to the init_lock_initialized boolean, since it 216 * really only matters early in the process creation, before any 217 * separate thread normally starts doing anything. */ 218 if (!init_lock_initialized) 219 malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT); 220 init_lock_initialized = true; 221 } 222 223 #ifdef _MSC_VER 224 # pragma section(".CRT$XCU", read) 225 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) 226 static const void (WINAPI *init_init_lock)(void) = _init_init_lock; 227 #endif 228 #endif 229 #else 230 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; 231 #endif 232 233 typedef struct { 234 void *p; /* Input pointer (as in realloc(p, s)). */ 235 size_t s; /* Request size. */ 236 void *r; /* Result pointer. */ 237 } malloc_utrace_t; 238 239 #ifdef JEMALLOC_UTRACE 240 # define UTRACE(a, b, c) do { \ 241 if (unlikely(opt_utrace)) { \ 242 int utrace_serrno = errno; \ 243 malloc_utrace_t ut; \ 244 ut.p = (a); \ 245 ut.s = (b); \ 246 ut.r = (c); \ 247 utrace(&ut, sizeof(ut)); \ 248 errno = utrace_serrno; \ 249 } \ 250 } while (0) 251 #else 252 # define UTRACE(a, b, c) 253 #endif 254 255 /******************************************************************************/ 256 /* 257 * Function prototypes for static functions that are referenced prior to 258 * definition. 259 */ 260 261 static bool malloc_init_hard_a0(void); 262 static bool malloc_init_hard(void); 263 264 /******************************************************************************/ 265 /* 266 * Begin miscellaneous support functions. 267 */ 268 269 JEMALLOC_ALWAYS_INLINE_C bool 270 malloc_initialized(void) 271 { 272 273 return (malloc_init_state == malloc_init_initialized); 274 } 275 276 JEMALLOC_ALWAYS_INLINE_C void 277 malloc_thread_init(void) 278 { 279 280 /* 281 * TSD initialization can't be safely done as a side effect of 282 * deallocation, because it is possible for a thread to do nothing but 283 * deallocate its TLS data via free(), in which case writing to TLS 284 * would cause write-after-free memory corruption. The quarantine 285 * facility *only* gets used as a side effect of deallocation, so make 286 * a best effort attempt at initializing its TSD by hooking all 287 * allocation events. 288 */ 289 if (config_fill && unlikely(opt_quarantine)) 290 quarantine_alloc_hook(); 291 } 292 293 JEMALLOC_ALWAYS_INLINE_C bool 294 malloc_init_a0(void) 295 { 296 297 if (unlikely(malloc_init_state == malloc_init_uninitialized)) 298 return (malloc_init_hard_a0()); 299 return (false); 300 } 301 302 JEMALLOC_ALWAYS_INLINE_C bool 303 malloc_init(void) 304 { 305 306 if (unlikely(!malloc_initialized()) && malloc_init_hard()) 307 return (true); 308 malloc_thread_init(); 309 310 return (false); 311 } 312 313 /* 314 * The a0*() functions are used instead of i{d,}alloc() in situations that 315 * cannot tolerate TLS variable access. 316 */ 317 318 static void * 319 a0ialloc(size_t size, bool zero, bool is_metadata) 320 { 321 322 if (unlikely(malloc_init_a0())) 323 return (NULL); 324 325 return (iallocztm(TSDN_NULL, size, size2index(size), zero, NULL, 326 is_metadata, arena_get(TSDN_NULL, 0, true), true)); 327 } 328 329 static void 330 a0idalloc(void *ptr, bool is_metadata) 331 { 332 333 idalloctm(TSDN_NULL, ptr, false, is_metadata, true); 334 } 335 336 void * 337 a0malloc(size_t size) 338 { 339 340 return (a0ialloc(size, false, true)); 341 } 342 343 void 344 a0dalloc(void *ptr) 345 { 346 347 a0idalloc(ptr, true); 348 } 349 350 /* 351 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive 352 * situations that cannot tolerate TLS variable access (TLS allocation and very 353 * early internal data structure initialization). 354 */ 355 356 void * 357 bootstrap_malloc(size_t size) 358 { 359 360 if (unlikely(size == 0)) 361 size = 1; 362 363 return (a0ialloc(size, false, false)); 364 } 365 366 void * 367 bootstrap_calloc(size_t num, size_t size) 368 { 369 size_t num_size; 370 371 num_size = num * size; 372 if (unlikely(num_size == 0)) { 373 assert(num == 0 || size == 0); 374 num_size = 1; 375 } 376 377 return (a0ialloc(num_size, true, false)); 378 } 379 380 void 381 bootstrap_free(void *ptr) 382 { 383 384 if (unlikely(ptr == NULL)) 385 return; 386 387 a0idalloc(ptr, false); 388 } 389 390 static void 391 arena_set(unsigned ind, arena_t *arena) 392 { 393 394 atomic_write_p((void **)&arenas[ind], arena); 395 } 396 397 static void 398 narenas_total_set(unsigned narenas) 399 { 400 401 atomic_write_u(&narenas_total, narenas); 402 } 403 404 static void 405 narenas_total_inc(void) 406 { 407 408 atomic_add_u(&narenas_total, 1); 409 } 410 411 unsigned 412 narenas_total_get(void) 413 { 414 415 return (atomic_read_u(&narenas_total)); 416 } 417 418 /* Create a new arena and insert it into the arenas array at index ind. */ 419 static arena_t * 420 arena_init_locked(tsdn_t *tsdn, unsigned ind) 421 { 422 arena_t *arena; 423 424 assert(ind <= narenas_total_get()); 425 if (ind > MALLOCX_ARENA_MAX) 426 return (NULL); 427 if (ind == narenas_total_get()) 428 narenas_total_inc(); 429 430 /* 431 * Another thread may have already initialized arenas[ind] if it's an 432 * auto arena. 433 */ 434 arena = arena_get(tsdn, ind, false); 435 if (arena != NULL) { 436 assert(ind < narenas_auto); 437 return (arena); 438 } 439 440 /* Actually initialize the arena. */ 441 arena = arena_new(tsdn, ind); 442 arena_set(ind, arena); 443 return (arena); 444 } 445 446 arena_t * 447 arena_init(tsdn_t *tsdn, unsigned ind) 448 { 449 arena_t *arena; 450 451 malloc_mutex_lock(tsdn, &arenas_lock); 452 arena = arena_init_locked(tsdn, ind); 453 malloc_mutex_unlock(tsdn, &arenas_lock); 454 return (arena); 455 } 456 457 static void 458 arena_bind(tsd_t *tsd, unsigned ind, bool internal) 459 { 460 arena_t *arena; 461 462 arena = arena_get(tsd_tsdn(tsd), ind, false); 463 arena_nthreads_inc(arena, internal); 464 465 if (tsd_nominal(tsd)) { 466 if (internal) 467 tsd_iarena_set(tsd, arena); 468 else 469 tsd_arena_set(tsd, arena); 470 } 471 } 472 473 void 474 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) 475 { 476 arena_t *oldarena, *newarena; 477 478 oldarena = arena_get(tsd_tsdn(tsd), oldind, false); 479 newarena = arena_get(tsd_tsdn(tsd), newind, false); 480 arena_nthreads_dec(oldarena, false); 481 arena_nthreads_inc(newarena, false); 482 tsd_arena_set(tsd, newarena); 483 } 484 485 static void 486 arena_unbind(tsd_t *tsd, unsigned ind, bool internal) 487 { 488 arena_t *arena; 489 490 arena = arena_get(tsd_tsdn(tsd), ind, false); 491 arena_nthreads_dec(arena, internal); 492 if (internal) 493 tsd_iarena_set(tsd, NULL); 494 else 495 tsd_arena_set(tsd, NULL); 496 } 497 498 arena_tdata_t * 499 arena_tdata_get_hard(tsd_t *tsd, unsigned ind) 500 { 501 arena_tdata_t *tdata, *arenas_tdata_old; 502 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); 503 unsigned narenas_tdata_old, i; 504 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd); 505 unsigned narenas_actual = narenas_total_get(); 506 507 /* 508 * Dissociate old tdata array (and set up for deallocation upon return) 509 * if it's too small. 510 */ 511 if (arenas_tdata != NULL && narenas_tdata < narenas_actual) { 512 arenas_tdata_old = arenas_tdata; 513 narenas_tdata_old = narenas_tdata; 514 arenas_tdata = NULL; 515 narenas_tdata = 0; 516 tsd_arenas_tdata_set(tsd, arenas_tdata); 517 tsd_narenas_tdata_set(tsd, narenas_tdata); 518 } else { 519 arenas_tdata_old = NULL; 520 narenas_tdata_old = 0; 521 } 522 523 /* Allocate tdata array if it's missing. */ 524 if (arenas_tdata == NULL) { 525 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd); 526 narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1; 527 528 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) { 529 *arenas_tdata_bypassp = true; 530 arenas_tdata = (arena_tdata_t *)a0malloc( 531 sizeof(arena_tdata_t) * narenas_tdata); 532 *arenas_tdata_bypassp = false; 533 } 534 if (arenas_tdata == NULL) { 535 tdata = NULL; 536 goto label_return; 537 } 538 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp); 539 tsd_arenas_tdata_set(tsd, arenas_tdata); 540 tsd_narenas_tdata_set(tsd, narenas_tdata); 541 } 542 543 /* 544 * Copy to tdata array. It's possible that the actual number of arenas 545 * has increased since narenas_total_get() was called above, but that 546 * causes no correctness issues unless two threads concurrently execute 547 * the arenas.extend mallctl, which we trust mallctl synchronization to 548 * prevent. 549 */ 550 551 /* Copy/initialize tickers. */ 552 for (i = 0; i < narenas_actual; i++) { 553 if (i < narenas_tdata_old) { 554 ticker_copy(&arenas_tdata[i].decay_ticker, 555 &arenas_tdata_old[i].decay_ticker); 556 } else { 557 ticker_init(&arenas_tdata[i].decay_ticker, 558 DECAY_NTICKS_PER_UPDATE); 559 } 560 } 561 if (narenas_tdata > narenas_actual) { 562 memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t) 563 * (narenas_tdata - narenas_actual)); 564 } 565 566 /* Read the refreshed tdata array. */ 567 tdata = &arenas_tdata[ind]; 568 label_return: 569 if (arenas_tdata_old != NULL) 570 a0dalloc(arenas_tdata_old); 571 return (tdata); 572 } 573 574 /* Slow path, called only by arena_choose(). */ 575 arena_t * 576 arena_choose_hard(tsd_t *tsd, bool internal) 577 { 578 arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL); 579 580 if (narenas_auto > 1) { 581 unsigned i, j, choose[2], first_null; 582 583 /* 584 * Determine binding for both non-internal and internal 585 * allocation. 586 * 587 * choose[0]: For application allocation. 588 * choose[1]: For internal metadata allocation. 589 */ 590 591 for (j = 0; j < 2; j++) 592 choose[j] = 0; 593 594 first_null = narenas_auto; 595 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock); 596 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL); 597 for (i = 1; i < narenas_auto; i++) { 598 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) { 599 /* 600 * Choose the first arena that has the lowest 601 * number of threads assigned to it. 602 */ 603 for (j = 0; j < 2; j++) { 604 if (arena_nthreads_get(arena_get( 605 tsd_tsdn(tsd), i, false), !!j) < 606 arena_nthreads_get(arena_get( 607 tsd_tsdn(tsd), choose[j], false), 608 !!j)) 609 choose[j] = i; 610 } 611 } else if (first_null == narenas_auto) { 612 /* 613 * Record the index of the first uninitialized 614 * arena, in case all extant arenas are in use. 615 * 616 * NB: It is possible for there to be 617 * discontinuities in terms of initialized 618 * versus uninitialized arenas, due to the 619 * "thread.arena" mallctl. 620 */ 621 first_null = i; 622 } 623 } 624 625 for (j = 0; j < 2; j++) { 626 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd), 627 choose[j], false), !!j) == 0 || first_null == 628 narenas_auto) { 629 /* 630 * Use an unloaded arena, or the least loaded 631 * arena if all arenas are already initialized. 632 */ 633 if (!!j == internal) { 634 ret = arena_get(tsd_tsdn(tsd), 635 choose[j], false); 636 } 637 } else { 638 arena_t *arena; 639 640 /* Initialize a new arena. */ 641 choose[j] = first_null; 642 arena = arena_init_locked(tsd_tsdn(tsd), 643 choose[j]); 644 if (arena == NULL) { 645 malloc_mutex_unlock(tsd_tsdn(tsd), 646 &arenas_lock); 647 return (NULL); 648 } 649 if (!!j == internal) 650 ret = arena; 651 } 652 arena_bind(tsd, choose[j], !!j); 653 } 654 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock); 655 } else { 656 ret = arena_get(tsd_tsdn(tsd), 0, false); 657 arena_bind(tsd, 0, false); 658 arena_bind(tsd, 0, true); 659 } 660 661 return (ret); 662 } 663 664 void 665 thread_allocated_cleanup(tsd_t *tsd) 666 { 667 668 /* Do nothing. */ 669 } 670 671 void 672 thread_deallocated_cleanup(tsd_t *tsd) 673 { 674 675 /* Do nothing. */ 676 } 677 678 void 679 iarena_cleanup(tsd_t *tsd) 680 { 681 arena_t *iarena; 682 683 iarena = tsd_iarena_get(tsd); 684 if (iarena != NULL) 685 arena_unbind(tsd, iarena->ind, true); 686 } 687 688 void 689 arena_cleanup(tsd_t *tsd) 690 { 691 arena_t *arena; 692 693 arena = tsd_arena_get(tsd); 694 if (arena != NULL) 695 arena_unbind(tsd, arena->ind, false); 696 } 697 698 void 699 arenas_tdata_cleanup(tsd_t *tsd) 700 { 701 arena_tdata_t *arenas_tdata; 702 703 /* Prevent tsd->arenas_tdata from being (re)created. */ 704 *tsd_arenas_tdata_bypassp_get(tsd) = true; 705 706 arenas_tdata = tsd_arenas_tdata_get(tsd); 707 if (arenas_tdata != NULL) { 708 tsd_arenas_tdata_set(tsd, NULL); 709 a0dalloc(arenas_tdata); 710 } 711 } 712 713 void 714 narenas_tdata_cleanup(tsd_t *tsd) 715 { 716 717 /* Do nothing. */ 718 } 719 720 void 721 arenas_tdata_bypass_cleanup(tsd_t *tsd) 722 { 723 724 /* Do nothing. */ 725 } 726 727 static void 728 stats_print_atexit(void) 729 { 730 731 if (config_tcache && config_stats) { 732 tsdn_t *tsdn; 733 unsigned narenas, i; 734 735 tsdn = tsdn_fetch(); 736 737 /* 738 * Merge stats from extant threads. This is racy, since 739 * individual threads do not lock when recording tcache stats 740 * events. As a consequence, the final stats may be slightly 741 * out of date by the time they are reported, if other threads 742 * continue to allocate. 743 */ 744 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 745 arena_t *arena = arena_get(tsdn, i, false); 746 if (arena != NULL) { 747 tcache_t *tcache; 748 749 /* 750 * tcache_stats_merge() locks bins, so if any 751 * code is introduced that acquires both arena 752 * and bin locks in the opposite order, 753 * deadlocks may result. 754 */ 755 malloc_mutex_lock(tsdn, &arena->lock); 756 ql_foreach(tcache, &arena->tcache_ql, link) { 757 tcache_stats_merge(tsdn, tcache, arena); 758 } 759 malloc_mutex_unlock(tsdn, &arena->lock); 760 } 761 } 762 } 763 je_malloc_stats_print(NULL, NULL, NULL); 764 } 765 766 /* 767 * End miscellaneous support functions. 768 */ 769 /******************************************************************************/ 770 /* 771 * Begin initialization functions. 772 */ 773 774 #ifndef JEMALLOC_HAVE_SECURE_GETENV 775 static char * 776 secure_getenv(const char *name) 777 { 778 779 # ifdef JEMALLOC_HAVE_ISSETUGID 780 if (issetugid() != 0) 781 return (NULL); 782 # endif 783 return (getenv(name)); 784 } 785 #endif 786 787 static unsigned 788 malloc_ncpus(void) 789 { 790 long result; 791 792 #ifdef _WIN32 793 SYSTEM_INFO si; 794 GetSystemInfo(&si); 795 result = si.dwNumberOfProcessors; 796 #else 797 result = sysconf(_SC_NPROCESSORS_ONLN); 798 #endif 799 return ((result == -1) ? 1 : (unsigned)result); 800 } 801 802 static bool 803 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, 804 char const **v_p, size_t *vlen_p) 805 { 806 bool accept; 807 const char *opts = *opts_p; 808 809 *k_p = opts; 810 811 for (accept = false; !accept;) { 812 switch (*opts) { 813 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 814 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 815 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 816 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 817 case 'Y': case 'Z': 818 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 819 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 820 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 821 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 822 case 'y': case 'z': 823 case '0': case '1': case '2': case '3': case '4': case '5': 824 case '6': case '7': case '8': case '9': 825 case '_': 826 opts++; 827 break; 828 case ':': 829 opts++; 830 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; 831 *v_p = opts; 832 accept = true; 833 break; 834 case '\0': 835 if (opts != *opts_p) { 836 malloc_write("<jemalloc>: Conf string ends " 837 "with key\n"); 838 } 839 return (true); 840 default: 841 malloc_write("<jemalloc>: Malformed conf string\n"); 842 return (true); 843 } 844 } 845 846 for (accept = false; !accept;) { 847 switch (*opts) { 848 case ',': 849 opts++; 850 /* 851 * Look ahead one character here, because the next time 852 * this function is called, it will assume that end of 853 * input has been cleanly reached if no input remains, 854 * but we have optimistically already consumed the 855 * comma if one exists. 856 */ 857 if (*opts == '\0') { 858 malloc_write("<jemalloc>: Conf string ends " 859 "with comma\n"); 860 } 861 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; 862 accept = true; 863 break; 864 case '\0': 865 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; 866 accept = true; 867 break; 868 default: 869 opts++; 870 break; 871 } 872 } 873 874 *opts_p = opts; 875 return (false); 876 } 877 878 static void 879 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, 880 size_t vlen) 881 { 882 883 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, 884 (int)vlen, v); 885 } 886 887 static void 888 malloc_slow_flag_init(void) 889 { 890 /* 891 * Combine the runtime options into malloc_slow for fast path. Called 892 * after processing all the options. 893 */ 894 malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0) 895 | (opt_junk_free ? flag_opt_junk_free : 0) 896 | (opt_quarantine ? flag_opt_quarantine : 0) 897 | (opt_zero ? flag_opt_zero : 0) 898 | (opt_utrace ? flag_opt_utrace : 0) 899 | (opt_xmalloc ? flag_opt_xmalloc : 0); 900 901 if (config_valgrind) 902 malloc_slow_flags |= (in_valgrind ? flag_in_valgrind : 0); 903 904 malloc_slow = (malloc_slow_flags != 0); 905 } 906 907 static void 908 malloc_conf_init(void) 909 { 910 unsigned i; 911 char buf[PATH_MAX + 1]; 912 const char *opts, *k, *v; 913 size_t klen, vlen; 914 915 /* 916 * Automatically configure valgrind before processing options. The 917 * valgrind option remains in jemalloc 3.x for compatibility reasons. 918 */ 919 if (config_valgrind) { 920 in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; 921 if (config_fill && unlikely(in_valgrind)) { 922 opt_junk = "false"; 923 opt_junk_alloc = false; 924 opt_junk_free = false; 925 assert(!opt_zero); 926 opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; 927 opt_redzone = true; 928 } 929 if (config_tcache && unlikely(in_valgrind)) 930 opt_tcache = false; 931 } 932 933 for (i = 0; i < 4; i++) { 934 /* Get runtime configuration. */ 935 switch (i) { 936 case 0: 937 opts = config_malloc_conf; 938 break; 939 case 1: 940 if (je_malloc_conf != NULL) { 941 /* 942 * Use options that were compiled into the 943 * program. 944 */ 945 opts = je_malloc_conf; 946 } else { 947 /* No configuration specified. */ 948 buf[0] = '\0'; 949 opts = buf; 950 } 951 break; 952 case 2: { 953 ssize_t linklen = 0; 954 #ifndef _WIN32 955 int saved_errno = errno; 956 const char *linkname = 957 # ifdef JEMALLOC_PREFIX 958 "/etc/"JEMALLOC_PREFIX"malloc.conf" 959 # else 960 "/etc/malloc.conf" 961 # endif 962 ; 963 964 /* 965 * Try to use the contents of the "/etc/malloc.conf" 966 * symbolic link's name. 967 */ 968 linklen = readlink(linkname, buf, sizeof(buf) - 1); 969 if (linklen == -1) { 970 /* No configuration specified. */ 971 linklen = 0; 972 /* Restore errno. */ 973 set_errno(saved_errno); 974 } 975 #endif 976 buf[linklen] = '\0'; 977 opts = buf; 978 break; 979 } case 3: { 980 const char *envname = 981 #ifdef JEMALLOC_PREFIX 982 JEMALLOC_CPREFIX"MALLOC_CONF" 983 #else 984 "MALLOC_CONF" 985 #endif 986 ; 987 988 if ((opts = secure_getenv(envname)) != NULL) { 989 /* 990 * Do nothing; opts is already initialized to 991 * the value of the MALLOC_CONF environment 992 * variable. 993 */ 994 } else { 995 /* No configuration specified. */ 996 buf[0] = '\0'; 997 opts = buf; 998 } 999 break; 1000 } default: 1001 not_reached(); 1002 buf[0] = '\0'; 1003 opts = buf; 1004 } 1005 1006 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, 1007 &vlen)) { 1008 #define CONF_MATCH(n) \ 1009 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) 1010 #define CONF_MATCH_VALUE(n) \ 1011 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0) 1012 #define CONF_HANDLE_BOOL(o, n, cont) \ 1013 if (CONF_MATCH(n)) { \ 1014 if (CONF_MATCH_VALUE("true")) \ 1015 o = true; \ 1016 else if (CONF_MATCH_VALUE("false")) \ 1017 o = false; \ 1018 else { \ 1019 malloc_conf_error( \ 1020 "Invalid conf value", \ 1021 k, klen, v, vlen); \ 1022 } \ 1023 if (cont) \ 1024 continue; \ 1025 } 1026 #define CONF_HANDLE_T_U(t, o, n, min, max, clip) \ 1027 if (CONF_MATCH(n)) { \ 1028 uintmax_t um; \ 1029 char *end; \ 1030 \ 1031 set_errno(0); \ 1032 um = malloc_strtoumax(v, &end, 0); \ 1033 if (get_errno() != 0 || (uintptr_t)end -\ 1034 (uintptr_t)v != vlen) { \ 1035 malloc_conf_error( \ 1036 "Invalid conf value", \ 1037 k, klen, v, vlen); \ 1038 } else if (clip) { \ 1039 if ((min) != 0 && um < (min)) \ 1040 o = (t)(min); \ 1041 else if (um > (max)) \ 1042 o = (t)(max); \ 1043 else \ 1044 o = (t)um; \ 1045 } else { \ 1046 if (((min) != 0 && um < (min)) \ 1047 || um > (max)) { \ 1048 malloc_conf_error( \ 1049 "Out-of-range " \ 1050 "conf value", \ 1051 k, klen, v, vlen); \ 1052 } else \ 1053 o = (t)um; \ 1054 } \ 1055 continue; \ 1056 } 1057 #define CONF_HANDLE_UNSIGNED(o, n, min, max, clip) \ 1058 CONF_HANDLE_T_U(unsigned, o, n, min, max, clip) 1059 #define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ 1060 CONF_HANDLE_T_U(size_t, o, n, min, max, clip) 1061 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ 1062 if (CONF_MATCH(n)) { \ 1063 long l; \ 1064 char *end; \ 1065 \ 1066 set_errno(0); \ 1067 l = strtol(v, &end, 0); \ 1068 if (get_errno() != 0 || (uintptr_t)end -\ 1069 (uintptr_t)v != vlen) { \ 1070 malloc_conf_error( \ 1071 "Invalid conf value", \ 1072 k, klen, v, vlen); \ 1073 } else if (l < (ssize_t)(min) || l > \ 1074 (ssize_t)(max)) { \ 1075 malloc_conf_error( \ 1076 "Out-of-range conf value", \ 1077 k, klen, v, vlen); \ 1078 } else \ 1079 o = l; \ 1080 continue; \ 1081 } 1082 #define CONF_HANDLE_CHAR_P(o, n, d) \ 1083 if (CONF_MATCH(n)) { \ 1084 size_t cpylen = (vlen <= \ 1085 sizeof(o)-1) ? vlen : \ 1086 sizeof(o)-1; \ 1087 strncpy(o, v, cpylen); \ 1088 o[cpylen] = '\0'; \ 1089 continue; \ 1090 } 1091 1092 CONF_HANDLE_BOOL(opt_abort, "abort", true) 1093 /* 1094 * Chunks always require at least one header page, 1095 * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and 1096 * possibly an additional page in the presence of 1097 * redzones. In order to simplify options processing, 1098 * use a conservative bound that accommodates all these 1099 * constraints. 1100 */ 1101 CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + 1102 LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1), 1103 (sizeof(size_t) << 3) - 1, true) 1104 if (strncmp("dss", k, klen) == 0) { 1105 int i; 1106 bool match = false; 1107 for (i = 0; i < dss_prec_limit; i++) { 1108 if (strncmp(dss_prec_names[i], v, vlen) 1109 == 0) { 1110 if (chunk_dss_prec_set(NULL, 1111 i)) { 1112 malloc_conf_error( 1113 "Error setting dss", 1114 k, klen, v, vlen); 1115 } else { 1116 opt_dss = 1117 dss_prec_names[i]; 1118 match = true; 1119 break; 1120 } 1121 } 1122 } 1123 if (!match) { 1124 malloc_conf_error("Invalid conf value", 1125 k, klen, v, vlen); 1126 } 1127 continue; 1128 } 1129 CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, 1130 UINT_MAX, false) 1131 if (strncmp("purge", k, klen) == 0) { 1132 int i; 1133 bool match = false; 1134 for (i = 0; i < purge_mode_limit; i++) { 1135 if (strncmp(purge_mode_names[i], v, 1136 vlen) == 0) { 1137 opt_purge = (purge_mode_t)i; 1138 match = true; 1139 break; 1140 } 1141 } 1142 if (!match) { 1143 malloc_conf_error("Invalid conf value", 1144 k, klen, v, vlen); 1145 } 1146 continue; 1147 } 1148 CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", 1149 -1, (sizeof(size_t) << 3) - 1) 1150 CONF_HANDLE_SSIZE_T(opt_decay_time, "decay_time", -1, 1151 NSTIME_SEC_MAX); 1152 CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) 1153 if (config_fill) { 1154 if (CONF_MATCH("junk")) { 1155 if (CONF_MATCH_VALUE("true")) { 1156 opt_junk = "true"; 1157 opt_junk_alloc = opt_junk_free = 1158 true; 1159 } else if (CONF_MATCH_VALUE("false")) { 1160 opt_junk = "false"; 1161 opt_junk_alloc = opt_junk_free = 1162 false; 1163 } else if (CONF_MATCH_VALUE("alloc")) { 1164 opt_junk = "alloc"; 1165 opt_junk_alloc = true; 1166 opt_junk_free = false; 1167 } else if (CONF_MATCH_VALUE("free")) { 1168 opt_junk = "free"; 1169 opt_junk_alloc = false; 1170 opt_junk_free = true; 1171 } else { 1172 malloc_conf_error( 1173 "Invalid conf value", k, 1174 klen, v, vlen); 1175 } 1176 continue; 1177 } 1178 CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 1179 0, SIZE_T_MAX, false) 1180 CONF_HANDLE_BOOL(opt_redzone, "redzone", true) 1181 CONF_HANDLE_BOOL(opt_zero, "zero", true) 1182 } 1183 if (config_utrace) { 1184 CONF_HANDLE_BOOL(opt_utrace, "utrace", true) 1185 } 1186 if (config_xmalloc) { 1187 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true) 1188 } 1189 if (config_tcache) { 1190 CONF_HANDLE_BOOL(opt_tcache, "tcache", 1191 !config_valgrind || !in_valgrind) 1192 if (CONF_MATCH("tcache")) { 1193 assert(config_valgrind && in_valgrind); 1194 if (opt_tcache) { 1195 opt_tcache = false; 1196 malloc_conf_error( 1197 "tcache cannot be enabled " 1198 "while running inside Valgrind", 1199 k, klen, v, vlen); 1200 } 1201 continue; 1202 } 1203 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, 1204 "lg_tcache_max", -1, 1205 (sizeof(size_t) << 3) - 1) 1206 } 1207 if (config_prof) { 1208 CONF_HANDLE_BOOL(opt_prof, "prof", true) 1209 CONF_HANDLE_CHAR_P(opt_prof_prefix, 1210 "prof_prefix", "jeprof") 1211 CONF_HANDLE_BOOL(opt_prof_active, "prof_active", 1212 true) 1213 CONF_HANDLE_BOOL(opt_prof_thread_active_init, 1214 "prof_thread_active_init", true) 1215 CONF_HANDLE_SIZE_T(opt_lg_prof_sample, 1216 "lg_prof_sample", 0, 1217 (sizeof(uint64_t) << 3) - 1, true) 1218 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", 1219 true) 1220 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, 1221 "lg_prof_interval", -1, 1222 (sizeof(uint64_t) << 3) - 1) 1223 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump", 1224 true) 1225 CONF_HANDLE_BOOL(opt_prof_final, "prof_final", 1226 true) 1227 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak", 1228 true) 1229 } 1230 malloc_conf_error("Invalid conf pair", k, klen, v, 1231 vlen); 1232 #undef CONF_MATCH 1233 #undef CONF_HANDLE_BOOL 1234 #undef CONF_HANDLE_SIZE_T 1235 #undef CONF_HANDLE_SSIZE_T 1236 #undef CONF_HANDLE_CHAR_P 1237 } 1238 } 1239 } 1240 1241 static bool 1242 malloc_init_hard_needed(void) 1243 { 1244 1245 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == 1246 malloc_init_recursible)) { 1247 /* 1248 * Another thread initialized the allocator before this one 1249 * acquired init_lock, or this thread is the initializing 1250 * thread, and it is recursively allocating. 1251 */ 1252 return (false); 1253 } 1254 #ifdef JEMALLOC_THREADED_INIT 1255 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) { 1256 /* Busy-wait until the initializing thread completes. */ 1257 do { 1258 malloc_mutex_unlock(NULL, &init_lock); 1259 CPU_SPINWAIT; 1260 malloc_mutex_lock(NULL, &init_lock); 1261 } while (!malloc_initialized()); 1262 return (false); 1263 } 1264 #endif 1265 return (true); 1266 } 1267 1268 static bool 1269 malloc_init_hard_a0_locked() 1270 { 1271 1272 malloc_initializer = INITIALIZER; 1273 1274 if (config_prof) 1275 prof_boot0(); 1276 malloc_conf_init(); 1277 if (opt_stats_print) { 1278 /* Print statistics at exit. */ 1279 if (atexit(stats_print_atexit) != 0) { 1280 malloc_write("<jemalloc>: Error in atexit()\n"); 1281 if (opt_abort) 1282 abort(); 1283 } 1284 } 1285 pages_boot(); 1286 if (base_boot()) 1287 return (true); 1288 if (chunk_boot()) 1289 return (true); 1290 if (ctl_boot()) 1291 return (true); 1292 if (config_prof) 1293 prof_boot1(); 1294 if (arena_boot()) 1295 return (true); 1296 if (config_tcache && tcache_boot(TSDN_NULL)) 1297 return (true); 1298 if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS)) 1299 return (true); 1300 /* 1301 * Create enough scaffolding to allow recursive allocation in 1302 * malloc_ncpus(). 1303 */ 1304 narenas_auto = 1; 1305 narenas_total_set(narenas_auto); 1306 arenas = &a0; 1307 memset(arenas, 0, sizeof(arena_t *) * narenas_auto); 1308 /* 1309 * Initialize one arena here. The rest are lazily created in 1310 * arena_choose_hard(). 1311 */ 1312 if (arena_init(TSDN_NULL, 0) == NULL) 1313 return (true); 1314 1315 malloc_init_state = malloc_init_a0_initialized; 1316 1317 return (false); 1318 } 1319 1320 static bool 1321 malloc_init_hard_a0(void) 1322 { 1323 bool ret; 1324 1325 malloc_mutex_lock(TSDN_NULL, &init_lock); 1326 ret = malloc_init_hard_a0_locked(); 1327 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1328 return (ret); 1329 } 1330 1331 /* Initialize data structures which may trigger recursive allocation. */ 1332 static bool 1333 malloc_init_hard_recursible(void) 1334 { 1335 1336 malloc_init_state = malloc_init_recursible; 1337 1338 ncpus = malloc_ncpus(); 1339 1340 #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ 1341 && !defined(_WIN32) && !defined(__native_client__)) 1342 /* LinuxThreads' pthread_atfork() allocates. */ 1343 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, 1344 jemalloc_postfork_child) != 0) { 1345 malloc_write("<jemalloc>: Error in pthread_atfork()\n"); 1346 if (opt_abort) 1347 abort(); 1348 return (true); 1349 } 1350 #endif 1351 1352 return (false); 1353 } 1354 1355 static bool 1356 malloc_init_hard_finish(tsdn_t *tsdn) 1357 { 1358 1359 if (malloc_mutex_boot()) 1360 return (true); 1361 1362 if (opt_narenas == 0) { 1363 /* 1364 * For SMP systems, create more than one arena per CPU by 1365 * default. 1366 */ 1367 if (ncpus > 1) 1368 opt_narenas = ncpus << 2; 1369 else 1370 opt_narenas = 1; 1371 } 1372 narenas_auto = opt_narenas; 1373 /* 1374 * Limit the number of arenas to the indexing range of MALLOCX_ARENA(). 1375 */ 1376 if (narenas_auto > MALLOCX_ARENA_MAX) { 1377 narenas_auto = MALLOCX_ARENA_MAX; 1378 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", 1379 narenas_auto); 1380 } 1381 narenas_total_set(narenas_auto); 1382 1383 /* Allocate and initialize arenas. */ 1384 arenas = (arena_t **)base_alloc(tsdn, sizeof(arena_t *) * 1385 (MALLOCX_ARENA_MAX+1)); 1386 if (arenas == NULL) 1387 return (true); 1388 /* Copy the pointer to the one arena that was already initialized. */ 1389 arena_set(0, a0); 1390 1391 malloc_init_state = malloc_init_initialized; 1392 malloc_slow_flag_init(); 1393 1394 return (false); 1395 } 1396 1397 static bool 1398 malloc_init_hard(void) 1399 { 1400 tsd_t *tsd; 1401 1402 #if defined(_WIN32) && _WIN32_WINNT < 0x0600 1403 _init_init_lock(); 1404 #endif 1405 malloc_mutex_lock(TSDN_NULL, &init_lock); 1406 if (!malloc_init_hard_needed()) { 1407 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1408 return (false); 1409 } 1410 1411 if (malloc_init_state != malloc_init_a0_initialized && 1412 malloc_init_hard_a0_locked()) { 1413 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1414 return (true); 1415 } 1416 1417 malloc_mutex_unlock(TSDN_NULL, &init_lock); 1418 /* Recursive allocation relies on functional tsd. */ 1419 tsd = malloc_tsd_boot0(); 1420 if (tsd == NULL) 1421 return (true); 1422 if (malloc_init_hard_recursible()) 1423 return (true); 1424 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock); 1425 1426 if (config_prof && prof_boot2(tsd_tsdn(tsd))) { 1427 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1428 return (true); 1429 } 1430 1431 if (malloc_init_hard_finish(tsd_tsdn(tsd))) { 1432 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1433 return (true); 1434 } 1435 1436 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); 1437 malloc_tsd_boot1(); 1438 return (false); 1439 } 1440 1441 /* 1442 * End initialization functions. 1443 */ 1444 /******************************************************************************/ 1445 /* 1446 * Begin malloc(3)-compatible functions. 1447 */ 1448 1449 static void * 1450 ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero, 1451 prof_tctx_t *tctx, bool slow_path) 1452 { 1453 void *p; 1454 1455 if (tctx == NULL) 1456 return (NULL); 1457 if (usize <= SMALL_MAXCLASS) { 1458 szind_t ind_large = size2index(LARGE_MINCLASS); 1459 p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path); 1460 if (p == NULL) 1461 return (NULL); 1462 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 1463 } else 1464 p = ialloc(tsd, usize, ind, zero, slow_path); 1465 1466 return (p); 1467 } 1468 1469 JEMALLOC_ALWAYS_INLINE_C void * 1470 ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path) 1471 { 1472 void *p; 1473 prof_tctx_t *tctx; 1474 1475 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); 1476 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1477 p = ialloc_prof_sample(tsd, usize, ind, zero, tctx, slow_path); 1478 else 1479 p = ialloc(tsd, usize, ind, zero, slow_path); 1480 if (unlikely(p == NULL)) { 1481 prof_alloc_rollback(tsd, tctx, true); 1482 return (NULL); 1483 } 1484 prof_malloc(tsd_tsdn(tsd), p, usize, tctx); 1485 1486 return (p); 1487 } 1488 1489 /* 1490 * ialloc_body() is inlined so that fast and slow paths are generated separately 1491 * with statically known slow_path. 1492 * 1493 * This function guarantees that *tsdn is non-NULL on success. 1494 */ 1495 JEMALLOC_ALWAYS_INLINE_C void * 1496 ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize, 1497 bool slow_path) 1498 { 1499 tsd_t *tsd; 1500 szind_t ind; 1501 1502 if (slow_path && unlikely(malloc_init())) { 1503 *tsdn = NULL; 1504 return (NULL); 1505 } 1506 1507 tsd = tsd_fetch(); 1508 *tsdn = tsd_tsdn(tsd); 1509 witness_assert_lockless(tsd_tsdn(tsd)); 1510 1511 ind = size2index(size); 1512 if (unlikely(ind >= NSIZES)) 1513 return (NULL); 1514 1515 if (config_stats || (config_prof && opt_prof) || (slow_path && 1516 config_valgrind && unlikely(in_valgrind))) { 1517 *usize = index2size(ind); 1518 assert(*usize > 0 && *usize <= HUGE_MAXCLASS); 1519 } 1520 1521 if (config_prof && opt_prof) 1522 return (ialloc_prof(tsd, *usize, ind, zero, slow_path)); 1523 1524 return (ialloc(tsd, size, ind, zero, slow_path)); 1525 } 1526 1527 JEMALLOC_ALWAYS_INLINE_C void 1528 ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func, 1529 bool update_errno, bool slow_path) 1530 { 1531 1532 assert(!tsdn_null(tsdn) || ret == NULL); 1533 1534 if (unlikely(ret == NULL)) { 1535 if (slow_path && config_xmalloc && unlikely(opt_xmalloc)) { 1536 malloc_printf("<jemalloc>: Error in %s(): out of " 1537 "memory\n", func); 1538 abort(); 1539 } 1540 if (update_errno) 1541 set_errno(ENOMEM); 1542 } 1543 if (config_stats && likely(ret != NULL)) { 1544 assert(usize == isalloc(tsdn, ret, config_prof)); 1545 *tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize; 1546 } 1547 witness_assert_lockless(tsdn); 1548 } 1549 1550 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1551 void JEMALLOC_NOTHROW * 1552 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 1553 je_malloc(size_t size) 1554 { 1555 void *ret; 1556 tsdn_t *tsdn; 1557 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1558 1559 if (size == 0) 1560 size = 1; 1561 1562 if (likely(!malloc_slow)) { 1563 ret = ialloc_body(size, false, &tsdn, &usize, false); 1564 ialloc_post_check(ret, tsdn, usize, "malloc", true, false); 1565 } else { 1566 ret = ialloc_body(size, false, &tsdn, &usize, true); 1567 ialloc_post_check(ret, tsdn, usize, "malloc", true, true); 1568 UTRACE(0, size, ret); 1569 JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, false); 1570 } 1571 1572 return (ret); 1573 } 1574 1575 static void * 1576 imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize, 1577 prof_tctx_t *tctx) 1578 { 1579 void *p; 1580 1581 if (tctx == NULL) 1582 return (NULL); 1583 if (usize <= SMALL_MAXCLASS) { 1584 assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS); 1585 p = ipalloc(tsd, LARGE_MINCLASS, alignment, false); 1586 if (p == NULL) 1587 return (NULL); 1588 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 1589 } else 1590 p = ipalloc(tsd, usize, alignment, false); 1591 1592 return (p); 1593 } 1594 1595 JEMALLOC_ALWAYS_INLINE_C void * 1596 imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize) 1597 { 1598 void *p; 1599 prof_tctx_t *tctx; 1600 1601 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); 1602 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1603 p = imemalign_prof_sample(tsd, alignment, usize, tctx); 1604 else 1605 p = ipalloc(tsd, usize, alignment, false); 1606 if (unlikely(p == NULL)) { 1607 prof_alloc_rollback(tsd, tctx, true); 1608 return (NULL); 1609 } 1610 prof_malloc(tsd_tsdn(tsd), p, usize, tctx); 1611 1612 return (p); 1613 } 1614 1615 JEMALLOC_ATTR(nonnull(1)) 1616 static int 1617 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) 1618 { 1619 int ret; 1620 tsd_t *tsd; 1621 size_t usize; 1622 void *result; 1623 1624 assert(min_alignment != 0); 1625 1626 if (unlikely(malloc_init())) { 1627 tsd = NULL; 1628 result = NULL; 1629 goto label_oom; 1630 } 1631 tsd = tsd_fetch(); 1632 witness_assert_lockless(tsd_tsdn(tsd)); 1633 if (size == 0) 1634 size = 1; 1635 1636 /* Make sure that alignment is a large enough power of 2. */ 1637 if (unlikely(((alignment - 1) & alignment) != 0 1638 || (alignment < min_alignment))) { 1639 if (config_xmalloc && unlikely(opt_xmalloc)) { 1640 malloc_write("<jemalloc>: Error allocating " 1641 "aligned memory: invalid alignment\n"); 1642 abort(); 1643 } 1644 result = NULL; 1645 ret = EINVAL; 1646 goto label_return; 1647 } 1648 1649 usize = sa2u(size, alignment); 1650 if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) { 1651 result = NULL; 1652 goto label_oom; 1653 } 1654 1655 if (config_prof && opt_prof) 1656 result = imemalign_prof(tsd, alignment, usize); 1657 else 1658 result = ipalloc(tsd, usize, alignment, false); 1659 if (unlikely(result == NULL)) 1660 goto label_oom; 1661 assert(((uintptr_t)result & (alignment - 1)) == ZU(0)); 1662 1663 *memptr = result; 1664 ret = 0; 1665 label_return: 1666 if (config_stats && likely(result != NULL)) { 1667 assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof)); 1668 *tsd_thread_allocatedp_get(tsd) += usize; 1669 } 1670 UTRACE(0, size, result); 1671 JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize, 1672 false); 1673 witness_assert_lockless(tsd_tsdn(tsd)); 1674 return (ret); 1675 label_oom: 1676 assert(result == NULL); 1677 if (config_xmalloc && unlikely(opt_xmalloc)) { 1678 malloc_write("<jemalloc>: Error allocating aligned memory: " 1679 "out of memory\n"); 1680 abort(); 1681 } 1682 ret = ENOMEM; 1683 witness_assert_lockless(tsd_tsdn(tsd)); 1684 goto label_return; 1685 } 1686 1687 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 1688 JEMALLOC_ATTR(nonnull(1)) 1689 je_posix_memalign(void **memptr, size_t alignment, size_t size) 1690 { 1691 int ret; 1692 1693 ret = imemalign(memptr, alignment, size, sizeof(void *)); 1694 1695 return (ret); 1696 } 1697 1698 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1699 void JEMALLOC_NOTHROW * 1700 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2) 1701 je_aligned_alloc(size_t alignment, size_t size) 1702 { 1703 void *ret; 1704 int err; 1705 1706 if (unlikely((err = imemalign(&ret, alignment, size, 1)) != 0)) { 1707 ret = NULL; 1708 set_errno(err); 1709 } 1710 1711 return (ret); 1712 } 1713 1714 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1715 void JEMALLOC_NOTHROW * 1716 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2) 1717 je_calloc(size_t num, size_t size) 1718 { 1719 void *ret; 1720 tsdn_t *tsdn; 1721 size_t num_size; 1722 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1723 1724 num_size = num * size; 1725 if (unlikely(num_size == 0)) { 1726 if (num == 0 || size == 0) 1727 num_size = 1; 1728 else 1729 num_size = HUGE_MAXCLASS + 1; /* Trigger OOM. */ 1730 /* 1731 * Try to avoid division here. We know that it isn't possible to 1732 * overflow during multiplication if neither operand uses any of the 1733 * most significant half of the bits in a size_t. 1734 */ 1735 } else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 1736 2))) && (num_size / size != num))) 1737 num_size = HUGE_MAXCLASS + 1; /* size_t overflow. */ 1738 1739 if (likely(!malloc_slow)) { 1740 ret = ialloc_body(num_size, true, &tsdn, &usize, false); 1741 ialloc_post_check(ret, tsdn, usize, "calloc", true, false); 1742 } else { 1743 ret = ialloc_body(num_size, true, &tsdn, &usize, true); 1744 ialloc_post_check(ret, tsdn, usize, "calloc", true, true); 1745 UTRACE(0, num_size, ret); 1746 JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, true); 1747 } 1748 1749 return (ret); 1750 } 1751 1752 static void * 1753 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, 1754 prof_tctx_t *tctx) 1755 { 1756 void *p; 1757 1758 if (tctx == NULL) 1759 return (NULL); 1760 if (usize <= SMALL_MAXCLASS) { 1761 p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); 1762 if (p == NULL) 1763 return (NULL); 1764 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 1765 } else 1766 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); 1767 1768 return (p); 1769 } 1770 1771 JEMALLOC_ALWAYS_INLINE_C void * 1772 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize) 1773 { 1774 void *p; 1775 bool prof_active; 1776 prof_tctx_t *old_tctx, *tctx; 1777 1778 prof_active = prof_active_get_unlocked(); 1779 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr); 1780 tctx = prof_alloc_prep(tsd, usize, prof_active, true); 1781 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) 1782 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); 1783 else 1784 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); 1785 if (unlikely(p == NULL)) { 1786 prof_alloc_rollback(tsd, tctx, true); 1787 return (NULL); 1788 } 1789 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize, 1790 old_tctx); 1791 1792 return (p); 1793 } 1794 1795 JEMALLOC_INLINE_C void 1796 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) 1797 { 1798 size_t usize; 1799 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1800 1801 witness_assert_lockless(tsd_tsdn(tsd)); 1802 1803 assert(ptr != NULL); 1804 assert(malloc_initialized() || IS_INITIALIZER); 1805 1806 if (config_prof && opt_prof) { 1807 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1808 prof_free(tsd, ptr, usize); 1809 } else if (config_stats || config_valgrind) 1810 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1811 if (config_stats) 1812 *tsd_thread_deallocatedp_get(tsd) += usize; 1813 1814 if (likely(!slow_path)) 1815 iqalloc(tsd, ptr, tcache, false); 1816 else { 1817 if (config_valgrind && unlikely(in_valgrind)) 1818 rzsize = p2rz(tsd_tsdn(tsd), ptr); 1819 iqalloc(tsd, ptr, tcache, true); 1820 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1821 } 1822 } 1823 1824 JEMALLOC_INLINE_C void 1825 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) 1826 { 1827 UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); 1828 1829 witness_assert_lockless(tsd_tsdn(tsd)); 1830 1831 assert(ptr != NULL); 1832 assert(malloc_initialized() || IS_INITIALIZER); 1833 1834 if (config_prof && opt_prof) 1835 prof_free(tsd, ptr, usize); 1836 if (config_stats) 1837 *tsd_thread_deallocatedp_get(tsd) += usize; 1838 if (config_valgrind && unlikely(in_valgrind)) 1839 rzsize = p2rz(tsd_tsdn(tsd), ptr); 1840 isqalloc(tsd, ptr, usize, tcache, slow_path); 1841 JEMALLOC_VALGRIND_FREE(ptr, rzsize); 1842 } 1843 1844 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1845 void JEMALLOC_NOTHROW * 1846 JEMALLOC_ALLOC_SIZE(2) 1847 je_realloc(void *ptr, size_t size) 1848 { 1849 void *ret; 1850 tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL); 1851 size_t usize JEMALLOC_CC_SILENCE_INIT(0); 1852 size_t old_usize = 0; 1853 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 1854 1855 if (unlikely(size == 0)) { 1856 if (ptr != NULL) { 1857 tsd_t *tsd; 1858 1859 /* realloc(ptr, 0) is equivalent to free(ptr). */ 1860 UTRACE(ptr, 0, 0); 1861 tsd = tsd_fetch(); 1862 ifree(tsd, ptr, tcache_get(tsd, false), true); 1863 return (NULL); 1864 } 1865 size = 1; 1866 } 1867 1868 if (likely(ptr != NULL)) { 1869 tsd_t *tsd; 1870 1871 assert(malloc_initialized() || IS_INITIALIZER); 1872 malloc_thread_init(); 1873 tsd = tsd_fetch(); 1874 1875 witness_assert_lockless(tsd_tsdn(tsd)); 1876 1877 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1878 if (config_valgrind && unlikely(in_valgrind)) { 1879 old_rzsize = config_prof ? p2rz(tsd_tsdn(tsd), ptr) : 1880 u2rz(old_usize); 1881 } 1882 1883 if (config_prof && opt_prof) { 1884 usize = s2u(size); 1885 ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ? 1886 NULL : irealloc_prof(tsd, ptr, old_usize, usize); 1887 } else { 1888 if (config_stats || (config_valgrind && 1889 unlikely(in_valgrind))) 1890 usize = s2u(size); 1891 ret = iralloc(tsd, ptr, old_usize, size, 0, false); 1892 } 1893 tsdn = tsd_tsdn(tsd); 1894 } else { 1895 /* realloc(NULL, size) is equivalent to malloc(size). */ 1896 if (likely(!malloc_slow)) 1897 ret = ialloc_body(size, false, &tsdn, &usize, false); 1898 else 1899 ret = ialloc_body(size, false, &tsdn, &usize, true); 1900 assert(!tsdn_null(tsdn) || ret == NULL); 1901 } 1902 1903 if (unlikely(ret == NULL)) { 1904 if (config_xmalloc && unlikely(opt_xmalloc)) { 1905 malloc_write("<jemalloc>: Error in realloc(): " 1906 "out of memory\n"); 1907 abort(); 1908 } 1909 set_errno(ENOMEM); 1910 } 1911 if (config_stats && likely(ret != NULL)) { 1912 tsd_t *tsd; 1913 1914 assert(usize == isalloc(tsdn, ret, config_prof)); 1915 tsd = tsdn_tsd(tsdn); 1916 *tsd_thread_allocatedp_get(tsd) += usize; 1917 *tsd_thread_deallocatedp_get(tsd) += old_usize; 1918 } 1919 UTRACE(ptr, size, ret); 1920 JEMALLOC_VALGRIND_REALLOC(true, tsdn, ret, usize, true, ptr, old_usize, 1921 old_rzsize, true, false); 1922 witness_assert_lockless(tsdn); 1923 return (ret); 1924 } 1925 1926 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 1927 je_free(void *ptr) 1928 { 1929 1930 UTRACE(ptr, 0, 0); 1931 if (likely(ptr != NULL)) { 1932 tsd_t *tsd = tsd_fetch(); 1933 witness_assert_lockless(tsd_tsdn(tsd)); 1934 if (likely(!malloc_slow)) 1935 ifree(tsd, ptr, tcache_get(tsd, false), false); 1936 else 1937 ifree(tsd, ptr, tcache_get(tsd, false), true); 1938 witness_assert_lockless(tsd_tsdn(tsd)); 1939 } 1940 } 1941 1942 /* 1943 * End malloc(3)-compatible functions. 1944 */ 1945 /******************************************************************************/ 1946 /* 1947 * Begin non-standard override functions. 1948 */ 1949 1950 #ifdef JEMALLOC_OVERRIDE_MEMALIGN 1951 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1952 void JEMALLOC_NOTHROW * 1953 JEMALLOC_ATTR(malloc) 1954 je_memalign(size_t alignment, size_t size) 1955 { 1956 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1957 if (unlikely(imemalign(&ret, alignment, size, 1) != 0)) 1958 ret = NULL; 1959 return (ret); 1960 } 1961 #endif 1962 1963 #ifdef JEMALLOC_OVERRIDE_VALLOC 1964 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 1965 void JEMALLOC_NOTHROW * 1966 JEMALLOC_ATTR(malloc) 1967 je_valloc(size_t size) 1968 { 1969 void *ret JEMALLOC_CC_SILENCE_INIT(NULL); 1970 if (unlikely(imemalign(&ret, PAGE, size, 1) != 0)) 1971 ret = NULL; 1972 return (ret); 1973 } 1974 #endif 1975 1976 /* 1977 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has 1978 * #define je_malloc malloc 1979 */ 1980 #define malloc_is_malloc 1 1981 #define is_malloc_(a) malloc_is_ ## a 1982 #define is_malloc(a) is_malloc_(a) 1983 1984 #if ((is_malloc(je_malloc) == 1) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)) 1985 /* 1986 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible 1987 * to inconsistently reference libc's malloc(3)-compatible functions 1988 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). 1989 * 1990 * These definitions interpose hooks in glibc. The functions are actually 1991 * passed an extra argument for the caller return address, which will be 1992 * ignored. 1993 */ 1994 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; 1995 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; 1996 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; 1997 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK 1998 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = 1999 je_memalign; 2000 # endif 2001 #endif 2002 2003 /* 2004 * End non-standard override functions. 2005 */ 2006 /******************************************************************************/ 2007 /* 2008 * Begin non-standard functions. 2009 */ 2010 2011 JEMALLOC_ALWAYS_INLINE_C bool 2012 imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize, 2013 size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena) 2014 { 2015 2016 if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) { 2017 *alignment = 0; 2018 *usize = s2u(size); 2019 } else { 2020 *alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); 2021 *usize = sa2u(size, *alignment); 2022 } 2023 if (unlikely(*usize == 0 || *usize > HUGE_MAXCLASS)) 2024 return (true); 2025 *zero = MALLOCX_ZERO_GET(flags); 2026 if ((flags & MALLOCX_TCACHE_MASK) != 0) { 2027 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2028 *tcache = NULL; 2029 else 2030 *tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2031 } else 2032 *tcache = tcache_get(tsd, true); 2033 if ((flags & MALLOCX_ARENA_MASK) != 0) { 2034 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2035 *arena = arena_get(tsd_tsdn(tsd), arena_ind, true); 2036 if (unlikely(*arena == NULL)) 2037 return (true); 2038 } else 2039 *arena = NULL; 2040 return (false); 2041 } 2042 2043 JEMALLOC_ALWAYS_INLINE_C void * 2044 imallocx_flags(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, 2045 tcache_t *tcache, arena_t *arena, bool slow_path) 2046 { 2047 szind_t ind; 2048 2049 if (unlikely(alignment != 0)) 2050 return (ipalloct(tsdn, usize, alignment, zero, tcache, arena)); 2051 ind = size2index(usize); 2052 assert(ind < NSIZES); 2053 return (iallocztm(tsdn, usize, ind, zero, tcache, false, arena, 2054 slow_path)); 2055 } 2056 2057 static void * 2058 imallocx_prof_sample(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, 2059 tcache_t *tcache, arena_t *arena, bool slow_path) 2060 { 2061 void *p; 2062 2063 if (usize <= SMALL_MAXCLASS) { 2064 assert(((alignment == 0) ? s2u(LARGE_MINCLASS) : 2065 sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS); 2066 p = imallocx_flags(tsdn, LARGE_MINCLASS, alignment, zero, 2067 tcache, arena, slow_path); 2068 if (p == NULL) 2069 return (NULL); 2070 arena_prof_promoted(tsdn, p, usize); 2071 } else { 2072 p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena, 2073 slow_path); 2074 } 2075 2076 return (p); 2077 } 2078 2079 JEMALLOC_ALWAYS_INLINE_C void * 2080 imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path) 2081 { 2082 void *p; 2083 size_t alignment; 2084 bool zero; 2085 tcache_t *tcache; 2086 arena_t *arena; 2087 prof_tctx_t *tctx; 2088 2089 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, 2090 &zero, &tcache, &arena))) 2091 return (NULL); 2092 tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true); 2093 if (likely((uintptr_t)tctx == (uintptr_t)1U)) { 2094 p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, 2095 tcache, arena, slow_path); 2096 } else if ((uintptr_t)tctx > (uintptr_t)1U) { 2097 p = imallocx_prof_sample(tsd_tsdn(tsd), *usize, alignment, zero, 2098 tcache, arena, slow_path); 2099 } else 2100 p = NULL; 2101 if (unlikely(p == NULL)) { 2102 prof_alloc_rollback(tsd, tctx, true); 2103 return (NULL); 2104 } 2105 prof_malloc(tsd_tsdn(tsd), p, *usize, tctx); 2106 2107 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2108 return (p); 2109 } 2110 2111 JEMALLOC_ALWAYS_INLINE_C void * 2112 imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, 2113 bool slow_path) 2114 { 2115 void *p; 2116 size_t alignment; 2117 bool zero; 2118 tcache_t *tcache; 2119 arena_t *arena; 2120 2121 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, 2122 &zero, &tcache, &arena))) 2123 return (NULL); 2124 p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, tcache, 2125 arena, slow_path); 2126 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2127 return (p); 2128 } 2129 2130 /* This function guarantees that *tsdn is non-NULL on success. */ 2131 JEMALLOC_ALWAYS_INLINE_C void * 2132 imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize, 2133 bool slow_path) 2134 { 2135 tsd_t *tsd; 2136 2137 if (slow_path && unlikely(malloc_init())) { 2138 *tsdn = NULL; 2139 return (NULL); 2140 } 2141 2142 tsd = tsd_fetch(); 2143 *tsdn = tsd_tsdn(tsd); 2144 witness_assert_lockless(tsd_tsdn(tsd)); 2145 2146 if (likely(flags == 0)) { 2147 szind_t ind = size2index(size); 2148 if (unlikely(ind >= NSIZES)) 2149 return (NULL); 2150 if (config_stats || (config_prof && opt_prof) || (slow_path && 2151 config_valgrind && unlikely(in_valgrind))) { 2152 *usize = index2size(ind); 2153 assert(*usize > 0 && *usize <= HUGE_MAXCLASS); 2154 } 2155 2156 if (config_prof && opt_prof) { 2157 return (ialloc_prof(tsd, *usize, ind, false, 2158 slow_path)); 2159 } 2160 2161 return (ialloc(tsd, size, ind, false, slow_path)); 2162 } 2163 2164 if (config_prof && opt_prof) 2165 return (imallocx_prof(tsd, size, flags, usize, slow_path)); 2166 2167 return (imallocx_no_prof(tsd, size, flags, usize, slow_path)); 2168 } 2169 2170 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2171 void JEMALLOC_NOTHROW * 2172 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) 2173 je_mallocx(size_t size, int flags) 2174 { 2175 tsdn_t *tsdn; 2176 void *p; 2177 size_t usize; 2178 2179 assert(size != 0); 2180 2181 if (likely(!malloc_slow)) { 2182 p = imallocx_body(size, flags, &tsdn, &usize, false); 2183 ialloc_post_check(p, tsdn, usize, "mallocx", false, false); 2184 } else { 2185 p = imallocx_body(size, flags, &tsdn, &usize, true); 2186 ialloc_post_check(p, tsdn, usize, "mallocx", false, true); 2187 UTRACE(0, size, p); 2188 JEMALLOC_VALGRIND_MALLOC(p != NULL, tsdn, p, usize, 2189 MALLOCX_ZERO_GET(flags)); 2190 } 2191 2192 return (p); 2193 } 2194 2195 static void * 2196 irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, 2197 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, 2198 prof_tctx_t *tctx) 2199 { 2200 void *p; 2201 2202 if (tctx == NULL) 2203 return (NULL); 2204 if (usize <= SMALL_MAXCLASS) { 2205 p = iralloct(tsd, old_ptr, old_usize, LARGE_MINCLASS, alignment, 2206 zero, tcache, arena); 2207 if (p == NULL) 2208 return (NULL); 2209 arena_prof_promoted(tsd_tsdn(tsd), p, usize); 2210 } else { 2211 p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero, 2212 tcache, arena); 2213 } 2214 2215 return (p); 2216 } 2217 2218 JEMALLOC_ALWAYS_INLINE_C void * 2219 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, 2220 size_t alignment, size_t *usize, bool zero, tcache_t *tcache, 2221 arena_t *arena) 2222 { 2223 void *p; 2224 bool prof_active; 2225 prof_tctx_t *old_tctx, *tctx; 2226 2227 prof_active = prof_active_get_unlocked(); 2228 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr); 2229 tctx = prof_alloc_prep(tsd, *usize, prof_active, false); 2230 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2231 p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize, 2232 alignment, zero, tcache, arena, tctx); 2233 } else { 2234 p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero, 2235 tcache, arena); 2236 } 2237 if (unlikely(p == NULL)) { 2238 prof_alloc_rollback(tsd, tctx, false); 2239 return (NULL); 2240 } 2241 2242 if (p == old_ptr && alignment != 0) { 2243 /* 2244 * The allocation did not move, so it is possible that the size 2245 * class is smaller than would guarantee the requested 2246 * alignment, and that the alignment constraint was 2247 * serendipitously satisfied. Additionally, old_usize may not 2248 * be the same as the current usize because of in-place large 2249 * reallocation. Therefore, query the actual value of usize. 2250 */ 2251 *usize = isalloc(tsd_tsdn(tsd), p, config_prof); 2252 } 2253 prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr, 2254 old_usize, old_tctx); 2255 2256 return (p); 2257 } 2258 2259 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN 2260 void JEMALLOC_NOTHROW * 2261 JEMALLOC_ALLOC_SIZE(2) 2262 je_rallocx(void *ptr, size_t size, int flags) 2263 { 2264 void *p; 2265 tsd_t *tsd; 2266 size_t usize; 2267 size_t old_usize; 2268 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2269 size_t alignment = MALLOCX_ALIGN_GET(flags); 2270 bool zero = flags & MALLOCX_ZERO; 2271 arena_t *arena; 2272 tcache_t *tcache; 2273 2274 assert(ptr != NULL); 2275 assert(size != 0); 2276 assert(malloc_initialized() || IS_INITIALIZER); 2277 malloc_thread_init(); 2278 tsd = tsd_fetch(); 2279 witness_assert_lockless(tsd_tsdn(tsd)); 2280 2281 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { 2282 unsigned arena_ind = MALLOCX_ARENA_GET(flags); 2283 arena = arena_get(tsd_tsdn(tsd), arena_ind, true); 2284 if (unlikely(arena == NULL)) 2285 goto label_oom; 2286 } else 2287 arena = NULL; 2288 2289 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2290 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2291 tcache = NULL; 2292 else 2293 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2294 } else 2295 tcache = tcache_get(tsd, true); 2296 2297 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 2298 if (config_valgrind && unlikely(in_valgrind)) 2299 old_rzsize = u2rz(old_usize); 2300 2301 if (config_prof && opt_prof) { 2302 usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); 2303 if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) 2304 goto label_oom; 2305 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, 2306 zero, tcache, arena); 2307 if (unlikely(p == NULL)) 2308 goto label_oom; 2309 } else { 2310 p = iralloct(tsd, ptr, old_usize, size, alignment, zero, 2311 tcache, arena); 2312 if (unlikely(p == NULL)) 2313 goto label_oom; 2314 if (config_stats || (config_valgrind && unlikely(in_valgrind))) 2315 usize = isalloc(tsd_tsdn(tsd), p, config_prof); 2316 } 2317 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); 2318 2319 if (config_stats) { 2320 *tsd_thread_allocatedp_get(tsd) += usize; 2321 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2322 } 2323 UTRACE(ptr, size, p); 2324 JEMALLOC_VALGRIND_REALLOC(true, tsd_tsdn(tsd), p, usize, false, ptr, 2325 old_usize, old_rzsize, false, zero); 2326 witness_assert_lockless(tsd_tsdn(tsd)); 2327 return (p); 2328 label_oom: 2329 if (config_xmalloc && unlikely(opt_xmalloc)) { 2330 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n"); 2331 abort(); 2332 } 2333 UTRACE(ptr, size, 0); 2334 witness_assert_lockless(tsd_tsdn(tsd)); 2335 return (NULL); 2336 } 2337 2338 JEMALLOC_ALWAYS_INLINE_C size_t 2339 ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, 2340 size_t extra, size_t alignment, bool zero) 2341 { 2342 size_t usize; 2343 2344 if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero)) 2345 return (old_usize); 2346 usize = isalloc(tsdn, ptr, config_prof); 2347 2348 return (usize); 2349 } 2350 2351 static size_t 2352 ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, 2353 size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) 2354 { 2355 size_t usize; 2356 2357 if (tctx == NULL) 2358 return (old_usize); 2359 usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment, 2360 zero); 2361 2362 return (usize); 2363 } 2364 2365 JEMALLOC_ALWAYS_INLINE_C size_t 2366 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, 2367 size_t extra, size_t alignment, bool zero) 2368 { 2369 size_t usize_max, usize; 2370 bool prof_active; 2371 prof_tctx_t *old_tctx, *tctx; 2372 2373 prof_active = prof_active_get_unlocked(); 2374 old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr); 2375 /* 2376 * usize isn't knowable before ixalloc() returns when extra is non-zero. 2377 * Therefore, compute its maximum possible value and use that in 2378 * prof_alloc_prep() to decide whether to capture a backtrace. 2379 * prof_realloc() will use the actual usize to decide whether to sample. 2380 */ 2381 if (alignment == 0) { 2382 usize_max = s2u(size+extra); 2383 assert(usize_max > 0 && usize_max <= HUGE_MAXCLASS); 2384 } else { 2385 usize_max = sa2u(size+extra, alignment); 2386 if (unlikely(usize_max == 0 || usize_max > HUGE_MAXCLASS)) { 2387 /* 2388 * usize_max is out of range, and chances are that 2389 * allocation will fail, but use the maximum possible 2390 * value and carry on with prof_alloc_prep(), just in 2391 * case allocation succeeds. 2392 */ 2393 usize_max = HUGE_MAXCLASS; 2394 } 2395 } 2396 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); 2397 2398 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { 2399 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize, 2400 size, extra, alignment, zero, tctx); 2401 } else { 2402 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, 2403 extra, alignment, zero); 2404 } 2405 if (usize == old_usize) { 2406 prof_alloc_rollback(tsd, tctx, false); 2407 return (usize); 2408 } 2409 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize, 2410 old_tctx); 2411 2412 return (usize); 2413 } 2414 2415 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2416 je_xallocx(void *ptr, size_t size, size_t extra, int flags) 2417 { 2418 tsd_t *tsd; 2419 size_t usize, old_usize; 2420 UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); 2421 size_t alignment = MALLOCX_ALIGN_GET(flags); 2422 bool zero = flags & MALLOCX_ZERO; 2423 2424 assert(ptr != NULL); 2425 assert(size != 0); 2426 assert(SIZE_T_MAX - size >= extra); 2427 assert(malloc_initialized() || IS_INITIALIZER); 2428 malloc_thread_init(); 2429 tsd = tsd_fetch(); 2430 witness_assert_lockless(tsd_tsdn(tsd)); 2431 2432 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 2433 2434 /* 2435 * The API explicitly absolves itself of protecting against (size + 2436 * extra) numerical overflow, but we may need to clamp extra to avoid 2437 * exceeding HUGE_MAXCLASS. 2438 * 2439 * Ordinarily, size limit checking is handled deeper down, but here we 2440 * have to check as part of (size + extra) clamping, since we need the 2441 * clamped value in the above helper functions. 2442 */ 2443 if (unlikely(size > HUGE_MAXCLASS)) { 2444 usize = old_usize; 2445 goto label_not_resized; 2446 } 2447 if (unlikely(HUGE_MAXCLASS - size < extra)) 2448 extra = HUGE_MAXCLASS - size; 2449 2450 if (config_valgrind && unlikely(in_valgrind)) 2451 old_rzsize = u2rz(old_usize); 2452 2453 if (config_prof && opt_prof) { 2454 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, 2455 alignment, zero); 2456 } else { 2457 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, 2458 extra, alignment, zero); 2459 } 2460 if (unlikely(usize == old_usize)) 2461 goto label_not_resized; 2462 2463 if (config_stats) { 2464 *tsd_thread_allocatedp_get(tsd) += usize; 2465 *tsd_thread_deallocatedp_get(tsd) += old_usize; 2466 } 2467 JEMALLOC_VALGRIND_REALLOC(false, tsd_tsdn(tsd), ptr, usize, false, ptr, 2468 old_usize, old_rzsize, false, zero); 2469 label_not_resized: 2470 UTRACE(ptr, size, ptr); 2471 witness_assert_lockless(tsd_tsdn(tsd)); 2472 return (usize); 2473 } 2474 2475 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2476 JEMALLOC_ATTR(pure) 2477 je_sallocx(const void *ptr, int flags) 2478 { 2479 size_t usize; 2480 tsdn_t *tsdn; 2481 2482 assert(malloc_initialized() || IS_INITIALIZER); 2483 malloc_thread_init(); 2484 2485 tsdn = tsdn_fetch(); 2486 witness_assert_lockless(tsdn); 2487 2488 if (config_ivsalloc) 2489 usize = ivsalloc(tsdn, ptr, config_prof); 2490 else 2491 usize = isalloc(tsdn, ptr, config_prof); 2492 2493 witness_assert_lockless(tsdn); 2494 return (usize); 2495 } 2496 2497 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2498 je_dallocx(void *ptr, int flags) 2499 { 2500 tsd_t *tsd; 2501 tcache_t *tcache; 2502 2503 assert(ptr != NULL); 2504 assert(malloc_initialized() || IS_INITIALIZER); 2505 2506 tsd = tsd_fetch(); 2507 witness_assert_lockless(tsd_tsdn(tsd)); 2508 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2509 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2510 tcache = NULL; 2511 else 2512 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2513 } else 2514 tcache = tcache_get(tsd, false); 2515 2516 UTRACE(ptr, 0, 0); 2517 if (likely(!malloc_slow)) 2518 ifree(tsd, ptr, tcache, false); 2519 else 2520 ifree(tsd, ptr, tcache, true); 2521 witness_assert_lockless(tsd_tsdn(tsd)); 2522 } 2523 2524 JEMALLOC_ALWAYS_INLINE_C size_t 2525 inallocx(tsdn_t *tsdn, size_t size, int flags) 2526 { 2527 size_t usize; 2528 2529 witness_assert_lockless(tsdn); 2530 2531 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) 2532 usize = s2u(size); 2533 else 2534 usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); 2535 witness_assert_lockless(tsdn); 2536 return (usize); 2537 } 2538 2539 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2540 je_sdallocx(void *ptr, size_t size, int flags) 2541 { 2542 tsd_t *tsd; 2543 tcache_t *tcache; 2544 size_t usize; 2545 2546 assert(ptr != NULL); 2547 assert(malloc_initialized() || IS_INITIALIZER); 2548 tsd = tsd_fetch(); 2549 usize = inallocx(tsd_tsdn(tsd), size, flags); 2550 assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof)); 2551 2552 witness_assert_lockless(tsd_tsdn(tsd)); 2553 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { 2554 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) 2555 tcache = NULL; 2556 else 2557 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); 2558 } else 2559 tcache = tcache_get(tsd, false); 2560 2561 UTRACE(ptr, 0, 0); 2562 if (likely(!malloc_slow)) 2563 isfree(tsd, ptr, usize, tcache, false); 2564 else 2565 isfree(tsd, ptr, usize, tcache, true); 2566 witness_assert_lockless(tsd_tsdn(tsd)); 2567 } 2568 2569 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2570 JEMALLOC_ATTR(pure) 2571 je_nallocx(size_t size, int flags) 2572 { 2573 size_t usize; 2574 tsdn_t *tsdn; 2575 2576 assert(size != 0); 2577 2578 if (unlikely(malloc_init())) 2579 return (0); 2580 2581 tsdn = tsdn_fetch(); 2582 witness_assert_lockless(tsdn); 2583 2584 usize = inallocx(tsdn, size, flags); 2585 if (unlikely(usize > HUGE_MAXCLASS)) 2586 return (0); 2587 2588 witness_assert_lockless(tsdn); 2589 return (usize); 2590 } 2591 2592 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2593 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, 2594 size_t newlen) 2595 { 2596 int ret; 2597 tsd_t *tsd; 2598 2599 if (unlikely(malloc_init())) 2600 return (EAGAIN); 2601 2602 tsd = tsd_fetch(); 2603 witness_assert_lockless(tsd_tsdn(tsd)); 2604 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); 2605 witness_assert_lockless(tsd_tsdn(tsd)); 2606 return (ret); 2607 } 2608 2609 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2610 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) 2611 { 2612 int ret; 2613 tsdn_t *tsdn; 2614 2615 if (unlikely(malloc_init())) 2616 return (EAGAIN); 2617 2618 tsdn = tsdn_fetch(); 2619 witness_assert_lockless(tsdn); 2620 ret = ctl_nametomib(tsdn, name, mibp, miblenp); 2621 witness_assert_lockless(tsdn); 2622 return (ret); 2623 } 2624 2625 JEMALLOC_EXPORT int JEMALLOC_NOTHROW 2626 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 2627 void *newp, size_t newlen) 2628 { 2629 int ret; 2630 tsd_t *tsd; 2631 2632 if (unlikely(malloc_init())) 2633 return (EAGAIN); 2634 2635 tsd = tsd_fetch(); 2636 witness_assert_lockless(tsd_tsdn(tsd)); 2637 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); 2638 witness_assert_lockless(tsd_tsdn(tsd)); 2639 return (ret); 2640 } 2641 2642 JEMALLOC_EXPORT void JEMALLOC_NOTHROW 2643 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, 2644 const char *opts) 2645 { 2646 tsdn_t *tsdn; 2647 2648 tsdn = tsdn_fetch(); 2649 witness_assert_lockless(tsdn); 2650 stats_print(write_cb, cbopaque, opts); 2651 witness_assert_lockless(tsdn); 2652 } 2653 2654 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW 2655 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) 2656 { 2657 size_t ret; 2658 tsdn_t *tsdn; 2659 2660 assert(malloc_initialized() || IS_INITIALIZER); 2661 malloc_thread_init(); 2662 2663 tsdn = tsdn_fetch(); 2664 witness_assert_lockless(tsdn); 2665 2666 if (config_ivsalloc) 2667 ret = ivsalloc(tsdn, ptr, config_prof); 2668 else 2669 ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof); 2670 2671 witness_assert_lockless(tsdn); 2672 return (ret); 2673 } 2674 2675 /* 2676 * End non-standard functions. 2677 */ 2678 /******************************************************************************/ 2679 /* 2680 * Begin compatibility functions. 2681 */ 2682 2683 #define ALLOCM_LG_ALIGN(la) (la) 2684 #define ALLOCM_ALIGN(a) (ffsl(a)-1) 2685 #define ALLOCM_ZERO ((int)0x40) 2686 #define ALLOCM_NO_MOVE ((int)0x80) 2687 2688 #define ALLOCM_SUCCESS 0 2689 #define ALLOCM_ERR_OOM 1 2690 #define ALLOCM_ERR_NOT_MOVED 2 2691 2692 int 2693 je_allocm(void **ptr, size_t *rsize, size_t size, int flags) 2694 { 2695 void *p; 2696 2697 assert(ptr != NULL); 2698 2699 p = je_mallocx(size, flags); 2700 if (p == NULL) 2701 return (ALLOCM_ERR_OOM); 2702 if (rsize != NULL) 2703 *rsize = isalloc(tsdn_fetch(), p, config_prof); 2704 *ptr = p; 2705 return (ALLOCM_SUCCESS); 2706 } 2707 2708 int 2709 je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) 2710 { 2711 int ret; 2712 bool no_move = flags & ALLOCM_NO_MOVE; 2713 2714 assert(ptr != NULL); 2715 assert(*ptr != NULL); 2716 assert(size != 0); 2717 assert(SIZE_T_MAX - size >= extra); 2718 2719 if (no_move) { 2720 size_t usize = je_xallocx(*ptr, size, extra, flags); 2721 ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; 2722 if (rsize != NULL) 2723 *rsize = usize; 2724 } else { 2725 void *p = je_rallocx(*ptr, size+extra, flags); 2726 if (p != NULL) { 2727 *ptr = p; 2728 ret = ALLOCM_SUCCESS; 2729 } else 2730 ret = ALLOCM_ERR_OOM; 2731 if (rsize != NULL) 2732 *rsize = isalloc(tsdn_fetch(), *ptr, config_prof); 2733 } 2734 return (ret); 2735 } 2736 2737 int 2738 je_sallocm(const void *ptr, size_t *rsize, int flags) 2739 { 2740 2741 assert(rsize != NULL); 2742 *rsize = je_sallocx(ptr, flags); 2743 return (ALLOCM_SUCCESS); 2744 } 2745 2746 int 2747 je_dallocm(void *ptr, int flags) 2748 { 2749 2750 je_dallocx(ptr, flags); 2751 return (ALLOCM_SUCCESS); 2752 } 2753 2754 int 2755 je_nallocm(size_t *rsize, size_t size, int flags) 2756 { 2757 size_t usize; 2758 2759 usize = je_nallocx(size, flags); 2760 if (usize == 0) 2761 return (ALLOCM_ERR_OOM); 2762 if (rsize != NULL) 2763 *rsize = usize; 2764 return (ALLOCM_SUCCESS); 2765 } 2766 2767 #undef ALLOCM_LG_ALIGN 2768 #undef ALLOCM_ALIGN 2769 #undef ALLOCM_ZERO 2770 #undef ALLOCM_NO_MOVE 2771 2772 #undef ALLOCM_SUCCESS 2773 #undef ALLOCM_ERR_OOM 2774 #undef ALLOCM_ERR_NOT_MOVED 2775 2776 /* 2777 * End compatibility functions. 2778 */ 2779 /******************************************************************************/ 2780 /* 2781 * The following functions are used by threading libraries for protection of 2782 * malloc during fork(). 2783 */ 2784 2785 /* 2786 * If an application creates a thread before doing any allocation in the main 2787 * thread, then calls fork(2) in the main thread followed by memory allocation 2788 * in the child process, a race can occur that results in deadlock within the 2789 * child: the main thread may have forked while the created thread had 2790 * partially initialized the allocator. Ordinarily jemalloc prevents 2791 * fork/malloc races via the following functions it registers during 2792 * initialization using pthread_atfork(), but of course that does no good if 2793 * the allocator isn't fully initialized at fork time. The following library 2794 * constructor is a partial solution to this problem. It may still be possible 2795 * to trigger the deadlock described above, but doing so would involve forking 2796 * via a library constructor that runs before jemalloc's runs. 2797 */ 2798 #ifndef JEMALLOC_JET 2799 JEMALLOC_ATTR(constructor) 2800 static void 2801 jemalloc_constructor(void) 2802 { 2803 2804 malloc_init(); 2805 } 2806 #endif 2807 2808 #ifndef JEMALLOC_MUTEX_INIT_CB 2809 void 2810 jemalloc_prefork(void) 2811 #else 2812 JEMALLOC_EXPORT void 2813 _malloc_prefork(void) 2814 #endif 2815 { 2816 tsd_t *tsd; 2817 unsigned i, j, narenas; 2818 arena_t *arena; 2819 2820 #ifdef JEMALLOC_MUTEX_INIT_CB 2821 if (!malloc_initialized()) 2822 return; 2823 #endif 2824 assert(malloc_initialized()); 2825 2826 tsd = tsd_fetch(); 2827 2828 narenas = narenas_total_get(); 2829 2830 witness_prefork(tsd); 2831 /* Acquire all mutexes in a safe order. */ 2832 ctl_prefork(tsd_tsdn(tsd)); 2833 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock); 2834 prof_prefork0(tsd_tsdn(tsd)); 2835 for (i = 0; i < 3; i++) { 2836 for (j = 0; j < narenas; j++) { 2837 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) != 2838 NULL) { 2839 switch (i) { 2840 case 0: 2841 arena_prefork0(tsd_tsdn(tsd), arena); 2842 break; 2843 case 1: 2844 arena_prefork1(tsd_tsdn(tsd), arena); 2845 break; 2846 case 2: 2847 arena_prefork2(tsd_tsdn(tsd), arena); 2848 break; 2849 default: not_reached(); 2850 } 2851 } 2852 } 2853 } 2854 base_prefork(tsd_tsdn(tsd)); 2855 chunk_prefork(tsd_tsdn(tsd)); 2856 for (i = 0; i < narenas; i++) { 2857 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) 2858 arena_prefork3(tsd_tsdn(tsd), arena); 2859 } 2860 prof_prefork1(tsd_tsdn(tsd)); 2861 } 2862 2863 #ifndef JEMALLOC_MUTEX_INIT_CB 2864 void 2865 jemalloc_postfork_parent(void) 2866 #else 2867 JEMALLOC_EXPORT void 2868 _malloc_postfork(void) 2869 #endif 2870 { 2871 tsd_t *tsd; 2872 unsigned i, narenas; 2873 2874 #ifdef JEMALLOC_MUTEX_INIT_CB 2875 if (!malloc_initialized()) 2876 return; 2877 #endif 2878 assert(malloc_initialized()); 2879 2880 tsd = tsd_fetch(); 2881 2882 witness_postfork_parent(tsd); 2883 /* Release all mutexes, now that fork() has completed. */ 2884 chunk_postfork_parent(tsd_tsdn(tsd)); 2885 base_postfork_parent(tsd_tsdn(tsd)); 2886 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 2887 arena_t *arena; 2888 2889 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) 2890 arena_postfork_parent(tsd_tsdn(tsd), arena); 2891 } 2892 prof_postfork_parent(tsd_tsdn(tsd)); 2893 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock); 2894 ctl_postfork_parent(tsd_tsdn(tsd)); 2895 } 2896 2897 void 2898 jemalloc_postfork_child(void) 2899 { 2900 tsd_t *tsd; 2901 unsigned i, narenas; 2902 2903 assert(malloc_initialized()); 2904 2905 tsd = tsd_fetch(); 2906 2907 witness_postfork_child(tsd); 2908 /* Release all mutexes, now that fork() has completed. */ 2909 chunk_postfork_child(tsd_tsdn(tsd)); 2910 base_postfork_child(tsd_tsdn(tsd)); 2911 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { 2912 arena_t *arena; 2913 2914 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) 2915 arena_postfork_child(tsd_tsdn(tsd), arena); 2916 } 2917 prof_postfork_child(tsd_tsdn(tsd)); 2918 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock); 2919 ctl_postfork_child(tsd_tsdn(tsd)); 2920 } 2921 2922 void 2923 _malloc_first_thread(void) 2924 { 2925 2926 (void)malloc_mutex_first_thread(); 2927 } 2928 2929 /******************************************************************************/ 2930