1 #define JEMALLOC_PROF_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 /******************************************************************************/ 4 5 #ifdef JEMALLOC_PROF_LIBUNWIND 6 #define UNW_LOCAL_ONLY 7 #include <libunwind.h> 8 #endif 9 10 #ifdef JEMALLOC_PROF_LIBGCC 11 #include <unwind.h> 12 #endif 13 14 /******************************************************************************/ 15 /* Data. */ 16 17 bool opt_prof = false; 18 bool opt_prof_active = true; 19 bool opt_prof_thread_active_init = true; 20 size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; 21 ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; 22 bool opt_prof_gdump = false; 23 bool opt_prof_final = false; 24 bool opt_prof_leak = false; 25 bool opt_prof_accum = false; 26 char opt_prof_prefix[ 27 /* Minimize memory bloat for non-prof builds. */ 28 #ifdef JEMALLOC_PROF 29 PATH_MAX + 30 #endif 31 1]; 32 33 /* 34 * Initialized as opt_prof_active, and accessed via 35 * prof_active_[gs]et{_unlocked,}(). 36 */ 37 bool prof_active; 38 static malloc_mutex_t prof_active_mtx; 39 40 /* 41 * Initialized as opt_prof_thread_active_init, and accessed via 42 * prof_thread_active_init_[gs]et(). 43 */ 44 static bool prof_thread_active_init; 45 static malloc_mutex_t prof_thread_active_init_mtx; 46 47 /* 48 * Initialized as opt_prof_gdump, and accessed via 49 * prof_gdump_[gs]et{_unlocked,}(). 50 */ 51 bool prof_gdump_val; 52 static malloc_mutex_t prof_gdump_mtx; 53 54 uint64_t prof_interval = 0; 55 56 size_t lg_prof_sample; 57 58 /* 59 * Table of mutexes that are shared among gctx's. These are leaf locks, so 60 * there is no problem with using them for more than one gctx at the same time. 61 * The primary motivation for this sharing though is that gctx's are ephemeral, 62 * and destroying mutexes causes complications for systems that allocate when 63 * creating/destroying mutexes. 64 */ 65 static malloc_mutex_t *gctx_locks; 66 static unsigned cum_gctxs; /* Atomic counter. */ 67 68 /* 69 * Table of mutexes that are shared among tdata's. No operations require 70 * holding multiple tdata locks, so there is no problem with using them for more 71 * than one tdata at the same time, even though a gctx lock may be acquired 72 * while holding a tdata lock. 73 */ 74 static malloc_mutex_t *tdata_locks; 75 76 /* 77 * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data 78 * structure that knows about all backtraces currently captured. 79 */ 80 static ckh_t bt2gctx; 81 static malloc_mutex_t bt2gctx_mtx; 82 83 /* 84 * Tree of all extant prof_tdata_t structures, regardless of state, 85 * {attached,detached,expired}. 86 */ 87 static prof_tdata_tree_t tdatas; 88 static malloc_mutex_t tdatas_mtx; 89 90 static uint64_t next_thr_uid; 91 static malloc_mutex_t next_thr_uid_mtx; 92 93 static malloc_mutex_t prof_dump_seq_mtx; 94 static uint64_t prof_dump_seq; 95 static uint64_t prof_dump_iseq; 96 static uint64_t prof_dump_mseq; 97 static uint64_t prof_dump_useq; 98 99 /* 100 * This buffer is rather large for stack allocation, so use a single buffer for 101 * all profile dumps. 102 */ 103 static malloc_mutex_t prof_dump_mtx; 104 static char prof_dump_buf[ 105 /* Minimize memory bloat for non-prof builds. */ 106 #ifdef JEMALLOC_PROF 107 PROF_DUMP_BUFSIZE 108 #else 109 1 110 #endif 111 ]; 112 static unsigned prof_dump_buf_end; 113 static int prof_dump_fd; 114 115 /* Do not dump any profiles until bootstrapping is complete. */ 116 static bool prof_booted = false; 117 118 /******************************************************************************/ 119 /* 120 * Function prototypes for static functions that are referenced prior to 121 * definition. 122 */ 123 124 static bool prof_tctx_should_destroy(prof_tctx_t *tctx); 125 static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx); 126 static bool prof_tdata_should_destroy(prof_tdata_t *tdata, 127 bool even_if_attached); 128 static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, 129 bool even_if_attached); 130 static char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name); 131 132 /******************************************************************************/ 133 /* Red-black trees. */ 134 135 JEMALLOC_INLINE_C int 136 prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) 137 { 138 uint64_t a_thr_uid = a->thr_uid; 139 uint64_t b_thr_uid = b->thr_uid; 140 int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid); 141 if (ret == 0) { 142 uint64_t a_thr_discrim = a->thr_discrim; 143 uint64_t b_thr_discrim = b->thr_discrim; 144 ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim < 145 b_thr_discrim); 146 if (ret == 0) { 147 uint64_t a_tctx_uid = a->tctx_uid; 148 uint64_t b_tctx_uid = b->tctx_uid; 149 ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid < 150 b_tctx_uid); 151 } 152 } 153 return (ret); 154 } 155 156 rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, 157 tctx_link, prof_tctx_comp) 158 159 JEMALLOC_INLINE_C int 160 prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) 161 { 162 unsigned a_len = a->bt.len; 163 unsigned b_len = b->bt.len; 164 unsigned comp_len = (a_len < b_len) ? a_len : b_len; 165 int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *)); 166 if (ret == 0) 167 ret = (a_len > b_len) - (a_len < b_len); 168 return (ret); 169 } 170 171 rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link, 172 prof_gctx_comp) 173 174 JEMALLOC_INLINE_C int 175 prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) 176 { 177 int ret; 178 uint64_t a_uid = a->thr_uid; 179 uint64_t b_uid = b->thr_uid; 180 181 ret = ((a_uid > b_uid) - (a_uid < b_uid)); 182 if (ret == 0) { 183 uint64_t a_discrim = a->thr_discrim; 184 uint64_t b_discrim = b->thr_discrim; 185 186 ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim)); 187 } 188 return (ret); 189 } 190 191 rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, 192 prof_tdata_comp) 193 194 /******************************************************************************/ 195 196 void 197 prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) 198 { 199 prof_tdata_t *tdata; 200 201 cassert(config_prof); 202 203 if (updated) { 204 /* 205 * Compute a new sample threshold. This isn't very important in 206 * practice, because this function is rarely executed, so the 207 * potential for sample bias is minimal except in contrived 208 * programs. 209 */ 210 tdata = prof_tdata_get(tsd, true); 211 if (tdata != NULL) 212 prof_sample_threshold_update(tdata); 213 } 214 215 if ((uintptr_t)tctx > (uintptr_t)1U) { 216 malloc_mutex_lock(tctx->tdata->lock); 217 tctx->prepared = false; 218 if (prof_tctx_should_destroy(tctx)) 219 prof_tctx_destroy(tsd, tctx); 220 else 221 malloc_mutex_unlock(tctx->tdata->lock); 222 } 223 } 224 225 void 226 prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) 227 { 228 229 prof_tctx_set(ptr, usize, tctx); 230 231 malloc_mutex_lock(tctx->tdata->lock); 232 tctx->cnts.curobjs++; 233 tctx->cnts.curbytes += usize; 234 if (opt_prof_accum) { 235 tctx->cnts.accumobjs++; 236 tctx->cnts.accumbytes += usize; 237 } 238 tctx->prepared = false; 239 malloc_mutex_unlock(tctx->tdata->lock); 240 } 241 242 void 243 prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) 244 { 245 246 malloc_mutex_lock(tctx->tdata->lock); 247 assert(tctx->cnts.curobjs > 0); 248 assert(tctx->cnts.curbytes >= usize); 249 tctx->cnts.curobjs--; 250 tctx->cnts.curbytes -= usize; 251 252 if (prof_tctx_should_destroy(tctx)) 253 prof_tctx_destroy(tsd, tctx); 254 else 255 malloc_mutex_unlock(tctx->tdata->lock); 256 } 257 258 void 259 bt_init(prof_bt_t *bt, void **vec) 260 { 261 262 cassert(config_prof); 263 264 bt->vec = vec; 265 bt->len = 0; 266 } 267 268 JEMALLOC_INLINE_C void 269 prof_enter(tsd_t *tsd, prof_tdata_t *tdata) 270 { 271 272 cassert(config_prof); 273 assert(tdata == prof_tdata_get(tsd, false)); 274 275 if (tdata != NULL) { 276 assert(!tdata->enq); 277 tdata->enq = true; 278 } 279 280 malloc_mutex_lock(&bt2gctx_mtx); 281 } 282 283 JEMALLOC_INLINE_C void 284 prof_leave(tsd_t *tsd, prof_tdata_t *tdata) 285 { 286 287 cassert(config_prof); 288 assert(tdata == prof_tdata_get(tsd, false)); 289 290 malloc_mutex_unlock(&bt2gctx_mtx); 291 292 if (tdata != NULL) { 293 bool idump, gdump; 294 295 assert(tdata->enq); 296 tdata->enq = false; 297 idump = tdata->enq_idump; 298 tdata->enq_idump = false; 299 gdump = tdata->enq_gdump; 300 tdata->enq_gdump = false; 301 302 if (idump) 303 prof_idump(); 304 if (gdump) 305 prof_gdump(); 306 } 307 } 308 309 #ifdef JEMALLOC_PROF_LIBUNWIND 310 void 311 prof_backtrace(prof_bt_t *bt) 312 { 313 int nframes; 314 315 cassert(config_prof); 316 assert(bt->len == 0); 317 assert(bt->vec != NULL); 318 319 nframes = unw_backtrace(bt->vec, PROF_BT_MAX); 320 if (nframes <= 0) 321 return; 322 bt->len = nframes; 323 } 324 #elif (defined(JEMALLOC_PROF_LIBGCC)) 325 static _Unwind_Reason_Code 326 prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) 327 { 328 329 cassert(config_prof); 330 331 return (_URC_NO_REASON); 332 } 333 334 static _Unwind_Reason_Code 335 prof_unwind_callback(struct _Unwind_Context *context, void *arg) 336 { 337 prof_unwind_data_t *data = (prof_unwind_data_t *)arg; 338 void *ip; 339 340 cassert(config_prof); 341 342 ip = (void *)_Unwind_GetIP(context); 343 if (ip == NULL) 344 return (_URC_END_OF_STACK); 345 data->bt->vec[data->bt->len] = ip; 346 data->bt->len++; 347 if (data->bt->len == data->max) 348 return (_URC_END_OF_STACK); 349 350 return (_URC_NO_REASON); 351 } 352 353 void 354 prof_backtrace(prof_bt_t *bt) 355 { 356 prof_unwind_data_t data = {bt, PROF_BT_MAX}; 357 358 cassert(config_prof); 359 360 _Unwind_Backtrace(prof_unwind_callback, &data); 361 } 362 #elif (defined(JEMALLOC_PROF_GCC)) 363 void 364 prof_backtrace(prof_bt_t *bt) 365 { 366 #define BT_FRAME(i) \ 367 if ((i) < PROF_BT_MAX) { \ 368 void *p; \ 369 if (__builtin_frame_address(i) == 0) \ 370 return; \ 371 p = __builtin_return_address(i); \ 372 if (p == NULL) \ 373 return; \ 374 bt->vec[(i)] = p; \ 375 bt->len = (i) + 1; \ 376 } else \ 377 return; 378 379 cassert(config_prof); 380 381 BT_FRAME(0) 382 BT_FRAME(1) 383 BT_FRAME(2) 384 BT_FRAME(3) 385 BT_FRAME(4) 386 BT_FRAME(5) 387 BT_FRAME(6) 388 BT_FRAME(7) 389 BT_FRAME(8) 390 BT_FRAME(9) 391 392 BT_FRAME(10) 393 BT_FRAME(11) 394 BT_FRAME(12) 395 BT_FRAME(13) 396 BT_FRAME(14) 397 BT_FRAME(15) 398 BT_FRAME(16) 399 BT_FRAME(17) 400 BT_FRAME(18) 401 BT_FRAME(19) 402 403 BT_FRAME(20) 404 BT_FRAME(21) 405 BT_FRAME(22) 406 BT_FRAME(23) 407 BT_FRAME(24) 408 BT_FRAME(25) 409 BT_FRAME(26) 410 BT_FRAME(27) 411 BT_FRAME(28) 412 BT_FRAME(29) 413 414 BT_FRAME(30) 415 BT_FRAME(31) 416 BT_FRAME(32) 417 BT_FRAME(33) 418 BT_FRAME(34) 419 BT_FRAME(35) 420 BT_FRAME(36) 421 BT_FRAME(37) 422 BT_FRAME(38) 423 BT_FRAME(39) 424 425 BT_FRAME(40) 426 BT_FRAME(41) 427 BT_FRAME(42) 428 BT_FRAME(43) 429 BT_FRAME(44) 430 BT_FRAME(45) 431 BT_FRAME(46) 432 BT_FRAME(47) 433 BT_FRAME(48) 434 BT_FRAME(49) 435 436 BT_FRAME(50) 437 BT_FRAME(51) 438 BT_FRAME(52) 439 BT_FRAME(53) 440 BT_FRAME(54) 441 BT_FRAME(55) 442 BT_FRAME(56) 443 BT_FRAME(57) 444 BT_FRAME(58) 445 BT_FRAME(59) 446 447 BT_FRAME(60) 448 BT_FRAME(61) 449 BT_FRAME(62) 450 BT_FRAME(63) 451 BT_FRAME(64) 452 BT_FRAME(65) 453 BT_FRAME(66) 454 BT_FRAME(67) 455 BT_FRAME(68) 456 BT_FRAME(69) 457 458 BT_FRAME(70) 459 BT_FRAME(71) 460 BT_FRAME(72) 461 BT_FRAME(73) 462 BT_FRAME(74) 463 BT_FRAME(75) 464 BT_FRAME(76) 465 BT_FRAME(77) 466 BT_FRAME(78) 467 BT_FRAME(79) 468 469 BT_FRAME(80) 470 BT_FRAME(81) 471 BT_FRAME(82) 472 BT_FRAME(83) 473 BT_FRAME(84) 474 BT_FRAME(85) 475 BT_FRAME(86) 476 BT_FRAME(87) 477 BT_FRAME(88) 478 BT_FRAME(89) 479 480 BT_FRAME(90) 481 BT_FRAME(91) 482 BT_FRAME(92) 483 BT_FRAME(93) 484 BT_FRAME(94) 485 BT_FRAME(95) 486 BT_FRAME(96) 487 BT_FRAME(97) 488 BT_FRAME(98) 489 BT_FRAME(99) 490 491 BT_FRAME(100) 492 BT_FRAME(101) 493 BT_FRAME(102) 494 BT_FRAME(103) 495 BT_FRAME(104) 496 BT_FRAME(105) 497 BT_FRAME(106) 498 BT_FRAME(107) 499 BT_FRAME(108) 500 BT_FRAME(109) 501 502 BT_FRAME(110) 503 BT_FRAME(111) 504 BT_FRAME(112) 505 BT_FRAME(113) 506 BT_FRAME(114) 507 BT_FRAME(115) 508 BT_FRAME(116) 509 BT_FRAME(117) 510 BT_FRAME(118) 511 BT_FRAME(119) 512 513 BT_FRAME(120) 514 BT_FRAME(121) 515 BT_FRAME(122) 516 BT_FRAME(123) 517 BT_FRAME(124) 518 BT_FRAME(125) 519 BT_FRAME(126) 520 BT_FRAME(127) 521 #undef BT_FRAME 522 } 523 #else 524 void 525 prof_backtrace(prof_bt_t *bt) 526 { 527 528 cassert(config_prof); 529 not_reached(); 530 } 531 #endif 532 533 static malloc_mutex_t * 534 prof_gctx_mutex_choose(void) 535 { 536 unsigned ngctxs = atomic_add_u(&cum_gctxs, 1); 537 538 return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]); 539 } 540 541 static malloc_mutex_t * 542 prof_tdata_mutex_choose(uint64_t thr_uid) 543 { 544 545 return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]); 546 } 547 548 static prof_gctx_t * 549 prof_gctx_create(tsd_t *tsd, prof_bt_t *bt) 550 { 551 /* 552 * Create a single allocation that has space for vec of length bt->len. 553 */ 554 prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsd, offsetof(prof_gctx_t, 555 vec) + (bt->len * sizeof(void *)), false, tcache_get(tsd, true), 556 true, NULL); 557 if (gctx == NULL) 558 return (NULL); 559 gctx->lock = prof_gctx_mutex_choose(); 560 /* 561 * Set nlimbo to 1, in order to avoid a race condition with 562 * prof_tctx_destroy()/prof_gctx_try_destroy(). 563 */ 564 gctx->nlimbo = 1; 565 tctx_tree_new(&gctx->tctxs); 566 /* Duplicate bt. */ 567 memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *)); 568 gctx->bt.vec = gctx->vec; 569 gctx->bt.len = bt->len; 570 return (gctx); 571 } 572 573 static void 574 prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx, 575 prof_tdata_t *tdata) 576 { 577 578 cassert(config_prof); 579 580 /* 581 * Check that gctx is still unused by any thread cache before destroying 582 * it. prof_lookup() increments gctx->nlimbo in order to avoid a race 583 * condition with this function, as does prof_tctx_destroy() in order to 584 * avoid a race between the main body of prof_tctx_destroy() and entry 585 * into this function. 586 */ 587 prof_enter(tsd, tdata_self); 588 malloc_mutex_lock(gctx->lock); 589 assert(gctx->nlimbo != 0); 590 if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { 591 /* Remove gctx from bt2gctx. */ 592 if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) 593 not_reached(); 594 prof_leave(tsd, tdata_self); 595 /* Destroy gctx. */ 596 malloc_mutex_unlock(gctx->lock); 597 idalloctm(tsd, gctx, tcache_get(tsd, false), true); 598 } else { 599 /* 600 * Compensate for increment in prof_tctx_destroy() or 601 * prof_lookup(). 602 */ 603 gctx->nlimbo--; 604 malloc_mutex_unlock(gctx->lock); 605 prof_leave(tsd, tdata_self); 606 } 607 } 608 609 /* tctx->tdata->lock must be held. */ 610 static bool 611 prof_tctx_should_destroy(prof_tctx_t *tctx) 612 { 613 614 if (opt_prof_accum) 615 return (false); 616 if (tctx->cnts.curobjs != 0) 617 return (false); 618 if (tctx->prepared) 619 return (false); 620 return (true); 621 } 622 623 static bool 624 prof_gctx_should_destroy(prof_gctx_t *gctx) 625 { 626 627 if (opt_prof_accum) 628 return (false); 629 if (!tctx_tree_empty(&gctx->tctxs)) 630 return (false); 631 if (gctx->nlimbo != 0) 632 return (false); 633 return (true); 634 } 635 636 /* tctx->tdata->lock is held upon entry, and released before return. */ 637 static void 638 prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) 639 { 640 prof_tdata_t *tdata = tctx->tdata; 641 prof_gctx_t *gctx = tctx->gctx; 642 bool destroy_tdata, destroy_tctx, destroy_gctx; 643 644 assert(tctx->cnts.curobjs == 0); 645 assert(tctx->cnts.curbytes == 0); 646 assert(!opt_prof_accum); 647 assert(tctx->cnts.accumobjs == 0); 648 assert(tctx->cnts.accumbytes == 0); 649 650 ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL); 651 destroy_tdata = prof_tdata_should_destroy(tdata, false); 652 malloc_mutex_unlock(tdata->lock); 653 654 malloc_mutex_lock(gctx->lock); 655 switch (tctx->state) { 656 case prof_tctx_state_nominal: 657 tctx_tree_remove(&gctx->tctxs, tctx); 658 destroy_tctx = true; 659 if (prof_gctx_should_destroy(gctx)) { 660 /* 661 * Increment gctx->nlimbo in order to keep another 662 * thread from winning the race to destroy gctx while 663 * this one has gctx->lock dropped. Without this, it 664 * would be possible for another thread to: 665 * 666 * 1) Sample an allocation associated with gctx. 667 * 2) Deallocate the sampled object. 668 * 3) Successfully prof_gctx_try_destroy(gctx). 669 * 670 * The result would be that gctx no longer exists by the 671 * time this thread accesses it in 672 * prof_gctx_try_destroy(). 673 */ 674 gctx->nlimbo++; 675 destroy_gctx = true; 676 } else 677 destroy_gctx = false; 678 break; 679 case prof_tctx_state_dumping: 680 /* 681 * A dumping thread needs tctx to remain valid until dumping 682 * has finished. Change state such that the dumping thread will 683 * complete destruction during a late dump iteration phase. 684 */ 685 tctx->state = prof_tctx_state_purgatory; 686 destroy_tctx = false; 687 destroy_gctx = false; 688 break; 689 default: 690 not_reached(); 691 destroy_tctx = false; 692 destroy_gctx = false; 693 } 694 malloc_mutex_unlock(gctx->lock); 695 if (destroy_gctx) { 696 prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx, 697 tdata); 698 } 699 700 if (destroy_tdata) 701 prof_tdata_destroy(tsd, tdata, false); 702 703 if (destroy_tctx) 704 idalloctm(tsd, tctx, tcache_get(tsd, false), true); 705 } 706 707 static bool 708 prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, 709 void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) 710 { 711 union { 712 prof_gctx_t *p; 713 void *v; 714 } gctx; 715 union { 716 prof_bt_t *p; 717 void *v; 718 } btkey; 719 bool new_gctx; 720 721 prof_enter(tsd, tdata); 722 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) { 723 /* bt has never been seen before. Insert it. */ 724 gctx.p = prof_gctx_create(tsd, bt); 725 if (gctx.v == NULL) { 726 prof_leave(tsd, tdata); 727 return (true); 728 } 729 btkey.p = &gctx.p->bt; 730 if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) { 731 /* OOM. */ 732 prof_leave(tsd, tdata); 733 idalloctm(tsd, gctx.v, tcache_get(tsd, false), true); 734 return (true); 735 } 736 new_gctx = true; 737 } else { 738 /* 739 * Increment nlimbo, in order to avoid a race condition with 740 * prof_tctx_destroy()/prof_gctx_try_destroy(). 741 */ 742 malloc_mutex_lock(gctx.p->lock); 743 gctx.p->nlimbo++; 744 malloc_mutex_unlock(gctx.p->lock); 745 new_gctx = false; 746 } 747 prof_leave(tsd, tdata); 748 749 *p_btkey = btkey.v; 750 *p_gctx = gctx.p; 751 *p_new_gctx = new_gctx; 752 return (false); 753 } 754 755 prof_tctx_t * 756 prof_lookup(tsd_t *tsd, prof_bt_t *bt) 757 { 758 union { 759 prof_tctx_t *p; 760 void *v; 761 } ret; 762 prof_tdata_t *tdata; 763 bool not_found; 764 765 cassert(config_prof); 766 767 tdata = prof_tdata_get(tsd, false); 768 if (tdata == NULL) 769 return (NULL); 770 771 malloc_mutex_lock(tdata->lock); 772 not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v); 773 if (!not_found) /* Note double negative! */ 774 ret.p->prepared = true; 775 malloc_mutex_unlock(tdata->lock); 776 if (not_found) { 777 tcache_t *tcache; 778 void *btkey; 779 prof_gctx_t *gctx; 780 bool new_gctx, error; 781 782 /* 783 * This thread's cache lacks bt. Look for it in the global 784 * cache. 785 */ 786 if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx, 787 &new_gctx)) 788 return (NULL); 789 790 /* Link a prof_tctx_t into gctx for this thread. */ 791 tcache = tcache_get(tsd, true); 792 ret.v = iallocztm(tsd, sizeof(prof_tctx_t), false, tcache, true, 793 NULL); 794 if (ret.p == NULL) { 795 if (new_gctx) 796 prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 797 return (NULL); 798 } 799 ret.p->tdata = tdata; 800 ret.p->thr_uid = tdata->thr_uid; 801 ret.p->thr_discrim = tdata->thr_discrim; 802 memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); 803 ret.p->gctx = gctx; 804 ret.p->tctx_uid = tdata->tctx_uid_next++; 805 ret.p->prepared = true; 806 ret.p->state = prof_tctx_state_initializing; 807 malloc_mutex_lock(tdata->lock); 808 error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v); 809 malloc_mutex_unlock(tdata->lock); 810 if (error) { 811 if (new_gctx) 812 prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 813 idalloctm(tsd, ret.v, tcache, true); 814 return (NULL); 815 } 816 malloc_mutex_lock(gctx->lock); 817 ret.p->state = prof_tctx_state_nominal; 818 tctx_tree_insert(&gctx->tctxs, ret.p); 819 gctx->nlimbo--; 820 malloc_mutex_unlock(gctx->lock); 821 } 822 823 return (ret.p); 824 } 825 826 void 827 prof_sample_threshold_update(prof_tdata_t *tdata) 828 { 829 /* 830 * The body of this function is compiled out unless heap profiling is 831 * enabled, so that it is possible to compile jemalloc with floating 832 * point support completely disabled. Avoiding floating point code is 833 * important on memory-constrained systems, but it also enables a 834 * workaround for versions of glibc that don't properly save/restore 835 * floating point registers during dynamic lazy symbol loading (which 836 * internally calls into whatever malloc implementation happens to be 837 * integrated into the application). Note that some compilers (e.g. 838 * gcc 4.8) may use floating point registers for fast memory moves, so 839 * jemalloc must be compiled with such optimizations disabled (e.g. 840 * -mno-sse) in order for the workaround to be complete. 841 */ 842 #ifdef JEMALLOC_PROF 843 uint64_t r; 844 double u; 845 846 if (!config_prof) 847 return; 848 849 if (lg_prof_sample == 0) { 850 tdata->bytes_until_sample = 0; 851 return; 852 } 853 854 /* 855 * Compute sample interval as a geometrically distributed random 856 * variable with mean (2^lg_prof_sample). 857 * 858 * __ __ 859 * | log(u) | 1 860 * tdata->bytes_until_sample = | -------- |, where p = --------------- 861 * | log(1-p) | lg_prof_sample 862 * 2 863 * 864 * For more information on the math, see: 865 * 866 * Non-Uniform Random Variate Generation 867 * Luc Devroye 868 * Springer-Verlag, New York, 1986 869 * pp 500 870 * (http://luc.devroye.org/rnbookindex.html) 871 */ 872 prng64(r, 53, tdata->prng_state, UINT64_C(6364136223846793005), 873 UINT64_C(1442695040888963407)); 874 u = (double)r * (1.0/9007199254740992.0L); 875 tdata->bytes_until_sample = (uint64_t)(log(u) / 876 log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) 877 + (uint64_t)1U; 878 #endif 879 } 880 881 #ifdef JEMALLOC_JET 882 static prof_tdata_t * 883 prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 884 { 885 size_t *tdata_count = (size_t *)arg; 886 887 (*tdata_count)++; 888 889 return (NULL); 890 } 891 892 size_t 893 prof_tdata_count(void) 894 { 895 size_t tdata_count = 0; 896 897 malloc_mutex_lock(&tdatas_mtx); 898 tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter, 899 (void *)&tdata_count); 900 malloc_mutex_unlock(&tdatas_mtx); 901 902 return (tdata_count); 903 } 904 #endif 905 906 #ifdef JEMALLOC_JET 907 size_t 908 prof_bt_count(void) 909 { 910 size_t bt_count; 911 tsd_t *tsd; 912 prof_tdata_t *tdata; 913 914 tsd = tsd_fetch(); 915 tdata = prof_tdata_get(tsd, false); 916 if (tdata == NULL) 917 return (0); 918 919 malloc_mutex_lock(&bt2gctx_mtx); 920 bt_count = ckh_count(&bt2gctx); 921 malloc_mutex_unlock(&bt2gctx_mtx); 922 923 return (bt_count); 924 } 925 #endif 926 927 #ifdef JEMALLOC_JET 928 #undef prof_dump_open 929 #define prof_dump_open JEMALLOC_N(prof_dump_open_impl) 930 #endif 931 static int 932 prof_dump_open(bool propagate_err, const char *filename) 933 { 934 int fd; 935 936 fd = creat(filename, 0644); 937 if (fd == -1 && !propagate_err) { 938 malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n", 939 filename); 940 if (opt_abort) 941 abort(); 942 } 943 944 return (fd); 945 } 946 #ifdef JEMALLOC_JET 947 #undef prof_dump_open 948 #define prof_dump_open JEMALLOC_N(prof_dump_open) 949 prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl); 950 #endif 951 952 static bool 953 prof_dump_flush(bool propagate_err) 954 { 955 bool ret = false; 956 ssize_t err; 957 958 cassert(config_prof); 959 960 err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); 961 if (err == -1) { 962 if (!propagate_err) { 963 malloc_write("<jemalloc>: write() failed during heap " 964 "profile flush\n"); 965 if (opt_abort) 966 abort(); 967 } 968 ret = true; 969 } 970 prof_dump_buf_end = 0; 971 972 return (ret); 973 } 974 975 static bool 976 prof_dump_close(bool propagate_err) 977 { 978 bool ret; 979 980 assert(prof_dump_fd != -1); 981 ret = prof_dump_flush(propagate_err); 982 close(prof_dump_fd); 983 prof_dump_fd = -1; 984 985 return (ret); 986 } 987 988 static bool 989 prof_dump_write(bool propagate_err, const char *s) 990 { 991 unsigned i, slen, n; 992 993 cassert(config_prof); 994 995 i = 0; 996 slen = strlen(s); 997 while (i < slen) { 998 /* Flush the buffer if it is full. */ 999 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) 1000 if (prof_dump_flush(propagate_err) && propagate_err) 1001 return (true); 1002 1003 if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { 1004 /* Finish writing. */ 1005 n = slen - i; 1006 } else { 1007 /* Write as much of s as will fit. */ 1008 n = PROF_DUMP_BUFSIZE - prof_dump_buf_end; 1009 } 1010 memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); 1011 prof_dump_buf_end += n; 1012 i += n; 1013 } 1014 1015 return (false); 1016 } 1017 1018 JEMALLOC_FORMAT_PRINTF(2, 3) 1019 static bool 1020 prof_dump_printf(bool propagate_err, const char *format, ...) 1021 { 1022 bool ret; 1023 va_list ap; 1024 char buf[PROF_PRINTF_BUFSIZE]; 1025 1026 va_start(ap, format); 1027 malloc_vsnprintf(buf, sizeof(buf), format, ap); 1028 va_end(ap); 1029 ret = prof_dump_write(propagate_err, buf); 1030 1031 return (ret); 1032 } 1033 1034 /* tctx->tdata->lock is held. */ 1035 static void 1036 prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata) 1037 { 1038 1039 malloc_mutex_lock(tctx->gctx->lock); 1040 1041 switch (tctx->state) { 1042 case prof_tctx_state_initializing: 1043 malloc_mutex_unlock(tctx->gctx->lock); 1044 return; 1045 case prof_tctx_state_nominal: 1046 tctx->state = prof_tctx_state_dumping; 1047 malloc_mutex_unlock(tctx->gctx->lock); 1048 1049 memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); 1050 1051 tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; 1052 tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes; 1053 if (opt_prof_accum) { 1054 tdata->cnt_summed.accumobjs += 1055 tctx->dump_cnts.accumobjs; 1056 tdata->cnt_summed.accumbytes += 1057 tctx->dump_cnts.accumbytes; 1058 } 1059 break; 1060 case prof_tctx_state_dumping: 1061 case prof_tctx_state_purgatory: 1062 not_reached(); 1063 } 1064 } 1065 1066 /* gctx->lock is held. */ 1067 static void 1068 prof_tctx_merge_gctx(prof_tctx_t *tctx, prof_gctx_t *gctx) 1069 { 1070 1071 gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs; 1072 gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes; 1073 if (opt_prof_accum) { 1074 gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; 1075 gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes; 1076 } 1077 } 1078 1079 /* tctx->gctx is held. */ 1080 static prof_tctx_t * 1081 prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) 1082 { 1083 1084 switch (tctx->state) { 1085 case prof_tctx_state_nominal: 1086 /* New since dumping started; ignore. */ 1087 break; 1088 case prof_tctx_state_dumping: 1089 case prof_tctx_state_purgatory: 1090 prof_tctx_merge_gctx(tctx, tctx->gctx); 1091 break; 1092 default: 1093 not_reached(); 1094 } 1095 1096 return (NULL); 1097 } 1098 1099 /* gctx->lock is held. */ 1100 static prof_tctx_t * 1101 prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) 1102 { 1103 bool propagate_err = *(bool *)arg; 1104 1105 if (prof_dump_printf(propagate_err, 1106 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", 1107 tctx->thr_uid, tctx->dump_cnts.curobjs, tctx->dump_cnts.curbytes, 1108 tctx->dump_cnts.accumobjs, tctx->dump_cnts.accumbytes)) 1109 return (tctx); 1110 return (NULL); 1111 } 1112 1113 /* tctx->gctx is held. */ 1114 static prof_tctx_t * 1115 prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) 1116 { 1117 prof_tctx_t *ret; 1118 1119 switch (tctx->state) { 1120 case prof_tctx_state_nominal: 1121 /* New since dumping started; ignore. */ 1122 break; 1123 case prof_tctx_state_dumping: 1124 tctx->state = prof_tctx_state_nominal; 1125 break; 1126 case prof_tctx_state_purgatory: 1127 ret = tctx; 1128 goto label_return; 1129 default: 1130 not_reached(); 1131 } 1132 1133 ret = NULL; 1134 label_return: 1135 return (ret); 1136 } 1137 1138 static void 1139 prof_dump_gctx_prep(prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) 1140 { 1141 1142 cassert(config_prof); 1143 1144 malloc_mutex_lock(gctx->lock); 1145 1146 /* 1147 * Increment nlimbo so that gctx won't go away before dump. 1148 * Additionally, link gctx into the dump list so that it is included in 1149 * prof_dump()'s second pass. 1150 */ 1151 gctx->nlimbo++; 1152 gctx_tree_insert(gctxs, gctx); 1153 1154 memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t)); 1155 1156 malloc_mutex_unlock(gctx->lock); 1157 } 1158 1159 static prof_gctx_t * 1160 prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) 1161 { 1162 size_t *leak_ngctx = (size_t *)arg; 1163 1164 malloc_mutex_lock(gctx->lock); 1165 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, NULL); 1166 if (gctx->cnt_summed.curobjs != 0) 1167 (*leak_ngctx)++; 1168 malloc_mutex_unlock(gctx->lock); 1169 1170 return (NULL); 1171 } 1172 1173 static void 1174 prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) 1175 { 1176 prof_tdata_t *tdata = prof_tdata_get(tsd, false); 1177 prof_gctx_t *gctx; 1178 1179 /* 1180 * Standard tree iteration won't work here, because as soon as we 1181 * decrement gctx->nlimbo and unlock gctx, another thread can 1182 * concurrently destroy it, which will corrupt the tree. Therefore, 1183 * tear down the tree one node at a time during iteration. 1184 */ 1185 while ((gctx = gctx_tree_first(gctxs)) != NULL) { 1186 gctx_tree_remove(gctxs, gctx); 1187 malloc_mutex_lock(gctx->lock); 1188 { 1189 prof_tctx_t *next; 1190 1191 next = NULL; 1192 do { 1193 prof_tctx_t *to_destroy = 1194 tctx_tree_iter(&gctx->tctxs, next, 1195 prof_tctx_finish_iter, NULL); 1196 if (to_destroy != NULL) { 1197 next = tctx_tree_next(&gctx->tctxs, 1198 to_destroy); 1199 tctx_tree_remove(&gctx->tctxs, 1200 to_destroy); 1201 idalloctm(tsd, to_destroy, 1202 tcache_get(tsd, false), true); 1203 } else 1204 next = NULL; 1205 } while (next != NULL); 1206 } 1207 gctx->nlimbo--; 1208 if (prof_gctx_should_destroy(gctx)) { 1209 gctx->nlimbo++; 1210 malloc_mutex_unlock(gctx->lock); 1211 prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 1212 } else 1213 malloc_mutex_unlock(gctx->lock); 1214 } 1215 } 1216 1217 static prof_tdata_t * 1218 prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 1219 { 1220 prof_cnt_t *cnt_all = (prof_cnt_t *)arg; 1221 1222 malloc_mutex_lock(tdata->lock); 1223 if (!tdata->expired) { 1224 size_t tabind; 1225 union { 1226 prof_tctx_t *p; 1227 void *v; 1228 } tctx; 1229 1230 tdata->dumping = true; 1231 memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t)); 1232 for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL, 1233 &tctx.v);) 1234 prof_tctx_merge_tdata(tctx.p, tdata); 1235 1236 cnt_all->curobjs += tdata->cnt_summed.curobjs; 1237 cnt_all->curbytes += tdata->cnt_summed.curbytes; 1238 if (opt_prof_accum) { 1239 cnt_all->accumobjs += tdata->cnt_summed.accumobjs; 1240 cnt_all->accumbytes += tdata->cnt_summed.accumbytes; 1241 } 1242 } else 1243 tdata->dumping = false; 1244 malloc_mutex_unlock(tdata->lock); 1245 1246 return (NULL); 1247 } 1248 1249 static prof_tdata_t * 1250 prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 1251 { 1252 bool propagate_err = *(bool *)arg; 1253 1254 if (!tdata->dumping) 1255 return (NULL); 1256 1257 if (prof_dump_printf(propagate_err, 1258 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n", 1259 tdata->thr_uid, tdata->cnt_summed.curobjs, 1260 tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs, 1261 tdata->cnt_summed.accumbytes, 1262 (tdata->thread_name != NULL) ? " " : "", 1263 (tdata->thread_name != NULL) ? tdata->thread_name : "")) 1264 return (tdata); 1265 return (NULL); 1266 } 1267 1268 #ifdef JEMALLOC_JET 1269 #undef prof_dump_header 1270 #define prof_dump_header JEMALLOC_N(prof_dump_header_impl) 1271 #endif 1272 static bool 1273 prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all) 1274 { 1275 bool ret; 1276 1277 if (prof_dump_printf(propagate_err, 1278 "heap_v2/%"FMTu64"\n" 1279 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", 1280 ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs, 1281 cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) 1282 return (true); 1283 1284 malloc_mutex_lock(&tdatas_mtx); 1285 ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, 1286 (void *)&propagate_err) != NULL); 1287 malloc_mutex_unlock(&tdatas_mtx); 1288 return (ret); 1289 } 1290 #ifdef JEMALLOC_JET 1291 #undef prof_dump_header 1292 #define prof_dump_header JEMALLOC_N(prof_dump_header) 1293 prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl); 1294 #endif 1295 1296 /* gctx->lock is held. */ 1297 static bool 1298 prof_dump_gctx(bool propagate_err, prof_gctx_t *gctx, const prof_bt_t *bt, 1299 prof_gctx_tree_t *gctxs) 1300 { 1301 bool ret; 1302 unsigned i; 1303 1304 cassert(config_prof); 1305 1306 /* Avoid dumping such gctx's that have no useful data. */ 1307 if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) || 1308 (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) { 1309 assert(gctx->cnt_summed.curobjs == 0); 1310 assert(gctx->cnt_summed.curbytes == 0); 1311 assert(gctx->cnt_summed.accumobjs == 0); 1312 assert(gctx->cnt_summed.accumbytes == 0); 1313 ret = false; 1314 goto label_return; 1315 } 1316 1317 if (prof_dump_printf(propagate_err, "@")) { 1318 ret = true; 1319 goto label_return; 1320 } 1321 for (i = 0; i < bt->len; i++) { 1322 if (prof_dump_printf(propagate_err, " %#"FMTxPTR, 1323 (uintptr_t)bt->vec[i])) { 1324 ret = true; 1325 goto label_return; 1326 } 1327 } 1328 1329 if (prof_dump_printf(propagate_err, 1330 "\n" 1331 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", 1332 gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes, 1333 gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) { 1334 ret = true; 1335 goto label_return; 1336 } 1337 1338 if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, 1339 (void *)&propagate_err) != NULL) { 1340 ret = true; 1341 goto label_return; 1342 } 1343 1344 ret = false; 1345 label_return: 1346 return (ret); 1347 } 1348 1349 JEMALLOC_FORMAT_PRINTF(1, 2) 1350 static int 1351 prof_open_maps(const char *format, ...) 1352 { 1353 int mfd; 1354 va_list ap; 1355 char filename[PATH_MAX + 1]; 1356 1357 va_start(ap, format); 1358 malloc_vsnprintf(filename, sizeof(filename), format, ap); 1359 va_end(ap); 1360 mfd = open(filename, O_RDONLY); 1361 1362 return (mfd); 1363 } 1364 1365 static bool 1366 prof_dump_maps(bool propagate_err) 1367 { 1368 bool ret; 1369 int mfd; 1370 1371 cassert(config_prof); 1372 #ifdef __FreeBSD__ 1373 mfd = prof_open_maps("/proc/curproc/map"); 1374 #else 1375 { 1376 int pid = getpid(); 1377 1378 mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid); 1379 if (mfd == -1) 1380 mfd = prof_open_maps("/proc/%d/maps", pid); 1381 } 1382 #endif 1383 if (mfd != -1) { 1384 ssize_t nread; 1385 1386 if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && 1387 propagate_err) { 1388 ret = true; 1389 goto label_return; 1390 } 1391 nread = 0; 1392 do { 1393 prof_dump_buf_end += nread; 1394 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) { 1395 /* Make space in prof_dump_buf before read(). */ 1396 if (prof_dump_flush(propagate_err) && 1397 propagate_err) { 1398 ret = true; 1399 goto label_return; 1400 } 1401 } 1402 nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], 1403 PROF_DUMP_BUFSIZE - prof_dump_buf_end); 1404 } while (nread > 0); 1405 } else { 1406 ret = true; 1407 goto label_return; 1408 } 1409 1410 ret = false; 1411 label_return: 1412 if (mfd != -1) 1413 close(mfd); 1414 return (ret); 1415 } 1416 1417 static void 1418 prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx, 1419 const char *filename) 1420 { 1421 1422 if (cnt_all->curbytes != 0) { 1423 malloc_printf("<jemalloc>: Leak summary: %"FMTu64" byte%s, %" 1424 FMTu64" object%s, %zu context%s\n", 1425 cnt_all->curbytes, (cnt_all->curbytes != 1) ? "s" : "", 1426 cnt_all->curobjs, (cnt_all->curobjs != 1) ? "s" : "", 1427 leak_ngctx, (leak_ngctx != 1) ? "s" : ""); 1428 malloc_printf( 1429 "<jemalloc>: Run jeprof on \"%s\" for leak detail\n", 1430 filename); 1431 } 1432 } 1433 1434 static prof_gctx_t * 1435 prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) 1436 { 1437 prof_gctx_t *ret; 1438 bool propagate_err = *(bool *)arg; 1439 1440 malloc_mutex_lock(gctx->lock); 1441 1442 if (prof_dump_gctx(propagate_err, gctx, &gctx->bt, gctxs)) { 1443 ret = gctx; 1444 goto label_return; 1445 } 1446 1447 ret = NULL; 1448 label_return: 1449 malloc_mutex_unlock(gctx->lock); 1450 return (ret); 1451 } 1452 1453 static bool 1454 prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) 1455 { 1456 prof_tdata_t *tdata; 1457 prof_cnt_t cnt_all; 1458 size_t tabind; 1459 union { 1460 prof_gctx_t *p; 1461 void *v; 1462 } gctx; 1463 size_t leak_ngctx; 1464 prof_gctx_tree_t gctxs; 1465 1466 cassert(config_prof); 1467 1468 tdata = prof_tdata_get(tsd, true); 1469 if (tdata == NULL) 1470 return (true); 1471 1472 malloc_mutex_lock(&prof_dump_mtx); 1473 prof_enter(tsd, tdata); 1474 1475 /* 1476 * Put gctx's in limbo and clear their counters in preparation for 1477 * summing. 1478 */ 1479 gctx_tree_new(&gctxs); 1480 for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) 1481 prof_dump_gctx_prep(gctx.p, &gctxs); 1482 1483 /* 1484 * Iterate over tdatas, and for the non-expired ones snapshot their tctx 1485 * stats and merge them into the associated gctx's. 1486 */ 1487 memset(&cnt_all, 0, sizeof(prof_cnt_t)); 1488 malloc_mutex_lock(&tdatas_mtx); 1489 tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, (void *)&cnt_all); 1490 malloc_mutex_unlock(&tdatas_mtx); 1491 1492 /* Merge tctx stats into gctx's. */ 1493 leak_ngctx = 0; 1494 gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, (void *)&leak_ngctx); 1495 1496 prof_leave(tsd, tdata); 1497 1498 /* Create dump file. */ 1499 if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) 1500 goto label_open_close_error; 1501 1502 /* Dump profile header. */ 1503 if (prof_dump_header(propagate_err, &cnt_all)) 1504 goto label_write_error; 1505 1506 /* Dump per gctx profile stats. */ 1507 if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter, 1508 (void *)&propagate_err) != NULL) 1509 goto label_write_error; 1510 1511 /* Dump /proc/<pid>/maps if possible. */ 1512 if (prof_dump_maps(propagate_err)) 1513 goto label_write_error; 1514 1515 if (prof_dump_close(propagate_err)) 1516 goto label_open_close_error; 1517 1518 prof_gctx_finish(tsd, &gctxs); 1519 malloc_mutex_unlock(&prof_dump_mtx); 1520 1521 if (leakcheck) 1522 prof_leakcheck(&cnt_all, leak_ngctx, filename); 1523 1524 return (false); 1525 label_write_error: 1526 prof_dump_close(propagate_err); 1527 label_open_close_error: 1528 prof_gctx_finish(tsd, &gctxs); 1529 malloc_mutex_unlock(&prof_dump_mtx); 1530 return (true); 1531 } 1532 1533 #define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) 1534 #define VSEQ_INVALID UINT64_C(0xffffffffffffffff) 1535 static void 1536 prof_dump_filename(char *filename, char v, uint64_t vseq) 1537 { 1538 1539 cassert(config_prof); 1540 1541 if (vseq != VSEQ_INVALID) { 1542 /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */ 1543 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 1544 "%s.%d.%"FMTu64".%c%"FMTu64".heap", 1545 opt_prof_prefix, (int)getpid(), prof_dump_seq, v, vseq); 1546 } else { 1547 /* "<prefix>.<pid>.<seq>.<v>.heap" */ 1548 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 1549 "%s.%d.%"FMTu64".%c.heap", 1550 opt_prof_prefix, (int)getpid(), prof_dump_seq, v); 1551 } 1552 prof_dump_seq++; 1553 } 1554 1555 static void 1556 prof_fdump(void) 1557 { 1558 tsd_t *tsd; 1559 char filename[DUMP_FILENAME_BUFSIZE]; 1560 1561 cassert(config_prof); 1562 assert(opt_prof_final); 1563 assert(opt_prof_prefix[0] != '\0'); 1564 1565 if (!prof_booted) 1566 return; 1567 tsd = tsd_fetch(); 1568 1569 malloc_mutex_lock(&prof_dump_seq_mtx); 1570 prof_dump_filename(filename, 'f', VSEQ_INVALID); 1571 malloc_mutex_unlock(&prof_dump_seq_mtx); 1572 prof_dump(tsd, false, filename, opt_prof_leak); 1573 } 1574 1575 void 1576 prof_idump(void) 1577 { 1578 tsd_t *tsd; 1579 prof_tdata_t *tdata; 1580 1581 cassert(config_prof); 1582 1583 if (!prof_booted) 1584 return; 1585 tsd = tsd_fetch(); 1586 tdata = prof_tdata_get(tsd, false); 1587 if (tdata == NULL) 1588 return; 1589 if (tdata->enq) { 1590 tdata->enq_idump = true; 1591 return; 1592 } 1593 1594 if (opt_prof_prefix[0] != '\0') { 1595 char filename[PATH_MAX + 1]; 1596 malloc_mutex_lock(&prof_dump_seq_mtx); 1597 prof_dump_filename(filename, 'i', prof_dump_iseq); 1598 prof_dump_iseq++; 1599 malloc_mutex_unlock(&prof_dump_seq_mtx); 1600 prof_dump(tsd, false, filename, false); 1601 } 1602 } 1603 1604 bool 1605 prof_mdump(const char *filename) 1606 { 1607 tsd_t *tsd; 1608 char filename_buf[DUMP_FILENAME_BUFSIZE]; 1609 1610 cassert(config_prof); 1611 1612 if (!opt_prof || !prof_booted) 1613 return (true); 1614 tsd = tsd_fetch(); 1615 1616 if (filename == NULL) { 1617 /* No filename specified, so automatically generate one. */ 1618 if (opt_prof_prefix[0] == '\0') 1619 return (true); 1620 malloc_mutex_lock(&prof_dump_seq_mtx); 1621 prof_dump_filename(filename_buf, 'm', prof_dump_mseq); 1622 prof_dump_mseq++; 1623 malloc_mutex_unlock(&prof_dump_seq_mtx); 1624 filename = filename_buf; 1625 } 1626 return (prof_dump(tsd, true, filename, false)); 1627 } 1628 1629 void 1630 prof_gdump(void) 1631 { 1632 tsd_t *tsd; 1633 prof_tdata_t *tdata; 1634 1635 cassert(config_prof); 1636 1637 if (!prof_booted) 1638 return; 1639 tsd = tsd_fetch(); 1640 tdata = prof_tdata_get(tsd, false); 1641 if (tdata == NULL) 1642 return; 1643 if (tdata->enq) { 1644 tdata->enq_gdump = true; 1645 return; 1646 } 1647 1648 if (opt_prof_prefix[0] != '\0') { 1649 char filename[DUMP_FILENAME_BUFSIZE]; 1650 malloc_mutex_lock(&prof_dump_seq_mtx); 1651 prof_dump_filename(filename, 'u', prof_dump_useq); 1652 prof_dump_useq++; 1653 malloc_mutex_unlock(&prof_dump_seq_mtx); 1654 prof_dump(tsd, false, filename, false); 1655 } 1656 } 1657 1658 static void 1659 prof_bt_hash(const void *key, size_t r_hash[2]) 1660 { 1661 prof_bt_t *bt = (prof_bt_t *)key; 1662 1663 cassert(config_prof); 1664 1665 hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash); 1666 } 1667 1668 static bool 1669 prof_bt_keycomp(const void *k1, const void *k2) 1670 { 1671 const prof_bt_t *bt1 = (prof_bt_t *)k1; 1672 const prof_bt_t *bt2 = (prof_bt_t *)k2; 1673 1674 cassert(config_prof); 1675 1676 if (bt1->len != bt2->len) 1677 return (false); 1678 return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); 1679 } 1680 1681 JEMALLOC_INLINE_C uint64_t 1682 prof_thr_uid_alloc(void) 1683 { 1684 uint64_t thr_uid; 1685 1686 malloc_mutex_lock(&next_thr_uid_mtx); 1687 thr_uid = next_thr_uid; 1688 next_thr_uid++; 1689 malloc_mutex_unlock(&next_thr_uid_mtx); 1690 1691 return (thr_uid); 1692 } 1693 1694 static prof_tdata_t * 1695 prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, 1696 char *thread_name, bool active) 1697 { 1698 prof_tdata_t *tdata; 1699 tcache_t *tcache; 1700 1701 cassert(config_prof); 1702 1703 /* Initialize an empty cache for this thread. */ 1704 tcache = tcache_get(tsd, true); 1705 tdata = (prof_tdata_t *)iallocztm(tsd, sizeof(prof_tdata_t), false, 1706 tcache, true, NULL); 1707 if (tdata == NULL) 1708 return (NULL); 1709 1710 tdata->lock = prof_tdata_mutex_choose(thr_uid); 1711 tdata->thr_uid = thr_uid; 1712 tdata->thr_discrim = thr_discrim; 1713 tdata->thread_name = thread_name; 1714 tdata->attached = true; 1715 tdata->expired = false; 1716 tdata->tctx_uid_next = 0; 1717 1718 if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, 1719 prof_bt_hash, prof_bt_keycomp)) { 1720 idalloctm(tsd, tdata, tcache, true); 1721 return (NULL); 1722 } 1723 1724 tdata->prng_state = (uint64_t)(uintptr_t)tdata; 1725 prof_sample_threshold_update(tdata); 1726 1727 tdata->enq = false; 1728 tdata->enq_idump = false; 1729 tdata->enq_gdump = false; 1730 1731 tdata->dumping = false; 1732 tdata->active = active; 1733 1734 malloc_mutex_lock(&tdatas_mtx); 1735 tdata_tree_insert(&tdatas, tdata); 1736 malloc_mutex_unlock(&tdatas_mtx); 1737 1738 return (tdata); 1739 } 1740 1741 prof_tdata_t * 1742 prof_tdata_init(tsd_t *tsd) 1743 { 1744 1745 return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(), 0, NULL, 1746 prof_thread_active_init_get())); 1747 } 1748 1749 /* tdata->lock must be held. */ 1750 static bool 1751 prof_tdata_should_destroy(prof_tdata_t *tdata, bool even_if_attached) 1752 { 1753 1754 if (tdata->attached && !even_if_attached) 1755 return (false); 1756 if (ckh_count(&tdata->bt2tctx) != 0) 1757 return (false); 1758 return (true); 1759 } 1760 1761 /* tdatas_mtx must be held. */ 1762 static void 1763 prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata, 1764 bool even_if_attached) 1765 { 1766 tcache_t *tcache; 1767 1768 assert(prof_tdata_should_destroy(tdata, even_if_attached)); 1769 assert(tsd_prof_tdata_get(tsd) != tdata); 1770 1771 tdata_tree_remove(&tdatas, tdata); 1772 1773 tcache = tcache_get(tsd, false); 1774 if (tdata->thread_name != NULL) 1775 idalloctm(tsd, tdata->thread_name, tcache, true); 1776 ckh_delete(tsd, &tdata->bt2tctx); 1777 idalloctm(tsd, tdata, tcache, true); 1778 } 1779 1780 static void 1781 prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) 1782 { 1783 1784 malloc_mutex_lock(&tdatas_mtx); 1785 prof_tdata_destroy_locked(tsd, tdata, even_if_attached); 1786 malloc_mutex_unlock(&tdatas_mtx); 1787 } 1788 1789 static void 1790 prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) 1791 { 1792 bool destroy_tdata; 1793 1794 malloc_mutex_lock(tdata->lock); 1795 if (tdata->attached) { 1796 destroy_tdata = prof_tdata_should_destroy(tdata, true); 1797 /* 1798 * Only detach if !destroy_tdata, because detaching would allow 1799 * another thread to win the race to destroy tdata. 1800 */ 1801 if (!destroy_tdata) 1802 tdata->attached = false; 1803 tsd_prof_tdata_set(tsd, NULL); 1804 } else 1805 destroy_tdata = false; 1806 malloc_mutex_unlock(tdata->lock); 1807 if (destroy_tdata) 1808 prof_tdata_destroy(tsd, tdata, true); 1809 } 1810 1811 prof_tdata_t * 1812 prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) 1813 { 1814 uint64_t thr_uid = tdata->thr_uid; 1815 uint64_t thr_discrim = tdata->thr_discrim + 1; 1816 char *thread_name = (tdata->thread_name != NULL) ? 1817 prof_thread_name_alloc(tsd, tdata->thread_name) : NULL; 1818 bool active = tdata->active; 1819 1820 prof_tdata_detach(tsd, tdata); 1821 return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name, 1822 active)); 1823 } 1824 1825 static bool 1826 prof_tdata_expire(prof_tdata_t *tdata) 1827 { 1828 bool destroy_tdata; 1829 1830 malloc_mutex_lock(tdata->lock); 1831 if (!tdata->expired) { 1832 tdata->expired = true; 1833 destroy_tdata = tdata->attached ? false : 1834 prof_tdata_should_destroy(tdata, false); 1835 } else 1836 destroy_tdata = false; 1837 malloc_mutex_unlock(tdata->lock); 1838 1839 return (destroy_tdata); 1840 } 1841 1842 static prof_tdata_t * 1843 prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 1844 { 1845 1846 return (prof_tdata_expire(tdata) ? tdata : NULL); 1847 } 1848 1849 void 1850 prof_reset(tsd_t *tsd, size_t lg_sample) 1851 { 1852 prof_tdata_t *next; 1853 1854 assert(lg_sample < (sizeof(uint64_t) << 3)); 1855 1856 malloc_mutex_lock(&prof_dump_mtx); 1857 malloc_mutex_lock(&tdatas_mtx); 1858 1859 lg_prof_sample = lg_sample; 1860 1861 next = NULL; 1862 do { 1863 prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next, 1864 prof_tdata_reset_iter, NULL); 1865 if (to_destroy != NULL) { 1866 next = tdata_tree_next(&tdatas, to_destroy); 1867 prof_tdata_destroy_locked(tsd, to_destroy, false); 1868 } else 1869 next = NULL; 1870 } while (next != NULL); 1871 1872 malloc_mutex_unlock(&tdatas_mtx); 1873 malloc_mutex_unlock(&prof_dump_mtx); 1874 } 1875 1876 void 1877 prof_tdata_cleanup(tsd_t *tsd) 1878 { 1879 prof_tdata_t *tdata; 1880 1881 if (!config_prof) 1882 return; 1883 1884 tdata = tsd_prof_tdata_get(tsd); 1885 if (tdata != NULL) 1886 prof_tdata_detach(tsd, tdata); 1887 } 1888 1889 bool 1890 prof_active_get(void) 1891 { 1892 bool prof_active_current; 1893 1894 malloc_mutex_lock(&prof_active_mtx); 1895 prof_active_current = prof_active; 1896 malloc_mutex_unlock(&prof_active_mtx); 1897 return (prof_active_current); 1898 } 1899 1900 bool 1901 prof_active_set(bool active) 1902 { 1903 bool prof_active_old; 1904 1905 malloc_mutex_lock(&prof_active_mtx); 1906 prof_active_old = prof_active; 1907 prof_active = active; 1908 malloc_mutex_unlock(&prof_active_mtx); 1909 return (prof_active_old); 1910 } 1911 1912 const char * 1913 prof_thread_name_get(void) 1914 { 1915 tsd_t *tsd; 1916 prof_tdata_t *tdata; 1917 1918 tsd = tsd_fetch(); 1919 tdata = prof_tdata_get(tsd, true); 1920 if (tdata == NULL) 1921 return (""); 1922 return (tdata->thread_name != NULL ? tdata->thread_name : ""); 1923 } 1924 1925 static char * 1926 prof_thread_name_alloc(tsd_t *tsd, const char *thread_name) 1927 { 1928 char *ret; 1929 size_t size; 1930 1931 if (thread_name == NULL) 1932 return (NULL); 1933 1934 size = strlen(thread_name) + 1; 1935 if (size == 1) 1936 return (""); 1937 1938 ret = iallocztm(tsd, size, false, tcache_get(tsd, true), true, NULL); 1939 if (ret == NULL) 1940 return (NULL); 1941 memcpy(ret, thread_name, size); 1942 return (ret); 1943 } 1944 1945 int 1946 prof_thread_name_set(tsd_t *tsd, const char *thread_name) 1947 { 1948 prof_tdata_t *tdata; 1949 unsigned i; 1950 char *s; 1951 1952 tdata = prof_tdata_get(tsd, true); 1953 if (tdata == NULL) 1954 return (EAGAIN); 1955 1956 /* Validate input. */ 1957 if (thread_name == NULL) 1958 return (EFAULT); 1959 for (i = 0; thread_name[i] != '\0'; i++) { 1960 char c = thread_name[i]; 1961 if (!isgraph(c) && !isblank(c)) 1962 return (EFAULT); 1963 } 1964 1965 s = prof_thread_name_alloc(tsd, thread_name); 1966 if (s == NULL) 1967 return (EAGAIN); 1968 1969 if (tdata->thread_name != NULL) { 1970 idalloctm(tsd, tdata->thread_name, tcache_get(tsd, false), 1971 true); 1972 tdata->thread_name = NULL; 1973 } 1974 if (strlen(s) > 0) 1975 tdata->thread_name = s; 1976 return (0); 1977 } 1978 1979 bool 1980 prof_thread_active_get(void) 1981 { 1982 tsd_t *tsd; 1983 prof_tdata_t *tdata; 1984 1985 tsd = tsd_fetch(); 1986 tdata = prof_tdata_get(tsd, true); 1987 if (tdata == NULL) 1988 return (false); 1989 return (tdata->active); 1990 } 1991 1992 bool 1993 prof_thread_active_set(bool active) 1994 { 1995 tsd_t *tsd; 1996 prof_tdata_t *tdata; 1997 1998 tsd = tsd_fetch(); 1999 tdata = prof_tdata_get(tsd, true); 2000 if (tdata == NULL) 2001 return (true); 2002 tdata->active = active; 2003 return (false); 2004 } 2005 2006 bool 2007 prof_thread_active_init_get(void) 2008 { 2009 bool active_init; 2010 2011 malloc_mutex_lock(&prof_thread_active_init_mtx); 2012 active_init = prof_thread_active_init; 2013 malloc_mutex_unlock(&prof_thread_active_init_mtx); 2014 return (active_init); 2015 } 2016 2017 bool 2018 prof_thread_active_init_set(bool active_init) 2019 { 2020 bool active_init_old; 2021 2022 malloc_mutex_lock(&prof_thread_active_init_mtx); 2023 active_init_old = prof_thread_active_init; 2024 prof_thread_active_init = active_init; 2025 malloc_mutex_unlock(&prof_thread_active_init_mtx); 2026 return (active_init_old); 2027 } 2028 2029 bool 2030 prof_gdump_get(void) 2031 { 2032 bool prof_gdump_current; 2033 2034 malloc_mutex_lock(&prof_gdump_mtx); 2035 prof_gdump_current = prof_gdump_val; 2036 malloc_mutex_unlock(&prof_gdump_mtx); 2037 return (prof_gdump_current); 2038 } 2039 2040 bool 2041 prof_gdump_set(bool gdump) 2042 { 2043 bool prof_gdump_old; 2044 2045 malloc_mutex_lock(&prof_gdump_mtx); 2046 prof_gdump_old = prof_gdump_val; 2047 prof_gdump_val = gdump; 2048 malloc_mutex_unlock(&prof_gdump_mtx); 2049 return (prof_gdump_old); 2050 } 2051 2052 void 2053 prof_boot0(void) 2054 { 2055 2056 cassert(config_prof); 2057 2058 memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, 2059 sizeof(PROF_PREFIX_DEFAULT)); 2060 } 2061 2062 void 2063 prof_boot1(void) 2064 { 2065 2066 cassert(config_prof); 2067 2068 /* 2069 * opt_prof must be in its final state before any arenas are 2070 * initialized, so this function must be executed early. 2071 */ 2072 2073 if (opt_prof_leak && !opt_prof) { 2074 /* 2075 * Enable opt_prof, but in such a way that profiles are never 2076 * automatically dumped. 2077 */ 2078 opt_prof = true; 2079 opt_prof_gdump = false; 2080 } else if (opt_prof) { 2081 if (opt_lg_prof_interval >= 0) { 2082 prof_interval = (((uint64_t)1U) << 2083 opt_lg_prof_interval); 2084 } 2085 } 2086 } 2087 2088 bool 2089 prof_boot2(void) 2090 { 2091 2092 cassert(config_prof); 2093 2094 if (opt_prof) { 2095 tsd_t *tsd; 2096 unsigned i; 2097 2098 lg_prof_sample = opt_lg_prof_sample; 2099 2100 prof_active = opt_prof_active; 2101 if (malloc_mutex_init(&prof_active_mtx)) 2102 return (true); 2103 2104 prof_gdump_val = opt_prof_gdump; 2105 if (malloc_mutex_init(&prof_gdump_mtx)) 2106 return (true); 2107 2108 prof_thread_active_init = opt_prof_thread_active_init; 2109 if (malloc_mutex_init(&prof_thread_active_init_mtx)) 2110 return (true); 2111 2112 tsd = tsd_fetch(); 2113 if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, 2114 prof_bt_keycomp)) 2115 return (true); 2116 if (malloc_mutex_init(&bt2gctx_mtx)) 2117 return (true); 2118 2119 tdata_tree_new(&tdatas); 2120 if (malloc_mutex_init(&tdatas_mtx)) 2121 return (true); 2122 2123 next_thr_uid = 0; 2124 if (malloc_mutex_init(&next_thr_uid_mtx)) 2125 return (true); 2126 2127 if (malloc_mutex_init(&prof_dump_seq_mtx)) 2128 return (true); 2129 if (malloc_mutex_init(&prof_dump_mtx)) 2130 return (true); 2131 2132 if (opt_prof_final && opt_prof_prefix[0] != '\0' && 2133 atexit(prof_fdump) != 0) { 2134 malloc_write("<jemalloc>: Error in atexit()\n"); 2135 if (opt_abort) 2136 abort(); 2137 } 2138 2139 gctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS * 2140 sizeof(malloc_mutex_t)); 2141 if (gctx_locks == NULL) 2142 return (true); 2143 for (i = 0; i < PROF_NCTX_LOCKS; i++) { 2144 if (malloc_mutex_init(&gctx_locks[i])) 2145 return (true); 2146 } 2147 2148 tdata_locks = (malloc_mutex_t *)base_alloc(PROF_NTDATA_LOCKS * 2149 sizeof(malloc_mutex_t)); 2150 if (tdata_locks == NULL) 2151 return (true); 2152 for (i = 0; i < PROF_NTDATA_LOCKS; i++) { 2153 if (malloc_mutex_init(&tdata_locks[i])) 2154 return (true); 2155 } 2156 } 2157 2158 #ifdef JEMALLOC_PROF_LIBGCC 2159 /* 2160 * Cause the backtracing machinery to allocate its internal state 2161 * before enabling profiling. 2162 */ 2163 _Unwind_Backtrace(prof_unwind_init_callback, NULL); 2164 #endif 2165 2166 prof_booted = true; 2167 2168 return (false); 2169 } 2170 2171 void 2172 prof_prefork(void) 2173 { 2174 2175 if (opt_prof) { 2176 unsigned i; 2177 2178 malloc_mutex_prefork(&tdatas_mtx); 2179 malloc_mutex_prefork(&bt2gctx_mtx); 2180 malloc_mutex_prefork(&next_thr_uid_mtx); 2181 malloc_mutex_prefork(&prof_dump_seq_mtx); 2182 for (i = 0; i < PROF_NCTX_LOCKS; i++) 2183 malloc_mutex_prefork(&gctx_locks[i]); 2184 for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2185 malloc_mutex_prefork(&tdata_locks[i]); 2186 } 2187 } 2188 2189 void 2190 prof_postfork_parent(void) 2191 { 2192 2193 if (opt_prof) { 2194 unsigned i; 2195 2196 for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2197 malloc_mutex_postfork_parent(&tdata_locks[i]); 2198 for (i = 0; i < PROF_NCTX_LOCKS; i++) 2199 malloc_mutex_postfork_parent(&gctx_locks[i]); 2200 malloc_mutex_postfork_parent(&prof_dump_seq_mtx); 2201 malloc_mutex_postfork_parent(&next_thr_uid_mtx); 2202 malloc_mutex_postfork_parent(&bt2gctx_mtx); 2203 malloc_mutex_postfork_parent(&tdatas_mtx); 2204 } 2205 } 2206 2207 void 2208 prof_postfork_child(void) 2209 { 2210 2211 if (opt_prof) { 2212 unsigned i; 2213 2214 for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2215 malloc_mutex_postfork_child(&tdata_locks[i]); 2216 for (i = 0; i < PROF_NCTX_LOCKS; i++) 2217 malloc_mutex_postfork_child(&gctx_locks[i]); 2218 malloc_mutex_postfork_child(&prof_dump_seq_mtx); 2219 malloc_mutex_postfork_child(&next_thr_uid_mtx); 2220 malloc_mutex_postfork_child(&bt2gctx_mtx); 2221 malloc_mutex_postfork_child(&tdatas_mtx); 2222 } 2223 } 2224 2225 /******************************************************************************/ 2226