1 #define JEMALLOC_PROF_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 /******************************************************************************/ 4 5 #ifdef JEMALLOC_PROF_LIBUNWIND 6 #define UNW_LOCAL_ONLY 7 #include <libunwind.h> 8 #endif 9 10 #ifdef JEMALLOC_PROF_LIBGCC 11 #include <unwind.h> 12 #endif 13 14 /******************************************************************************/ 15 /* Data. */ 16 17 bool opt_prof = false; 18 bool opt_prof_active = true; 19 bool opt_prof_thread_active_init = true; 20 size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; 21 ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; 22 bool opt_prof_gdump = false; 23 bool opt_prof_final = false; 24 bool opt_prof_leak = false; 25 bool opt_prof_accum = false; 26 char opt_prof_prefix[ 27 /* Minimize memory bloat for non-prof builds. */ 28 #ifdef JEMALLOC_PROF 29 PATH_MAX + 30 #endif 31 1]; 32 33 /* 34 * Initialized as opt_prof_active, and accessed via 35 * prof_active_[gs]et{_unlocked,}(). 36 */ 37 bool prof_active; 38 static malloc_mutex_t prof_active_mtx; 39 40 /* 41 * Initialized as opt_prof_thread_active_init, and accessed via 42 * prof_thread_active_init_[gs]et(). 43 */ 44 static bool prof_thread_active_init; 45 static malloc_mutex_t prof_thread_active_init_mtx; 46 47 /* 48 * Initialized as opt_prof_gdump, and accessed via 49 * prof_gdump_[gs]et{_unlocked,}(). 50 */ 51 bool prof_gdump_val; 52 static malloc_mutex_t prof_gdump_mtx; 53 54 uint64_t prof_interval = 0; 55 56 size_t lg_prof_sample; 57 58 /* 59 * Table of mutexes that are shared among gctx's. These are leaf locks, so 60 * there is no problem with using them for more than one gctx at the same time. 61 * The primary motivation for this sharing though is that gctx's are ephemeral, 62 * and destroying mutexes causes complications for systems that allocate when 63 * creating/destroying mutexes. 64 */ 65 static malloc_mutex_t *gctx_locks; 66 static unsigned cum_gctxs; /* Atomic counter. */ 67 68 /* 69 * Table of mutexes that are shared among tdata's. No operations require 70 * holding multiple tdata locks, so there is no problem with using them for more 71 * than one tdata at the same time, even though a gctx lock may be acquired 72 * while holding a tdata lock. 73 */ 74 static malloc_mutex_t *tdata_locks; 75 76 /* 77 * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data 78 * structure that knows about all backtraces currently captured. 79 */ 80 static ckh_t bt2gctx; 81 static malloc_mutex_t bt2gctx_mtx; 82 83 /* 84 * Tree of all extant prof_tdata_t structures, regardless of state, 85 * {attached,detached,expired}. 86 */ 87 static prof_tdata_tree_t tdatas; 88 static malloc_mutex_t tdatas_mtx; 89 90 static uint64_t next_thr_uid; 91 static malloc_mutex_t next_thr_uid_mtx; 92 93 static malloc_mutex_t prof_dump_seq_mtx; 94 static uint64_t prof_dump_seq; 95 static uint64_t prof_dump_iseq; 96 static uint64_t prof_dump_mseq; 97 static uint64_t prof_dump_useq; 98 99 /* 100 * This buffer is rather large for stack allocation, so use a single buffer for 101 * all profile dumps. 102 */ 103 static malloc_mutex_t prof_dump_mtx; 104 static char prof_dump_buf[ 105 /* Minimize memory bloat for non-prof builds. */ 106 #ifdef JEMALLOC_PROF 107 PROF_DUMP_BUFSIZE 108 #else 109 1 110 #endif 111 ]; 112 static unsigned prof_dump_buf_end; 113 static int prof_dump_fd; 114 115 /* Do not dump any profiles until bootstrapping is complete. */ 116 static bool prof_booted = false; 117 118 /******************************************************************************/ 119 /* 120 * Function prototypes for static functions that are referenced prior to 121 * definition. 122 */ 123 124 static bool prof_tctx_should_destroy(prof_tctx_t *tctx); 125 static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx); 126 static bool prof_tdata_should_destroy(prof_tdata_t *tdata, 127 bool even_if_attached); 128 static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, 129 bool even_if_attached); 130 static char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name); 131 132 /******************************************************************************/ 133 /* Red-black trees. */ 134 135 JEMALLOC_INLINE_C int 136 prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) 137 { 138 uint64_t a_thr_uid = a->thr_uid; 139 uint64_t b_thr_uid = b->thr_uid; 140 int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid); 141 if (ret == 0) { 142 uint64_t a_thr_discrim = a->thr_discrim; 143 uint64_t b_thr_discrim = b->thr_discrim; 144 ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim < 145 b_thr_discrim); 146 if (ret == 0) { 147 uint64_t a_tctx_uid = a->tctx_uid; 148 uint64_t b_tctx_uid = b->tctx_uid; 149 ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid < 150 b_tctx_uid); 151 } 152 } 153 return (ret); 154 } 155 156 rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, 157 tctx_link, prof_tctx_comp) 158 159 JEMALLOC_INLINE_C int 160 prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) 161 { 162 unsigned a_len = a->bt.len; 163 unsigned b_len = b->bt.len; 164 unsigned comp_len = (a_len < b_len) ? a_len : b_len; 165 int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *)); 166 if (ret == 0) 167 ret = (a_len > b_len) - (a_len < b_len); 168 return (ret); 169 } 170 171 rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link, 172 prof_gctx_comp) 173 174 JEMALLOC_INLINE_C int 175 prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) 176 { 177 int ret; 178 uint64_t a_uid = a->thr_uid; 179 uint64_t b_uid = b->thr_uid; 180 181 ret = ((a_uid > b_uid) - (a_uid < b_uid)); 182 if (ret == 0) { 183 uint64_t a_discrim = a->thr_discrim; 184 uint64_t b_discrim = b->thr_discrim; 185 186 ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim)); 187 } 188 return (ret); 189 } 190 191 rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, 192 prof_tdata_comp) 193 194 /******************************************************************************/ 195 196 void 197 prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) 198 { 199 prof_tdata_t *tdata; 200 201 cassert(config_prof); 202 203 if (updated) { 204 /* 205 * Compute a new sample threshold. This isn't very important in 206 * practice, because this function is rarely executed, so the 207 * potential for sample bias is minimal except in contrived 208 * programs. 209 */ 210 tdata = prof_tdata_get(tsd, true); 211 if (tdata != NULL) 212 prof_sample_threshold_update(tdata); 213 } 214 215 if ((uintptr_t)tctx > (uintptr_t)1U) { 216 malloc_mutex_lock(tctx->tdata->lock); 217 tctx->prepared = false; 218 if (prof_tctx_should_destroy(tctx)) 219 prof_tctx_destroy(tsd, tctx); 220 else 221 malloc_mutex_unlock(tctx->tdata->lock); 222 } 223 } 224 225 void 226 prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) 227 { 228 229 prof_tctx_set(ptr, usize, tctx); 230 231 malloc_mutex_lock(tctx->tdata->lock); 232 tctx->cnts.curobjs++; 233 tctx->cnts.curbytes += usize; 234 if (opt_prof_accum) { 235 tctx->cnts.accumobjs++; 236 tctx->cnts.accumbytes += usize; 237 } 238 tctx->prepared = false; 239 malloc_mutex_unlock(tctx->tdata->lock); 240 } 241 242 void 243 prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) 244 { 245 246 malloc_mutex_lock(tctx->tdata->lock); 247 assert(tctx->cnts.curobjs > 0); 248 assert(tctx->cnts.curbytes >= usize); 249 tctx->cnts.curobjs--; 250 tctx->cnts.curbytes -= usize; 251 252 if (prof_tctx_should_destroy(tctx)) 253 prof_tctx_destroy(tsd, tctx); 254 else 255 malloc_mutex_unlock(tctx->tdata->lock); 256 } 257 258 void 259 bt_init(prof_bt_t *bt, void **vec) 260 { 261 262 cassert(config_prof); 263 264 bt->vec = vec; 265 bt->len = 0; 266 } 267 268 JEMALLOC_INLINE_C void 269 prof_enter(tsd_t *tsd, prof_tdata_t *tdata) 270 { 271 272 cassert(config_prof); 273 assert(tdata == prof_tdata_get(tsd, false)); 274 275 if (tdata != NULL) { 276 assert(!tdata->enq); 277 tdata->enq = true; 278 } 279 280 malloc_mutex_lock(&bt2gctx_mtx); 281 } 282 283 JEMALLOC_INLINE_C void 284 prof_leave(tsd_t *tsd, prof_tdata_t *tdata) 285 { 286 287 cassert(config_prof); 288 assert(tdata == prof_tdata_get(tsd, false)); 289 290 malloc_mutex_unlock(&bt2gctx_mtx); 291 292 if (tdata != NULL) { 293 bool idump, gdump; 294 295 assert(tdata->enq); 296 tdata->enq = false; 297 idump = tdata->enq_idump; 298 tdata->enq_idump = false; 299 gdump = tdata->enq_gdump; 300 tdata->enq_gdump = false; 301 302 if (idump) 303 prof_idump(); 304 if (gdump) 305 prof_gdump(); 306 } 307 } 308 309 #ifdef JEMALLOC_PROF_LIBUNWIND 310 void 311 prof_backtrace(prof_bt_t *bt) 312 { 313 int nframes; 314 315 cassert(config_prof); 316 assert(bt->len == 0); 317 assert(bt->vec != NULL); 318 319 nframes = unw_backtrace(bt->vec, PROF_BT_MAX); 320 if (nframes <= 0) 321 return; 322 bt->len = nframes; 323 } 324 #elif (defined(JEMALLOC_PROF_LIBGCC)) 325 static _Unwind_Reason_Code 326 prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) 327 { 328 329 cassert(config_prof); 330 331 return (_URC_NO_REASON); 332 } 333 334 static _Unwind_Reason_Code 335 prof_unwind_callback(struct _Unwind_Context *context, void *arg) 336 { 337 prof_unwind_data_t *data = (prof_unwind_data_t *)arg; 338 void *ip; 339 340 cassert(config_prof); 341 342 ip = (void *)_Unwind_GetIP(context); 343 if (ip == NULL) 344 return (_URC_END_OF_STACK); 345 data->bt->vec[data->bt->len] = ip; 346 data->bt->len++; 347 if (data->bt->len == data->max) 348 return (_URC_END_OF_STACK); 349 350 return (_URC_NO_REASON); 351 } 352 353 void 354 prof_backtrace(prof_bt_t *bt) 355 { 356 prof_unwind_data_t data = {bt, PROF_BT_MAX}; 357 358 cassert(config_prof); 359 360 _Unwind_Backtrace(prof_unwind_callback, &data); 361 } 362 #elif (defined(JEMALLOC_PROF_GCC)) 363 void 364 prof_backtrace(prof_bt_t *bt) 365 { 366 #define BT_FRAME(i) \ 367 if ((i) < PROF_BT_MAX) { \ 368 void *p; \ 369 if (__builtin_frame_address(i) == 0) \ 370 return; \ 371 p = __builtin_return_address(i); \ 372 if (p == NULL) \ 373 return; \ 374 bt->vec[(i)] = p; \ 375 bt->len = (i) + 1; \ 376 } else \ 377 return; 378 379 cassert(config_prof); 380 381 BT_FRAME(0) 382 BT_FRAME(1) 383 BT_FRAME(2) 384 BT_FRAME(3) 385 BT_FRAME(4) 386 BT_FRAME(5) 387 BT_FRAME(6) 388 BT_FRAME(7) 389 BT_FRAME(8) 390 BT_FRAME(9) 391 392 BT_FRAME(10) 393 BT_FRAME(11) 394 BT_FRAME(12) 395 BT_FRAME(13) 396 BT_FRAME(14) 397 BT_FRAME(15) 398 BT_FRAME(16) 399 BT_FRAME(17) 400 BT_FRAME(18) 401 BT_FRAME(19) 402 403 BT_FRAME(20) 404 BT_FRAME(21) 405 BT_FRAME(22) 406 BT_FRAME(23) 407 BT_FRAME(24) 408 BT_FRAME(25) 409 BT_FRAME(26) 410 BT_FRAME(27) 411 BT_FRAME(28) 412 BT_FRAME(29) 413 414 BT_FRAME(30) 415 BT_FRAME(31) 416 BT_FRAME(32) 417 BT_FRAME(33) 418 BT_FRAME(34) 419 BT_FRAME(35) 420 BT_FRAME(36) 421 BT_FRAME(37) 422 BT_FRAME(38) 423 BT_FRAME(39) 424 425 BT_FRAME(40) 426 BT_FRAME(41) 427 BT_FRAME(42) 428 BT_FRAME(43) 429 BT_FRAME(44) 430 BT_FRAME(45) 431 BT_FRAME(46) 432 BT_FRAME(47) 433 BT_FRAME(48) 434 BT_FRAME(49) 435 436 BT_FRAME(50) 437 BT_FRAME(51) 438 BT_FRAME(52) 439 BT_FRAME(53) 440 BT_FRAME(54) 441 BT_FRAME(55) 442 BT_FRAME(56) 443 BT_FRAME(57) 444 BT_FRAME(58) 445 BT_FRAME(59) 446 447 BT_FRAME(60) 448 BT_FRAME(61) 449 BT_FRAME(62) 450 BT_FRAME(63) 451 BT_FRAME(64) 452 BT_FRAME(65) 453 BT_FRAME(66) 454 BT_FRAME(67) 455 BT_FRAME(68) 456 BT_FRAME(69) 457 458 BT_FRAME(70) 459 BT_FRAME(71) 460 BT_FRAME(72) 461 BT_FRAME(73) 462 BT_FRAME(74) 463 BT_FRAME(75) 464 BT_FRAME(76) 465 BT_FRAME(77) 466 BT_FRAME(78) 467 BT_FRAME(79) 468 469 BT_FRAME(80) 470 BT_FRAME(81) 471 BT_FRAME(82) 472 BT_FRAME(83) 473 BT_FRAME(84) 474 BT_FRAME(85) 475 BT_FRAME(86) 476 BT_FRAME(87) 477 BT_FRAME(88) 478 BT_FRAME(89) 479 480 BT_FRAME(90) 481 BT_FRAME(91) 482 BT_FRAME(92) 483 BT_FRAME(93) 484 BT_FRAME(94) 485 BT_FRAME(95) 486 BT_FRAME(96) 487 BT_FRAME(97) 488 BT_FRAME(98) 489 BT_FRAME(99) 490 491 BT_FRAME(100) 492 BT_FRAME(101) 493 BT_FRAME(102) 494 BT_FRAME(103) 495 BT_FRAME(104) 496 BT_FRAME(105) 497 BT_FRAME(106) 498 BT_FRAME(107) 499 BT_FRAME(108) 500 BT_FRAME(109) 501 502 BT_FRAME(110) 503 BT_FRAME(111) 504 BT_FRAME(112) 505 BT_FRAME(113) 506 BT_FRAME(114) 507 BT_FRAME(115) 508 BT_FRAME(116) 509 BT_FRAME(117) 510 BT_FRAME(118) 511 BT_FRAME(119) 512 513 BT_FRAME(120) 514 BT_FRAME(121) 515 BT_FRAME(122) 516 BT_FRAME(123) 517 BT_FRAME(124) 518 BT_FRAME(125) 519 BT_FRAME(126) 520 BT_FRAME(127) 521 #undef BT_FRAME 522 } 523 #else 524 void 525 prof_backtrace(prof_bt_t *bt) 526 { 527 528 cassert(config_prof); 529 not_reached(); 530 } 531 #endif 532 533 static malloc_mutex_t * 534 prof_gctx_mutex_choose(void) 535 { 536 unsigned ngctxs = atomic_add_u(&cum_gctxs, 1); 537 538 return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]); 539 } 540 541 static malloc_mutex_t * 542 prof_tdata_mutex_choose(uint64_t thr_uid) 543 { 544 545 return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]); 546 } 547 548 static prof_gctx_t * 549 prof_gctx_create(tsd_t *tsd, prof_bt_t *bt) 550 { 551 /* 552 * Create a single allocation that has space for vec of length bt->len. 553 */ 554 prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsd, offsetof(prof_gctx_t, 555 vec) + (bt->len * sizeof(void *)), false, tcache_get(tsd, true), 556 true, NULL); 557 if (gctx == NULL) 558 return (NULL); 559 gctx->lock = prof_gctx_mutex_choose(); 560 /* 561 * Set nlimbo to 1, in order to avoid a race condition with 562 * prof_tctx_destroy()/prof_gctx_try_destroy(). 563 */ 564 gctx->nlimbo = 1; 565 tctx_tree_new(&gctx->tctxs); 566 /* Duplicate bt. */ 567 memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *)); 568 gctx->bt.vec = gctx->vec; 569 gctx->bt.len = bt->len; 570 return (gctx); 571 } 572 573 static void 574 prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx, 575 prof_tdata_t *tdata) 576 { 577 578 cassert(config_prof); 579 580 /* 581 * Check that gctx is still unused by any thread cache before destroying 582 * it. prof_lookup() increments gctx->nlimbo in order to avoid a race 583 * condition with this function, as does prof_tctx_destroy() in order to 584 * avoid a race between the main body of prof_tctx_destroy() and entry 585 * into this function. 586 */ 587 prof_enter(tsd, tdata_self); 588 malloc_mutex_lock(gctx->lock); 589 assert(gctx->nlimbo != 0); 590 if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { 591 /* Remove gctx from bt2gctx. */ 592 if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) 593 not_reached(); 594 prof_leave(tsd, tdata_self); 595 /* Destroy gctx. */ 596 malloc_mutex_unlock(gctx->lock); 597 idalloctm(tsd, gctx, tcache_get(tsd, false), true); 598 } else { 599 /* 600 * Compensate for increment in prof_tctx_destroy() or 601 * prof_lookup(). 602 */ 603 gctx->nlimbo--; 604 malloc_mutex_unlock(gctx->lock); 605 prof_leave(tsd, tdata_self); 606 } 607 } 608 609 /* tctx->tdata->lock must be held. */ 610 static bool 611 prof_tctx_should_destroy(prof_tctx_t *tctx) 612 { 613 614 if (opt_prof_accum) 615 return (false); 616 if (tctx->cnts.curobjs != 0) 617 return (false); 618 if (tctx->prepared) 619 return (false); 620 return (true); 621 } 622 623 static bool 624 prof_gctx_should_destroy(prof_gctx_t *gctx) 625 { 626 627 if (opt_prof_accum) 628 return (false); 629 if (!tctx_tree_empty(&gctx->tctxs)) 630 return (false); 631 if (gctx->nlimbo != 0) 632 return (false); 633 return (true); 634 } 635 636 /* tctx->tdata->lock is held upon entry, and released before return. */ 637 static void 638 prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) 639 { 640 prof_tdata_t *tdata = tctx->tdata; 641 prof_gctx_t *gctx = tctx->gctx; 642 bool destroy_tdata, destroy_tctx, destroy_gctx; 643 644 assert(tctx->cnts.curobjs == 0); 645 assert(tctx->cnts.curbytes == 0); 646 assert(!opt_prof_accum); 647 assert(tctx->cnts.accumobjs == 0); 648 assert(tctx->cnts.accumbytes == 0); 649 650 ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL); 651 destroy_tdata = prof_tdata_should_destroy(tdata, false); 652 malloc_mutex_unlock(tdata->lock); 653 654 malloc_mutex_lock(gctx->lock); 655 switch (tctx->state) { 656 case prof_tctx_state_nominal: 657 tctx_tree_remove(&gctx->tctxs, tctx); 658 destroy_tctx = true; 659 if (prof_gctx_should_destroy(gctx)) { 660 /* 661 * Increment gctx->nlimbo in order to keep another 662 * thread from winning the race to destroy gctx while 663 * this one has gctx->lock dropped. Without this, it 664 * would be possible for another thread to: 665 * 666 * 1) Sample an allocation associated with gctx. 667 * 2) Deallocate the sampled object. 668 * 3) Successfully prof_gctx_try_destroy(gctx). 669 * 670 * The result would be that gctx no longer exists by the 671 * time this thread accesses it in 672 * prof_gctx_try_destroy(). 673 */ 674 gctx->nlimbo++; 675 destroy_gctx = true; 676 } else 677 destroy_gctx = false; 678 break; 679 case prof_tctx_state_dumping: 680 /* 681 * A dumping thread needs tctx to remain valid until dumping 682 * has finished. Change state such that the dumping thread will 683 * complete destruction during a late dump iteration phase. 684 */ 685 tctx->state = prof_tctx_state_purgatory; 686 destroy_tctx = false; 687 destroy_gctx = false; 688 break; 689 default: 690 not_reached(); 691 destroy_tctx = false; 692 destroy_gctx = false; 693 } 694 malloc_mutex_unlock(gctx->lock); 695 if (destroy_gctx) { 696 prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx, 697 tdata); 698 } 699 700 if (destroy_tdata) 701 prof_tdata_destroy(tsd, tdata, false); 702 703 if (destroy_tctx) 704 idalloctm(tsd, tctx, tcache_get(tsd, false), true); 705 } 706 707 static bool 708 prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, 709 void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) 710 { 711 union { 712 prof_gctx_t *p; 713 void *v; 714 } gctx; 715 union { 716 prof_bt_t *p; 717 void *v; 718 } btkey; 719 bool new_gctx; 720 721 prof_enter(tsd, tdata); 722 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) { 723 /* bt has never been seen before. Insert it. */ 724 gctx.p = prof_gctx_create(tsd, bt); 725 if (gctx.v == NULL) { 726 prof_leave(tsd, tdata); 727 return (true); 728 } 729 btkey.p = &gctx.p->bt; 730 if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) { 731 /* OOM. */ 732 prof_leave(tsd, tdata); 733 idalloctm(tsd, gctx.v, tcache_get(tsd, false), true); 734 return (true); 735 } 736 new_gctx = true; 737 } else { 738 /* 739 * Increment nlimbo, in order to avoid a race condition with 740 * prof_tctx_destroy()/prof_gctx_try_destroy(). 741 */ 742 malloc_mutex_lock(gctx.p->lock); 743 gctx.p->nlimbo++; 744 malloc_mutex_unlock(gctx.p->lock); 745 new_gctx = false; 746 } 747 prof_leave(tsd, tdata); 748 749 *p_btkey = btkey.v; 750 *p_gctx = gctx.p; 751 *p_new_gctx = new_gctx; 752 return (false); 753 } 754 755 prof_tctx_t * 756 prof_lookup(tsd_t *tsd, prof_bt_t *bt) 757 { 758 union { 759 prof_tctx_t *p; 760 void *v; 761 } ret; 762 prof_tdata_t *tdata; 763 bool not_found; 764 765 cassert(config_prof); 766 767 tdata = prof_tdata_get(tsd, false); 768 if (tdata == NULL) 769 return (NULL); 770 771 malloc_mutex_lock(tdata->lock); 772 not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v); 773 if (!not_found) /* Note double negative! */ 774 ret.p->prepared = true; 775 malloc_mutex_unlock(tdata->lock); 776 if (not_found) { 777 tcache_t *tcache; 778 void *btkey; 779 prof_gctx_t *gctx; 780 bool new_gctx, error; 781 782 /* 783 * This thread's cache lacks bt. Look for it in the global 784 * cache. 785 */ 786 if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx, 787 &new_gctx)) 788 return (NULL); 789 790 /* Link a prof_tctx_t into gctx for this thread. */ 791 tcache = tcache_get(tsd, true); 792 ret.v = iallocztm(tsd, sizeof(prof_tctx_t), false, tcache, true, 793 NULL); 794 if (ret.p == NULL) { 795 if (new_gctx) 796 prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 797 return (NULL); 798 } 799 ret.p->tdata = tdata; 800 ret.p->thr_uid = tdata->thr_uid; 801 ret.p->thr_discrim = tdata->thr_discrim; 802 memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); 803 ret.p->gctx = gctx; 804 ret.p->tctx_uid = tdata->tctx_uid_next++; 805 ret.p->prepared = true; 806 ret.p->state = prof_tctx_state_initializing; 807 malloc_mutex_lock(tdata->lock); 808 error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v); 809 malloc_mutex_unlock(tdata->lock); 810 if (error) { 811 if (new_gctx) 812 prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 813 idalloctm(tsd, ret.v, tcache, true); 814 return (NULL); 815 } 816 malloc_mutex_lock(gctx->lock); 817 ret.p->state = prof_tctx_state_nominal; 818 tctx_tree_insert(&gctx->tctxs, ret.p); 819 gctx->nlimbo--; 820 malloc_mutex_unlock(gctx->lock); 821 } 822 823 return (ret.p); 824 } 825 826 void 827 prof_sample_threshold_update(prof_tdata_t *tdata) 828 { 829 /* 830 * The body of this function is compiled out unless heap profiling is 831 * enabled, so that it is possible to compile jemalloc with floating 832 * point support completely disabled. Avoiding floating point code is 833 * important on memory-constrained systems, but it also enables a 834 * workaround for versions of glibc that don't properly save/restore 835 * floating point registers during dynamic lazy symbol loading (which 836 * internally calls into whatever malloc implementation happens to be 837 * integrated into the application). Note that some compilers (e.g. 838 * gcc 4.8) may use floating point registers for fast memory moves, so 839 * jemalloc must be compiled with such optimizations disabled (e.g. 840 * -mno-sse) in order for the workaround to be complete. 841 */ 842 #ifdef JEMALLOC_PROF 843 uint64_t r; 844 double u; 845 846 if (!config_prof) 847 return; 848 849 if (lg_prof_sample == 0) { 850 tdata->bytes_until_sample = 0; 851 return; 852 } 853 854 /* 855 * Compute sample interval as a geometrically distributed random 856 * variable with mean (2^lg_prof_sample). 857 * 858 * __ __ 859 * | log(u) | 1 860 * tdata->bytes_until_sample = | -------- |, where p = --------------- 861 * | log(1-p) | lg_prof_sample 862 * 2 863 * 864 * For more information on the math, see: 865 * 866 * Non-Uniform Random Variate Generation 867 * Luc Devroye 868 * Springer-Verlag, New York, 1986 869 * pp 500 870 * (http://luc.devroye.org/rnbookindex.html) 871 */ 872 prng64(r, 53, tdata->prng_state, UINT64_C(6364136223846793005), 873 UINT64_C(1442695040888963407)); 874 u = (double)r * (1.0/9007199254740992.0L); 875 tdata->bytes_until_sample = (uint64_t)(log(u) / 876 log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) 877 + (uint64_t)1U; 878 #endif 879 } 880 881 #ifdef JEMALLOC_JET 882 static prof_tdata_t * 883 prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 884 { 885 size_t *tdata_count = (size_t *)arg; 886 887 (*tdata_count)++; 888 889 return (NULL); 890 } 891 892 size_t 893 prof_tdata_count(void) 894 { 895 size_t tdata_count = 0; 896 897 malloc_mutex_lock(&tdatas_mtx); 898 tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter, 899 (void *)&tdata_count); 900 malloc_mutex_unlock(&tdatas_mtx); 901 902 return (tdata_count); 903 } 904 #endif 905 906 #ifdef JEMALLOC_JET 907 size_t 908 prof_bt_count(void) 909 { 910 size_t bt_count; 911 tsd_t *tsd; 912 prof_tdata_t *tdata; 913 914 tsd = tsd_fetch(); 915 tdata = prof_tdata_get(tsd, false); 916 if (tdata == NULL) 917 return (0); 918 919 malloc_mutex_lock(&bt2gctx_mtx); 920 bt_count = ckh_count(&bt2gctx); 921 malloc_mutex_unlock(&bt2gctx_mtx); 922 923 return (bt_count); 924 } 925 #endif 926 927 #ifdef JEMALLOC_JET 928 #undef prof_dump_open 929 #define prof_dump_open JEMALLOC_N(prof_dump_open_impl) 930 #endif 931 static int 932 prof_dump_open(bool propagate_err, const char *filename) 933 { 934 int fd; 935 936 fd = creat(filename, 0644); 937 if (fd == -1 && !propagate_err) { 938 malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n", 939 filename); 940 if (opt_abort) 941 abort(); 942 } 943 944 return (fd); 945 } 946 #ifdef JEMALLOC_JET 947 #undef prof_dump_open 948 #define prof_dump_open JEMALLOC_N(prof_dump_open) 949 prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl); 950 #endif 951 952 static bool 953 prof_dump_flush(bool propagate_err) 954 { 955 bool ret = false; 956 ssize_t err; 957 958 cassert(config_prof); 959 960 err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); 961 if (err == -1) { 962 if (!propagate_err) { 963 malloc_write("<jemalloc>: write() failed during heap " 964 "profile flush\n"); 965 if (opt_abort) 966 abort(); 967 } 968 ret = true; 969 } 970 prof_dump_buf_end = 0; 971 972 return (ret); 973 } 974 975 static bool 976 prof_dump_close(bool propagate_err) 977 { 978 bool ret; 979 980 assert(prof_dump_fd != -1); 981 ret = prof_dump_flush(propagate_err); 982 close(prof_dump_fd); 983 prof_dump_fd = -1; 984 985 return (ret); 986 } 987 988 static bool 989 prof_dump_write(bool propagate_err, const char *s) 990 { 991 unsigned i, slen, n; 992 993 cassert(config_prof); 994 995 i = 0; 996 slen = strlen(s); 997 while (i < slen) { 998 /* Flush the buffer if it is full. */ 999 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) 1000 if (prof_dump_flush(propagate_err) && propagate_err) 1001 return (true); 1002 1003 if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { 1004 /* Finish writing. */ 1005 n = slen - i; 1006 } else { 1007 /* Write as much of s as will fit. */ 1008 n = PROF_DUMP_BUFSIZE - prof_dump_buf_end; 1009 } 1010 memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); 1011 prof_dump_buf_end += n; 1012 i += n; 1013 } 1014 1015 return (false); 1016 } 1017 1018 JEMALLOC_FORMAT_PRINTF(2, 3) 1019 static bool 1020 prof_dump_printf(bool propagate_err, const char *format, ...) 1021 { 1022 bool ret; 1023 va_list ap; 1024 char buf[PROF_PRINTF_BUFSIZE]; 1025 1026 va_start(ap, format); 1027 malloc_vsnprintf(buf, sizeof(buf), format, ap); 1028 va_end(ap); 1029 ret = prof_dump_write(propagate_err, buf); 1030 1031 return (ret); 1032 } 1033 1034 /* tctx->tdata->lock is held. */ 1035 static void 1036 prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata) 1037 { 1038 1039 malloc_mutex_lock(tctx->gctx->lock); 1040 1041 switch (tctx->state) { 1042 case prof_tctx_state_initializing: 1043 malloc_mutex_unlock(tctx->gctx->lock); 1044 return; 1045 case prof_tctx_state_nominal: 1046 tctx->state = prof_tctx_state_dumping; 1047 malloc_mutex_unlock(tctx->gctx->lock); 1048 1049 memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); 1050 1051 tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; 1052 tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes; 1053 if (opt_prof_accum) { 1054 tdata->cnt_summed.accumobjs += 1055 tctx->dump_cnts.accumobjs; 1056 tdata->cnt_summed.accumbytes += 1057 tctx->dump_cnts.accumbytes; 1058 } 1059 break; 1060 case prof_tctx_state_dumping: 1061 case prof_tctx_state_purgatory: 1062 not_reached(); 1063 } 1064 } 1065 1066 /* gctx->lock is held. */ 1067 static void 1068 prof_tctx_merge_gctx(prof_tctx_t *tctx, prof_gctx_t *gctx) 1069 { 1070 1071 gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs; 1072 gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes; 1073 if (opt_prof_accum) { 1074 gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; 1075 gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes; 1076 } 1077 } 1078 1079 /* tctx->gctx is held. */ 1080 static prof_tctx_t * 1081 prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) 1082 { 1083 1084 switch (tctx->state) { 1085 case prof_tctx_state_nominal: 1086 /* New since dumping started; ignore. */ 1087 break; 1088 case prof_tctx_state_dumping: 1089 case prof_tctx_state_purgatory: 1090 prof_tctx_merge_gctx(tctx, tctx->gctx); 1091 break; 1092 default: 1093 not_reached(); 1094 } 1095 1096 return (NULL); 1097 } 1098 1099 /* gctx->lock is held. */ 1100 static prof_tctx_t * 1101 prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) 1102 { 1103 bool propagate_err = *(bool *)arg; 1104 1105 switch (tctx->state) { 1106 case prof_tctx_state_initializing: 1107 case prof_tctx_state_nominal: 1108 /* Not captured by this dump. */ 1109 break; 1110 case prof_tctx_state_dumping: 1111 case prof_tctx_state_purgatory: 1112 if (prof_dump_printf(propagate_err, 1113 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": " 1114 "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs, 1115 tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs, 1116 tctx->dump_cnts.accumbytes)) 1117 return (tctx); 1118 break; 1119 default: 1120 not_reached(); 1121 } 1122 return (NULL); 1123 } 1124 1125 /* tctx->gctx is held. */ 1126 static prof_tctx_t * 1127 prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) 1128 { 1129 prof_tctx_t *ret; 1130 1131 switch (tctx->state) { 1132 case prof_tctx_state_nominal: 1133 /* New since dumping started; ignore. */ 1134 break; 1135 case prof_tctx_state_dumping: 1136 tctx->state = prof_tctx_state_nominal; 1137 break; 1138 case prof_tctx_state_purgatory: 1139 ret = tctx; 1140 goto label_return; 1141 default: 1142 not_reached(); 1143 } 1144 1145 ret = NULL; 1146 label_return: 1147 return (ret); 1148 } 1149 1150 static void 1151 prof_dump_gctx_prep(prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) 1152 { 1153 1154 cassert(config_prof); 1155 1156 malloc_mutex_lock(gctx->lock); 1157 1158 /* 1159 * Increment nlimbo so that gctx won't go away before dump. 1160 * Additionally, link gctx into the dump list so that it is included in 1161 * prof_dump()'s second pass. 1162 */ 1163 gctx->nlimbo++; 1164 gctx_tree_insert(gctxs, gctx); 1165 1166 memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t)); 1167 1168 malloc_mutex_unlock(gctx->lock); 1169 } 1170 1171 static prof_gctx_t * 1172 prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) 1173 { 1174 size_t *leak_ngctx = (size_t *)arg; 1175 1176 malloc_mutex_lock(gctx->lock); 1177 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, NULL); 1178 if (gctx->cnt_summed.curobjs != 0) 1179 (*leak_ngctx)++; 1180 malloc_mutex_unlock(gctx->lock); 1181 1182 return (NULL); 1183 } 1184 1185 static void 1186 prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) 1187 { 1188 prof_tdata_t *tdata = prof_tdata_get(tsd, false); 1189 prof_gctx_t *gctx; 1190 1191 /* 1192 * Standard tree iteration won't work here, because as soon as we 1193 * decrement gctx->nlimbo and unlock gctx, another thread can 1194 * concurrently destroy it, which will corrupt the tree. Therefore, 1195 * tear down the tree one node at a time during iteration. 1196 */ 1197 while ((gctx = gctx_tree_first(gctxs)) != NULL) { 1198 gctx_tree_remove(gctxs, gctx); 1199 malloc_mutex_lock(gctx->lock); 1200 { 1201 prof_tctx_t *next; 1202 1203 next = NULL; 1204 do { 1205 prof_tctx_t *to_destroy = 1206 tctx_tree_iter(&gctx->tctxs, next, 1207 prof_tctx_finish_iter, NULL); 1208 if (to_destroy != NULL) { 1209 next = tctx_tree_next(&gctx->tctxs, 1210 to_destroy); 1211 tctx_tree_remove(&gctx->tctxs, 1212 to_destroy); 1213 idalloctm(tsd, to_destroy, 1214 tcache_get(tsd, false), true); 1215 } else 1216 next = NULL; 1217 } while (next != NULL); 1218 } 1219 gctx->nlimbo--; 1220 if (prof_gctx_should_destroy(gctx)) { 1221 gctx->nlimbo++; 1222 malloc_mutex_unlock(gctx->lock); 1223 prof_gctx_try_destroy(tsd, tdata, gctx, tdata); 1224 } else 1225 malloc_mutex_unlock(gctx->lock); 1226 } 1227 } 1228 1229 static prof_tdata_t * 1230 prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 1231 { 1232 prof_cnt_t *cnt_all = (prof_cnt_t *)arg; 1233 1234 malloc_mutex_lock(tdata->lock); 1235 if (!tdata->expired) { 1236 size_t tabind; 1237 union { 1238 prof_tctx_t *p; 1239 void *v; 1240 } tctx; 1241 1242 tdata->dumping = true; 1243 memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t)); 1244 for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL, 1245 &tctx.v);) 1246 prof_tctx_merge_tdata(tctx.p, tdata); 1247 1248 cnt_all->curobjs += tdata->cnt_summed.curobjs; 1249 cnt_all->curbytes += tdata->cnt_summed.curbytes; 1250 if (opt_prof_accum) { 1251 cnt_all->accumobjs += tdata->cnt_summed.accumobjs; 1252 cnt_all->accumbytes += tdata->cnt_summed.accumbytes; 1253 } 1254 } else 1255 tdata->dumping = false; 1256 malloc_mutex_unlock(tdata->lock); 1257 1258 return (NULL); 1259 } 1260 1261 static prof_tdata_t * 1262 prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 1263 { 1264 bool propagate_err = *(bool *)arg; 1265 1266 if (!tdata->dumping) 1267 return (NULL); 1268 1269 if (prof_dump_printf(propagate_err, 1270 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n", 1271 tdata->thr_uid, tdata->cnt_summed.curobjs, 1272 tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs, 1273 tdata->cnt_summed.accumbytes, 1274 (tdata->thread_name != NULL) ? " " : "", 1275 (tdata->thread_name != NULL) ? tdata->thread_name : "")) 1276 return (tdata); 1277 return (NULL); 1278 } 1279 1280 #ifdef JEMALLOC_JET 1281 #undef prof_dump_header 1282 #define prof_dump_header JEMALLOC_N(prof_dump_header_impl) 1283 #endif 1284 static bool 1285 prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all) 1286 { 1287 bool ret; 1288 1289 if (prof_dump_printf(propagate_err, 1290 "heap_v2/%"FMTu64"\n" 1291 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", 1292 ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs, 1293 cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) 1294 return (true); 1295 1296 malloc_mutex_lock(&tdatas_mtx); 1297 ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, 1298 (void *)&propagate_err) != NULL); 1299 malloc_mutex_unlock(&tdatas_mtx); 1300 return (ret); 1301 } 1302 #ifdef JEMALLOC_JET 1303 #undef prof_dump_header 1304 #define prof_dump_header JEMALLOC_N(prof_dump_header) 1305 prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl); 1306 #endif 1307 1308 /* gctx->lock is held. */ 1309 static bool 1310 prof_dump_gctx(bool propagate_err, prof_gctx_t *gctx, const prof_bt_t *bt, 1311 prof_gctx_tree_t *gctxs) 1312 { 1313 bool ret; 1314 unsigned i; 1315 1316 cassert(config_prof); 1317 1318 /* Avoid dumping such gctx's that have no useful data. */ 1319 if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) || 1320 (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) { 1321 assert(gctx->cnt_summed.curobjs == 0); 1322 assert(gctx->cnt_summed.curbytes == 0); 1323 assert(gctx->cnt_summed.accumobjs == 0); 1324 assert(gctx->cnt_summed.accumbytes == 0); 1325 ret = false; 1326 goto label_return; 1327 } 1328 1329 if (prof_dump_printf(propagate_err, "@")) { 1330 ret = true; 1331 goto label_return; 1332 } 1333 for (i = 0; i < bt->len; i++) { 1334 if (prof_dump_printf(propagate_err, " %#"FMTxPTR, 1335 (uintptr_t)bt->vec[i])) { 1336 ret = true; 1337 goto label_return; 1338 } 1339 } 1340 1341 if (prof_dump_printf(propagate_err, 1342 "\n" 1343 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", 1344 gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes, 1345 gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) { 1346 ret = true; 1347 goto label_return; 1348 } 1349 1350 if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, 1351 (void *)&propagate_err) != NULL) { 1352 ret = true; 1353 goto label_return; 1354 } 1355 1356 ret = false; 1357 label_return: 1358 return (ret); 1359 } 1360 1361 JEMALLOC_FORMAT_PRINTF(1, 2) 1362 static int 1363 prof_open_maps(const char *format, ...) 1364 { 1365 int mfd; 1366 va_list ap; 1367 char filename[PATH_MAX + 1]; 1368 1369 va_start(ap, format); 1370 malloc_vsnprintf(filename, sizeof(filename), format, ap); 1371 va_end(ap); 1372 mfd = open(filename, O_RDONLY); 1373 1374 return (mfd); 1375 } 1376 1377 static bool 1378 prof_dump_maps(bool propagate_err) 1379 { 1380 bool ret; 1381 int mfd; 1382 1383 cassert(config_prof); 1384 #ifdef __FreeBSD__ 1385 mfd = prof_open_maps("/proc/curproc/map"); 1386 #else 1387 { 1388 int pid = getpid(); 1389 1390 mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid); 1391 if (mfd == -1) 1392 mfd = prof_open_maps("/proc/%d/maps", pid); 1393 } 1394 #endif 1395 if (mfd != -1) { 1396 ssize_t nread; 1397 1398 if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && 1399 propagate_err) { 1400 ret = true; 1401 goto label_return; 1402 } 1403 nread = 0; 1404 do { 1405 prof_dump_buf_end += nread; 1406 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) { 1407 /* Make space in prof_dump_buf before read(). */ 1408 if (prof_dump_flush(propagate_err) && 1409 propagate_err) { 1410 ret = true; 1411 goto label_return; 1412 } 1413 } 1414 nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], 1415 PROF_DUMP_BUFSIZE - prof_dump_buf_end); 1416 } while (nread > 0); 1417 } else { 1418 ret = true; 1419 goto label_return; 1420 } 1421 1422 ret = false; 1423 label_return: 1424 if (mfd != -1) 1425 close(mfd); 1426 return (ret); 1427 } 1428 1429 static void 1430 prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx, 1431 const char *filename) 1432 { 1433 1434 if (cnt_all->curbytes != 0) { 1435 malloc_printf("<jemalloc>: Leak summary: %"FMTu64" byte%s, %" 1436 FMTu64" object%s, %zu context%s\n", 1437 cnt_all->curbytes, (cnt_all->curbytes != 1) ? "s" : "", 1438 cnt_all->curobjs, (cnt_all->curobjs != 1) ? "s" : "", 1439 leak_ngctx, (leak_ngctx != 1) ? "s" : ""); 1440 malloc_printf( 1441 "<jemalloc>: Run jeprof on \"%s\" for leak detail\n", 1442 filename); 1443 } 1444 } 1445 1446 static prof_gctx_t * 1447 prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) 1448 { 1449 prof_gctx_t *ret; 1450 bool propagate_err = *(bool *)arg; 1451 1452 malloc_mutex_lock(gctx->lock); 1453 1454 if (prof_dump_gctx(propagate_err, gctx, &gctx->bt, gctxs)) { 1455 ret = gctx; 1456 goto label_return; 1457 } 1458 1459 ret = NULL; 1460 label_return: 1461 malloc_mutex_unlock(gctx->lock); 1462 return (ret); 1463 } 1464 1465 static bool 1466 prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) 1467 { 1468 prof_tdata_t *tdata; 1469 prof_cnt_t cnt_all; 1470 size_t tabind; 1471 union { 1472 prof_gctx_t *p; 1473 void *v; 1474 } gctx; 1475 size_t leak_ngctx; 1476 prof_gctx_tree_t gctxs; 1477 1478 cassert(config_prof); 1479 1480 tdata = prof_tdata_get(tsd, true); 1481 if (tdata == NULL) 1482 return (true); 1483 1484 malloc_mutex_lock(&prof_dump_mtx); 1485 prof_enter(tsd, tdata); 1486 1487 /* 1488 * Put gctx's in limbo and clear their counters in preparation for 1489 * summing. 1490 */ 1491 gctx_tree_new(&gctxs); 1492 for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) 1493 prof_dump_gctx_prep(gctx.p, &gctxs); 1494 1495 /* 1496 * Iterate over tdatas, and for the non-expired ones snapshot their tctx 1497 * stats and merge them into the associated gctx's. 1498 */ 1499 memset(&cnt_all, 0, sizeof(prof_cnt_t)); 1500 malloc_mutex_lock(&tdatas_mtx); 1501 tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, (void *)&cnt_all); 1502 malloc_mutex_unlock(&tdatas_mtx); 1503 1504 /* Merge tctx stats into gctx's. */ 1505 leak_ngctx = 0; 1506 gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, (void *)&leak_ngctx); 1507 1508 prof_leave(tsd, tdata); 1509 1510 /* Create dump file. */ 1511 if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) 1512 goto label_open_close_error; 1513 1514 /* Dump profile header. */ 1515 if (prof_dump_header(propagate_err, &cnt_all)) 1516 goto label_write_error; 1517 1518 /* Dump per gctx profile stats. */ 1519 if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter, 1520 (void *)&propagate_err) != NULL) 1521 goto label_write_error; 1522 1523 /* Dump /proc/<pid>/maps if possible. */ 1524 if (prof_dump_maps(propagate_err)) 1525 goto label_write_error; 1526 1527 if (prof_dump_close(propagate_err)) 1528 goto label_open_close_error; 1529 1530 prof_gctx_finish(tsd, &gctxs); 1531 malloc_mutex_unlock(&prof_dump_mtx); 1532 1533 if (leakcheck) 1534 prof_leakcheck(&cnt_all, leak_ngctx, filename); 1535 1536 return (false); 1537 label_write_error: 1538 prof_dump_close(propagate_err); 1539 label_open_close_error: 1540 prof_gctx_finish(tsd, &gctxs); 1541 malloc_mutex_unlock(&prof_dump_mtx); 1542 return (true); 1543 } 1544 1545 #define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) 1546 #define VSEQ_INVALID UINT64_C(0xffffffffffffffff) 1547 static void 1548 prof_dump_filename(char *filename, char v, uint64_t vseq) 1549 { 1550 1551 cassert(config_prof); 1552 1553 if (vseq != VSEQ_INVALID) { 1554 /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */ 1555 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 1556 "%s.%d.%"FMTu64".%c%"FMTu64".heap", 1557 opt_prof_prefix, (int)getpid(), prof_dump_seq, v, vseq); 1558 } else { 1559 /* "<prefix>.<pid>.<seq>.<v>.heap" */ 1560 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 1561 "%s.%d.%"FMTu64".%c.heap", 1562 opt_prof_prefix, (int)getpid(), prof_dump_seq, v); 1563 } 1564 prof_dump_seq++; 1565 } 1566 1567 static void 1568 prof_fdump(void) 1569 { 1570 tsd_t *tsd; 1571 char filename[DUMP_FILENAME_BUFSIZE]; 1572 1573 cassert(config_prof); 1574 assert(opt_prof_final); 1575 assert(opt_prof_prefix[0] != '\0'); 1576 1577 if (!prof_booted) 1578 return; 1579 tsd = tsd_fetch(); 1580 1581 malloc_mutex_lock(&prof_dump_seq_mtx); 1582 prof_dump_filename(filename, 'f', VSEQ_INVALID); 1583 malloc_mutex_unlock(&prof_dump_seq_mtx); 1584 prof_dump(tsd, false, filename, opt_prof_leak); 1585 } 1586 1587 void 1588 prof_idump(void) 1589 { 1590 tsd_t *tsd; 1591 prof_tdata_t *tdata; 1592 1593 cassert(config_prof); 1594 1595 if (!prof_booted) 1596 return; 1597 tsd = tsd_fetch(); 1598 tdata = prof_tdata_get(tsd, false); 1599 if (tdata == NULL) 1600 return; 1601 if (tdata->enq) { 1602 tdata->enq_idump = true; 1603 return; 1604 } 1605 1606 if (opt_prof_prefix[0] != '\0') { 1607 char filename[PATH_MAX + 1]; 1608 malloc_mutex_lock(&prof_dump_seq_mtx); 1609 prof_dump_filename(filename, 'i', prof_dump_iseq); 1610 prof_dump_iseq++; 1611 malloc_mutex_unlock(&prof_dump_seq_mtx); 1612 prof_dump(tsd, false, filename, false); 1613 } 1614 } 1615 1616 bool 1617 prof_mdump(const char *filename) 1618 { 1619 tsd_t *tsd; 1620 char filename_buf[DUMP_FILENAME_BUFSIZE]; 1621 1622 cassert(config_prof); 1623 1624 if (!opt_prof || !prof_booted) 1625 return (true); 1626 tsd = tsd_fetch(); 1627 1628 if (filename == NULL) { 1629 /* No filename specified, so automatically generate one. */ 1630 if (opt_prof_prefix[0] == '\0') 1631 return (true); 1632 malloc_mutex_lock(&prof_dump_seq_mtx); 1633 prof_dump_filename(filename_buf, 'm', prof_dump_mseq); 1634 prof_dump_mseq++; 1635 malloc_mutex_unlock(&prof_dump_seq_mtx); 1636 filename = filename_buf; 1637 } 1638 return (prof_dump(tsd, true, filename, false)); 1639 } 1640 1641 void 1642 prof_gdump(void) 1643 { 1644 tsd_t *tsd; 1645 prof_tdata_t *tdata; 1646 1647 cassert(config_prof); 1648 1649 if (!prof_booted) 1650 return; 1651 tsd = tsd_fetch(); 1652 tdata = prof_tdata_get(tsd, false); 1653 if (tdata == NULL) 1654 return; 1655 if (tdata->enq) { 1656 tdata->enq_gdump = true; 1657 return; 1658 } 1659 1660 if (opt_prof_prefix[0] != '\0') { 1661 char filename[DUMP_FILENAME_BUFSIZE]; 1662 malloc_mutex_lock(&prof_dump_seq_mtx); 1663 prof_dump_filename(filename, 'u', prof_dump_useq); 1664 prof_dump_useq++; 1665 malloc_mutex_unlock(&prof_dump_seq_mtx); 1666 prof_dump(tsd, false, filename, false); 1667 } 1668 } 1669 1670 static void 1671 prof_bt_hash(const void *key, size_t r_hash[2]) 1672 { 1673 prof_bt_t *bt = (prof_bt_t *)key; 1674 1675 cassert(config_prof); 1676 1677 hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash); 1678 } 1679 1680 static bool 1681 prof_bt_keycomp(const void *k1, const void *k2) 1682 { 1683 const prof_bt_t *bt1 = (prof_bt_t *)k1; 1684 const prof_bt_t *bt2 = (prof_bt_t *)k2; 1685 1686 cassert(config_prof); 1687 1688 if (bt1->len != bt2->len) 1689 return (false); 1690 return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); 1691 } 1692 1693 JEMALLOC_INLINE_C uint64_t 1694 prof_thr_uid_alloc(void) 1695 { 1696 uint64_t thr_uid; 1697 1698 malloc_mutex_lock(&next_thr_uid_mtx); 1699 thr_uid = next_thr_uid; 1700 next_thr_uid++; 1701 malloc_mutex_unlock(&next_thr_uid_mtx); 1702 1703 return (thr_uid); 1704 } 1705 1706 static prof_tdata_t * 1707 prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, 1708 char *thread_name, bool active) 1709 { 1710 prof_tdata_t *tdata; 1711 tcache_t *tcache; 1712 1713 cassert(config_prof); 1714 1715 /* Initialize an empty cache for this thread. */ 1716 tcache = tcache_get(tsd, true); 1717 tdata = (prof_tdata_t *)iallocztm(tsd, sizeof(prof_tdata_t), false, 1718 tcache, true, NULL); 1719 if (tdata == NULL) 1720 return (NULL); 1721 1722 tdata->lock = prof_tdata_mutex_choose(thr_uid); 1723 tdata->thr_uid = thr_uid; 1724 tdata->thr_discrim = thr_discrim; 1725 tdata->thread_name = thread_name; 1726 tdata->attached = true; 1727 tdata->expired = false; 1728 tdata->tctx_uid_next = 0; 1729 1730 if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, 1731 prof_bt_hash, prof_bt_keycomp)) { 1732 idalloctm(tsd, tdata, tcache, true); 1733 return (NULL); 1734 } 1735 1736 tdata->prng_state = (uint64_t)(uintptr_t)tdata; 1737 prof_sample_threshold_update(tdata); 1738 1739 tdata->enq = false; 1740 tdata->enq_idump = false; 1741 tdata->enq_gdump = false; 1742 1743 tdata->dumping = false; 1744 tdata->active = active; 1745 1746 malloc_mutex_lock(&tdatas_mtx); 1747 tdata_tree_insert(&tdatas, tdata); 1748 malloc_mutex_unlock(&tdatas_mtx); 1749 1750 return (tdata); 1751 } 1752 1753 prof_tdata_t * 1754 prof_tdata_init(tsd_t *tsd) 1755 { 1756 1757 return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(), 0, NULL, 1758 prof_thread_active_init_get())); 1759 } 1760 1761 /* tdata->lock must be held. */ 1762 static bool 1763 prof_tdata_should_destroy(prof_tdata_t *tdata, bool even_if_attached) 1764 { 1765 1766 if (tdata->attached && !even_if_attached) 1767 return (false); 1768 if (ckh_count(&tdata->bt2tctx) != 0) 1769 return (false); 1770 return (true); 1771 } 1772 1773 /* tdatas_mtx must be held. */ 1774 static void 1775 prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata, 1776 bool even_if_attached) 1777 { 1778 tcache_t *tcache; 1779 1780 assert(prof_tdata_should_destroy(tdata, even_if_attached)); 1781 assert(tsd_prof_tdata_get(tsd) != tdata); 1782 1783 tdata_tree_remove(&tdatas, tdata); 1784 1785 tcache = tcache_get(tsd, false); 1786 if (tdata->thread_name != NULL) 1787 idalloctm(tsd, tdata->thread_name, tcache, true); 1788 ckh_delete(tsd, &tdata->bt2tctx); 1789 idalloctm(tsd, tdata, tcache, true); 1790 } 1791 1792 static void 1793 prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) 1794 { 1795 1796 malloc_mutex_lock(&tdatas_mtx); 1797 prof_tdata_destroy_locked(tsd, tdata, even_if_attached); 1798 malloc_mutex_unlock(&tdatas_mtx); 1799 } 1800 1801 static void 1802 prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) 1803 { 1804 bool destroy_tdata; 1805 1806 malloc_mutex_lock(tdata->lock); 1807 if (tdata->attached) { 1808 destroy_tdata = prof_tdata_should_destroy(tdata, true); 1809 /* 1810 * Only detach if !destroy_tdata, because detaching would allow 1811 * another thread to win the race to destroy tdata. 1812 */ 1813 if (!destroy_tdata) 1814 tdata->attached = false; 1815 tsd_prof_tdata_set(tsd, NULL); 1816 } else 1817 destroy_tdata = false; 1818 malloc_mutex_unlock(tdata->lock); 1819 if (destroy_tdata) 1820 prof_tdata_destroy(tsd, tdata, true); 1821 } 1822 1823 prof_tdata_t * 1824 prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) 1825 { 1826 uint64_t thr_uid = tdata->thr_uid; 1827 uint64_t thr_discrim = tdata->thr_discrim + 1; 1828 char *thread_name = (tdata->thread_name != NULL) ? 1829 prof_thread_name_alloc(tsd, tdata->thread_name) : NULL; 1830 bool active = tdata->active; 1831 1832 prof_tdata_detach(tsd, tdata); 1833 return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name, 1834 active)); 1835 } 1836 1837 static bool 1838 prof_tdata_expire(prof_tdata_t *tdata) 1839 { 1840 bool destroy_tdata; 1841 1842 malloc_mutex_lock(tdata->lock); 1843 if (!tdata->expired) { 1844 tdata->expired = true; 1845 destroy_tdata = tdata->attached ? false : 1846 prof_tdata_should_destroy(tdata, false); 1847 } else 1848 destroy_tdata = false; 1849 malloc_mutex_unlock(tdata->lock); 1850 1851 return (destroy_tdata); 1852 } 1853 1854 static prof_tdata_t * 1855 prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) 1856 { 1857 1858 return (prof_tdata_expire(tdata) ? tdata : NULL); 1859 } 1860 1861 void 1862 prof_reset(tsd_t *tsd, size_t lg_sample) 1863 { 1864 prof_tdata_t *next; 1865 1866 assert(lg_sample < (sizeof(uint64_t) << 3)); 1867 1868 malloc_mutex_lock(&prof_dump_mtx); 1869 malloc_mutex_lock(&tdatas_mtx); 1870 1871 lg_prof_sample = lg_sample; 1872 1873 next = NULL; 1874 do { 1875 prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next, 1876 prof_tdata_reset_iter, NULL); 1877 if (to_destroy != NULL) { 1878 next = tdata_tree_next(&tdatas, to_destroy); 1879 prof_tdata_destroy_locked(tsd, to_destroy, false); 1880 } else 1881 next = NULL; 1882 } while (next != NULL); 1883 1884 malloc_mutex_unlock(&tdatas_mtx); 1885 malloc_mutex_unlock(&prof_dump_mtx); 1886 } 1887 1888 void 1889 prof_tdata_cleanup(tsd_t *tsd) 1890 { 1891 prof_tdata_t *tdata; 1892 1893 if (!config_prof) 1894 return; 1895 1896 tdata = tsd_prof_tdata_get(tsd); 1897 if (tdata != NULL) 1898 prof_tdata_detach(tsd, tdata); 1899 } 1900 1901 bool 1902 prof_active_get(void) 1903 { 1904 bool prof_active_current; 1905 1906 malloc_mutex_lock(&prof_active_mtx); 1907 prof_active_current = prof_active; 1908 malloc_mutex_unlock(&prof_active_mtx); 1909 return (prof_active_current); 1910 } 1911 1912 bool 1913 prof_active_set(bool active) 1914 { 1915 bool prof_active_old; 1916 1917 malloc_mutex_lock(&prof_active_mtx); 1918 prof_active_old = prof_active; 1919 prof_active = active; 1920 malloc_mutex_unlock(&prof_active_mtx); 1921 return (prof_active_old); 1922 } 1923 1924 const char * 1925 prof_thread_name_get(void) 1926 { 1927 tsd_t *tsd; 1928 prof_tdata_t *tdata; 1929 1930 tsd = tsd_fetch(); 1931 tdata = prof_tdata_get(tsd, true); 1932 if (tdata == NULL) 1933 return (""); 1934 return (tdata->thread_name != NULL ? tdata->thread_name : ""); 1935 } 1936 1937 static char * 1938 prof_thread_name_alloc(tsd_t *tsd, const char *thread_name) 1939 { 1940 char *ret; 1941 size_t size; 1942 1943 if (thread_name == NULL) 1944 return (NULL); 1945 1946 size = strlen(thread_name) + 1; 1947 if (size == 1) 1948 return (""); 1949 1950 ret = iallocztm(tsd, size, false, tcache_get(tsd, true), true, NULL); 1951 if (ret == NULL) 1952 return (NULL); 1953 memcpy(ret, thread_name, size); 1954 return (ret); 1955 } 1956 1957 int 1958 prof_thread_name_set(tsd_t *tsd, const char *thread_name) 1959 { 1960 prof_tdata_t *tdata; 1961 unsigned i; 1962 char *s; 1963 1964 tdata = prof_tdata_get(tsd, true); 1965 if (tdata == NULL) 1966 return (EAGAIN); 1967 1968 /* Validate input. */ 1969 if (thread_name == NULL) 1970 return (EFAULT); 1971 for (i = 0; thread_name[i] != '\0'; i++) { 1972 char c = thread_name[i]; 1973 if (!isgraph(c) && !isblank(c)) 1974 return (EFAULT); 1975 } 1976 1977 s = prof_thread_name_alloc(tsd, thread_name); 1978 if (s == NULL) 1979 return (EAGAIN); 1980 1981 if (tdata->thread_name != NULL) { 1982 idalloctm(tsd, tdata->thread_name, tcache_get(tsd, false), 1983 true); 1984 tdata->thread_name = NULL; 1985 } 1986 if (strlen(s) > 0) 1987 tdata->thread_name = s; 1988 return (0); 1989 } 1990 1991 bool 1992 prof_thread_active_get(void) 1993 { 1994 tsd_t *tsd; 1995 prof_tdata_t *tdata; 1996 1997 tsd = tsd_fetch(); 1998 tdata = prof_tdata_get(tsd, true); 1999 if (tdata == NULL) 2000 return (false); 2001 return (tdata->active); 2002 } 2003 2004 bool 2005 prof_thread_active_set(bool active) 2006 { 2007 tsd_t *tsd; 2008 prof_tdata_t *tdata; 2009 2010 tsd = tsd_fetch(); 2011 tdata = prof_tdata_get(tsd, true); 2012 if (tdata == NULL) 2013 return (true); 2014 tdata->active = active; 2015 return (false); 2016 } 2017 2018 bool 2019 prof_thread_active_init_get(void) 2020 { 2021 bool active_init; 2022 2023 malloc_mutex_lock(&prof_thread_active_init_mtx); 2024 active_init = prof_thread_active_init; 2025 malloc_mutex_unlock(&prof_thread_active_init_mtx); 2026 return (active_init); 2027 } 2028 2029 bool 2030 prof_thread_active_init_set(bool active_init) 2031 { 2032 bool active_init_old; 2033 2034 malloc_mutex_lock(&prof_thread_active_init_mtx); 2035 active_init_old = prof_thread_active_init; 2036 prof_thread_active_init = active_init; 2037 malloc_mutex_unlock(&prof_thread_active_init_mtx); 2038 return (active_init_old); 2039 } 2040 2041 bool 2042 prof_gdump_get(void) 2043 { 2044 bool prof_gdump_current; 2045 2046 malloc_mutex_lock(&prof_gdump_mtx); 2047 prof_gdump_current = prof_gdump_val; 2048 malloc_mutex_unlock(&prof_gdump_mtx); 2049 return (prof_gdump_current); 2050 } 2051 2052 bool 2053 prof_gdump_set(bool gdump) 2054 { 2055 bool prof_gdump_old; 2056 2057 malloc_mutex_lock(&prof_gdump_mtx); 2058 prof_gdump_old = prof_gdump_val; 2059 prof_gdump_val = gdump; 2060 malloc_mutex_unlock(&prof_gdump_mtx); 2061 return (prof_gdump_old); 2062 } 2063 2064 void 2065 prof_boot0(void) 2066 { 2067 2068 cassert(config_prof); 2069 2070 memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, 2071 sizeof(PROF_PREFIX_DEFAULT)); 2072 } 2073 2074 void 2075 prof_boot1(void) 2076 { 2077 2078 cassert(config_prof); 2079 2080 /* 2081 * opt_prof must be in its final state before any arenas are 2082 * initialized, so this function must be executed early. 2083 */ 2084 2085 if (opt_prof_leak && !opt_prof) { 2086 /* 2087 * Enable opt_prof, but in such a way that profiles are never 2088 * automatically dumped. 2089 */ 2090 opt_prof = true; 2091 opt_prof_gdump = false; 2092 } else if (opt_prof) { 2093 if (opt_lg_prof_interval >= 0) { 2094 prof_interval = (((uint64_t)1U) << 2095 opt_lg_prof_interval); 2096 } 2097 } 2098 } 2099 2100 bool 2101 prof_boot2(void) 2102 { 2103 2104 cassert(config_prof); 2105 2106 if (opt_prof) { 2107 tsd_t *tsd; 2108 unsigned i; 2109 2110 lg_prof_sample = opt_lg_prof_sample; 2111 2112 prof_active = opt_prof_active; 2113 if (malloc_mutex_init(&prof_active_mtx)) 2114 return (true); 2115 2116 prof_gdump_val = opt_prof_gdump; 2117 if (malloc_mutex_init(&prof_gdump_mtx)) 2118 return (true); 2119 2120 prof_thread_active_init = opt_prof_thread_active_init; 2121 if (malloc_mutex_init(&prof_thread_active_init_mtx)) 2122 return (true); 2123 2124 tsd = tsd_fetch(); 2125 if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, 2126 prof_bt_keycomp)) 2127 return (true); 2128 if (malloc_mutex_init(&bt2gctx_mtx)) 2129 return (true); 2130 2131 tdata_tree_new(&tdatas); 2132 if (malloc_mutex_init(&tdatas_mtx)) 2133 return (true); 2134 2135 next_thr_uid = 0; 2136 if (malloc_mutex_init(&next_thr_uid_mtx)) 2137 return (true); 2138 2139 if (malloc_mutex_init(&prof_dump_seq_mtx)) 2140 return (true); 2141 if (malloc_mutex_init(&prof_dump_mtx)) 2142 return (true); 2143 2144 if (opt_prof_final && opt_prof_prefix[0] != '\0' && 2145 atexit(prof_fdump) != 0) { 2146 malloc_write("<jemalloc>: Error in atexit()\n"); 2147 if (opt_abort) 2148 abort(); 2149 } 2150 2151 gctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS * 2152 sizeof(malloc_mutex_t)); 2153 if (gctx_locks == NULL) 2154 return (true); 2155 for (i = 0; i < PROF_NCTX_LOCKS; i++) { 2156 if (malloc_mutex_init(&gctx_locks[i])) 2157 return (true); 2158 } 2159 2160 tdata_locks = (malloc_mutex_t *)base_alloc(PROF_NTDATA_LOCKS * 2161 sizeof(malloc_mutex_t)); 2162 if (tdata_locks == NULL) 2163 return (true); 2164 for (i = 0; i < PROF_NTDATA_LOCKS; i++) { 2165 if (malloc_mutex_init(&tdata_locks[i])) 2166 return (true); 2167 } 2168 } 2169 2170 #ifdef JEMALLOC_PROF_LIBGCC 2171 /* 2172 * Cause the backtracing machinery to allocate its internal state 2173 * before enabling profiling. 2174 */ 2175 _Unwind_Backtrace(prof_unwind_init_callback, NULL); 2176 #endif 2177 2178 prof_booted = true; 2179 2180 return (false); 2181 } 2182 2183 void 2184 prof_prefork(void) 2185 { 2186 2187 if (opt_prof) { 2188 unsigned i; 2189 2190 malloc_mutex_prefork(&tdatas_mtx); 2191 malloc_mutex_prefork(&bt2gctx_mtx); 2192 malloc_mutex_prefork(&next_thr_uid_mtx); 2193 malloc_mutex_prefork(&prof_dump_seq_mtx); 2194 for (i = 0; i < PROF_NCTX_LOCKS; i++) 2195 malloc_mutex_prefork(&gctx_locks[i]); 2196 for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2197 malloc_mutex_prefork(&tdata_locks[i]); 2198 } 2199 } 2200 2201 void 2202 prof_postfork_parent(void) 2203 { 2204 2205 if (opt_prof) { 2206 unsigned i; 2207 2208 for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2209 malloc_mutex_postfork_parent(&tdata_locks[i]); 2210 for (i = 0; i < PROF_NCTX_LOCKS; i++) 2211 malloc_mutex_postfork_parent(&gctx_locks[i]); 2212 malloc_mutex_postfork_parent(&prof_dump_seq_mtx); 2213 malloc_mutex_postfork_parent(&next_thr_uid_mtx); 2214 malloc_mutex_postfork_parent(&bt2gctx_mtx); 2215 malloc_mutex_postfork_parent(&tdatas_mtx); 2216 } 2217 } 2218 2219 void 2220 prof_postfork_child(void) 2221 { 2222 2223 if (opt_prof) { 2224 unsigned i; 2225 2226 for (i = 0; i < PROF_NTDATA_LOCKS; i++) 2227 malloc_mutex_postfork_child(&tdata_locks[i]); 2228 for (i = 0; i < PROF_NCTX_LOCKS; i++) 2229 malloc_mutex_postfork_child(&gctx_locks[i]); 2230 malloc_mutex_postfork_child(&prof_dump_seq_mtx); 2231 malloc_mutex_postfork_child(&next_thr_uid_mtx); 2232 malloc_mutex_postfork_child(&bt2gctx_mtx); 2233 malloc_mutex_postfork_child(&tdatas_mtx); 2234 } 2235 } 2236 2237 /******************************************************************************/ 2238