1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/assert.h" 5 6 JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS 7 8 /******************************************************************************/ 9 /* Data. */ 10 11 /* This option should be opt-in only. */ 12 #define BACKGROUND_THREAD_DEFAULT false 13 /* Read-only after initialization. */ 14 bool opt_background_thread = BACKGROUND_THREAD_DEFAULT; 15 size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1; 16 17 /* Used for thread creation, termination and stats. */ 18 malloc_mutex_t background_thread_lock; 19 /* Indicates global state. Atomic because decay reads this w/o locking. */ 20 atomic_b_t background_thread_enabled_state; 21 size_t n_background_threads; 22 size_t max_background_threads; 23 /* Thread info per-index. */ 24 background_thread_info_t *background_thread_info; 25 26 /******************************************************************************/ 27 28 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER 29 30 static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, 31 void *(*)(void *), void *__restrict); 32 33 static void 34 pthread_create_wrapper_init(void) { 35 #ifdef JEMALLOC_LAZY_LOCK 36 if (!isthreaded) { 37 isthreaded = true; 38 } 39 #endif 40 } 41 42 int 43 pthread_create_wrapper(pthread_t *__restrict thread, const pthread_attr_t *attr, 44 void *(*start_routine)(void *), void *__restrict arg) { 45 pthread_create_wrapper_init(); 46 47 return pthread_create_fptr(thread, attr, start_routine, arg); 48 } 49 #endif /* JEMALLOC_PTHREAD_CREATE_WRAPPER */ 50 51 #ifndef JEMALLOC_BACKGROUND_THREAD 52 #define NOT_REACHED { not_reached(); } 53 bool background_thread_create(tsd_t *tsd, unsigned arena_ind) NOT_REACHED 54 bool background_threads_enable(tsd_t *tsd) NOT_REACHED 55 bool background_threads_disable(tsd_t *tsd) NOT_REACHED 56 bool background_thread_is_started(background_thread_info_t *info) NOT_REACHED 57 void background_thread_wakeup_early(background_thread_info_t *info, 58 nstime_t *remaining_sleep) NOT_REACHED 59 void background_thread_prefork0(tsdn_t *tsdn) NOT_REACHED 60 void background_thread_prefork1(tsdn_t *tsdn) NOT_REACHED 61 void background_thread_postfork_parent(tsdn_t *tsdn) NOT_REACHED 62 void background_thread_postfork_child(tsdn_t *tsdn) NOT_REACHED 63 bool background_thread_stats_read(tsdn_t *tsdn, 64 background_thread_stats_t *stats) NOT_REACHED 65 void background_thread_ctl_init(tsdn_t *tsdn) NOT_REACHED 66 #undef NOT_REACHED 67 #else 68 69 static bool background_thread_enabled_at_fork; 70 71 static void 72 background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) { 73 background_thread_wakeup_time_set(tsdn, info, 0); 74 info->npages_to_purge_new = 0; 75 if (config_stats) { 76 info->tot_n_runs = 0; 77 nstime_init_zero(&info->tot_sleep_time); 78 } 79 } 80 81 static inline bool 82 set_current_thread_affinity(int cpu) { 83 #if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) 84 cpu_set_t cpuset; 85 #else 86 # ifndef __NetBSD__ 87 cpuset_t cpuset; 88 # else 89 cpuset_t *cpuset; 90 # endif 91 #endif 92 93 #ifndef __NetBSD__ 94 CPU_ZERO(&cpuset); 95 CPU_SET(cpu, &cpuset); 96 #else 97 cpuset = cpuset_create(); 98 #endif 99 100 #if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) 101 return (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0); 102 #else 103 # ifndef __NetBSD__ 104 int ret = pthread_setaffinity_np(pthread_self(), sizeof(cpuset_t), 105 &cpuset); 106 # else 107 int ret = pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), 108 cpuset); 109 cpuset_destroy(cpuset); 110 # endif 111 return ret != 0; 112 #endif 113 } 114 115 #define BILLION UINT64_C(1000000000) 116 /* Minimal sleep interval 100 ms. */ 117 #define BACKGROUND_THREAD_MIN_INTERVAL_NS (BILLION / 10) 118 119 static void 120 background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info, 121 uint64_t interval) { 122 if (config_stats) { 123 info->tot_n_runs++; 124 } 125 info->npages_to_purge_new = 0; 126 127 struct timeval tv; 128 /* Specific clock required by timedwait. */ 129 gettimeofday(&tv, NULL); 130 nstime_t before_sleep; 131 nstime_init2(&before_sleep, tv.tv_sec, tv.tv_usec * 1000); 132 133 int ret; 134 if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) { 135 background_thread_wakeup_time_set(tsdn, info, 136 BACKGROUND_THREAD_INDEFINITE_SLEEP); 137 ret = pthread_cond_wait(&info->cond, &info->mtx.lock); 138 assert(ret == 0); 139 } else { 140 assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS && 141 interval <= BACKGROUND_THREAD_INDEFINITE_SLEEP); 142 /* We need malloc clock (can be different from tv). */ 143 nstime_t next_wakeup; 144 nstime_init_update(&next_wakeup); 145 nstime_iadd(&next_wakeup, interval); 146 assert(nstime_ns(&next_wakeup) < 147 BACKGROUND_THREAD_INDEFINITE_SLEEP); 148 background_thread_wakeup_time_set(tsdn, info, 149 nstime_ns(&next_wakeup)); 150 151 nstime_t ts_wakeup; 152 nstime_copy(&ts_wakeup, &before_sleep); 153 nstime_iadd(&ts_wakeup, interval); 154 struct timespec ts; 155 ts.tv_sec = (size_t)nstime_sec(&ts_wakeup); 156 ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup); 157 158 assert(!background_thread_indefinite_sleep(info)); 159 ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, &ts); 160 assert(ret == ETIMEDOUT || ret == 0); 161 } 162 if (config_stats) { 163 gettimeofday(&tv, NULL); 164 nstime_t after_sleep; 165 nstime_init2(&after_sleep, tv.tv_sec, tv.tv_usec * 1000); 166 if (nstime_compare(&after_sleep, &before_sleep) > 0) { 167 nstime_subtract(&after_sleep, &before_sleep); 168 nstime_add(&info->tot_sleep_time, &after_sleep); 169 } 170 } 171 } 172 173 static bool 174 background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) { 175 if (unlikely(info->state == background_thread_paused)) { 176 malloc_mutex_unlock(tsdn, &info->mtx); 177 /* Wait on global lock to update status. */ 178 malloc_mutex_lock(tsdn, &background_thread_lock); 179 malloc_mutex_unlock(tsdn, &background_thread_lock); 180 malloc_mutex_lock(tsdn, &info->mtx); 181 return true; 182 } 183 184 return false; 185 } 186 187 static inline void 188 background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info, 189 unsigned ind) { 190 uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX; 191 unsigned narenas = narenas_total_get(); 192 bool slept_indefinitely = background_thread_indefinite_sleep(info); 193 194 for (unsigned i = ind; i < narenas; i += max_background_threads) { 195 arena_t *arena = arena_get(tsdn, i, false); 196 if (!arena) { 197 continue; 198 } 199 /* 200 * If thread was woken up from the indefinite sleep, don't 201 * do the work instantly, but rather check when the deferred 202 * work that caused this thread to wake up is scheduled for. 203 */ 204 if (!slept_indefinitely) { 205 arena_do_deferred_work(tsdn, arena); 206 } 207 if (ns_until_deferred <= BACKGROUND_THREAD_MIN_INTERVAL_NS) { 208 /* Min interval will be used. */ 209 continue; 210 } 211 uint64_t ns_arena_deferred = pa_shard_time_until_deferred_work( 212 tsdn, &arena->pa_shard); 213 if (ns_arena_deferred < ns_until_deferred) { 214 ns_until_deferred = ns_arena_deferred; 215 } 216 } 217 218 uint64_t sleep_ns; 219 if (ns_until_deferred == BACKGROUND_THREAD_DEFERRED_MAX) { 220 sleep_ns = BACKGROUND_THREAD_INDEFINITE_SLEEP; 221 } else { 222 sleep_ns = 223 (ns_until_deferred < BACKGROUND_THREAD_MIN_INTERVAL_NS) 224 ? BACKGROUND_THREAD_MIN_INTERVAL_NS 225 : ns_until_deferred; 226 227 } 228 229 background_thread_sleep(tsdn, info, sleep_ns); 230 } 231 232 static bool 233 background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) { 234 if (info == &background_thread_info[0]) { 235 malloc_mutex_assert_owner(tsd_tsdn(tsd), 236 &background_thread_lock); 237 } else { 238 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), 239 &background_thread_lock); 240 } 241 242 pre_reentrancy(tsd, NULL); 243 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 244 bool has_thread; 245 assert(info->state != background_thread_paused); 246 if (info->state == background_thread_started) { 247 has_thread = true; 248 info->state = background_thread_stopped; 249 pthread_cond_signal(&info->cond); 250 } else { 251 has_thread = false; 252 } 253 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 254 255 if (!has_thread) { 256 post_reentrancy(tsd); 257 return false; 258 } 259 void *ret; 260 if (pthread_join(info->thread, &ret)) { 261 post_reentrancy(tsd); 262 return true; 263 } 264 assert(ret == NULL); 265 n_background_threads--; 266 post_reentrancy(tsd); 267 268 return false; 269 } 270 271 static void *background_thread_entry(void *ind_arg); 272 273 static int 274 background_thread_create_signals_masked(pthread_t *thread, 275 const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { 276 /* 277 * Mask signals during thread creation so that the thread inherits 278 * an empty signal set. 279 */ 280 sigset_t set; 281 sigfillset(&set); 282 sigset_t oldset; 283 int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset); 284 if (mask_err != 0) { 285 return mask_err; 286 } 287 int create_err = pthread_create_wrapper(thread, attr, start_routine, 288 arg); 289 /* 290 * Restore the signal mask. Failure to restore the signal mask here 291 * changes program behavior. 292 */ 293 int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL); 294 if (restore_err != 0) { 295 malloc_printf("<jemalloc>: background thread creation " 296 "failed (%d), and signal mask restoration failed " 297 "(%d)\n", create_err, restore_err); 298 if (opt_abort) { 299 abort(); 300 } 301 } 302 return create_err; 303 } 304 305 static bool 306 check_background_thread_creation(tsd_t *tsd, unsigned *n_created, 307 bool *created_threads) { 308 bool ret = false; 309 if (likely(*n_created == n_background_threads)) { 310 return ret; 311 } 312 313 tsdn_t *tsdn = tsd_tsdn(tsd); 314 malloc_mutex_unlock(tsdn, &background_thread_info[0].mtx); 315 for (unsigned i = 1; i < max_background_threads; i++) { 316 if (created_threads[i]) { 317 continue; 318 } 319 background_thread_info_t *info = &background_thread_info[i]; 320 malloc_mutex_lock(tsdn, &info->mtx); 321 /* 322 * In case of the background_thread_paused state because of 323 * arena reset, delay the creation. 324 */ 325 bool create = (info->state == background_thread_started); 326 malloc_mutex_unlock(tsdn, &info->mtx); 327 if (!create) { 328 continue; 329 } 330 331 pre_reentrancy(tsd, NULL); 332 int err = background_thread_create_signals_masked(&info->thread, 333 NULL, background_thread_entry, (void *)(uintptr_t)i); 334 post_reentrancy(tsd); 335 336 if (err == 0) { 337 (*n_created)++; 338 created_threads[i] = true; 339 } else { 340 malloc_printf("<jemalloc>: background thread " 341 "creation failed (%d)\n", err); 342 if (opt_abort) { 343 abort(); 344 } 345 } 346 /* Return to restart the loop since we unlocked. */ 347 ret = true; 348 break; 349 } 350 malloc_mutex_lock(tsdn, &background_thread_info[0].mtx); 351 352 return ret; 353 } 354 355 static void 356 background_thread0_work(tsd_t *tsd) { 357 /* Thread0 is also responsible for launching / terminating threads. */ 358 VARIABLE_ARRAY(bool, created_threads, max_background_threads); 359 unsigned i; 360 for (i = 1; i < max_background_threads; i++) { 361 created_threads[i] = false; 362 } 363 /* Start working, and create more threads when asked. */ 364 unsigned n_created = 1; 365 while (background_thread_info[0].state != background_thread_stopped) { 366 if (background_thread_pause_check(tsd_tsdn(tsd), 367 &background_thread_info[0])) { 368 continue; 369 } 370 if (check_background_thread_creation(tsd, &n_created, 371 (bool *)&created_threads)) { 372 continue; 373 } 374 background_work_sleep_once(tsd_tsdn(tsd), 375 &background_thread_info[0], 0); 376 } 377 378 /* 379 * Shut down other threads at exit. Note that the ctl thread is holding 380 * the global background_thread mutex (and is waiting) for us. 381 */ 382 assert(!background_thread_enabled()); 383 for (i = 1; i < max_background_threads; i++) { 384 background_thread_info_t *info = &background_thread_info[i]; 385 assert(info->state != background_thread_paused); 386 if (created_threads[i]) { 387 background_threads_disable_single(tsd, info); 388 } else { 389 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 390 if (info->state != background_thread_stopped) { 391 /* The thread was not created. */ 392 assert(info->state == 393 background_thread_started); 394 n_background_threads--; 395 info->state = background_thread_stopped; 396 } 397 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 398 } 399 } 400 background_thread_info[0].state = background_thread_stopped; 401 assert(n_background_threads == 1); 402 } 403 404 static void 405 background_work(tsd_t *tsd, unsigned ind) { 406 background_thread_info_t *info = &background_thread_info[ind]; 407 408 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 409 background_thread_wakeup_time_set(tsd_tsdn(tsd), info, 410 BACKGROUND_THREAD_INDEFINITE_SLEEP); 411 if (ind == 0) { 412 background_thread0_work(tsd); 413 } else { 414 while (info->state != background_thread_stopped) { 415 if (background_thread_pause_check(tsd_tsdn(tsd), 416 info)) { 417 continue; 418 } 419 background_work_sleep_once(tsd_tsdn(tsd), info, ind); 420 } 421 } 422 assert(info->state == background_thread_stopped); 423 background_thread_wakeup_time_set(tsd_tsdn(tsd), info, 0); 424 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 425 } 426 427 static void * 428 background_thread_entry(void *ind_arg) { 429 unsigned thread_ind = (unsigned)(uintptr_t)ind_arg; 430 assert(thread_ind < max_background_threads); 431 #ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP 432 pthread_setname_np(pthread_self(), "jemalloc_bg_thd"); 433 #elif defined(__FreeBSD__) || defined(__DragonFly__) 434 pthread_set_name_np(pthread_self(), "jemalloc_bg_thd"); 435 #endif 436 if (opt_percpu_arena != percpu_arena_disabled) { 437 set_current_thread_affinity((int)thread_ind); 438 } 439 /* 440 * Start periodic background work. We use internal tsd which avoids 441 * side effects, for example triggering new arena creation (which in 442 * turn triggers another background thread creation). 443 */ 444 background_work(tsd_internal_fetch(), thread_ind); 445 assert(pthread_equal(pthread_self(), 446 background_thread_info[thread_ind].thread)); 447 448 return NULL; 449 } 450 451 static void 452 background_thread_init(tsd_t *tsd, background_thread_info_t *info) { 453 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); 454 info->state = background_thread_started; 455 background_thread_info_init(tsd_tsdn(tsd), info); 456 n_background_threads++; 457 } 458 459 static bool 460 background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { 461 assert(have_background_thread); 462 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); 463 464 /* We create at most NCPUs threads. */ 465 size_t thread_ind = arena_ind % max_background_threads; 466 background_thread_info_t *info = &background_thread_info[thread_ind]; 467 468 bool need_new_thread; 469 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 470 need_new_thread = background_thread_enabled() && 471 (info->state == background_thread_stopped); 472 if (need_new_thread) { 473 background_thread_init(tsd, info); 474 } 475 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 476 if (!need_new_thread) { 477 return false; 478 } 479 if (arena_ind != 0) { 480 /* Threads are created asynchronously by Thread 0. */ 481 background_thread_info_t *t0 = &background_thread_info[0]; 482 malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx); 483 assert(t0->state == background_thread_started); 484 pthread_cond_signal(&t0->cond); 485 malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx); 486 487 return false; 488 } 489 490 pre_reentrancy(tsd, NULL); 491 /* 492 * To avoid complications (besides reentrancy), create internal 493 * background threads with the underlying pthread_create. 494 */ 495 int err = background_thread_create_signals_masked(&info->thread, NULL, 496 background_thread_entry, (void *)thread_ind); 497 post_reentrancy(tsd); 498 499 if (err != 0) { 500 malloc_printf("<jemalloc>: arena 0 background thread creation " 501 "failed (%d)\n", err); 502 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 503 info->state = background_thread_stopped; 504 n_background_threads--; 505 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 506 507 return true; 508 } 509 510 return false; 511 } 512 513 /* Create a new background thread if needed. */ 514 bool 515 background_thread_create(tsd_t *tsd, unsigned arena_ind) { 516 assert(have_background_thread); 517 518 bool ret; 519 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 520 ret = background_thread_create_locked(tsd, arena_ind); 521 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 522 523 return ret; 524 } 525 526 bool 527 background_threads_enable(tsd_t *tsd) { 528 assert(n_background_threads == 0); 529 assert(background_thread_enabled()); 530 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); 531 532 VARIABLE_ARRAY(bool, marked, max_background_threads); 533 unsigned nmarked; 534 for (unsigned i = 0; i < max_background_threads; i++) { 535 marked[i] = false; 536 } 537 nmarked = 0; 538 /* Thread 0 is required and created at the end. */ 539 marked[0] = true; 540 /* Mark the threads we need to create for thread 0. */ 541 unsigned narenas = narenas_total_get(); 542 for (unsigned i = 1; i < narenas; i++) { 543 if (marked[i % max_background_threads] || 544 arena_get(tsd_tsdn(tsd), i, false) == NULL) { 545 continue; 546 } 547 background_thread_info_t *info = &background_thread_info[ 548 i % max_background_threads]; 549 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 550 assert(info->state == background_thread_stopped); 551 background_thread_init(tsd, info); 552 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 553 marked[i % max_background_threads] = true; 554 if (++nmarked == max_background_threads) { 555 break; 556 } 557 } 558 559 bool err = background_thread_create_locked(tsd, 0); 560 if (err) { 561 return true; 562 } 563 for (unsigned i = 0; i < narenas; i++) { 564 arena_t *arena = arena_get(tsd_tsdn(tsd), i, false); 565 if (arena != NULL) { 566 pa_shard_set_deferral_allowed(tsd_tsdn(tsd), 567 &arena->pa_shard, true); 568 } 569 } 570 return false; 571 } 572 573 bool 574 background_threads_disable(tsd_t *tsd) { 575 assert(!background_thread_enabled()); 576 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); 577 578 /* Thread 0 will be responsible for terminating other threads. */ 579 if (background_threads_disable_single(tsd, 580 &background_thread_info[0])) { 581 return true; 582 } 583 assert(n_background_threads == 0); 584 unsigned narenas = narenas_total_get(); 585 for (unsigned i = 0; i < narenas; i++) { 586 arena_t *arena = arena_get(tsd_tsdn(tsd), i, false); 587 if (arena != NULL) { 588 pa_shard_set_deferral_allowed(tsd_tsdn(tsd), 589 &arena->pa_shard, false); 590 } 591 } 592 593 return false; 594 } 595 596 bool 597 background_thread_is_started(background_thread_info_t *info) { 598 return info->state == background_thread_started; 599 } 600 601 void 602 background_thread_wakeup_early(background_thread_info_t *info, 603 nstime_t *remaining_sleep) { 604 /* 605 * This is an optimization to increase batching. At this point 606 * we know that background thread wakes up soon, so the time to cache 607 * the just freed memory is bounded and low. 608 */ 609 if (remaining_sleep != NULL && nstime_ns(remaining_sleep) < 610 BACKGROUND_THREAD_MIN_INTERVAL_NS) { 611 return; 612 } 613 pthread_cond_signal(&info->cond); 614 } 615 616 void 617 background_thread_prefork0(tsdn_t *tsdn) { 618 malloc_mutex_prefork(tsdn, &background_thread_lock); 619 background_thread_enabled_at_fork = background_thread_enabled(); 620 } 621 622 void 623 background_thread_prefork1(tsdn_t *tsdn) { 624 for (unsigned i = 0; i < max_background_threads; i++) { 625 malloc_mutex_prefork(tsdn, &background_thread_info[i].mtx); 626 } 627 } 628 629 void 630 background_thread_postfork_parent(tsdn_t *tsdn) { 631 for (unsigned i = 0; i < max_background_threads; i++) { 632 malloc_mutex_postfork_parent(tsdn, 633 &background_thread_info[i].mtx); 634 } 635 malloc_mutex_postfork_parent(tsdn, &background_thread_lock); 636 } 637 638 void 639 background_thread_postfork_child(tsdn_t *tsdn) { 640 for (unsigned i = 0; i < max_background_threads; i++) { 641 malloc_mutex_postfork_child(tsdn, 642 &background_thread_info[i].mtx); 643 } 644 malloc_mutex_postfork_child(tsdn, &background_thread_lock); 645 if (!background_thread_enabled_at_fork) { 646 return; 647 } 648 649 /* Clear background_thread state (reset to disabled for child). */ 650 malloc_mutex_lock(tsdn, &background_thread_lock); 651 n_background_threads = 0; 652 background_thread_enabled_set(tsdn, false); 653 for (unsigned i = 0; i < max_background_threads; i++) { 654 background_thread_info_t *info = &background_thread_info[i]; 655 malloc_mutex_lock(tsdn, &info->mtx); 656 info->state = background_thread_stopped; 657 int ret = pthread_cond_init(&info->cond, NULL); 658 assert(ret == 0); 659 background_thread_info_init(tsdn, info); 660 malloc_mutex_unlock(tsdn, &info->mtx); 661 } 662 malloc_mutex_unlock(tsdn, &background_thread_lock); 663 } 664 665 bool 666 background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { 667 assert(config_stats); 668 malloc_mutex_lock(tsdn, &background_thread_lock); 669 if (!background_thread_enabled()) { 670 malloc_mutex_unlock(tsdn, &background_thread_lock); 671 return true; 672 } 673 674 nstime_init_zero(&stats->run_interval); 675 memset(&stats->max_counter_per_bg_thd, 0, sizeof(mutex_prof_data_t)); 676 677 uint64_t num_runs = 0; 678 stats->num_threads = n_background_threads; 679 for (unsigned i = 0; i < max_background_threads; i++) { 680 background_thread_info_t *info = &background_thread_info[i]; 681 if (malloc_mutex_trylock(tsdn, &info->mtx)) { 682 /* 683 * Each background thread run may take a long time; 684 * avoid waiting on the stats if the thread is active. 685 */ 686 continue; 687 } 688 if (info->state != background_thread_stopped) { 689 num_runs += info->tot_n_runs; 690 nstime_add(&stats->run_interval, &info->tot_sleep_time); 691 malloc_mutex_prof_max_update(tsdn, 692 &stats->max_counter_per_bg_thd, &info->mtx); 693 } 694 malloc_mutex_unlock(tsdn, &info->mtx); 695 } 696 stats->num_runs = num_runs; 697 if (num_runs > 0) { 698 nstime_idivide(&stats->run_interval, num_runs); 699 } 700 malloc_mutex_unlock(tsdn, &background_thread_lock); 701 702 return false; 703 } 704 705 #undef BACKGROUND_THREAD_NPAGES_THRESHOLD 706 #undef BILLION 707 #undef BACKGROUND_THREAD_MIN_INTERVAL_NS 708 709 #ifdef JEMALLOC_HAVE_DLSYM 710 #include <dlfcn.h> 711 #endif 712 713 static bool 714 pthread_create_fptr_init(void) { 715 if (pthread_create_fptr != NULL) { 716 return false; 717 } 718 /* 719 * Try the next symbol first, because 1) when use lazy_lock we have a 720 * wrapper for pthread_create; and 2) application may define its own 721 * wrapper as well (and can call malloc within the wrapper). 722 */ 723 #ifdef JEMALLOC_HAVE_DLSYM 724 pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); 725 #else 726 pthread_create_fptr = NULL; 727 #endif 728 if (pthread_create_fptr == NULL) { 729 if (config_lazy_lock) { 730 malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, " 731 "\"pthread_create\")\n"); 732 abort(); 733 } else { 734 /* Fall back to the default symbol. */ 735 pthread_create_fptr = pthread_create; 736 } 737 } 738 739 return false; 740 } 741 742 /* 743 * When lazy lock is enabled, we need to make sure setting isthreaded before 744 * taking any background_thread locks. This is called early in ctl (instead of 745 * wait for the pthread_create calls to trigger) because the mutex is required 746 * before creating background threads. 747 */ 748 void 749 background_thread_ctl_init(tsdn_t *tsdn) { 750 malloc_mutex_assert_not_owner(tsdn, &background_thread_lock); 751 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER 752 pthread_create_fptr_init(); 753 pthread_create_wrapper_init(); 754 #endif 755 } 756 757 #endif /* defined(JEMALLOC_BACKGROUND_THREAD) */ 758 759 bool 760 background_thread_boot0(void) { 761 if (!have_background_thread && opt_background_thread) { 762 malloc_printf("<jemalloc>: option background_thread currently " 763 "supports pthread only\n"); 764 return true; 765 } 766 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER 767 if ((config_lazy_lock || opt_background_thread) && 768 pthread_create_fptr_init()) { 769 return true; 770 } 771 #endif 772 return false; 773 } 774 775 bool 776 background_thread_boot1(tsdn_t *tsdn, base_t *base) { 777 #ifdef JEMALLOC_BACKGROUND_THREAD 778 assert(have_background_thread); 779 assert(narenas_total_get() > 0); 780 781 if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) { 782 opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD; 783 } 784 max_background_threads = opt_max_background_threads; 785 786 background_thread_enabled_set(tsdn, opt_background_thread); 787 if (malloc_mutex_init(&background_thread_lock, 788 "background_thread_global", 789 WITNESS_RANK_BACKGROUND_THREAD_GLOBAL, 790 malloc_mutex_rank_exclusive)) { 791 return true; 792 } 793 794 background_thread_info = (background_thread_info_t *)base_alloc(tsdn, 795 base, opt_max_background_threads * 796 sizeof(background_thread_info_t), CACHELINE); 797 if (background_thread_info == NULL) { 798 return true; 799 } 800 801 for (unsigned i = 0; i < max_background_threads; i++) { 802 background_thread_info_t *info = &background_thread_info[i]; 803 /* Thread mutex is rank_inclusive because of thread0. */ 804 if (malloc_mutex_init(&info->mtx, "background_thread", 805 WITNESS_RANK_BACKGROUND_THREAD, 806 malloc_mutex_address_ordered)) { 807 return true; 808 } 809 if (pthread_cond_init(&info->cond, NULL)) { 810 return true; 811 } 812 malloc_mutex_lock(tsdn, &info->mtx); 813 info->state = background_thread_stopped; 814 background_thread_info_init(tsdn, info); 815 malloc_mutex_unlock(tsdn, &info->mtx); 816 } 817 #endif 818 819 return false; 820 } 821