1 #define JEMALLOC_BACKGROUND_THREAD_C_ 2 #include "jemalloc/internal/jemalloc_preamble.h" 3 #include "jemalloc/internal/jemalloc_internal_includes.h" 4 5 #include "jemalloc/internal/assert.h" 6 7 JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS 8 9 /******************************************************************************/ 10 /* Data. */ 11 12 /* This option should be opt-in only. */ 13 #define BACKGROUND_THREAD_DEFAULT false 14 /* Read-only after initialization. */ 15 bool opt_background_thread = BACKGROUND_THREAD_DEFAULT; 16 size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1; 17 18 /* Used for thread creation, termination and stats. */ 19 malloc_mutex_t background_thread_lock; 20 /* Indicates global state. Atomic because decay reads this w/o locking. */ 21 atomic_b_t background_thread_enabled_state; 22 size_t n_background_threads; 23 size_t max_background_threads; 24 /* Thread info per-index. */ 25 background_thread_info_t *background_thread_info; 26 27 /******************************************************************************/ 28 29 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER 30 31 static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, 32 void *(*)(void *), void *__restrict); 33 34 static void 35 pthread_create_wrapper_init(void) { 36 #ifdef JEMALLOC_LAZY_LOCK 37 if (!isthreaded) { 38 isthreaded = true; 39 } 40 #endif 41 } 42 43 int 44 pthread_create_wrapper(pthread_t *__restrict thread, const pthread_attr_t *attr, 45 void *(*start_routine)(void *), void *__restrict arg) { 46 pthread_create_wrapper_init(); 47 48 return pthread_create_fptr(thread, attr, start_routine, arg); 49 } 50 #endif /* JEMALLOC_PTHREAD_CREATE_WRAPPER */ 51 52 #ifndef JEMALLOC_BACKGROUND_THREAD 53 #define NOT_REACHED { not_reached(); } 54 bool background_thread_create(tsd_t *tsd, unsigned arena_ind) NOT_REACHED 55 bool background_threads_enable(tsd_t *tsd) NOT_REACHED 56 bool background_threads_disable(tsd_t *tsd) NOT_REACHED 57 void background_thread_interval_check(tsdn_t *tsdn, arena_t *arena, 58 arena_decay_t *decay, size_t npages_new) NOT_REACHED 59 void background_thread_prefork0(tsdn_t *tsdn) NOT_REACHED 60 void background_thread_prefork1(tsdn_t *tsdn) NOT_REACHED 61 void background_thread_postfork_parent(tsdn_t *tsdn) NOT_REACHED 62 void background_thread_postfork_child(tsdn_t *tsdn) NOT_REACHED 63 bool background_thread_stats_read(tsdn_t *tsdn, 64 background_thread_stats_t *stats) NOT_REACHED 65 void background_thread_ctl_init(tsdn_t *tsdn) NOT_REACHED 66 #undef NOT_REACHED 67 #else 68 69 static bool background_thread_enabled_at_fork; 70 71 static void 72 background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) { 73 background_thread_wakeup_time_set(tsdn, info, 0); 74 info->npages_to_purge_new = 0; 75 if (config_stats) { 76 info->tot_n_runs = 0; 77 nstime_init(&info->tot_sleep_time, 0); 78 } 79 } 80 81 static inline bool 82 set_current_thread_affinity(int cpu) { 83 #if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) 84 cpu_set_t cpuset; 85 CPU_ZERO(&cpuset); 86 CPU_SET(cpu, &cpuset); 87 int ret = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); 88 89 return (ret != 0); 90 #else 91 return false; 92 #endif 93 } 94 95 /* Threshold for determining when to wake up the background thread. */ 96 #define BACKGROUND_THREAD_NPAGES_THRESHOLD UINT64_C(1024) 97 #define BILLION UINT64_C(1000000000) 98 /* Minimal sleep interval 100 ms. */ 99 #define BACKGROUND_THREAD_MIN_INTERVAL_NS (BILLION / 10) 100 101 static inline size_t 102 decay_npurge_after_interval(arena_decay_t *decay, size_t interval) { 103 size_t i; 104 uint64_t sum = 0; 105 for (i = 0; i < interval; i++) { 106 sum += decay->backlog[i] * h_steps[i]; 107 } 108 for (; i < SMOOTHSTEP_NSTEPS; i++) { 109 sum += decay->backlog[i] * (h_steps[i] - h_steps[i - interval]); 110 } 111 112 return (size_t)(sum >> SMOOTHSTEP_BFP); 113 } 114 115 static uint64_t 116 arena_decay_compute_purge_interval_impl(tsdn_t *tsdn, arena_decay_t *decay, 117 extents_t *extents) { 118 if (malloc_mutex_trylock(tsdn, &decay->mtx)) { 119 /* Use minimal interval if decay is contended. */ 120 return BACKGROUND_THREAD_MIN_INTERVAL_NS; 121 } 122 123 uint64_t interval; 124 ssize_t decay_time = atomic_load_zd(&decay->time_ms, ATOMIC_RELAXED); 125 if (decay_time <= 0) { 126 /* Purging is eagerly done or disabled currently. */ 127 interval = BACKGROUND_THREAD_INDEFINITE_SLEEP; 128 goto label_done; 129 } 130 131 uint64_t decay_interval_ns = nstime_ns(&decay->interval); 132 assert(decay_interval_ns > 0); 133 size_t npages = extents_npages_get(extents); 134 if (npages == 0) { 135 unsigned i; 136 for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) { 137 if (decay->backlog[i] > 0) { 138 break; 139 } 140 } 141 if (i == SMOOTHSTEP_NSTEPS) { 142 /* No dirty pages recorded. Sleep indefinitely. */ 143 interval = BACKGROUND_THREAD_INDEFINITE_SLEEP; 144 goto label_done; 145 } 146 } 147 if (npages <= BACKGROUND_THREAD_NPAGES_THRESHOLD) { 148 /* Use max interval. */ 149 interval = decay_interval_ns * SMOOTHSTEP_NSTEPS; 150 goto label_done; 151 } 152 153 size_t lb = BACKGROUND_THREAD_MIN_INTERVAL_NS / decay_interval_ns; 154 size_t ub = SMOOTHSTEP_NSTEPS; 155 /* Minimal 2 intervals to ensure reaching next epoch deadline. */ 156 lb = (lb < 2) ? 2 : lb; 157 if ((decay_interval_ns * ub <= BACKGROUND_THREAD_MIN_INTERVAL_NS) || 158 (lb + 2 > ub)) { 159 interval = BACKGROUND_THREAD_MIN_INTERVAL_NS; 160 goto label_done; 161 } 162 163 assert(lb + 2 <= ub); 164 size_t npurge_lb, npurge_ub; 165 npurge_lb = decay_npurge_after_interval(decay, lb); 166 if (npurge_lb > BACKGROUND_THREAD_NPAGES_THRESHOLD) { 167 interval = decay_interval_ns * lb; 168 goto label_done; 169 } 170 npurge_ub = decay_npurge_after_interval(decay, ub); 171 if (npurge_ub < BACKGROUND_THREAD_NPAGES_THRESHOLD) { 172 interval = decay_interval_ns * ub; 173 goto label_done; 174 } 175 176 unsigned n_search = 0; 177 size_t target, npurge; 178 while ((npurge_lb + BACKGROUND_THREAD_NPAGES_THRESHOLD < npurge_ub) 179 && (lb + 2 < ub)) { 180 target = (lb + ub) / 2; 181 npurge = decay_npurge_after_interval(decay, target); 182 if (npurge > BACKGROUND_THREAD_NPAGES_THRESHOLD) { 183 ub = target; 184 npurge_ub = npurge; 185 } else { 186 lb = target; 187 npurge_lb = npurge; 188 } 189 assert(n_search++ < lg_floor(SMOOTHSTEP_NSTEPS) + 1); 190 } 191 interval = decay_interval_ns * (ub + lb) / 2; 192 label_done: 193 interval = (interval < BACKGROUND_THREAD_MIN_INTERVAL_NS) ? 194 BACKGROUND_THREAD_MIN_INTERVAL_NS : interval; 195 malloc_mutex_unlock(tsdn, &decay->mtx); 196 197 return interval; 198 } 199 200 /* Compute purge interval for background threads. */ 201 static uint64_t 202 arena_decay_compute_purge_interval(tsdn_t *tsdn, arena_t *arena) { 203 uint64_t i1, i2; 204 i1 = arena_decay_compute_purge_interval_impl(tsdn, &arena->decay_dirty, 205 &arena->extents_dirty); 206 if (i1 == BACKGROUND_THREAD_MIN_INTERVAL_NS) { 207 return i1; 208 } 209 i2 = arena_decay_compute_purge_interval_impl(tsdn, &arena->decay_muzzy, 210 &arena->extents_muzzy); 211 212 return i1 < i2 ? i1 : i2; 213 } 214 215 static void 216 background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info, 217 uint64_t interval) { 218 if (config_stats) { 219 info->tot_n_runs++; 220 } 221 info->npages_to_purge_new = 0; 222 223 struct timeval tv; 224 /* Specific clock required by timedwait. */ 225 gettimeofday(&tv, NULL); 226 nstime_t before_sleep; 227 nstime_init2(&before_sleep, tv.tv_sec, tv.tv_usec * 1000); 228 229 int ret; 230 if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) { 231 assert(background_thread_indefinite_sleep(info)); 232 ret = pthread_cond_wait(&info->cond, &info->mtx.lock); 233 assert(ret == 0); 234 } else { 235 assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS && 236 interval <= BACKGROUND_THREAD_INDEFINITE_SLEEP); 237 /* We need malloc clock (can be different from tv). */ 238 nstime_t next_wakeup; 239 nstime_init(&next_wakeup, 0); 240 nstime_update(&next_wakeup); 241 nstime_iadd(&next_wakeup, interval); 242 assert(nstime_ns(&next_wakeup) < 243 BACKGROUND_THREAD_INDEFINITE_SLEEP); 244 background_thread_wakeup_time_set(tsdn, info, 245 nstime_ns(&next_wakeup)); 246 247 nstime_t ts_wakeup; 248 nstime_copy(&ts_wakeup, &before_sleep); 249 nstime_iadd(&ts_wakeup, interval); 250 struct timespec ts; 251 ts.tv_sec = (size_t)nstime_sec(&ts_wakeup); 252 ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup); 253 254 assert(!background_thread_indefinite_sleep(info)); 255 ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, &ts); 256 assert(ret == ETIMEDOUT || ret == 0); 257 background_thread_wakeup_time_set(tsdn, info, 258 BACKGROUND_THREAD_INDEFINITE_SLEEP); 259 } 260 if (config_stats) { 261 gettimeofday(&tv, NULL); 262 nstime_t after_sleep; 263 nstime_init2(&after_sleep, tv.tv_sec, tv.tv_usec * 1000); 264 if (nstime_compare(&after_sleep, &before_sleep) > 0) { 265 nstime_subtract(&after_sleep, &before_sleep); 266 nstime_add(&info->tot_sleep_time, &after_sleep); 267 } 268 } 269 } 270 271 static bool 272 background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) { 273 if (unlikely(info->state == background_thread_paused)) { 274 malloc_mutex_unlock(tsdn, &info->mtx); 275 /* Wait on global lock to update status. */ 276 malloc_mutex_lock(tsdn, &background_thread_lock); 277 malloc_mutex_unlock(tsdn, &background_thread_lock); 278 malloc_mutex_lock(tsdn, &info->mtx); 279 return true; 280 } 281 282 return false; 283 } 284 285 static inline void 286 background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info, unsigned ind) { 287 uint64_t min_interval = BACKGROUND_THREAD_INDEFINITE_SLEEP; 288 unsigned narenas = narenas_total_get(); 289 290 for (unsigned i = ind; i < narenas; i += max_background_threads) { 291 arena_t *arena = arena_get(tsdn, i, false); 292 if (!arena) { 293 continue; 294 } 295 arena_decay(tsdn, arena, true, false); 296 if (min_interval == BACKGROUND_THREAD_MIN_INTERVAL_NS) { 297 /* Min interval will be used. */ 298 continue; 299 } 300 uint64_t interval = arena_decay_compute_purge_interval(tsdn, 301 arena); 302 assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS); 303 if (min_interval > interval) { 304 min_interval = interval; 305 } 306 } 307 background_thread_sleep(tsdn, info, min_interval); 308 } 309 310 static bool 311 background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) { 312 if (info == &background_thread_info[0]) { 313 malloc_mutex_assert_owner(tsd_tsdn(tsd), 314 &background_thread_lock); 315 } else { 316 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), 317 &background_thread_lock); 318 } 319 320 pre_reentrancy(tsd, NULL); 321 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 322 bool has_thread; 323 assert(info->state != background_thread_paused); 324 if (info->state == background_thread_started) { 325 has_thread = true; 326 info->state = background_thread_stopped; 327 pthread_cond_signal(&info->cond); 328 } else { 329 has_thread = false; 330 } 331 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 332 333 if (!has_thread) { 334 post_reentrancy(tsd); 335 return false; 336 } 337 void *ret; 338 if (pthread_join(info->thread, &ret)) { 339 post_reentrancy(tsd); 340 return true; 341 } 342 assert(ret == NULL); 343 n_background_threads--; 344 post_reentrancy(tsd); 345 346 return false; 347 } 348 349 static void *background_thread_entry(void *ind_arg); 350 351 static int 352 background_thread_create_signals_masked(pthread_t *thread, 353 const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { 354 /* 355 * Mask signals during thread creation so that the thread inherits 356 * an empty signal set. 357 */ 358 sigset_t set; 359 sigfillset(&set); 360 sigset_t oldset; 361 int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset); 362 if (mask_err != 0) { 363 return mask_err; 364 } 365 int create_err = pthread_create_wrapper(thread, attr, start_routine, 366 arg); 367 /* 368 * Restore the signal mask. Failure to restore the signal mask here 369 * changes program behavior. 370 */ 371 int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL); 372 if (restore_err != 0) { 373 malloc_printf("<jemalloc>: background thread creation " 374 "failed (%d), and signal mask restoration failed " 375 "(%d)\n", create_err, restore_err); 376 if (opt_abort) { 377 abort(); 378 } 379 } 380 return create_err; 381 } 382 383 static bool 384 check_background_thread_creation(tsd_t *tsd, unsigned *n_created, 385 bool *created_threads) { 386 bool ret = false; 387 if (likely(*n_created == n_background_threads)) { 388 return ret; 389 } 390 391 tsdn_t *tsdn = tsd_tsdn(tsd); 392 malloc_mutex_unlock(tsdn, &background_thread_info[0].mtx); 393 for (unsigned i = 1; i < max_background_threads; i++) { 394 if (created_threads[i]) { 395 continue; 396 } 397 background_thread_info_t *info = &background_thread_info[i]; 398 malloc_mutex_lock(tsdn, &info->mtx); 399 /* 400 * In case of the background_thread_paused state because of 401 * arena reset, delay the creation. 402 */ 403 bool create = (info->state == background_thread_started); 404 malloc_mutex_unlock(tsdn, &info->mtx); 405 if (!create) { 406 continue; 407 } 408 409 pre_reentrancy(tsd, NULL); 410 int err = background_thread_create_signals_masked(&info->thread, 411 NULL, background_thread_entry, (void *)(uintptr_t)i); 412 post_reentrancy(tsd); 413 414 if (err == 0) { 415 (*n_created)++; 416 created_threads[i] = true; 417 } else { 418 malloc_printf("<jemalloc>: background thread " 419 "creation failed (%d)\n", err); 420 if (opt_abort) { 421 abort(); 422 } 423 } 424 /* Return to restart the loop since we unlocked. */ 425 ret = true; 426 break; 427 } 428 malloc_mutex_lock(tsdn, &background_thread_info[0].mtx); 429 430 return ret; 431 } 432 433 static void 434 background_thread0_work(tsd_t *tsd) { 435 /* Thread0 is also responsible for launching / terminating threads. */ 436 VARIABLE_ARRAY(bool, created_threads, max_background_threads); 437 unsigned i; 438 for (i = 1; i < max_background_threads; i++) { 439 created_threads[i] = false; 440 } 441 /* Start working, and create more threads when asked. */ 442 unsigned n_created = 1; 443 while (background_thread_info[0].state != background_thread_stopped) { 444 if (background_thread_pause_check(tsd_tsdn(tsd), 445 &background_thread_info[0])) { 446 continue; 447 } 448 if (check_background_thread_creation(tsd, &n_created, 449 (bool *)&created_threads)) { 450 continue; 451 } 452 background_work_sleep_once(tsd_tsdn(tsd), 453 &background_thread_info[0], 0); 454 } 455 456 /* 457 * Shut down other threads at exit. Note that the ctl thread is holding 458 * the global background_thread mutex (and is waiting) for us. 459 */ 460 assert(!background_thread_enabled()); 461 for (i = 1; i < max_background_threads; i++) { 462 background_thread_info_t *info = &background_thread_info[i]; 463 assert(info->state != background_thread_paused); 464 if (created_threads[i]) { 465 background_threads_disable_single(tsd, info); 466 } else { 467 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 468 if (info->state != background_thread_stopped) { 469 /* The thread was not created. */ 470 assert(info->state == 471 background_thread_started); 472 n_background_threads--; 473 info->state = background_thread_stopped; 474 } 475 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 476 } 477 } 478 background_thread_info[0].state = background_thread_stopped; 479 assert(n_background_threads == 1); 480 } 481 482 static void 483 background_work(tsd_t *tsd, unsigned ind) { 484 background_thread_info_t *info = &background_thread_info[ind]; 485 486 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 487 background_thread_wakeup_time_set(tsd_tsdn(tsd), info, 488 BACKGROUND_THREAD_INDEFINITE_SLEEP); 489 if (ind == 0) { 490 background_thread0_work(tsd); 491 } else { 492 while (info->state != background_thread_stopped) { 493 if (background_thread_pause_check(tsd_tsdn(tsd), 494 info)) { 495 continue; 496 } 497 background_work_sleep_once(tsd_tsdn(tsd), info, ind); 498 } 499 } 500 assert(info->state == background_thread_stopped); 501 background_thread_wakeup_time_set(tsd_tsdn(tsd), info, 0); 502 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 503 } 504 505 static void * 506 background_thread_entry(void *ind_arg) { 507 unsigned thread_ind = (unsigned)(uintptr_t)ind_arg; 508 assert(thread_ind < max_background_threads); 509 #ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP 510 pthread_setname_np(pthread_self(), "jemalloc_bg_thd"); 511 #elif defined(__FreeBSD__) 512 pthread_set_name_np(pthread_self(), "jemalloc_bg_thd"); 513 #endif 514 if (opt_percpu_arena != percpu_arena_disabled) { 515 set_current_thread_affinity((int)thread_ind); 516 } 517 /* 518 * Start periodic background work. We use internal tsd which avoids 519 * side effects, for example triggering new arena creation (which in 520 * turn triggers another background thread creation). 521 */ 522 background_work(tsd_internal_fetch(), thread_ind); 523 assert(pthread_equal(pthread_self(), 524 background_thread_info[thread_ind].thread)); 525 526 return NULL; 527 } 528 529 static void 530 background_thread_init(tsd_t *tsd, background_thread_info_t *info) { 531 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); 532 info->state = background_thread_started; 533 background_thread_info_init(tsd_tsdn(tsd), info); 534 n_background_threads++; 535 } 536 537 static bool 538 background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { 539 assert(have_background_thread); 540 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); 541 542 /* We create at most NCPUs threads. */ 543 size_t thread_ind = arena_ind % max_background_threads; 544 background_thread_info_t *info = &background_thread_info[thread_ind]; 545 546 bool need_new_thread; 547 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 548 need_new_thread = background_thread_enabled() && 549 (info->state == background_thread_stopped); 550 if (need_new_thread) { 551 background_thread_init(tsd, info); 552 } 553 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 554 if (!need_new_thread) { 555 return false; 556 } 557 if (arena_ind != 0) { 558 /* Threads are created asynchronously by Thread 0. */ 559 background_thread_info_t *t0 = &background_thread_info[0]; 560 malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx); 561 assert(t0->state == background_thread_started); 562 pthread_cond_signal(&t0->cond); 563 malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx); 564 565 return false; 566 } 567 568 pre_reentrancy(tsd, NULL); 569 /* 570 * To avoid complications (besides reentrancy), create internal 571 * background threads with the underlying pthread_create. 572 */ 573 int err = background_thread_create_signals_masked(&info->thread, NULL, 574 background_thread_entry, (void *)thread_ind); 575 post_reentrancy(tsd); 576 577 if (err != 0) { 578 malloc_printf("<jemalloc>: arena 0 background thread creation " 579 "failed (%d)\n", err); 580 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 581 info->state = background_thread_stopped; 582 n_background_threads--; 583 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 584 585 return true; 586 } 587 588 return false; 589 } 590 591 /* Create a new background thread if needed. */ 592 bool 593 background_thread_create(tsd_t *tsd, unsigned arena_ind) { 594 assert(have_background_thread); 595 596 bool ret; 597 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 598 ret = background_thread_create_locked(tsd, arena_ind); 599 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 600 601 return ret; 602 } 603 604 bool 605 background_threads_enable(tsd_t *tsd) { 606 assert(n_background_threads == 0); 607 assert(background_thread_enabled()); 608 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); 609 610 VARIABLE_ARRAY(bool, marked, max_background_threads); 611 unsigned i, nmarked; 612 for (i = 0; i < max_background_threads; i++) { 613 marked[i] = false; 614 } 615 nmarked = 0; 616 /* Thread 0 is required and created at the end. */ 617 marked[0] = true; 618 /* Mark the threads we need to create for thread 0. */ 619 unsigned n = narenas_total_get(); 620 for (i = 1; i < n; i++) { 621 if (marked[i % max_background_threads] || 622 arena_get(tsd_tsdn(tsd), i, false) == NULL) { 623 continue; 624 } 625 background_thread_info_t *info = &background_thread_info[ 626 i % max_background_threads]; 627 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 628 assert(info->state == background_thread_stopped); 629 background_thread_init(tsd, info); 630 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 631 marked[i % max_background_threads] = true; 632 if (++nmarked == max_background_threads) { 633 break; 634 } 635 } 636 637 return background_thread_create_locked(tsd, 0); 638 } 639 640 bool 641 background_threads_disable(tsd_t *tsd) { 642 assert(!background_thread_enabled()); 643 malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); 644 645 /* Thread 0 will be responsible for terminating other threads. */ 646 if (background_threads_disable_single(tsd, 647 &background_thread_info[0])) { 648 return true; 649 } 650 assert(n_background_threads == 0); 651 652 return false; 653 } 654 655 /* Check if we need to signal the background thread early. */ 656 void 657 background_thread_interval_check(tsdn_t *tsdn, arena_t *arena, 658 arena_decay_t *decay, size_t npages_new) { 659 background_thread_info_t *info = arena_background_thread_info_get( 660 arena); 661 if (malloc_mutex_trylock(tsdn, &info->mtx)) { 662 /* 663 * Background thread may hold the mutex for a long period of 664 * time. We'd like to avoid the variance on application 665 * threads. So keep this non-blocking, and leave the work to a 666 * future epoch. 667 */ 668 return; 669 } 670 671 if (info->state != background_thread_started) { 672 goto label_done; 673 } 674 if (malloc_mutex_trylock(tsdn, &decay->mtx)) { 675 goto label_done; 676 } 677 678 ssize_t decay_time = atomic_load_zd(&decay->time_ms, ATOMIC_RELAXED); 679 if (decay_time <= 0) { 680 /* Purging is eagerly done or disabled currently. */ 681 goto label_done_unlock2; 682 } 683 uint64_t decay_interval_ns = nstime_ns(&decay->interval); 684 assert(decay_interval_ns > 0); 685 686 nstime_t diff; 687 nstime_init(&diff, background_thread_wakeup_time_get(info)); 688 if (nstime_compare(&diff, &decay->epoch) <= 0) { 689 goto label_done_unlock2; 690 } 691 nstime_subtract(&diff, &decay->epoch); 692 if (nstime_ns(&diff) < BACKGROUND_THREAD_MIN_INTERVAL_NS) { 693 goto label_done_unlock2; 694 } 695 696 if (npages_new > 0) { 697 size_t n_epoch = (size_t)(nstime_ns(&diff) / decay_interval_ns); 698 /* 699 * Compute how many new pages we would need to purge by the next 700 * wakeup, which is used to determine if we should signal the 701 * background thread. 702 */ 703 uint64_t npurge_new; 704 if (n_epoch >= SMOOTHSTEP_NSTEPS) { 705 npurge_new = npages_new; 706 } else { 707 uint64_t h_steps_max = h_steps[SMOOTHSTEP_NSTEPS - 1]; 708 assert(h_steps_max >= 709 h_steps[SMOOTHSTEP_NSTEPS - 1 - n_epoch]); 710 npurge_new = npages_new * (h_steps_max - 711 h_steps[SMOOTHSTEP_NSTEPS - 1 - n_epoch]); 712 npurge_new >>= SMOOTHSTEP_BFP; 713 } 714 info->npages_to_purge_new += npurge_new; 715 } 716 717 bool should_signal; 718 if (info->npages_to_purge_new > BACKGROUND_THREAD_NPAGES_THRESHOLD) { 719 should_signal = true; 720 } else if (unlikely(background_thread_indefinite_sleep(info)) && 721 (extents_npages_get(&arena->extents_dirty) > 0 || 722 extents_npages_get(&arena->extents_muzzy) > 0 || 723 info->npages_to_purge_new > 0)) { 724 should_signal = true; 725 } else { 726 should_signal = false; 727 } 728 729 if (should_signal) { 730 info->npages_to_purge_new = 0; 731 pthread_cond_signal(&info->cond); 732 } 733 label_done_unlock2: 734 malloc_mutex_unlock(tsdn, &decay->mtx); 735 label_done: 736 malloc_mutex_unlock(tsdn, &info->mtx); 737 } 738 739 void 740 background_thread_prefork0(tsdn_t *tsdn) { 741 malloc_mutex_prefork(tsdn, &background_thread_lock); 742 background_thread_enabled_at_fork = background_thread_enabled(); 743 } 744 745 void 746 background_thread_prefork1(tsdn_t *tsdn) { 747 for (unsigned i = 0; i < max_background_threads; i++) { 748 malloc_mutex_prefork(tsdn, &background_thread_info[i].mtx); 749 } 750 } 751 752 void 753 background_thread_postfork_parent(tsdn_t *tsdn) { 754 for (unsigned i = 0; i < max_background_threads; i++) { 755 malloc_mutex_postfork_parent(tsdn, 756 &background_thread_info[i].mtx); 757 } 758 malloc_mutex_postfork_parent(tsdn, &background_thread_lock); 759 } 760 761 void 762 background_thread_postfork_child(tsdn_t *tsdn) { 763 for (unsigned i = 0; i < max_background_threads; i++) { 764 malloc_mutex_postfork_child(tsdn, 765 &background_thread_info[i].mtx); 766 } 767 malloc_mutex_postfork_child(tsdn, &background_thread_lock); 768 if (!background_thread_enabled_at_fork) { 769 return; 770 } 771 772 /* Clear background_thread state (reset to disabled for child). */ 773 malloc_mutex_lock(tsdn, &background_thread_lock); 774 n_background_threads = 0; 775 background_thread_enabled_set(tsdn, false); 776 for (unsigned i = 0; i < max_background_threads; i++) { 777 background_thread_info_t *info = &background_thread_info[i]; 778 malloc_mutex_lock(tsdn, &info->mtx); 779 info->state = background_thread_stopped; 780 int ret = pthread_cond_init(&info->cond, NULL); 781 assert(ret == 0); 782 background_thread_info_init(tsdn, info); 783 malloc_mutex_unlock(tsdn, &info->mtx); 784 } 785 malloc_mutex_unlock(tsdn, &background_thread_lock); 786 } 787 788 bool 789 background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { 790 assert(config_stats); 791 malloc_mutex_lock(tsdn, &background_thread_lock); 792 if (!background_thread_enabled()) { 793 malloc_mutex_unlock(tsdn, &background_thread_lock); 794 return true; 795 } 796 797 stats->num_threads = n_background_threads; 798 uint64_t num_runs = 0; 799 nstime_init(&stats->run_interval, 0); 800 for (unsigned i = 0; i < max_background_threads; i++) { 801 background_thread_info_t *info = &background_thread_info[i]; 802 if (malloc_mutex_trylock(tsdn, &info->mtx)) { 803 /* 804 * Each background thread run may take a long time; 805 * avoid waiting on the stats if the thread is active. 806 */ 807 continue; 808 } 809 if (info->state != background_thread_stopped) { 810 num_runs += info->tot_n_runs; 811 nstime_add(&stats->run_interval, &info->tot_sleep_time); 812 } 813 malloc_mutex_unlock(tsdn, &info->mtx); 814 } 815 stats->num_runs = num_runs; 816 if (num_runs > 0) { 817 nstime_idivide(&stats->run_interval, num_runs); 818 } 819 malloc_mutex_unlock(tsdn, &background_thread_lock); 820 821 return false; 822 } 823 824 #undef BACKGROUND_THREAD_NPAGES_THRESHOLD 825 #undef BILLION 826 #undef BACKGROUND_THREAD_MIN_INTERVAL_NS 827 828 #ifdef JEMALLOC_HAVE_DLSYM 829 #include <dlfcn.h> 830 #endif 831 832 static bool 833 pthread_create_fptr_init(void) { 834 if (pthread_create_fptr != NULL) { 835 return false; 836 } 837 /* 838 * Try the next symbol first, because 1) when use lazy_lock we have a 839 * wrapper for pthread_create; and 2) application may define its own 840 * wrapper as well (and can call malloc within the wrapper). 841 */ 842 #ifdef JEMALLOC_HAVE_DLSYM 843 pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); 844 #else 845 pthread_create_fptr = NULL; 846 #endif 847 if (pthread_create_fptr == NULL) { 848 if (config_lazy_lock) { 849 malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, " 850 "\"pthread_create\")\n"); 851 abort(); 852 } else { 853 /* Fall back to the default symbol. */ 854 pthread_create_fptr = pthread_create; 855 } 856 } 857 858 return false; 859 } 860 861 /* 862 * When lazy lock is enabled, we need to make sure setting isthreaded before 863 * taking any background_thread locks. This is called early in ctl (instead of 864 * wait for the pthread_create calls to trigger) because the mutex is required 865 * before creating background threads. 866 */ 867 void 868 background_thread_ctl_init(tsdn_t *tsdn) { 869 malloc_mutex_assert_not_owner(tsdn, &background_thread_lock); 870 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER 871 pthread_create_fptr_init(); 872 pthread_create_wrapper_init(); 873 #endif 874 } 875 876 #endif /* defined(JEMALLOC_BACKGROUND_THREAD) */ 877 878 bool 879 background_thread_boot0(void) { 880 if (!have_background_thread && opt_background_thread) { 881 malloc_printf("<jemalloc>: option background_thread currently " 882 "supports pthread only\n"); 883 return true; 884 } 885 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER 886 if ((config_lazy_lock || opt_background_thread) && 887 pthread_create_fptr_init()) { 888 return true; 889 } 890 #endif 891 return false; 892 } 893 894 bool 895 background_thread_boot1(tsdn_t *tsdn) { 896 #ifdef JEMALLOC_BACKGROUND_THREAD 897 assert(have_background_thread); 898 assert(narenas_total_get() > 0); 899 900 if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) { 901 opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD; 902 } 903 max_background_threads = opt_max_background_threads; 904 905 background_thread_enabled_set(tsdn, opt_background_thread); 906 if (malloc_mutex_init(&background_thread_lock, 907 "background_thread_global", 908 WITNESS_RANK_BACKGROUND_THREAD_GLOBAL, 909 malloc_mutex_rank_exclusive)) { 910 return true; 911 } 912 913 background_thread_info = (background_thread_info_t *)base_alloc(tsdn, 914 b0get(), opt_max_background_threads * 915 sizeof(background_thread_info_t), CACHELINE); 916 if (background_thread_info == NULL) { 917 return true; 918 } 919 920 for (unsigned i = 0; i < max_background_threads; i++) { 921 background_thread_info_t *info = &background_thread_info[i]; 922 /* Thread mutex is rank_inclusive because of thread0. */ 923 if (malloc_mutex_init(&info->mtx, "background_thread", 924 WITNESS_RANK_BACKGROUND_THREAD, 925 malloc_mutex_address_ordered)) { 926 return true; 927 } 928 if (pthread_cond_init(&info->cond, NULL)) { 929 return true; 930 } 931 malloc_mutex_lock(tsdn, &info->mtx); 932 info->state = background_thread_stopped; 933 background_thread_info_init(tsdn, info); 934 malloc_mutex_unlock(tsdn, &info->mtx); 935 } 936 #endif 937 938 return false; 939 } 940