58     arena_decay_t *decay, extents_t *extents, bool all, size_t npages_limit,
519 arena_decay_ms_read(arena_decay_t *decay) {  in arena_decay_ms_read()  argument
520 	return atomic_load_zd(&decay->time_ms, ATOMIC_RELAXED);  in arena_decay_ms_read()
524 arena_decay_ms_write(arena_decay_t *decay, ssize_t decay_ms) {  in arena_decay_ms_write()  argument
525 	atomic_store_zd(&decay->time_ms, decay_ms, ATOMIC_RELAXED);  in arena_decay_ms_write()
529 arena_decay_deadline_init(arena_decay_t *decay) {  in arena_decay_deadline_init()  argument
534 	nstime_copy(&decay->deadline, &decay->epoch);  in arena_decay_deadline_init()
535 	nstime_add(&decay->deadline, &decay->interval);  in arena_decay_deadline_init()
536 	if (arena_decay_ms_read(decay) > 0) {  in arena_decay_deadline_init()
539 		nstime_init(&jitter, prng_range_u64(&decay->jitter_state,  in arena_decay_deadline_init()
540 		    nstime_ns(&decay->interval)));  in arena_decay_deadline_init()
541 		nstime_add(&decay->deadline, &jitter);  in arena_decay_deadline_init()
546 arena_decay_deadline_reached(const arena_decay_t *decay, const nstime_t *time) {  in arena_decay_deadline_reached()  argument
547 	return (nstime_compare(&decay->deadline, time) <= 0);  in arena_decay_deadline_reached()
551 arena_decay_backlog_npages_limit(const arena_decay_t *decay) {  in arena_decay_backlog_npages_limit()  argument
563 		sum += decay->backlog[i] * h_steps[i];  in arena_decay_backlog_npages_limit()
571 arena_decay_backlog_update_last(arena_decay_t *decay, size_t current_npages) {  in arena_decay_backlog_update_last()  argument
572 	size_t npages_delta = (current_npages > decay->nunpurged) ?  in arena_decay_backlog_update_last()
573 	    current_npages - decay->nunpurged : 0;  in arena_decay_backlog_update_last()
574 	decay->backlog[SMOOTHSTEP_NSTEPS-1] = npages_delta;  in arena_decay_backlog_update_last()
577 		if (current_npages > decay->ceil_npages) {  in arena_decay_backlog_update_last()
578 			decay->ceil_npages = current_npages;  in arena_decay_backlog_update_last()
580 		size_t npages_limit = arena_decay_backlog_npages_limit(decay);  in arena_decay_backlog_update_last()
581 		assert(decay->ceil_npages >= npages_limit);  in arena_decay_backlog_update_last()
582 		if (decay->ceil_npages > npages_limit) {  in arena_decay_backlog_update_last()
583 			decay->ceil_npages = npages_limit;  in arena_decay_backlog_update_last()
589 arena_decay_backlog_update(arena_decay_t *decay, uint64_t nadvance_u64,  in arena_decay_backlog_update()  argument
592 		memset(decay->backlog, 0, (SMOOTHSTEP_NSTEPS-1) *  in arena_decay_backlog_update()
599 		memmove(decay->backlog, &decay->backlog[nadvance_z],  in arena_decay_backlog_update()
602 			memset(&decay->backlog[SMOOTHSTEP_NSTEPS -  in arena_decay_backlog_update()
607 	arena_decay_backlog_update_last(decay, current_npages);  in arena_decay_backlog_update()
611 arena_decay_try_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,  in arena_decay_try_purge()  argument
615 		arena_decay_to_limit(tsdn, arena, decay, extents, false,  in arena_decay_try_purge()
622 arena_decay_epoch_advance_helper(arena_decay_t *decay, const nstime_t *time,  in arena_decay_epoch_advance_helper()  argument
624 	assert(arena_decay_deadline_reached(decay, time));  in arena_decay_epoch_advance_helper()
628 	nstime_subtract(&delta, &decay->epoch);  in arena_decay_epoch_advance_helper()
630 	uint64_t nadvance_u64 = nstime_divide(&delta, &decay->interval);  in arena_decay_epoch_advance_helper()
634 	nstime_copy(&delta, &decay->interval);  in arena_decay_epoch_advance_helper()
636 	nstime_add(&decay->epoch, &delta);  in arena_decay_epoch_advance_helper()
639 	arena_decay_deadline_init(decay);  in arena_decay_epoch_advance_helper()
642 	arena_decay_backlog_update(decay, nadvance_u64, current_npages);  in arena_decay_epoch_advance_helper()
646 arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,  in arena_decay_epoch_advance()  argument
649 	arena_decay_epoch_advance_helper(decay, time, current_npages);  in arena_decay_epoch_advance()
651 	size_t npages_limit = arena_decay_backlog_npages_limit(decay);  in arena_decay_epoch_advance()
653 	decay->nunpurged = (npages_limit > current_npages) ? npages_limit :  in arena_decay_epoch_advance()
657 		arena_decay_try_purge(tsdn, arena, decay, extents,  in arena_decay_epoch_advance()
663 arena_decay_reinit(arena_decay_t *decay, ssize_t decay_ms) {  in arena_decay_reinit()  argument
664 	arena_decay_ms_write(decay, decay_ms);  in arena_decay_reinit()
666 		nstime_init(&decay->interval, (uint64_t)decay_ms *  in arena_decay_reinit()
668 		nstime_idivide(&decay->interval, SMOOTHSTEP_NSTEPS);  in arena_decay_reinit()
671 	nstime_init(&decay->epoch, 0);  in arena_decay_reinit()
672 	nstime_update(&decay->epoch);  in arena_decay_reinit()
673 	decay->jitter_state = (uint64_t)(uintptr_t)decay;  in arena_decay_reinit()
674 	arena_decay_deadline_init(decay);  in arena_decay_reinit()
675 	decay->nunpurged = 0;  in arena_decay_reinit()
676 	memset(decay->backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t));  in arena_decay_reinit()
680 arena_decay_init(arena_decay_t *decay, ssize_t decay_ms,  in arena_decay_init()  argument
684 			assert(((char *)decay)[i] == 0);  in arena_decay_init()
686 		decay->ceil_npages = 0;  in arena_decay_init()
688 	if (malloc_mutex_init(&decay->mtx, "decay", WITNESS_RANK_DECAY,  in arena_decay_init()
692 	decay->purging = false;  in arena_decay_init()
693 	arena_decay_reinit(decay, decay_ms);  in arena_decay_init()
696 		decay->stats = stats;  in arena_decay_init()
714 arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,  in arena_maybe_decay()  argument
716 	malloc_mutex_assert_owner(tsdn, &decay->mtx);  in arena_maybe_decay()
719 	ssize_t decay_ms = arena_decay_ms_read(decay);  in arena_maybe_decay()
722 			arena_decay_to_limit(tsdn, arena, decay, extents, false,  in arena_maybe_decay()
732 	if (unlikely(!nstime_monotonic() && nstime_compare(&decay->epoch, &time)  in arena_maybe_decay()
743 		nstime_copy(&decay->epoch, &time);  in arena_maybe_decay()
744 		arena_decay_deadline_init(decay);  in arena_maybe_decay()
747 		assert(nstime_compare(&decay->epoch, &time) <= 0);  in arena_maybe_decay()
757 	bool advance_epoch = arena_decay_deadline_reached(decay, &time);  in arena_maybe_decay()
759 		arena_decay_epoch_advance(tsdn, arena, decay, extents, &time,  in arena_maybe_decay()
762 		arena_decay_try_purge(tsdn, arena, decay, extents,  in arena_maybe_decay()
764 		    arena_decay_backlog_npages_limit(decay),  in arena_maybe_decay()
772 arena_decay_ms_get(arena_decay_t *decay) {  in arena_decay_ms_get()  argument
773 	return arena_decay_ms_read(decay);  in arena_decay_ms_get()
787 arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,  in arena_decay_ms_set()  argument
793 	malloc_mutex_lock(tsdn, &decay->mtx);  in arena_decay_ms_set()
802 	arena_decay_reinit(decay, decay_ms);  in arena_decay_ms_set()
803 	arena_maybe_decay(tsdn, arena, decay, extents, false);  in arena_decay_ms_set()
804 	malloc_mutex_unlock(tsdn, &decay->mtx);  in arena_decay_ms_set()
844     extent_hooks_t **r_extent_hooks, arena_decay_t *decay, extents_t *extents,  in arena_decay_stashed()  argument
894 		arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->npurge,  in arena_decay_stashed()
897 		    &decay->stats->nmadvise, nmadvise);  in arena_decay_stashed()
898 		arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->purged,  in arena_decay_stashed()
916 arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,  in arena_decay_to_limit()  argument
921 	malloc_mutex_assert_owner(tsdn, &decay->mtx);  in arena_decay_to_limit()
923 	if (decay->purging) {  in arena_decay_to_limit()
926 	decay->purging = true;  in arena_decay_to_limit()
927 	malloc_mutex_unlock(tsdn, &decay->mtx);  in arena_decay_to_limit()
938 		    &extent_hooks, decay, extents, all, &decay_extents,  in arena_decay_to_limit()
943 	malloc_mutex_lock(tsdn, &decay->mtx);  in arena_decay_to_limit()
944 	decay->purging = false;  in arena_decay_to_limit()
948 arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,  in arena_decay_impl()  argument
951 		malloc_mutex_lock(tsdn, &decay->mtx);  in arena_decay_impl()
952 		arena_decay_to_limit(tsdn, arena, decay, extents, all, 0,  in arena_decay_impl()
954 		malloc_mutex_unlock(tsdn, &decay->mtx);  in arena_decay_impl()
959 	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {  in arena_decay_impl()
964 	bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, extents,  in arena_decay_impl()
969 		npages_new = decay->backlog[SMOOTHSTEP_NSTEPS-1];  in arena_decay_impl()
971 	malloc_mutex_unlock(tsdn, &decay->mtx);  in arena_decay_impl()
975 		background_thread_interval_check(tsdn, arena, decay,  in arena_decay_impl()