1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 /* 5 * This file is logically part of the PA module. While pa.c contains the core 6 * allocator functionality, this file contains boring integration functionality; 7 * things like the pre- and post- fork handlers, and stats merging for CTL 8 * refreshes. 9 */ 10 11 void 12 pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) { 13 malloc_mutex_prefork(tsdn, &shard->pac.decay_dirty.mtx); 14 malloc_mutex_prefork(tsdn, &shard->pac.decay_muzzy.mtx); 15 } 16 17 void 18 pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) { 19 if (shard->ever_used_hpa) { 20 sec_prefork2(tsdn, &shard->hpa_sec); 21 } 22 } 23 24 void 25 pa_shard_prefork3(tsdn_t *tsdn, pa_shard_t *shard) { 26 malloc_mutex_prefork(tsdn, &shard->pac.grow_mtx); 27 if (shard->ever_used_hpa) { 28 hpa_shard_prefork3(tsdn, &shard->hpa_shard); 29 } 30 } 31 32 void 33 pa_shard_prefork4(tsdn_t *tsdn, pa_shard_t *shard) { 34 ecache_prefork(tsdn, &shard->pac.ecache_dirty); 35 ecache_prefork(tsdn, &shard->pac.ecache_muzzy); 36 ecache_prefork(tsdn, &shard->pac.ecache_retained); 37 if (shard->ever_used_hpa) { 38 hpa_shard_prefork4(tsdn, &shard->hpa_shard); 39 } 40 } 41 42 void 43 pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard) { 44 edata_cache_prefork(tsdn, &shard->edata_cache); 45 } 46 47 void 48 pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) { 49 edata_cache_postfork_parent(tsdn, &shard->edata_cache); 50 ecache_postfork_parent(tsdn, &shard->pac.ecache_dirty); 51 ecache_postfork_parent(tsdn, &shard->pac.ecache_muzzy); 52 ecache_postfork_parent(tsdn, &shard->pac.ecache_retained); 53 malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx); 54 malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx); 55 malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx); 56 if (shard->ever_used_hpa) { 57 sec_postfork_parent(tsdn, &shard->hpa_sec); 58 hpa_shard_postfork_parent(tsdn, &shard->hpa_shard); 59 } 60 } 61 62 void 63 pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) { 64 edata_cache_postfork_child(tsdn, &shard->edata_cache); 65 ecache_postfork_child(tsdn, &shard->pac.ecache_dirty); 66 ecache_postfork_child(tsdn, &shard->pac.ecache_muzzy); 67 ecache_postfork_child(tsdn, &shard->pac.ecache_retained); 68 malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx); 69 malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx); 70 malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx); 71 if (shard->ever_used_hpa) { 72 sec_postfork_child(tsdn, &shard->hpa_sec); 73 hpa_shard_postfork_child(tsdn, &shard->hpa_shard); 74 } 75 } 76 77 void 78 pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, size_t *ndirty, 79 size_t *nmuzzy) { 80 *nactive += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED); 81 *ndirty += ecache_npages_get(&shard->pac.ecache_dirty); 82 *nmuzzy += ecache_npages_get(&shard->pac.ecache_muzzy); 83 } 84 85 void 86 pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, 87 pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out, 88 hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out, 89 size_t *resident) { 90 cassert(config_stats); 91 92 pa_shard_stats_out->pac_stats.retained += 93 ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE; 94 pa_shard_stats_out->edata_avail += atomic_load_zu( 95 &shard->edata_cache.count, ATOMIC_RELAXED); 96 97 size_t resident_pgs = 0; 98 resident_pgs += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED); 99 resident_pgs += ecache_npages_get(&shard->pac.ecache_dirty); 100 *resident += (resident_pgs << LG_PAGE); 101 102 /* Dirty decay stats */ 103 locked_inc_u64_unsynchronized( 104 &pa_shard_stats_out->pac_stats.decay_dirty.npurge, 105 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), 106 &shard->pac.stats->decay_dirty.npurge)); 107 locked_inc_u64_unsynchronized( 108 &pa_shard_stats_out->pac_stats.decay_dirty.nmadvise, 109 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), 110 &shard->pac.stats->decay_dirty.nmadvise)); 111 locked_inc_u64_unsynchronized( 112 &pa_shard_stats_out->pac_stats.decay_dirty.purged, 113 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), 114 &shard->pac.stats->decay_dirty.purged)); 115 116 /* Muzzy decay stats */ 117 locked_inc_u64_unsynchronized( 118 &pa_shard_stats_out->pac_stats.decay_muzzy.npurge, 119 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), 120 &shard->pac.stats->decay_muzzy.npurge)); 121 locked_inc_u64_unsynchronized( 122 &pa_shard_stats_out->pac_stats.decay_muzzy.nmadvise, 123 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), 124 &shard->pac.stats->decay_muzzy.nmadvise)); 125 locked_inc_u64_unsynchronized( 126 &pa_shard_stats_out->pac_stats.decay_muzzy.purged, 127 locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), 128 &shard->pac.stats->decay_muzzy.purged)); 129 130 atomic_load_add_store_zu(&pa_shard_stats_out->pac_stats.abandoned_vm, 131 atomic_load_zu(&shard->pac.stats->abandoned_vm, ATOMIC_RELAXED)); 132 133 for (pszind_t i = 0; i < SC_NPSIZES; i++) { 134 size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes, 135 retained_bytes; 136 dirty = ecache_nextents_get(&shard->pac.ecache_dirty, i); 137 muzzy = ecache_nextents_get(&shard->pac.ecache_muzzy, i); 138 retained = ecache_nextents_get(&shard->pac.ecache_retained, i); 139 dirty_bytes = ecache_nbytes_get(&shard->pac.ecache_dirty, i); 140 muzzy_bytes = ecache_nbytes_get(&shard->pac.ecache_muzzy, i); 141 retained_bytes = ecache_nbytes_get(&shard->pac.ecache_retained, 142 i); 143 144 estats_out[i].ndirty = dirty; 145 estats_out[i].nmuzzy = muzzy; 146 estats_out[i].nretained = retained; 147 estats_out[i].dirty_bytes = dirty_bytes; 148 estats_out[i].muzzy_bytes = muzzy_bytes; 149 estats_out[i].retained_bytes = retained_bytes; 150 } 151 152 if (shard->ever_used_hpa) { 153 hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out); 154 sec_stats_merge(tsdn, &shard->hpa_sec, sec_stats_out); 155 } 156 } 157 158 static void 159 pa_shard_mtx_stats_read_single(tsdn_t *tsdn, mutex_prof_data_t *mutex_prof_data, 160 malloc_mutex_t *mtx, int ind) { 161 malloc_mutex_lock(tsdn, mtx); 162 malloc_mutex_prof_read(tsdn, &mutex_prof_data[ind], mtx); 163 malloc_mutex_unlock(tsdn, mtx); 164 } 165 166 void 167 pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard, 168 mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]) { 169 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, 170 &shard->edata_cache.mtx, arena_prof_mutex_extent_avail); 171 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, 172 &shard->pac.ecache_dirty.mtx, arena_prof_mutex_extents_dirty); 173 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, 174 &shard->pac.ecache_muzzy.mtx, arena_prof_mutex_extents_muzzy); 175 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, 176 &shard->pac.ecache_retained.mtx, arena_prof_mutex_extents_retained); 177 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, 178 &shard->pac.decay_dirty.mtx, arena_prof_mutex_decay_dirty); 179 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, 180 &shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy); 181 182 if (shard->ever_used_hpa) { 183 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, 184 &shard->hpa_shard.mtx, arena_prof_mutex_hpa_shard); 185 pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, 186 &shard->hpa_shard.grow_mtx, 187 arena_prof_mutex_hpa_shard_grow); 188 sec_mutex_stats_read(tsdn, &shard->hpa_sec, 189 &mutex_prof_data[arena_prof_mutex_hpa_sec]); 190 } 191 } 192