1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/san.h" 5 #include "jemalloc/internal/hpa.h" 6 7 static void 8 pa_nactive_add(pa_shard_t *shard, size_t add_pages) { 9 atomic_fetch_add_zu(&shard->nactive, add_pages, ATOMIC_RELAXED); 10 } 11 12 static void 13 pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) { 14 assert(atomic_load_zu(&shard->nactive, ATOMIC_RELAXED) >= sub_pages); 15 atomic_fetch_sub_zu(&shard->nactive, sub_pages, ATOMIC_RELAXED); 16 } 17 18 bool 19 pa_central_init(pa_central_t *central, base_t *base, bool hpa, 20 hpa_hooks_t *hpa_hooks) { 21 bool err; 22 if (hpa) { 23 err = hpa_central_init(¢ral->hpa, base, hpa_hooks); 24 if (err) { 25 return true; 26 } 27 } 28 return false; 29 } 30 31 bool 32 pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central, 33 emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats, 34 malloc_mutex_t *stats_mtx, nstime_t *cur_time, 35 size_t pac_oversize_threshold, ssize_t dirty_decay_ms, 36 ssize_t muzzy_decay_ms) { 37 /* This will change eventually, but for now it should hold. */ 38 assert(base_ind_get(base) == ind); 39 if (edata_cache_init(&shard->edata_cache, base)) { 40 return true; 41 } 42 43 if (pac_init(tsdn, &shard->pac, base, emap, &shard->edata_cache, 44 cur_time, pac_oversize_threshold, dirty_decay_ms, muzzy_decay_ms, 45 &stats->pac_stats, stats_mtx)) { 46 return true; 47 } 48 49 shard->ind = ind; 50 51 shard->ever_used_hpa = false; 52 atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED); 53 54 atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED); 55 56 shard->stats_mtx = stats_mtx; 57 shard->stats = stats; 58 memset(shard->stats, 0, sizeof(*shard->stats)); 59 60 shard->central = central; 61 shard->emap = emap; 62 shard->base = base; 63 64 return false; 65 } 66 67 bool 68 pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard, 69 const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) { 70 if (hpa_shard_init(&shard->hpa_shard, &shard->central->hpa, shard->emap, 71 shard->base, &shard->edata_cache, shard->ind, hpa_opts)) { 72 return true; 73 } 74 if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai, 75 hpa_sec_opts)) { 76 return true; 77 } 78 shard->ever_used_hpa = true; 79 atomic_store_b(&shard->use_hpa, true, ATOMIC_RELAXED); 80 81 return false; 82 } 83 84 void 85 pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) { 86 atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED); 87 if (shard->ever_used_hpa) { 88 sec_disable(tsdn, &shard->hpa_sec); 89 hpa_shard_disable(tsdn, &shard->hpa_shard); 90 } 91 } 92 93 void 94 pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) { 95 atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED); 96 if (shard->ever_used_hpa) { 97 sec_flush(tsdn, &shard->hpa_sec); 98 } 99 } 100 101 static bool 102 pa_shard_uses_hpa(pa_shard_t *shard) { 103 return atomic_load_b(&shard->use_hpa, ATOMIC_RELAXED); 104 } 105 106 void 107 pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) { 108 pac_destroy(tsdn, &shard->pac); 109 if (shard->ever_used_hpa) { 110 sec_flush(tsdn, &shard->hpa_sec); 111 hpa_shard_disable(tsdn, &shard->hpa_shard); 112 } 113 } 114 115 static pai_t * 116 pa_get_pai(pa_shard_t *shard, edata_t *edata) { 117 return (edata_pai_get(edata) == EXTENT_PAI_PAC 118 ? &shard->pac.pai : &shard->hpa_sec.pai); 119 } 120 121 edata_t * 122 pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, 123 bool slab, szind_t szind, bool zero, bool guarded, 124 bool *deferred_work_generated) { 125 witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), 126 WITNESS_RANK_CORE, 0); 127 assert(!guarded || alignment <= PAGE); 128 129 edata_t *edata = NULL; 130 if (!guarded && pa_shard_uses_hpa(shard)) { 131 edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment, 132 zero, /* guarded */ false, slab, deferred_work_generated); 133 } 134 /* 135 * Fall back to the PAC if the HPA is off or couldn't serve the given 136 * allocation request. 137 */ 138 if (edata == NULL) { 139 edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero, 140 guarded, slab, deferred_work_generated); 141 } 142 if (edata != NULL) { 143 assert(edata_size_get(edata) == size); 144 pa_nactive_add(shard, size >> LG_PAGE); 145 emap_remap(tsdn, shard->emap, edata, szind, slab); 146 edata_szind_set(edata, szind); 147 edata_slab_set(edata, slab); 148 if (slab && (size > 2 * PAGE)) { 149 emap_register_interior(tsdn, shard->emap, edata, szind); 150 } 151 assert(edata_arena_ind_get(edata) == shard->ind); 152 } 153 return edata; 154 } 155 156 bool 157 pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, 158 size_t new_size, szind_t szind, bool zero, bool *deferred_work_generated) { 159 assert(new_size > old_size); 160 assert(edata_size_get(edata) == old_size); 161 assert((new_size & PAGE_MASK) == 0); 162 if (edata_guarded_get(edata)) { 163 return true; 164 } 165 size_t expand_amount = new_size - old_size; 166 167 pai_t *pai = pa_get_pai(shard, edata); 168 169 bool error = pai_expand(tsdn, pai, edata, old_size, new_size, zero, 170 deferred_work_generated); 171 if (error) { 172 return true; 173 } 174 175 pa_nactive_add(shard, expand_amount >> LG_PAGE); 176 edata_szind_set(edata, szind); 177 emap_remap(tsdn, shard->emap, edata, szind, /* slab */ false); 178 return false; 179 } 180 181 bool 182 pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, 183 size_t new_size, szind_t szind, bool *deferred_work_generated) { 184 assert(new_size < old_size); 185 assert(edata_size_get(edata) == old_size); 186 assert((new_size & PAGE_MASK) == 0); 187 if (edata_guarded_get(edata)) { 188 return true; 189 } 190 size_t shrink_amount = old_size - new_size; 191 192 pai_t *pai = pa_get_pai(shard, edata); 193 bool error = pai_shrink(tsdn, pai, edata, old_size, new_size, 194 deferred_work_generated); 195 if (error) { 196 return true; 197 } 198 pa_nactive_sub(shard, shrink_amount >> LG_PAGE); 199 200 edata_szind_set(edata, szind); 201 emap_remap(tsdn, shard->emap, edata, szind, /* slab */ false); 202 return false; 203 } 204 205 void 206 pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, 207 bool *deferred_work_generated) { 208 emap_remap(tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false); 209 if (edata_slab_get(edata)) { 210 emap_deregister_interior(tsdn, shard->emap, edata); 211 /* 212 * The slab state of the extent isn't cleared. It may be used 213 * by the pai implementation, e.g. to make caching decisions. 214 */ 215 } 216 edata_addr_set(edata, edata_base_get(edata)); 217 edata_szind_set(edata, SC_NSIZES); 218 pa_nactive_sub(shard, edata_size_get(edata) >> LG_PAGE); 219 pai_t *pai = pa_get_pai(shard, edata); 220 pai_dalloc(tsdn, pai, edata, deferred_work_generated); 221 } 222 223 bool 224 pa_shard_retain_grow_limit_get_set(tsdn_t *tsdn, pa_shard_t *shard, 225 size_t *old_limit, size_t *new_limit) { 226 return pac_retain_grow_limit_get_set(tsdn, &shard->pac, old_limit, 227 new_limit); 228 } 229 230 bool 231 pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state, 232 ssize_t decay_ms, pac_purge_eagerness_t eagerness) { 233 return pac_decay_ms_set(tsdn, &shard->pac, state, decay_ms, eagerness); 234 } 235 236 ssize_t 237 pa_decay_ms_get(pa_shard_t *shard, extent_state_t state) { 238 return pac_decay_ms_get(&shard->pac, state); 239 } 240 241 void 242 pa_shard_set_deferral_allowed(tsdn_t *tsdn, pa_shard_t *shard, 243 bool deferral_allowed) { 244 if (pa_shard_uses_hpa(shard)) { 245 hpa_shard_set_deferral_allowed(tsdn, &shard->hpa_shard, 246 deferral_allowed); 247 } 248 } 249 250 void 251 pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) { 252 if (pa_shard_uses_hpa(shard)) { 253 hpa_shard_do_deferred_work(tsdn, &shard->hpa_shard); 254 } 255 } 256 257 /* 258 * Get time until next deferred work ought to happen. If there are multiple 259 * things that have been deferred, this function calculates the time until 260 * the soonest of those things. 261 */ 262 uint64_t 263 pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) { 264 uint64_t time = pai_time_until_deferred_work(tsdn, &shard->pac.pai); 265 if (time == BACKGROUND_THREAD_DEFERRED_MIN) { 266 return time; 267 } 268 269 if (pa_shard_uses_hpa(shard)) { 270 uint64_t hpa = 271 pai_time_until_deferred_work(tsdn, &shard->hpa_shard.pai); 272 if (hpa < time) { 273 time = hpa; 274 } 275 } 276 return time; 277 } 278