1 #define JEMALLOC_CTL_C_ 2 #include "jemalloc/internal/jemalloc_preamble.h" 3 #include "jemalloc/internal/jemalloc_internal_includes.h" 4 5 #include "jemalloc/internal/assert.h" 6 #include "jemalloc/internal/ctl.h" 7 #include "jemalloc/internal/extent_dss.h" 8 #include "jemalloc/internal/extent_mmap.h" 9 #include "jemalloc/internal/mutex.h" 10 #include "jemalloc/internal/nstime.h" 11 #include "jemalloc/internal/sc.h" 12 #include "jemalloc/internal/util.h" 13 14 /******************************************************************************/ 15 /* Data. */ 16 17 /* 18 * ctl_mtx protects the following: 19 * - ctl_stats->* 20 */ 21 static malloc_mutex_t ctl_mtx; 22 static bool ctl_initialized; 23 static ctl_stats_t *ctl_stats; 24 static ctl_arenas_t *ctl_arenas; 25 26 /******************************************************************************/ 27 /* Helpers for named and indexed nodes. */ 28 29 static const ctl_named_node_t * ctl_named_node(const ctl_node_t * node)30 ctl_named_node(const ctl_node_t *node) { 31 return ((node->named) ? (const ctl_named_node_t *)node : NULL); 32 } 33 34 static const ctl_named_node_t * ctl_named_children(const ctl_named_node_t * node,size_t index)35 ctl_named_children(const ctl_named_node_t *node, size_t index) { 36 const ctl_named_node_t *children = ctl_named_node(node->children); 37 38 return (children ? &children[index] : NULL); 39 } 40 41 static const ctl_indexed_node_t * ctl_indexed_node(const ctl_node_t * node)42 ctl_indexed_node(const ctl_node_t *node) { 43 return (!node->named ? (const ctl_indexed_node_t *)node : NULL); 44 } 45 46 /******************************************************************************/ 47 /* Function prototypes for non-inline static functions. */ 48 49 #define CTL_PROTO(n) \ 50 static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 51 void *oldp, size_t *oldlenp, void *newp, size_t newlen); 52 53 #define INDEX_PROTO(n) \ 54 static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \ 55 const size_t *mib, size_t miblen, size_t i); 56 57 CTL_PROTO(version) 58 CTL_PROTO(epoch) 59 CTL_PROTO(background_thread) 60 CTL_PROTO(max_background_threads) 61 CTL_PROTO(thread_tcache_enabled) 62 CTL_PROTO(thread_tcache_flush) 63 CTL_PROTO(thread_prof_name) 64 CTL_PROTO(thread_prof_active) 65 CTL_PROTO(thread_arena) 66 CTL_PROTO(thread_allocated) 67 CTL_PROTO(thread_allocatedp) 68 CTL_PROTO(thread_deallocated) 69 CTL_PROTO(thread_deallocatedp) 70 CTL_PROTO(config_cache_oblivious) 71 CTL_PROTO(config_debug) 72 CTL_PROTO(config_fill) 73 CTL_PROTO(config_lazy_lock) 74 CTL_PROTO(config_malloc_conf) 75 CTL_PROTO(config_opt_safety_checks) 76 CTL_PROTO(config_prof) 77 CTL_PROTO(config_prof_libgcc) 78 CTL_PROTO(config_prof_libunwind) 79 CTL_PROTO(config_stats) 80 CTL_PROTO(config_utrace) 81 CTL_PROTO(config_xmalloc) 82 CTL_PROTO(opt_abort) 83 CTL_PROTO(opt_abort_conf) 84 CTL_PROTO(opt_confirm_conf) 85 CTL_PROTO(opt_metadata_thp) 86 CTL_PROTO(opt_retain) 87 CTL_PROTO(opt_dss) 88 CTL_PROTO(opt_narenas) 89 CTL_PROTO(opt_percpu_arena) 90 CTL_PROTO(opt_oversize_threshold) 91 CTL_PROTO(opt_background_thread) 92 CTL_PROTO(opt_max_background_threads) 93 CTL_PROTO(opt_dirty_decay_ms) 94 CTL_PROTO(opt_muzzy_decay_ms) 95 CTL_PROTO(opt_stats_print) 96 CTL_PROTO(opt_stats_print_opts) 97 CTL_PROTO(opt_junk) 98 CTL_PROTO(opt_zero) 99 CTL_PROTO(opt_utrace) 100 CTL_PROTO(opt_xmalloc) 101 CTL_PROTO(opt_tcache) 102 CTL_PROTO(opt_thp) 103 CTL_PROTO(opt_lg_extent_max_active_fit) 104 CTL_PROTO(opt_lg_tcache_max) 105 CTL_PROTO(opt_prof) 106 CTL_PROTO(opt_prof_prefix) 107 CTL_PROTO(opt_prof_active) 108 CTL_PROTO(opt_prof_thread_active_init) 109 CTL_PROTO(opt_lg_prof_sample) 110 CTL_PROTO(opt_lg_prof_interval) 111 CTL_PROTO(opt_prof_gdump) 112 CTL_PROTO(opt_prof_final) 113 CTL_PROTO(opt_prof_leak) 114 CTL_PROTO(opt_prof_accum) 115 CTL_PROTO(tcache_create) 116 CTL_PROTO(tcache_flush) 117 CTL_PROTO(tcache_destroy) 118 CTL_PROTO(arena_i_initialized) 119 CTL_PROTO(arena_i_decay) 120 CTL_PROTO(arena_i_purge) 121 CTL_PROTO(arena_i_reset) 122 CTL_PROTO(arena_i_destroy) 123 CTL_PROTO(arena_i_dss) 124 CTL_PROTO(arena_i_dirty_decay_ms) 125 CTL_PROTO(arena_i_muzzy_decay_ms) 126 CTL_PROTO(arena_i_extent_hooks) 127 CTL_PROTO(arena_i_retain_grow_limit) 128 INDEX_PROTO(arena_i) 129 CTL_PROTO(arenas_bin_i_size) 130 CTL_PROTO(arenas_bin_i_nregs) 131 CTL_PROTO(arenas_bin_i_slab_size) 132 CTL_PROTO(arenas_bin_i_nshards) 133 INDEX_PROTO(arenas_bin_i) 134 CTL_PROTO(arenas_lextent_i_size) 135 INDEX_PROTO(arenas_lextent_i) 136 CTL_PROTO(arenas_narenas) 137 CTL_PROTO(arenas_dirty_decay_ms) 138 CTL_PROTO(arenas_muzzy_decay_ms) 139 CTL_PROTO(arenas_quantum) 140 CTL_PROTO(arenas_page) 141 CTL_PROTO(arenas_tcache_max) 142 CTL_PROTO(arenas_nbins) 143 CTL_PROTO(arenas_nhbins) 144 CTL_PROTO(arenas_nlextents) 145 CTL_PROTO(arenas_create) 146 CTL_PROTO(arenas_lookup) 147 CTL_PROTO(prof_thread_active_init) 148 CTL_PROTO(prof_active) 149 CTL_PROTO(prof_dump) 150 CTL_PROTO(prof_gdump) 151 CTL_PROTO(prof_reset) 152 CTL_PROTO(prof_interval) 153 CTL_PROTO(lg_prof_sample) 154 CTL_PROTO(prof_log_start) 155 CTL_PROTO(prof_log_stop) 156 CTL_PROTO(stats_arenas_i_small_allocated) 157 CTL_PROTO(stats_arenas_i_small_nmalloc) 158 CTL_PROTO(stats_arenas_i_small_ndalloc) 159 CTL_PROTO(stats_arenas_i_small_nrequests) 160 CTL_PROTO(stats_arenas_i_small_nfills) 161 CTL_PROTO(stats_arenas_i_small_nflushes) 162 CTL_PROTO(stats_arenas_i_large_allocated) 163 CTL_PROTO(stats_arenas_i_large_nmalloc) 164 CTL_PROTO(stats_arenas_i_large_ndalloc) 165 CTL_PROTO(stats_arenas_i_large_nrequests) 166 CTL_PROTO(stats_arenas_i_large_nfills) 167 CTL_PROTO(stats_arenas_i_large_nflushes) 168 CTL_PROTO(stats_arenas_i_bins_j_nmalloc) 169 CTL_PROTO(stats_arenas_i_bins_j_ndalloc) 170 CTL_PROTO(stats_arenas_i_bins_j_nrequests) 171 CTL_PROTO(stats_arenas_i_bins_j_curregs) 172 CTL_PROTO(stats_arenas_i_bins_j_nfills) 173 CTL_PROTO(stats_arenas_i_bins_j_nflushes) 174 CTL_PROTO(stats_arenas_i_bins_j_nslabs) 175 CTL_PROTO(stats_arenas_i_bins_j_nreslabs) 176 CTL_PROTO(stats_arenas_i_bins_j_curslabs) 177 CTL_PROTO(stats_arenas_i_bins_j_nonfull_slabs) 178 INDEX_PROTO(stats_arenas_i_bins_j) 179 CTL_PROTO(stats_arenas_i_lextents_j_nmalloc) 180 CTL_PROTO(stats_arenas_i_lextents_j_ndalloc) 181 CTL_PROTO(stats_arenas_i_lextents_j_nrequests) 182 CTL_PROTO(stats_arenas_i_lextents_j_curlextents) 183 INDEX_PROTO(stats_arenas_i_lextents_j) 184 CTL_PROTO(stats_arenas_i_extents_j_ndirty) 185 CTL_PROTO(stats_arenas_i_extents_j_nmuzzy) 186 CTL_PROTO(stats_arenas_i_extents_j_nretained) 187 CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes) 188 CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes) 189 CTL_PROTO(stats_arenas_i_extents_j_retained_bytes) 190 INDEX_PROTO(stats_arenas_i_extents_j) 191 CTL_PROTO(stats_arenas_i_nthreads) 192 CTL_PROTO(stats_arenas_i_uptime) 193 CTL_PROTO(stats_arenas_i_dss) 194 CTL_PROTO(stats_arenas_i_dirty_decay_ms) 195 CTL_PROTO(stats_arenas_i_muzzy_decay_ms) 196 CTL_PROTO(stats_arenas_i_pactive) 197 CTL_PROTO(stats_arenas_i_pdirty) 198 CTL_PROTO(stats_arenas_i_pmuzzy) 199 CTL_PROTO(stats_arenas_i_mapped) 200 CTL_PROTO(stats_arenas_i_retained) 201 CTL_PROTO(stats_arenas_i_extent_avail) 202 CTL_PROTO(stats_arenas_i_dirty_npurge) 203 CTL_PROTO(stats_arenas_i_dirty_nmadvise) 204 CTL_PROTO(stats_arenas_i_dirty_purged) 205 CTL_PROTO(stats_arenas_i_muzzy_npurge) 206 CTL_PROTO(stats_arenas_i_muzzy_nmadvise) 207 CTL_PROTO(stats_arenas_i_muzzy_purged) 208 CTL_PROTO(stats_arenas_i_base) 209 CTL_PROTO(stats_arenas_i_internal) 210 CTL_PROTO(stats_arenas_i_metadata_thp) 211 CTL_PROTO(stats_arenas_i_tcache_bytes) 212 CTL_PROTO(stats_arenas_i_resident) 213 CTL_PROTO(stats_arenas_i_abandoned_vm) 214 INDEX_PROTO(stats_arenas_i) 215 CTL_PROTO(stats_allocated) 216 CTL_PROTO(stats_active) 217 CTL_PROTO(stats_background_thread_num_threads) 218 CTL_PROTO(stats_background_thread_num_runs) 219 CTL_PROTO(stats_background_thread_run_interval) 220 CTL_PROTO(stats_metadata) 221 CTL_PROTO(stats_metadata_thp) 222 CTL_PROTO(stats_resident) 223 CTL_PROTO(stats_mapped) 224 CTL_PROTO(stats_retained) 225 CTL_PROTO(experimental_hooks_install) 226 CTL_PROTO(experimental_hooks_remove) 227 CTL_PROTO(experimental_utilization_query) 228 CTL_PROTO(experimental_utilization_batch_query) 229 CTL_PROTO(experimental_arenas_i_pactivep) 230 INDEX_PROTO(experimental_arenas_i) 231 232 #define MUTEX_STATS_CTL_PROTO_GEN(n) \ 233 CTL_PROTO(stats_##n##_num_ops) \ 234 CTL_PROTO(stats_##n##_num_wait) \ 235 CTL_PROTO(stats_##n##_num_spin_acq) \ 236 CTL_PROTO(stats_##n##_num_owner_switch) \ 237 CTL_PROTO(stats_##n##_total_wait_time) \ 238 CTL_PROTO(stats_##n##_max_wait_time) \ 239 CTL_PROTO(stats_##n##_max_num_thds) 240 241 /* Global mutexes. */ 242 #define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx) 243 MUTEX_PROF_GLOBAL_MUTEXES 244 #undef OP 245 246 /* Per arena mutexes. */ 247 #define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##mtx) 248 MUTEX_PROF_ARENA_MUTEXES 249 #undef OP 250 251 /* Arena bin mutexes. */ 252 MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex) 253 #undef MUTEX_STATS_CTL_PROTO_GEN 254 255 CTL_PROTO(stats_mutexes_reset) 256 257 /******************************************************************************/ 258 /* mallctl tree. */ 259 260 #define NAME(n) {true}, n 261 #define CHILD(t, c) \ 262 sizeof(c##_node) / sizeof(ctl_##t##_node_t), \ 263 (ctl_node_t *)c##_node, \ 264 NULL 265 #define CTL(c) 0, NULL, c##_ctl 266 267 /* 268 * Only handles internal indexed nodes, since there are currently no external 269 * ones. 270 */ 271 #define INDEX(i) {false}, i##_index 272 273 static const ctl_named_node_t thread_tcache_node[] = { 274 {NAME("enabled"), CTL(thread_tcache_enabled)}, 275 {NAME("flush"), CTL(thread_tcache_flush)} 276 }; 277 278 static const ctl_named_node_t thread_prof_node[] = { 279 {NAME("name"), CTL(thread_prof_name)}, 280 {NAME("active"), CTL(thread_prof_active)} 281 }; 282 283 static const ctl_named_node_t thread_node[] = { 284 {NAME("arena"), CTL(thread_arena)}, 285 {NAME("allocated"), CTL(thread_allocated)}, 286 {NAME("allocatedp"), CTL(thread_allocatedp)}, 287 {NAME("deallocated"), CTL(thread_deallocated)}, 288 {NAME("deallocatedp"), CTL(thread_deallocatedp)}, 289 {NAME("tcache"), CHILD(named, thread_tcache)}, 290 {NAME("prof"), CHILD(named, thread_prof)} 291 }; 292 293 static const ctl_named_node_t config_node[] = { 294 {NAME("cache_oblivious"), CTL(config_cache_oblivious)}, 295 {NAME("debug"), CTL(config_debug)}, 296 {NAME("fill"), CTL(config_fill)}, 297 {NAME("lazy_lock"), CTL(config_lazy_lock)}, 298 {NAME("malloc_conf"), CTL(config_malloc_conf)}, 299 {NAME("opt_safety_checks"), CTL(config_opt_safety_checks)}, 300 {NAME("prof"), CTL(config_prof)}, 301 {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, 302 {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, 303 {NAME("stats"), CTL(config_stats)}, 304 {NAME("utrace"), CTL(config_utrace)}, 305 {NAME("xmalloc"), CTL(config_xmalloc)} 306 }; 307 308 static const ctl_named_node_t opt_node[] = { 309 {NAME("abort"), CTL(opt_abort)}, 310 {NAME("abort_conf"), CTL(opt_abort_conf)}, 311 {NAME("confirm_conf"), CTL(opt_confirm_conf)}, 312 {NAME("metadata_thp"), CTL(opt_metadata_thp)}, 313 {NAME("retain"), CTL(opt_retain)}, 314 {NAME("dss"), CTL(opt_dss)}, 315 {NAME("narenas"), CTL(opt_narenas)}, 316 {NAME("percpu_arena"), CTL(opt_percpu_arena)}, 317 {NAME("oversize_threshold"), CTL(opt_oversize_threshold)}, 318 {NAME("background_thread"), CTL(opt_background_thread)}, 319 {NAME("max_background_threads"), CTL(opt_max_background_threads)}, 320 {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)}, 321 {NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)}, 322 {NAME("stats_print"), CTL(opt_stats_print)}, 323 {NAME("stats_print_opts"), CTL(opt_stats_print_opts)}, 324 {NAME("junk"), CTL(opt_junk)}, 325 {NAME("zero"), CTL(opt_zero)}, 326 {NAME("utrace"), CTL(opt_utrace)}, 327 {NAME("xmalloc"), CTL(opt_xmalloc)}, 328 {NAME("tcache"), CTL(opt_tcache)}, 329 {NAME("thp"), CTL(opt_thp)}, 330 {NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)}, 331 {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, 332 {NAME("prof"), CTL(opt_prof)}, 333 {NAME("prof_prefix"), CTL(opt_prof_prefix)}, 334 {NAME("prof_active"), CTL(opt_prof_active)}, 335 {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)}, 336 {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, 337 {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, 338 {NAME("prof_gdump"), CTL(opt_prof_gdump)}, 339 {NAME("prof_final"), CTL(opt_prof_final)}, 340 {NAME("prof_leak"), CTL(opt_prof_leak)}, 341 {NAME("prof_accum"), CTL(opt_prof_accum)} 342 }; 343 344 static const ctl_named_node_t tcache_node[] = { 345 {NAME("create"), CTL(tcache_create)}, 346 {NAME("flush"), CTL(tcache_flush)}, 347 {NAME("destroy"), CTL(tcache_destroy)} 348 }; 349 350 static const ctl_named_node_t arena_i_node[] = { 351 {NAME("initialized"), CTL(arena_i_initialized)}, 352 {NAME("decay"), CTL(arena_i_decay)}, 353 {NAME("purge"), CTL(arena_i_purge)}, 354 {NAME("reset"), CTL(arena_i_reset)}, 355 {NAME("destroy"), CTL(arena_i_destroy)}, 356 {NAME("dss"), CTL(arena_i_dss)}, 357 {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)}, 358 {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)}, 359 {NAME("extent_hooks"), CTL(arena_i_extent_hooks)}, 360 {NAME("retain_grow_limit"), CTL(arena_i_retain_grow_limit)} 361 }; 362 static const ctl_named_node_t super_arena_i_node[] = { 363 {NAME(""), CHILD(named, arena_i)} 364 }; 365 366 static const ctl_indexed_node_t arena_node[] = { 367 {INDEX(arena_i)} 368 }; 369 370 static const ctl_named_node_t arenas_bin_i_node[] = { 371 {NAME("size"), CTL(arenas_bin_i_size)}, 372 {NAME("nregs"), CTL(arenas_bin_i_nregs)}, 373 {NAME("slab_size"), CTL(arenas_bin_i_slab_size)}, 374 {NAME("nshards"), CTL(arenas_bin_i_nshards)} 375 }; 376 static const ctl_named_node_t super_arenas_bin_i_node[] = { 377 {NAME(""), CHILD(named, arenas_bin_i)} 378 }; 379 380 static const ctl_indexed_node_t arenas_bin_node[] = { 381 {INDEX(arenas_bin_i)} 382 }; 383 384 static const ctl_named_node_t arenas_lextent_i_node[] = { 385 {NAME("size"), CTL(arenas_lextent_i_size)} 386 }; 387 static const ctl_named_node_t super_arenas_lextent_i_node[] = { 388 {NAME(""), CHILD(named, arenas_lextent_i)} 389 }; 390 391 static const ctl_indexed_node_t arenas_lextent_node[] = { 392 {INDEX(arenas_lextent_i)} 393 }; 394 395 static const ctl_named_node_t arenas_node[] = { 396 {NAME("narenas"), CTL(arenas_narenas)}, 397 {NAME("dirty_decay_ms"), CTL(arenas_dirty_decay_ms)}, 398 {NAME("muzzy_decay_ms"), CTL(arenas_muzzy_decay_ms)}, 399 {NAME("quantum"), CTL(arenas_quantum)}, 400 {NAME("page"), CTL(arenas_page)}, 401 {NAME("tcache_max"), CTL(arenas_tcache_max)}, 402 {NAME("nbins"), CTL(arenas_nbins)}, 403 {NAME("nhbins"), CTL(arenas_nhbins)}, 404 {NAME("bin"), CHILD(indexed, arenas_bin)}, 405 {NAME("nlextents"), CTL(arenas_nlextents)}, 406 {NAME("lextent"), CHILD(indexed, arenas_lextent)}, 407 {NAME("create"), CTL(arenas_create)}, 408 {NAME("lookup"), CTL(arenas_lookup)} 409 }; 410 411 static const ctl_named_node_t prof_node[] = { 412 {NAME("thread_active_init"), CTL(prof_thread_active_init)}, 413 {NAME("active"), CTL(prof_active)}, 414 {NAME("dump"), CTL(prof_dump)}, 415 {NAME("gdump"), CTL(prof_gdump)}, 416 {NAME("reset"), CTL(prof_reset)}, 417 {NAME("interval"), CTL(prof_interval)}, 418 {NAME("lg_sample"), CTL(lg_prof_sample)}, 419 {NAME("log_start"), CTL(prof_log_start)}, 420 {NAME("log_stop"), CTL(prof_log_stop)} 421 }; 422 static const ctl_named_node_t stats_arenas_i_small_node[] = { 423 {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, 424 {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, 425 {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, 426 {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}, 427 {NAME("nfills"), CTL(stats_arenas_i_small_nfills)}, 428 {NAME("nflushes"), CTL(stats_arenas_i_small_nflushes)} 429 }; 430 431 static const ctl_named_node_t stats_arenas_i_large_node[] = { 432 {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, 433 {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, 434 {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, 435 {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}, 436 {NAME("nfills"), CTL(stats_arenas_i_large_nfills)}, 437 {NAME("nflushes"), CTL(stats_arenas_i_large_nflushes)} 438 }; 439 440 #define MUTEX_PROF_DATA_NODE(prefix) \ 441 static const ctl_named_node_t stats_##prefix##_node[] = { \ 442 {NAME("num_ops"), \ 443 CTL(stats_##prefix##_num_ops)}, \ 444 {NAME("num_wait"), \ 445 CTL(stats_##prefix##_num_wait)}, \ 446 {NAME("num_spin_acq"), \ 447 CTL(stats_##prefix##_num_spin_acq)}, \ 448 {NAME("num_owner_switch"), \ 449 CTL(stats_##prefix##_num_owner_switch)}, \ 450 {NAME("total_wait_time"), \ 451 CTL(stats_##prefix##_total_wait_time)}, \ 452 {NAME("max_wait_time"), \ 453 CTL(stats_##prefix##_max_wait_time)}, \ 454 {NAME("max_num_thds"), \ 455 CTL(stats_##prefix##_max_num_thds)} \ 456 /* Note that # of current waiting thread not provided. */ \ 457 }; 458 459 MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex) 460 461 static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { 462 {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, 463 {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, 464 {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, 465 {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)}, 466 {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, 467 {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, 468 {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)}, 469 {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)}, 470 {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)}, 471 {NAME("nonfull_slabs"), CTL(stats_arenas_i_bins_j_nonfull_slabs)}, 472 {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)} 473 }; 474 475 static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { 476 {NAME(""), CHILD(named, stats_arenas_i_bins_j)} 477 }; 478 479 static const ctl_indexed_node_t stats_arenas_i_bins_node[] = { 480 {INDEX(stats_arenas_i_bins_j)} 481 }; 482 483 static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = { 484 {NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)}, 485 {NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)}, 486 {NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)}, 487 {NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)} 488 }; 489 static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = { 490 {NAME(""), CHILD(named, stats_arenas_i_lextents_j)} 491 }; 492 493 static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { 494 {INDEX(stats_arenas_i_lextents_j)} 495 }; 496 497 static const ctl_named_node_t stats_arenas_i_extents_j_node[] = { 498 {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)}, 499 {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)}, 500 {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)}, 501 {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)}, 502 {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)}, 503 {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)} 504 }; 505 506 static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = { 507 {NAME(""), CHILD(named, stats_arenas_i_extents_j)} 508 }; 509 510 static const ctl_indexed_node_t stats_arenas_i_extents_node[] = { 511 {INDEX(stats_arenas_i_extents_j)} 512 }; 513 514 #define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) 515 MUTEX_PROF_ARENA_MUTEXES 516 #undef OP 517 518 static const ctl_named_node_t stats_arenas_i_mutexes_node[] = { 519 #define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)}, 520 MUTEX_PROF_ARENA_MUTEXES 521 #undef OP 522 }; 523 524 static const ctl_named_node_t stats_arenas_i_node[] = { 525 {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, 526 {NAME("uptime"), CTL(stats_arenas_i_uptime)}, 527 {NAME("dss"), CTL(stats_arenas_i_dss)}, 528 {NAME("dirty_decay_ms"), CTL(stats_arenas_i_dirty_decay_ms)}, 529 {NAME("muzzy_decay_ms"), CTL(stats_arenas_i_muzzy_decay_ms)}, 530 {NAME("pactive"), CTL(stats_arenas_i_pactive)}, 531 {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, 532 {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, 533 {NAME("mapped"), CTL(stats_arenas_i_mapped)}, 534 {NAME("retained"), CTL(stats_arenas_i_retained)}, 535 {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)}, 536 {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, 537 {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, 538 {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, 539 {NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)}, 540 {NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)}, 541 {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)}, 542 {NAME("base"), CTL(stats_arenas_i_base)}, 543 {NAME("internal"), CTL(stats_arenas_i_internal)}, 544 {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)}, 545 {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, 546 {NAME("resident"), CTL(stats_arenas_i_resident)}, 547 {NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)}, 548 {NAME("small"), CHILD(named, stats_arenas_i_small)}, 549 {NAME("large"), CHILD(named, stats_arenas_i_large)}, 550 {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, 551 {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, 552 {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)}, 553 {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)} 554 }; 555 static const ctl_named_node_t super_stats_arenas_i_node[] = { 556 {NAME(""), CHILD(named, stats_arenas_i)} 557 }; 558 559 static const ctl_indexed_node_t stats_arenas_node[] = { 560 {INDEX(stats_arenas_i)} 561 }; 562 563 static const ctl_named_node_t stats_background_thread_node[] = { 564 {NAME("num_threads"), CTL(stats_background_thread_num_threads)}, 565 {NAME("num_runs"), CTL(stats_background_thread_num_runs)}, 566 {NAME("run_interval"), CTL(stats_background_thread_run_interval)} 567 }; 568 569 #define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx) 570 MUTEX_PROF_GLOBAL_MUTEXES 571 #undef OP 572 573 static const ctl_named_node_t stats_mutexes_node[] = { 574 #define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)}, 575 MUTEX_PROF_GLOBAL_MUTEXES 576 #undef OP 577 {NAME("reset"), CTL(stats_mutexes_reset)} 578 }; 579 #undef MUTEX_PROF_DATA_NODE 580 581 static const ctl_named_node_t stats_node[] = { 582 {NAME("allocated"), CTL(stats_allocated)}, 583 {NAME("active"), CTL(stats_active)}, 584 {NAME("metadata"), CTL(stats_metadata)}, 585 {NAME("metadata_thp"), CTL(stats_metadata_thp)}, 586 {NAME("resident"), CTL(stats_resident)}, 587 {NAME("mapped"), CTL(stats_mapped)}, 588 {NAME("retained"), CTL(stats_retained)}, 589 {NAME("background_thread"), 590 CHILD(named, stats_background_thread)}, 591 {NAME("mutexes"), CHILD(named, stats_mutexes)}, 592 {NAME("arenas"), CHILD(indexed, stats_arenas)} 593 }; 594 595 static const ctl_named_node_t experimental_hooks_node[] = { 596 {NAME("install"), CTL(experimental_hooks_install)}, 597 {NAME("remove"), CTL(experimental_hooks_remove)} 598 }; 599 600 static const ctl_named_node_t experimental_utilization_node[] = { 601 {NAME("query"), CTL(experimental_utilization_query)}, 602 {NAME("batch_query"), CTL(experimental_utilization_batch_query)} 603 }; 604 605 static const ctl_named_node_t experimental_arenas_i_node[] = { 606 {NAME("pactivep"), CTL(experimental_arenas_i_pactivep)} 607 }; 608 static const ctl_named_node_t super_experimental_arenas_i_node[] = { 609 {NAME(""), CHILD(named, experimental_arenas_i)} 610 }; 611 612 static const ctl_indexed_node_t experimental_arenas_node[] = { 613 {INDEX(experimental_arenas_i)} 614 }; 615 616 static const ctl_named_node_t experimental_node[] = { 617 {NAME("hooks"), CHILD(named, experimental_hooks)}, 618 {NAME("utilization"), CHILD(named, experimental_utilization)}, 619 {NAME("arenas"), CHILD(indexed, experimental_arenas)} 620 }; 621 622 static const ctl_named_node_t root_node[] = { 623 {NAME("version"), CTL(version)}, 624 {NAME("epoch"), CTL(epoch)}, 625 {NAME("background_thread"), CTL(background_thread)}, 626 {NAME("max_background_threads"), CTL(max_background_threads)}, 627 {NAME("thread"), CHILD(named, thread)}, 628 {NAME("config"), CHILD(named, config)}, 629 {NAME("opt"), CHILD(named, opt)}, 630 {NAME("tcache"), CHILD(named, tcache)}, 631 {NAME("arena"), CHILD(indexed, arena)}, 632 {NAME("arenas"), CHILD(named, arenas)}, 633 {NAME("prof"), CHILD(named, prof)}, 634 {NAME("stats"), CHILD(named, stats)}, 635 {NAME("experimental"), CHILD(named, experimental)} 636 }; 637 static const ctl_named_node_t super_root_node[] = { 638 {NAME(""), CHILD(named, root)} 639 }; 640 641 #undef NAME 642 #undef CHILD 643 #undef CTL 644 #undef INDEX 645 646 /******************************************************************************/ 647 648 /* 649 * Sets *dst + *src non-atomically. This is safe, since everything is 650 * synchronized by the ctl mutex. 651 */ 652 static void ctl_accum_arena_stats_u64(arena_stats_u64_t * dst,arena_stats_u64_t * src)653 ctl_accum_arena_stats_u64(arena_stats_u64_t *dst, arena_stats_u64_t *src) { 654 #ifdef JEMALLOC_ATOMIC_U64 655 uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED); 656 uint64_t cur_src = atomic_load_u64(src, ATOMIC_RELAXED); 657 atomic_store_u64(dst, cur_dst + cur_src, ATOMIC_RELAXED); 658 #else 659 *dst += *src; 660 #endif 661 } 662 663 /* Likewise: with ctl mutex synchronization, reading is simple. */ 664 static uint64_t ctl_arena_stats_read_u64(arena_stats_u64_t * p)665 ctl_arena_stats_read_u64(arena_stats_u64_t *p) { 666 #ifdef JEMALLOC_ATOMIC_U64 667 return atomic_load_u64(p, ATOMIC_RELAXED); 668 #else 669 return *p; 670 #endif 671 } 672 673 static void accum_atomic_zu(atomic_zu_t * dst,atomic_zu_t * src)674 accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) { 675 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED); 676 size_t cur_src = atomic_load_zu(src, ATOMIC_RELAXED); 677 atomic_store_zu(dst, cur_dst + cur_src, ATOMIC_RELAXED); 678 } 679 680 /******************************************************************************/ 681 682 static unsigned arenas_i2a_impl(size_t i,bool compat,bool validate)683 arenas_i2a_impl(size_t i, bool compat, bool validate) { 684 unsigned a; 685 686 switch (i) { 687 case MALLCTL_ARENAS_ALL: 688 a = 0; 689 break; 690 case MALLCTL_ARENAS_DESTROYED: 691 a = 1; 692 break; 693 default: 694 if (compat && i == ctl_arenas->narenas) { 695 /* 696 * Provide deprecated backward compatibility for 697 * accessing the merged stats at index narenas rather 698 * than via MALLCTL_ARENAS_ALL. This is scheduled for 699 * removal in 6.0.0. 700 */ 701 a = 0; 702 } else if (validate && i >= ctl_arenas->narenas) { 703 a = UINT_MAX; 704 } else { 705 /* 706 * This function should never be called for an index 707 * more than one past the range of indices that have 708 * initialized ctl data. 709 */ 710 assert(i < ctl_arenas->narenas || (!validate && i == 711 ctl_arenas->narenas)); 712 a = (unsigned)i + 2; 713 } 714 break; 715 } 716 717 return a; 718 } 719 720 static unsigned arenas_i2a(size_t i)721 arenas_i2a(size_t i) { 722 return arenas_i2a_impl(i, true, false); 723 } 724 725 static ctl_arena_t * arenas_i_impl(tsd_t * tsd,size_t i,bool compat,bool init)726 arenas_i_impl(tsd_t *tsd, size_t i, bool compat, bool init) { 727 ctl_arena_t *ret; 728 729 assert(!compat || !init); 730 731 ret = ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)]; 732 if (init && ret == NULL) { 733 if (config_stats) { 734 struct container_s { 735 ctl_arena_t ctl_arena; 736 ctl_arena_stats_t astats; 737 }; 738 struct container_s *cont = 739 (struct container_s *)base_alloc(tsd_tsdn(tsd), 740 b0get(), sizeof(struct container_s), QUANTUM); 741 if (cont == NULL) { 742 return NULL; 743 } 744 ret = &cont->ctl_arena; 745 ret->astats = &cont->astats; 746 } else { 747 ret = (ctl_arena_t *)base_alloc(tsd_tsdn(tsd), b0get(), 748 sizeof(ctl_arena_t), QUANTUM); 749 if (ret == NULL) { 750 return NULL; 751 } 752 } 753 ret->arena_ind = (unsigned)i; 754 ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)] = ret; 755 } 756 757 assert(ret == NULL || arenas_i2a(ret->arena_ind) == arenas_i2a(i)); 758 return ret; 759 } 760 761 static ctl_arena_t * arenas_i(size_t i)762 arenas_i(size_t i) { 763 ctl_arena_t *ret = arenas_i_impl(tsd_fetch(), i, true, false); 764 assert(ret != NULL); 765 return ret; 766 } 767 768 static void ctl_arena_clear(ctl_arena_t * ctl_arena)769 ctl_arena_clear(ctl_arena_t *ctl_arena) { 770 ctl_arena->nthreads = 0; 771 ctl_arena->dss = dss_prec_names[dss_prec_limit]; 772 ctl_arena->dirty_decay_ms = -1; 773 ctl_arena->muzzy_decay_ms = -1; 774 ctl_arena->pactive = 0; 775 ctl_arena->pdirty = 0; 776 ctl_arena->pmuzzy = 0; 777 if (config_stats) { 778 memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t)); 779 ctl_arena->astats->allocated_small = 0; 780 ctl_arena->astats->nmalloc_small = 0; 781 ctl_arena->astats->ndalloc_small = 0; 782 ctl_arena->astats->nrequests_small = 0; 783 ctl_arena->astats->nfills_small = 0; 784 ctl_arena->astats->nflushes_small = 0; 785 memset(ctl_arena->astats->bstats, 0, SC_NBINS * 786 sizeof(bin_stats_t)); 787 memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) * 788 sizeof(arena_stats_large_t)); 789 memset(ctl_arena->astats->estats, 0, SC_NPSIZES * 790 sizeof(arena_stats_extents_t)); 791 } 792 } 793 794 static void ctl_arena_stats_amerge(tsdn_t * tsdn,ctl_arena_t * ctl_arena,arena_t * arena)795 ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { 796 unsigned i; 797 798 if (config_stats) { 799 arena_stats_merge(tsdn, arena, &ctl_arena->nthreads, 800 &ctl_arena->dss, &ctl_arena->dirty_decay_ms, 801 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, 802 &ctl_arena->pdirty, &ctl_arena->pmuzzy, 803 &ctl_arena->astats->astats, ctl_arena->astats->bstats, 804 ctl_arena->astats->lstats, ctl_arena->astats->estats); 805 806 for (i = 0; i < SC_NBINS; i++) { 807 ctl_arena->astats->allocated_small += 808 ctl_arena->astats->bstats[i].curregs * 809 sz_index2size(i); 810 ctl_arena->astats->nmalloc_small += 811 ctl_arena->astats->bstats[i].nmalloc; 812 ctl_arena->astats->ndalloc_small += 813 ctl_arena->astats->bstats[i].ndalloc; 814 ctl_arena->astats->nrequests_small += 815 ctl_arena->astats->bstats[i].nrequests; 816 ctl_arena->astats->nfills_small += 817 ctl_arena->astats->bstats[i].nfills; 818 ctl_arena->astats->nflushes_small += 819 ctl_arena->astats->bstats[i].nflushes; 820 } 821 } else { 822 arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads, 823 &ctl_arena->dss, &ctl_arena->dirty_decay_ms, 824 &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, 825 &ctl_arena->pdirty, &ctl_arena->pmuzzy); 826 } 827 } 828 829 static void ctl_arena_stats_sdmerge(ctl_arena_t * ctl_sdarena,ctl_arena_t * ctl_arena,bool destroyed)830 ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, 831 bool destroyed) { 832 unsigned i; 833 834 if (!destroyed) { 835 ctl_sdarena->nthreads += ctl_arena->nthreads; 836 ctl_sdarena->pactive += ctl_arena->pactive; 837 ctl_sdarena->pdirty += ctl_arena->pdirty; 838 ctl_sdarena->pmuzzy += ctl_arena->pmuzzy; 839 } else { 840 assert(ctl_arena->nthreads == 0); 841 assert(ctl_arena->pactive == 0); 842 assert(ctl_arena->pdirty == 0); 843 assert(ctl_arena->pmuzzy == 0); 844 } 845 846 if (config_stats) { 847 ctl_arena_stats_t *sdstats = ctl_sdarena->astats; 848 ctl_arena_stats_t *astats = ctl_arena->astats; 849 850 if (!destroyed) { 851 accum_atomic_zu(&sdstats->astats.mapped, 852 &astats->astats.mapped); 853 accum_atomic_zu(&sdstats->astats.retained, 854 &astats->astats.retained); 855 accum_atomic_zu(&sdstats->astats.extent_avail, 856 &astats->astats.extent_avail); 857 } 858 859 ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge, 860 &astats->astats.decay_dirty.npurge); 861 ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.nmadvise, 862 &astats->astats.decay_dirty.nmadvise); 863 ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.purged, 864 &astats->astats.decay_dirty.purged); 865 866 ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.npurge, 867 &astats->astats.decay_muzzy.npurge); 868 ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.nmadvise, 869 &astats->astats.decay_muzzy.nmadvise); 870 ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged, 871 &astats->astats.decay_muzzy.purged); 872 873 #define OP(mtx) malloc_mutex_prof_merge( \ 874 &(sdstats->astats.mutex_prof_data[ \ 875 arena_prof_mutex_##mtx]), \ 876 &(astats->astats.mutex_prof_data[ \ 877 arena_prof_mutex_##mtx])); 878 MUTEX_PROF_ARENA_MUTEXES 879 #undef OP 880 if (!destroyed) { 881 accum_atomic_zu(&sdstats->astats.base, 882 &astats->astats.base); 883 accum_atomic_zu(&sdstats->astats.internal, 884 &astats->astats.internal); 885 accum_atomic_zu(&sdstats->astats.resident, 886 &astats->astats.resident); 887 accum_atomic_zu(&sdstats->astats.metadata_thp, 888 &astats->astats.metadata_thp); 889 } else { 890 assert(atomic_load_zu( 891 &astats->astats.internal, ATOMIC_RELAXED) == 0); 892 } 893 894 if (!destroyed) { 895 sdstats->allocated_small += astats->allocated_small; 896 } else { 897 assert(astats->allocated_small == 0); 898 } 899 sdstats->nmalloc_small += astats->nmalloc_small; 900 sdstats->ndalloc_small += astats->ndalloc_small; 901 sdstats->nrequests_small += astats->nrequests_small; 902 sdstats->nfills_small += astats->nfills_small; 903 sdstats->nflushes_small += astats->nflushes_small; 904 905 if (!destroyed) { 906 accum_atomic_zu(&sdstats->astats.allocated_large, 907 &astats->astats.allocated_large); 908 } else { 909 assert(atomic_load_zu(&astats->astats.allocated_large, 910 ATOMIC_RELAXED) == 0); 911 } 912 ctl_accum_arena_stats_u64(&sdstats->astats.nmalloc_large, 913 &astats->astats.nmalloc_large); 914 ctl_accum_arena_stats_u64(&sdstats->astats.ndalloc_large, 915 &astats->astats.ndalloc_large); 916 ctl_accum_arena_stats_u64(&sdstats->astats.nrequests_large, 917 &astats->astats.nrequests_large); 918 accum_atomic_zu(&sdstats->astats.abandoned_vm, 919 &astats->astats.abandoned_vm); 920 921 accum_atomic_zu(&sdstats->astats.tcache_bytes, 922 &astats->astats.tcache_bytes); 923 924 if (ctl_arena->arena_ind == 0) { 925 sdstats->astats.uptime = astats->astats.uptime; 926 } 927 928 /* Merge bin stats. */ 929 for (i = 0; i < SC_NBINS; i++) { 930 sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; 931 sdstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; 932 sdstats->bstats[i].nrequests += 933 astats->bstats[i].nrequests; 934 if (!destroyed) { 935 sdstats->bstats[i].curregs += 936 astats->bstats[i].curregs; 937 } else { 938 assert(astats->bstats[i].curregs == 0); 939 } 940 sdstats->bstats[i].nfills += astats->bstats[i].nfills; 941 sdstats->bstats[i].nflushes += 942 astats->bstats[i].nflushes; 943 sdstats->bstats[i].nslabs += astats->bstats[i].nslabs; 944 sdstats->bstats[i].reslabs += astats->bstats[i].reslabs; 945 if (!destroyed) { 946 sdstats->bstats[i].curslabs += 947 astats->bstats[i].curslabs; 948 sdstats->bstats[i].nonfull_slabs += 949 astats->bstats[i].nonfull_slabs; 950 } else { 951 assert(astats->bstats[i].curslabs == 0); 952 assert(astats->bstats[i].nonfull_slabs == 0); 953 } 954 malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data, 955 &astats->bstats[i].mutex_data); 956 } 957 958 /* Merge stats for large allocations. */ 959 for (i = 0; i < SC_NSIZES - SC_NBINS; i++) { 960 ctl_accum_arena_stats_u64(&sdstats->lstats[i].nmalloc, 961 &astats->lstats[i].nmalloc); 962 ctl_accum_arena_stats_u64(&sdstats->lstats[i].ndalloc, 963 &astats->lstats[i].ndalloc); 964 ctl_accum_arena_stats_u64(&sdstats->lstats[i].nrequests, 965 &astats->lstats[i].nrequests); 966 if (!destroyed) { 967 sdstats->lstats[i].curlextents += 968 astats->lstats[i].curlextents; 969 } else { 970 assert(astats->lstats[i].curlextents == 0); 971 } 972 } 973 974 /* Merge extents stats. */ 975 for (i = 0; i < SC_NPSIZES; i++) { 976 accum_atomic_zu(&sdstats->estats[i].ndirty, 977 &astats->estats[i].ndirty); 978 accum_atomic_zu(&sdstats->estats[i].nmuzzy, 979 &astats->estats[i].nmuzzy); 980 accum_atomic_zu(&sdstats->estats[i].nretained, 981 &astats->estats[i].nretained); 982 accum_atomic_zu(&sdstats->estats[i].dirty_bytes, 983 &astats->estats[i].dirty_bytes); 984 accum_atomic_zu(&sdstats->estats[i].muzzy_bytes, 985 &astats->estats[i].muzzy_bytes); 986 accum_atomic_zu(&sdstats->estats[i].retained_bytes, 987 &astats->estats[i].retained_bytes); 988 } 989 } 990 } 991 992 static void ctl_arena_refresh(tsdn_t * tsdn,arena_t * arena,ctl_arena_t * ctl_sdarena,unsigned i,bool destroyed)993 ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena, 994 unsigned i, bool destroyed) { 995 ctl_arena_t *ctl_arena = arenas_i(i); 996 997 ctl_arena_clear(ctl_arena); 998 ctl_arena_stats_amerge(tsdn, ctl_arena, arena); 999 /* Merge into sum stats as well. */ 1000 ctl_arena_stats_sdmerge(ctl_sdarena, ctl_arena, destroyed); 1001 } 1002 1003 static unsigned ctl_arena_init(tsd_t * tsd,extent_hooks_t * extent_hooks)1004 ctl_arena_init(tsd_t *tsd, extent_hooks_t *extent_hooks) { 1005 unsigned arena_ind; 1006 ctl_arena_t *ctl_arena; 1007 1008 if ((ctl_arena = ql_last(&ctl_arenas->destroyed, destroyed_link)) != 1009 NULL) { 1010 ql_remove(&ctl_arenas->destroyed, ctl_arena, destroyed_link); 1011 arena_ind = ctl_arena->arena_ind; 1012 } else { 1013 arena_ind = ctl_arenas->narenas; 1014 } 1015 1016 /* Trigger stats allocation. */ 1017 if (arenas_i_impl(tsd, arena_ind, false, true) == NULL) { 1018 return UINT_MAX; 1019 } 1020 1021 /* Initialize new arena. */ 1022 if (arena_init(tsd_tsdn(tsd), arena_ind, extent_hooks) == NULL) { 1023 return UINT_MAX; 1024 } 1025 1026 if (arena_ind == ctl_arenas->narenas) { 1027 ctl_arenas->narenas++; 1028 } 1029 1030 return arena_ind; 1031 } 1032 1033 static void ctl_background_thread_stats_read(tsdn_t * tsdn)1034 ctl_background_thread_stats_read(tsdn_t *tsdn) { 1035 background_thread_stats_t *stats = &ctl_stats->background_thread; 1036 if (!have_background_thread || 1037 background_thread_stats_read(tsdn, stats)) { 1038 memset(stats, 0, sizeof(background_thread_stats_t)); 1039 nstime_init(&stats->run_interval, 0); 1040 } 1041 } 1042 1043 static void ctl_refresh(tsdn_t * tsdn)1044 ctl_refresh(tsdn_t *tsdn) { 1045 unsigned i; 1046 ctl_arena_t *ctl_sarena = arenas_i(MALLCTL_ARENAS_ALL); 1047 VARIABLE_ARRAY(arena_t *, tarenas, ctl_arenas->narenas); 1048 1049 /* 1050 * Clear sum stats, since they will be merged into by 1051 * ctl_arena_refresh(). 1052 */ 1053 ctl_arena_clear(ctl_sarena); 1054 1055 for (i = 0; i < ctl_arenas->narenas; i++) { 1056 tarenas[i] = arena_get(tsdn, i, false); 1057 } 1058 1059 for (i = 0; i < ctl_arenas->narenas; i++) { 1060 ctl_arena_t *ctl_arena = arenas_i(i); 1061 bool initialized = (tarenas[i] != NULL); 1062 1063 ctl_arena->initialized = initialized; 1064 if (initialized) { 1065 ctl_arena_refresh(tsdn, tarenas[i], ctl_sarena, i, 1066 false); 1067 } 1068 } 1069 1070 if (config_stats) { 1071 ctl_stats->allocated = ctl_sarena->astats->allocated_small + 1072 atomic_load_zu(&ctl_sarena->astats->astats.allocated_large, 1073 ATOMIC_RELAXED); 1074 ctl_stats->active = (ctl_sarena->pactive << LG_PAGE); 1075 ctl_stats->metadata = atomic_load_zu( 1076 &ctl_sarena->astats->astats.base, ATOMIC_RELAXED) + 1077 atomic_load_zu(&ctl_sarena->astats->astats.internal, 1078 ATOMIC_RELAXED); 1079 ctl_stats->metadata_thp = atomic_load_zu( 1080 &ctl_sarena->astats->astats.metadata_thp, ATOMIC_RELAXED); 1081 ctl_stats->resident = atomic_load_zu( 1082 &ctl_sarena->astats->astats.resident, ATOMIC_RELAXED); 1083 ctl_stats->mapped = atomic_load_zu( 1084 &ctl_sarena->astats->astats.mapped, ATOMIC_RELAXED); 1085 ctl_stats->retained = atomic_load_zu( 1086 &ctl_sarena->astats->astats.retained, ATOMIC_RELAXED); 1087 1088 ctl_background_thread_stats_read(tsdn); 1089 1090 #define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \ 1091 malloc_mutex_lock(tsdn, &mtx); \ 1092 malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \ 1093 malloc_mutex_unlock(tsdn, &mtx); 1094 1095 if (config_prof && opt_prof) { 1096 READ_GLOBAL_MUTEX_PROF_DATA(global_prof_mutex_prof, 1097 bt2gctx_mtx); 1098 } 1099 if (have_background_thread) { 1100 READ_GLOBAL_MUTEX_PROF_DATA( 1101 global_prof_mutex_background_thread, 1102 background_thread_lock); 1103 } else { 1104 memset(&ctl_stats->mutex_prof_data[ 1105 global_prof_mutex_background_thread], 0, 1106 sizeof(mutex_prof_data_t)); 1107 } 1108 /* We own ctl mutex already. */ 1109 malloc_mutex_prof_read(tsdn, 1110 &ctl_stats->mutex_prof_data[global_prof_mutex_ctl], 1111 &ctl_mtx); 1112 #undef READ_GLOBAL_MUTEX_PROF_DATA 1113 } 1114 ctl_arenas->epoch++; 1115 } 1116 1117 static bool ctl_init(tsd_t * tsd)1118 ctl_init(tsd_t *tsd) { 1119 bool ret; 1120 tsdn_t *tsdn = tsd_tsdn(tsd); 1121 1122 malloc_mutex_lock(tsdn, &ctl_mtx); 1123 if (!ctl_initialized) { 1124 ctl_arena_t *ctl_sarena, *ctl_darena; 1125 unsigned i; 1126 1127 /* 1128 * Allocate demand-zeroed space for pointers to the full 1129 * range of supported arena indices. 1130 */ 1131 if (ctl_arenas == NULL) { 1132 ctl_arenas = (ctl_arenas_t *)base_alloc(tsdn, 1133 b0get(), sizeof(ctl_arenas_t), QUANTUM); 1134 if (ctl_arenas == NULL) { 1135 ret = true; 1136 goto label_return; 1137 } 1138 } 1139 1140 if (config_stats && ctl_stats == NULL) { 1141 ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(), 1142 sizeof(ctl_stats_t), QUANTUM); 1143 if (ctl_stats == NULL) { 1144 ret = true; 1145 goto label_return; 1146 } 1147 } 1148 1149 /* 1150 * Allocate space for the current full range of arenas 1151 * here rather than doing it lazily elsewhere, in order 1152 * to limit when OOM-caused errors can occur. 1153 */ 1154 if ((ctl_sarena = arenas_i_impl(tsd, MALLCTL_ARENAS_ALL, false, 1155 true)) == NULL) { 1156 ret = true; 1157 goto label_return; 1158 } 1159 ctl_sarena->initialized = true; 1160 1161 if ((ctl_darena = arenas_i_impl(tsd, MALLCTL_ARENAS_DESTROYED, 1162 false, true)) == NULL) { 1163 ret = true; 1164 goto label_return; 1165 } 1166 ctl_arena_clear(ctl_darena); 1167 /* 1168 * Don't toggle ctl_darena to initialized until an arena is 1169 * actually destroyed, so that arena.<i>.initialized can be used 1170 * to query whether the stats are relevant. 1171 */ 1172 1173 ctl_arenas->narenas = narenas_total_get(); 1174 for (i = 0; i < ctl_arenas->narenas; i++) { 1175 if (arenas_i_impl(tsd, i, false, true) == NULL) { 1176 ret = true; 1177 goto label_return; 1178 } 1179 } 1180 1181 ql_new(&ctl_arenas->destroyed); 1182 ctl_refresh(tsdn); 1183 1184 ctl_initialized = true; 1185 } 1186 1187 ret = false; 1188 label_return: 1189 malloc_mutex_unlock(tsdn, &ctl_mtx); 1190 return ret; 1191 } 1192 1193 static int ctl_lookup(tsdn_t * tsdn,const char * name,ctl_node_t const ** nodesp,size_t * mibp,size_t * depthp)1194 ctl_lookup(tsdn_t *tsdn, const char *name, ctl_node_t const **nodesp, 1195 size_t *mibp, size_t *depthp) { 1196 int ret; 1197 const char *elm, *tdot, *dot; 1198 size_t elen, i, j; 1199 const ctl_named_node_t *node; 1200 1201 elm = name; 1202 /* Equivalent to strchrnul(). */ 1203 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0'); 1204 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); 1205 if (elen == 0) { 1206 ret = ENOENT; 1207 goto label_return; 1208 } 1209 node = super_root_node; 1210 for (i = 0; i < *depthp; i++) { 1211 assert(node); 1212 assert(node->nchildren > 0); 1213 if (ctl_named_node(node->children) != NULL) { 1214 const ctl_named_node_t *pnode = node; 1215 1216 /* Children are named. */ 1217 for (j = 0; j < node->nchildren; j++) { 1218 const ctl_named_node_t *child = 1219 ctl_named_children(node, j); 1220 if (strlen(child->name) == elen && 1221 strncmp(elm, child->name, elen) == 0) { 1222 node = child; 1223 if (nodesp != NULL) { 1224 nodesp[i] = 1225 (const ctl_node_t *)node; 1226 } 1227 mibp[i] = j; 1228 break; 1229 } 1230 } 1231 if (node == pnode) { 1232 ret = ENOENT; 1233 goto label_return; 1234 } 1235 } else { 1236 uintmax_t index; 1237 const ctl_indexed_node_t *inode; 1238 1239 /* Children are indexed. */ 1240 index = malloc_strtoumax(elm, NULL, 10); 1241 if (index == UINTMAX_MAX || index > SIZE_T_MAX) { 1242 ret = ENOENT; 1243 goto label_return; 1244 } 1245 1246 inode = ctl_indexed_node(node->children); 1247 node = inode->index(tsdn, mibp, *depthp, (size_t)index); 1248 if (node == NULL) { 1249 ret = ENOENT; 1250 goto label_return; 1251 } 1252 1253 if (nodesp != NULL) { 1254 nodesp[i] = (const ctl_node_t *)node; 1255 } 1256 mibp[i] = (size_t)index; 1257 } 1258 1259 if (node->ctl != NULL) { 1260 /* Terminal node. */ 1261 if (*dot != '\0') { 1262 /* 1263 * The name contains more elements than are 1264 * in this path through the tree. 1265 */ 1266 ret = ENOENT; 1267 goto label_return; 1268 } 1269 /* Complete lookup successful. */ 1270 *depthp = i + 1; 1271 break; 1272 } 1273 1274 /* Update elm. */ 1275 if (*dot == '\0') { 1276 /* No more elements. */ 1277 ret = ENOENT; 1278 goto label_return; 1279 } 1280 elm = &dot[1]; 1281 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : 1282 strchr(elm, '\0'); 1283 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); 1284 } 1285 1286 ret = 0; 1287 label_return: 1288 return ret; 1289 } 1290 1291 int ctl_byname(tsd_t * tsd,const char * name,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1292 ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, 1293 void *newp, size_t newlen) { 1294 int ret; 1295 size_t depth; 1296 ctl_node_t const *nodes[CTL_MAX_DEPTH]; 1297 size_t mib[CTL_MAX_DEPTH]; 1298 const ctl_named_node_t *node; 1299 1300 if (!ctl_initialized && ctl_init(tsd)) { 1301 ret = EAGAIN; 1302 goto label_return; 1303 } 1304 1305 depth = CTL_MAX_DEPTH; 1306 ret = ctl_lookup(tsd_tsdn(tsd), name, nodes, mib, &depth); 1307 if (ret != 0) { 1308 goto label_return; 1309 } 1310 1311 node = ctl_named_node(nodes[depth-1]); 1312 if (node != NULL && node->ctl) { 1313 ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen); 1314 } else { 1315 /* The name refers to a partial path through the ctl tree. */ 1316 ret = ENOENT; 1317 } 1318 1319 label_return: 1320 return(ret); 1321 } 1322 1323 int ctl_nametomib(tsd_t * tsd,const char * name,size_t * mibp,size_t * miblenp)1324 ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp) { 1325 int ret; 1326 1327 if (!ctl_initialized && ctl_init(tsd)) { 1328 ret = EAGAIN; 1329 goto label_return; 1330 } 1331 1332 ret = ctl_lookup(tsd_tsdn(tsd), name, NULL, mibp, miblenp); 1333 label_return: 1334 return(ret); 1335 } 1336 1337 int ctl_bymib(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1338 ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 1339 size_t *oldlenp, void *newp, size_t newlen) { 1340 int ret; 1341 const ctl_named_node_t *node; 1342 size_t i; 1343 1344 if (!ctl_initialized && ctl_init(tsd)) { 1345 ret = EAGAIN; 1346 goto label_return; 1347 } 1348 1349 /* Iterate down the tree. */ 1350 node = super_root_node; 1351 for (i = 0; i < miblen; i++) { 1352 assert(node); 1353 assert(node->nchildren > 0); 1354 if (ctl_named_node(node->children) != NULL) { 1355 /* Children are named. */ 1356 if (node->nchildren <= mib[i]) { 1357 ret = ENOENT; 1358 goto label_return; 1359 } 1360 node = ctl_named_children(node, mib[i]); 1361 } else { 1362 const ctl_indexed_node_t *inode; 1363 1364 /* Indexed element. */ 1365 inode = ctl_indexed_node(node->children); 1366 node = inode->index(tsd_tsdn(tsd), mib, miblen, mib[i]); 1367 if (node == NULL) { 1368 ret = ENOENT; 1369 goto label_return; 1370 } 1371 } 1372 } 1373 1374 /* Call the ctl function. */ 1375 if (node && node->ctl) { 1376 ret = node->ctl(tsd, mib, miblen, oldp, oldlenp, newp, newlen); 1377 } else { 1378 /* Partial MIB. */ 1379 ret = ENOENT; 1380 } 1381 1382 label_return: 1383 return(ret); 1384 } 1385 1386 bool ctl_boot(void)1387 ctl_boot(void) { 1388 if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL, 1389 malloc_mutex_rank_exclusive)) { 1390 return true; 1391 } 1392 1393 ctl_initialized = false; 1394 1395 return false; 1396 } 1397 1398 void ctl_prefork(tsdn_t * tsdn)1399 ctl_prefork(tsdn_t *tsdn) { 1400 malloc_mutex_prefork(tsdn, &ctl_mtx); 1401 } 1402 1403 void ctl_postfork_parent(tsdn_t * tsdn)1404 ctl_postfork_parent(tsdn_t *tsdn) { 1405 malloc_mutex_postfork_parent(tsdn, &ctl_mtx); 1406 } 1407 1408 void ctl_postfork_child(tsdn_t * tsdn)1409 ctl_postfork_child(tsdn_t *tsdn) { 1410 malloc_mutex_postfork_child(tsdn, &ctl_mtx); 1411 } 1412 1413 /******************************************************************************/ 1414 /* *_ctl() functions. */ 1415 1416 #define READONLY() do { \ 1417 if (newp != NULL || newlen != 0) { \ 1418 ret = EPERM; \ 1419 goto label_return; \ 1420 } \ 1421 } while (0) 1422 1423 #define WRITEONLY() do { \ 1424 if (oldp != NULL || oldlenp != NULL) { \ 1425 ret = EPERM; \ 1426 goto label_return; \ 1427 } \ 1428 } while (0) 1429 1430 #define READ_XOR_WRITE() do { \ 1431 if ((oldp != NULL && oldlenp != NULL) && (newp != NULL || \ 1432 newlen != 0)) { \ 1433 ret = EPERM; \ 1434 goto label_return; \ 1435 } \ 1436 } while (0) 1437 1438 #define READ(v, t) do { \ 1439 if (oldp != NULL && oldlenp != NULL) { \ 1440 if (*oldlenp != sizeof(t)) { \ 1441 size_t copylen = (sizeof(t) <= *oldlenp) \ 1442 ? sizeof(t) : *oldlenp; \ 1443 memcpy(oldp, (void *)&(v), copylen); \ 1444 ret = EINVAL; \ 1445 goto label_return; \ 1446 } \ 1447 *(t *)oldp = (v); \ 1448 } \ 1449 } while (0) 1450 1451 #define WRITE(v, t) do { \ 1452 if (newp != NULL) { \ 1453 if (newlen != sizeof(t)) { \ 1454 ret = EINVAL; \ 1455 goto label_return; \ 1456 } \ 1457 (v) = *(t *)newp; \ 1458 } \ 1459 } while (0) 1460 1461 #define MIB_UNSIGNED(v, i) do { \ 1462 if (mib[i] > UINT_MAX) { \ 1463 ret = EFAULT; \ 1464 goto label_return; \ 1465 } \ 1466 v = (unsigned)mib[i]; \ 1467 } while (0) 1468 1469 /* 1470 * There's a lot of code duplication in the following macros due to limitations 1471 * in how nested cpp macros are expanded. 1472 */ 1473 #define CTL_RO_CLGEN(c, l, n, v, t) \ 1474 static int \ 1475 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1476 size_t *oldlenp, void *newp, size_t newlen) { \ 1477 int ret; \ 1478 t oldval; \ 1479 \ 1480 if (!(c)) { \ 1481 return ENOENT; \ 1482 } \ 1483 if (l) { \ 1484 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1485 } \ 1486 READONLY(); \ 1487 oldval = (v); \ 1488 READ(oldval, t); \ 1489 \ 1490 ret = 0; \ 1491 label_return: \ 1492 if (l) { \ 1493 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1494 } \ 1495 return ret; \ 1496 } 1497 1498 #define CTL_RO_CGEN(c, n, v, t) \ 1499 static int \ 1500 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 1501 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ 1502 int ret; \ 1503 t oldval; \ 1504 \ 1505 if (!(c)) { \ 1506 return ENOENT; \ 1507 } \ 1508 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1509 READONLY(); \ 1510 oldval = (v); \ 1511 READ(oldval, t); \ 1512 \ 1513 ret = 0; \ 1514 label_return: \ 1515 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1516 return ret; \ 1517 } 1518 1519 #define CTL_RO_GEN(n, v, t) \ 1520 static int \ 1521 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1522 size_t *oldlenp, void *newp, size_t newlen) { \ 1523 int ret; \ 1524 t oldval; \ 1525 \ 1526 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \ 1527 READONLY(); \ 1528 oldval = (v); \ 1529 READ(oldval, t); \ 1530 \ 1531 ret = 0; \ 1532 label_return: \ 1533 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \ 1534 return ret; \ 1535 } 1536 1537 /* 1538 * ctl_mtx is not acquired, under the assumption that no pertinent data will 1539 * mutate during the call. 1540 */ 1541 #define CTL_RO_NL_CGEN(c, n, v, t) \ 1542 static int \ 1543 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 1544 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ 1545 int ret; \ 1546 t oldval; \ 1547 \ 1548 if (!(c)) { \ 1549 return ENOENT; \ 1550 } \ 1551 READONLY(); \ 1552 oldval = (v); \ 1553 READ(oldval, t); \ 1554 \ 1555 ret = 0; \ 1556 label_return: \ 1557 return ret; \ 1558 } 1559 1560 #define CTL_RO_NL_GEN(n, v, t) \ 1561 static int \ 1562 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 1563 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ 1564 int ret; \ 1565 t oldval; \ 1566 \ 1567 READONLY(); \ 1568 oldval = (v); \ 1569 READ(oldval, t); \ 1570 \ 1571 ret = 0; \ 1572 label_return: \ 1573 return ret; \ 1574 } 1575 1576 #define CTL_TSD_RO_NL_CGEN(c, n, m, t) \ 1577 static int \ 1578 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ 1579 size_t *oldlenp, void *newp, size_t newlen) { \ 1580 int ret; \ 1581 t oldval; \ 1582 \ 1583 if (!(c)) { \ 1584 return ENOENT; \ 1585 } \ 1586 READONLY(); \ 1587 oldval = (m(tsd)); \ 1588 READ(oldval, t); \ 1589 \ 1590 ret = 0; \ 1591 label_return: \ 1592 return ret; \ 1593 } 1594 1595 #define CTL_RO_CONFIG_GEN(n, t) \ 1596 static int \ 1597 n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ 1598 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ 1599 int ret; \ 1600 t oldval; \ 1601 \ 1602 READONLY(); \ 1603 oldval = n; \ 1604 READ(oldval, t); \ 1605 \ 1606 ret = 0; \ 1607 label_return: \ 1608 return ret; \ 1609 } 1610 1611 /******************************************************************************/ 1612 CTL_RO_NL_GEN(version,JEMALLOC_VERSION,const char *)1613 CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) 1614 1615 static int 1616 epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1617 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1618 int ret; 1619 UNUSED uint64_t newval; 1620 1621 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 1622 WRITE(newval, uint64_t); 1623 if (newp != NULL) { 1624 ctl_refresh(tsd_tsdn(tsd)); 1625 } 1626 READ(ctl_arenas->epoch, uint64_t); 1627 1628 ret = 0; 1629 label_return: 1630 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 1631 return ret; 1632 } 1633 1634 static int background_thread_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1635 background_thread_ctl(tsd_t *tsd, const size_t *mib, 1636 size_t miblen, void *oldp, size_t *oldlenp, 1637 void *newp, size_t newlen) { 1638 int ret; 1639 bool oldval; 1640 1641 if (!have_background_thread) { 1642 return ENOENT; 1643 } 1644 background_thread_ctl_init(tsd_tsdn(tsd)); 1645 1646 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 1647 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 1648 if (newp == NULL) { 1649 oldval = background_thread_enabled(); 1650 READ(oldval, bool); 1651 } else { 1652 if (newlen != sizeof(bool)) { 1653 ret = EINVAL; 1654 goto label_return; 1655 } 1656 oldval = background_thread_enabled(); 1657 READ(oldval, bool); 1658 1659 bool newval = *(bool *)newp; 1660 if (newval == oldval) { 1661 ret = 0; 1662 goto label_return; 1663 } 1664 1665 background_thread_enabled_set(tsd_tsdn(tsd), newval); 1666 if (newval) { 1667 if (background_threads_enable(tsd)) { 1668 ret = EFAULT; 1669 goto label_return; 1670 } 1671 } else { 1672 if (background_threads_disable(tsd)) { 1673 ret = EFAULT; 1674 goto label_return; 1675 } 1676 } 1677 } 1678 ret = 0; 1679 label_return: 1680 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 1681 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 1682 1683 return ret; 1684 } 1685 1686 static int max_background_threads_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1687 max_background_threads_ctl(tsd_t *tsd, const size_t *mib, 1688 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 1689 size_t newlen) { 1690 int ret; 1691 size_t oldval; 1692 1693 if (!have_background_thread) { 1694 return ENOENT; 1695 } 1696 background_thread_ctl_init(tsd_tsdn(tsd)); 1697 1698 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 1699 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 1700 if (newp == NULL) { 1701 oldval = max_background_threads; 1702 READ(oldval, size_t); 1703 } else { 1704 if (newlen != sizeof(size_t)) { 1705 ret = EINVAL; 1706 goto label_return; 1707 } 1708 oldval = max_background_threads; 1709 READ(oldval, size_t); 1710 1711 size_t newval = *(size_t *)newp; 1712 if (newval == oldval) { 1713 ret = 0; 1714 goto label_return; 1715 } 1716 if (newval > opt_max_background_threads) { 1717 ret = EINVAL; 1718 goto label_return; 1719 } 1720 1721 if (background_thread_enabled()) { 1722 background_thread_enabled_set(tsd_tsdn(tsd), false); 1723 if (background_threads_disable(tsd)) { 1724 ret = EFAULT; 1725 goto label_return; 1726 } 1727 max_background_threads = newval; 1728 background_thread_enabled_set(tsd_tsdn(tsd), true); 1729 if (background_threads_enable(tsd)) { 1730 ret = EFAULT; 1731 goto label_return; 1732 } 1733 } else { 1734 max_background_threads = newval; 1735 } 1736 } 1737 ret = 0; 1738 label_return: 1739 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 1740 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 1741 1742 return ret; 1743 } 1744 1745 /******************************************************************************/ 1746 CTL_RO_CONFIG_GEN(config_cache_oblivious,bool)1747 CTL_RO_CONFIG_GEN(config_cache_oblivious, bool) 1748 CTL_RO_CONFIG_GEN(config_debug, bool) 1749 CTL_RO_CONFIG_GEN(config_fill, bool) 1750 CTL_RO_CONFIG_GEN(config_lazy_lock, bool) 1751 CTL_RO_CONFIG_GEN(config_malloc_conf, const char *) 1752 CTL_RO_CONFIG_GEN(config_opt_safety_checks, bool) 1753 CTL_RO_CONFIG_GEN(config_prof, bool) 1754 CTL_RO_CONFIG_GEN(config_prof_libgcc, bool) 1755 CTL_RO_CONFIG_GEN(config_prof_libunwind, bool) 1756 CTL_RO_CONFIG_GEN(config_stats, bool) 1757 CTL_RO_CONFIG_GEN(config_utrace, bool) 1758 CTL_RO_CONFIG_GEN(config_xmalloc, bool) 1759 1760 /******************************************************************************/ 1761 1762 CTL_RO_NL_GEN(opt_abort, opt_abort, bool) 1763 CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) 1764 CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool) 1765 CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], 1766 const char *) 1767 CTL_RO_NL_GEN(opt_retain, opt_retain, bool) 1768 CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) 1769 CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) 1770 CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena], 1771 const char *) 1772 CTL_RO_NL_GEN(opt_oversize_threshold, opt_oversize_threshold, size_t) 1773 CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool) 1774 CTL_RO_NL_GEN(opt_max_background_threads, opt_max_background_threads, size_t) 1775 CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t) 1776 CTL_RO_NL_GEN(opt_muzzy_decay_ms, opt_muzzy_decay_ms, ssize_t) 1777 CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) 1778 CTL_RO_NL_GEN(opt_stats_print_opts, opt_stats_print_opts, const char *) 1779 CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *) 1780 CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) 1781 CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) 1782 CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) 1783 CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) 1784 CTL_RO_NL_GEN(opt_thp, thp_mode_names[opt_thp], const char *) 1785 CTL_RO_NL_GEN(opt_lg_extent_max_active_fit, opt_lg_extent_max_active_fit, 1786 size_t) 1787 CTL_RO_NL_GEN(opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) 1788 CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) 1789 CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) 1790 CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) 1791 CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init, 1792 opt_prof_thread_active_init, bool) 1793 CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) 1794 CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) 1795 CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) 1796 CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool) 1797 CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) 1798 CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) 1799 1800 /******************************************************************************/ 1801 1802 static int 1803 thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1804 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1805 int ret; 1806 arena_t *oldarena; 1807 unsigned newind, oldind; 1808 1809 oldarena = arena_choose(tsd, NULL); 1810 if (oldarena == NULL) { 1811 return EAGAIN; 1812 } 1813 newind = oldind = arena_ind_get(oldarena); 1814 WRITE(newind, unsigned); 1815 READ(oldind, unsigned); 1816 1817 if (newind != oldind) { 1818 arena_t *newarena; 1819 1820 if (newind >= narenas_total_get()) { 1821 /* New arena index is out of range. */ 1822 ret = EFAULT; 1823 goto label_return; 1824 } 1825 1826 if (have_percpu_arena && 1827 PERCPU_ARENA_ENABLED(opt_percpu_arena)) { 1828 if (newind < percpu_arena_ind_limit(opt_percpu_arena)) { 1829 /* 1830 * If perCPU arena is enabled, thread_arena 1831 * control is not allowed for the auto arena 1832 * range. 1833 */ 1834 ret = EPERM; 1835 goto label_return; 1836 } 1837 } 1838 1839 /* Initialize arena if necessary. */ 1840 newarena = arena_get(tsd_tsdn(tsd), newind, true); 1841 if (newarena == NULL) { 1842 ret = EAGAIN; 1843 goto label_return; 1844 } 1845 /* Set new arena/tcache associations. */ 1846 arena_migrate(tsd, oldind, newind); 1847 if (tcache_available(tsd)) { 1848 tcache_arena_reassociate(tsd_tsdn(tsd), 1849 tsd_tcachep_get(tsd), newarena); 1850 } 1851 } 1852 1853 ret = 0; 1854 label_return: 1855 return ret; 1856 } 1857 CTL_TSD_RO_NL_CGEN(config_stats,thread_allocated,tsd_thread_allocated_get,uint64_t)1858 CTL_TSD_RO_NL_CGEN(config_stats, thread_allocated, tsd_thread_allocated_get, 1859 uint64_t) 1860 CTL_TSD_RO_NL_CGEN(config_stats, thread_allocatedp, tsd_thread_allocatedp_get, 1861 uint64_t *) 1862 CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocated, tsd_thread_deallocated_get, 1863 uint64_t) 1864 CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp, 1865 tsd_thread_deallocatedp_get, uint64_t *) 1866 1867 static int 1868 thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, 1869 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 1870 size_t newlen) { 1871 int ret; 1872 bool oldval; 1873 1874 oldval = tcache_enabled_get(tsd); 1875 if (newp != NULL) { 1876 if (newlen != sizeof(bool)) { 1877 ret = EINVAL; 1878 goto label_return; 1879 } 1880 tcache_enabled_set(tsd, *(bool *)newp); 1881 } 1882 READ(oldval, bool); 1883 1884 ret = 0; 1885 label_return: 1886 return ret; 1887 } 1888 1889 static int thread_tcache_flush_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1890 thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, 1891 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 1892 size_t newlen) { 1893 int ret; 1894 1895 if (!tcache_available(tsd)) { 1896 ret = EFAULT; 1897 goto label_return; 1898 } 1899 1900 READONLY(); 1901 WRITEONLY(); 1902 1903 tcache_flush(tsd); 1904 1905 ret = 0; 1906 label_return: 1907 return ret; 1908 } 1909 1910 static int thread_prof_name_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1911 thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, 1912 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 1913 size_t newlen) { 1914 int ret; 1915 1916 if (!config_prof) { 1917 return ENOENT; 1918 } 1919 1920 READ_XOR_WRITE(); 1921 1922 if (newp != NULL) { 1923 if (newlen != sizeof(const char *)) { 1924 ret = EINVAL; 1925 goto label_return; 1926 } 1927 1928 if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) != 1929 0) { 1930 goto label_return; 1931 } 1932 } else { 1933 const char *oldname = prof_thread_name_get(tsd); 1934 READ(oldname, const char *); 1935 } 1936 1937 ret = 0; 1938 label_return: 1939 return ret; 1940 } 1941 1942 static int thread_prof_active_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1943 thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, 1944 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 1945 size_t newlen) { 1946 int ret; 1947 bool oldval; 1948 1949 if (!config_prof) { 1950 return ENOENT; 1951 } 1952 1953 oldval = prof_thread_active_get(tsd); 1954 if (newp != NULL) { 1955 if (newlen != sizeof(bool)) { 1956 ret = EINVAL; 1957 goto label_return; 1958 } 1959 if (prof_thread_active_set(tsd, *(bool *)newp)) { 1960 ret = EAGAIN; 1961 goto label_return; 1962 } 1963 } 1964 READ(oldval, bool); 1965 1966 ret = 0; 1967 label_return: 1968 return ret; 1969 } 1970 1971 /******************************************************************************/ 1972 1973 static int tcache_create_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1974 tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1975 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1976 int ret; 1977 unsigned tcache_ind; 1978 1979 READONLY(); 1980 if (tcaches_create(tsd, &tcache_ind)) { 1981 ret = EFAULT; 1982 goto label_return; 1983 } 1984 READ(tcache_ind, unsigned); 1985 1986 ret = 0; 1987 label_return: 1988 return ret; 1989 } 1990 1991 static int tcache_flush_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)1992 tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 1993 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 1994 int ret; 1995 unsigned tcache_ind; 1996 1997 WRITEONLY(); 1998 tcache_ind = UINT_MAX; 1999 WRITE(tcache_ind, unsigned); 2000 if (tcache_ind == UINT_MAX) { 2001 ret = EFAULT; 2002 goto label_return; 2003 } 2004 tcaches_flush(tsd, tcache_ind); 2005 2006 ret = 0; 2007 label_return: 2008 return ret; 2009 } 2010 2011 static int tcache_destroy_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2012 tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2013 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2014 int ret; 2015 unsigned tcache_ind; 2016 2017 WRITEONLY(); 2018 tcache_ind = UINT_MAX; 2019 WRITE(tcache_ind, unsigned); 2020 if (tcache_ind == UINT_MAX) { 2021 ret = EFAULT; 2022 goto label_return; 2023 } 2024 tcaches_destroy(tsd, tcache_ind); 2025 2026 ret = 0; 2027 label_return: 2028 return ret; 2029 } 2030 2031 /******************************************************************************/ 2032 2033 static int arena_i_initialized_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2034 arena_i_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2035 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2036 int ret; 2037 tsdn_t *tsdn = tsd_tsdn(tsd); 2038 unsigned arena_ind; 2039 bool initialized; 2040 2041 READONLY(); 2042 MIB_UNSIGNED(arena_ind, 1); 2043 2044 malloc_mutex_lock(tsdn, &ctl_mtx); 2045 initialized = arenas_i(arena_ind)->initialized; 2046 malloc_mutex_unlock(tsdn, &ctl_mtx); 2047 2048 READ(initialized, bool); 2049 2050 ret = 0; 2051 label_return: 2052 return ret; 2053 } 2054 2055 static void arena_i_decay(tsdn_t * tsdn,unsigned arena_ind,bool all)2056 arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) { 2057 malloc_mutex_lock(tsdn, &ctl_mtx); 2058 { 2059 unsigned narenas = ctl_arenas->narenas; 2060 2061 /* 2062 * Access via index narenas is deprecated, and scheduled for 2063 * removal in 6.0.0. 2064 */ 2065 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == narenas) { 2066 unsigned i; 2067 VARIABLE_ARRAY(arena_t *, tarenas, narenas); 2068 2069 for (i = 0; i < narenas; i++) { 2070 tarenas[i] = arena_get(tsdn, i, false); 2071 } 2072 2073 /* 2074 * No further need to hold ctl_mtx, since narenas and 2075 * tarenas contain everything needed below. 2076 */ 2077 malloc_mutex_unlock(tsdn, &ctl_mtx); 2078 2079 for (i = 0; i < narenas; i++) { 2080 if (tarenas[i] != NULL) { 2081 arena_decay(tsdn, tarenas[i], false, 2082 all); 2083 } 2084 } 2085 } else { 2086 arena_t *tarena; 2087 2088 assert(arena_ind < narenas); 2089 2090 tarena = arena_get(tsdn, arena_ind, false); 2091 2092 /* No further need to hold ctl_mtx. */ 2093 malloc_mutex_unlock(tsdn, &ctl_mtx); 2094 2095 if (tarena != NULL) { 2096 arena_decay(tsdn, tarena, false, all); 2097 } 2098 } 2099 } 2100 } 2101 2102 static int arena_i_decay_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2103 arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2104 size_t *oldlenp, void *newp, size_t newlen) { 2105 int ret; 2106 unsigned arena_ind; 2107 2108 READONLY(); 2109 WRITEONLY(); 2110 MIB_UNSIGNED(arena_ind, 1); 2111 arena_i_decay(tsd_tsdn(tsd), arena_ind, false); 2112 2113 ret = 0; 2114 label_return: 2115 return ret; 2116 } 2117 2118 static int arena_i_purge_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2119 arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2120 size_t *oldlenp, void *newp, size_t newlen) { 2121 int ret; 2122 unsigned arena_ind; 2123 2124 READONLY(); 2125 WRITEONLY(); 2126 MIB_UNSIGNED(arena_ind, 1); 2127 arena_i_decay(tsd_tsdn(tsd), arena_ind, true); 2128 2129 ret = 0; 2130 label_return: 2131 return ret; 2132 } 2133 2134 static int arena_i_reset_destroy_helper(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen,unsigned * arena_ind,arena_t ** arena)2135 arena_i_reset_destroy_helper(tsd_t *tsd, const size_t *mib, size_t miblen, 2136 void *oldp, size_t *oldlenp, void *newp, size_t newlen, unsigned *arena_ind, 2137 arena_t **arena) { 2138 int ret; 2139 2140 READONLY(); 2141 WRITEONLY(); 2142 MIB_UNSIGNED(*arena_ind, 1); 2143 2144 *arena = arena_get(tsd_tsdn(tsd), *arena_ind, false); 2145 if (*arena == NULL || arena_is_auto(*arena)) { 2146 ret = EFAULT; 2147 goto label_return; 2148 } 2149 2150 ret = 0; 2151 label_return: 2152 return ret; 2153 } 2154 2155 static void arena_reset_prepare_background_thread(tsd_t * tsd,unsigned arena_ind)2156 arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) { 2157 /* Temporarily disable the background thread during arena reset. */ 2158 if (have_background_thread) { 2159 malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); 2160 if (background_thread_enabled()) { 2161 background_thread_info_t *info = 2162 background_thread_info_get(arena_ind); 2163 assert(info->state == background_thread_started); 2164 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 2165 info->state = background_thread_paused; 2166 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 2167 } 2168 } 2169 } 2170 2171 static void arena_reset_finish_background_thread(tsd_t * tsd,unsigned arena_ind)2172 arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) { 2173 if (have_background_thread) { 2174 if (background_thread_enabled()) { 2175 background_thread_info_t *info = 2176 background_thread_info_get(arena_ind); 2177 assert(info->state == background_thread_paused); 2178 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); 2179 info->state = background_thread_started; 2180 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); 2181 } 2182 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); 2183 } 2184 } 2185 2186 static int arena_i_reset_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2187 arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2188 size_t *oldlenp, void *newp, size_t newlen) { 2189 int ret; 2190 unsigned arena_ind; 2191 arena_t *arena; 2192 2193 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp, 2194 newp, newlen, &arena_ind, &arena); 2195 if (ret != 0) { 2196 return ret; 2197 } 2198 2199 arena_reset_prepare_background_thread(tsd, arena_ind); 2200 arena_reset(tsd, arena); 2201 arena_reset_finish_background_thread(tsd, arena_ind); 2202 2203 return ret; 2204 } 2205 2206 static int arena_i_destroy_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2207 arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2208 size_t *oldlenp, void *newp, size_t newlen) { 2209 int ret; 2210 unsigned arena_ind; 2211 arena_t *arena; 2212 ctl_arena_t *ctl_darena, *ctl_arena; 2213 2214 ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp, 2215 newp, newlen, &arena_ind, &arena); 2216 if (ret != 0) { 2217 goto label_return; 2218 } 2219 2220 if (arena_nthreads_get(arena, false) != 0 || arena_nthreads_get(arena, 2221 true) != 0) { 2222 ret = EFAULT; 2223 goto label_return; 2224 } 2225 2226 arena_reset_prepare_background_thread(tsd, arena_ind); 2227 /* Merge stats after resetting and purging arena. */ 2228 arena_reset(tsd, arena); 2229 arena_decay(tsd_tsdn(tsd), arena, false, true); 2230 ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED); 2231 ctl_darena->initialized = true; 2232 ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true); 2233 /* Destroy arena. */ 2234 arena_destroy(tsd, arena); 2235 ctl_arena = arenas_i(arena_ind); 2236 ctl_arena->initialized = false; 2237 /* Record arena index for later recycling via arenas.create. */ 2238 ql_elm_new(ctl_arena, destroyed_link); 2239 ql_tail_insert(&ctl_arenas->destroyed, ctl_arena, destroyed_link); 2240 arena_reset_finish_background_thread(tsd, arena_ind); 2241 2242 assert(ret == 0); 2243 label_return: 2244 return ret; 2245 } 2246 2247 static int arena_i_dss_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2248 arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2249 size_t *oldlenp, void *newp, size_t newlen) { 2250 int ret; 2251 const char *dss = NULL; 2252 unsigned arena_ind; 2253 dss_prec_t dss_prec_old = dss_prec_limit; 2254 dss_prec_t dss_prec = dss_prec_limit; 2255 2256 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2257 WRITE(dss, const char *); 2258 MIB_UNSIGNED(arena_ind, 1); 2259 if (dss != NULL) { 2260 int i; 2261 bool match = false; 2262 2263 for (i = 0; i < dss_prec_limit; i++) { 2264 if (strcmp(dss_prec_names[i], dss) == 0) { 2265 dss_prec = i; 2266 match = true; 2267 break; 2268 } 2269 } 2270 2271 if (!match) { 2272 ret = EINVAL; 2273 goto label_return; 2274 } 2275 } 2276 2277 /* 2278 * Access via index narenas is deprecated, and scheduled for removal in 2279 * 6.0.0. 2280 */ 2281 if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == 2282 ctl_arenas->narenas) { 2283 if (dss_prec != dss_prec_limit && 2284 extent_dss_prec_set(dss_prec)) { 2285 ret = EFAULT; 2286 goto label_return; 2287 } 2288 dss_prec_old = extent_dss_prec_get(); 2289 } else { 2290 arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2291 if (arena == NULL || (dss_prec != dss_prec_limit && 2292 arena_dss_prec_set(arena, dss_prec))) { 2293 ret = EFAULT; 2294 goto label_return; 2295 } 2296 dss_prec_old = arena_dss_prec_get(arena); 2297 } 2298 2299 dss = dss_prec_names[dss_prec_old]; 2300 READ(dss, const char *); 2301 2302 ret = 0; 2303 label_return: 2304 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2305 return ret; 2306 } 2307 2308 static int arena_i_decay_ms_ctl_impl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen,bool dirty)2309 arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, 2310 void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { 2311 int ret; 2312 unsigned arena_ind; 2313 arena_t *arena; 2314 2315 MIB_UNSIGNED(arena_ind, 1); 2316 arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2317 if (arena == NULL) { 2318 ret = EFAULT; 2319 goto label_return; 2320 } 2321 2322 if (oldp != NULL && oldlenp != NULL) { 2323 size_t oldval = dirty ? arena_dirty_decay_ms_get(arena) : 2324 arena_muzzy_decay_ms_get(arena); 2325 READ(oldval, ssize_t); 2326 } 2327 if (newp != NULL) { 2328 if (newlen != sizeof(ssize_t)) { 2329 ret = EINVAL; 2330 goto label_return; 2331 } 2332 if (arena_is_huge(arena_ind) && *(ssize_t *)newp > 0) { 2333 /* 2334 * By default the huge arena purges eagerly. If it is 2335 * set to non-zero decay time afterwards, background 2336 * thread might be needed. 2337 */ 2338 if (background_thread_create(tsd, arena_ind)) { 2339 ret = EFAULT; 2340 goto label_return; 2341 } 2342 } 2343 if (dirty ? arena_dirty_decay_ms_set(tsd_tsdn(tsd), arena, 2344 *(ssize_t *)newp) : arena_muzzy_decay_ms_set(tsd_tsdn(tsd), 2345 arena, *(ssize_t *)newp)) { 2346 ret = EFAULT; 2347 goto label_return; 2348 } 2349 } 2350 2351 ret = 0; 2352 label_return: 2353 return ret; 2354 } 2355 2356 static int arena_i_dirty_decay_ms_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2357 arena_i_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2358 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2359 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2360 newlen, true); 2361 } 2362 2363 static int arena_i_muzzy_decay_ms_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2364 arena_i_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2365 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2366 return arena_i_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2367 newlen, false); 2368 } 2369 2370 static int arena_i_extent_hooks_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2371 arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2372 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2373 int ret; 2374 unsigned arena_ind; 2375 arena_t *arena; 2376 2377 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2378 MIB_UNSIGNED(arena_ind, 1); 2379 if (arena_ind < narenas_total_get()) { 2380 extent_hooks_t *old_extent_hooks; 2381 arena = arena_get(tsd_tsdn(tsd), arena_ind, false); 2382 if (arena == NULL) { 2383 if (arena_ind >= narenas_auto) { 2384 ret = EFAULT; 2385 goto label_return; 2386 } 2387 old_extent_hooks = 2388 (extent_hooks_t *)&extent_hooks_default; 2389 READ(old_extent_hooks, extent_hooks_t *); 2390 if (newp != NULL) { 2391 /* Initialize a new arena as a side effect. */ 2392 extent_hooks_t *new_extent_hooks 2393 JEMALLOC_CC_SILENCE_INIT(NULL); 2394 WRITE(new_extent_hooks, extent_hooks_t *); 2395 arena = arena_init(tsd_tsdn(tsd), arena_ind, 2396 new_extent_hooks); 2397 if (arena == NULL) { 2398 ret = EFAULT; 2399 goto label_return; 2400 } 2401 } 2402 } else { 2403 if (newp != NULL) { 2404 extent_hooks_t *new_extent_hooks 2405 JEMALLOC_CC_SILENCE_INIT(NULL); 2406 WRITE(new_extent_hooks, extent_hooks_t *); 2407 old_extent_hooks = extent_hooks_set(tsd, arena, 2408 new_extent_hooks); 2409 READ(old_extent_hooks, extent_hooks_t *); 2410 } else { 2411 old_extent_hooks = extent_hooks_get(arena); 2412 READ(old_extent_hooks, extent_hooks_t *); 2413 } 2414 } 2415 } else { 2416 ret = EFAULT; 2417 goto label_return; 2418 } 2419 ret = 0; 2420 label_return: 2421 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2422 return ret; 2423 } 2424 2425 static int arena_i_retain_grow_limit_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2426 arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, 2427 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2428 size_t newlen) { 2429 int ret; 2430 unsigned arena_ind; 2431 arena_t *arena; 2432 2433 if (!opt_retain) { 2434 /* Only relevant when retain is enabled. */ 2435 return ENOENT; 2436 } 2437 2438 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2439 MIB_UNSIGNED(arena_ind, 1); 2440 if (arena_ind < narenas_total_get() && (arena = 2441 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { 2442 size_t old_limit, new_limit; 2443 if (newp != NULL) { 2444 WRITE(new_limit, size_t); 2445 } 2446 bool err = arena_retain_grow_limit_get_set(tsd, arena, 2447 &old_limit, newp != NULL ? &new_limit : NULL); 2448 if (!err) { 2449 READ(old_limit, size_t); 2450 ret = 0; 2451 } else { 2452 ret = EFAULT; 2453 } 2454 } else { 2455 ret = EFAULT; 2456 } 2457 label_return: 2458 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2459 return ret; 2460 } 2461 2462 static const ctl_named_node_t * arena_i_index(tsdn_t * tsdn,const size_t * mib,size_t miblen,size_t i)2463 arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, 2464 size_t i) { 2465 const ctl_named_node_t *ret; 2466 2467 malloc_mutex_lock(tsdn, &ctl_mtx); 2468 switch (i) { 2469 case MALLCTL_ARENAS_ALL: 2470 case MALLCTL_ARENAS_DESTROYED: 2471 break; 2472 default: 2473 if (i > ctl_arenas->narenas) { 2474 ret = NULL; 2475 goto label_return; 2476 } 2477 break; 2478 } 2479 2480 ret = super_arena_i_node; 2481 label_return: 2482 malloc_mutex_unlock(tsdn, &ctl_mtx); 2483 return ret; 2484 } 2485 2486 /******************************************************************************/ 2487 2488 static int arenas_narenas_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2489 arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2490 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2491 int ret; 2492 unsigned narenas; 2493 2494 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2495 READONLY(); 2496 if (*oldlenp != sizeof(unsigned)) { 2497 ret = EINVAL; 2498 goto label_return; 2499 } 2500 narenas = ctl_arenas->narenas; 2501 READ(narenas, unsigned); 2502 2503 ret = 0; 2504 label_return: 2505 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2506 return ret; 2507 } 2508 2509 static int arenas_decay_ms_ctl_impl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen,bool dirty)2510 arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, 2511 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2512 size_t newlen, bool dirty) { 2513 int ret; 2514 2515 if (oldp != NULL && oldlenp != NULL) { 2516 size_t oldval = (dirty ? arena_dirty_decay_ms_default_get() : 2517 arena_muzzy_decay_ms_default_get()); 2518 READ(oldval, ssize_t); 2519 } 2520 if (newp != NULL) { 2521 if (newlen != sizeof(ssize_t)) { 2522 ret = EINVAL; 2523 goto label_return; 2524 } 2525 if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp) 2526 : arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) { 2527 ret = EFAULT; 2528 goto label_return; 2529 } 2530 } 2531 2532 ret = 0; 2533 label_return: 2534 return ret; 2535 } 2536 2537 static int arenas_dirty_decay_ms_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2538 arenas_dirty_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2539 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2540 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2541 newlen, true); 2542 } 2543 2544 static int arenas_muzzy_decay_ms_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2545 arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2546 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2547 return arenas_decay_ms_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, 2548 newlen, false); 2549 } 2550 CTL_RO_NL_GEN(arenas_quantum,QUANTUM,size_t)2551 CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) 2552 CTL_RO_NL_GEN(arenas_page, PAGE, size_t) 2553 CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) 2554 CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned) 2555 CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) 2556 CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t) 2557 CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t) 2558 CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t) 2559 CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t) 2560 static const ctl_named_node_t * 2561 arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, 2562 size_t miblen, size_t i) { 2563 if (i > SC_NBINS) { 2564 return NULL; 2565 } 2566 return super_arenas_bin_i_node; 2567 } 2568 2569 CTL_RO_NL_GEN(arenas_nlextents, SC_NSIZES - SC_NBINS, unsigned) 2570 CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(SC_NBINS+(szind_t)mib[2]), 2571 size_t) 2572 static const ctl_named_node_t * arenas_lextent_i_index(tsdn_t * tsdn,const size_t * mib,size_t miblen,size_t i)2573 arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, 2574 size_t miblen, size_t i) { 2575 if (i > SC_NSIZES - SC_NBINS) { 2576 return NULL; 2577 } 2578 return super_arenas_lextent_i_node; 2579 } 2580 2581 static int arenas_create_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2582 arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2583 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2584 int ret; 2585 extent_hooks_t *extent_hooks; 2586 unsigned arena_ind; 2587 2588 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2589 2590 extent_hooks = (extent_hooks_t *)&extent_hooks_default; 2591 WRITE(extent_hooks, extent_hooks_t *); 2592 if ((arena_ind = ctl_arena_init(tsd, extent_hooks)) == UINT_MAX) { 2593 ret = EAGAIN; 2594 goto label_return; 2595 } 2596 READ(arena_ind, unsigned); 2597 2598 ret = 0; 2599 label_return: 2600 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2601 return ret; 2602 } 2603 2604 static int arenas_lookup_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2605 arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, 2606 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2607 size_t newlen) { 2608 int ret; 2609 unsigned arena_ind; 2610 void *ptr; 2611 extent_t *extent; 2612 arena_t *arena; 2613 2614 ptr = NULL; 2615 ret = EINVAL; 2616 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 2617 WRITE(ptr, void *); 2618 extent = iealloc(tsd_tsdn(tsd), ptr); 2619 if (extent == NULL) 2620 goto label_return; 2621 2622 arena = extent_arena_get(extent); 2623 if (arena == NULL) 2624 goto label_return; 2625 2626 arena_ind = arena_ind_get(arena); 2627 READ(arena_ind, unsigned); 2628 2629 ret = 0; 2630 label_return: 2631 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 2632 return ret; 2633 } 2634 2635 /******************************************************************************/ 2636 2637 static int prof_thread_active_init_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2638 prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, 2639 size_t miblen, void *oldp, size_t *oldlenp, void *newp, 2640 size_t newlen) { 2641 int ret; 2642 bool oldval; 2643 2644 if (!config_prof) { 2645 return ENOENT; 2646 } 2647 2648 if (newp != NULL) { 2649 if (newlen != sizeof(bool)) { 2650 ret = EINVAL; 2651 goto label_return; 2652 } 2653 oldval = prof_thread_active_init_set(tsd_tsdn(tsd), 2654 *(bool *)newp); 2655 } else { 2656 oldval = prof_thread_active_init_get(tsd_tsdn(tsd)); 2657 } 2658 READ(oldval, bool); 2659 2660 ret = 0; 2661 label_return: 2662 return ret; 2663 } 2664 2665 static int prof_active_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2666 prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2667 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2668 int ret; 2669 bool oldval; 2670 2671 if (!config_prof) { 2672 return ENOENT; 2673 } 2674 2675 if (newp != NULL) { 2676 if (newlen != sizeof(bool)) { 2677 ret = EINVAL; 2678 goto label_return; 2679 } 2680 oldval = prof_active_set(tsd_tsdn(tsd), *(bool *)newp); 2681 } else { 2682 oldval = prof_active_get(tsd_tsdn(tsd)); 2683 } 2684 READ(oldval, bool); 2685 2686 ret = 0; 2687 label_return: 2688 return ret; 2689 } 2690 2691 static int prof_dump_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2692 prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2693 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2694 int ret; 2695 const char *filename = NULL; 2696 2697 if (!config_prof) { 2698 return ENOENT; 2699 } 2700 2701 WRITEONLY(); 2702 WRITE(filename, const char *); 2703 2704 if (prof_mdump(tsd, filename)) { 2705 ret = EFAULT; 2706 goto label_return; 2707 } 2708 2709 ret = 0; 2710 label_return: 2711 return ret; 2712 } 2713 2714 static int prof_gdump_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2715 prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2716 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2717 int ret; 2718 bool oldval; 2719 2720 if (!config_prof) { 2721 return ENOENT; 2722 } 2723 2724 if (newp != NULL) { 2725 if (newlen != sizeof(bool)) { 2726 ret = EINVAL; 2727 goto label_return; 2728 } 2729 oldval = prof_gdump_set(tsd_tsdn(tsd), *(bool *)newp); 2730 } else { 2731 oldval = prof_gdump_get(tsd_tsdn(tsd)); 2732 } 2733 READ(oldval, bool); 2734 2735 ret = 0; 2736 label_return: 2737 return ret; 2738 } 2739 2740 static int prof_reset_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2741 prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 2742 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 2743 int ret; 2744 size_t lg_sample = lg_prof_sample; 2745 2746 if (!config_prof) { 2747 return ENOENT; 2748 } 2749 2750 WRITEONLY(); 2751 WRITE(lg_sample, size_t); 2752 if (lg_sample >= (sizeof(uint64_t) << 3)) { 2753 lg_sample = (sizeof(uint64_t) << 3) - 1; 2754 } 2755 2756 prof_reset(tsd, lg_sample); 2757 2758 ret = 0; 2759 label_return: 2760 return ret; 2761 } 2762 CTL_RO_NL_CGEN(config_prof,prof_interval,prof_interval,uint64_t)2763 CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) 2764 CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) 2765 2766 static int 2767 prof_log_start_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2768 size_t *oldlenp, void *newp, size_t newlen) { 2769 int ret; 2770 2771 const char *filename = NULL; 2772 2773 if (!config_prof) { 2774 return ENOENT; 2775 } 2776 2777 WRITEONLY(); 2778 WRITE(filename, const char *); 2779 2780 if (prof_log_start(tsd_tsdn(tsd), filename)) { 2781 ret = EFAULT; 2782 goto label_return; 2783 } 2784 2785 ret = 0; 2786 label_return: 2787 return ret; 2788 } 2789 2790 static int prof_log_stop_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2791 prof_log_stop_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, 2792 size_t *oldlenp, void *newp, size_t newlen) { 2793 if (!config_prof) { 2794 return ENOENT; 2795 } 2796 2797 if (prof_log_stop(tsd_tsdn(tsd))) { 2798 return EFAULT; 2799 } 2800 2801 return 0; 2802 } 2803 2804 /******************************************************************************/ 2805 2806 CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t) 2807 CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t) 2808 CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t) 2809 CTL_RO_CGEN(config_stats, stats_metadata_thp, ctl_stats->metadata_thp, size_t) 2810 CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t) 2811 CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t) 2812 CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t) 2813 2814 CTL_RO_CGEN(config_stats, stats_background_thread_num_threads, 2815 ctl_stats->background_thread.num_threads, size_t) 2816 CTL_RO_CGEN(config_stats, stats_background_thread_num_runs, 2817 ctl_stats->background_thread.num_runs, uint64_t) 2818 CTL_RO_CGEN(config_stats, stats_background_thread_run_interval, 2819 nstime_ns(&ctl_stats->background_thread.run_interval), uint64_t) 2820 2821 CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *) 2822 CTL_RO_GEN(stats_arenas_i_dirty_decay_ms, arenas_i(mib[2])->dirty_decay_ms, 2823 ssize_t) 2824 CTL_RO_GEN(stats_arenas_i_muzzy_decay_ms, arenas_i(mib[2])->muzzy_decay_ms, 2825 ssize_t) 2826 CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned) 2827 CTL_RO_GEN(stats_arenas_i_uptime, 2828 nstime_ns(&arenas_i(mib[2])->astats->astats.uptime), uint64_t) 2829 CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t) 2830 CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t) 2831 CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t) 2832 CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, 2833 atomic_load_zu(&arenas_i(mib[2])->astats->astats.mapped, ATOMIC_RELAXED), 2834 size_t) 2835 CTL_RO_CGEN(config_stats, stats_arenas_i_retained, 2836 atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED), 2837 size_t) 2838 CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail, 2839 atomic_load_zu(&arenas_i(mib[2])->astats->astats.extent_avail, 2840 ATOMIC_RELAXED), 2841 size_t) 2842 2843 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, 2844 ctl_arena_stats_read_u64( 2845 &arenas_i(mib[2])->astats->astats.decay_dirty.npurge), uint64_t) 2846 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise, 2847 ctl_arena_stats_read_u64( 2848 &arenas_i(mib[2])->astats->astats.decay_dirty.nmadvise), uint64_t) 2849 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged, 2850 ctl_arena_stats_read_u64( 2851 &arenas_i(mib[2])->astats->astats.decay_dirty.purged), uint64_t) 2852 2853 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge, 2854 ctl_arena_stats_read_u64( 2855 &arenas_i(mib[2])->astats->astats.decay_muzzy.npurge), uint64_t) 2856 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise, 2857 ctl_arena_stats_read_u64( 2858 &arenas_i(mib[2])->astats->astats.decay_muzzy.nmadvise), uint64_t) 2859 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged, 2860 ctl_arena_stats_read_u64( 2861 &arenas_i(mib[2])->astats->astats.decay_muzzy.purged), uint64_t) 2862 2863 CTL_RO_CGEN(config_stats, stats_arenas_i_base, 2864 atomic_load_zu(&arenas_i(mib[2])->astats->astats.base, ATOMIC_RELAXED), 2865 size_t) 2866 CTL_RO_CGEN(config_stats, stats_arenas_i_internal, 2867 atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED), 2868 size_t) 2869 CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_thp, 2870 atomic_load_zu(&arenas_i(mib[2])->astats->astats.metadata_thp, 2871 ATOMIC_RELAXED), size_t) 2872 CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes, 2873 atomic_load_zu(&arenas_i(mib[2])->astats->astats.tcache_bytes, 2874 ATOMIC_RELAXED), size_t) 2875 CTL_RO_CGEN(config_stats, stats_arenas_i_resident, 2876 atomic_load_zu(&arenas_i(mib[2])->astats->astats.resident, ATOMIC_RELAXED), 2877 size_t) 2878 CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm, 2879 atomic_load_zu(&arenas_i(mib[2])->astats->astats.abandoned_vm, 2880 ATOMIC_RELAXED), size_t) 2881 2882 CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, 2883 arenas_i(mib[2])->astats->allocated_small, size_t) 2884 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc, 2885 arenas_i(mib[2])->astats->nmalloc_small, uint64_t) 2886 CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc, 2887 arenas_i(mib[2])->astats->ndalloc_small, uint64_t) 2888 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests, 2889 arenas_i(mib[2])->astats->nrequests_small, uint64_t) 2890 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nfills, 2891 arenas_i(mib[2])->astats->nfills_small, uint64_t) 2892 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nflushes, 2893 arenas_i(mib[2])->astats->nflushes_small, uint64_t) 2894 CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated, 2895 atomic_load_zu(&arenas_i(mib[2])->astats->astats.allocated_large, 2896 ATOMIC_RELAXED), size_t) 2897 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc, 2898 ctl_arena_stats_read_u64( 2899 &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t) 2900 CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, 2901 ctl_arena_stats_read_u64( 2902 &arenas_i(mib[2])->astats->astats.ndalloc_large), uint64_t) 2903 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, 2904 ctl_arena_stats_read_u64( 2905 &arenas_i(mib[2])->astats->astats.nrequests_large), uint64_t) 2906 /* 2907 * Note: "nmalloc_large" here instead of "nfills" in the read. This is 2908 * intentional (large has no batch fill). 2909 */ 2910 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nfills, 2911 ctl_arena_stats_read_u64( 2912 &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t) 2913 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nflushes, 2914 ctl_arena_stats_read_u64( 2915 &arenas_i(mib[2])->astats->astats.nflushes_large), uint64_t) 2916 2917 /* Lock profiling related APIs below. */ 2918 #define RO_MUTEX_CTL_GEN(n, l) \ 2919 CTL_RO_CGEN(config_stats, stats_##n##_num_ops, \ 2920 l.n_lock_ops, uint64_t) \ 2921 CTL_RO_CGEN(config_stats, stats_##n##_num_wait, \ 2922 l.n_wait_times, uint64_t) \ 2923 CTL_RO_CGEN(config_stats, stats_##n##_num_spin_acq, \ 2924 l.n_spin_acquired, uint64_t) \ 2925 CTL_RO_CGEN(config_stats, stats_##n##_num_owner_switch, \ 2926 l.n_owner_switches, uint64_t) \ 2927 CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \ 2928 nstime_ns(&l.tot_wait_time), uint64_t) \ 2929 CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \ 2930 nstime_ns(&l.max_wait_time), uint64_t) \ 2931 CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \ 2932 l.max_n_thds, uint32_t) 2933 2934 /* Global mutexes. */ 2935 #define OP(mtx) \ 2936 RO_MUTEX_CTL_GEN(mutexes_##mtx, \ 2937 ctl_stats->mutex_prof_data[global_prof_mutex_##mtx]) 2938 MUTEX_PROF_GLOBAL_MUTEXES 2939 #undef OP 2940 2941 /* Per arena mutexes */ 2942 #define OP(mtx) RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \ 2943 arenas_i(mib[2])->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx]) 2944 MUTEX_PROF_ARENA_MUTEXES 2945 #undef OP 2946 2947 /* tcache bin mutex */ 2948 RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, 2949 arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data) 2950 #undef RO_MUTEX_CTL_GEN 2951 2952 /* Resets all mutex stats, including global, arena and bin mutexes. */ 2953 static int stats_mutexes_reset_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)2954 stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, 2955 size_t miblen, void *oldp, size_t *oldlenp, 2956 void *newp, size_t newlen) { 2957 if (!config_stats) { 2958 return ENOENT; 2959 } 2960 2961 tsdn_t *tsdn = tsd_tsdn(tsd); 2962 2963 #define MUTEX_PROF_RESET(mtx) \ 2964 malloc_mutex_lock(tsdn, &mtx); \ 2965 malloc_mutex_prof_data_reset(tsdn, &mtx); \ 2966 malloc_mutex_unlock(tsdn, &mtx); 2967 2968 /* Global mutexes: ctl and prof. */ 2969 MUTEX_PROF_RESET(ctl_mtx); 2970 if (have_background_thread) { 2971 MUTEX_PROF_RESET(background_thread_lock); 2972 } 2973 if (config_prof && opt_prof) { 2974 MUTEX_PROF_RESET(bt2gctx_mtx); 2975 } 2976 2977 2978 /* Per arena mutexes. */ 2979 unsigned n = narenas_total_get(); 2980 2981 for (unsigned i = 0; i < n; i++) { 2982 arena_t *arena = arena_get(tsdn, i, false); 2983 if (!arena) { 2984 continue; 2985 } 2986 MUTEX_PROF_RESET(arena->large_mtx); 2987 MUTEX_PROF_RESET(arena->extent_avail_mtx); 2988 MUTEX_PROF_RESET(arena->extents_dirty.mtx); 2989 MUTEX_PROF_RESET(arena->extents_muzzy.mtx); 2990 MUTEX_PROF_RESET(arena->extents_retained.mtx); 2991 MUTEX_PROF_RESET(arena->decay_dirty.mtx); 2992 MUTEX_PROF_RESET(arena->decay_muzzy.mtx); 2993 MUTEX_PROF_RESET(arena->tcache_ql_mtx); 2994 MUTEX_PROF_RESET(arena->base->mtx); 2995 2996 for (szind_t i = 0; i < SC_NBINS; i++) { 2997 for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { 2998 bin_t *bin = &arena->bins[i].bin_shards[j]; 2999 MUTEX_PROF_RESET(bin->lock); 3000 } 3001 } 3002 } 3003 #undef MUTEX_PROF_RESET 3004 return 0; 3005 } 3006 3007 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc, 3008 arenas_i(mib[2])->astats->bstats[mib[4]].nmalloc, uint64_t) 3009 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc, 3010 arenas_i(mib[2])->astats->bstats[mib[4]].ndalloc, uint64_t) 3011 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests, 3012 arenas_i(mib[2])->astats->bstats[mib[4]].nrequests, uint64_t) 3013 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs, 3014 arenas_i(mib[2])->astats->bstats[mib[4]].curregs, size_t) 3015 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nfills, 3016 arenas_i(mib[2])->astats->bstats[mib[4]].nfills, uint64_t) 3017 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nflushes, 3018 arenas_i(mib[2])->astats->bstats[mib[4]].nflushes, uint64_t) 3019 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nslabs, 3020 arenas_i(mib[2])->astats->bstats[mib[4]].nslabs, uint64_t) 3021 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreslabs, 3022 arenas_i(mib[2])->astats->bstats[mib[4]].reslabs, uint64_t) 3023 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs, 3024 arenas_i(mib[2])->astats->bstats[mib[4]].curslabs, size_t) 3025 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nonfull_slabs, 3026 arenas_i(mib[2])->astats->bstats[mib[4]].nonfull_slabs, size_t) 3027 3028 static const ctl_named_node_t * stats_arenas_i_bins_j_index(tsdn_t * tsdn,const size_t * mib,size_t miblen,size_t j)3029 stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, 3030 size_t miblen, size_t j) { 3031 if (j > SC_NBINS) { 3032 return NULL; 3033 } 3034 return super_stats_arenas_i_bins_j_node; 3035 } 3036 3037 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc, 3038 ctl_arena_stats_read_u64( 3039 &arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), uint64_t) 3040 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc, 3041 ctl_arena_stats_read_u64( 3042 &arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), uint64_t) 3043 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests, 3044 ctl_arena_stats_read_u64( 3045 &arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), uint64_t) 3046 CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, 3047 arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) 3048 3049 static const ctl_named_node_t * stats_arenas_i_lextents_j_index(tsdn_t * tsdn,const size_t * mib,size_t miblen,size_t j)3050 stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, 3051 size_t miblen, size_t j) { 3052 if (j > SC_NSIZES - SC_NBINS) { 3053 return NULL; 3054 } 3055 return super_stats_arenas_i_lextents_j_node; 3056 } 3057 3058 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty, 3059 atomic_load_zu( 3060 &arenas_i(mib[2])->astats->estats[mib[4]].ndirty, 3061 ATOMIC_RELAXED), size_t); 3062 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy, 3063 atomic_load_zu( 3064 &arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, 3065 ATOMIC_RELAXED), size_t); 3066 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained, 3067 atomic_load_zu( 3068 &arenas_i(mib[2])->astats->estats[mib[4]].nretained, 3069 ATOMIC_RELAXED), size_t); 3070 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes, 3071 atomic_load_zu( 3072 &arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, 3073 ATOMIC_RELAXED), size_t); 3074 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes, 3075 atomic_load_zu( 3076 &arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, 3077 ATOMIC_RELAXED), size_t); 3078 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes, 3079 atomic_load_zu( 3080 &arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, 3081 ATOMIC_RELAXED), size_t); 3082 3083 static const ctl_named_node_t * stats_arenas_i_extents_j_index(tsdn_t * tsdn,const size_t * mib,size_t miblen,size_t j)3084 stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib, 3085 size_t miblen, size_t j) { 3086 if (j >= SC_NPSIZES) { 3087 return NULL; 3088 } 3089 return super_stats_arenas_i_extents_j_node; 3090 } 3091 3092 static bool ctl_arenas_i_verify(size_t i)3093 ctl_arenas_i_verify(size_t i) { 3094 size_t a = arenas_i2a_impl(i, true, true); 3095 if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) { 3096 return true; 3097 } 3098 3099 return false; 3100 } 3101 3102 static const ctl_named_node_t * stats_arenas_i_index(tsdn_t * tsdn,const size_t * mib,size_t miblen,size_t i)3103 stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, 3104 size_t miblen, size_t i) { 3105 const ctl_named_node_t *ret; 3106 3107 malloc_mutex_lock(tsdn, &ctl_mtx); 3108 if (ctl_arenas_i_verify(i)) { 3109 ret = NULL; 3110 goto label_return; 3111 } 3112 3113 ret = super_stats_arenas_i_node; 3114 label_return: 3115 malloc_mutex_unlock(tsdn, &ctl_mtx); 3116 return ret; 3117 } 3118 3119 static int experimental_hooks_install_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)3120 experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3121 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3122 int ret; 3123 if (oldp == NULL || oldlenp == NULL|| newp == NULL) { 3124 ret = EINVAL; 3125 goto label_return; 3126 } 3127 /* 3128 * Note: this is a *private* struct. This is an experimental interface; 3129 * forcing the user to know the jemalloc internals well enough to 3130 * extract the ABI hopefully ensures nobody gets too comfortable with 3131 * this API, which can change at a moment's notice. 3132 */ 3133 hooks_t hooks; 3134 WRITE(hooks, hooks_t); 3135 void *handle = hook_install(tsd_tsdn(tsd), &hooks); 3136 if (handle == NULL) { 3137 ret = EAGAIN; 3138 goto label_return; 3139 } 3140 READ(handle, void *); 3141 3142 ret = 0; 3143 label_return: 3144 return ret; 3145 } 3146 3147 static int experimental_hooks_remove_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)3148 experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, 3149 void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3150 int ret; 3151 WRITEONLY(); 3152 void *handle = NULL; 3153 WRITE(handle, void *); 3154 if (handle == NULL) { 3155 ret = EINVAL; 3156 goto label_return; 3157 } 3158 hook_remove(tsd_tsdn(tsd), handle); 3159 ret = 0; 3160 label_return: 3161 return ret; 3162 } 3163 3164 /* 3165 * Output six memory utilization entries for an input pointer, the first one of 3166 * type (void *) and the remaining five of type size_t, describing the following 3167 * (in the same order): 3168 * 3169 * (a) memory address of the extent a potential reallocation would go into, 3170 * == the five fields below describe about the extent the pointer resides in == 3171 * (b) number of free regions in the extent, 3172 * (c) number of regions in the extent, 3173 * (d) size of the extent in terms of bytes, 3174 * (e) total number of free regions in the bin the extent belongs to, and 3175 * (f) total number of regions in the bin the extent belongs to. 3176 * 3177 * Note that "(e)" and "(f)" are only available when stats are enabled; 3178 * otherwise their values are undefined. 3179 * 3180 * This API is mainly intended for small class allocations, where extents are 3181 * used as slab. 3182 * 3183 * In case of large class allocations, "(a)" will be NULL, and "(e)" and "(f)" 3184 * will be zero (if stats are enabled; otherwise undefined). The other three 3185 * fields will be properly set though the values are trivial: "(b)" will be 0, 3186 * "(c)" will be 1, and "(d)" will be the usable size. 3187 * 3188 * The input pointer and size are respectively passed in by newp and newlen, 3189 * and the output fields and size are respectively oldp and *oldlenp. 3190 * 3191 * It can be beneficial to define the following macros to make it easier to 3192 * access the output: 3193 * 3194 * #define SLABCUR_READ(out) (*(void **)out) 3195 * #define COUNTS(out) ((size_t *)((void **)out + 1)) 3196 * #define NFREE_READ(out) COUNTS(out)[0] 3197 * #define NREGS_READ(out) COUNTS(out)[1] 3198 * #define SIZE_READ(out) COUNTS(out)[2] 3199 * #define BIN_NFREE_READ(out) COUNTS(out)[3] 3200 * #define BIN_NREGS_READ(out) COUNTS(out)[4] 3201 * 3202 * and then write e.g. NFREE_READ(oldp) to fetch the output. See the unit test 3203 * test_query in test/unit/extent_util.c for an example. 3204 * 3205 * For a typical defragmentation workflow making use of this API for 3206 * understanding the fragmentation level, please refer to the comment for 3207 * experimental_utilization_batch_query_ctl. 3208 * 3209 * It's up to the application how to determine the significance of 3210 * fragmentation relying on the outputs returned. Possible choices are: 3211 * 3212 * (a) if extent utilization ratio is below certain threshold, 3213 * (b) if extent memory consumption is above certain threshold, 3214 * (c) if extent utilization ratio is significantly below bin utilization ratio, 3215 * (d) if input pointer deviates a lot from potential reallocation address, or 3216 * (e) some selection/combination of the above. 3217 * 3218 * The caller needs to make sure that the input/output arguments are valid, 3219 * in particular, that the size of the output is correct, i.e.: 3220 * 3221 * *oldlenp = sizeof(void *) + sizeof(size_t) * 5 3222 * 3223 * Otherwise, the function immediately returns EINVAL without touching anything. 3224 * 3225 * In the rare case where there's no associated extent found for the input 3226 * pointer, the function zeros out all output fields and return. Please refer 3227 * to the comment for experimental_utilization_batch_query_ctl to understand the 3228 * motivation from C++. 3229 */ 3230 static int experimental_utilization_query_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)3231 experimental_utilization_query_ctl(tsd_t *tsd, const size_t *mib, 3232 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3233 int ret; 3234 3235 assert(sizeof(extent_util_stats_verbose_t) 3236 == sizeof(void *) + sizeof(size_t) * 5); 3237 3238 if (oldp == NULL || oldlenp == NULL 3239 || *oldlenp != sizeof(extent_util_stats_verbose_t) 3240 || newp == NULL) { 3241 ret = EINVAL; 3242 goto label_return; 3243 } 3244 3245 void *ptr = NULL; 3246 WRITE(ptr, void *); 3247 extent_util_stats_verbose_t *util_stats 3248 = (extent_util_stats_verbose_t *)oldp; 3249 extent_util_stats_verbose_get(tsd_tsdn(tsd), ptr, 3250 &util_stats->nfree, &util_stats->nregs, &util_stats->size, 3251 &util_stats->bin_nfree, &util_stats->bin_nregs, 3252 &util_stats->slabcur_addr); 3253 ret = 0; 3254 3255 label_return: 3256 return ret; 3257 } 3258 3259 /* 3260 * Given an input array of pointers, output three memory utilization entries of 3261 * type size_t for each input pointer about the extent it resides in: 3262 * 3263 * (a) number of free regions in the extent, 3264 * (b) number of regions in the extent, and 3265 * (c) size of the extent in terms of bytes. 3266 * 3267 * This API is mainly intended for small class allocations, where extents are 3268 * used as slab. In case of large class allocations, the outputs are trivial: 3269 * "(a)" will be 0, "(b)" will be 1, and "(c)" will be the usable size. 3270 * 3271 * Note that multiple input pointers may reside on a same extent so the output 3272 * fields may contain duplicates. 3273 * 3274 * The format of the input/output looks like: 3275 * 3276 * input[0]: 1st_pointer_to_query | output[0]: 1st_extent_n_free_regions 3277 * | output[1]: 1st_extent_n_regions 3278 * | output[2]: 1st_extent_size 3279 * input[1]: 2nd_pointer_to_query | output[3]: 2nd_extent_n_free_regions 3280 * | output[4]: 2nd_extent_n_regions 3281 * | output[5]: 2nd_extent_size 3282 * ... | ... 3283 * 3284 * The input array and size are respectively passed in by newp and newlen, and 3285 * the output array and size are respectively oldp and *oldlenp. 3286 * 3287 * It can be beneficial to define the following macros to make it easier to 3288 * access the output: 3289 * 3290 * #define NFREE_READ(out, i) out[(i) * 3] 3291 * #define NREGS_READ(out, i) out[(i) * 3 + 1] 3292 * #define SIZE_READ(out, i) out[(i) * 3 + 2] 3293 * 3294 * and then write e.g. NFREE_READ(oldp, i) to fetch the output. See the unit 3295 * test test_batch in test/unit/extent_util.c for a concrete example. 3296 * 3297 * A typical workflow would be composed of the following steps: 3298 * 3299 * (1) flush tcache: mallctl("thread.tcache.flush", ...) 3300 * (2) initialize input array of pointers to query fragmentation 3301 * (3) allocate output array to hold utilization statistics 3302 * (4) query utilization: mallctl("experimental.utilization.batch_query", ...) 3303 * (5) (optional) decide if it's worthwhile to defragment; otherwise stop here 3304 * (6) disable tcache: mallctl("thread.tcache.enabled", ...) 3305 * (7) defragment allocations with significant fragmentation, e.g.: 3306 * for each allocation { 3307 * if it's fragmented { 3308 * malloc(...); 3309 * memcpy(...); 3310 * free(...); 3311 * } 3312 * } 3313 * (8) enable tcache: mallctl("thread.tcache.enabled", ...) 3314 * 3315 * The application can determine the significance of fragmentation themselves 3316 * relying on the statistics returned, both at the overall level i.e. step "(5)" 3317 * and at individual allocation level i.e. within step "(7)". Possible choices 3318 * are: 3319 * 3320 * (a) whether memory utilization ratio is below certain threshold, 3321 * (b) whether memory consumption is above certain threshold, or 3322 * (c) some combination of the two. 3323 * 3324 * The caller needs to make sure that the input/output arrays are valid and 3325 * their sizes are proper as well as matched, meaning: 3326 * 3327 * (a) newlen = n_pointers * sizeof(const void *) 3328 * (b) *oldlenp = n_pointers * sizeof(size_t) * 3 3329 * (c) n_pointers > 0 3330 * 3331 * Otherwise, the function immediately returns EINVAL without touching anything. 3332 * 3333 * In the rare case where there's no associated extent found for some pointers, 3334 * rather than immediately terminating the computation and raising an error, 3335 * the function simply zeros out the corresponding output fields and continues 3336 * the computation until all input pointers are handled. The motivations of 3337 * such a design are as follows: 3338 * 3339 * (a) The function always either processes nothing or processes everything, and 3340 * never leaves the output half touched and half untouched. 3341 * 3342 * (b) It facilitates usage needs especially common in C++. A vast variety of 3343 * C++ objects are instantiated with multiple dynamic memory allocations. For 3344 * example, std::string and std::vector typically use at least two allocations, 3345 * one for the metadata and one for the actual content. Other types may use 3346 * even more allocations. When inquiring about utilization statistics, the 3347 * caller often wants to examine into all such allocations, especially internal 3348 * one(s), rather than just the topmost one. The issue comes when some 3349 * implementations do certain optimizations to reduce/aggregate some internal 3350 * allocations, e.g. putting short strings directly into the metadata, and such 3351 * decisions are not known to the caller. Therefore, we permit pointers to 3352 * memory usages that may not be returned by previous malloc calls, and we 3353 * provide the caller a convenient way to identify such cases. 3354 */ 3355 static int experimental_utilization_batch_query_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)3356 experimental_utilization_batch_query_ctl(tsd_t *tsd, const size_t *mib, 3357 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3358 int ret; 3359 3360 assert(sizeof(extent_util_stats_t) == sizeof(size_t) * 3); 3361 3362 const size_t len = newlen / sizeof(const void *); 3363 if (oldp == NULL || oldlenp == NULL || newp == NULL || newlen == 0 3364 || newlen != len * sizeof(const void *) 3365 || *oldlenp != len * sizeof(extent_util_stats_t)) { 3366 ret = EINVAL; 3367 goto label_return; 3368 } 3369 3370 void **ptrs = (void **)newp; 3371 extent_util_stats_t *util_stats = (extent_util_stats_t *)oldp; 3372 size_t i; 3373 for (i = 0; i < len; ++i) { 3374 extent_util_stats_get(tsd_tsdn(tsd), ptrs[i], 3375 &util_stats[i].nfree, &util_stats[i].nregs, 3376 &util_stats[i].size); 3377 } 3378 ret = 0; 3379 3380 label_return: 3381 return ret; 3382 } 3383 3384 static const ctl_named_node_t * experimental_arenas_i_index(tsdn_t * tsdn,const size_t * mib,size_t miblen,size_t i)3385 experimental_arenas_i_index(tsdn_t *tsdn, const size_t *mib, 3386 size_t miblen, size_t i) { 3387 const ctl_named_node_t *ret; 3388 3389 malloc_mutex_lock(tsdn, &ctl_mtx); 3390 if (ctl_arenas_i_verify(i)) { 3391 ret = NULL; 3392 goto label_return; 3393 } 3394 ret = super_experimental_arenas_i_node; 3395 label_return: 3396 malloc_mutex_unlock(tsdn, &ctl_mtx); 3397 return ret; 3398 } 3399 3400 static int experimental_arenas_i_pactivep_ctl(tsd_t * tsd,const size_t * mib,size_t miblen,void * oldp,size_t * oldlenp,void * newp,size_t newlen)3401 experimental_arenas_i_pactivep_ctl(tsd_t *tsd, const size_t *mib, 3402 size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { 3403 if (!config_stats) { 3404 return ENOENT; 3405 } 3406 if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(size_t *)) { 3407 return EINVAL; 3408 } 3409 3410 unsigned arena_ind; 3411 arena_t *arena; 3412 int ret; 3413 size_t *pactivep; 3414 3415 malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); 3416 READONLY(); 3417 MIB_UNSIGNED(arena_ind, 2); 3418 if (arena_ind < narenas_total_get() && (arena = 3419 arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { 3420 #if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) || \ 3421 defined(JEMALLOC_GCC_SYNC_ATOMICS) || defined(_MSC_VER) 3422 /* Expose the underlying counter for fast read. */ 3423 pactivep = (size_t *)&(arena->nactive.repr); 3424 READ(pactivep, size_t *); 3425 ret = 0; 3426 #else 3427 ret = EFAULT; 3428 #endif 3429 } else { 3430 ret = EFAULT; 3431 } 3432 label_return: 3433 malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); 3434 return ret; 3435 } 3436