1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/assert.h" 5 #include "jemalloc/internal/buf_writer.h" 6 #include "jemalloc/internal/emitter.h" 7 #include "jemalloc/internal/prof_data.h" 8 #include "jemalloc/internal/prof_recent.h" 9 10 ssize_t opt_prof_recent_alloc_max = PROF_RECENT_ALLOC_MAX_DEFAULT; 11 malloc_mutex_t prof_recent_alloc_mtx; /* Protects the fields below */ 12 static atomic_zd_t prof_recent_alloc_max; 13 static ssize_t prof_recent_alloc_count = 0; 14 prof_recent_list_t prof_recent_alloc_list; 15 16 malloc_mutex_t prof_recent_dump_mtx; /* Protects dumping. */ 17 18 static void 19 prof_recent_alloc_max_init() { 20 atomic_store_zd(&prof_recent_alloc_max, opt_prof_recent_alloc_max, 21 ATOMIC_RELAXED); 22 } 23 24 static inline ssize_t 25 prof_recent_alloc_max_get_no_lock() { 26 return atomic_load_zd(&prof_recent_alloc_max, ATOMIC_RELAXED); 27 } 28 29 static inline ssize_t 30 prof_recent_alloc_max_get(tsd_t *tsd) { 31 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 32 return prof_recent_alloc_max_get_no_lock(); 33 } 34 35 static inline ssize_t 36 prof_recent_alloc_max_update(tsd_t *tsd, ssize_t max) { 37 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 38 ssize_t old_max = prof_recent_alloc_max_get(tsd); 39 atomic_store_zd(&prof_recent_alloc_max, max, ATOMIC_RELAXED); 40 return old_max; 41 } 42 43 static prof_recent_t * 44 prof_recent_allocate_node(tsdn_t *tsdn) { 45 return (prof_recent_t *)iallocztm(tsdn, sizeof(prof_recent_t), 46 sz_size2index(sizeof(prof_recent_t)), false, NULL, true, 47 arena_get(tsdn, 0, false), true); 48 } 49 50 static void 51 prof_recent_free_node(tsdn_t *tsdn, prof_recent_t *node) { 52 assert(node != NULL); 53 assert(isalloc(tsdn, node) == sz_s2u(sizeof(prof_recent_t))); 54 idalloctm(tsdn, node, NULL, NULL, true, true); 55 } 56 57 static inline void 58 increment_recent_count(tsd_t *tsd, prof_tctx_t *tctx) { 59 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); 60 ++tctx->recent_count; 61 assert(tctx->recent_count > 0); 62 } 63 64 bool 65 prof_recent_alloc_prepare(tsd_t *tsd, prof_tctx_t *tctx) { 66 cassert(config_prof); 67 assert(opt_prof && prof_booted); 68 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); 69 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 70 71 /* 72 * Check whether last-N mode is turned on without trying to acquire the 73 * lock, so as to optimize for the following two scenarios: 74 * (1) Last-N mode is switched off; 75 * (2) Dumping, during which last-N mode is temporarily turned off so 76 * as not to block sampled allocations. 77 */ 78 if (prof_recent_alloc_max_get_no_lock() == 0) { 79 return false; 80 } 81 82 /* 83 * Increment recent_count to hold the tctx so that it won't be gone 84 * even after tctx->tdata->lock is released. This acts as a 85 * "placeholder"; the real recording of the allocation requires a lock 86 * on prof_recent_alloc_mtx and is done in prof_recent_alloc (when 87 * tctx->tdata->lock has been released). 88 */ 89 increment_recent_count(tsd, tctx); 90 return true; 91 } 92 93 static void 94 decrement_recent_count(tsd_t *tsd, prof_tctx_t *tctx) { 95 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 96 assert(tctx != NULL); 97 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); 98 assert(tctx->recent_count > 0); 99 --tctx->recent_count; 100 prof_tctx_try_destroy(tsd, tctx); 101 } 102 103 static inline edata_t * 104 prof_recent_alloc_edata_get_no_lock(const prof_recent_t *n) { 105 return (edata_t *)atomic_load_p(&n->alloc_edata, ATOMIC_ACQUIRE); 106 } 107 108 edata_t * 109 prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *n) { 110 cassert(config_prof); 111 return prof_recent_alloc_edata_get_no_lock(n); 112 } 113 114 static inline edata_t * 115 prof_recent_alloc_edata_get(tsd_t *tsd, const prof_recent_t *n) { 116 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 117 return prof_recent_alloc_edata_get_no_lock(n); 118 } 119 120 static void 121 prof_recent_alloc_edata_set(tsd_t *tsd, prof_recent_t *n, edata_t *edata) { 122 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 123 atomic_store_p(&n->alloc_edata, edata, ATOMIC_RELEASE); 124 } 125 126 void 127 edata_prof_recent_alloc_init(edata_t *edata) { 128 cassert(config_prof); 129 edata_prof_recent_alloc_set_dont_call_directly(edata, NULL); 130 } 131 132 static inline prof_recent_t * 133 edata_prof_recent_alloc_get_no_lock(const edata_t *edata) { 134 cassert(config_prof); 135 return edata_prof_recent_alloc_get_dont_call_directly(edata); 136 } 137 138 prof_recent_t * 139 edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata) { 140 cassert(config_prof); 141 return edata_prof_recent_alloc_get_no_lock(edata); 142 } 143 144 static inline prof_recent_t * 145 edata_prof_recent_alloc_get(tsd_t *tsd, const edata_t *edata) { 146 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 147 prof_recent_t *recent_alloc = 148 edata_prof_recent_alloc_get_no_lock(edata); 149 assert(recent_alloc == NULL || 150 prof_recent_alloc_edata_get(tsd, recent_alloc) == edata); 151 return recent_alloc; 152 } 153 154 static prof_recent_t * 155 edata_prof_recent_alloc_update_internal(tsd_t *tsd, edata_t *edata, 156 prof_recent_t *recent_alloc) { 157 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 158 prof_recent_t *old_recent_alloc = 159 edata_prof_recent_alloc_get(tsd, edata); 160 edata_prof_recent_alloc_set_dont_call_directly(edata, recent_alloc); 161 return old_recent_alloc; 162 } 163 164 static void 165 edata_prof_recent_alloc_set(tsd_t *tsd, edata_t *edata, 166 prof_recent_t *recent_alloc) { 167 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 168 assert(recent_alloc != NULL); 169 prof_recent_t *old_recent_alloc = 170 edata_prof_recent_alloc_update_internal(tsd, edata, recent_alloc); 171 assert(old_recent_alloc == NULL); 172 prof_recent_alloc_edata_set(tsd, recent_alloc, edata); 173 } 174 175 static void 176 edata_prof_recent_alloc_reset(tsd_t *tsd, edata_t *edata, 177 prof_recent_t *recent_alloc) { 178 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 179 assert(recent_alloc != NULL); 180 prof_recent_t *old_recent_alloc = 181 edata_prof_recent_alloc_update_internal(tsd, edata, NULL); 182 assert(old_recent_alloc == recent_alloc); 183 assert(edata == prof_recent_alloc_edata_get(tsd, recent_alloc)); 184 prof_recent_alloc_edata_set(tsd, recent_alloc, NULL); 185 } 186 187 /* 188 * This function should be called right before an allocation is released, so 189 * that the associated recent allocation record can contain the following 190 * information: 191 * (1) The allocation is released; 192 * (2) The time of the deallocation; and 193 * (3) The prof_tctx associated with the deallocation. 194 */ 195 void 196 prof_recent_alloc_reset(tsd_t *tsd, edata_t *edata) { 197 cassert(config_prof); 198 /* 199 * Check whether the recent allocation record still exists without 200 * trying to acquire the lock. 201 */ 202 if (edata_prof_recent_alloc_get_no_lock(edata) == NULL) { 203 return; 204 } 205 206 prof_tctx_t *dalloc_tctx = prof_tctx_create(tsd); 207 /* 208 * In case dalloc_tctx is NULL, e.g. due to OOM, we will not record the 209 * deallocation time / tctx, which is handled later, after we check 210 * again when holding the lock. 211 */ 212 213 if (dalloc_tctx != NULL) { 214 malloc_mutex_lock(tsd_tsdn(tsd), dalloc_tctx->tdata->lock); 215 increment_recent_count(tsd, dalloc_tctx); 216 dalloc_tctx->prepared = false; 217 malloc_mutex_unlock(tsd_tsdn(tsd), dalloc_tctx->tdata->lock); 218 } 219 220 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 221 /* Check again after acquiring the lock. */ 222 prof_recent_t *recent = edata_prof_recent_alloc_get(tsd, edata); 223 if (recent != NULL) { 224 assert(nstime_equals_zero(&recent->dalloc_time)); 225 assert(recent->dalloc_tctx == NULL); 226 if (dalloc_tctx != NULL) { 227 nstime_prof_update(&recent->dalloc_time); 228 recent->dalloc_tctx = dalloc_tctx; 229 dalloc_tctx = NULL; 230 } 231 edata_prof_recent_alloc_reset(tsd, edata, recent); 232 } 233 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 234 235 if (dalloc_tctx != NULL) { 236 /* We lost the rase - the allocation record was just gone. */ 237 decrement_recent_count(tsd, dalloc_tctx); 238 } 239 } 240 241 static void 242 prof_recent_alloc_evict_edata(tsd_t *tsd, prof_recent_t *recent_alloc) { 243 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 244 edata_t *edata = prof_recent_alloc_edata_get(tsd, recent_alloc); 245 if (edata != NULL) { 246 edata_prof_recent_alloc_reset(tsd, edata, recent_alloc); 247 } 248 } 249 250 static bool 251 prof_recent_alloc_is_empty(tsd_t *tsd) { 252 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 253 if (ql_empty(&prof_recent_alloc_list)) { 254 assert(prof_recent_alloc_count == 0); 255 return true; 256 } else { 257 assert(prof_recent_alloc_count > 0); 258 return false; 259 } 260 } 261 262 static void 263 prof_recent_alloc_assert_count(tsd_t *tsd) { 264 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 265 if (!config_debug) { 266 return; 267 } 268 ssize_t count = 0; 269 prof_recent_t *n; 270 ql_foreach(n, &prof_recent_alloc_list, link) { 271 ++count; 272 } 273 assert(count == prof_recent_alloc_count); 274 assert(prof_recent_alloc_max_get(tsd) == -1 || 275 count <= prof_recent_alloc_max_get(tsd)); 276 } 277 278 void 279 prof_recent_alloc(tsd_t *tsd, edata_t *edata, size_t size, size_t usize) { 280 cassert(config_prof); 281 assert(edata != NULL); 282 prof_tctx_t *tctx = edata_prof_tctx_get(edata); 283 284 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock); 285 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 286 prof_recent_alloc_assert_count(tsd); 287 288 /* 289 * Reserve a new prof_recent_t node if needed. If needed, we release 290 * the prof_recent_alloc_mtx lock and allocate. Then, rather than 291 * immediately checking for OOM, we regain the lock and try to make use 292 * of the reserve node if needed. There are six scenarios: 293 * 294 * \ now | no need | need but OOMed | need and allocated 295 * later \ | | | 296 * ------------------------------------------------------------ 297 * no need | (1) | (2) | (3) 298 * ------------------------------------------------------------ 299 * need | (4) | (5) | (6) 300 * 301 * First, "(4)" never happens, because we don't release the lock in the 302 * middle if there's no need for a new node; in such cases "(1)" always 303 * takes place, which is trivial. 304 * 305 * Out of the remaining four scenarios, "(6)" is the common case and is 306 * trivial. "(5)" is also trivial, in which case we'll rollback the 307 * effect of prof_recent_alloc_prepare() as expected. 308 * 309 * "(2)" / "(3)" occurs when the need for a new node is gone after we 310 * regain the lock. If the new node is successfully allocated, i.e. in 311 * the case of "(3)", we'll release it in the end; otherwise, i.e. in 312 * the case of "(2)", we do nothing - we're lucky that the OOM ends up 313 * doing no harm at all. 314 * 315 * Therefore, the only performance cost of the "release lock" -> 316 * "allocate" -> "regain lock" design is the "(3)" case, but it happens 317 * very rarely, so the cost is relatively small compared to the gain of 318 * not having to have the lock order of prof_recent_alloc_mtx above all 319 * the allocation locks. 320 */ 321 prof_recent_t *reserve = NULL; 322 if (prof_recent_alloc_max_get(tsd) == -1 || 323 prof_recent_alloc_count < prof_recent_alloc_max_get(tsd)) { 324 assert(prof_recent_alloc_max_get(tsd) != 0); 325 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 326 reserve = prof_recent_allocate_node(tsd_tsdn(tsd)); 327 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 328 prof_recent_alloc_assert_count(tsd); 329 } 330 331 if (prof_recent_alloc_max_get(tsd) == 0) { 332 assert(prof_recent_alloc_is_empty(tsd)); 333 goto label_rollback; 334 } 335 336 prof_tctx_t *old_alloc_tctx, *old_dalloc_tctx; 337 if (prof_recent_alloc_count == prof_recent_alloc_max_get(tsd)) { 338 /* If upper limit is reached, rotate the head. */ 339 assert(prof_recent_alloc_max_get(tsd) != -1); 340 assert(!prof_recent_alloc_is_empty(tsd)); 341 prof_recent_t *head = ql_first(&prof_recent_alloc_list); 342 old_alloc_tctx = head->alloc_tctx; 343 assert(old_alloc_tctx != NULL); 344 old_dalloc_tctx = head->dalloc_tctx; 345 prof_recent_alloc_evict_edata(tsd, head); 346 ql_rotate(&prof_recent_alloc_list, link); 347 } else { 348 /* Otherwise make use of the new node. */ 349 assert(prof_recent_alloc_max_get(tsd) == -1 || 350 prof_recent_alloc_count < prof_recent_alloc_max_get(tsd)); 351 if (reserve == NULL) { 352 goto label_rollback; 353 } 354 ql_elm_new(reserve, link); 355 ql_tail_insert(&prof_recent_alloc_list, reserve, link); 356 reserve = NULL; 357 old_alloc_tctx = NULL; 358 old_dalloc_tctx = NULL; 359 ++prof_recent_alloc_count; 360 } 361 362 /* Fill content into the tail node. */ 363 prof_recent_t *tail = ql_last(&prof_recent_alloc_list, link); 364 assert(tail != NULL); 365 tail->size = size; 366 tail->usize = usize; 367 nstime_copy(&tail->alloc_time, edata_prof_alloc_time_get(edata)); 368 tail->alloc_tctx = tctx; 369 nstime_init_zero(&tail->dalloc_time); 370 tail->dalloc_tctx = NULL; 371 edata_prof_recent_alloc_set(tsd, edata, tail); 372 373 assert(!prof_recent_alloc_is_empty(tsd)); 374 prof_recent_alloc_assert_count(tsd); 375 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 376 377 if (reserve != NULL) { 378 prof_recent_free_node(tsd_tsdn(tsd), reserve); 379 } 380 381 /* 382 * Asynchronously handle the tctx of the old node, so that there's no 383 * simultaneous holdings of prof_recent_alloc_mtx and tdata->lock. 384 * In the worst case this may delay the tctx release but it's better 385 * than holding prof_recent_alloc_mtx for longer. 386 */ 387 if (old_alloc_tctx != NULL) { 388 decrement_recent_count(tsd, old_alloc_tctx); 389 } 390 if (old_dalloc_tctx != NULL) { 391 decrement_recent_count(tsd, old_dalloc_tctx); 392 } 393 return; 394 395 label_rollback: 396 assert(edata_prof_recent_alloc_get(tsd, edata) == NULL); 397 prof_recent_alloc_assert_count(tsd); 398 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 399 if (reserve != NULL) { 400 prof_recent_free_node(tsd_tsdn(tsd), reserve); 401 } 402 decrement_recent_count(tsd, tctx); 403 } 404 405 ssize_t 406 prof_recent_alloc_max_ctl_read() { 407 cassert(config_prof); 408 /* Don't bother to acquire the lock. */ 409 return prof_recent_alloc_max_get_no_lock(); 410 } 411 412 static void 413 prof_recent_alloc_restore_locked(tsd_t *tsd, prof_recent_list_t *to_delete) { 414 malloc_mutex_assert_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 415 ssize_t max = prof_recent_alloc_max_get(tsd); 416 if (max == -1 || prof_recent_alloc_count <= max) { 417 /* Easy case - no need to alter the list. */ 418 ql_new(to_delete); 419 prof_recent_alloc_assert_count(tsd); 420 return; 421 } 422 423 prof_recent_t *node; 424 ql_foreach(node, &prof_recent_alloc_list, link) { 425 if (prof_recent_alloc_count == max) { 426 break; 427 } 428 prof_recent_alloc_evict_edata(tsd, node); 429 --prof_recent_alloc_count; 430 } 431 assert(prof_recent_alloc_count == max); 432 433 ql_move(to_delete, &prof_recent_alloc_list); 434 if (max == 0) { 435 assert(node == NULL); 436 } else { 437 assert(node != NULL); 438 ql_split(to_delete, node, &prof_recent_alloc_list, link); 439 } 440 assert(!ql_empty(to_delete)); 441 prof_recent_alloc_assert_count(tsd); 442 } 443 444 static void 445 prof_recent_alloc_async_cleanup(tsd_t *tsd, prof_recent_list_t *to_delete) { 446 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_dump_mtx); 447 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 448 while (!ql_empty(to_delete)) { 449 prof_recent_t *node = ql_first(to_delete); 450 ql_remove(to_delete, node, link); 451 decrement_recent_count(tsd, node->alloc_tctx); 452 if (node->dalloc_tctx != NULL) { 453 decrement_recent_count(tsd, node->dalloc_tctx); 454 } 455 prof_recent_free_node(tsd_tsdn(tsd), node); 456 } 457 } 458 459 ssize_t 460 prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max) { 461 cassert(config_prof); 462 assert(max >= -1); 463 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 464 prof_recent_alloc_assert_count(tsd); 465 const ssize_t old_max = prof_recent_alloc_max_update(tsd, max); 466 prof_recent_list_t to_delete; 467 prof_recent_alloc_restore_locked(tsd, &to_delete); 468 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 469 prof_recent_alloc_async_cleanup(tsd, &to_delete); 470 return old_max; 471 } 472 473 static void 474 prof_recent_alloc_dump_bt(emitter_t *emitter, prof_tctx_t *tctx) { 475 char bt_buf[2 * sizeof(intptr_t) + 3]; 476 char *s = bt_buf; 477 assert(tctx != NULL); 478 prof_bt_t *bt = &tctx->gctx->bt; 479 for (size_t i = 0; i < bt->len; ++i) { 480 malloc_snprintf(bt_buf, sizeof(bt_buf), "%p", bt->vec[i]); 481 emitter_json_value(emitter, emitter_type_string, &s); 482 } 483 } 484 485 static void 486 prof_recent_alloc_dump_node(emitter_t *emitter, prof_recent_t *node) { 487 emitter_json_object_begin(emitter); 488 489 emitter_json_kv(emitter, "size", emitter_type_size, &node->size); 490 emitter_json_kv(emitter, "usize", emitter_type_size, &node->usize); 491 bool released = prof_recent_alloc_edata_get_no_lock(node) == NULL; 492 emitter_json_kv(emitter, "released", emitter_type_bool, &released); 493 494 emitter_json_kv(emitter, "alloc_thread_uid", emitter_type_uint64, 495 &node->alloc_tctx->thr_uid); 496 prof_tdata_t *alloc_tdata = node->alloc_tctx->tdata; 497 assert(alloc_tdata != NULL); 498 if (alloc_tdata->thread_name != NULL) { 499 emitter_json_kv(emitter, "alloc_thread_name", 500 emitter_type_string, &alloc_tdata->thread_name); 501 } 502 uint64_t alloc_time_ns = nstime_ns(&node->alloc_time); 503 emitter_json_kv(emitter, "alloc_time", emitter_type_uint64, 504 &alloc_time_ns); 505 emitter_json_array_kv_begin(emitter, "alloc_trace"); 506 prof_recent_alloc_dump_bt(emitter, node->alloc_tctx); 507 emitter_json_array_end(emitter); 508 509 if (released && node->dalloc_tctx != NULL) { 510 emitter_json_kv(emitter, "dalloc_thread_uid", 511 emitter_type_uint64, &node->dalloc_tctx->thr_uid); 512 prof_tdata_t *dalloc_tdata = node->dalloc_tctx->tdata; 513 assert(dalloc_tdata != NULL); 514 if (dalloc_tdata->thread_name != NULL) { 515 emitter_json_kv(emitter, "dalloc_thread_name", 516 emitter_type_string, &dalloc_tdata->thread_name); 517 } 518 assert(!nstime_equals_zero(&node->dalloc_time)); 519 uint64_t dalloc_time_ns = nstime_ns(&node->dalloc_time); 520 emitter_json_kv(emitter, "dalloc_time", emitter_type_uint64, 521 &dalloc_time_ns); 522 emitter_json_array_kv_begin(emitter, "dalloc_trace"); 523 prof_recent_alloc_dump_bt(emitter, node->dalloc_tctx); 524 emitter_json_array_end(emitter); 525 } 526 527 emitter_json_object_end(emitter); 528 } 529 530 #define PROF_RECENT_PRINT_BUFSIZE 65536 531 JEMALLOC_COLD 532 void 533 prof_recent_alloc_dump(tsd_t *tsd, write_cb_t *write_cb, void *cbopaque) { 534 cassert(config_prof); 535 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_dump_mtx); 536 buf_writer_t buf_writer; 537 buf_writer_init(tsd_tsdn(tsd), &buf_writer, write_cb, cbopaque, NULL, 538 PROF_RECENT_PRINT_BUFSIZE); 539 emitter_t emitter; 540 emitter_init(&emitter, emitter_output_json_compact, buf_writer_cb, 541 &buf_writer); 542 prof_recent_list_t temp_list; 543 544 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 545 prof_recent_alloc_assert_count(tsd); 546 ssize_t dump_max = prof_recent_alloc_max_get(tsd); 547 ql_move(&temp_list, &prof_recent_alloc_list); 548 ssize_t dump_count = prof_recent_alloc_count; 549 prof_recent_alloc_count = 0; 550 prof_recent_alloc_assert_count(tsd); 551 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 552 553 emitter_begin(&emitter); 554 uint64_t sample_interval = (uint64_t)1U << lg_prof_sample; 555 emitter_json_kv(&emitter, "sample_interval", emitter_type_uint64, 556 &sample_interval); 557 emitter_json_kv(&emitter, "recent_alloc_max", emitter_type_ssize, 558 &dump_max); 559 emitter_json_array_kv_begin(&emitter, "recent_alloc"); 560 prof_recent_t *node; 561 ql_foreach(node, &temp_list, link) { 562 prof_recent_alloc_dump_node(&emitter, node); 563 } 564 emitter_json_array_end(&emitter); 565 emitter_end(&emitter); 566 567 malloc_mutex_lock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 568 prof_recent_alloc_assert_count(tsd); 569 ql_concat(&temp_list, &prof_recent_alloc_list, link); 570 ql_move(&prof_recent_alloc_list, &temp_list); 571 prof_recent_alloc_count += dump_count; 572 prof_recent_alloc_restore_locked(tsd, &temp_list); 573 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_alloc_mtx); 574 575 buf_writer_terminate(tsd_tsdn(tsd), &buf_writer); 576 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_recent_dump_mtx); 577 578 prof_recent_alloc_async_cleanup(tsd, &temp_list); 579 } 580 #undef PROF_RECENT_PRINT_BUFSIZE 581 582 bool 583 prof_recent_init() { 584 cassert(config_prof); 585 prof_recent_alloc_max_init(); 586 587 if (malloc_mutex_init(&prof_recent_alloc_mtx, "prof_recent_alloc", 588 WITNESS_RANK_PROF_RECENT_ALLOC, malloc_mutex_rank_exclusive)) { 589 return true; 590 } 591 592 if (malloc_mutex_init(&prof_recent_dump_mtx, "prof_recent_dump", 593 WITNESS_RANK_PROF_RECENT_DUMP, malloc_mutex_rank_exclusive)) { 594 return true; 595 } 596 597 ql_new(&prof_recent_alloc_list); 598 599 return false; 600 } 601