1 #define JEMALLOC_ARENA_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 purge_mode_t opt_purge = PURGE_DEFAULT; 8 const char *purge_mode_names[] = { 9 "ratio", 10 "decay", 11 "N/A" 12 }; 13 ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 14 static ssize_t lg_dirty_mult_default; 15 ssize_t opt_decay_time = DECAY_TIME_DEFAULT; 16 static ssize_t decay_time_default; 17 18 arena_bin_info_t arena_bin_info[NBINS]; 19 20 size_t map_bias; 21 size_t map_misc_offset; 22 size_t arena_maxrun; /* Max run size for arenas. */ 23 size_t large_maxclass; /* Max large size class. */ 24 unsigned nlclasses; /* Number of large size classes. */ 25 unsigned nhclasses; /* Number of huge size classes. */ 26 27 /******************************************************************************/ 28 /* 29 * Function prototypes for static functions that are referenced prior to 30 * definition. 31 */ 32 33 static void arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, 34 arena_chunk_t *chunk); 35 static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, 36 size_t ndirty_limit); 37 static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, 38 bool dirty, bool cleaned, bool decommitted); 39 static void arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, 40 arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin); 41 static void arena_bin_lower_run(arena_t *arena, arena_run_t *run, 42 arena_bin_t *bin); 43 44 /******************************************************************************/ 45 46 JEMALLOC_INLINE_C size_t 47 arena_miscelm_size_get(const arena_chunk_map_misc_t *miscelm) 48 { 49 arena_chunk_t *chunk; 50 size_t pageind, mapbits; 51 52 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 53 pageind = arena_miscelm_to_pageind(miscelm); 54 mapbits = arena_mapbits_get(chunk, pageind); 55 return (arena_mapbits_size_decode(mapbits)); 56 } 57 58 JEMALLOC_INLINE_C const extent_node_t * 59 arena_miscelm_extent_get(const arena_chunk_map_misc_t *miscelm) 60 { 61 arena_chunk_t *chunk; 62 63 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 64 return (&chunk->node); 65 } 66 67 JEMALLOC_INLINE_C int 68 arena_sn_comp(const arena_chunk_map_misc_t *a, const arena_chunk_map_misc_t *b) 69 { 70 size_t a_sn, b_sn; 71 72 assert(a != NULL); 73 assert(b != NULL); 74 75 a_sn = extent_node_sn_get(arena_miscelm_extent_get(a)); 76 b_sn = extent_node_sn_get(arena_miscelm_extent_get(b)); 77 78 return ((a_sn > b_sn) - (a_sn < b_sn)); 79 } 80 81 JEMALLOC_INLINE_C int 82 arena_ad_comp(const arena_chunk_map_misc_t *a, 83 const arena_chunk_map_misc_t *b) 84 { 85 uintptr_t a_miscelm = (uintptr_t)a; 86 uintptr_t b_miscelm = (uintptr_t)b; 87 88 assert(a != NULL); 89 assert(b != NULL); 90 91 return ((a_miscelm > b_miscelm) - (a_miscelm < b_miscelm)); 92 } 93 94 JEMALLOC_INLINE_C int 95 arena_snad_comp(const arena_chunk_map_misc_t *a, 96 const arena_chunk_map_misc_t *b) 97 { 98 int ret; 99 100 assert(a != NULL); 101 assert(b != NULL); 102 103 ret = arena_sn_comp(a, b); 104 if (ret != 0) 105 return (ret); 106 107 ret = arena_ad_comp(a, b); 108 return (ret); 109 } 110 111 /* Generate pairing heap functions. */ 112 ph_gen(static UNUSED, arena_run_heap_, arena_run_heap_t, arena_chunk_map_misc_t, 113 ph_link, arena_snad_comp) 114 115 #ifdef JEMALLOC_JET 116 #undef run_quantize_floor 117 #define run_quantize_floor JEMALLOC_N(n_run_quantize_floor) 118 #endif 119 static size_t 120 run_quantize_floor(size_t size) 121 { 122 size_t ret; 123 pszind_t pind; 124 125 assert(size > 0); 126 assert(size <= HUGE_MAXCLASS); 127 assert((size & PAGE_MASK) == 0); 128 129 assert(size != 0); 130 assert(size == PAGE_CEILING(size)); 131 132 pind = psz2ind(size - large_pad + 1); 133 if (pind == 0) { 134 /* 135 * Avoid underflow. This short-circuit would also do the right 136 * thing for all sizes in the range for which there are 137 * PAGE-spaced size classes, but it's simplest to just handle 138 * the one case that would cause erroneous results. 139 */ 140 return (size); 141 } 142 ret = pind2sz(pind - 1) + large_pad; 143 assert(ret <= size); 144 return (ret); 145 } 146 #ifdef JEMALLOC_JET 147 #undef run_quantize_floor 148 #define run_quantize_floor JEMALLOC_N(run_quantize_floor) 149 run_quantize_t *run_quantize_floor = JEMALLOC_N(n_run_quantize_floor); 150 #endif 151 152 #ifdef JEMALLOC_JET 153 #undef run_quantize_ceil 154 #define run_quantize_ceil JEMALLOC_N(n_run_quantize_ceil) 155 #endif 156 static size_t 157 run_quantize_ceil(size_t size) 158 { 159 size_t ret; 160 161 assert(size > 0); 162 assert(size <= HUGE_MAXCLASS); 163 assert((size & PAGE_MASK) == 0); 164 165 ret = run_quantize_floor(size); 166 if (ret < size) { 167 /* 168 * Skip a quantization that may have an adequately large run, 169 * because under-sized runs may be mixed in. This only happens 170 * when an unusual size is requested, i.e. for aligned 171 * allocation, and is just one of several places where linear 172 * search would potentially find sufficiently aligned available 173 * memory somewhere lower. 174 */ 175 ret = pind2sz(psz2ind(ret - large_pad + 1)) + large_pad; 176 } 177 return (ret); 178 } 179 #ifdef JEMALLOC_JET 180 #undef run_quantize_ceil 181 #define run_quantize_ceil JEMALLOC_N(run_quantize_ceil) 182 run_quantize_t *run_quantize_ceil = JEMALLOC_N(n_run_quantize_ceil); 183 #endif 184 185 static void 186 arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 187 size_t npages) 188 { 189 pszind_t pind = psz2ind(run_quantize_floor(arena_miscelm_size_get( 190 arena_miscelm_get_const(chunk, pageind)))); 191 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 192 LG_PAGE)); 193 assert((npages << LG_PAGE) < chunksize); 194 assert(pind2sz(pind) <= chunksize); 195 arena_run_heap_insert(&arena->runs_avail[pind], 196 arena_miscelm_get_mutable(chunk, pageind)); 197 } 198 199 static void 200 arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 201 size_t npages) 202 { 203 pszind_t pind = psz2ind(run_quantize_floor(arena_miscelm_size_get( 204 arena_miscelm_get_const(chunk, pageind)))); 205 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 206 LG_PAGE)); 207 assert((npages << LG_PAGE) < chunksize); 208 assert(pind2sz(pind) <= chunksize); 209 arena_run_heap_remove(&arena->runs_avail[pind], 210 arena_miscelm_get_mutable(chunk, pageind)); 211 } 212 213 static void 214 arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 215 size_t npages) 216 { 217 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, 218 pageind); 219 220 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 221 LG_PAGE)); 222 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); 223 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == 224 CHUNK_MAP_DIRTY); 225 226 qr_new(&miscelm->rd, rd_link); 227 qr_meld(&arena->runs_dirty, &miscelm->rd, rd_link); 228 arena->ndirty += npages; 229 } 230 231 static void 232 arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 233 size_t npages) 234 { 235 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, 236 pageind); 237 238 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 239 LG_PAGE)); 240 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); 241 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == 242 CHUNK_MAP_DIRTY); 243 244 qr_remove(&miscelm->rd, rd_link); 245 assert(arena->ndirty >= npages); 246 arena->ndirty -= npages; 247 } 248 249 static size_t 250 arena_chunk_dirty_npages(const extent_node_t *node) 251 { 252 253 return (extent_node_size_get(node) >> LG_PAGE); 254 } 255 256 void 257 arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, bool cache) 258 { 259 260 if (cache) { 261 extent_node_dirty_linkage_init(node); 262 extent_node_dirty_insert(node, &arena->runs_dirty, 263 &arena->chunks_cache); 264 arena->ndirty += arena_chunk_dirty_npages(node); 265 } 266 } 267 268 void 269 arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, bool dirty) 270 { 271 272 if (dirty) { 273 extent_node_dirty_remove(node); 274 assert(arena->ndirty >= arena_chunk_dirty_npages(node)); 275 arena->ndirty -= arena_chunk_dirty_npages(node); 276 } 277 } 278 279 JEMALLOC_INLINE_C void * 280 arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 281 { 282 void *ret; 283 size_t regind; 284 arena_chunk_map_misc_t *miscelm; 285 void *rpages; 286 287 assert(run->nfree > 0); 288 assert(!bitmap_full(run->bitmap, &bin_info->bitmap_info)); 289 290 regind = (unsigned)bitmap_sfu(run->bitmap, &bin_info->bitmap_info); 291 miscelm = arena_run_to_miscelm(run); 292 rpages = arena_miscelm_to_rpages(miscelm); 293 ret = (void *)((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset + 294 (uintptr_t)(bin_info->reg_interval * regind)); 295 run->nfree--; 296 return (ret); 297 } 298 299 JEMALLOC_INLINE_C void 300 arena_run_reg_dalloc(arena_run_t *run, void *ptr) 301 { 302 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 303 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 304 size_t mapbits = arena_mapbits_get(chunk, pageind); 305 szind_t binind = arena_ptr_small_binind_get(ptr, mapbits); 306 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 307 size_t regind = arena_run_regind(run, bin_info, ptr); 308 309 assert(run->nfree < bin_info->nregs); 310 /* Freeing an interior pointer can cause assertion failure. */ 311 assert(((uintptr_t)ptr - 312 ((uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + 313 (uintptr_t)bin_info->reg0_offset)) % 314 (uintptr_t)bin_info->reg_interval == 0); 315 assert((uintptr_t)ptr >= 316 (uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + 317 (uintptr_t)bin_info->reg0_offset); 318 /* Freeing an unallocated pointer can cause assertion failure. */ 319 assert(bitmap_get(run->bitmap, &bin_info->bitmap_info, regind)); 320 321 bitmap_unset(run->bitmap, &bin_info->bitmap_info, regind); 322 run->nfree++; 323 } 324 325 JEMALLOC_INLINE_C void 326 arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) 327 { 328 329 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 330 (run_ind << LG_PAGE)), (npages << LG_PAGE)); 331 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, 332 (npages << LG_PAGE)); 333 } 334 335 JEMALLOC_INLINE_C void 336 arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) 337 { 338 339 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind 340 << LG_PAGE)), PAGE); 341 } 342 343 JEMALLOC_INLINE_C void 344 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 345 { 346 size_t i; 347 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 348 349 arena_run_page_mark_zeroed(chunk, run_ind); 350 for (i = 0; i < PAGE / sizeof(size_t); i++) 351 assert(p[i] == 0); 352 } 353 354 static void 355 arena_nactive_add(arena_t *arena, size_t add_pages) 356 { 357 358 if (config_stats) { 359 size_t cactive_add = CHUNK_CEILING((arena->nactive + 360 add_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << 361 LG_PAGE); 362 if (cactive_add != 0) 363 stats_cactive_add(cactive_add); 364 } 365 arena->nactive += add_pages; 366 } 367 368 static void 369 arena_nactive_sub(arena_t *arena, size_t sub_pages) 370 { 371 372 if (config_stats) { 373 size_t cactive_sub = CHUNK_CEILING(arena->nactive << LG_PAGE) - 374 CHUNK_CEILING((arena->nactive - sub_pages) << LG_PAGE); 375 if (cactive_sub != 0) 376 stats_cactive_sub(cactive_sub); 377 } 378 arena->nactive -= sub_pages; 379 } 380 381 static void 382 arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, 383 size_t flag_dirty, size_t flag_decommitted, size_t need_pages) 384 { 385 size_t total_pages, rem_pages; 386 387 assert(flag_dirty == 0 || flag_decommitted == 0); 388 389 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 390 LG_PAGE; 391 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 392 flag_dirty); 393 assert(need_pages <= total_pages); 394 rem_pages = total_pages - need_pages; 395 396 arena_avail_remove(arena, chunk, run_ind, total_pages); 397 if (flag_dirty != 0) 398 arena_run_dirty_remove(arena, chunk, run_ind, total_pages); 399 arena_nactive_add(arena, need_pages); 400 401 /* Keep track of trailing unused pages for later use. */ 402 if (rem_pages > 0) { 403 size_t flags = flag_dirty | flag_decommitted; 404 size_t flag_unzeroed_mask = (flags == 0) ? CHUNK_MAP_UNZEROED : 405 0; 406 407 arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 408 (rem_pages << LG_PAGE), flags | 409 (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) & 410 flag_unzeroed_mask)); 411 arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, 412 (rem_pages << LG_PAGE), flags | 413 (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) & 414 flag_unzeroed_mask)); 415 if (flag_dirty != 0) { 416 arena_run_dirty_insert(arena, chunk, run_ind+need_pages, 417 rem_pages); 418 } 419 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages); 420 } 421 } 422 423 static bool 424 arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, 425 bool remove, bool zero) 426 { 427 arena_chunk_t *chunk; 428 arena_chunk_map_misc_t *miscelm; 429 size_t flag_dirty, flag_decommitted, run_ind, need_pages; 430 size_t flag_unzeroed_mask; 431 432 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 433 miscelm = arena_run_to_miscelm(run); 434 run_ind = arena_miscelm_to_pageind(miscelm); 435 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 436 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); 437 need_pages = (size >> LG_PAGE); 438 assert(need_pages > 0); 439 440 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, 441 run_ind << LG_PAGE, size, arena->ind)) 442 return (true); 443 444 if (remove) { 445 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 446 flag_decommitted, need_pages); 447 } 448 449 if (zero) { 450 if (flag_decommitted != 0) { 451 /* The run is untouched, and therefore zeroed. */ 452 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void 453 *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 454 (need_pages << LG_PAGE)); 455 } else if (flag_dirty != 0) { 456 /* The run is dirty, so all pages must be zeroed. */ 457 arena_run_zero(chunk, run_ind, need_pages); 458 } else { 459 /* 460 * The run is clean, so some pages may be zeroed (i.e. 461 * never before touched). 462 */ 463 size_t i; 464 for (i = 0; i < need_pages; i++) { 465 if (arena_mapbits_unzeroed_get(chunk, run_ind+i) 466 != 0) 467 arena_run_zero(chunk, run_ind+i, 1); 468 else if (config_debug) { 469 arena_run_page_validate_zeroed(chunk, 470 run_ind+i); 471 } else { 472 arena_run_page_mark_zeroed(chunk, 473 run_ind+i); 474 } 475 } 476 } 477 } else { 478 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 479 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 480 } 481 482 /* 483 * Set the last element first, in case the run only contains one page 484 * (i.e. both statements set the same element). 485 */ 486 flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? 487 CHUNK_MAP_UNZEROED : 0; 488 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty | 489 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 490 run_ind+need_pages-1))); 491 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty | 492 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind))); 493 return (false); 494 } 495 496 static bool 497 arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 498 { 499 500 return (arena_run_split_large_helper(arena, run, size, true, zero)); 501 } 502 503 static bool 504 arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 505 { 506 507 return (arena_run_split_large_helper(arena, run, size, false, zero)); 508 } 509 510 static bool 511 arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, 512 szind_t binind) 513 { 514 arena_chunk_t *chunk; 515 arena_chunk_map_misc_t *miscelm; 516 size_t flag_dirty, flag_decommitted, run_ind, need_pages, i; 517 518 assert(binind != BININD_INVALID); 519 520 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 521 miscelm = arena_run_to_miscelm(run); 522 run_ind = arena_miscelm_to_pageind(miscelm); 523 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 524 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); 525 need_pages = (size >> LG_PAGE); 526 assert(need_pages > 0); 527 528 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, 529 run_ind << LG_PAGE, size, arena->ind)) 530 return (true); 531 532 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 533 flag_decommitted, need_pages); 534 535 for (i = 0; i < need_pages; i++) { 536 size_t flag_unzeroed = arena_mapbits_unzeroed_get(chunk, 537 run_ind+i); 538 arena_mapbits_small_set(chunk, run_ind+i, i, binind, 539 flag_unzeroed); 540 if (config_debug && flag_dirty == 0 && flag_unzeroed == 0) 541 arena_run_page_validate_zeroed(chunk, run_ind+i); 542 } 543 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 544 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 545 return (false); 546 } 547 548 static arena_chunk_t * 549 arena_chunk_init_spare(arena_t *arena) 550 { 551 arena_chunk_t *chunk; 552 553 assert(arena->spare != NULL); 554 555 chunk = arena->spare; 556 arena->spare = NULL; 557 558 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 559 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 560 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 561 arena_maxrun); 562 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 563 arena_maxrun); 564 assert(arena_mapbits_dirty_get(chunk, map_bias) == 565 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 566 567 return (chunk); 568 } 569 570 static bool 571 arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 572 size_t sn, bool zero) 573 { 574 575 /* 576 * The extent node notion of "committed" doesn't directly apply to 577 * arena chunks. Arbitrarily mark them as committed. The commit state 578 * of runs is tracked individually, and upon chunk deallocation the 579 * entire chunk is in a consistent commit state. 580 */ 581 extent_node_init(&chunk->node, arena, chunk, chunksize, sn, zero, true); 582 extent_node_achunk_set(&chunk->node, true); 583 return (chunk_register(tsdn, chunk, &chunk->node)); 584 } 585 586 static arena_chunk_t * 587 arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena, 588 chunk_hooks_t *chunk_hooks, bool *zero, bool *commit) 589 { 590 arena_chunk_t *chunk; 591 size_t sn; 592 593 malloc_mutex_unlock(tsdn, &arena->lock); 594 595 chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks, 596 NULL, chunksize, chunksize, &sn, zero, commit); 597 if (chunk != NULL && !*commit) { 598 /* Commit header. */ 599 if (chunk_hooks->commit(chunk, chunksize, 0, map_bias << 600 LG_PAGE, arena->ind)) { 601 chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, 602 (void *)chunk, chunksize, sn, *zero, *commit); 603 chunk = NULL; 604 } 605 } 606 if (chunk != NULL && arena_chunk_register(tsdn, arena, chunk, sn, 607 *zero)) { 608 if (!*commit) { 609 /* Undo commit of header. */ 610 chunk_hooks->decommit(chunk, chunksize, 0, map_bias << 611 LG_PAGE, arena->ind); 612 } 613 chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, (void *)chunk, 614 chunksize, sn, *zero, *commit); 615 chunk = NULL; 616 } 617 618 malloc_mutex_lock(tsdn, &arena->lock); 619 return (chunk); 620 } 621 622 static arena_chunk_t * 623 arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero, 624 bool *commit) 625 { 626 arena_chunk_t *chunk; 627 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 628 size_t sn; 629 630 chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize, 631 chunksize, &sn, zero, commit, true); 632 if (chunk != NULL) { 633 if (arena_chunk_register(tsdn, arena, chunk, sn, *zero)) { 634 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, 635 chunksize, sn, true); 636 return (NULL); 637 } 638 } 639 if (chunk == NULL) { 640 chunk = arena_chunk_alloc_internal_hard(tsdn, arena, 641 &chunk_hooks, zero, commit); 642 } 643 644 if (config_stats && chunk != NULL) { 645 arena->stats.mapped += chunksize; 646 arena->stats.metadata_mapped += (map_bias << LG_PAGE); 647 } 648 649 return (chunk); 650 } 651 652 static arena_chunk_t * 653 arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena) 654 { 655 arena_chunk_t *chunk; 656 bool zero, commit; 657 size_t flag_unzeroed, flag_decommitted, i; 658 659 assert(arena->spare == NULL); 660 661 zero = false; 662 commit = false; 663 chunk = arena_chunk_alloc_internal(tsdn, arena, &zero, &commit); 664 if (chunk == NULL) 665 return (NULL); 666 667 chunk->hugepage = true; 668 669 /* 670 * Initialize the map to contain one maximal free untouched run. Mark 671 * the pages as zeroed if arena_chunk_alloc_internal() returned a zeroed 672 * or decommitted chunk. 673 */ 674 flag_unzeroed = (zero || !commit) ? 0 : CHUNK_MAP_UNZEROED; 675 flag_decommitted = commit ? 0 : CHUNK_MAP_DECOMMITTED; 676 arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun, 677 flag_unzeroed | flag_decommitted); 678 /* 679 * There is no need to initialize the internal page map entries unless 680 * the chunk is not zeroed. 681 */ 682 if (!zero) { 683 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( 684 (void *)arena_bitselm_get_const(chunk, map_bias+1), 685 (size_t)((uintptr_t)arena_bitselm_get_const(chunk, 686 chunk_npages-1) - 687 (uintptr_t)arena_bitselm_get_const(chunk, map_bias+1))); 688 for (i = map_bias+1; i < chunk_npages-1; i++) 689 arena_mapbits_internal_set(chunk, i, flag_unzeroed); 690 } else { 691 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void 692 *)arena_bitselm_get_const(chunk, map_bias+1), 693 (size_t)((uintptr_t)arena_bitselm_get_const(chunk, 694 chunk_npages-1) - 695 (uintptr_t)arena_bitselm_get_const(chunk, map_bias+1))); 696 if (config_debug) { 697 for (i = map_bias+1; i < chunk_npages-1; i++) { 698 assert(arena_mapbits_unzeroed_get(chunk, i) == 699 flag_unzeroed); 700 } 701 } 702 } 703 arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun, 704 flag_unzeroed); 705 706 return (chunk); 707 } 708 709 static arena_chunk_t * 710 arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena) 711 { 712 arena_chunk_t *chunk; 713 714 if (arena->spare != NULL) 715 chunk = arena_chunk_init_spare(arena); 716 else { 717 chunk = arena_chunk_init_hard(tsdn, arena); 718 if (chunk == NULL) 719 return (NULL); 720 } 721 722 ql_elm_new(&chunk->node, ql_link); 723 ql_tail_insert(&arena->achunks, &chunk->node, ql_link); 724 arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias); 725 726 return (chunk); 727 } 728 729 static void 730 arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) 731 { 732 size_t sn, hugepage; 733 bool committed; 734 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 735 736 chunk_deregister(chunk, &chunk->node); 737 738 sn = extent_node_sn_get(&chunk->node); 739 hugepage = chunk->hugepage; 740 committed = (arena_mapbits_decommitted_get(chunk, map_bias) == 0); 741 if (!committed) { 742 /* 743 * Decommit the header. Mark the chunk as decommitted even if 744 * header decommit fails, since treating a partially committed 745 * chunk as committed has a high potential for causing later 746 * access of decommitted memory. 747 */ 748 chunk_hooks = chunk_hooks_get(tsdn, arena); 749 chunk_hooks.decommit(chunk, chunksize, 0, map_bias << LG_PAGE, 750 arena->ind); 751 } 752 if (!hugepage) { 753 /* 754 * Convert chunk back to the default state, so that all 755 * subsequent chunk allocations start out with chunks that can 756 * be backed by transparent huge pages. 757 */ 758 pages_huge(chunk, chunksize); 759 } 760 761 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, (void *)chunk, chunksize, 762 sn, committed); 763 764 if (config_stats) { 765 arena->stats.mapped -= chunksize; 766 arena->stats.metadata_mapped -= (map_bias << LG_PAGE); 767 } 768 } 769 770 static void 771 arena_spare_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *spare) 772 { 773 774 assert(arena->spare != spare); 775 776 if (arena_mapbits_dirty_get(spare, map_bias) != 0) { 777 arena_run_dirty_remove(arena, spare, map_bias, 778 chunk_npages-map_bias); 779 } 780 781 arena_chunk_discard(tsdn, arena, spare); 782 } 783 784 static void 785 arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) 786 { 787 arena_chunk_t *spare; 788 789 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 790 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 791 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 792 arena_maxrun); 793 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 794 arena_maxrun); 795 assert(arena_mapbits_dirty_get(chunk, map_bias) == 796 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 797 assert(arena_mapbits_decommitted_get(chunk, map_bias) == 798 arena_mapbits_decommitted_get(chunk, chunk_npages-1)); 799 800 /* Remove run from runs_avail, so that the arena does not use it. */ 801 arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias); 802 803 ql_remove(&arena->achunks, &chunk->node, ql_link); 804 spare = arena->spare; 805 arena->spare = chunk; 806 if (spare != NULL) 807 arena_spare_discard(tsdn, arena, spare); 808 } 809 810 static void 811 arena_huge_malloc_stats_update(arena_t *arena, size_t usize) 812 { 813 szind_t index = size2index(usize) - nlclasses - NBINS; 814 815 cassert(config_stats); 816 817 arena->stats.nmalloc_huge++; 818 arena->stats.allocated_huge += usize; 819 arena->stats.hstats[index].nmalloc++; 820 arena->stats.hstats[index].curhchunks++; 821 } 822 823 static void 824 arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize) 825 { 826 szind_t index = size2index(usize) - nlclasses - NBINS; 827 828 cassert(config_stats); 829 830 arena->stats.nmalloc_huge--; 831 arena->stats.allocated_huge -= usize; 832 arena->stats.hstats[index].nmalloc--; 833 arena->stats.hstats[index].curhchunks--; 834 } 835 836 static void 837 arena_huge_dalloc_stats_update(arena_t *arena, size_t usize) 838 { 839 szind_t index = size2index(usize) - nlclasses - NBINS; 840 841 cassert(config_stats); 842 843 arena->stats.ndalloc_huge++; 844 arena->stats.allocated_huge -= usize; 845 arena->stats.hstats[index].ndalloc++; 846 arena->stats.hstats[index].curhchunks--; 847 } 848 849 static void 850 arena_huge_reset_stats_cancel(arena_t *arena, size_t usize) 851 { 852 szind_t index = size2index(usize) - nlclasses - NBINS; 853 854 cassert(config_stats); 855 856 arena->stats.ndalloc_huge++; 857 arena->stats.hstats[index].ndalloc--; 858 } 859 860 static void 861 arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize) 862 { 863 szind_t index = size2index(usize) - nlclasses - NBINS; 864 865 cassert(config_stats); 866 867 arena->stats.ndalloc_huge--; 868 arena->stats.allocated_huge += usize; 869 arena->stats.hstats[index].ndalloc--; 870 arena->stats.hstats[index].curhchunks++; 871 } 872 873 static void 874 arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize) 875 { 876 877 arena_huge_dalloc_stats_update(arena, oldsize); 878 arena_huge_malloc_stats_update(arena, usize); 879 } 880 881 static void 882 arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize, 883 size_t usize) 884 { 885 886 arena_huge_dalloc_stats_update_undo(arena, oldsize); 887 arena_huge_malloc_stats_update_undo(arena, usize); 888 } 889 890 extent_node_t * 891 arena_node_alloc(tsdn_t *tsdn, arena_t *arena) 892 { 893 extent_node_t *node; 894 895 malloc_mutex_lock(tsdn, &arena->node_cache_mtx); 896 node = ql_last(&arena->node_cache, ql_link); 897 if (node == NULL) { 898 malloc_mutex_unlock(tsdn, &arena->node_cache_mtx); 899 return (base_alloc(tsdn, sizeof(extent_node_t))); 900 } 901 ql_tail_remove(&arena->node_cache, extent_node_t, ql_link); 902 malloc_mutex_unlock(tsdn, &arena->node_cache_mtx); 903 return (node); 904 } 905 906 void 907 arena_node_dalloc(tsdn_t *tsdn, arena_t *arena, extent_node_t *node) 908 { 909 910 malloc_mutex_lock(tsdn, &arena->node_cache_mtx); 911 ql_elm_new(node, ql_link); 912 ql_tail_insert(&arena->node_cache, node, ql_link); 913 malloc_mutex_unlock(tsdn, &arena->node_cache_mtx); 914 } 915 916 static void * 917 arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena, 918 chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, size_t *sn, 919 bool *zero, size_t csize) 920 { 921 void *ret; 922 bool commit = true; 923 924 ret = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, csize, 925 alignment, sn, zero, &commit); 926 if (ret == NULL) { 927 /* Revert optimistic stats updates. */ 928 malloc_mutex_lock(tsdn, &arena->lock); 929 if (config_stats) { 930 arena_huge_malloc_stats_update_undo(arena, usize); 931 arena->stats.mapped -= usize; 932 } 933 arena_nactive_sub(arena, usize >> LG_PAGE); 934 malloc_mutex_unlock(tsdn, &arena->lock); 935 } 936 937 return (ret); 938 } 939 940 void * 941 arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, 942 size_t alignment, size_t *sn, bool *zero) 943 { 944 void *ret; 945 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 946 size_t csize = CHUNK_CEILING(usize); 947 bool commit = true; 948 949 malloc_mutex_lock(tsdn, &arena->lock); 950 951 /* Optimistically update stats. */ 952 if (config_stats) { 953 arena_huge_malloc_stats_update(arena, usize); 954 arena->stats.mapped += usize; 955 } 956 arena_nactive_add(arena, usize >> LG_PAGE); 957 958 ret = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, csize, 959 alignment, sn, zero, &commit, true); 960 malloc_mutex_unlock(tsdn, &arena->lock); 961 if (ret == NULL) { 962 ret = arena_chunk_alloc_huge_hard(tsdn, arena, &chunk_hooks, 963 usize, alignment, sn, zero, csize); 964 } 965 966 return (ret); 967 } 968 969 void 970 arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize, 971 size_t sn) 972 { 973 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 974 size_t csize; 975 976 csize = CHUNK_CEILING(usize); 977 malloc_mutex_lock(tsdn, &arena->lock); 978 if (config_stats) { 979 arena_huge_dalloc_stats_update(arena, usize); 980 arena->stats.mapped -= usize; 981 } 982 arena_nactive_sub(arena, usize >> LG_PAGE); 983 984 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, csize, sn, true); 985 malloc_mutex_unlock(tsdn, &arena->lock); 986 } 987 988 void 989 arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, void *chunk, 990 size_t oldsize, size_t usize) 991 { 992 993 assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize)); 994 assert(oldsize != usize); 995 996 malloc_mutex_lock(tsdn, &arena->lock); 997 if (config_stats) 998 arena_huge_ralloc_stats_update(arena, oldsize, usize); 999 if (oldsize < usize) 1000 arena_nactive_add(arena, (usize - oldsize) >> LG_PAGE); 1001 else 1002 arena_nactive_sub(arena, (oldsize - usize) >> LG_PAGE); 1003 malloc_mutex_unlock(tsdn, &arena->lock); 1004 } 1005 1006 void 1007 arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk, 1008 size_t oldsize, size_t usize, size_t sn) 1009 { 1010 size_t udiff = oldsize - usize; 1011 size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); 1012 1013 malloc_mutex_lock(tsdn, &arena->lock); 1014 if (config_stats) { 1015 arena_huge_ralloc_stats_update(arena, oldsize, usize); 1016 if (cdiff != 0) 1017 arena->stats.mapped -= cdiff; 1018 } 1019 arena_nactive_sub(arena, udiff >> LG_PAGE); 1020 1021 if (cdiff != 0) { 1022 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 1023 void *nchunk = (void *)((uintptr_t)chunk + 1024 CHUNK_CEILING(usize)); 1025 1026 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff, 1027 sn, true); 1028 } 1029 malloc_mutex_unlock(tsdn, &arena->lock); 1030 } 1031 1032 static bool 1033 arena_chunk_ralloc_huge_expand_hard(tsdn_t *tsdn, arena_t *arena, 1034 chunk_hooks_t *chunk_hooks, void *chunk, size_t oldsize, size_t usize, 1035 size_t *sn, bool *zero, void *nchunk, size_t udiff, size_t cdiff) 1036 { 1037 bool err; 1038 bool commit = true; 1039 1040 err = (chunk_alloc_wrapper(tsdn, arena, chunk_hooks, nchunk, cdiff, 1041 chunksize, sn, zero, &commit) == NULL); 1042 if (err) { 1043 /* Revert optimistic stats updates. */ 1044 malloc_mutex_lock(tsdn, &arena->lock); 1045 if (config_stats) { 1046 arena_huge_ralloc_stats_update_undo(arena, oldsize, 1047 usize); 1048 arena->stats.mapped -= cdiff; 1049 } 1050 arena_nactive_sub(arena, udiff >> LG_PAGE); 1051 malloc_mutex_unlock(tsdn, &arena->lock); 1052 } else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk, 1053 cdiff, true, arena->ind)) { 1054 chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, nchunk, cdiff, 1055 *sn, *zero, true); 1056 err = true; 1057 } 1058 return (err); 1059 } 1060 1061 bool 1062 arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk, 1063 size_t oldsize, size_t usize, bool *zero) 1064 { 1065 bool err; 1066 chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena); 1067 void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)); 1068 size_t udiff = usize - oldsize; 1069 size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); 1070 size_t sn; 1071 bool commit = true; 1072 1073 malloc_mutex_lock(tsdn, &arena->lock); 1074 1075 /* Optimistically update stats. */ 1076 if (config_stats) { 1077 arena_huge_ralloc_stats_update(arena, oldsize, usize); 1078 arena->stats.mapped += cdiff; 1079 } 1080 arena_nactive_add(arena, udiff >> LG_PAGE); 1081 1082 err = (chunk_alloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff, 1083 chunksize, &sn, zero, &commit, true) == NULL); 1084 malloc_mutex_unlock(tsdn, &arena->lock); 1085 if (err) { 1086 err = arena_chunk_ralloc_huge_expand_hard(tsdn, arena, 1087 &chunk_hooks, chunk, oldsize, usize, &sn, zero, nchunk, 1088 udiff, cdiff); 1089 } else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk, 1090 cdiff, true, arena->ind)) { 1091 chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, nchunk, cdiff, 1092 sn, *zero, true); 1093 err = true; 1094 } 1095 1096 return (err); 1097 } 1098 1099 /* 1100 * Do first-best-fit run selection, i.e. select the lowest run that best fits. 1101 * Run sizes are indexed, so not all candidate runs are necessarily exactly the 1102 * same size. 1103 */ 1104 static arena_run_t * 1105 arena_run_first_best_fit(arena_t *arena, size_t size) 1106 { 1107 pszind_t pind, i; 1108 1109 pind = psz2ind(run_quantize_ceil(size)); 1110 1111 for (i = pind; pind2sz(i) <= chunksize; i++) { 1112 arena_chunk_map_misc_t *miscelm = arena_run_heap_first( 1113 &arena->runs_avail[i]); 1114 if (miscelm != NULL) 1115 return (&miscelm->run); 1116 } 1117 1118 return (NULL); 1119 } 1120 1121 static arena_run_t * 1122 arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) 1123 { 1124 arena_run_t *run = arena_run_first_best_fit(arena, size); 1125 if (run != NULL) { 1126 if (arena_run_split_large(arena, run, size, zero)) 1127 run = NULL; 1128 } 1129 return (run); 1130 } 1131 1132 static arena_run_t * 1133 arena_run_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t size, bool zero) 1134 { 1135 arena_chunk_t *chunk; 1136 arena_run_t *run; 1137 1138 assert(size <= arena_maxrun); 1139 assert(size == PAGE_CEILING(size)); 1140 1141 /* Search the arena's chunks for the lowest best fit. */ 1142 run = arena_run_alloc_large_helper(arena, size, zero); 1143 if (run != NULL) 1144 return (run); 1145 1146 /* 1147 * No usable runs. Create a new chunk from which to allocate the run. 1148 */ 1149 chunk = arena_chunk_alloc(tsdn, arena); 1150 if (chunk != NULL) { 1151 run = &arena_miscelm_get_mutable(chunk, map_bias)->run; 1152 if (arena_run_split_large(arena, run, size, zero)) 1153 run = NULL; 1154 return (run); 1155 } 1156 1157 /* 1158 * arena_chunk_alloc() failed, but another thread may have made 1159 * sufficient memory available while this one dropped arena->lock in 1160 * arena_chunk_alloc(), so search one more time. 1161 */ 1162 return (arena_run_alloc_large_helper(arena, size, zero)); 1163 } 1164 1165 static arena_run_t * 1166 arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind) 1167 { 1168 arena_run_t *run = arena_run_first_best_fit(arena, size); 1169 if (run != NULL) { 1170 if (arena_run_split_small(arena, run, size, binind)) 1171 run = NULL; 1172 } 1173 return (run); 1174 } 1175 1176 static arena_run_t * 1177 arena_run_alloc_small(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t binind) 1178 { 1179 arena_chunk_t *chunk; 1180 arena_run_t *run; 1181 1182 assert(size <= arena_maxrun); 1183 assert(size == PAGE_CEILING(size)); 1184 assert(binind != BININD_INVALID); 1185 1186 /* Search the arena's chunks for the lowest best fit. */ 1187 run = arena_run_alloc_small_helper(arena, size, binind); 1188 if (run != NULL) 1189 return (run); 1190 1191 /* 1192 * No usable runs. Create a new chunk from which to allocate the run. 1193 */ 1194 chunk = arena_chunk_alloc(tsdn, arena); 1195 if (chunk != NULL) { 1196 run = &arena_miscelm_get_mutable(chunk, map_bias)->run; 1197 if (arena_run_split_small(arena, run, size, binind)) 1198 run = NULL; 1199 return (run); 1200 } 1201 1202 /* 1203 * arena_chunk_alloc() failed, but another thread may have made 1204 * sufficient memory available while this one dropped arena->lock in 1205 * arena_chunk_alloc(), so search one more time. 1206 */ 1207 return (arena_run_alloc_small_helper(arena, size, binind)); 1208 } 1209 1210 static bool 1211 arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult) 1212 { 1213 1214 return (lg_dirty_mult >= -1 && lg_dirty_mult < (ssize_t)(sizeof(size_t) 1215 << 3)); 1216 } 1217 1218 ssize_t 1219 arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena) 1220 { 1221 ssize_t lg_dirty_mult; 1222 1223 malloc_mutex_lock(tsdn, &arena->lock); 1224 lg_dirty_mult = arena->lg_dirty_mult; 1225 malloc_mutex_unlock(tsdn, &arena->lock); 1226 1227 return (lg_dirty_mult); 1228 } 1229 1230 bool 1231 arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena, ssize_t lg_dirty_mult) 1232 { 1233 1234 if (!arena_lg_dirty_mult_valid(lg_dirty_mult)) 1235 return (true); 1236 1237 malloc_mutex_lock(tsdn, &arena->lock); 1238 arena->lg_dirty_mult = lg_dirty_mult; 1239 arena_maybe_purge(tsdn, arena); 1240 malloc_mutex_unlock(tsdn, &arena->lock); 1241 1242 return (false); 1243 } 1244 1245 static void 1246 arena_decay_deadline_init(arena_t *arena) 1247 { 1248 1249 assert(opt_purge == purge_mode_decay); 1250 1251 /* 1252 * Generate a new deadline that is uniformly random within the next 1253 * epoch after the current one. 1254 */ 1255 nstime_copy(&arena->decay.deadline, &arena->decay.epoch); 1256 nstime_add(&arena->decay.deadline, &arena->decay.interval); 1257 if (arena->decay.time > 0) { 1258 nstime_t jitter; 1259 1260 nstime_init(&jitter, prng_range_u64(&arena->decay.jitter_state, 1261 nstime_ns(&arena->decay.interval))); 1262 nstime_add(&arena->decay.deadline, &jitter); 1263 } 1264 } 1265 1266 static bool 1267 arena_decay_deadline_reached(const arena_t *arena, const nstime_t *time) 1268 { 1269 1270 assert(opt_purge == purge_mode_decay); 1271 1272 return (nstime_compare(&arena->decay.deadline, time) <= 0); 1273 } 1274 1275 static size_t 1276 arena_decay_backlog_npages_limit(const arena_t *arena) 1277 { 1278 static const uint64_t h_steps[] = { 1279 #define STEP(step, h, x, y) \ 1280 h, 1281 SMOOTHSTEP 1282 #undef STEP 1283 }; 1284 uint64_t sum; 1285 size_t npages_limit_backlog; 1286 unsigned i; 1287 1288 assert(opt_purge == purge_mode_decay); 1289 1290 /* 1291 * For each element of decay_backlog, multiply by the corresponding 1292 * fixed-point smoothstep decay factor. Sum the products, then divide 1293 * to round down to the nearest whole number of pages. 1294 */ 1295 sum = 0; 1296 for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) 1297 sum += arena->decay.backlog[i] * h_steps[i]; 1298 npages_limit_backlog = (size_t)(sum >> SMOOTHSTEP_BFP); 1299 1300 return (npages_limit_backlog); 1301 } 1302 1303 static void 1304 arena_decay_backlog_update_last(arena_t *arena) 1305 { 1306 size_t ndirty_delta = (arena->ndirty > arena->decay.ndirty) ? 1307 arena->ndirty - arena->decay.ndirty : 0; 1308 arena->decay.backlog[SMOOTHSTEP_NSTEPS-1] = ndirty_delta; 1309 } 1310 1311 static void 1312 arena_decay_backlog_update(arena_t *arena, uint64_t nadvance_u64) 1313 { 1314 1315 if (nadvance_u64 >= SMOOTHSTEP_NSTEPS) { 1316 memset(arena->decay.backlog, 0, (SMOOTHSTEP_NSTEPS-1) * 1317 sizeof(size_t)); 1318 } else { 1319 size_t nadvance_z = (size_t)nadvance_u64; 1320 1321 assert((uint64_t)nadvance_z == nadvance_u64); 1322 1323 memmove(arena->decay.backlog, &arena->decay.backlog[nadvance_z], 1324 (SMOOTHSTEP_NSTEPS - nadvance_z) * sizeof(size_t)); 1325 if (nadvance_z > 1) { 1326 memset(&arena->decay.backlog[SMOOTHSTEP_NSTEPS - 1327 nadvance_z], 0, (nadvance_z-1) * sizeof(size_t)); 1328 } 1329 } 1330 1331 arena_decay_backlog_update_last(arena); 1332 } 1333 1334 static void 1335 arena_decay_epoch_advance_helper(arena_t *arena, const nstime_t *time) 1336 { 1337 uint64_t nadvance_u64; 1338 nstime_t delta; 1339 1340 assert(opt_purge == purge_mode_decay); 1341 assert(arena_decay_deadline_reached(arena, time)); 1342 1343 nstime_copy(&delta, time); 1344 nstime_subtract(&delta, &arena->decay.epoch); 1345 nadvance_u64 = nstime_divide(&delta, &arena->decay.interval); 1346 assert(nadvance_u64 > 0); 1347 1348 /* Add nadvance_u64 decay intervals to epoch. */ 1349 nstime_copy(&delta, &arena->decay.interval); 1350 nstime_imultiply(&delta, nadvance_u64); 1351 nstime_add(&arena->decay.epoch, &delta); 1352 1353 /* Set a new deadline. */ 1354 arena_decay_deadline_init(arena); 1355 1356 /* Update the backlog. */ 1357 arena_decay_backlog_update(arena, nadvance_u64); 1358 } 1359 1360 static void 1361 arena_decay_epoch_advance_purge(tsdn_t *tsdn, arena_t *arena) 1362 { 1363 size_t ndirty_limit = arena_decay_backlog_npages_limit(arena); 1364 1365 if (arena->ndirty > ndirty_limit) 1366 arena_purge_to_limit(tsdn, arena, ndirty_limit); 1367 arena->decay.ndirty = arena->ndirty; 1368 } 1369 1370 static void 1371 arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, const nstime_t *time) 1372 { 1373 1374 arena_decay_epoch_advance_helper(arena, time); 1375 arena_decay_epoch_advance_purge(tsdn, arena); 1376 } 1377 1378 static void 1379 arena_decay_init(arena_t *arena, ssize_t decay_time) 1380 { 1381 1382 arena->decay.time = decay_time; 1383 if (decay_time > 0) { 1384 nstime_init2(&arena->decay.interval, decay_time, 0); 1385 nstime_idivide(&arena->decay.interval, SMOOTHSTEP_NSTEPS); 1386 } 1387 1388 nstime_init(&arena->decay.epoch, 0); 1389 nstime_update(&arena->decay.epoch); 1390 arena->decay.jitter_state = (uint64_t)(uintptr_t)arena; 1391 arena_decay_deadline_init(arena); 1392 arena->decay.ndirty = arena->ndirty; 1393 memset(arena->decay.backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t)); 1394 } 1395 1396 static bool 1397 arena_decay_time_valid(ssize_t decay_time) 1398 { 1399 1400 if (decay_time < -1) 1401 return (false); 1402 if (decay_time == -1 || (uint64_t)decay_time <= NSTIME_SEC_MAX) 1403 return (true); 1404 return (false); 1405 } 1406 1407 ssize_t 1408 arena_decay_time_get(tsdn_t *tsdn, arena_t *arena) 1409 { 1410 ssize_t decay_time; 1411 1412 malloc_mutex_lock(tsdn, &arena->lock); 1413 decay_time = arena->decay.time; 1414 malloc_mutex_unlock(tsdn, &arena->lock); 1415 1416 return (decay_time); 1417 } 1418 1419 bool 1420 arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) 1421 { 1422 1423 if (!arena_decay_time_valid(decay_time)) 1424 return (true); 1425 1426 malloc_mutex_lock(tsdn, &arena->lock); 1427 /* 1428 * Restart decay backlog from scratch, which may cause many dirty pages 1429 * to be immediately purged. It would conceptually be possible to map 1430 * the old backlog onto the new backlog, but there is no justification 1431 * for such complexity since decay_time changes are intended to be 1432 * infrequent, either between the {-1, 0, >0} states, or a one-time 1433 * arbitrary change during initial arena configuration. 1434 */ 1435 arena_decay_init(arena, decay_time); 1436 arena_maybe_purge(tsdn, arena); 1437 malloc_mutex_unlock(tsdn, &arena->lock); 1438 1439 return (false); 1440 } 1441 1442 static void 1443 arena_maybe_purge_ratio(tsdn_t *tsdn, arena_t *arena) 1444 { 1445 1446 assert(opt_purge == purge_mode_ratio); 1447 1448 /* Don't purge if the option is disabled. */ 1449 if (arena->lg_dirty_mult < 0) 1450 return; 1451 1452 /* 1453 * Iterate, since preventing recursive purging could otherwise leave too 1454 * many dirty pages. 1455 */ 1456 while (true) { 1457 size_t threshold = (arena->nactive >> arena->lg_dirty_mult); 1458 if (threshold < chunk_npages) 1459 threshold = chunk_npages; 1460 /* 1461 * Don't purge unless the number of purgeable pages exceeds the 1462 * threshold. 1463 */ 1464 if (arena->ndirty <= threshold) 1465 return; 1466 arena_purge_to_limit(tsdn, arena, threshold); 1467 } 1468 } 1469 1470 static void 1471 arena_maybe_purge_decay(tsdn_t *tsdn, arena_t *arena) 1472 { 1473 nstime_t time; 1474 1475 assert(opt_purge == purge_mode_decay); 1476 1477 /* Purge all or nothing if the option is disabled. */ 1478 if (arena->decay.time <= 0) { 1479 if (arena->decay.time == 0) 1480 arena_purge_to_limit(tsdn, arena, 0); 1481 return; 1482 } 1483 1484 nstime_init(&time, 0); 1485 nstime_update(&time); 1486 if (unlikely(!nstime_monotonic() && nstime_compare(&arena->decay.epoch, 1487 &time) > 0)) { 1488 /* 1489 * Time went backwards. Move the epoch back in time and 1490 * generate a new deadline, with the expectation that time 1491 * typically flows forward for long enough periods of time that 1492 * epochs complete. Unfortunately, this strategy is susceptible 1493 * to clock jitter triggering premature epoch advances, but 1494 * clock jitter estimation and compensation isn't feasible here 1495 * because calls into this code are event-driven. 1496 */ 1497 nstime_copy(&arena->decay.epoch, &time); 1498 arena_decay_deadline_init(arena); 1499 } else { 1500 /* Verify that time does not go backwards. */ 1501 assert(nstime_compare(&arena->decay.epoch, &time) <= 0); 1502 } 1503 1504 /* 1505 * If the deadline has been reached, advance to the current epoch and 1506 * purge to the new limit if necessary. Note that dirty pages created 1507 * during the current epoch are not subject to purge until a future 1508 * epoch, so as a result purging only happens during epoch advances. 1509 */ 1510 if (arena_decay_deadline_reached(arena, &time)) 1511 arena_decay_epoch_advance(tsdn, arena, &time); 1512 } 1513 1514 void 1515 arena_maybe_purge(tsdn_t *tsdn, arena_t *arena) 1516 { 1517 1518 /* Don't recursively purge. */ 1519 if (arena->purging) 1520 return; 1521 1522 if (opt_purge == purge_mode_ratio) 1523 arena_maybe_purge_ratio(tsdn, arena); 1524 else 1525 arena_maybe_purge_decay(tsdn, arena); 1526 } 1527 1528 static size_t 1529 arena_dirty_count(arena_t *arena) 1530 { 1531 size_t ndirty = 0; 1532 arena_runs_dirty_link_t *rdelm; 1533 extent_node_t *chunkselm; 1534 1535 for (rdelm = qr_next(&arena->runs_dirty, rd_link), 1536 chunkselm = qr_next(&arena->chunks_cache, cc_link); 1537 rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) { 1538 size_t npages; 1539 1540 if (rdelm == &chunkselm->rd) { 1541 npages = extent_node_size_get(chunkselm) >> LG_PAGE; 1542 chunkselm = qr_next(chunkselm, cc_link); 1543 } else { 1544 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( 1545 rdelm); 1546 arena_chunk_map_misc_t *miscelm = 1547 arena_rd_to_miscelm(rdelm); 1548 size_t pageind = arena_miscelm_to_pageind(miscelm); 1549 assert(arena_mapbits_allocated_get(chunk, pageind) == 1550 0); 1551 assert(arena_mapbits_large_get(chunk, pageind) == 0); 1552 assert(arena_mapbits_dirty_get(chunk, pageind) != 0); 1553 npages = arena_mapbits_unallocated_size_get(chunk, 1554 pageind) >> LG_PAGE; 1555 } 1556 ndirty += npages; 1557 } 1558 1559 return (ndirty); 1560 } 1561 1562 static size_t 1563 arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, 1564 size_t ndirty_limit, arena_runs_dirty_link_t *purge_runs_sentinel, 1565 extent_node_t *purge_chunks_sentinel) 1566 { 1567 arena_runs_dirty_link_t *rdelm, *rdelm_next; 1568 extent_node_t *chunkselm; 1569 size_t nstashed = 0; 1570 1571 /* Stash runs/chunks according to ndirty_limit. */ 1572 for (rdelm = qr_next(&arena->runs_dirty, rd_link), 1573 chunkselm = qr_next(&arena->chunks_cache, cc_link); 1574 rdelm != &arena->runs_dirty; rdelm = rdelm_next) { 1575 size_t npages; 1576 rdelm_next = qr_next(rdelm, rd_link); 1577 1578 if (rdelm == &chunkselm->rd) { 1579 extent_node_t *chunkselm_next; 1580 size_t sn; 1581 bool zero, commit; 1582 UNUSED void *chunk; 1583 1584 npages = extent_node_size_get(chunkselm) >> LG_PAGE; 1585 if (opt_purge == purge_mode_decay && arena->ndirty - 1586 (nstashed + npages) < ndirty_limit) 1587 break; 1588 1589 chunkselm_next = qr_next(chunkselm, cc_link); 1590 /* 1591 * Allocate. chunkselm remains valid due to the 1592 * dalloc_node=false argument to chunk_alloc_cache(). 1593 */ 1594 zero = false; 1595 commit = false; 1596 chunk = chunk_alloc_cache(tsdn, arena, chunk_hooks, 1597 extent_node_addr_get(chunkselm), 1598 extent_node_size_get(chunkselm), chunksize, &sn, 1599 &zero, &commit, false); 1600 assert(chunk == extent_node_addr_get(chunkselm)); 1601 assert(zero == extent_node_zeroed_get(chunkselm)); 1602 extent_node_dirty_insert(chunkselm, purge_runs_sentinel, 1603 purge_chunks_sentinel); 1604 assert(npages == (extent_node_size_get(chunkselm) >> 1605 LG_PAGE)); 1606 chunkselm = chunkselm_next; 1607 } else { 1608 arena_chunk_t *chunk = 1609 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); 1610 arena_chunk_map_misc_t *miscelm = 1611 arena_rd_to_miscelm(rdelm); 1612 size_t pageind = arena_miscelm_to_pageind(miscelm); 1613 arena_run_t *run = &miscelm->run; 1614 size_t run_size = 1615 arena_mapbits_unallocated_size_get(chunk, pageind); 1616 1617 npages = run_size >> LG_PAGE; 1618 if (opt_purge == purge_mode_decay && arena->ndirty - 1619 (nstashed + npages) < ndirty_limit) 1620 break; 1621 1622 assert(pageind + npages <= chunk_npages); 1623 assert(arena_mapbits_dirty_get(chunk, pageind) == 1624 arena_mapbits_dirty_get(chunk, pageind+npages-1)); 1625 1626 /* 1627 * If purging the spare chunk's run, make it available 1628 * prior to allocation. 1629 */ 1630 if (chunk == arena->spare) 1631 arena_chunk_alloc(tsdn, arena); 1632 1633 /* Temporarily allocate the free dirty run. */ 1634 arena_run_split_large(arena, run, run_size, false); 1635 /* Stash. */ 1636 if (false) 1637 qr_new(rdelm, rd_link); /* Redundant. */ 1638 else { 1639 assert(qr_next(rdelm, rd_link) == rdelm); 1640 assert(qr_prev(rdelm, rd_link) == rdelm); 1641 } 1642 qr_meld(purge_runs_sentinel, rdelm, rd_link); 1643 } 1644 1645 nstashed += npages; 1646 if (opt_purge == purge_mode_ratio && arena->ndirty - nstashed <= 1647 ndirty_limit) 1648 break; 1649 } 1650 1651 return (nstashed); 1652 } 1653 1654 static size_t 1655 arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, 1656 arena_runs_dirty_link_t *purge_runs_sentinel, 1657 extent_node_t *purge_chunks_sentinel) 1658 { 1659 size_t npurged, nmadvise; 1660 arena_runs_dirty_link_t *rdelm; 1661 extent_node_t *chunkselm; 1662 1663 if (config_stats) 1664 nmadvise = 0; 1665 npurged = 0; 1666 1667 malloc_mutex_unlock(tsdn, &arena->lock); 1668 for (rdelm = qr_next(purge_runs_sentinel, rd_link), 1669 chunkselm = qr_next(purge_chunks_sentinel, cc_link); 1670 rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) { 1671 size_t npages; 1672 1673 if (rdelm == &chunkselm->rd) { 1674 /* 1675 * Don't actually purge the chunk here because 1) 1676 * chunkselm is embedded in the chunk and must remain 1677 * valid, and 2) we deallocate the chunk in 1678 * arena_unstash_purged(), where it is destroyed, 1679 * decommitted, or purged, depending on chunk 1680 * deallocation policy. 1681 */ 1682 size_t size = extent_node_size_get(chunkselm); 1683 npages = size >> LG_PAGE; 1684 chunkselm = qr_next(chunkselm, cc_link); 1685 } else { 1686 size_t pageind, run_size, flag_unzeroed, flags, i; 1687 bool decommitted; 1688 arena_chunk_t *chunk = 1689 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); 1690 arena_chunk_map_misc_t *miscelm = 1691 arena_rd_to_miscelm(rdelm); 1692 pageind = arena_miscelm_to_pageind(miscelm); 1693 run_size = arena_mapbits_large_size_get(chunk, pageind); 1694 npages = run_size >> LG_PAGE; 1695 1696 /* 1697 * If this is the first run purged within chunk, mark 1698 * the chunk as non-huge. This will prevent all use of 1699 * transparent huge pages for this chunk until the chunk 1700 * as a whole is deallocated. 1701 */ 1702 if (chunk->hugepage) { 1703 pages_nohuge(chunk, chunksize); 1704 chunk->hugepage = false; 1705 } 1706 1707 assert(pageind + npages <= chunk_npages); 1708 assert(!arena_mapbits_decommitted_get(chunk, pageind)); 1709 assert(!arena_mapbits_decommitted_get(chunk, 1710 pageind+npages-1)); 1711 decommitted = !chunk_hooks->decommit(chunk, chunksize, 1712 pageind << LG_PAGE, npages << LG_PAGE, arena->ind); 1713 if (decommitted) { 1714 flag_unzeroed = 0; 1715 flags = CHUNK_MAP_DECOMMITTED; 1716 } else { 1717 flag_unzeroed = chunk_purge_wrapper(tsdn, arena, 1718 chunk_hooks, chunk, chunksize, pageind << 1719 LG_PAGE, run_size) ? CHUNK_MAP_UNZEROED : 0; 1720 flags = flag_unzeroed; 1721 } 1722 arena_mapbits_large_set(chunk, pageind+npages-1, 0, 1723 flags); 1724 arena_mapbits_large_set(chunk, pageind, run_size, 1725 flags); 1726 1727 /* 1728 * Set the unzeroed flag for internal pages, now that 1729 * chunk_purge_wrapper() has returned whether the pages 1730 * were zeroed as a side effect of purging. This chunk 1731 * map modification is safe even though the arena mutex 1732 * isn't currently owned by this thread, because the run 1733 * is marked as allocated, thus protecting it from being 1734 * modified by any other thread. As long as these 1735 * writes don't perturb the first and last elements' 1736 * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 1737 */ 1738 for (i = 1; i < npages-1; i++) { 1739 arena_mapbits_internal_set(chunk, pageind+i, 1740 flag_unzeroed); 1741 } 1742 } 1743 1744 npurged += npages; 1745 if (config_stats) 1746 nmadvise++; 1747 } 1748 malloc_mutex_lock(tsdn, &arena->lock); 1749 1750 if (config_stats) { 1751 arena->stats.nmadvise += nmadvise; 1752 arena->stats.purged += npurged; 1753 } 1754 1755 return (npurged); 1756 } 1757 1758 static void 1759 arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, 1760 arena_runs_dirty_link_t *purge_runs_sentinel, 1761 extent_node_t *purge_chunks_sentinel) 1762 { 1763 arena_runs_dirty_link_t *rdelm, *rdelm_next; 1764 extent_node_t *chunkselm; 1765 1766 /* Deallocate chunks/runs. */ 1767 for (rdelm = qr_next(purge_runs_sentinel, rd_link), 1768 chunkselm = qr_next(purge_chunks_sentinel, cc_link); 1769 rdelm != purge_runs_sentinel; rdelm = rdelm_next) { 1770 rdelm_next = qr_next(rdelm, rd_link); 1771 if (rdelm == &chunkselm->rd) { 1772 extent_node_t *chunkselm_next = qr_next(chunkselm, 1773 cc_link); 1774 void *addr = extent_node_addr_get(chunkselm); 1775 size_t size = extent_node_size_get(chunkselm); 1776 size_t sn = extent_node_sn_get(chunkselm); 1777 bool zeroed = extent_node_zeroed_get(chunkselm); 1778 bool committed = extent_node_committed_get(chunkselm); 1779 extent_node_dirty_remove(chunkselm); 1780 arena_node_dalloc(tsdn, arena, chunkselm); 1781 chunkselm = chunkselm_next; 1782 chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, addr, 1783 size, sn, zeroed, committed); 1784 } else { 1785 arena_chunk_t *chunk = 1786 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); 1787 arena_chunk_map_misc_t *miscelm = 1788 arena_rd_to_miscelm(rdelm); 1789 size_t pageind = arena_miscelm_to_pageind(miscelm); 1790 bool decommitted = (arena_mapbits_decommitted_get(chunk, 1791 pageind) != 0); 1792 arena_run_t *run = &miscelm->run; 1793 qr_remove(rdelm, rd_link); 1794 arena_run_dalloc(tsdn, arena, run, false, true, 1795 decommitted); 1796 } 1797 } 1798 } 1799 1800 /* 1801 * NB: ndirty_limit is interpreted differently depending on opt_purge: 1802 * - purge_mode_ratio: Purge as few dirty run/chunks as possible to reach the 1803 * desired state: 1804 * (arena->ndirty <= ndirty_limit) 1805 * - purge_mode_decay: Purge as many dirty runs/chunks as possible without 1806 * violating the invariant: 1807 * (arena->ndirty >= ndirty_limit) 1808 */ 1809 static void 1810 arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit) 1811 { 1812 chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena); 1813 size_t npurge, npurged; 1814 arena_runs_dirty_link_t purge_runs_sentinel; 1815 extent_node_t purge_chunks_sentinel; 1816 1817 arena->purging = true; 1818 1819 /* 1820 * Calls to arena_dirty_count() are disabled even for debug builds 1821 * because overhead grows nonlinearly as memory usage increases. 1822 */ 1823 if (false && config_debug) { 1824 size_t ndirty = arena_dirty_count(arena); 1825 assert(ndirty == arena->ndirty); 1826 } 1827 assert(opt_purge != purge_mode_ratio || (arena->nactive >> 1828 arena->lg_dirty_mult) < arena->ndirty || ndirty_limit == 0); 1829 1830 qr_new(&purge_runs_sentinel, rd_link); 1831 extent_node_dirty_linkage_init(&purge_chunks_sentinel); 1832 1833 npurge = arena_stash_dirty(tsdn, arena, &chunk_hooks, ndirty_limit, 1834 &purge_runs_sentinel, &purge_chunks_sentinel); 1835 if (npurge == 0) 1836 goto label_return; 1837 npurged = arena_purge_stashed(tsdn, arena, &chunk_hooks, 1838 &purge_runs_sentinel, &purge_chunks_sentinel); 1839 assert(npurged == npurge); 1840 arena_unstash_purged(tsdn, arena, &chunk_hooks, &purge_runs_sentinel, 1841 &purge_chunks_sentinel); 1842 1843 if (config_stats) 1844 arena->stats.npurge++; 1845 1846 label_return: 1847 arena->purging = false; 1848 } 1849 1850 void 1851 arena_purge(tsdn_t *tsdn, arena_t *arena, bool all) 1852 { 1853 1854 malloc_mutex_lock(tsdn, &arena->lock); 1855 if (all) 1856 arena_purge_to_limit(tsdn, arena, 0); 1857 else 1858 arena_maybe_purge(tsdn, arena); 1859 malloc_mutex_unlock(tsdn, &arena->lock); 1860 } 1861 1862 static void 1863 arena_achunk_prof_reset(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk) 1864 { 1865 size_t pageind, npages; 1866 1867 cassert(config_prof); 1868 assert(opt_prof); 1869 1870 /* 1871 * Iterate over the allocated runs and remove profiled allocations from 1872 * the sample set. 1873 */ 1874 for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { 1875 if (arena_mapbits_allocated_get(chunk, pageind) != 0) { 1876 if (arena_mapbits_large_get(chunk, pageind) != 0) { 1877 void *ptr = (void *)((uintptr_t)chunk + (pageind 1878 << LG_PAGE)); 1879 size_t usize = isalloc(tsd_tsdn(tsd), ptr, 1880 config_prof); 1881 1882 prof_free(tsd, ptr, usize); 1883 npages = arena_mapbits_large_size_get(chunk, 1884 pageind) >> LG_PAGE; 1885 } else { 1886 /* Skip small run. */ 1887 size_t binind = arena_mapbits_binind_get(chunk, 1888 pageind); 1889 arena_bin_info_t *bin_info = 1890 &arena_bin_info[binind]; 1891 npages = bin_info->run_size >> LG_PAGE; 1892 } 1893 } else { 1894 /* Skip unallocated run. */ 1895 npages = arena_mapbits_unallocated_size_get(chunk, 1896 pageind) >> LG_PAGE; 1897 } 1898 assert(pageind + npages <= chunk_npages); 1899 } 1900 } 1901 1902 void 1903 arena_reset(tsd_t *tsd, arena_t *arena) 1904 { 1905 unsigned i; 1906 extent_node_t *node; 1907 1908 /* 1909 * Locking in this function is unintuitive. The caller guarantees that 1910 * no concurrent operations are happening in this arena, but there are 1911 * still reasons that some locking is necessary: 1912 * 1913 * - Some of the functions in the transitive closure of calls assume 1914 * appropriate locks are held, and in some cases these locks are 1915 * temporarily dropped to avoid lock order reversal or deadlock due to 1916 * reentry. 1917 * - mallctl("epoch", ...) may concurrently refresh stats. While 1918 * strictly speaking this is a "concurrent operation", disallowing 1919 * stats refreshes would impose an inconvenient burden. 1920 */ 1921 1922 /* Remove large allocations from prof sample set. */ 1923 if (config_prof && opt_prof) { 1924 ql_foreach(node, &arena->achunks, ql_link) { 1925 arena_achunk_prof_reset(tsd, arena, 1926 extent_node_addr_get(node)); 1927 } 1928 } 1929 1930 /* Reset curruns for large size classes. */ 1931 if (config_stats) { 1932 for (i = 0; i < nlclasses; i++) 1933 arena->stats.lstats[i].curruns = 0; 1934 } 1935 1936 /* Huge allocations. */ 1937 malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx); 1938 for (node = ql_last(&arena->huge, ql_link); node != NULL; node = 1939 ql_last(&arena->huge, ql_link)) { 1940 void *ptr = extent_node_addr_get(node); 1941 size_t usize; 1942 1943 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx); 1944 if (config_stats || (config_prof && opt_prof)) 1945 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 1946 /* Remove huge allocation from prof sample set. */ 1947 if (config_prof && opt_prof) 1948 prof_free(tsd, ptr, usize); 1949 huge_dalloc(tsd_tsdn(tsd), ptr); 1950 malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx); 1951 /* Cancel out unwanted effects on stats. */ 1952 if (config_stats) 1953 arena_huge_reset_stats_cancel(arena, usize); 1954 } 1955 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx); 1956 1957 malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock); 1958 1959 /* Bins. */ 1960 for (i = 0; i < NBINS; i++) { 1961 arena_bin_t *bin = &arena->bins[i]; 1962 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); 1963 bin->runcur = NULL; 1964 arena_run_heap_new(&bin->runs); 1965 if (config_stats) { 1966 bin->stats.curregs = 0; 1967 bin->stats.curruns = 0; 1968 } 1969 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); 1970 } 1971 1972 /* 1973 * Re-initialize runs_dirty such that the chunks_cache and runs_dirty 1974 * chains directly correspond. 1975 */ 1976 qr_new(&arena->runs_dirty, rd_link); 1977 for (node = qr_next(&arena->chunks_cache, cc_link); 1978 node != &arena->chunks_cache; node = qr_next(node, cc_link)) { 1979 qr_new(&node->rd, rd_link); 1980 qr_meld(&arena->runs_dirty, &node->rd, rd_link); 1981 } 1982 1983 /* Arena chunks. */ 1984 for (node = ql_last(&arena->achunks, ql_link); node != NULL; node = 1985 ql_last(&arena->achunks, ql_link)) { 1986 ql_remove(&arena->achunks, node, ql_link); 1987 arena_chunk_discard(tsd_tsdn(tsd), arena, 1988 extent_node_addr_get(node)); 1989 } 1990 1991 /* Spare. */ 1992 if (arena->spare != NULL) { 1993 arena_chunk_discard(tsd_tsdn(tsd), arena, arena->spare); 1994 arena->spare = NULL; 1995 } 1996 1997 assert(!arena->purging); 1998 arena->nactive = 0; 1999 2000 for (i = 0; i < NPSIZES; i++) 2001 arena_run_heap_new(&arena->runs_avail[i]); 2002 2003 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock); 2004 } 2005 2006 static void 2007 arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, 2008 size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty, 2009 size_t flag_decommitted) 2010 { 2011 size_t size = *p_size; 2012 size_t run_ind = *p_run_ind; 2013 size_t run_pages = *p_run_pages; 2014 2015 /* Try to coalesce forward. */ 2016 if (run_ind + run_pages < chunk_npages && 2017 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 2018 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty && 2019 arena_mapbits_decommitted_get(chunk, run_ind+run_pages) == 2020 flag_decommitted) { 2021 size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 2022 run_ind+run_pages); 2023 size_t nrun_pages = nrun_size >> LG_PAGE; 2024 2025 /* 2026 * Remove successor from runs_avail; the coalesced run is 2027 * inserted later. 2028 */ 2029 assert(arena_mapbits_unallocated_size_get(chunk, 2030 run_ind+run_pages+nrun_pages-1) == nrun_size); 2031 assert(arena_mapbits_dirty_get(chunk, 2032 run_ind+run_pages+nrun_pages-1) == flag_dirty); 2033 assert(arena_mapbits_decommitted_get(chunk, 2034 run_ind+run_pages+nrun_pages-1) == flag_decommitted); 2035 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); 2036 2037 /* 2038 * If the successor is dirty, remove it from the set of dirty 2039 * pages. 2040 */ 2041 if (flag_dirty != 0) { 2042 arena_run_dirty_remove(arena, chunk, run_ind+run_pages, 2043 nrun_pages); 2044 } 2045 2046 size += nrun_size; 2047 run_pages += nrun_pages; 2048 2049 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 2050 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 2051 size); 2052 } 2053 2054 /* Try to coalesce backward. */ 2055 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, 2056 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == 2057 flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) == 2058 flag_decommitted) { 2059 size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 2060 run_ind-1); 2061 size_t prun_pages = prun_size >> LG_PAGE; 2062 2063 run_ind -= prun_pages; 2064 2065 /* 2066 * Remove predecessor from runs_avail; the coalesced run is 2067 * inserted later. 2068 */ 2069 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 2070 prun_size); 2071 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 2072 assert(arena_mapbits_decommitted_get(chunk, run_ind) == 2073 flag_decommitted); 2074 arena_avail_remove(arena, chunk, run_ind, prun_pages); 2075 2076 /* 2077 * If the predecessor is dirty, remove it from the set of dirty 2078 * pages. 2079 */ 2080 if (flag_dirty != 0) { 2081 arena_run_dirty_remove(arena, chunk, run_ind, 2082 prun_pages); 2083 } 2084 2085 size += prun_size; 2086 run_pages += prun_pages; 2087 2088 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 2089 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 2090 size); 2091 } 2092 2093 *p_size = size; 2094 *p_run_ind = run_ind; 2095 *p_run_pages = run_pages; 2096 } 2097 2098 static size_t 2099 arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 2100 size_t run_ind) 2101 { 2102 size_t size; 2103 2104 assert(run_ind >= map_bias); 2105 assert(run_ind < chunk_npages); 2106 2107 if (arena_mapbits_large_get(chunk, run_ind) != 0) { 2108 size = arena_mapbits_large_size_get(chunk, run_ind); 2109 assert(size == PAGE || arena_mapbits_large_size_get(chunk, 2110 run_ind+(size>>LG_PAGE)-1) == 0); 2111 } else { 2112 arena_bin_info_t *bin_info = &arena_bin_info[run->binind]; 2113 size = bin_info->run_size; 2114 } 2115 2116 return (size); 2117 } 2118 2119 static void 2120 arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, bool dirty, 2121 bool cleaned, bool decommitted) 2122 { 2123 arena_chunk_t *chunk; 2124 arena_chunk_map_misc_t *miscelm; 2125 size_t size, run_ind, run_pages, flag_dirty, flag_decommitted; 2126 2127 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 2128 miscelm = arena_run_to_miscelm(run); 2129 run_ind = arena_miscelm_to_pageind(miscelm); 2130 assert(run_ind >= map_bias); 2131 assert(run_ind < chunk_npages); 2132 size = arena_run_size_get(arena, chunk, run, run_ind); 2133 run_pages = (size >> LG_PAGE); 2134 arena_nactive_sub(arena, run_pages); 2135 2136 /* 2137 * The run is dirty if the caller claims to have dirtied it, as well as 2138 * if it was already dirty before being allocated and the caller 2139 * doesn't claim to have cleaned it. 2140 */ 2141 assert(arena_mapbits_dirty_get(chunk, run_ind) == 2142 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 2143 if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind) 2144 != 0) 2145 dirty = true; 2146 flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 2147 flag_decommitted = decommitted ? CHUNK_MAP_DECOMMITTED : 0; 2148 2149 /* Mark pages as unallocated in the chunk map. */ 2150 if (dirty || decommitted) { 2151 size_t flags = flag_dirty | flag_decommitted; 2152 arena_mapbits_unallocated_set(chunk, run_ind, size, flags); 2153 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 2154 flags); 2155 } else { 2156 arena_mapbits_unallocated_set(chunk, run_ind, size, 2157 arena_mapbits_unzeroed_get(chunk, run_ind)); 2158 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 2159 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 2160 } 2161 2162 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, 2163 flag_dirty, flag_decommitted); 2164 2165 /* Insert into runs_avail, now that coalescing is complete. */ 2166 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 2167 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 2168 assert(arena_mapbits_dirty_get(chunk, run_ind) == 2169 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 2170 assert(arena_mapbits_decommitted_get(chunk, run_ind) == 2171 arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1)); 2172 arena_avail_insert(arena, chunk, run_ind, run_pages); 2173 2174 if (dirty) 2175 arena_run_dirty_insert(arena, chunk, run_ind, run_pages); 2176 2177 /* Deallocate chunk if it is now completely unused. */ 2178 if (size == arena_maxrun) { 2179 assert(run_ind == map_bias); 2180 assert(run_pages == (arena_maxrun >> LG_PAGE)); 2181 arena_chunk_dalloc(tsdn, arena, chunk); 2182 } 2183 2184 /* 2185 * It is okay to do dirty page processing here even if the chunk was 2186 * deallocated above, since in that case it is the spare. Waiting 2187 * until after possible chunk deallocation to do dirty processing 2188 * allows for an old spare to be fully deallocated, thus decreasing the 2189 * chances of spuriously crossing the dirty page purging threshold. 2190 */ 2191 if (dirty) 2192 arena_maybe_purge(tsdn, arena); 2193 } 2194 2195 static void 2196 arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2197 arena_run_t *run, size_t oldsize, size_t newsize) 2198 { 2199 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 2200 size_t pageind = arena_miscelm_to_pageind(miscelm); 2201 size_t head_npages = (oldsize - newsize) >> LG_PAGE; 2202 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 2203 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); 2204 size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? 2205 CHUNK_MAP_UNZEROED : 0; 2206 2207 assert(oldsize > newsize); 2208 2209 /* 2210 * Update the chunk map so that arena_run_dalloc() can treat the 2211 * leading run as separately allocated. Set the last element of each 2212 * run first, in case of single-page runs. 2213 */ 2214 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 2215 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | 2216 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 2217 pageind+head_npages-1))); 2218 arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty | 2219 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); 2220 2221 if (config_debug) { 2222 UNUSED size_t tail_npages = newsize >> LG_PAGE; 2223 assert(arena_mapbits_large_size_get(chunk, 2224 pageind+head_npages+tail_npages-1) == 0); 2225 assert(arena_mapbits_dirty_get(chunk, 2226 pageind+head_npages+tail_npages-1) == flag_dirty); 2227 } 2228 arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 2229 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 2230 pageind+head_npages))); 2231 2232 arena_run_dalloc(tsdn, arena, run, false, false, (flag_decommitted != 2233 0)); 2234 } 2235 2236 static void 2237 arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2238 arena_run_t *run, size_t oldsize, size_t newsize, bool dirty) 2239 { 2240 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 2241 size_t pageind = arena_miscelm_to_pageind(miscelm); 2242 size_t head_npages = newsize >> LG_PAGE; 2243 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 2244 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); 2245 size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? 2246 CHUNK_MAP_UNZEROED : 0; 2247 arena_chunk_map_misc_t *tail_miscelm; 2248 arena_run_t *tail_run; 2249 2250 assert(oldsize > newsize); 2251 2252 /* 2253 * Update the chunk map so that arena_run_dalloc() can treat the 2254 * trailing run as separately allocated. Set the last element of each 2255 * run first, in case of single-page runs. 2256 */ 2257 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 2258 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | 2259 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 2260 pageind+head_npages-1))); 2261 arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty | 2262 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); 2263 2264 if (config_debug) { 2265 UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 2266 assert(arena_mapbits_large_size_get(chunk, 2267 pageind+head_npages+tail_npages-1) == 0); 2268 assert(arena_mapbits_dirty_get(chunk, 2269 pageind+head_npages+tail_npages-1) == flag_dirty); 2270 } 2271 arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 2272 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 2273 pageind+head_npages))); 2274 2275 tail_miscelm = arena_miscelm_get_mutable(chunk, pageind + head_npages); 2276 tail_run = &tail_miscelm->run; 2277 arena_run_dalloc(tsdn, arena, tail_run, dirty, false, (flag_decommitted 2278 != 0)); 2279 } 2280 2281 static void 2282 arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 2283 { 2284 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 2285 2286 arena_run_heap_insert(&bin->runs, miscelm); 2287 } 2288 2289 static arena_run_t * 2290 arena_bin_nonfull_run_tryget(arena_bin_t *bin) 2291 { 2292 arena_chunk_map_misc_t *miscelm; 2293 2294 miscelm = arena_run_heap_remove_first(&bin->runs); 2295 if (miscelm == NULL) 2296 return (NULL); 2297 if (config_stats) 2298 bin->stats.reruns++; 2299 2300 return (&miscelm->run); 2301 } 2302 2303 static arena_run_t * 2304 arena_bin_nonfull_run_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin) 2305 { 2306 arena_run_t *run; 2307 szind_t binind; 2308 arena_bin_info_t *bin_info; 2309 2310 /* Look for a usable run. */ 2311 run = arena_bin_nonfull_run_tryget(bin); 2312 if (run != NULL) 2313 return (run); 2314 /* No existing runs have any space available. */ 2315 2316 binind = arena_bin_index(arena, bin); 2317 bin_info = &arena_bin_info[binind]; 2318 2319 /* Allocate a new run. */ 2320 malloc_mutex_unlock(tsdn, &bin->lock); 2321 /******************************/ 2322 malloc_mutex_lock(tsdn, &arena->lock); 2323 run = arena_run_alloc_small(tsdn, arena, bin_info->run_size, binind); 2324 if (run != NULL) { 2325 /* Initialize run internals. */ 2326 run->binind = binind; 2327 run->nfree = bin_info->nregs; 2328 bitmap_init(run->bitmap, &bin_info->bitmap_info); 2329 } 2330 malloc_mutex_unlock(tsdn, &arena->lock); 2331 /********************************/ 2332 malloc_mutex_lock(tsdn, &bin->lock); 2333 if (run != NULL) { 2334 if (config_stats) { 2335 bin->stats.nruns++; 2336 bin->stats.curruns++; 2337 } 2338 return (run); 2339 } 2340 2341 /* 2342 * arena_run_alloc_small() failed, but another thread may have made 2343 * sufficient memory available while this one dropped bin->lock above, 2344 * so search one more time. 2345 */ 2346 run = arena_bin_nonfull_run_tryget(bin); 2347 if (run != NULL) 2348 return (run); 2349 2350 return (NULL); 2351 } 2352 2353 /* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 2354 static void * 2355 arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin) 2356 { 2357 szind_t binind; 2358 arena_bin_info_t *bin_info; 2359 arena_run_t *run; 2360 2361 binind = arena_bin_index(arena, bin); 2362 bin_info = &arena_bin_info[binind]; 2363 bin->runcur = NULL; 2364 run = arena_bin_nonfull_run_get(tsdn, arena, bin); 2365 if (bin->runcur != NULL && bin->runcur->nfree > 0) { 2366 /* 2367 * Another thread updated runcur while this one ran without the 2368 * bin lock in arena_bin_nonfull_run_get(). 2369 */ 2370 void *ret; 2371 assert(bin->runcur->nfree > 0); 2372 ret = arena_run_reg_alloc(bin->runcur, bin_info); 2373 if (run != NULL) { 2374 arena_chunk_t *chunk; 2375 2376 /* 2377 * arena_run_alloc_small() may have allocated run, or 2378 * it may have pulled run from the bin's run tree. 2379 * Therefore it is unsafe to make any assumptions about 2380 * how run has previously been used, and 2381 * arena_bin_lower_run() must be called, as if a region 2382 * were just deallocated from the run. 2383 */ 2384 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 2385 if (run->nfree == bin_info->nregs) { 2386 arena_dalloc_bin_run(tsdn, arena, chunk, run, 2387 bin); 2388 } else 2389 arena_bin_lower_run(arena, run, bin); 2390 } 2391 return (ret); 2392 } 2393 2394 if (run == NULL) 2395 return (NULL); 2396 2397 bin->runcur = run; 2398 2399 assert(bin->runcur->nfree > 0); 2400 2401 return (arena_run_reg_alloc(bin->runcur, bin_info)); 2402 } 2403 2404 void 2405 arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_bin_t *tbin, 2406 szind_t binind, uint64_t prof_accumbytes) 2407 { 2408 unsigned i, nfill; 2409 arena_bin_t *bin; 2410 2411 assert(tbin->ncached == 0); 2412 2413 if (config_prof && arena_prof_accum(tsdn, arena, prof_accumbytes)) 2414 prof_idump(tsdn); 2415 bin = &arena->bins[binind]; 2416 malloc_mutex_lock(tsdn, &bin->lock); 2417 for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 2418 tbin->lg_fill_div); i < nfill; i++) { 2419 arena_run_t *run; 2420 void *ptr; 2421 if ((run = bin->runcur) != NULL && run->nfree > 0) 2422 ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 2423 else 2424 ptr = arena_bin_malloc_hard(tsdn, arena, bin); 2425 if (ptr == NULL) { 2426 /* 2427 * OOM. tbin->avail isn't yet filled down to its first 2428 * element, so the successful allocations (if any) must 2429 * be moved just before tbin->avail before bailing out. 2430 */ 2431 if (i > 0) { 2432 memmove(tbin->avail - i, tbin->avail - nfill, 2433 i * sizeof(void *)); 2434 } 2435 break; 2436 } 2437 if (config_fill && unlikely(opt_junk_alloc)) { 2438 arena_alloc_junk_small(ptr, &arena_bin_info[binind], 2439 true); 2440 } 2441 /* Insert such that low regions get used first. */ 2442 *(tbin->avail - nfill + i) = ptr; 2443 } 2444 if (config_stats) { 2445 bin->stats.nmalloc += i; 2446 bin->stats.nrequests += tbin->tstats.nrequests; 2447 bin->stats.curregs += i; 2448 bin->stats.nfills++; 2449 tbin->tstats.nrequests = 0; 2450 } 2451 malloc_mutex_unlock(tsdn, &bin->lock); 2452 tbin->ncached = i; 2453 arena_decay_tick(tsdn, arena); 2454 } 2455 2456 void 2457 arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 2458 { 2459 2460 size_t redzone_size = bin_info->redzone_size; 2461 2462 if (zero) { 2463 memset((void *)((uintptr_t)ptr - redzone_size), 2464 JEMALLOC_ALLOC_JUNK, redzone_size); 2465 memset((void *)((uintptr_t)ptr + bin_info->reg_size), 2466 JEMALLOC_ALLOC_JUNK, redzone_size); 2467 } else { 2468 memset((void *)((uintptr_t)ptr - redzone_size), 2469 JEMALLOC_ALLOC_JUNK, bin_info->reg_interval); 2470 } 2471 } 2472 2473 #ifdef JEMALLOC_JET 2474 #undef arena_redzone_corruption 2475 #define arena_redzone_corruption JEMALLOC_N(n_arena_redzone_corruption) 2476 #endif 2477 static void 2478 arena_redzone_corruption(void *ptr, size_t usize, bool after, 2479 size_t offset, uint8_t byte) 2480 { 2481 2482 malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p " 2483 "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", 2484 after ? "after" : "before", ptr, usize, byte); 2485 } 2486 #ifdef JEMALLOC_JET 2487 #undef arena_redzone_corruption 2488 #define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) 2489 arena_redzone_corruption_t *arena_redzone_corruption = 2490 JEMALLOC_N(n_arena_redzone_corruption); 2491 #endif 2492 2493 static void 2494 arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) 2495 { 2496 bool error = false; 2497 2498 if (opt_junk_alloc) { 2499 size_t size = bin_info->reg_size; 2500 size_t redzone_size = bin_info->redzone_size; 2501 size_t i; 2502 2503 for (i = 1; i <= redzone_size; i++) { 2504 uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); 2505 if (*byte != JEMALLOC_ALLOC_JUNK) { 2506 error = true; 2507 arena_redzone_corruption(ptr, size, false, i, 2508 *byte); 2509 if (reset) 2510 *byte = JEMALLOC_ALLOC_JUNK; 2511 } 2512 } 2513 for (i = 0; i < redzone_size; i++) { 2514 uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); 2515 if (*byte != JEMALLOC_ALLOC_JUNK) { 2516 error = true; 2517 arena_redzone_corruption(ptr, size, true, i, 2518 *byte); 2519 if (reset) 2520 *byte = JEMALLOC_ALLOC_JUNK; 2521 } 2522 } 2523 } 2524 2525 if (opt_abort && error) 2526 abort(); 2527 } 2528 2529 #ifdef JEMALLOC_JET 2530 #undef arena_dalloc_junk_small 2531 #define arena_dalloc_junk_small JEMALLOC_N(n_arena_dalloc_junk_small) 2532 #endif 2533 void 2534 arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 2535 { 2536 size_t redzone_size = bin_info->redzone_size; 2537 2538 arena_redzones_validate(ptr, bin_info, false); 2539 memset((void *)((uintptr_t)ptr - redzone_size), JEMALLOC_FREE_JUNK, 2540 bin_info->reg_interval); 2541 } 2542 #ifdef JEMALLOC_JET 2543 #undef arena_dalloc_junk_small 2544 #define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) 2545 arena_dalloc_junk_small_t *arena_dalloc_junk_small = 2546 JEMALLOC_N(n_arena_dalloc_junk_small); 2547 #endif 2548 2549 void 2550 arena_quarantine_junk_small(void *ptr, size_t usize) 2551 { 2552 szind_t binind; 2553 arena_bin_info_t *bin_info; 2554 cassert(config_fill); 2555 assert(opt_junk_free); 2556 assert(opt_quarantine); 2557 assert(usize <= SMALL_MAXCLASS); 2558 2559 binind = size2index(usize); 2560 bin_info = &arena_bin_info[binind]; 2561 arena_redzones_validate(ptr, bin_info, true); 2562 } 2563 2564 static void * 2565 arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) 2566 { 2567 void *ret; 2568 arena_bin_t *bin; 2569 size_t usize; 2570 arena_run_t *run; 2571 2572 assert(binind < NBINS); 2573 bin = &arena->bins[binind]; 2574 usize = index2size(binind); 2575 2576 malloc_mutex_lock(tsdn, &bin->lock); 2577 if ((run = bin->runcur) != NULL && run->nfree > 0) 2578 ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 2579 else 2580 ret = arena_bin_malloc_hard(tsdn, arena, bin); 2581 2582 if (ret == NULL) { 2583 malloc_mutex_unlock(tsdn, &bin->lock); 2584 return (NULL); 2585 } 2586 2587 if (config_stats) { 2588 bin->stats.nmalloc++; 2589 bin->stats.nrequests++; 2590 bin->stats.curregs++; 2591 } 2592 malloc_mutex_unlock(tsdn, &bin->lock); 2593 if (config_prof && !isthreaded && arena_prof_accum(tsdn, arena, usize)) 2594 prof_idump(tsdn); 2595 2596 if (!zero) { 2597 if (config_fill) { 2598 if (unlikely(opt_junk_alloc)) { 2599 arena_alloc_junk_small(ret, 2600 &arena_bin_info[binind], false); 2601 } else if (unlikely(opt_zero)) 2602 memset(ret, 0, usize); 2603 } 2604 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, usize); 2605 } else { 2606 if (config_fill && unlikely(opt_junk_alloc)) { 2607 arena_alloc_junk_small(ret, &arena_bin_info[binind], 2608 true); 2609 } 2610 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, usize); 2611 memset(ret, 0, usize); 2612 } 2613 2614 arena_decay_tick(tsdn, arena); 2615 return (ret); 2616 } 2617 2618 void * 2619 arena_malloc_large(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) 2620 { 2621 void *ret; 2622 size_t usize; 2623 uintptr_t random_offset; 2624 arena_run_t *run; 2625 arena_chunk_map_misc_t *miscelm; 2626 UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false); 2627 2628 /* Large allocation. */ 2629 usize = index2size(binind); 2630 malloc_mutex_lock(tsdn, &arena->lock); 2631 if (config_cache_oblivious) { 2632 uint64_t r; 2633 2634 /* 2635 * Compute a uniformly distributed offset within the first page 2636 * that is a multiple of the cacheline size, e.g. [0 .. 63) * 64 2637 * for 4 KiB pages and 64-byte cachelines. 2638 */ 2639 r = prng_lg_range_zu(&arena->offset_state, LG_PAGE - 2640 LG_CACHELINE, false); 2641 random_offset = ((uintptr_t)r) << LG_CACHELINE; 2642 } else 2643 random_offset = 0; 2644 run = arena_run_alloc_large(tsdn, arena, usize + large_pad, zero); 2645 if (run == NULL) { 2646 malloc_mutex_unlock(tsdn, &arena->lock); 2647 return (NULL); 2648 } 2649 miscelm = arena_run_to_miscelm(run); 2650 ret = (void *)((uintptr_t)arena_miscelm_to_rpages(miscelm) + 2651 random_offset); 2652 if (config_stats) { 2653 szind_t index = binind - NBINS; 2654 2655 arena->stats.nmalloc_large++; 2656 arena->stats.nrequests_large++; 2657 arena->stats.allocated_large += usize; 2658 arena->stats.lstats[index].nmalloc++; 2659 arena->stats.lstats[index].nrequests++; 2660 arena->stats.lstats[index].curruns++; 2661 } 2662 if (config_prof) 2663 idump = arena_prof_accum_locked(arena, usize); 2664 malloc_mutex_unlock(tsdn, &arena->lock); 2665 if (config_prof && idump) 2666 prof_idump(tsdn); 2667 2668 if (!zero) { 2669 if (config_fill) { 2670 if (unlikely(opt_junk_alloc)) 2671 memset(ret, JEMALLOC_ALLOC_JUNK, usize); 2672 else if (unlikely(opt_zero)) 2673 memset(ret, 0, usize); 2674 } 2675 } 2676 2677 arena_decay_tick(tsdn, arena); 2678 return (ret); 2679 } 2680 2681 void * 2682 arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, 2683 bool zero) 2684 { 2685 2686 assert(!tsdn_null(tsdn) || arena != NULL); 2687 2688 if (likely(!tsdn_null(tsdn))) 2689 arena = arena_choose(tsdn_tsd(tsdn), arena); 2690 if (unlikely(arena == NULL)) 2691 return (NULL); 2692 2693 if (likely(size <= SMALL_MAXCLASS)) 2694 return (arena_malloc_small(tsdn, arena, ind, zero)); 2695 if (likely(size <= large_maxclass)) 2696 return (arena_malloc_large(tsdn, arena, ind, zero)); 2697 return (huge_malloc(tsdn, arena, index2size(ind), zero)); 2698 } 2699 2700 /* Only handles large allocations that require more than page alignment. */ 2701 static void * 2702 arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, 2703 bool zero) 2704 { 2705 void *ret; 2706 size_t alloc_size, leadsize, trailsize; 2707 arena_run_t *run; 2708 arena_chunk_t *chunk; 2709 arena_chunk_map_misc_t *miscelm; 2710 void *rpages; 2711 2712 assert(!tsdn_null(tsdn) || arena != NULL); 2713 assert(usize == PAGE_CEILING(usize)); 2714 2715 if (likely(!tsdn_null(tsdn))) 2716 arena = arena_choose(tsdn_tsd(tsdn), arena); 2717 if (unlikely(arena == NULL)) 2718 return (NULL); 2719 2720 alignment = PAGE_CEILING(alignment); 2721 alloc_size = usize + large_pad + alignment - PAGE; 2722 2723 malloc_mutex_lock(tsdn, &arena->lock); 2724 run = arena_run_alloc_large(tsdn, arena, alloc_size, false); 2725 if (run == NULL) { 2726 malloc_mutex_unlock(tsdn, &arena->lock); 2727 return (NULL); 2728 } 2729 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 2730 miscelm = arena_run_to_miscelm(run); 2731 rpages = arena_miscelm_to_rpages(miscelm); 2732 2733 leadsize = ALIGNMENT_CEILING((uintptr_t)rpages, alignment) - 2734 (uintptr_t)rpages; 2735 assert(alloc_size >= leadsize + usize); 2736 trailsize = alloc_size - leadsize - usize - large_pad; 2737 if (leadsize != 0) { 2738 arena_chunk_map_misc_t *head_miscelm = miscelm; 2739 arena_run_t *head_run = run; 2740 2741 miscelm = arena_miscelm_get_mutable(chunk, 2742 arena_miscelm_to_pageind(head_miscelm) + (leadsize >> 2743 LG_PAGE)); 2744 run = &miscelm->run; 2745 2746 arena_run_trim_head(tsdn, arena, chunk, head_run, alloc_size, 2747 alloc_size - leadsize); 2748 } 2749 if (trailsize != 0) { 2750 arena_run_trim_tail(tsdn, arena, chunk, run, usize + large_pad + 2751 trailsize, usize + large_pad, false); 2752 } 2753 if (arena_run_init_large(arena, run, usize + large_pad, zero)) { 2754 size_t run_ind = 2755 arena_miscelm_to_pageind(arena_run_to_miscelm(run)); 2756 bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0); 2757 bool decommitted = (arena_mapbits_decommitted_get(chunk, 2758 run_ind) != 0); 2759 2760 assert(decommitted); /* Cause of OOM. */ 2761 arena_run_dalloc(tsdn, arena, run, dirty, false, decommitted); 2762 malloc_mutex_unlock(tsdn, &arena->lock); 2763 return (NULL); 2764 } 2765 ret = arena_miscelm_to_rpages(miscelm); 2766 2767 if (config_stats) { 2768 szind_t index = size2index(usize) - NBINS; 2769 2770 arena->stats.nmalloc_large++; 2771 arena->stats.nrequests_large++; 2772 arena->stats.allocated_large += usize; 2773 arena->stats.lstats[index].nmalloc++; 2774 arena->stats.lstats[index].nrequests++; 2775 arena->stats.lstats[index].curruns++; 2776 } 2777 malloc_mutex_unlock(tsdn, &arena->lock); 2778 2779 if (config_fill && !zero) { 2780 if (unlikely(opt_junk_alloc)) 2781 memset(ret, JEMALLOC_ALLOC_JUNK, usize); 2782 else if (unlikely(opt_zero)) 2783 memset(ret, 0, usize); 2784 } 2785 arena_decay_tick(tsdn, arena); 2786 return (ret); 2787 } 2788 2789 void * 2790 arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, 2791 bool zero, tcache_t *tcache) 2792 { 2793 void *ret; 2794 2795 if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE 2796 && (usize & PAGE_MASK) == 0))) { 2797 /* Small; alignment doesn't require special run placement. */ 2798 ret = arena_malloc(tsdn, arena, usize, size2index(usize), zero, 2799 tcache, true); 2800 } else if (usize <= large_maxclass && alignment <= PAGE) { 2801 /* 2802 * Large; alignment doesn't require special run placement. 2803 * However, the cached pointer may be at a random offset from 2804 * the base of the run, so do some bit manipulation to retrieve 2805 * the base. 2806 */ 2807 ret = arena_malloc(tsdn, arena, usize, size2index(usize), zero, 2808 tcache, true); 2809 if (config_cache_oblivious) 2810 ret = (void *)((uintptr_t)ret & ~PAGE_MASK); 2811 } else { 2812 if (likely(usize <= large_maxclass)) { 2813 ret = arena_palloc_large(tsdn, arena, usize, alignment, 2814 zero); 2815 } else if (likely(alignment <= chunksize)) 2816 ret = huge_malloc(tsdn, arena, usize, zero); 2817 else { 2818 ret = huge_palloc(tsdn, arena, usize, alignment, zero); 2819 } 2820 } 2821 return (ret); 2822 } 2823 2824 void 2825 arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size) 2826 { 2827 arena_chunk_t *chunk; 2828 size_t pageind; 2829 szind_t binind; 2830 2831 cassert(config_prof); 2832 assert(ptr != NULL); 2833 assert(CHUNK_ADDR2BASE(ptr) != ptr); 2834 assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS); 2835 assert(isalloc(tsdn, ptr, true) == LARGE_MINCLASS); 2836 assert(size <= SMALL_MAXCLASS); 2837 2838 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2839 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2840 binind = size2index(size); 2841 assert(binind < NBINS); 2842 arena_mapbits_large_binind_set(chunk, pageind, binind); 2843 2844 assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS); 2845 assert(isalloc(tsdn, ptr, true) == size); 2846 } 2847 2848 static void 2849 arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 2850 arena_bin_t *bin) 2851 { 2852 2853 /* Dissociate run from bin. */ 2854 if (run == bin->runcur) 2855 bin->runcur = NULL; 2856 else { 2857 szind_t binind = arena_bin_index(extent_node_arena_get( 2858 &chunk->node), bin); 2859 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 2860 2861 /* 2862 * The following block's conditional is necessary because if the 2863 * run only contains one region, then it never gets inserted 2864 * into the non-full runs tree. 2865 */ 2866 if (bin_info->nregs != 1) { 2867 arena_chunk_map_misc_t *miscelm = 2868 arena_run_to_miscelm(run); 2869 2870 arena_run_heap_remove(&bin->runs, miscelm); 2871 } 2872 } 2873 } 2874 2875 static void 2876 arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2877 arena_run_t *run, arena_bin_t *bin) 2878 { 2879 2880 assert(run != bin->runcur); 2881 2882 malloc_mutex_unlock(tsdn, &bin->lock); 2883 /******************************/ 2884 malloc_mutex_lock(tsdn, &arena->lock); 2885 arena_run_dalloc(tsdn, arena, run, true, false, false); 2886 malloc_mutex_unlock(tsdn, &arena->lock); 2887 /****************************/ 2888 malloc_mutex_lock(tsdn, &bin->lock); 2889 if (config_stats) 2890 bin->stats.curruns--; 2891 } 2892 2893 static void 2894 arena_bin_lower_run(arena_t *arena, arena_run_t *run, arena_bin_t *bin) 2895 { 2896 2897 /* 2898 * Make sure that if bin->runcur is non-NULL, it refers to the 2899 * oldest/lowest non-full run. It is okay to NULL runcur out rather 2900 * than proactively keeping it pointing at the oldest/lowest non-full 2901 * run. 2902 */ 2903 if (bin->runcur != NULL && 2904 arena_snad_comp(arena_run_to_miscelm(bin->runcur), 2905 arena_run_to_miscelm(run)) > 0) { 2906 /* Switch runcur. */ 2907 if (bin->runcur->nfree > 0) 2908 arena_bin_runs_insert(bin, bin->runcur); 2909 bin->runcur = run; 2910 if (config_stats) 2911 bin->stats.reruns++; 2912 } else 2913 arena_bin_runs_insert(bin, run); 2914 } 2915 2916 static void 2917 arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2918 void *ptr, arena_chunk_map_bits_t *bitselm, bool junked) 2919 { 2920 size_t pageind, rpages_ind; 2921 arena_run_t *run; 2922 arena_bin_t *bin; 2923 arena_bin_info_t *bin_info; 2924 szind_t binind; 2925 2926 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2927 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); 2928 run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; 2929 binind = run->binind; 2930 bin = &arena->bins[binind]; 2931 bin_info = &arena_bin_info[binind]; 2932 2933 if (!junked && config_fill && unlikely(opt_junk_free)) 2934 arena_dalloc_junk_small(ptr, bin_info); 2935 2936 arena_run_reg_dalloc(run, ptr); 2937 if (run->nfree == bin_info->nregs) { 2938 arena_dissociate_bin_run(chunk, run, bin); 2939 arena_dalloc_bin_run(tsdn, arena, chunk, run, bin); 2940 } else if (run->nfree == 1 && run != bin->runcur) 2941 arena_bin_lower_run(arena, run, bin); 2942 2943 if (config_stats) { 2944 bin->stats.ndalloc++; 2945 bin->stats.curregs--; 2946 } 2947 } 2948 2949 void 2950 arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, 2951 arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm) 2952 { 2953 2954 arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, true); 2955 } 2956 2957 void 2958 arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void *ptr, 2959 size_t pageind, arena_chunk_map_bits_t *bitselm) 2960 { 2961 arena_run_t *run; 2962 arena_bin_t *bin; 2963 size_t rpages_ind; 2964 2965 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); 2966 run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; 2967 bin = &arena->bins[run->binind]; 2968 malloc_mutex_lock(tsdn, &bin->lock); 2969 arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, false); 2970 malloc_mutex_unlock(tsdn, &bin->lock); 2971 } 2972 2973 void 2974 arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 2975 void *ptr, size_t pageind) 2976 { 2977 arena_chunk_map_bits_t *bitselm; 2978 2979 if (config_debug) { 2980 /* arena_ptr_small_binind_get() does extra sanity checking. */ 2981 assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 2982 pageind)) != BININD_INVALID); 2983 } 2984 bitselm = arena_bitselm_get_mutable(chunk, pageind); 2985 arena_dalloc_bin(tsdn, arena, chunk, ptr, pageind, bitselm); 2986 arena_decay_tick(tsdn, arena); 2987 } 2988 2989 #ifdef JEMALLOC_JET 2990 #undef arena_dalloc_junk_large 2991 #define arena_dalloc_junk_large JEMALLOC_N(n_arena_dalloc_junk_large) 2992 #endif 2993 void 2994 arena_dalloc_junk_large(void *ptr, size_t usize) 2995 { 2996 2997 if (config_fill && unlikely(opt_junk_free)) 2998 memset(ptr, JEMALLOC_FREE_JUNK, usize); 2999 } 3000 #ifdef JEMALLOC_JET 3001 #undef arena_dalloc_junk_large 3002 #define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) 3003 arena_dalloc_junk_large_t *arena_dalloc_junk_large = 3004 JEMALLOC_N(n_arena_dalloc_junk_large); 3005 #endif 3006 3007 static void 3008 arena_dalloc_large_locked_impl(tsdn_t *tsdn, arena_t *arena, 3009 arena_chunk_t *chunk, void *ptr, bool junked) 3010 { 3011 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 3012 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, 3013 pageind); 3014 arena_run_t *run = &miscelm->run; 3015 3016 if (config_fill || config_stats) { 3017 size_t usize = arena_mapbits_large_size_get(chunk, pageind) - 3018 large_pad; 3019 3020 if (!junked) 3021 arena_dalloc_junk_large(ptr, usize); 3022 if (config_stats) { 3023 szind_t index = size2index(usize) - NBINS; 3024 3025 arena->stats.ndalloc_large++; 3026 arena->stats.allocated_large -= usize; 3027 arena->stats.lstats[index].ndalloc++; 3028 arena->stats.lstats[index].curruns--; 3029 } 3030 } 3031 3032 arena_run_dalloc(tsdn, arena, run, true, false, false); 3033 } 3034 3035 void 3036 arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena, 3037 arena_chunk_t *chunk, void *ptr) 3038 { 3039 3040 arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, true); 3041 } 3042 3043 void 3044 arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 3045 void *ptr) 3046 { 3047 3048 malloc_mutex_lock(tsdn, &arena->lock); 3049 arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, false); 3050 malloc_mutex_unlock(tsdn, &arena->lock); 3051 arena_decay_tick(tsdn, arena); 3052 } 3053 3054 static void 3055 arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 3056 void *ptr, size_t oldsize, size_t size) 3057 { 3058 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 3059 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, 3060 pageind); 3061 arena_run_t *run = &miscelm->run; 3062 3063 assert(size < oldsize); 3064 3065 /* 3066 * Shrink the run, and make trailing pages available for other 3067 * allocations. 3068 */ 3069 malloc_mutex_lock(tsdn, &arena->lock); 3070 arena_run_trim_tail(tsdn, arena, chunk, run, oldsize + large_pad, size + 3071 large_pad, true); 3072 if (config_stats) { 3073 szind_t oldindex = size2index(oldsize) - NBINS; 3074 szind_t index = size2index(size) - NBINS; 3075 3076 arena->stats.ndalloc_large++; 3077 arena->stats.allocated_large -= oldsize; 3078 arena->stats.lstats[oldindex].ndalloc++; 3079 arena->stats.lstats[oldindex].curruns--; 3080 3081 arena->stats.nmalloc_large++; 3082 arena->stats.nrequests_large++; 3083 arena->stats.allocated_large += size; 3084 arena->stats.lstats[index].nmalloc++; 3085 arena->stats.lstats[index].nrequests++; 3086 arena->stats.lstats[index].curruns++; 3087 } 3088 malloc_mutex_unlock(tsdn, &arena->lock); 3089 } 3090 3091 static bool 3092 arena_ralloc_large_grow(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 3093 void *ptr, size_t oldsize, size_t usize_min, size_t usize_max, bool zero) 3094 { 3095 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 3096 size_t npages = (oldsize + large_pad) >> LG_PAGE; 3097 size_t followsize; 3098 3099 assert(oldsize == arena_mapbits_large_size_get(chunk, pageind) - 3100 large_pad); 3101 3102 /* Try to extend the run. */ 3103 malloc_mutex_lock(tsdn, &arena->lock); 3104 if (pageind+npages >= chunk_npages || arena_mapbits_allocated_get(chunk, 3105 pageind+npages) != 0) 3106 goto label_fail; 3107 followsize = arena_mapbits_unallocated_size_get(chunk, pageind+npages); 3108 if (oldsize + followsize >= usize_min) { 3109 /* 3110 * The next run is available and sufficiently large. Split the 3111 * following run, then merge the first part with the existing 3112 * allocation. 3113 */ 3114 arena_run_t *run; 3115 size_t usize, splitsize, size, flag_dirty, flag_unzeroed_mask; 3116 3117 usize = usize_max; 3118 while (oldsize + followsize < usize) 3119 usize = index2size(size2index(usize)-1); 3120 assert(usize >= usize_min); 3121 assert(usize >= oldsize); 3122 splitsize = usize - oldsize; 3123 if (splitsize == 0) 3124 goto label_fail; 3125 3126 run = &arena_miscelm_get_mutable(chunk, pageind+npages)->run; 3127 if (arena_run_split_large(arena, run, splitsize, zero)) 3128 goto label_fail; 3129 3130 if (config_cache_oblivious && zero) { 3131 /* 3132 * Zero the trailing bytes of the original allocation's 3133 * last page, since they are in an indeterminate state. 3134 * There will always be trailing bytes, because ptr's 3135 * offset from the beginning of the run is a multiple of 3136 * CACHELINE in [0 .. PAGE). 3137 */ 3138 void *zbase = (void *)((uintptr_t)ptr + oldsize); 3139 void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase + 3140 PAGE)); 3141 size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase; 3142 assert(nzero > 0); 3143 memset(zbase, 0, nzero); 3144 } 3145 3146 size = oldsize + splitsize; 3147 npages = (size + large_pad) >> LG_PAGE; 3148 3149 /* 3150 * Mark the extended run as dirty if either portion of the run 3151 * was dirty before allocation. This is rather pedantic, 3152 * because there's not actually any sequence of events that 3153 * could cause the resulting run to be passed to 3154 * arena_run_dalloc() with the dirty argument set to false 3155 * (which is when dirty flag consistency would really matter). 3156 */ 3157 flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 3158 arena_mapbits_dirty_get(chunk, pageind+npages-1); 3159 flag_unzeroed_mask = flag_dirty == 0 ? CHUNK_MAP_UNZEROED : 0; 3160 arena_mapbits_large_set(chunk, pageind, size + large_pad, 3161 flag_dirty | (flag_unzeroed_mask & 3162 arena_mapbits_unzeroed_get(chunk, pageind))); 3163 arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty | 3164 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 3165 pageind+npages-1))); 3166 3167 if (config_stats) { 3168 szind_t oldindex = size2index(oldsize) - NBINS; 3169 szind_t index = size2index(size) - NBINS; 3170 3171 arena->stats.ndalloc_large++; 3172 arena->stats.allocated_large -= oldsize; 3173 arena->stats.lstats[oldindex].ndalloc++; 3174 arena->stats.lstats[oldindex].curruns--; 3175 3176 arena->stats.nmalloc_large++; 3177 arena->stats.nrequests_large++; 3178 arena->stats.allocated_large += size; 3179 arena->stats.lstats[index].nmalloc++; 3180 arena->stats.lstats[index].nrequests++; 3181 arena->stats.lstats[index].curruns++; 3182 } 3183 malloc_mutex_unlock(tsdn, &arena->lock); 3184 return (false); 3185 } 3186 label_fail: 3187 malloc_mutex_unlock(tsdn, &arena->lock); 3188 return (true); 3189 } 3190 3191 #ifdef JEMALLOC_JET 3192 #undef arena_ralloc_junk_large 3193 #define arena_ralloc_junk_large JEMALLOC_N(n_arena_ralloc_junk_large) 3194 #endif 3195 static void 3196 arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) 3197 { 3198 3199 if (config_fill && unlikely(opt_junk_free)) { 3200 memset((void *)((uintptr_t)ptr + usize), JEMALLOC_FREE_JUNK, 3201 old_usize - usize); 3202 } 3203 } 3204 #ifdef JEMALLOC_JET 3205 #undef arena_ralloc_junk_large 3206 #define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) 3207 arena_ralloc_junk_large_t *arena_ralloc_junk_large = 3208 JEMALLOC_N(n_arena_ralloc_junk_large); 3209 #endif 3210 3211 /* 3212 * Try to resize a large allocation, in order to avoid copying. This will 3213 * always fail if growing an object, and the following run is already in use. 3214 */ 3215 static bool 3216 arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min, 3217 size_t usize_max, bool zero) 3218 { 3219 arena_chunk_t *chunk; 3220 arena_t *arena; 3221 3222 if (oldsize == usize_max) { 3223 /* Current size class is compatible and maximal. */ 3224 return (false); 3225 } 3226 3227 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 3228 arena = extent_node_arena_get(&chunk->node); 3229 3230 if (oldsize < usize_max) { 3231 bool ret = arena_ralloc_large_grow(tsdn, arena, chunk, ptr, 3232 oldsize, usize_min, usize_max, zero); 3233 if (config_fill && !ret && !zero) { 3234 if (unlikely(opt_junk_alloc)) { 3235 memset((void *)((uintptr_t)ptr + oldsize), 3236 JEMALLOC_ALLOC_JUNK, 3237 isalloc(tsdn, ptr, config_prof) - oldsize); 3238 } else if (unlikely(opt_zero)) { 3239 memset((void *)((uintptr_t)ptr + oldsize), 0, 3240 isalloc(tsdn, ptr, config_prof) - oldsize); 3241 } 3242 } 3243 return (ret); 3244 } 3245 3246 assert(oldsize > usize_max); 3247 /* Fill before shrinking in order avoid a race. */ 3248 arena_ralloc_junk_large(ptr, oldsize, usize_max); 3249 arena_ralloc_large_shrink(tsdn, arena, chunk, ptr, oldsize, usize_max); 3250 return (false); 3251 } 3252 3253 bool 3254 arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, 3255 size_t extra, bool zero) 3256 { 3257 size_t usize_min, usize_max; 3258 3259 /* Calls with non-zero extra had to clamp extra. */ 3260 assert(extra == 0 || size + extra <= HUGE_MAXCLASS); 3261 3262 if (unlikely(size > HUGE_MAXCLASS)) 3263 return (true); 3264 3265 usize_min = s2u(size); 3266 usize_max = s2u(size + extra); 3267 if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) { 3268 arena_chunk_t *chunk; 3269 3270 /* 3271 * Avoid moving the allocation if the size class can be left the 3272 * same. 3273 */ 3274 if (oldsize <= SMALL_MAXCLASS) { 3275 assert(arena_bin_info[size2index(oldsize)].reg_size == 3276 oldsize); 3277 if ((usize_max > SMALL_MAXCLASS || 3278 size2index(usize_max) != size2index(oldsize)) && 3279 (size > oldsize || usize_max < oldsize)) 3280 return (true); 3281 } else { 3282 if (usize_max <= SMALL_MAXCLASS) 3283 return (true); 3284 if (arena_ralloc_large(tsdn, ptr, oldsize, usize_min, 3285 usize_max, zero)) 3286 return (true); 3287 } 3288 3289 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 3290 arena_decay_tick(tsdn, extent_node_arena_get(&chunk->node)); 3291 return (false); 3292 } else { 3293 return (huge_ralloc_no_move(tsdn, ptr, oldsize, usize_min, 3294 usize_max, zero)); 3295 } 3296 } 3297 3298 static void * 3299 arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, 3300 size_t alignment, bool zero, tcache_t *tcache) 3301 { 3302 3303 if (alignment == 0) 3304 return (arena_malloc(tsdn, arena, usize, size2index(usize), 3305 zero, tcache, true)); 3306 usize = sa2u(usize, alignment); 3307 if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) 3308 return (NULL); 3309 return (ipalloct(tsdn, usize, alignment, zero, tcache, arena)); 3310 } 3311 3312 void * 3313 arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, 3314 size_t alignment, bool zero, tcache_t *tcache) 3315 { 3316 void *ret; 3317 size_t usize; 3318 3319 usize = s2u(size); 3320 if (unlikely(usize == 0 || size > HUGE_MAXCLASS)) 3321 return (NULL); 3322 3323 if (likely(usize <= large_maxclass)) { 3324 size_t copysize; 3325 3326 /* Try to avoid moving the allocation. */ 3327 if (!arena_ralloc_no_move(tsd_tsdn(tsd), ptr, oldsize, usize, 0, 3328 zero)) 3329 return (ptr); 3330 3331 /* 3332 * size and oldsize are different enough that we need to move 3333 * the object. In that case, fall back to allocating new space 3334 * and copying. 3335 */ 3336 ret = arena_ralloc_move_helper(tsd_tsdn(tsd), arena, usize, 3337 alignment, zero, tcache); 3338 if (ret == NULL) 3339 return (NULL); 3340 3341 /* 3342 * Junk/zero-filling were already done by 3343 * ipalloc()/arena_malloc(). 3344 */ 3345 3346 copysize = (usize < oldsize) ? usize : oldsize; 3347 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 3348 memcpy(ret, ptr, copysize); 3349 isqalloc(tsd, ptr, oldsize, tcache, true); 3350 } else { 3351 ret = huge_ralloc(tsd, arena, ptr, oldsize, usize, alignment, 3352 zero, tcache); 3353 } 3354 return (ret); 3355 } 3356 3357 dss_prec_t 3358 arena_dss_prec_get(tsdn_t *tsdn, arena_t *arena) 3359 { 3360 dss_prec_t ret; 3361 3362 malloc_mutex_lock(tsdn, &arena->lock); 3363 ret = arena->dss_prec; 3364 malloc_mutex_unlock(tsdn, &arena->lock); 3365 return (ret); 3366 } 3367 3368 bool 3369 arena_dss_prec_set(tsdn_t *tsdn, arena_t *arena, dss_prec_t dss_prec) 3370 { 3371 3372 if (!have_dss) 3373 return (dss_prec != dss_prec_disabled); 3374 malloc_mutex_lock(tsdn, &arena->lock); 3375 arena->dss_prec = dss_prec; 3376 malloc_mutex_unlock(tsdn, &arena->lock); 3377 return (false); 3378 } 3379 3380 ssize_t 3381 arena_lg_dirty_mult_default_get(void) 3382 { 3383 3384 return ((ssize_t)atomic_read_z((size_t *)&lg_dirty_mult_default)); 3385 } 3386 3387 bool 3388 arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult) 3389 { 3390 3391 if (opt_purge != purge_mode_ratio) 3392 return (true); 3393 if (!arena_lg_dirty_mult_valid(lg_dirty_mult)) 3394 return (true); 3395 atomic_write_z((size_t *)&lg_dirty_mult_default, (size_t)lg_dirty_mult); 3396 return (false); 3397 } 3398 3399 ssize_t 3400 arena_decay_time_default_get(void) 3401 { 3402 3403 return ((ssize_t)atomic_read_z((size_t *)&decay_time_default)); 3404 } 3405 3406 bool 3407 arena_decay_time_default_set(ssize_t decay_time) 3408 { 3409 3410 if (opt_purge != purge_mode_decay) 3411 return (true); 3412 if (!arena_decay_time_valid(decay_time)) 3413 return (true); 3414 atomic_write_z((size_t *)&decay_time_default, (size_t)decay_time); 3415 return (false); 3416 } 3417 3418 static void 3419 arena_basic_stats_merge_locked(arena_t *arena, unsigned *nthreads, 3420 const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, 3421 size_t *nactive, size_t *ndirty) 3422 { 3423 3424 *nthreads += arena_nthreads_get(arena, false); 3425 *dss = dss_prec_names[arena->dss_prec]; 3426 *lg_dirty_mult = arena->lg_dirty_mult; 3427 *decay_time = arena->decay.time; 3428 *nactive += arena->nactive; 3429 *ndirty += arena->ndirty; 3430 } 3431 3432 void 3433 arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, 3434 const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, 3435 size_t *nactive, size_t *ndirty) 3436 { 3437 3438 malloc_mutex_lock(tsdn, &arena->lock); 3439 arena_basic_stats_merge_locked(arena, nthreads, dss, lg_dirty_mult, 3440 decay_time, nactive, ndirty); 3441 malloc_mutex_unlock(tsdn, &arena->lock); 3442 } 3443 3444 void 3445 arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, 3446 const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, 3447 size_t *nactive, size_t *ndirty, arena_stats_t *astats, 3448 malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats, 3449 malloc_huge_stats_t *hstats) 3450 { 3451 unsigned i; 3452 3453 cassert(config_stats); 3454 3455 malloc_mutex_lock(tsdn, &arena->lock); 3456 arena_basic_stats_merge_locked(arena, nthreads, dss, lg_dirty_mult, 3457 decay_time, nactive, ndirty); 3458 3459 astats->mapped += arena->stats.mapped; 3460 astats->retained += arena->stats.retained; 3461 astats->npurge += arena->stats.npurge; 3462 astats->nmadvise += arena->stats.nmadvise; 3463 astats->purged += arena->stats.purged; 3464 astats->metadata_mapped += arena->stats.metadata_mapped; 3465 astats->metadata_allocated += arena_metadata_allocated_get(arena); 3466 astats->allocated_large += arena->stats.allocated_large; 3467 astats->nmalloc_large += arena->stats.nmalloc_large; 3468 astats->ndalloc_large += arena->stats.ndalloc_large; 3469 astats->nrequests_large += arena->stats.nrequests_large; 3470 astats->allocated_huge += arena->stats.allocated_huge; 3471 astats->nmalloc_huge += arena->stats.nmalloc_huge; 3472 astats->ndalloc_huge += arena->stats.ndalloc_huge; 3473 3474 for (i = 0; i < nlclasses; i++) { 3475 lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 3476 lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 3477 lstats[i].nrequests += arena->stats.lstats[i].nrequests; 3478 lstats[i].curruns += arena->stats.lstats[i].curruns; 3479 } 3480 3481 for (i = 0; i < nhclasses; i++) { 3482 hstats[i].nmalloc += arena->stats.hstats[i].nmalloc; 3483 hstats[i].ndalloc += arena->stats.hstats[i].ndalloc; 3484 hstats[i].curhchunks += arena->stats.hstats[i].curhchunks; 3485 } 3486 malloc_mutex_unlock(tsdn, &arena->lock); 3487 3488 for (i = 0; i < NBINS; i++) { 3489 arena_bin_t *bin = &arena->bins[i]; 3490 3491 malloc_mutex_lock(tsdn, &bin->lock); 3492 bstats[i].nmalloc += bin->stats.nmalloc; 3493 bstats[i].ndalloc += bin->stats.ndalloc; 3494 bstats[i].nrequests += bin->stats.nrequests; 3495 bstats[i].curregs += bin->stats.curregs; 3496 if (config_tcache) { 3497 bstats[i].nfills += bin->stats.nfills; 3498 bstats[i].nflushes += bin->stats.nflushes; 3499 } 3500 bstats[i].nruns += bin->stats.nruns; 3501 bstats[i].reruns += bin->stats.reruns; 3502 bstats[i].curruns += bin->stats.curruns; 3503 malloc_mutex_unlock(tsdn, &bin->lock); 3504 } 3505 } 3506 3507 unsigned 3508 arena_nthreads_get(arena_t *arena, bool internal) 3509 { 3510 3511 return (atomic_read_u(&arena->nthreads[internal])); 3512 } 3513 3514 void 3515 arena_nthreads_inc(arena_t *arena, bool internal) 3516 { 3517 3518 atomic_add_u(&arena->nthreads[internal], 1); 3519 } 3520 3521 void 3522 arena_nthreads_dec(arena_t *arena, bool internal) 3523 { 3524 3525 atomic_sub_u(&arena->nthreads[internal], 1); 3526 } 3527 3528 size_t 3529 arena_extent_sn_next(arena_t *arena) 3530 { 3531 3532 return (atomic_add_z(&arena->extent_sn_next, 1) - 1); 3533 } 3534 3535 arena_t * 3536 arena_new(tsdn_t *tsdn, unsigned ind) 3537 { 3538 arena_t *arena; 3539 unsigned i; 3540 3541 /* 3542 * Allocate arena, arena->lstats, and arena->hstats contiguously, mainly 3543 * because there is no way to clean up if base_alloc() OOMs. 3544 */ 3545 if (config_stats) { 3546 arena = (arena_t *)base_alloc(tsdn, 3547 CACHELINE_CEILING(sizeof(arena_t)) + 3548 QUANTUM_CEILING((nlclasses * sizeof(malloc_large_stats_t))) 3549 + (nhclasses * sizeof(malloc_huge_stats_t))); 3550 } else 3551 arena = (arena_t *)base_alloc(tsdn, sizeof(arena_t)); 3552 if (arena == NULL) 3553 return (NULL); 3554 3555 arena->ind = ind; 3556 arena->nthreads[0] = arena->nthreads[1] = 0; 3557 if (malloc_mutex_init(&arena->lock, "arena", WITNESS_RANK_ARENA)) 3558 return (NULL); 3559 3560 if (config_stats) { 3561 memset(&arena->stats, 0, sizeof(arena_stats_t)); 3562 arena->stats.lstats = (malloc_large_stats_t *)((uintptr_t)arena 3563 + CACHELINE_CEILING(sizeof(arena_t))); 3564 memset(arena->stats.lstats, 0, nlclasses * 3565 sizeof(malloc_large_stats_t)); 3566 arena->stats.hstats = (malloc_huge_stats_t *)((uintptr_t)arena 3567 + CACHELINE_CEILING(sizeof(arena_t)) + 3568 QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t))); 3569 memset(arena->stats.hstats, 0, nhclasses * 3570 sizeof(malloc_huge_stats_t)); 3571 if (config_tcache) 3572 ql_new(&arena->tcache_ql); 3573 } 3574 3575 if (config_prof) 3576 arena->prof_accumbytes = 0; 3577 3578 if (config_cache_oblivious) { 3579 /* 3580 * A nondeterministic seed based on the address of arena reduces 3581 * the likelihood of lockstep non-uniform cache index 3582 * utilization among identical concurrent processes, but at the 3583 * cost of test repeatability. For debug builds, instead use a 3584 * deterministic seed. 3585 */ 3586 arena->offset_state = config_debug ? ind : 3587 (size_t)(uintptr_t)arena; 3588 } 3589 3590 arena->dss_prec = chunk_dss_prec_get(); 3591 3592 ql_new(&arena->achunks); 3593 3594 arena->extent_sn_next = 0; 3595 3596 arena->spare = NULL; 3597 3598 arena->lg_dirty_mult = arena_lg_dirty_mult_default_get(); 3599 arena->purging = false; 3600 arena->nactive = 0; 3601 arena->ndirty = 0; 3602 3603 for (i = 0; i < NPSIZES; i++) 3604 arena_run_heap_new(&arena->runs_avail[i]); 3605 3606 qr_new(&arena->runs_dirty, rd_link); 3607 qr_new(&arena->chunks_cache, cc_link); 3608 3609 if (opt_purge == purge_mode_decay) 3610 arena_decay_init(arena, arena_decay_time_default_get()); 3611 3612 ql_new(&arena->huge); 3613 if (malloc_mutex_init(&arena->huge_mtx, "arena_huge", 3614 WITNESS_RANK_ARENA_HUGE)) 3615 return (NULL); 3616 3617 extent_tree_szsnad_new(&arena->chunks_szsnad_cached); 3618 extent_tree_ad_new(&arena->chunks_ad_cached); 3619 extent_tree_szsnad_new(&arena->chunks_szsnad_retained); 3620 extent_tree_ad_new(&arena->chunks_ad_retained); 3621 if (malloc_mutex_init(&arena->chunks_mtx, "arena_chunks", 3622 WITNESS_RANK_ARENA_CHUNKS)) 3623 return (NULL); 3624 ql_new(&arena->node_cache); 3625 if (malloc_mutex_init(&arena->node_cache_mtx, "arena_node_cache", 3626 WITNESS_RANK_ARENA_NODE_CACHE)) 3627 return (NULL); 3628 3629 arena->chunk_hooks = chunk_hooks_default; 3630 3631 /* Initialize bins. */ 3632 for (i = 0; i < NBINS; i++) { 3633 arena_bin_t *bin = &arena->bins[i]; 3634 if (malloc_mutex_init(&bin->lock, "arena_bin", 3635 WITNESS_RANK_ARENA_BIN)) 3636 return (NULL); 3637 bin->runcur = NULL; 3638 arena_run_heap_new(&bin->runs); 3639 if (config_stats) 3640 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 3641 } 3642 3643 return (arena); 3644 } 3645 3646 /* 3647 * Calculate bin_info->run_size such that it meets the following constraints: 3648 * 3649 * *) bin_info->run_size <= arena_maxrun 3650 * *) bin_info->nregs <= RUN_MAXREGS 3651 * 3652 * bin_info->nregs and bin_info->reg0_offset are also calculated here, since 3653 * these settings are all interdependent. 3654 */ 3655 static void 3656 bin_info_run_size_calc(arena_bin_info_t *bin_info) 3657 { 3658 size_t pad_size; 3659 size_t try_run_size, perfect_run_size, actual_run_size; 3660 uint32_t try_nregs, perfect_nregs, actual_nregs; 3661 3662 /* 3663 * Determine redzone size based on minimum alignment and minimum 3664 * redzone size. Add padding to the end of the run if it is needed to 3665 * align the regions. The padding allows each redzone to be half the 3666 * minimum alignment; without the padding, each redzone would have to 3667 * be twice as large in order to maintain alignment. 3668 */ 3669 if (config_fill && unlikely(opt_redzone)) { 3670 size_t align_min = ZU(1) << (ffs_zu(bin_info->reg_size) - 1); 3671 if (align_min <= REDZONE_MINSIZE) { 3672 bin_info->redzone_size = REDZONE_MINSIZE; 3673 pad_size = 0; 3674 } else { 3675 bin_info->redzone_size = align_min >> 1; 3676 pad_size = bin_info->redzone_size; 3677 } 3678 } else { 3679 bin_info->redzone_size = 0; 3680 pad_size = 0; 3681 } 3682 bin_info->reg_interval = bin_info->reg_size + 3683 (bin_info->redzone_size << 1); 3684 3685 /* 3686 * Compute run size under ideal conditions (no redzones, no limit on run 3687 * size). 3688 */ 3689 try_run_size = PAGE; 3690 try_nregs = (uint32_t)(try_run_size / bin_info->reg_size); 3691 do { 3692 perfect_run_size = try_run_size; 3693 perfect_nregs = try_nregs; 3694 3695 try_run_size += PAGE; 3696 try_nregs = (uint32_t)(try_run_size / bin_info->reg_size); 3697 } while (perfect_run_size != perfect_nregs * bin_info->reg_size); 3698 assert(perfect_nregs <= RUN_MAXREGS); 3699 3700 actual_run_size = perfect_run_size; 3701 actual_nregs = (uint32_t)((actual_run_size - pad_size) / 3702 bin_info->reg_interval); 3703 3704 /* 3705 * Redzones can require enough padding that not even a single region can 3706 * fit within the number of pages that would normally be dedicated to a 3707 * run for this size class. Increase the run size until at least one 3708 * region fits. 3709 */ 3710 while (actual_nregs == 0) { 3711 assert(config_fill && unlikely(opt_redzone)); 3712 3713 actual_run_size += PAGE; 3714 actual_nregs = (uint32_t)((actual_run_size - pad_size) / 3715 bin_info->reg_interval); 3716 } 3717 3718 /* 3719 * Make sure that the run will fit within an arena chunk. 3720 */ 3721 while (actual_run_size > arena_maxrun) { 3722 actual_run_size -= PAGE; 3723 actual_nregs = (uint32_t)((actual_run_size - pad_size) / 3724 bin_info->reg_interval); 3725 } 3726 assert(actual_nregs > 0); 3727 assert(actual_run_size == s2u(actual_run_size)); 3728 3729 /* Copy final settings. */ 3730 bin_info->run_size = actual_run_size; 3731 bin_info->nregs = actual_nregs; 3732 bin_info->reg0_offset = (uint32_t)(actual_run_size - (actual_nregs * 3733 bin_info->reg_interval) - pad_size + bin_info->redzone_size); 3734 3735 assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 3736 * bin_info->reg_interval) + pad_size == bin_info->run_size); 3737 } 3738 3739 static void 3740 bin_info_init(void) 3741 { 3742 arena_bin_info_t *bin_info; 3743 3744 #define BIN_INFO_INIT_bin_yes(index, size) \ 3745 bin_info = &arena_bin_info[index]; \ 3746 bin_info->reg_size = size; \ 3747 bin_info_run_size_calc(bin_info); \ 3748 bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 3749 #define BIN_INFO_INIT_bin_no(index, size) 3750 #define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \ 3751 BIN_INFO_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)) 3752 SIZE_CLASSES 3753 #undef BIN_INFO_INIT_bin_yes 3754 #undef BIN_INFO_INIT_bin_no 3755 #undef SC 3756 } 3757 3758 void 3759 arena_boot(void) 3760 { 3761 unsigned i; 3762 3763 arena_lg_dirty_mult_default_set(opt_lg_dirty_mult); 3764 arena_decay_time_default_set(opt_decay_time); 3765 3766 /* 3767 * Compute the header size such that it is large enough to contain the 3768 * page map. The page map is biased to omit entries for the header 3769 * itself, so some iteration is necessary to compute the map bias. 3770 * 3771 * 1) Compute safe header_size and map_bias values that include enough 3772 * space for an unbiased page map. 3773 * 2) Refine map_bias based on (1) to omit the header pages in the page 3774 * map. The resulting map_bias may be one too small. 3775 * 3) Refine map_bias based on (2). The result will be >= the result 3776 * from (2), and will always be correct. 3777 */ 3778 map_bias = 0; 3779 for (i = 0; i < 3; i++) { 3780 size_t header_size = offsetof(arena_chunk_t, map_bits) + 3781 ((sizeof(arena_chunk_map_bits_t) + 3782 sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias)); 3783 map_bias = (header_size + PAGE_MASK) >> LG_PAGE; 3784 } 3785 assert(map_bias > 0); 3786 3787 map_misc_offset = offsetof(arena_chunk_t, map_bits) + 3788 sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias); 3789 3790 arena_maxrun = chunksize - (map_bias << LG_PAGE); 3791 assert(arena_maxrun > 0); 3792 large_maxclass = index2size(size2index(chunksize)-1); 3793 if (large_maxclass > arena_maxrun) { 3794 /* 3795 * For small chunk sizes it's possible for there to be fewer 3796 * non-header pages available than are necessary to serve the 3797 * size classes just below chunksize. 3798 */ 3799 large_maxclass = arena_maxrun; 3800 } 3801 assert(large_maxclass > 0); 3802 nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS); 3803 nhclasses = NSIZES - nlclasses - NBINS; 3804 3805 bin_info_init(); 3806 } 3807 3808 void 3809 arena_prefork0(tsdn_t *tsdn, arena_t *arena) 3810 { 3811 3812 malloc_mutex_prefork(tsdn, &arena->lock); 3813 } 3814 3815 void 3816 arena_prefork1(tsdn_t *tsdn, arena_t *arena) 3817 { 3818 3819 malloc_mutex_prefork(tsdn, &arena->chunks_mtx); 3820 } 3821 3822 void 3823 arena_prefork2(tsdn_t *tsdn, arena_t *arena) 3824 { 3825 3826 malloc_mutex_prefork(tsdn, &arena->node_cache_mtx); 3827 } 3828 3829 void 3830 arena_prefork3(tsdn_t *tsdn, arena_t *arena) 3831 { 3832 unsigned i; 3833 3834 for (i = 0; i < NBINS; i++) 3835 malloc_mutex_prefork(tsdn, &arena->bins[i].lock); 3836 malloc_mutex_prefork(tsdn, &arena->huge_mtx); 3837 } 3838 3839 void 3840 arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) 3841 { 3842 unsigned i; 3843 3844 malloc_mutex_postfork_parent(tsdn, &arena->huge_mtx); 3845 for (i = 0; i < NBINS; i++) 3846 malloc_mutex_postfork_parent(tsdn, &arena->bins[i].lock); 3847 malloc_mutex_postfork_parent(tsdn, &arena->node_cache_mtx); 3848 malloc_mutex_postfork_parent(tsdn, &arena->chunks_mtx); 3849 malloc_mutex_postfork_parent(tsdn, &arena->lock); 3850 } 3851 3852 void 3853 arena_postfork_child(tsdn_t *tsdn, arena_t *arena) 3854 { 3855 unsigned i; 3856 3857 malloc_mutex_postfork_child(tsdn, &arena->huge_mtx); 3858 for (i = 0; i < NBINS; i++) 3859 malloc_mutex_postfork_child(tsdn, &arena->bins[i].lock); 3860 malloc_mutex_postfork_child(tsdn, &arena->node_cache_mtx); 3861 malloc_mutex_postfork_child(tsdn, &arena->chunks_mtx); 3862 malloc_mutex_postfork_child(tsdn, &arena->lock); 3863 } 3864