1 #define JEMALLOC_ARENA_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8 static ssize_t lg_dirty_mult_default; 9 arena_bin_info_t arena_bin_info[NBINS]; 10 11 size_t map_bias; 12 size_t map_misc_offset; 13 size_t arena_maxrun; /* Max run size for arenas. */ 14 size_t large_maxclass; /* Max large size class. */ 15 static size_t small_maxrun; /* Max run size used for small size classes. */ 16 static bool *small_run_tab; /* Valid small run page multiples. */ 17 unsigned nlclasses; /* Number of large size classes. */ 18 unsigned nhclasses; /* Number of huge size classes. */ 19 20 /******************************************************************************/ 21 /* 22 * Function prototypes for static functions that are referenced prior to 23 * definition. 24 */ 25 26 static void arena_purge(arena_t *arena, bool all); 27 static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, 28 bool cleaned, bool decommitted); 29 static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 30 arena_run_t *run, arena_bin_t *bin); 31 static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 32 arena_run_t *run, arena_bin_t *bin); 33 34 /******************************************************************************/ 35 36 #define CHUNK_MAP_KEY ((uintptr_t)0x1U) 37 38 JEMALLOC_INLINE_C arena_chunk_map_misc_t * 39 arena_miscelm_key_create(size_t size) 40 { 41 42 return ((arena_chunk_map_misc_t *)(arena_mapbits_size_encode(size) | 43 CHUNK_MAP_KEY)); 44 } 45 46 JEMALLOC_INLINE_C bool 47 arena_miscelm_is_key(const arena_chunk_map_misc_t *miscelm) 48 { 49 50 return (((uintptr_t)miscelm & CHUNK_MAP_KEY) != 0); 51 } 52 53 #undef CHUNK_MAP_KEY 54 55 JEMALLOC_INLINE_C size_t 56 arena_miscelm_key_size_get(const arena_chunk_map_misc_t *miscelm) 57 { 58 59 assert(arena_miscelm_is_key(miscelm)); 60 61 return (arena_mapbits_size_decode((uintptr_t)miscelm)); 62 } 63 64 JEMALLOC_INLINE_C size_t 65 arena_miscelm_size_get(arena_chunk_map_misc_t *miscelm) 66 { 67 arena_chunk_t *chunk; 68 size_t pageind, mapbits; 69 70 assert(!arena_miscelm_is_key(miscelm)); 71 72 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 73 pageind = arena_miscelm_to_pageind(miscelm); 74 mapbits = arena_mapbits_get(chunk, pageind); 75 return (arena_mapbits_size_decode(mapbits)); 76 } 77 78 JEMALLOC_INLINE_C int 79 arena_run_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b) 80 { 81 uintptr_t a_miscelm = (uintptr_t)a; 82 uintptr_t b_miscelm = (uintptr_t)b; 83 84 assert(a != NULL); 85 assert(b != NULL); 86 87 return ((a_miscelm > b_miscelm) - (a_miscelm < b_miscelm)); 88 } 89 90 /* Generate red-black tree functions. */ 91 rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_misc_t, 92 rb_link, arena_run_comp) 93 94 static size_t 95 run_quantize(size_t size) 96 { 97 size_t qsize; 98 99 assert(size != 0); 100 assert(size == PAGE_CEILING(size)); 101 102 /* Don't change sizes that are valid small run sizes. */ 103 if (size <= small_maxrun && small_run_tab[size >> LG_PAGE]) 104 return (size); 105 106 /* 107 * Round down to the nearest run size that can actually be requested 108 * during normal large allocation. Add large_pad so that cache index 109 * randomization can offset the allocation from the page boundary. 110 */ 111 qsize = index2size(size2index(size - large_pad + 1) - 1) + large_pad; 112 if (qsize <= SMALL_MAXCLASS + large_pad) 113 return (run_quantize(size - large_pad)); 114 assert(qsize <= size); 115 return (qsize); 116 } 117 118 static size_t 119 run_quantize_next(size_t size) 120 { 121 size_t large_run_size_next; 122 123 assert(size != 0); 124 assert(size == PAGE_CEILING(size)); 125 126 /* 127 * Return the next quantized size greater than the input size. 128 * Quantized sizes comprise the union of run sizes that back small 129 * region runs, and run sizes that back large regions with no explicit 130 * alignment constraints. 131 */ 132 133 if (size > SMALL_MAXCLASS) { 134 large_run_size_next = PAGE_CEILING(index2size(size2index(size - 135 large_pad) + 1) + large_pad); 136 } else 137 large_run_size_next = SIZE_T_MAX; 138 if (size >= small_maxrun) 139 return (large_run_size_next); 140 141 while (true) { 142 size += PAGE; 143 assert(size <= small_maxrun); 144 if (small_run_tab[size >> LG_PAGE]) { 145 if (large_run_size_next < size) 146 return (large_run_size_next); 147 return (size); 148 } 149 } 150 } 151 152 static size_t 153 run_quantize_first(size_t size) 154 { 155 size_t qsize = run_quantize(size); 156 157 if (qsize < size) { 158 /* 159 * Skip a quantization that may have an adequately large run, 160 * because under-sized runs may be mixed in. This only happens 161 * when an unusual size is requested, i.e. for aligned 162 * allocation, and is just one of several places where linear 163 * search would potentially find sufficiently aligned available 164 * memory somewhere lower. 165 */ 166 qsize = run_quantize_next(size); 167 } 168 return (qsize); 169 } 170 171 JEMALLOC_INLINE_C int 172 arena_avail_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b) 173 { 174 int ret; 175 uintptr_t a_miscelm = (uintptr_t)a; 176 size_t a_qsize = run_quantize(arena_miscelm_is_key(a) ? 177 arena_miscelm_key_size_get(a) : arena_miscelm_size_get(a)); 178 size_t b_qsize = run_quantize(arena_miscelm_size_get(b)); 179 180 /* 181 * Compare based on quantized size rather than size, in order to sort 182 * equally useful runs only by address. 183 */ 184 ret = (a_qsize > b_qsize) - (a_qsize < b_qsize); 185 if (ret == 0) { 186 if (!arena_miscelm_is_key(a)) { 187 uintptr_t b_miscelm = (uintptr_t)b; 188 189 ret = (a_miscelm > b_miscelm) - (a_miscelm < b_miscelm); 190 } else { 191 /* 192 * Treat keys as if they are lower than anything else. 193 */ 194 ret = -1; 195 } 196 } 197 198 return (ret); 199 } 200 201 /* Generate red-black tree functions. */ 202 rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, 203 arena_chunk_map_misc_t, rb_link, arena_avail_comp) 204 205 static void 206 arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 207 size_t npages) 208 { 209 210 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 211 LG_PAGE)); 212 arena_avail_tree_insert(&arena->runs_avail, arena_miscelm_get(chunk, 213 pageind)); 214 } 215 216 static void 217 arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 218 size_t npages) 219 { 220 221 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 222 LG_PAGE)); 223 arena_avail_tree_remove(&arena->runs_avail, arena_miscelm_get(chunk, 224 pageind)); 225 } 226 227 static void 228 arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 229 size_t npages) 230 { 231 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); 232 233 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 234 LG_PAGE)); 235 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); 236 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == 237 CHUNK_MAP_DIRTY); 238 239 qr_new(&miscelm->rd, rd_link); 240 qr_meld(&arena->runs_dirty, &miscelm->rd, rd_link); 241 arena->ndirty += npages; 242 } 243 244 static void 245 arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 246 size_t npages) 247 { 248 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); 249 250 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 251 LG_PAGE)); 252 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); 253 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == 254 CHUNK_MAP_DIRTY); 255 256 qr_remove(&miscelm->rd, rd_link); 257 assert(arena->ndirty >= npages); 258 arena->ndirty -= npages; 259 } 260 261 static size_t 262 arena_chunk_dirty_npages(const extent_node_t *node) 263 { 264 265 return (extent_node_size_get(node) >> LG_PAGE); 266 } 267 268 void 269 arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, bool cache) 270 { 271 272 if (cache) { 273 extent_node_dirty_linkage_init(node); 274 extent_node_dirty_insert(node, &arena->runs_dirty, 275 &arena->chunks_cache); 276 arena->ndirty += arena_chunk_dirty_npages(node); 277 } 278 } 279 280 void 281 arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, bool dirty) 282 { 283 284 if (dirty) { 285 extent_node_dirty_remove(node); 286 assert(arena->ndirty >= arena_chunk_dirty_npages(node)); 287 arena->ndirty -= arena_chunk_dirty_npages(node); 288 } 289 } 290 291 JEMALLOC_INLINE_C void * 292 arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 293 { 294 void *ret; 295 unsigned regind; 296 arena_chunk_map_misc_t *miscelm; 297 void *rpages; 298 299 assert(run->nfree > 0); 300 assert(!bitmap_full(run->bitmap, &bin_info->bitmap_info)); 301 302 regind = bitmap_sfu(run->bitmap, &bin_info->bitmap_info); 303 miscelm = arena_run_to_miscelm(run); 304 rpages = arena_miscelm_to_rpages(miscelm); 305 ret = (void *)((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset + 306 (uintptr_t)(bin_info->reg_interval * regind)); 307 run->nfree--; 308 return (ret); 309 } 310 311 JEMALLOC_INLINE_C void 312 arena_run_reg_dalloc(arena_run_t *run, void *ptr) 313 { 314 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 315 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 316 size_t mapbits = arena_mapbits_get(chunk, pageind); 317 szind_t binind = arena_ptr_small_binind_get(ptr, mapbits); 318 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 319 unsigned regind = arena_run_regind(run, bin_info, ptr); 320 321 assert(run->nfree < bin_info->nregs); 322 /* Freeing an interior pointer can cause assertion failure. */ 323 assert(((uintptr_t)ptr - 324 ((uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + 325 (uintptr_t)bin_info->reg0_offset)) % 326 (uintptr_t)bin_info->reg_interval == 0); 327 assert((uintptr_t)ptr >= 328 (uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + 329 (uintptr_t)bin_info->reg0_offset); 330 /* Freeing an unallocated pointer can cause assertion failure. */ 331 assert(bitmap_get(run->bitmap, &bin_info->bitmap_info, regind)); 332 333 bitmap_unset(run->bitmap, &bin_info->bitmap_info, regind); 334 run->nfree++; 335 } 336 337 JEMALLOC_INLINE_C void 338 arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) 339 { 340 341 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 342 (run_ind << LG_PAGE)), (npages << LG_PAGE)); 343 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, 344 (npages << LG_PAGE)); 345 } 346 347 JEMALLOC_INLINE_C void 348 arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) 349 { 350 351 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind 352 << LG_PAGE)), PAGE); 353 } 354 355 JEMALLOC_INLINE_C void 356 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 357 { 358 size_t i; 359 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 360 361 arena_run_page_mark_zeroed(chunk, run_ind); 362 for (i = 0; i < PAGE / sizeof(size_t); i++) 363 assert(p[i] == 0); 364 } 365 366 static void 367 arena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages) 368 { 369 370 if (config_stats) { 371 ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + add_pages 372 - sub_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << 373 LG_PAGE); 374 if (cactive_diff != 0) 375 stats_cactive_add(cactive_diff); 376 } 377 } 378 379 static void 380 arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, 381 size_t flag_dirty, size_t flag_decommitted, size_t need_pages) 382 { 383 size_t total_pages, rem_pages; 384 385 assert(flag_dirty == 0 || flag_decommitted == 0); 386 387 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 388 LG_PAGE; 389 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 390 flag_dirty); 391 assert(need_pages <= total_pages); 392 rem_pages = total_pages - need_pages; 393 394 arena_avail_remove(arena, chunk, run_ind, total_pages); 395 if (flag_dirty != 0) 396 arena_run_dirty_remove(arena, chunk, run_ind, total_pages); 397 arena_cactive_update(arena, need_pages, 0); 398 arena->nactive += need_pages; 399 400 /* Keep track of trailing unused pages for later use. */ 401 if (rem_pages > 0) { 402 size_t flags = flag_dirty | flag_decommitted; 403 size_t flag_unzeroed_mask = (flags == 0) ? CHUNK_MAP_UNZEROED : 404 0; 405 406 arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 407 (rem_pages << LG_PAGE), flags | 408 (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) & 409 flag_unzeroed_mask)); 410 arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, 411 (rem_pages << LG_PAGE), flags | 412 (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) & 413 flag_unzeroed_mask)); 414 if (flag_dirty != 0) { 415 arena_run_dirty_insert(arena, chunk, run_ind+need_pages, 416 rem_pages); 417 } 418 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages); 419 } 420 } 421 422 static bool 423 arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, 424 bool remove, bool zero) 425 { 426 arena_chunk_t *chunk; 427 arena_chunk_map_misc_t *miscelm; 428 size_t flag_dirty, flag_decommitted, run_ind, need_pages; 429 size_t flag_unzeroed_mask; 430 431 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 432 miscelm = arena_run_to_miscelm(run); 433 run_ind = arena_miscelm_to_pageind(miscelm); 434 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 435 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); 436 need_pages = (size >> LG_PAGE); 437 assert(need_pages > 0); 438 439 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, 440 run_ind << LG_PAGE, size, arena->ind)) 441 return (true); 442 443 if (remove) { 444 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 445 flag_decommitted, need_pages); 446 } 447 448 if (zero) { 449 if (flag_decommitted != 0) { 450 /* The run is untouched, and therefore zeroed. */ 451 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void 452 *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 453 (need_pages << LG_PAGE)); 454 } else if (flag_dirty != 0) { 455 /* The run is dirty, so all pages must be zeroed. */ 456 arena_run_zero(chunk, run_ind, need_pages); 457 } else { 458 /* 459 * The run is clean, so some pages may be zeroed (i.e. 460 * never before touched). 461 */ 462 size_t i; 463 for (i = 0; i < need_pages; i++) { 464 if (arena_mapbits_unzeroed_get(chunk, run_ind+i) 465 != 0) 466 arena_run_zero(chunk, run_ind+i, 1); 467 else if (config_debug) { 468 arena_run_page_validate_zeroed(chunk, 469 run_ind+i); 470 } else { 471 arena_run_page_mark_zeroed(chunk, 472 run_ind+i); 473 } 474 } 475 } 476 } else { 477 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 478 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 479 } 480 481 /* 482 * Set the last element first, in case the run only contains one page 483 * (i.e. both statements set the same element). 484 */ 485 flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? 486 CHUNK_MAP_UNZEROED : 0; 487 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty | 488 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 489 run_ind+need_pages-1))); 490 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty | 491 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind))); 492 return (false); 493 } 494 495 static bool 496 arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 497 { 498 499 return (arena_run_split_large_helper(arena, run, size, true, zero)); 500 } 501 502 static bool 503 arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 504 { 505 506 return (arena_run_split_large_helper(arena, run, size, false, zero)); 507 } 508 509 static bool 510 arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, 511 szind_t binind) 512 { 513 arena_chunk_t *chunk; 514 arena_chunk_map_misc_t *miscelm; 515 size_t flag_dirty, flag_decommitted, run_ind, need_pages, i; 516 517 assert(binind != BININD_INVALID); 518 519 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 520 miscelm = arena_run_to_miscelm(run); 521 run_ind = arena_miscelm_to_pageind(miscelm); 522 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 523 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); 524 need_pages = (size >> LG_PAGE); 525 assert(need_pages > 0); 526 527 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, 528 run_ind << LG_PAGE, size, arena->ind)) 529 return (true); 530 531 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 532 flag_decommitted, need_pages); 533 534 for (i = 0; i < need_pages; i++) { 535 size_t flag_unzeroed = arena_mapbits_unzeroed_get(chunk, 536 run_ind+i); 537 arena_mapbits_small_set(chunk, run_ind+i, i, binind, 538 flag_unzeroed); 539 if (config_debug && flag_dirty == 0 && flag_unzeroed == 0) 540 arena_run_page_validate_zeroed(chunk, run_ind+i); 541 } 542 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 543 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 544 return (false); 545 } 546 547 static arena_chunk_t * 548 arena_chunk_init_spare(arena_t *arena) 549 { 550 arena_chunk_t *chunk; 551 552 assert(arena->spare != NULL); 553 554 chunk = arena->spare; 555 arena->spare = NULL; 556 557 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 558 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 559 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 560 arena_maxrun); 561 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 562 arena_maxrun); 563 assert(arena_mapbits_dirty_get(chunk, map_bias) == 564 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 565 566 return (chunk); 567 } 568 569 static bool 570 arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, bool zero) 571 { 572 573 /* 574 * The extent node notion of "committed" doesn't directly apply to 575 * arena chunks. Arbitrarily mark them as committed. The commit state 576 * of runs is tracked individually, and upon chunk deallocation the 577 * entire chunk is in a consistent commit state. 578 */ 579 extent_node_init(&chunk->node, arena, chunk, chunksize, zero, true); 580 extent_node_achunk_set(&chunk->node, true); 581 return (chunk_register(chunk, &chunk->node)); 582 } 583 584 static arena_chunk_t * 585 arena_chunk_alloc_internal_hard(arena_t *arena, chunk_hooks_t *chunk_hooks, 586 bool *zero, bool *commit) 587 { 588 arena_chunk_t *chunk; 589 590 malloc_mutex_unlock(&arena->lock); 591 592 chunk = (arena_chunk_t *)chunk_alloc_wrapper(arena, chunk_hooks, NULL, 593 chunksize, chunksize, zero, commit); 594 if (chunk != NULL && !*commit) { 595 /* Commit header. */ 596 if (chunk_hooks->commit(chunk, chunksize, 0, map_bias << 597 LG_PAGE, arena->ind)) { 598 chunk_dalloc_wrapper(arena, chunk_hooks, 599 (void *)chunk, chunksize, *commit); 600 chunk = NULL; 601 } 602 } 603 if (chunk != NULL && arena_chunk_register(arena, chunk, *zero)) { 604 if (!*commit) { 605 /* Undo commit of header. */ 606 chunk_hooks->decommit(chunk, chunksize, 0, map_bias << 607 LG_PAGE, arena->ind); 608 } 609 chunk_dalloc_wrapper(arena, chunk_hooks, (void *)chunk, 610 chunksize, *commit); 611 chunk = NULL; 612 } 613 614 malloc_mutex_lock(&arena->lock); 615 return (chunk); 616 } 617 618 static arena_chunk_t * 619 arena_chunk_alloc_internal(arena_t *arena, bool *zero, bool *commit) 620 { 621 arena_chunk_t *chunk; 622 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 623 624 chunk = chunk_alloc_cache(arena, &chunk_hooks, NULL, chunksize, 625 chunksize, zero, true); 626 if (chunk != NULL) { 627 if (arena_chunk_register(arena, chunk, *zero)) { 628 chunk_dalloc_cache(arena, &chunk_hooks, chunk, 629 chunksize, true); 630 return (NULL); 631 } 632 *commit = true; 633 } 634 if (chunk == NULL) { 635 chunk = arena_chunk_alloc_internal_hard(arena, &chunk_hooks, 636 zero, commit); 637 } 638 639 if (config_stats && chunk != NULL) { 640 arena->stats.mapped += chunksize; 641 arena->stats.metadata_mapped += (map_bias << LG_PAGE); 642 } 643 644 return (chunk); 645 } 646 647 static arena_chunk_t * 648 arena_chunk_init_hard(arena_t *arena) 649 { 650 arena_chunk_t *chunk; 651 bool zero, commit; 652 size_t flag_unzeroed, flag_decommitted, i; 653 654 assert(arena->spare == NULL); 655 656 zero = false; 657 commit = false; 658 chunk = arena_chunk_alloc_internal(arena, &zero, &commit); 659 if (chunk == NULL) 660 return (NULL); 661 662 /* 663 * Initialize the map to contain one maximal free untouched run. Mark 664 * the pages as zeroed if chunk_alloc() returned a zeroed or decommitted 665 * chunk. 666 */ 667 flag_unzeroed = (zero || !commit) ? 0 : CHUNK_MAP_UNZEROED; 668 flag_decommitted = commit ? 0 : CHUNK_MAP_DECOMMITTED; 669 arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun, 670 flag_unzeroed | flag_decommitted); 671 /* 672 * There is no need to initialize the internal page map entries unless 673 * the chunk is not zeroed. 674 */ 675 if (!zero) { 676 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( 677 (void *)arena_bitselm_get(chunk, map_bias+1), 678 (size_t)((uintptr_t) arena_bitselm_get(chunk, 679 chunk_npages-1) - (uintptr_t)arena_bitselm_get(chunk, 680 map_bias+1))); 681 for (i = map_bias+1; i < chunk_npages-1; i++) 682 arena_mapbits_internal_set(chunk, i, flag_unzeroed); 683 } else { 684 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void 685 *)arena_bitselm_get(chunk, map_bias+1), (size_t)((uintptr_t) 686 arena_bitselm_get(chunk, chunk_npages-1) - 687 (uintptr_t)arena_bitselm_get(chunk, map_bias+1))); 688 if (config_debug) { 689 for (i = map_bias+1; i < chunk_npages-1; i++) { 690 assert(arena_mapbits_unzeroed_get(chunk, i) == 691 flag_unzeroed); 692 } 693 } 694 } 695 arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun, 696 flag_unzeroed); 697 698 return (chunk); 699 } 700 701 static arena_chunk_t * 702 arena_chunk_alloc(arena_t *arena) 703 { 704 arena_chunk_t *chunk; 705 706 if (arena->spare != NULL) 707 chunk = arena_chunk_init_spare(arena); 708 else { 709 chunk = arena_chunk_init_hard(arena); 710 if (chunk == NULL) 711 return (NULL); 712 } 713 714 /* Insert the run into the runs_avail tree. */ 715 arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias); 716 717 return (chunk); 718 } 719 720 static void 721 arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) 722 { 723 724 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 725 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 726 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 727 arena_maxrun); 728 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 729 arena_maxrun); 730 assert(arena_mapbits_dirty_get(chunk, map_bias) == 731 arena_mapbits_dirty_get(chunk, chunk_npages-1)); 732 assert(arena_mapbits_decommitted_get(chunk, map_bias) == 733 arena_mapbits_decommitted_get(chunk, chunk_npages-1)); 734 735 /* 736 * Remove run from the runs_avail tree, so that the arena does not use 737 * it. 738 */ 739 arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias); 740 741 if (arena->spare != NULL) { 742 arena_chunk_t *spare = arena->spare; 743 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 744 bool committed; 745 746 arena->spare = chunk; 747 if (arena_mapbits_dirty_get(spare, map_bias) != 0) { 748 arena_run_dirty_remove(arena, spare, map_bias, 749 chunk_npages-map_bias); 750 } 751 752 chunk_deregister(spare, &spare->node); 753 754 committed = (arena_mapbits_decommitted_get(spare, map_bias) == 755 0); 756 if (!committed) { 757 /* 758 * Decommit the header. Mark the chunk as decommitted 759 * even if header decommit fails, since treating a 760 * partially committed chunk as committed has a high 761 * potential for causing later access of decommitted 762 * memory. 763 */ 764 chunk_hooks = chunk_hooks_get(arena); 765 chunk_hooks.decommit(spare, chunksize, 0, map_bias << 766 LG_PAGE, arena->ind); 767 } 768 769 chunk_dalloc_cache(arena, &chunk_hooks, (void *)spare, 770 chunksize, committed); 771 772 if (config_stats) { 773 arena->stats.mapped -= chunksize; 774 arena->stats.metadata_mapped -= (map_bias << LG_PAGE); 775 } 776 } else 777 arena->spare = chunk; 778 } 779 780 static void 781 arena_huge_malloc_stats_update(arena_t *arena, size_t usize) 782 { 783 szind_t index = size2index(usize) - nlclasses - NBINS; 784 785 cassert(config_stats); 786 787 arena->stats.nmalloc_huge++; 788 arena->stats.allocated_huge += usize; 789 arena->stats.hstats[index].nmalloc++; 790 arena->stats.hstats[index].curhchunks++; 791 } 792 793 static void 794 arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize) 795 { 796 szind_t index = size2index(usize) - nlclasses - NBINS; 797 798 cassert(config_stats); 799 800 arena->stats.nmalloc_huge--; 801 arena->stats.allocated_huge -= usize; 802 arena->stats.hstats[index].nmalloc--; 803 arena->stats.hstats[index].curhchunks--; 804 } 805 806 static void 807 arena_huge_dalloc_stats_update(arena_t *arena, size_t usize) 808 { 809 szind_t index = size2index(usize) - nlclasses - NBINS; 810 811 cassert(config_stats); 812 813 arena->stats.ndalloc_huge++; 814 arena->stats.allocated_huge -= usize; 815 arena->stats.hstats[index].ndalloc++; 816 arena->stats.hstats[index].curhchunks--; 817 } 818 819 static void 820 arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize) 821 { 822 szind_t index = size2index(usize) - nlclasses - NBINS; 823 824 cassert(config_stats); 825 826 arena->stats.ndalloc_huge--; 827 arena->stats.allocated_huge += usize; 828 arena->stats.hstats[index].ndalloc--; 829 arena->stats.hstats[index].curhchunks++; 830 } 831 832 static void 833 arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize) 834 { 835 836 arena_huge_dalloc_stats_update(arena, oldsize); 837 arena_huge_malloc_stats_update(arena, usize); 838 } 839 840 static void 841 arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize, 842 size_t usize) 843 { 844 845 arena_huge_dalloc_stats_update_undo(arena, oldsize); 846 arena_huge_malloc_stats_update_undo(arena, usize); 847 } 848 849 extent_node_t * 850 arena_node_alloc(arena_t *arena) 851 { 852 extent_node_t *node; 853 854 malloc_mutex_lock(&arena->node_cache_mtx); 855 node = ql_last(&arena->node_cache, ql_link); 856 if (node == NULL) { 857 malloc_mutex_unlock(&arena->node_cache_mtx); 858 return (base_alloc(sizeof(extent_node_t))); 859 } 860 ql_tail_remove(&arena->node_cache, extent_node_t, ql_link); 861 malloc_mutex_unlock(&arena->node_cache_mtx); 862 return (node); 863 } 864 865 void 866 arena_node_dalloc(arena_t *arena, extent_node_t *node) 867 { 868 869 malloc_mutex_lock(&arena->node_cache_mtx); 870 ql_elm_new(node, ql_link); 871 ql_tail_insert(&arena->node_cache, node, ql_link); 872 malloc_mutex_unlock(&arena->node_cache_mtx); 873 } 874 875 static void * 876 arena_chunk_alloc_huge_hard(arena_t *arena, chunk_hooks_t *chunk_hooks, 877 size_t usize, size_t alignment, bool *zero, size_t csize) 878 { 879 void *ret; 880 bool commit = true; 881 882 ret = chunk_alloc_wrapper(arena, chunk_hooks, NULL, csize, alignment, 883 zero, &commit); 884 if (ret == NULL) { 885 /* Revert optimistic stats updates. */ 886 malloc_mutex_lock(&arena->lock); 887 if (config_stats) { 888 arena_huge_malloc_stats_update_undo(arena, usize); 889 arena->stats.mapped -= usize; 890 } 891 arena->nactive -= (usize >> LG_PAGE); 892 malloc_mutex_unlock(&arena->lock); 893 } 894 895 return (ret); 896 } 897 898 void * 899 arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, 900 bool *zero) 901 { 902 void *ret; 903 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 904 size_t csize = CHUNK_CEILING(usize); 905 906 malloc_mutex_lock(&arena->lock); 907 908 /* Optimistically update stats. */ 909 if (config_stats) { 910 arena_huge_malloc_stats_update(arena, usize); 911 arena->stats.mapped += usize; 912 } 913 arena->nactive += (usize >> LG_PAGE); 914 915 ret = chunk_alloc_cache(arena, &chunk_hooks, NULL, csize, alignment, 916 zero, true); 917 malloc_mutex_unlock(&arena->lock); 918 if (ret == NULL) { 919 ret = arena_chunk_alloc_huge_hard(arena, &chunk_hooks, usize, 920 alignment, zero, csize); 921 } 922 923 if (config_stats && ret != NULL) 924 stats_cactive_add(usize); 925 return (ret); 926 } 927 928 void 929 arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize) 930 { 931 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 932 size_t csize; 933 934 csize = CHUNK_CEILING(usize); 935 malloc_mutex_lock(&arena->lock); 936 if (config_stats) { 937 arena_huge_dalloc_stats_update(arena, usize); 938 arena->stats.mapped -= usize; 939 stats_cactive_sub(usize); 940 } 941 arena->nactive -= (usize >> LG_PAGE); 942 943 chunk_dalloc_cache(arena, &chunk_hooks, chunk, csize, true); 944 malloc_mutex_unlock(&arena->lock); 945 } 946 947 void 948 arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, size_t oldsize, 949 size_t usize) 950 { 951 952 assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize)); 953 assert(oldsize != usize); 954 955 malloc_mutex_lock(&arena->lock); 956 if (config_stats) 957 arena_huge_ralloc_stats_update(arena, oldsize, usize); 958 if (oldsize < usize) { 959 size_t udiff = usize - oldsize; 960 arena->nactive += udiff >> LG_PAGE; 961 if (config_stats) 962 stats_cactive_add(udiff); 963 } else { 964 size_t udiff = oldsize - usize; 965 arena->nactive -= udiff >> LG_PAGE; 966 if (config_stats) 967 stats_cactive_sub(udiff); 968 } 969 malloc_mutex_unlock(&arena->lock); 970 } 971 972 void 973 arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize, 974 size_t usize) 975 { 976 size_t udiff = oldsize - usize; 977 size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); 978 979 malloc_mutex_lock(&arena->lock); 980 if (config_stats) { 981 arena_huge_ralloc_stats_update(arena, oldsize, usize); 982 if (cdiff != 0) { 983 arena->stats.mapped -= cdiff; 984 stats_cactive_sub(udiff); 985 } 986 } 987 arena->nactive -= udiff >> LG_PAGE; 988 989 if (cdiff != 0) { 990 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 991 void *nchunk = (void *)((uintptr_t)chunk + 992 CHUNK_CEILING(usize)); 993 994 chunk_dalloc_cache(arena, &chunk_hooks, nchunk, cdiff, true); 995 } 996 malloc_mutex_unlock(&arena->lock); 997 } 998 999 static bool 1000 arena_chunk_ralloc_huge_expand_hard(arena_t *arena, chunk_hooks_t *chunk_hooks, 1001 void *chunk, size_t oldsize, size_t usize, bool *zero, void *nchunk, 1002 size_t udiff, size_t cdiff) 1003 { 1004 bool err; 1005 bool commit = true; 1006 1007 err = (chunk_alloc_wrapper(arena, chunk_hooks, nchunk, cdiff, chunksize, 1008 zero, &commit) == NULL); 1009 if (err) { 1010 /* Revert optimistic stats updates. */ 1011 malloc_mutex_lock(&arena->lock); 1012 if (config_stats) { 1013 arena_huge_ralloc_stats_update_undo(arena, oldsize, 1014 usize); 1015 arena->stats.mapped -= cdiff; 1016 } 1017 arena->nactive -= (udiff >> LG_PAGE); 1018 malloc_mutex_unlock(&arena->lock); 1019 } else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk, 1020 cdiff, true, arena->ind)) { 1021 chunk_dalloc_arena(arena, chunk_hooks, nchunk, cdiff, *zero, 1022 true); 1023 err = true; 1024 } 1025 return (err); 1026 } 1027 1028 bool 1029 arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, 1030 size_t usize, bool *zero) 1031 { 1032 bool err; 1033 chunk_hooks_t chunk_hooks = chunk_hooks_get(arena); 1034 void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)); 1035 size_t udiff = usize - oldsize; 1036 size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); 1037 1038 malloc_mutex_lock(&arena->lock); 1039 1040 /* Optimistically update stats. */ 1041 if (config_stats) { 1042 arena_huge_ralloc_stats_update(arena, oldsize, usize); 1043 arena->stats.mapped += cdiff; 1044 } 1045 arena->nactive += (udiff >> LG_PAGE); 1046 1047 err = (chunk_alloc_cache(arena, &arena->chunk_hooks, nchunk, cdiff, 1048 chunksize, zero, true) == NULL); 1049 malloc_mutex_unlock(&arena->lock); 1050 if (err) { 1051 err = arena_chunk_ralloc_huge_expand_hard(arena, &chunk_hooks, 1052 chunk, oldsize, usize, zero, nchunk, udiff, 1053 cdiff); 1054 } else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk, 1055 cdiff, true, arena->ind)) { 1056 chunk_dalloc_arena(arena, &chunk_hooks, nchunk, cdiff, *zero, 1057 true); 1058 err = true; 1059 } 1060 1061 if (config_stats && !err) 1062 stats_cactive_add(udiff); 1063 return (err); 1064 } 1065 1066 /* 1067 * Do first-best-fit run selection, i.e. select the lowest run that best fits. 1068 * Run sizes are quantized, so not all candidate runs are necessarily exactly 1069 * the same size. 1070 */ 1071 static arena_run_t * 1072 arena_run_first_best_fit(arena_t *arena, size_t size) 1073 { 1074 size_t search_size = run_quantize_first(size); 1075 arena_chunk_map_misc_t *key = arena_miscelm_key_create(search_size); 1076 arena_chunk_map_misc_t *miscelm = 1077 arena_avail_tree_nsearch(&arena->runs_avail, key); 1078 if (miscelm == NULL) 1079 return (NULL); 1080 return (&miscelm->run); 1081 } 1082 1083 static arena_run_t * 1084 arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) 1085 { 1086 arena_run_t *run = arena_run_first_best_fit(arena, s2u(size)); 1087 if (run != NULL) { 1088 if (arena_run_split_large(arena, run, size, zero)) 1089 run = NULL; 1090 } 1091 return (run); 1092 } 1093 1094 static arena_run_t * 1095 arena_run_alloc_large(arena_t *arena, size_t size, bool zero) 1096 { 1097 arena_chunk_t *chunk; 1098 arena_run_t *run; 1099 1100 assert(size <= arena_maxrun); 1101 assert(size == PAGE_CEILING(size)); 1102 1103 /* Search the arena's chunks for the lowest best fit. */ 1104 run = arena_run_alloc_large_helper(arena, size, zero); 1105 if (run != NULL) 1106 return (run); 1107 1108 /* 1109 * No usable runs. Create a new chunk from which to allocate the run. 1110 */ 1111 chunk = arena_chunk_alloc(arena); 1112 if (chunk != NULL) { 1113 run = &arena_miscelm_get(chunk, map_bias)->run; 1114 if (arena_run_split_large(arena, run, size, zero)) 1115 run = NULL; 1116 return (run); 1117 } 1118 1119 /* 1120 * arena_chunk_alloc() failed, but another thread may have made 1121 * sufficient memory available while this one dropped arena->lock in 1122 * arena_chunk_alloc(), so search one more time. 1123 */ 1124 return (arena_run_alloc_large_helper(arena, size, zero)); 1125 } 1126 1127 static arena_run_t * 1128 arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind) 1129 { 1130 arena_run_t *run = arena_run_first_best_fit(arena, size); 1131 if (run != NULL) { 1132 if (arena_run_split_small(arena, run, size, binind)) 1133 run = NULL; 1134 } 1135 return (run); 1136 } 1137 1138 static arena_run_t * 1139 arena_run_alloc_small(arena_t *arena, size_t size, szind_t binind) 1140 { 1141 arena_chunk_t *chunk; 1142 arena_run_t *run; 1143 1144 assert(size <= arena_maxrun); 1145 assert(size == PAGE_CEILING(size)); 1146 assert(binind != BININD_INVALID); 1147 1148 /* Search the arena's chunks for the lowest best fit. */ 1149 run = arena_run_alloc_small_helper(arena, size, binind); 1150 if (run != NULL) 1151 return (run); 1152 1153 /* 1154 * No usable runs. Create a new chunk from which to allocate the run. 1155 */ 1156 chunk = arena_chunk_alloc(arena); 1157 if (chunk != NULL) { 1158 run = &arena_miscelm_get(chunk, map_bias)->run; 1159 if (arena_run_split_small(arena, run, size, binind)) 1160 run = NULL; 1161 return (run); 1162 } 1163 1164 /* 1165 * arena_chunk_alloc() failed, but another thread may have made 1166 * sufficient memory available while this one dropped arena->lock in 1167 * arena_chunk_alloc(), so search one more time. 1168 */ 1169 return (arena_run_alloc_small_helper(arena, size, binind)); 1170 } 1171 1172 static bool 1173 arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult) 1174 { 1175 1176 return (lg_dirty_mult >= -1 && lg_dirty_mult < (ssize_t)(sizeof(size_t) 1177 << 3)); 1178 } 1179 1180 ssize_t 1181 arena_lg_dirty_mult_get(arena_t *arena) 1182 { 1183 ssize_t lg_dirty_mult; 1184 1185 malloc_mutex_lock(&arena->lock); 1186 lg_dirty_mult = arena->lg_dirty_mult; 1187 malloc_mutex_unlock(&arena->lock); 1188 1189 return (lg_dirty_mult); 1190 } 1191 1192 bool 1193 arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult) 1194 { 1195 1196 if (!arena_lg_dirty_mult_valid(lg_dirty_mult)) 1197 return (true); 1198 1199 malloc_mutex_lock(&arena->lock); 1200 arena->lg_dirty_mult = lg_dirty_mult; 1201 arena_maybe_purge(arena); 1202 malloc_mutex_unlock(&arena->lock); 1203 1204 return (false); 1205 } 1206 1207 void 1208 arena_maybe_purge(arena_t *arena) 1209 { 1210 1211 /* Don't purge if the option is disabled. */ 1212 if (arena->lg_dirty_mult < 0) 1213 return; 1214 /* Don't recursively purge. */ 1215 if (arena->purging) 1216 return; 1217 /* 1218 * Iterate, since preventing recursive purging could otherwise leave too 1219 * many dirty pages. 1220 */ 1221 while (true) { 1222 size_t threshold = (arena->nactive >> arena->lg_dirty_mult); 1223 if (threshold < chunk_npages) 1224 threshold = chunk_npages; 1225 /* 1226 * Don't purge unless the number of purgeable pages exceeds the 1227 * threshold. 1228 */ 1229 if (arena->ndirty <= threshold) 1230 return; 1231 arena_purge(arena, false); 1232 } 1233 } 1234 1235 static size_t 1236 arena_dirty_count(arena_t *arena) 1237 { 1238 size_t ndirty = 0; 1239 arena_runs_dirty_link_t *rdelm; 1240 extent_node_t *chunkselm; 1241 1242 for (rdelm = qr_next(&arena->runs_dirty, rd_link), 1243 chunkselm = qr_next(&arena->chunks_cache, cc_link); 1244 rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) { 1245 size_t npages; 1246 1247 if (rdelm == &chunkselm->rd) { 1248 npages = extent_node_size_get(chunkselm) >> LG_PAGE; 1249 chunkselm = qr_next(chunkselm, cc_link); 1250 } else { 1251 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( 1252 rdelm); 1253 arena_chunk_map_misc_t *miscelm = 1254 arena_rd_to_miscelm(rdelm); 1255 size_t pageind = arena_miscelm_to_pageind(miscelm); 1256 assert(arena_mapbits_allocated_get(chunk, pageind) == 1257 0); 1258 assert(arena_mapbits_large_get(chunk, pageind) == 0); 1259 assert(arena_mapbits_dirty_get(chunk, pageind) != 0); 1260 npages = arena_mapbits_unallocated_size_get(chunk, 1261 pageind) >> LG_PAGE; 1262 } 1263 ndirty += npages; 1264 } 1265 1266 return (ndirty); 1267 } 1268 1269 static size_t 1270 arena_compute_npurge(arena_t *arena, bool all) 1271 { 1272 size_t npurge; 1273 1274 /* 1275 * Compute the minimum number of pages that this thread should try to 1276 * purge. 1277 */ 1278 if (!all) { 1279 size_t threshold = (arena->nactive >> arena->lg_dirty_mult); 1280 threshold = threshold < chunk_npages ? chunk_npages : threshold; 1281 1282 npurge = arena->ndirty - threshold; 1283 } else 1284 npurge = arena->ndirty; 1285 1286 return (npurge); 1287 } 1288 1289 static size_t 1290 arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks, bool all, 1291 size_t npurge, arena_runs_dirty_link_t *purge_runs_sentinel, 1292 extent_node_t *purge_chunks_sentinel) 1293 { 1294 arena_runs_dirty_link_t *rdelm, *rdelm_next; 1295 extent_node_t *chunkselm; 1296 size_t nstashed = 0; 1297 1298 /* Stash at least npurge pages. */ 1299 for (rdelm = qr_next(&arena->runs_dirty, rd_link), 1300 chunkselm = qr_next(&arena->chunks_cache, cc_link); 1301 rdelm != &arena->runs_dirty; rdelm = rdelm_next) { 1302 size_t npages; 1303 rdelm_next = qr_next(rdelm, rd_link); 1304 1305 if (rdelm == &chunkselm->rd) { 1306 extent_node_t *chunkselm_next; 1307 bool zero; 1308 UNUSED void *chunk; 1309 1310 chunkselm_next = qr_next(chunkselm, cc_link); 1311 /* 1312 * Allocate. chunkselm remains valid due to the 1313 * dalloc_node=false argument to chunk_alloc_cache(). 1314 */ 1315 zero = false; 1316 chunk = chunk_alloc_cache(arena, chunk_hooks, 1317 extent_node_addr_get(chunkselm), 1318 extent_node_size_get(chunkselm), chunksize, &zero, 1319 false); 1320 assert(chunk == extent_node_addr_get(chunkselm)); 1321 assert(zero == extent_node_zeroed_get(chunkselm)); 1322 extent_node_dirty_insert(chunkselm, purge_runs_sentinel, 1323 purge_chunks_sentinel); 1324 npages = extent_node_size_get(chunkselm) >> LG_PAGE; 1325 chunkselm = chunkselm_next; 1326 } else { 1327 arena_chunk_t *chunk = 1328 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); 1329 arena_chunk_map_misc_t *miscelm = 1330 arena_rd_to_miscelm(rdelm); 1331 size_t pageind = arena_miscelm_to_pageind(miscelm); 1332 arena_run_t *run = &miscelm->run; 1333 size_t run_size = 1334 arena_mapbits_unallocated_size_get(chunk, pageind); 1335 1336 npages = run_size >> LG_PAGE; 1337 1338 assert(pageind + npages <= chunk_npages); 1339 assert(arena_mapbits_dirty_get(chunk, pageind) == 1340 arena_mapbits_dirty_get(chunk, pageind+npages-1)); 1341 1342 /* 1343 * If purging the spare chunk's run, make it available 1344 * prior to allocation. 1345 */ 1346 if (chunk == arena->spare) 1347 arena_chunk_alloc(arena); 1348 1349 /* Temporarily allocate the free dirty run. */ 1350 arena_run_split_large(arena, run, run_size, false); 1351 /* Stash. */ 1352 if (false) 1353 qr_new(rdelm, rd_link); /* Redundant. */ 1354 else { 1355 assert(qr_next(rdelm, rd_link) == rdelm); 1356 assert(qr_prev(rdelm, rd_link) == rdelm); 1357 } 1358 qr_meld(purge_runs_sentinel, rdelm, rd_link); 1359 } 1360 1361 nstashed += npages; 1362 if (!all && nstashed >= npurge) 1363 break; 1364 } 1365 1366 return (nstashed); 1367 } 1368 1369 static size_t 1370 arena_purge_stashed(arena_t *arena, chunk_hooks_t *chunk_hooks, 1371 arena_runs_dirty_link_t *purge_runs_sentinel, 1372 extent_node_t *purge_chunks_sentinel) 1373 { 1374 size_t npurged, nmadvise; 1375 arena_runs_dirty_link_t *rdelm; 1376 extent_node_t *chunkselm; 1377 1378 if (config_stats) 1379 nmadvise = 0; 1380 npurged = 0; 1381 1382 malloc_mutex_unlock(&arena->lock); 1383 for (rdelm = qr_next(purge_runs_sentinel, rd_link), 1384 chunkselm = qr_next(purge_chunks_sentinel, cc_link); 1385 rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) { 1386 size_t npages; 1387 1388 if (rdelm == &chunkselm->rd) { 1389 /* 1390 * Don't actually purge the chunk here because 1) 1391 * chunkselm is embedded in the chunk and must remain 1392 * valid, and 2) we deallocate the chunk in 1393 * arena_unstash_purged(), where it is destroyed, 1394 * decommitted, or purged, depending on chunk 1395 * deallocation policy. 1396 */ 1397 size_t size = extent_node_size_get(chunkselm); 1398 npages = size >> LG_PAGE; 1399 chunkselm = qr_next(chunkselm, cc_link); 1400 } else { 1401 size_t pageind, run_size, flag_unzeroed, flags, i; 1402 bool decommitted; 1403 arena_chunk_t *chunk = 1404 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); 1405 arena_chunk_map_misc_t *miscelm = 1406 arena_rd_to_miscelm(rdelm); 1407 pageind = arena_miscelm_to_pageind(miscelm); 1408 run_size = arena_mapbits_large_size_get(chunk, pageind); 1409 npages = run_size >> LG_PAGE; 1410 1411 assert(pageind + npages <= chunk_npages); 1412 assert(!arena_mapbits_decommitted_get(chunk, pageind)); 1413 assert(!arena_mapbits_decommitted_get(chunk, 1414 pageind+npages-1)); 1415 decommitted = !chunk_hooks->decommit(chunk, chunksize, 1416 pageind << LG_PAGE, npages << LG_PAGE, arena->ind); 1417 if (decommitted) { 1418 flag_unzeroed = 0; 1419 flags = CHUNK_MAP_DECOMMITTED; 1420 } else { 1421 flag_unzeroed = chunk_purge_wrapper(arena, 1422 chunk_hooks, chunk, chunksize, pageind << 1423 LG_PAGE, run_size) ? CHUNK_MAP_UNZEROED : 0; 1424 flags = flag_unzeroed; 1425 } 1426 arena_mapbits_large_set(chunk, pageind+npages-1, 0, 1427 flags); 1428 arena_mapbits_large_set(chunk, pageind, run_size, 1429 flags); 1430 1431 /* 1432 * Set the unzeroed flag for internal pages, now that 1433 * chunk_purge_wrapper() has returned whether the pages 1434 * were zeroed as a side effect of purging. This chunk 1435 * map modification is safe even though the arena mutex 1436 * isn't currently owned by this thread, because the run 1437 * is marked as allocated, thus protecting it from being 1438 * modified by any other thread. As long as these 1439 * writes don't perturb the first and last elements' 1440 * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 1441 */ 1442 for (i = 1; i < npages-1; i++) { 1443 arena_mapbits_internal_set(chunk, pageind+i, 1444 flag_unzeroed); 1445 } 1446 } 1447 1448 npurged += npages; 1449 if (config_stats) 1450 nmadvise++; 1451 } 1452 malloc_mutex_lock(&arena->lock); 1453 1454 if (config_stats) { 1455 arena->stats.nmadvise += nmadvise; 1456 arena->stats.purged += npurged; 1457 } 1458 1459 return (npurged); 1460 } 1461 1462 static void 1463 arena_unstash_purged(arena_t *arena, chunk_hooks_t *chunk_hooks, 1464 arena_runs_dirty_link_t *purge_runs_sentinel, 1465 extent_node_t *purge_chunks_sentinel) 1466 { 1467 arena_runs_dirty_link_t *rdelm, *rdelm_next; 1468 extent_node_t *chunkselm; 1469 1470 /* Deallocate chunks/runs. */ 1471 for (rdelm = qr_next(purge_runs_sentinel, rd_link), 1472 chunkselm = qr_next(purge_chunks_sentinel, cc_link); 1473 rdelm != purge_runs_sentinel; rdelm = rdelm_next) { 1474 rdelm_next = qr_next(rdelm, rd_link); 1475 if (rdelm == &chunkselm->rd) { 1476 extent_node_t *chunkselm_next = qr_next(chunkselm, 1477 cc_link); 1478 void *addr = extent_node_addr_get(chunkselm); 1479 size_t size = extent_node_size_get(chunkselm); 1480 bool zeroed = extent_node_zeroed_get(chunkselm); 1481 bool committed = extent_node_committed_get(chunkselm); 1482 extent_node_dirty_remove(chunkselm); 1483 arena_node_dalloc(arena, chunkselm); 1484 chunkselm = chunkselm_next; 1485 chunk_dalloc_arena(arena, chunk_hooks, addr, size, 1486 zeroed, committed); 1487 } else { 1488 arena_chunk_t *chunk = 1489 (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); 1490 arena_chunk_map_misc_t *miscelm = 1491 arena_rd_to_miscelm(rdelm); 1492 size_t pageind = arena_miscelm_to_pageind(miscelm); 1493 bool decommitted = (arena_mapbits_decommitted_get(chunk, 1494 pageind) != 0); 1495 arena_run_t *run = &miscelm->run; 1496 qr_remove(rdelm, rd_link); 1497 arena_run_dalloc(arena, run, false, true, decommitted); 1498 } 1499 } 1500 } 1501 1502 static void 1503 arena_purge(arena_t *arena, bool all) 1504 { 1505 chunk_hooks_t chunk_hooks = chunk_hooks_get(arena); 1506 size_t npurge, npurgeable, npurged; 1507 arena_runs_dirty_link_t purge_runs_sentinel; 1508 extent_node_t purge_chunks_sentinel; 1509 1510 arena->purging = true; 1511 1512 /* 1513 * Calls to arena_dirty_count() are disabled even for debug builds 1514 * because overhead grows nonlinearly as memory usage increases. 1515 */ 1516 if (false && config_debug) { 1517 size_t ndirty = arena_dirty_count(arena); 1518 assert(ndirty == arena->ndirty); 1519 } 1520 assert((arena->nactive >> arena->lg_dirty_mult) < arena->ndirty || all); 1521 1522 if (config_stats) 1523 arena->stats.npurge++; 1524 1525 npurge = arena_compute_npurge(arena, all); 1526 qr_new(&purge_runs_sentinel, rd_link); 1527 extent_node_dirty_linkage_init(&purge_chunks_sentinel); 1528 1529 npurgeable = arena_stash_dirty(arena, &chunk_hooks, all, npurge, 1530 &purge_runs_sentinel, &purge_chunks_sentinel); 1531 assert(npurgeable >= npurge); 1532 npurged = arena_purge_stashed(arena, &chunk_hooks, &purge_runs_sentinel, 1533 &purge_chunks_sentinel); 1534 assert(npurged == npurgeable); 1535 arena_unstash_purged(arena, &chunk_hooks, &purge_runs_sentinel, 1536 &purge_chunks_sentinel); 1537 1538 arena->purging = false; 1539 } 1540 1541 void 1542 arena_purge_all(arena_t *arena) 1543 { 1544 1545 malloc_mutex_lock(&arena->lock); 1546 arena_purge(arena, true); 1547 malloc_mutex_unlock(&arena->lock); 1548 } 1549 1550 static void 1551 arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, 1552 size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty, 1553 size_t flag_decommitted) 1554 { 1555 size_t size = *p_size; 1556 size_t run_ind = *p_run_ind; 1557 size_t run_pages = *p_run_pages; 1558 1559 /* Try to coalesce forward. */ 1560 if (run_ind + run_pages < chunk_npages && 1561 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 1562 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty && 1563 arena_mapbits_decommitted_get(chunk, run_ind+run_pages) == 1564 flag_decommitted) { 1565 size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 1566 run_ind+run_pages); 1567 size_t nrun_pages = nrun_size >> LG_PAGE; 1568 1569 /* 1570 * Remove successor from runs_avail; the coalesced run is 1571 * inserted later. 1572 */ 1573 assert(arena_mapbits_unallocated_size_get(chunk, 1574 run_ind+run_pages+nrun_pages-1) == nrun_size); 1575 assert(arena_mapbits_dirty_get(chunk, 1576 run_ind+run_pages+nrun_pages-1) == flag_dirty); 1577 assert(arena_mapbits_decommitted_get(chunk, 1578 run_ind+run_pages+nrun_pages-1) == flag_decommitted); 1579 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); 1580 1581 /* 1582 * If the successor is dirty, remove it from the set of dirty 1583 * pages. 1584 */ 1585 if (flag_dirty != 0) { 1586 arena_run_dirty_remove(arena, chunk, run_ind+run_pages, 1587 nrun_pages); 1588 } 1589 1590 size += nrun_size; 1591 run_pages += nrun_pages; 1592 1593 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1594 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1595 size); 1596 } 1597 1598 /* Try to coalesce backward. */ 1599 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, 1600 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == 1601 flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) == 1602 flag_decommitted) { 1603 size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 1604 run_ind-1); 1605 size_t prun_pages = prun_size >> LG_PAGE; 1606 1607 run_ind -= prun_pages; 1608 1609 /* 1610 * Remove predecessor from runs_avail; the coalesced run is 1611 * inserted later. 1612 */ 1613 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1614 prun_size); 1615 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 1616 assert(arena_mapbits_decommitted_get(chunk, run_ind) == 1617 flag_decommitted); 1618 arena_avail_remove(arena, chunk, run_ind, prun_pages); 1619 1620 /* 1621 * If the predecessor is dirty, remove it from the set of dirty 1622 * pages. 1623 */ 1624 if (flag_dirty != 0) { 1625 arena_run_dirty_remove(arena, chunk, run_ind, 1626 prun_pages); 1627 } 1628 1629 size += prun_size; 1630 run_pages += prun_pages; 1631 1632 arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1633 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1634 size); 1635 } 1636 1637 *p_size = size; 1638 *p_run_ind = run_ind; 1639 *p_run_pages = run_pages; 1640 } 1641 1642 static size_t 1643 arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1644 size_t run_ind) 1645 { 1646 size_t size; 1647 1648 assert(run_ind >= map_bias); 1649 assert(run_ind < chunk_npages); 1650 1651 if (arena_mapbits_large_get(chunk, run_ind) != 0) { 1652 size = arena_mapbits_large_size_get(chunk, run_ind); 1653 assert(size == PAGE || arena_mapbits_large_size_get(chunk, 1654 run_ind+(size>>LG_PAGE)-1) == 0); 1655 } else { 1656 arena_bin_info_t *bin_info = &arena_bin_info[run->binind]; 1657 size = bin_info->run_size; 1658 } 1659 1660 return (size); 1661 } 1662 1663 static bool 1664 arena_run_decommit(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run) 1665 { 1666 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1667 size_t run_ind = arena_miscelm_to_pageind(miscelm); 1668 size_t offset = run_ind << LG_PAGE; 1669 size_t length = arena_run_size_get(arena, chunk, run, run_ind); 1670 1671 return (arena->chunk_hooks.decommit(chunk, chunksize, offset, length, 1672 arena->ind)); 1673 } 1674 1675 static void 1676 arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned, 1677 bool decommitted) 1678 { 1679 arena_chunk_t *chunk; 1680 arena_chunk_map_misc_t *miscelm; 1681 size_t size, run_ind, run_pages, flag_dirty, flag_decommitted; 1682 1683 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1684 miscelm = arena_run_to_miscelm(run); 1685 run_ind = arena_miscelm_to_pageind(miscelm); 1686 assert(run_ind >= map_bias); 1687 assert(run_ind < chunk_npages); 1688 size = arena_run_size_get(arena, chunk, run, run_ind); 1689 run_pages = (size >> LG_PAGE); 1690 arena_cactive_update(arena, 0, run_pages); 1691 arena->nactive -= run_pages; 1692 1693 /* 1694 * The run is dirty if the caller claims to have dirtied it, as well as 1695 * if it was already dirty before being allocated and the caller 1696 * doesn't claim to have cleaned it. 1697 */ 1698 assert(arena_mapbits_dirty_get(chunk, run_ind) == 1699 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1700 if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind) 1701 != 0) 1702 dirty = true; 1703 flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 1704 flag_decommitted = decommitted ? CHUNK_MAP_DECOMMITTED : 0; 1705 1706 /* Mark pages as unallocated in the chunk map. */ 1707 if (dirty || decommitted) { 1708 size_t flags = flag_dirty | flag_decommitted; 1709 arena_mapbits_unallocated_set(chunk, run_ind, size, flags); 1710 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1711 flags); 1712 } else { 1713 arena_mapbits_unallocated_set(chunk, run_ind, size, 1714 arena_mapbits_unzeroed_get(chunk, run_ind)); 1715 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1716 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 1717 } 1718 1719 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, 1720 flag_dirty, flag_decommitted); 1721 1722 /* Insert into runs_avail, now that coalescing is complete. */ 1723 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1724 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 1725 assert(arena_mapbits_dirty_get(chunk, run_ind) == 1726 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1727 assert(arena_mapbits_decommitted_get(chunk, run_ind) == 1728 arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1)); 1729 arena_avail_insert(arena, chunk, run_ind, run_pages); 1730 1731 if (dirty) 1732 arena_run_dirty_insert(arena, chunk, run_ind, run_pages); 1733 1734 /* Deallocate chunk if it is now completely unused. */ 1735 if (size == arena_maxrun) { 1736 assert(run_ind == map_bias); 1737 assert(run_pages == (arena_maxrun >> LG_PAGE)); 1738 arena_chunk_dalloc(arena, chunk); 1739 } 1740 1741 /* 1742 * It is okay to do dirty page processing here even if the chunk was 1743 * deallocated above, since in that case it is the spare. Waiting 1744 * until after possible chunk deallocation to do dirty processing 1745 * allows for an old spare to be fully deallocated, thus decreasing the 1746 * chances of spuriously crossing the dirty page purging threshold. 1747 */ 1748 if (dirty) 1749 arena_maybe_purge(arena); 1750 } 1751 1752 static void 1753 arena_run_dalloc_decommit(arena_t *arena, arena_chunk_t *chunk, 1754 arena_run_t *run) 1755 { 1756 bool committed = arena_run_decommit(arena, chunk, run); 1757 1758 arena_run_dalloc(arena, run, committed, false, !committed); 1759 } 1760 1761 static void 1762 arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1763 size_t oldsize, size_t newsize) 1764 { 1765 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1766 size_t pageind = arena_miscelm_to_pageind(miscelm); 1767 size_t head_npages = (oldsize - newsize) >> LG_PAGE; 1768 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1769 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); 1770 size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? 1771 CHUNK_MAP_UNZEROED : 0; 1772 1773 assert(oldsize > newsize); 1774 1775 /* 1776 * Update the chunk map so that arena_run_dalloc() can treat the 1777 * leading run as separately allocated. Set the last element of each 1778 * run first, in case of single-page runs. 1779 */ 1780 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1781 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | 1782 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 1783 pageind+head_npages-1))); 1784 arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty | 1785 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); 1786 1787 if (config_debug) { 1788 UNUSED size_t tail_npages = newsize >> LG_PAGE; 1789 assert(arena_mapbits_large_size_get(chunk, 1790 pageind+head_npages+tail_npages-1) == 0); 1791 assert(arena_mapbits_dirty_get(chunk, 1792 pageind+head_npages+tail_npages-1) == flag_dirty); 1793 } 1794 arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 1795 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 1796 pageind+head_npages))); 1797 1798 arena_run_dalloc(arena, run, false, false, (flag_decommitted != 0)); 1799 } 1800 1801 static void 1802 arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1803 size_t oldsize, size_t newsize, bool dirty) 1804 { 1805 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1806 size_t pageind = arena_miscelm_to_pageind(miscelm); 1807 size_t head_npages = newsize >> LG_PAGE; 1808 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1809 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); 1810 size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? 1811 CHUNK_MAP_UNZEROED : 0; 1812 arena_chunk_map_misc_t *tail_miscelm; 1813 arena_run_t *tail_run; 1814 1815 assert(oldsize > newsize); 1816 1817 /* 1818 * Update the chunk map so that arena_run_dalloc() can treat the 1819 * trailing run as separately allocated. Set the last element of each 1820 * run first, in case of single-page runs. 1821 */ 1822 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1823 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | 1824 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 1825 pageind+head_npages-1))); 1826 arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty | 1827 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); 1828 1829 if (config_debug) { 1830 UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 1831 assert(arena_mapbits_large_size_get(chunk, 1832 pageind+head_npages+tail_npages-1) == 0); 1833 assert(arena_mapbits_dirty_get(chunk, 1834 pageind+head_npages+tail_npages-1) == flag_dirty); 1835 } 1836 arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 1837 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 1838 pageind+head_npages))); 1839 1840 tail_miscelm = arena_miscelm_get(chunk, pageind + head_npages); 1841 tail_run = &tail_miscelm->run; 1842 arena_run_dalloc(arena, tail_run, dirty, false, (flag_decommitted != 1843 0)); 1844 } 1845 1846 static arena_run_t * 1847 arena_bin_runs_first(arena_bin_t *bin) 1848 { 1849 arena_chunk_map_misc_t *miscelm = arena_run_tree_first(&bin->runs); 1850 if (miscelm != NULL) 1851 return (&miscelm->run); 1852 1853 return (NULL); 1854 } 1855 1856 static void 1857 arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1858 { 1859 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1860 1861 assert(arena_run_tree_search(&bin->runs, miscelm) == NULL); 1862 1863 arena_run_tree_insert(&bin->runs, miscelm); 1864 } 1865 1866 static void 1867 arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1868 { 1869 arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 1870 1871 assert(arena_run_tree_search(&bin->runs, miscelm) != NULL); 1872 1873 arena_run_tree_remove(&bin->runs, miscelm); 1874 } 1875 1876 static arena_run_t * 1877 arena_bin_nonfull_run_tryget(arena_bin_t *bin) 1878 { 1879 arena_run_t *run = arena_bin_runs_first(bin); 1880 if (run != NULL) { 1881 arena_bin_runs_remove(bin, run); 1882 if (config_stats) 1883 bin->stats.reruns++; 1884 } 1885 return (run); 1886 } 1887 1888 static arena_run_t * 1889 arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1890 { 1891 arena_run_t *run; 1892 szind_t binind; 1893 arena_bin_info_t *bin_info; 1894 1895 /* Look for a usable run. */ 1896 run = arena_bin_nonfull_run_tryget(bin); 1897 if (run != NULL) 1898 return (run); 1899 /* No existing runs have any space available. */ 1900 1901 binind = arena_bin_index(arena, bin); 1902 bin_info = &arena_bin_info[binind]; 1903 1904 /* Allocate a new run. */ 1905 malloc_mutex_unlock(&bin->lock); 1906 /******************************/ 1907 malloc_mutex_lock(&arena->lock); 1908 run = arena_run_alloc_small(arena, bin_info->run_size, binind); 1909 if (run != NULL) { 1910 /* Initialize run internals. */ 1911 run->binind = binind; 1912 run->nfree = bin_info->nregs; 1913 bitmap_init(run->bitmap, &bin_info->bitmap_info); 1914 } 1915 malloc_mutex_unlock(&arena->lock); 1916 /********************************/ 1917 malloc_mutex_lock(&bin->lock); 1918 if (run != NULL) { 1919 if (config_stats) { 1920 bin->stats.nruns++; 1921 bin->stats.curruns++; 1922 } 1923 return (run); 1924 } 1925 1926 /* 1927 * arena_run_alloc_small() failed, but another thread may have made 1928 * sufficient memory available while this one dropped bin->lock above, 1929 * so search one more time. 1930 */ 1931 run = arena_bin_nonfull_run_tryget(bin); 1932 if (run != NULL) 1933 return (run); 1934 1935 return (NULL); 1936 } 1937 1938 /* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1939 static void * 1940 arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1941 { 1942 szind_t binind; 1943 arena_bin_info_t *bin_info; 1944 arena_run_t *run; 1945 1946 binind = arena_bin_index(arena, bin); 1947 bin_info = &arena_bin_info[binind]; 1948 bin->runcur = NULL; 1949 run = arena_bin_nonfull_run_get(arena, bin); 1950 if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1951 /* 1952 * Another thread updated runcur while this one ran without the 1953 * bin lock in arena_bin_nonfull_run_get(). 1954 */ 1955 void *ret; 1956 assert(bin->runcur->nfree > 0); 1957 ret = arena_run_reg_alloc(bin->runcur, bin_info); 1958 if (run != NULL) { 1959 arena_chunk_t *chunk; 1960 1961 /* 1962 * arena_run_alloc_small() may have allocated run, or 1963 * it may have pulled run from the bin's run tree. 1964 * Therefore it is unsafe to make any assumptions about 1965 * how run has previously been used, and 1966 * arena_bin_lower_run() must be called, as if a region 1967 * were just deallocated from the run. 1968 */ 1969 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1970 if (run->nfree == bin_info->nregs) 1971 arena_dalloc_bin_run(arena, chunk, run, bin); 1972 else 1973 arena_bin_lower_run(arena, chunk, run, bin); 1974 } 1975 return (ret); 1976 } 1977 1978 if (run == NULL) 1979 return (NULL); 1980 1981 bin->runcur = run; 1982 1983 assert(bin->runcur->nfree > 0); 1984 1985 return (arena_run_reg_alloc(bin->runcur, bin_info)); 1986 } 1987 1988 void 1989 arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, szind_t binind, 1990 uint64_t prof_accumbytes) 1991 { 1992 unsigned i, nfill; 1993 arena_bin_t *bin; 1994 1995 assert(tbin->ncached == 0); 1996 1997 if (config_prof && arena_prof_accum(arena, prof_accumbytes)) 1998 prof_idump(); 1999 bin = &arena->bins[binind]; 2000 malloc_mutex_lock(&bin->lock); 2001 for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 2002 tbin->lg_fill_div); i < nfill; i++) { 2003 arena_run_t *run; 2004 void *ptr; 2005 if ((run = bin->runcur) != NULL && run->nfree > 0) 2006 ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 2007 else 2008 ptr = arena_bin_malloc_hard(arena, bin); 2009 if (ptr == NULL) { 2010 /* 2011 * OOM. tbin->avail isn't yet filled down to its first 2012 * element, so the successful allocations (if any) must 2013 * be moved to the base of tbin->avail before bailing 2014 * out. 2015 */ 2016 if (i > 0) { 2017 memmove(tbin->avail, &tbin->avail[nfill - i], 2018 i * sizeof(void *)); 2019 } 2020 break; 2021 } 2022 if (config_fill && unlikely(opt_junk_alloc)) { 2023 arena_alloc_junk_small(ptr, &arena_bin_info[binind], 2024 true); 2025 } 2026 /* Insert such that low regions get used first. */ 2027 tbin->avail[nfill - 1 - i] = ptr; 2028 } 2029 if (config_stats) { 2030 bin->stats.nmalloc += i; 2031 bin->stats.nrequests += tbin->tstats.nrequests; 2032 bin->stats.curregs += i; 2033 bin->stats.nfills++; 2034 tbin->tstats.nrequests = 0; 2035 } 2036 malloc_mutex_unlock(&bin->lock); 2037 tbin->ncached = i; 2038 } 2039 2040 void 2041 arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 2042 { 2043 2044 if (zero) { 2045 size_t redzone_size = bin_info->redzone_size; 2046 memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, 2047 redzone_size); 2048 memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, 2049 redzone_size); 2050 } else { 2051 memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, 2052 bin_info->reg_interval); 2053 } 2054 } 2055 2056 #ifdef JEMALLOC_JET 2057 #undef arena_redzone_corruption 2058 #define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl) 2059 #endif 2060 static void 2061 arena_redzone_corruption(void *ptr, size_t usize, bool after, 2062 size_t offset, uint8_t byte) 2063 { 2064 2065 malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p " 2066 "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", 2067 after ? "after" : "before", ptr, usize, byte); 2068 } 2069 #ifdef JEMALLOC_JET 2070 #undef arena_redzone_corruption 2071 #define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) 2072 arena_redzone_corruption_t *arena_redzone_corruption = 2073 JEMALLOC_N(arena_redzone_corruption_impl); 2074 #endif 2075 2076 static void 2077 arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) 2078 { 2079 bool error = false; 2080 2081 if (opt_junk_alloc) { 2082 size_t size = bin_info->reg_size; 2083 size_t redzone_size = bin_info->redzone_size; 2084 size_t i; 2085 2086 for (i = 1; i <= redzone_size; i++) { 2087 uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); 2088 if (*byte != 0xa5) { 2089 error = true; 2090 arena_redzone_corruption(ptr, size, false, i, 2091 *byte); 2092 if (reset) 2093 *byte = 0xa5; 2094 } 2095 } 2096 for (i = 0; i < redzone_size; i++) { 2097 uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); 2098 if (*byte != 0xa5) { 2099 error = true; 2100 arena_redzone_corruption(ptr, size, true, i, 2101 *byte); 2102 if (reset) 2103 *byte = 0xa5; 2104 } 2105 } 2106 } 2107 2108 if (opt_abort && error) 2109 abort(); 2110 } 2111 2112 #ifdef JEMALLOC_JET 2113 #undef arena_dalloc_junk_small 2114 #define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl) 2115 #endif 2116 void 2117 arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 2118 { 2119 size_t redzone_size = bin_info->redzone_size; 2120 2121 arena_redzones_validate(ptr, bin_info, false); 2122 memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, 2123 bin_info->reg_interval); 2124 } 2125 #ifdef JEMALLOC_JET 2126 #undef arena_dalloc_junk_small 2127 #define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) 2128 arena_dalloc_junk_small_t *arena_dalloc_junk_small = 2129 JEMALLOC_N(arena_dalloc_junk_small_impl); 2130 #endif 2131 2132 void 2133 arena_quarantine_junk_small(void *ptr, size_t usize) 2134 { 2135 szind_t binind; 2136 arena_bin_info_t *bin_info; 2137 cassert(config_fill); 2138 assert(opt_junk_free); 2139 assert(opt_quarantine); 2140 assert(usize <= SMALL_MAXCLASS); 2141 2142 binind = size2index(usize); 2143 bin_info = &arena_bin_info[binind]; 2144 arena_redzones_validate(ptr, bin_info, true); 2145 } 2146 2147 void * 2148 arena_malloc_small(arena_t *arena, size_t size, bool zero) 2149 { 2150 void *ret; 2151 arena_bin_t *bin; 2152 arena_run_t *run; 2153 szind_t binind; 2154 2155 binind = size2index(size); 2156 assert(binind < NBINS); 2157 bin = &arena->bins[binind]; 2158 size = index2size(binind); 2159 2160 malloc_mutex_lock(&bin->lock); 2161 if ((run = bin->runcur) != NULL && run->nfree > 0) 2162 ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 2163 else 2164 ret = arena_bin_malloc_hard(arena, bin); 2165 2166 if (ret == NULL) { 2167 malloc_mutex_unlock(&bin->lock); 2168 return (NULL); 2169 } 2170 2171 if (config_stats) { 2172 bin->stats.nmalloc++; 2173 bin->stats.nrequests++; 2174 bin->stats.curregs++; 2175 } 2176 malloc_mutex_unlock(&bin->lock); 2177 if (config_prof && !isthreaded && arena_prof_accum(arena, size)) 2178 prof_idump(); 2179 2180 if (!zero) { 2181 if (config_fill) { 2182 if (unlikely(opt_junk_alloc)) { 2183 arena_alloc_junk_small(ret, 2184 &arena_bin_info[binind], false); 2185 } else if (unlikely(opt_zero)) 2186 memset(ret, 0, size); 2187 } 2188 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 2189 } else { 2190 if (config_fill && unlikely(opt_junk_alloc)) { 2191 arena_alloc_junk_small(ret, &arena_bin_info[binind], 2192 true); 2193 } 2194 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 2195 memset(ret, 0, size); 2196 } 2197 2198 return (ret); 2199 } 2200 2201 void * 2202 arena_malloc_large(arena_t *arena, size_t size, bool zero) 2203 { 2204 void *ret; 2205 size_t usize; 2206 uintptr_t random_offset; 2207 arena_run_t *run; 2208 arena_chunk_map_misc_t *miscelm; 2209 UNUSED bool idump; 2210 2211 /* Large allocation. */ 2212 usize = s2u(size); 2213 malloc_mutex_lock(&arena->lock); 2214 if (config_cache_oblivious) { 2215 uint64_t r; 2216 2217 /* 2218 * Compute a uniformly distributed offset within the first page 2219 * that is a multiple of the cacheline size, e.g. [0 .. 63) * 64 2220 * for 4 KiB pages and 64-byte cachelines. 2221 */ 2222 prng64(r, LG_PAGE - LG_CACHELINE, arena->offset_state, 2223 UINT64_C(6364136223846793009), 2224 UINT64_C(1442695040888963409)); 2225 random_offset = ((uintptr_t)r) << LG_CACHELINE; 2226 } else 2227 random_offset = 0; 2228 run = arena_run_alloc_large(arena, usize + large_pad, zero); 2229 if (run == NULL) { 2230 malloc_mutex_unlock(&arena->lock); 2231 return (NULL); 2232 } 2233 miscelm = arena_run_to_miscelm(run); 2234 ret = (void *)((uintptr_t)arena_miscelm_to_rpages(miscelm) + 2235 random_offset); 2236 if (config_stats) { 2237 szind_t index = size2index(usize) - NBINS; 2238 2239 arena->stats.nmalloc_large++; 2240 arena->stats.nrequests_large++; 2241 arena->stats.allocated_large += usize; 2242 arena->stats.lstats[index].nmalloc++; 2243 arena->stats.lstats[index].nrequests++; 2244 arena->stats.lstats[index].curruns++; 2245 } 2246 if (config_prof) 2247 idump = arena_prof_accum_locked(arena, usize); 2248 malloc_mutex_unlock(&arena->lock); 2249 if (config_prof && idump) 2250 prof_idump(); 2251 2252 if (!zero) { 2253 if (config_fill) { 2254 if (unlikely(opt_junk_alloc)) 2255 memset(ret, 0xa5, usize); 2256 else if (unlikely(opt_zero)) 2257 memset(ret, 0, usize); 2258 } 2259 } 2260 2261 return (ret); 2262 } 2263 2264 /* Only handles large allocations that require more than page alignment. */ 2265 static void * 2266 arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, 2267 bool zero) 2268 { 2269 void *ret; 2270 size_t alloc_size, leadsize, trailsize; 2271 arena_run_t *run; 2272 arena_chunk_t *chunk; 2273 arena_chunk_map_misc_t *miscelm; 2274 void *rpages; 2275 2276 assert(usize == PAGE_CEILING(usize)); 2277 2278 arena = arena_choose(tsd, arena); 2279 if (unlikely(arena == NULL)) 2280 return (NULL); 2281 2282 alignment = PAGE_CEILING(alignment); 2283 alloc_size = usize + large_pad + alignment - PAGE; 2284 2285 malloc_mutex_lock(&arena->lock); 2286 run = arena_run_alloc_large(arena, alloc_size, false); 2287 if (run == NULL) { 2288 malloc_mutex_unlock(&arena->lock); 2289 return (NULL); 2290 } 2291 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 2292 miscelm = arena_run_to_miscelm(run); 2293 rpages = arena_miscelm_to_rpages(miscelm); 2294 2295 leadsize = ALIGNMENT_CEILING((uintptr_t)rpages, alignment) - 2296 (uintptr_t)rpages; 2297 assert(alloc_size >= leadsize + usize); 2298 trailsize = alloc_size - leadsize - usize - large_pad; 2299 if (leadsize != 0) { 2300 arena_chunk_map_misc_t *head_miscelm = miscelm; 2301 arena_run_t *head_run = run; 2302 2303 miscelm = arena_miscelm_get(chunk, 2304 arena_miscelm_to_pageind(head_miscelm) + (leadsize >> 2305 LG_PAGE)); 2306 run = &miscelm->run; 2307 2308 arena_run_trim_head(arena, chunk, head_run, alloc_size, 2309 alloc_size - leadsize); 2310 } 2311 if (trailsize != 0) { 2312 arena_run_trim_tail(arena, chunk, run, usize + large_pad + 2313 trailsize, usize + large_pad, false); 2314 } 2315 if (arena_run_init_large(arena, run, usize + large_pad, zero)) { 2316 size_t run_ind = 2317 arena_miscelm_to_pageind(arena_run_to_miscelm(run)); 2318 bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0); 2319 bool decommitted = (arena_mapbits_decommitted_get(chunk, 2320 run_ind) != 0); 2321 2322 assert(decommitted); /* Cause of OOM. */ 2323 arena_run_dalloc(arena, run, dirty, false, decommitted); 2324 malloc_mutex_unlock(&arena->lock); 2325 return (NULL); 2326 } 2327 ret = arena_miscelm_to_rpages(miscelm); 2328 2329 if (config_stats) { 2330 szind_t index = size2index(usize) - NBINS; 2331 2332 arena->stats.nmalloc_large++; 2333 arena->stats.nrequests_large++; 2334 arena->stats.allocated_large += usize; 2335 arena->stats.lstats[index].nmalloc++; 2336 arena->stats.lstats[index].nrequests++; 2337 arena->stats.lstats[index].curruns++; 2338 } 2339 malloc_mutex_unlock(&arena->lock); 2340 2341 if (config_fill && !zero) { 2342 if (unlikely(opt_junk_alloc)) 2343 memset(ret, 0xa5, usize); 2344 else if (unlikely(opt_zero)) 2345 memset(ret, 0, usize); 2346 } 2347 return (ret); 2348 } 2349 2350 void * 2351 arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, 2352 bool zero, tcache_t *tcache) 2353 { 2354 void *ret; 2355 2356 if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE 2357 && (usize & PAGE_MASK) == 0))) { 2358 /* Small; alignment doesn't require special run placement. */ 2359 ret = arena_malloc(tsd, arena, usize, zero, tcache); 2360 } else if (usize <= large_maxclass && alignment <= PAGE) { 2361 /* 2362 * Large; alignment doesn't require special run placement. 2363 * However, the cached pointer may be at a random offset from 2364 * the base of the run, so do some bit manipulation to retrieve 2365 * the base. 2366 */ 2367 ret = arena_malloc(tsd, arena, usize, zero, tcache); 2368 if (config_cache_oblivious) 2369 ret = (void *)((uintptr_t)ret & ~PAGE_MASK); 2370 } else { 2371 if (likely(usize <= large_maxclass)) { 2372 ret = arena_palloc_large(tsd, arena, usize, alignment, 2373 zero); 2374 } else if (likely(alignment <= chunksize)) 2375 ret = huge_malloc(tsd, arena, usize, zero, tcache); 2376 else { 2377 ret = huge_palloc(tsd, arena, usize, alignment, zero, 2378 tcache); 2379 } 2380 } 2381 return (ret); 2382 } 2383 2384 void 2385 arena_prof_promoted(const void *ptr, size_t size) 2386 { 2387 arena_chunk_t *chunk; 2388 size_t pageind; 2389 szind_t binind; 2390 2391 cassert(config_prof); 2392 assert(ptr != NULL); 2393 assert(CHUNK_ADDR2BASE(ptr) != ptr); 2394 assert(isalloc(ptr, false) == LARGE_MINCLASS); 2395 assert(isalloc(ptr, true) == LARGE_MINCLASS); 2396 assert(size <= SMALL_MAXCLASS); 2397 2398 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2399 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2400 binind = size2index(size); 2401 assert(binind < NBINS); 2402 arena_mapbits_large_binind_set(chunk, pageind, binind); 2403 2404 assert(isalloc(ptr, false) == LARGE_MINCLASS); 2405 assert(isalloc(ptr, true) == size); 2406 } 2407 2408 static void 2409 arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 2410 arena_bin_t *bin) 2411 { 2412 2413 /* Dissociate run from bin. */ 2414 if (run == bin->runcur) 2415 bin->runcur = NULL; 2416 else { 2417 szind_t binind = arena_bin_index(extent_node_arena_get( 2418 &chunk->node), bin); 2419 arena_bin_info_t *bin_info = &arena_bin_info[binind]; 2420 2421 if (bin_info->nregs != 1) { 2422 /* 2423 * This block's conditional is necessary because if the 2424 * run only contains one region, then it never gets 2425 * inserted into the non-full runs tree. 2426 */ 2427 arena_bin_runs_remove(bin, run); 2428 } 2429 } 2430 } 2431 2432 static void 2433 arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 2434 arena_bin_t *bin) 2435 { 2436 2437 assert(run != bin->runcur); 2438 assert(arena_run_tree_search(&bin->runs, arena_run_to_miscelm(run)) == 2439 NULL); 2440 2441 malloc_mutex_unlock(&bin->lock); 2442 /******************************/ 2443 malloc_mutex_lock(&arena->lock); 2444 arena_run_dalloc_decommit(arena, chunk, run); 2445 malloc_mutex_unlock(&arena->lock); 2446 /****************************/ 2447 malloc_mutex_lock(&bin->lock); 2448 if (config_stats) 2449 bin->stats.curruns--; 2450 } 2451 2452 static void 2453 arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 2454 arena_bin_t *bin) 2455 { 2456 2457 /* 2458 * Make sure that if bin->runcur is non-NULL, it refers to the lowest 2459 * non-full run. It is okay to NULL runcur out rather than proactively 2460 * keeping it pointing at the lowest non-full run. 2461 */ 2462 if ((uintptr_t)run < (uintptr_t)bin->runcur) { 2463 /* Switch runcur. */ 2464 if (bin->runcur->nfree > 0) 2465 arena_bin_runs_insert(bin, bin->runcur); 2466 bin->runcur = run; 2467 if (config_stats) 2468 bin->stats.reruns++; 2469 } else 2470 arena_bin_runs_insert(bin, run); 2471 } 2472 2473 static void 2474 arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2475 arena_chunk_map_bits_t *bitselm, bool junked) 2476 { 2477 size_t pageind, rpages_ind; 2478 arena_run_t *run; 2479 arena_bin_t *bin; 2480 arena_bin_info_t *bin_info; 2481 szind_t binind; 2482 2483 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2484 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); 2485 run = &arena_miscelm_get(chunk, rpages_ind)->run; 2486 binind = run->binind; 2487 bin = &arena->bins[binind]; 2488 bin_info = &arena_bin_info[binind]; 2489 2490 if (!junked && config_fill && unlikely(opt_junk_free)) 2491 arena_dalloc_junk_small(ptr, bin_info); 2492 2493 arena_run_reg_dalloc(run, ptr); 2494 if (run->nfree == bin_info->nregs) { 2495 arena_dissociate_bin_run(chunk, run, bin); 2496 arena_dalloc_bin_run(arena, chunk, run, bin); 2497 } else if (run->nfree == 1 && run != bin->runcur) 2498 arena_bin_lower_run(arena, chunk, run, bin); 2499 2500 if (config_stats) { 2501 bin->stats.ndalloc++; 2502 bin->stats.curregs--; 2503 } 2504 } 2505 2506 void 2507 arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2508 arena_chunk_map_bits_t *bitselm) 2509 { 2510 2511 arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, true); 2512 } 2513 2514 void 2515 arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2516 size_t pageind, arena_chunk_map_bits_t *bitselm) 2517 { 2518 arena_run_t *run; 2519 arena_bin_t *bin; 2520 size_t rpages_ind; 2521 2522 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); 2523 run = &arena_miscelm_get(chunk, rpages_ind)->run; 2524 bin = &arena->bins[run->binind]; 2525 malloc_mutex_lock(&bin->lock); 2526 arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, false); 2527 malloc_mutex_unlock(&bin->lock); 2528 } 2529 2530 void 2531 arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2532 size_t pageind) 2533 { 2534 arena_chunk_map_bits_t *bitselm; 2535 2536 if (config_debug) { 2537 /* arena_ptr_small_binind_get() does extra sanity checking. */ 2538 assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 2539 pageind)) != BININD_INVALID); 2540 } 2541 bitselm = arena_bitselm_get(chunk, pageind); 2542 arena_dalloc_bin(arena, chunk, ptr, pageind, bitselm); 2543 } 2544 2545 #ifdef JEMALLOC_JET 2546 #undef arena_dalloc_junk_large 2547 #define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl) 2548 #endif 2549 void 2550 arena_dalloc_junk_large(void *ptr, size_t usize) 2551 { 2552 2553 if (config_fill && unlikely(opt_junk_free)) 2554 memset(ptr, 0x5a, usize); 2555 } 2556 #ifdef JEMALLOC_JET 2557 #undef arena_dalloc_junk_large 2558 #define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) 2559 arena_dalloc_junk_large_t *arena_dalloc_junk_large = 2560 JEMALLOC_N(arena_dalloc_junk_large_impl); 2561 #endif 2562 2563 static void 2564 arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk, 2565 void *ptr, bool junked) 2566 { 2567 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2568 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); 2569 arena_run_t *run = &miscelm->run; 2570 2571 if (config_fill || config_stats) { 2572 size_t usize = arena_mapbits_large_size_get(chunk, pageind) - 2573 large_pad; 2574 2575 if (!junked) 2576 arena_dalloc_junk_large(ptr, usize); 2577 if (config_stats) { 2578 szind_t index = size2index(usize) - NBINS; 2579 2580 arena->stats.ndalloc_large++; 2581 arena->stats.allocated_large -= usize; 2582 arena->stats.lstats[index].ndalloc++; 2583 arena->stats.lstats[index].curruns--; 2584 } 2585 } 2586 2587 arena_run_dalloc_decommit(arena, chunk, run); 2588 } 2589 2590 void 2591 arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk, 2592 void *ptr) 2593 { 2594 2595 arena_dalloc_large_locked_impl(arena, chunk, ptr, true); 2596 } 2597 2598 void 2599 arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 2600 { 2601 2602 malloc_mutex_lock(&arena->lock); 2603 arena_dalloc_large_locked_impl(arena, chunk, ptr, false); 2604 malloc_mutex_unlock(&arena->lock); 2605 } 2606 2607 static void 2608 arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2609 size_t oldsize, size_t size) 2610 { 2611 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2612 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); 2613 arena_run_t *run = &miscelm->run; 2614 2615 assert(size < oldsize); 2616 2617 /* 2618 * Shrink the run, and make trailing pages available for other 2619 * allocations. 2620 */ 2621 malloc_mutex_lock(&arena->lock); 2622 arena_run_trim_tail(arena, chunk, run, oldsize + large_pad, size + 2623 large_pad, true); 2624 if (config_stats) { 2625 szind_t oldindex = size2index(oldsize) - NBINS; 2626 szind_t index = size2index(size) - NBINS; 2627 2628 arena->stats.ndalloc_large++; 2629 arena->stats.allocated_large -= oldsize; 2630 arena->stats.lstats[oldindex].ndalloc++; 2631 arena->stats.lstats[oldindex].curruns--; 2632 2633 arena->stats.nmalloc_large++; 2634 arena->stats.nrequests_large++; 2635 arena->stats.allocated_large += size; 2636 arena->stats.lstats[index].nmalloc++; 2637 arena->stats.lstats[index].nrequests++; 2638 arena->stats.lstats[index].curruns++; 2639 } 2640 malloc_mutex_unlock(&arena->lock); 2641 } 2642 2643 static bool 2644 arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2645 size_t oldsize, size_t usize_min, size_t usize_max, bool zero) 2646 { 2647 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2648 size_t npages = (oldsize + large_pad) >> LG_PAGE; 2649 size_t followsize; 2650 2651 assert(oldsize == arena_mapbits_large_size_get(chunk, pageind) - 2652 large_pad); 2653 2654 /* Try to extend the run. */ 2655 malloc_mutex_lock(&arena->lock); 2656 if (pageind+npages >= chunk_npages || arena_mapbits_allocated_get(chunk, 2657 pageind+npages) != 0) 2658 goto label_fail; 2659 followsize = arena_mapbits_unallocated_size_get(chunk, pageind+npages); 2660 if (oldsize + followsize >= usize_min) { 2661 /* 2662 * The next run is available and sufficiently large. Split the 2663 * following run, then merge the first part with the existing 2664 * allocation. 2665 */ 2666 arena_run_t *run; 2667 size_t usize, splitsize, size, flag_dirty, flag_unzeroed_mask; 2668 2669 usize = usize_max; 2670 while (oldsize + followsize < usize) 2671 usize = index2size(size2index(usize)-1); 2672 assert(usize >= usize_min); 2673 assert(usize >= oldsize); 2674 splitsize = usize - oldsize; 2675 if (splitsize == 0) 2676 goto label_fail; 2677 2678 run = &arena_miscelm_get(chunk, pageind+npages)->run; 2679 if (arena_run_split_large(arena, run, splitsize, zero)) 2680 goto label_fail; 2681 2682 size = oldsize + splitsize; 2683 npages = (size + large_pad) >> LG_PAGE; 2684 2685 /* 2686 * Mark the extended run as dirty if either portion of the run 2687 * was dirty before allocation. This is rather pedantic, 2688 * because there's not actually any sequence of events that 2689 * could cause the resulting run to be passed to 2690 * arena_run_dalloc() with the dirty argument set to false 2691 * (which is when dirty flag consistency would really matter). 2692 */ 2693 flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 2694 arena_mapbits_dirty_get(chunk, pageind+npages-1); 2695 flag_unzeroed_mask = flag_dirty == 0 ? CHUNK_MAP_UNZEROED : 0; 2696 arena_mapbits_large_set(chunk, pageind, size + large_pad, 2697 flag_dirty | (flag_unzeroed_mask & 2698 arena_mapbits_unzeroed_get(chunk, pageind))); 2699 arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty | 2700 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, 2701 pageind+npages-1))); 2702 2703 if (config_stats) { 2704 szind_t oldindex = size2index(oldsize) - NBINS; 2705 szind_t index = size2index(size) - NBINS; 2706 2707 arena->stats.ndalloc_large++; 2708 arena->stats.allocated_large -= oldsize; 2709 arena->stats.lstats[oldindex].ndalloc++; 2710 arena->stats.lstats[oldindex].curruns--; 2711 2712 arena->stats.nmalloc_large++; 2713 arena->stats.nrequests_large++; 2714 arena->stats.allocated_large += size; 2715 arena->stats.lstats[index].nmalloc++; 2716 arena->stats.lstats[index].nrequests++; 2717 arena->stats.lstats[index].curruns++; 2718 } 2719 malloc_mutex_unlock(&arena->lock); 2720 return (false); 2721 } 2722 label_fail: 2723 malloc_mutex_unlock(&arena->lock); 2724 return (true); 2725 } 2726 2727 #ifdef JEMALLOC_JET 2728 #undef arena_ralloc_junk_large 2729 #define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl) 2730 #endif 2731 static void 2732 arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) 2733 { 2734 2735 if (config_fill && unlikely(opt_junk_free)) { 2736 memset((void *)((uintptr_t)ptr + usize), 0x5a, 2737 old_usize - usize); 2738 } 2739 } 2740 #ifdef JEMALLOC_JET 2741 #undef arena_ralloc_junk_large 2742 #define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) 2743 arena_ralloc_junk_large_t *arena_ralloc_junk_large = 2744 JEMALLOC_N(arena_ralloc_junk_large_impl); 2745 #endif 2746 2747 /* 2748 * Try to resize a large allocation, in order to avoid copying. This will 2749 * always fail if growing an object, and the following run is already in use. 2750 */ 2751 static bool 2752 arena_ralloc_large(void *ptr, size_t oldsize, size_t usize_min, 2753 size_t usize_max, bool zero) 2754 { 2755 arena_chunk_t *chunk; 2756 arena_t *arena; 2757 2758 if (oldsize == usize_max) { 2759 /* Current size class is compatible and maximal. */ 2760 return (false); 2761 } 2762 2763 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2764 arena = extent_node_arena_get(&chunk->node); 2765 2766 if (oldsize < usize_max) { 2767 bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize, 2768 usize_min, usize_max, zero); 2769 if (config_fill && !ret && !zero) { 2770 if (unlikely(opt_junk_alloc)) { 2771 memset((void *)((uintptr_t)ptr + oldsize), 0xa5, 2772 isalloc(ptr, config_prof) - oldsize); 2773 } else if (unlikely(opt_zero)) { 2774 memset((void *)((uintptr_t)ptr + oldsize), 0, 2775 isalloc(ptr, config_prof) - oldsize); 2776 } 2777 } 2778 return (ret); 2779 } 2780 2781 assert(oldsize > usize_max); 2782 /* Fill before shrinking in order avoid a race. */ 2783 arena_ralloc_junk_large(ptr, oldsize, usize_max); 2784 arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, usize_max); 2785 return (false); 2786 } 2787 2788 bool 2789 arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 2790 bool zero) 2791 { 2792 size_t usize_min, usize_max; 2793 2794 usize_min = s2u(size); 2795 usize_max = s2u(size + extra); 2796 if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) { 2797 /* 2798 * Avoid moving the allocation if the size class can be left the 2799 * same. 2800 */ 2801 if (oldsize <= SMALL_MAXCLASS) { 2802 assert(arena_bin_info[size2index(oldsize)].reg_size == 2803 oldsize); 2804 if ((usize_max <= SMALL_MAXCLASS && 2805 size2index(usize_max) == size2index(oldsize)) || 2806 (size <= oldsize && usize_max >= oldsize)) 2807 return (false); 2808 } else { 2809 if (usize_max > SMALL_MAXCLASS) { 2810 if (!arena_ralloc_large(ptr, oldsize, usize_min, 2811 usize_max, zero)) 2812 return (false); 2813 } 2814 } 2815 2816 /* Reallocation would require a move. */ 2817 return (true); 2818 } else { 2819 return (huge_ralloc_no_move(ptr, oldsize, usize_min, usize_max, 2820 zero)); 2821 } 2822 } 2823 2824 static void * 2825 arena_ralloc_move_helper(tsd_t *tsd, arena_t *arena, size_t usize, 2826 size_t alignment, bool zero, tcache_t *tcache) 2827 { 2828 2829 if (alignment == 0) 2830 return (arena_malloc(tsd, arena, usize, zero, tcache)); 2831 usize = sa2u(usize, alignment); 2832 if (usize == 0) 2833 return (NULL); 2834 return (ipalloct(tsd, usize, alignment, zero, tcache, arena)); 2835 } 2836 2837 void * 2838 arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, 2839 size_t alignment, bool zero, tcache_t *tcache) 2840 { 2841 void *ret; 2842 size_t usize; 2843 2844 usize = s2u(size); 2845 if (usize == 0) 2846 return (NULL); 2847 2848 if (likely(usize <= large_maxclass)) { 2849 size_t copysize; 2850 2851 /* Try to avoid moving the allocation. */ 2852 if (!arena_ralloc_no_move(ptr, oldsize, usize, 0, zero)) 2853 return (ptr); 2854 2855 /* 2856 * size and oldsize are different enough that we need to move 2857 * the object. In that case, fall back to allocating new space 2858 * and copying. 2859 */ 2860 ret = arena_ralloc_move_helper(tsd, arena, usize, alignment, 2861 zero, tcache); 2862 if (ret == NULL) 2863 return (NULL); 2864 2865 /* 2866 * Junk/zero-filling were already done by 2867 * ipalloc()/arena_malloc(). 2868 */ 2869 2870 copysize = (usize < oldsize) ? usize : oldsize; 2871 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 2872 memcpy(ret, ptr, copysize); 2873 isqalloc(tsd, ptr, oldsize, tcache); 2874 } else { 2875 ret = huge_ralloc(tsd, arena, ptr, oldsize, usize, alignment, 2876 zero, tcache); 2877 } 2878 return (ret); 2879 } 2880 2881 dss_prec_t 2882 arena_dss_prec_get(arena_t *arena) 2883 { 2884 dss_prec_t ret; 2885 2886 malloc_mutex_lock(&arena->lock); 2887 ret = arena->dss_prec; 2888 malloc_mutex_unlock(&arena->lock); 2889 return (ret); 2890 } 2891 2892 bool 2893 arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) 2894 { 2895 2896 if (!have_dss) 2897 return (dss_prec != dss_prec_disabled); 2898 malloc_mutex_lock(&arena->lock); 2899 arena->dss_prec = dss_prec; 2900 malloc_mutex_unlock(&arena->lock); 2901 return (false); 2902 } 2903 2904 ssize_t 2905 arena_lg_dirty_mult_default_get(void) 2906 { 2907 2908 return ((ssize_t)atomic_read_z((size_t *)&lg_dirty_mult_default)); 2909 } 2910 2911 bool 2912 arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult) 2913 { 2914 2915 if (!arena_lg_dirty_mult_valid(lg_dirty_mult)) 2916 return (true); 2917 atomic_write_z((size_t *)&lg_dirty_mult_default, (size_t)lg_dirty_mult); 2918 return (false); 2919 } 2920 2921 void 2922 arena_stats_merge(arena_t *arena, const char **dss, ssize_t *lg_dirty_mult, 2923 size_t *nactive, size_t *ndirty, arena_stats_t *astats, 2924 malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats, 2925 malloc_huge_stats_t *hstats) 2926 { 2927 unsigned i; 2928 2929 malloc_mutex_lock(&arena->lock); 2930 *dss = dss_prec_names[arena->dss_prec]; 2931 *lg_dirty_mult = arena->lg_dirty_mult; 2932 *nactive += arena->nactive; 2933 *ndirty += arena->ndirty; 2934 2935 astats->mapped += arena->stats.mapped; 2936 astats->npurge += arena->stats.npurge; 2937 astats->nmadvise += arena->stats.nmadvise; 2938 astats->purged += arena->stats.purged; 2939 astats->metadata_mapped += arena->stats.metadata_mapped; 2940 astats->metadata_allocated += arena_metadata_allocated_get(arena); 2941 astats->allocated_large += arena->stats.allocated_large; 2942 astats->nmalloc_large += arena->stats.nmalloc_large; 2943 astats->ndalloc_large += arena->stats.ndalloc_large; 2944 astats->nrequests_large += arena->stats.nrequests_large; 2945 astats->allocated_huge += arena->stats.allocated_huge; 2946 astats->nmalloc_huge += arena->stats.nmalloc_huge; 2947 astats->ndalloc_huge += arena->stats.ndalloc_huge; 2948 2949 for (i = 0; i < nlclasses; i++) { 2950 lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 2951 lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 2952 lstats[i].nrequests += arena->stats.lstats[i].nrequests; 2953 lstats[i].curruns += arena->stats.lstats[i].curruns; 2954 } 2955 2956 for (i = 0; i < nhclasses; i++) { 2957 hstats[i].nmalloc += arena->stats.hstats[i].nmalloc; 2958 hstats[i].ndalloc += arena->stats.hstats[i].ndalloc; 2959 hstats[i].curhchunks += arena->stats.hstats[i].curhchunks; 2960 } 2961 malloc_mutex_unlock(&arena->lock); 2962 2963 for (i = 0; i < NBINS; i++) { 2964 arena_bin_t *bin = &arena->bins[i]; 2965 2966 malloc_mutex_lock(&bin->lock); 2967 bstats[i].nmalloc += bin->stats.nmalloc; 2968 bstats[i].ndalloc += bin->stats.ndalloc; 2969 bstats[i].nrequests += bin->stats.nrequests; 2970 bstats[i].curregs += bin->stats.curregs; 2971 if (config_tcache) { 2972 bstats[i].nfills += bin->stats.nfills; 2973 bstats[i].nflushes += bin->stats.nflushes; 2974 } 2975 bstats[i].nruns += bin->stats.nruns; 2976 bstats[i].reruns += bin->stats.reruns; 2977 bstats[i].curruns += bin->stats.curruns; 2978 malloc_mutex_unlock(&bin->lock); 2979 } 2980 } 2981 2982 arena_t * 2983 arena_new(unsigned ind) 2984 { 2985 arena_t *arena; 2986 unsigned i; 2987 arena_bin_t *bin; 2988 2989 /* 2990 * Allocate arena, arena->lstats, and arena->hstats contiguously, mainly 2991 * because there is no way to clean up if base_alloc() OOMs. 2992 */ 2993 if (config_stats) { 2994 arena = (arena_t *)base_alloc(CACHELINE_CEILING(sizeof(arena_t)) 2995 + QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t) + 2996 nhclasses) * sizeof(malloc_huge_stats_t)); 2997 } else 2998 arena = (arena_t *)base_alloc(sizeof(arena_t)); 2999 if (arena == NULL) 3000 return (NULL); 3001 3002 arena->ind = ind; 3003 arena->nthreads = 0; 3004 if (malloc_mutex_init(&arena->lock)) 3005 return (NULL); 3006 3007 if (config_stats) { 3008 memset(&arena->stats, 0, sizeof(arena_stats_t)); 3009 arena->stats.lstats = (malloc_large_stats_t *)((uintptr_t)arena 3010 + CACHELINE_CEILING(sizeof(arena_t))); 3011 memset(arena->stats.lstats, 0, nlclasses * 3012 sizeof(malloc_large_stats_t)); 3013 arena->stats.hstats = (malloc_huge_stats_t *)((uintptr_t)arena 3014 + CACHELINE_CEILING(sizeof(arena_t)) + 3015 QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t))); 3016 memset(arena->stats.hstats, 0, nhclasses * 3017 sizeof(malloc_huge_stats_t)); 3018 if (config_tcache) 3019 ql_new(&arena->tcache_ql); 3020 } 3021 3022 if (config_prof) 3023 arena->prof_accumbytes = 0; 3024 3025 if (config_cache_oblivious) { 3026 /* 3027 * A nondeterministic seed based on the address of arena reduces 3028 * the likelihood of lockstep non-uniform cache index 3029 * utilization among identical concurrent processes, but at the 3030 * cost of test repeatability. For debug builds, instead use a 3031 * deterministic seed. 3032 */ 3033 arena->offset_state = config_debug ? ind : 3034 (uint64_t)(uintptr_t)arena; 3035 } 3036 3037 arena->dss_prec = chunk_dss_prec_get(); 3038 3039 arena->spare = NULL; 3040 3041 arena->lg_dirty_mult = arena_lg_dirty_mult_default_get(); 3042 arena->purging = false; 3043 arena->nactive = 0; 3044 arena->ndirty = 0; 3045 3046 arena_avail_tree_new(&arena->runs_avail); 3047 qr_new(&arena->runs_dirty, rd_link); 3048 qr_new(&arena->chunks_cache, cc_link); 3049 3050 ql_new(&arena->huge); 3051 if (malloc_mutex_init(&arena->huge_mtx)) 3052 return (NULL); 3053 3054 extent_tree_szad_new(&arena->chunks_szad_cached); 3055 extent_tree_ad_new(&arena->chunks_ad_cached); 3056 extent_tree_szad_new(&arena->chunks_szad_retained); 3057 extent_tree_ad_new(&arena->chunks_ad_retained); 3058 if (malloc_mutex_init(&arena->chunks_mtx)) 3059 return (NULL); 3060 ql_new(&arena->node_cache); 3061 if (malloc_mutex_init(&arena->node_cache_mtx)) 3062 return (NULL); 3063 3064 arena->chunk_hooks = chunk_hooks_default; 3065 3066 /* Initialize bins. */ 3067 for (i = 0; i < NBINS; i++) { 3068 bin = &arena->bins[i]; 3069 if (malloc_mutex_init(&bin->lock)) 3070 return (NULL); 3071 bin->runcur = NULL; 3072 arena_run_tree_new(&bin->runs); 3073 if (config_stats) 3074 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 3075 } 3076 3077 return (arena); 3078 } 3079 3080 /* 3081 * Calculate bin_info->run_size such that it meets the following constraints: 3082 * 3083 * *) bin_info->run_size <= arena_maxrun 3084 * *) bin_info->nregs <= RUN_MAXREGS 3085 * 3086 * bin_info->nregs and bin_info->reg0_offset are also calculated here, since 3087 * these settings are all interdependent. 3088 */ 3089 static void 3090 bin_info_run_size_calc(arena_bin_info_t *bin_info) 3091 { 3092 size_t pad_size; 3093 size_t try_run_size, perfect_run_size, actual_run_size; 3094 uint32_t try_nregs, perfect_nregs, actual_nregs; 3095 3096 /* 3097 * Determine redzone size based on minimum alignment and minimum 3098 * redzone size. Add padding to the end of the run if it is needed to 3099 * align the regions. The padding allows each redzone to be half the 3100 * minimum alignment; without the padding, each redzone would have to 3101 * be twice as large in order to maintain alignment. 3102 */ 3103 if (config_fill && unlikely(opt_redzone)) { 3104 size_t align_min = ZU(1) << (jemalloc_ffs(bin_info->reg_size) - 3105 1); 3106 if (align_min <= REDZONE_MINSIZE) { 3107 bin_info->redzone_size = REDZONE_MINSIZE; 3108 pad_size = 0; 3109 } else { 3110 bin_info->redzone_size = align_min >> 1; 3111 pad_size = bin_info->redzone_size; 3112 } 3113 } else { 3114 bin_info->redzone_size = 0; 3115 pad_size = 0; 3116 } 3117 bin_info->reg_interval = bin_info->reg_size + 3118 (bin_info->redzone_size << 1); 3119 3120 /* 3121 * Compute run size under ideal conditions (no redzones, no limit on run 3122 * size). 3123 */ 3124 try_run_size = PAGE; 3125 try_nregs = try_run_size / bin_info->reg_size; 3126 do { 3127 perfect_run_size = try_run_size; 3128 perfect_nregs = try_nregs; 3129 3130 try_run_size += PAGE; 3131 try_nregs = try_run_size / bin_info->reg_size; 3132 } while (perfect_run_size != perfect_nregs * bin_info->reg_size); 3133 assert(perfect_nregs <= RUN_MAXREGS); 3134 3135 actual_run_size = perfect_run_size; 3136 actual_nregs = (actual_run_size - pad_size) / bin_info->reg_interval; 3137 3138 /* 3139 * Redzones can require enough padding that not even a single region can 3140 * fit within the number of pages that would normally be dedicated to a 3141 * run for this size class. Increase the run size until at least one 3142 * region fits. 3143 */ 3144 while (actual_nregs == 0) { 3145 assert(config_fill && unlikely(opt_redzone)); 3146 3147 actual_run_size += PAGE; 3148 actual_nregs = (actual_run_size - pad_size) / 3149 bin_info->reg_interval; 3150 } 3151 3152 /* 3153 * Make sure that the run will fit within an arena chunk. 3154 */ 3155 while (actual_run_size > arena_maxrun) { 3156 actual_run_size -= PAGE; 3157 actual_nregs = (actual_run_size - pad_size) / 3158 bin_info->reg_interval; 3159 } 3160 assert(actual_nregs > 0); 3161 assert(actual_run_size == s2u(actual_run_size)); 3162 3163 /* Copy final settings. */ 3164 bin_info->run_size = actual_run_size; 3165 bin_info->nregs = actual_nregs; 3166 bin_info->reg0_offset = actual_run_size - (actual_nregs * 3167 bin_info->reg_interval) - pad_size + bin_info->redzone_size; 3168 3169 if (actual_run_size > small_maxrun) 3170 small_maxrun = actual_run_size; 3171 3172 assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 3173 * bin_info->reg_interval) + pad_size == bin_info->run_size); 3174 } 3175 3176 static void 3177 bin_info_init(void) 3178 { 3179 arena_bin_info_t *bin_info; 3180 3181 #define BIN_INFO_INIT_bin_yes(index, size) \ 3182 bin_info = &arena_bin_info[index]; \ 3183 bin_info->reg_size = size; \ 3184 bin_info_run_size_calc(bin_info); \ 3185 bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 3186 #define BIN_INFO_INIT_bin_no(index, size) 3187 #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 3188 BIN_INFO_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)) 3189 SIZE_CLASSES 3190 #undef BIN_INFO_INIT_bin_yes 3191 #undef BIN_INFO_INIT_bin_no 3192 #undef SC 3193 } 3194 3195 static bool 3196 small_run_size_init(void) 3197 { 3198 3199 assert(small_maxrun != 0); 3200 3201 small_run_tab = (bool *)base_alloc(sizeof(bool) * (small_maxrun >> 3202 LG_PAGE)); 3203 if (small_run_tab == NULL) 3204 return (true); 3205 3206 #define TAB_INIT_bin_yes(index, size) { \ 3207 arena_bin_info_t *bin_info = &arena_bin_info[index]; \ 3208 small_run_tab[bin_info->run_size >> LG_PAGE] = true; \ 3209 } 3210 #define TAB_INIT_bin_no(index, size) 3211 #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 3212 TAB_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)) 3213 SIZE_CLASSES 3214 #undef TAB_INIT_bin_yes 3215 #undef TAB_INIT_bin_no 3216 #undef SC 3217 3218 return (false); 3219 } 3220 3221 bool 3222 arena_boot(void) 3223 { 3224 unsigned i; 3225 3226 arena_lg_dirty_mult_default_set(opt_lg_dirty_mult); 3227 3228 /* 3229 * Compute the header size such that it is large enough to contain the 3230 * page map. The page map is biased to omit entries for the header 3231 * itself, so some iteration is necessary to compute the map bias. 3232 * 3233 * 1) Compute safe header_size and map_bias values that include enough 3234 * space for an unbiased page map. 3235 * 2) Refine map_bias based on (1) to omit the header pages in the page 3236 * map. The resulting map_bias may be one too small. 3237 * 3) Refine map_bias based on (2). The result will be >= the result 3238 * from (2), and will always be correct. 3239 */ 3240 map_bias = 0; 3241 for (i = 0; i < 3; i++) { 3242 size_t header_size = offsetof(arena_chunk_t, map_bits) + 3243 ((sizeof(arena_chunk_map_bits_t) + 3244 sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias)); 3245 map_bias = (header_size + PAGE_MASK) >> LG_PAGE; 3246 } 3247 assert(map_bias > 0); 3248 3249 map_misc_offset = offsetof(arena_chunk_t, map_bits) + 3250 sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias); 3251 3252 arena_maxrun = chunksize - (map_bias << LG_PAGE); 3253 assert(arena_maxrun > 0); 3254 large_maxclass = index2size(size2index(chunksize)-1); 3255 if (large_maxclass > arena_maxrun) { 3256 /* 3257 * For small chunk sizes it's possible for there to be fewer 3258 * non-header pages available than are necessary to serve the 3259 * size classes just below chunksize. 3260 */ 3261 large_maxclass = arena_maxrun; 3262 } 3263 assert(large_maxclass > 0); 3264 nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS); 3265 nhclasses = NSIZES - nlclasses - NBINS; 3266 3267 bin_info_init(); 3268 return (small_run_size_init()); 3269 } 3270 3271 void 3272 arena_prefork(arena_t *arena) 3273 { 3274 unsigned i; 3275 3276 malloc_mutex_prefork(&arena->lock); 3277 malloc_mutex_prefork(&arena->huge_mtx); 3278 malloc_mutex_prefork(&arena->chunks_mtx); 3279 malloc_mutex_prefork(&arena->node_cache_mtx); 3280 for (i = 0; i < NBINS; i++) 3281 malloc_mutex_prefork(&arena->bins[i].lock); 3282 } 3283 3284 void 3285 arena_postfork_parent(arena_t *arena) 3286 { 3287 unsigned i; 3288 3289 for (i = 0; i < NBINS; i++) 3290 malloc_mutex_postfork_parent(&arena->bins[i].lock); 3291 malloc_mutex_postfork_parent(&arena->node_cache_mtx); 3292 malloc_mutex_postfork_parent(&arena->chunks_mtx); 3293 malloc_mutex_postfork_parent(&arena->huge_mtx); 3294 malloc_mutex_postfork_parent(&arena->lock); 3295 } 3296 3297 void 3298 arena_postfork_child(arena_t *arena) 3299 { 3300 unsigned i; 3301 3302 for (i = 0; i < NBINS; i++) 3303 malloc_mutex_postfork_child(&arena->bins[i].lock); 3304 malloc_mutex_postfork_child(&arena->node_cache_mtx); 3305 malloc_mutex_postfork_child(&arena->chunks_mtx); 3306 malloc_mutex_postfork_child(&arena->huge_mtx); 3307 malloc_mutex_postfork_child(&arena->lock); 3308 } 3309