1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/bit_util.h" 5 #include "jemalloc/internal/cache_bin.h" 6 #include "jemalloc/internal/safety_check.h" 7 8 void 9 cache_bin_info_init(cache_bin_info_t *info, 10 cache_bin_sz_t ncached_max) { 11 assert(ncached_max <= CACHE_BIN_NCACHED_MAX); 12 size_t stack_size = (size_t)ncached_max * sizeof(void *); 13 assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8))); 14 info->ncached_max = (cache_bin_sz_t)ncached_max; 15 } 16 17 void 18 cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, 19 size_t *size, size_t *alignment) { 20 /* For the total bin stack region (per tcache), reserve 2 more slots so 21 * that 22 * 1) the empty position can be safely read on the fast path before 23 * checking "is_empty"; and 24 * 2) the cur_ptr can go beyond the empty position by 1 step safely on 25 * the fast path (i.e. no overflow). 26 */ 27 *size = sizeof(void *) * 2; 28 for (szind_t i = 0; i < ninfos; i++) { 29 assert(infos[i].ncached_max > 0); 30 *size += infos[i].ncached_max * sizeof(void *); 31 } 32 33 /* 34 * Align to at least PAGE, to minimize the # of TLBs needed by the 35 * smaller sizes; also helps if the larger sizes don't get used at all. 36 */ 37 *alignment = PAGE; 38 } 39 40 void 41 cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc, 42 size_t *cur_offset) { 43 if (config_debug) { 44 size_t computed_size; 45 size_t computed_alignment; 46 47 /* Pointer should be as aligned as we asked for. */ 48 cache_bin_info_compute_alloc(infos, ninfos, &computed_size, 49 &computed_alignment); 50 assert(((uintptr_t)alloc & (computed_alignment - 1)) == 0); 51 } 52 53 *(uintptr_t *)((uintptr_t)alloc + *cur_offset) = 54 cache_bin_preceding_junk; 55 *cur_offset += sizeof(void *); 56 } 57 58 void 59 cache_bin_postincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc, 60 size_t *cur_offset) { 61 *(uintptr_t *)((uintptr_t)alloc + *cur_offset) = 62 cache_bin_trailing_junk; 63 *cur_offset += sizeof(void *); 64 } 65 66 void 67 cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc, 68 size_t *cur_offset) { 69 /* 70 * The full_position points to the lowest available space. Allocations 71 * will access the slots toward higher addresses (for the benefit of 72 * adjacent prefetch). 73 */ 74 void *stack_cur = (void *)((uintptr_t)alloc + *cur_offset); 75 void *full_position = stack_cur; 76 uint16_t bin_stack_size = info->ncached_max * sizeof(void *); 77 78 *cur_offset += bin_stack_size; 79 void *empty_position = (void *)((uintptr_t)alloc + *cur_offset); 80 81 /* Init to the empty position. */ 82 bin->stack_head = (void **)empty_position; 83 bin->low_bits_low_water = (uint16_t)(uintptr_t)bin->stack_head; 84 bin->low_bits_full = (uint16_t)(uintptr_t)full_position; 85 bin->low_bits_empty = (uint16_t)(uintptr_t)empty_position; 86 cache_bin_sz_t free_spots = cache_bin_diff(bin, 87 bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head, 88 /* racy */ false); 89 assert(free_spots == bin_stack_size); 90 assert(cache_bin_ncached_get_local(bin, info) == 0); 91 assert(cache_bin_empty_position_get(bin) == empty_position); 92 93 assert(bin_stack_size > 0 || empty_position == full_position); 94 } 95 96 bool 97 cache_bin_still_zero_initialized(cache_bin_t *bin) { 98 return bin->stack_head == NULL; 99 } 100