1 #ifndef JEMALLOC_INTERNAL_EXTENT_STRUCTS_H 2 #define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H 3 4 #include "jemalloc/internal/atomic.h" 5 #include "jemalloc/internal/bit_util.h" 6 #include "jemalloc/internal/bitmap.h" 7 #include "jemalloc/internal/mutex.h" 8 #include "jemalloc/internal/ql.h" 9 #include "jemalloc/internal/ph.h" 10 #include "jemalloc/internal/sc.h" 11 12 typedef enum { 13 extent_state_active = 0, 14 extent_state_dirty = 1, 15 extent_state_muzzy = 2, 16 extent_state_retained = 3 17 } extent_state_t; 18 19 /* Extent (span of pages). Use accessor functions for e_* fields. */ 20 struct extent_s { 21 /* 22 * Bitfield containing several fields: 23 * 24 * a: arena_ind 25 * b: slab 26 * c: committed 27 * d: dumpable 28 * z: zeroed 29 * t: state 30 * i: szind 31 * f: nfree 32 * s: bin_shard 33 * n: sn 34 * 35 * nnnnnnnn ... nnnnnnss ssssffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa 36 * 37 * arena_ind: Arena from which this extent came, or all 1 bits if 38 * unassociated. 39 * 40 * slab: The slab flag indicates whether the extent is used for a slab 41 * of small regions. This helps differentiate small size classes, 42 * and it indicates whether interior pointers can be looked up via 43 * iealloc(). 44 * 45 * committed: The committed flag indicates whether physical memory is 46 * committed to the extent, whether explicitly or implicitly 47 * as on a system that overcommits and satisfies physical 48 * memory needs on demand via soft page faults. 49 * 50 * dumpable: The dumpable flag indicates whether or not we've set the 51 * memory in question to be dumpable. Note that this 52 * interacts somewhat subtly with user-specified extent hooks, 53 * since we don't know if *they* are fiddling with 54 * dumpability (in which case, we don't want to undo whatever 55 * they're doing). To deal with this scenario, we: 56 * - Make dumpable false only for memory allocated with the 57 * default hooks. 58 * - Only allow memory to go from non-dumpable to dumpable, 59 * and only once. 60 * - Never make the OS call to allow dumping when the 61 * dumpable bit is already set. 62 * These three constraints mean that we will never 63 * accidentally dump user memory that the user meant to set 64 * nondumpable with their extent hooks. 65 * 66 * 67 * zeroed: The zeroed flag is used by extent recycling code to track 68 * whether memory is zero-filled. 69 * 70 * state: The state flag is an extent_state_t. 71 * 72 * szind: The szind flag indicates usable size class index for 73 * allocations residing in this extent, regardless of whether the 74 * extent is a slab. Extent size and usable size often differ 75 * even for non-slabs, either due to sz_large_pad or promotion of 76 * sampled small regions. 77 * 78 * nfree: Number of free regions in slab. 79 * 80 * bin_shard: the shard of the bin from which this extent came. 81 * 82 * sn: Serial number (potentially non-unique). 83 * 84 * Serial numbers may wrap around if !opt_retain, but as long as 85 * comparison functions fall back on address comparison for equal 86 * serial numbers, stable (if imperfect) ordering is maintained. 87 * 88 * Serial numbers may not be unique even in the absence of 89 * wrap-around, e.g. when splitting an extent and assigning the same 90 * serial number to both resulting adjacent extents. 91 */ 92 uint64_t e_bits; 93 #define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT) ((((((uint64_t)0x1U) << (CURRENT_FIELD_WIDTH)) - 1)) << (CURRENT_FIELD_SHIFT)) 94 95 #define EXTENT_BITS_ARENA_WIDTH MALLOCX_ARENA_BITS 96 #define EXTENT_BITS_ARENA_SHIFT 0 97 #define EXTENT_BITS_ARENA_MASK MASK(EXTENT_BITS_ARENA_WIDTH, EXTENT_BITS_ARENA_SHIFT) 98 99 #define EXTENT_BITS_SLAB_WIDTH 1 100 #define EXTENT_BITS_SLAB_SHIFT (EXTENT_BITS_ARENA_WIDTH + EXTENT_BITS_ARENA_SHIFT) 101 #define EXTENT_BITS_SLAB_MASK MASK(EXTENT_BITS_SLAB_WIDTH, EXTENT_BITS_SLAB_SHIFT) 102 103 #define EXTENT_BITS_COMMITTED_WIDTH 1 104 #define EXTENT_BITS_COMMITTED_SHIFT (EXTENT_BITS_SLAB_WIDTH + EXTENT_BITS_SLAB_SHIFT) 105 #define EXTENT_BITS_COMMITTED_MASK MASK(EXTENT_BITS_COMMITTED_WIDTH, EXTENT_BITS_COMMITTED_SHIFT) 106 107 #define EXTENT_BITS_DUMPABLE_WIDTH 1 108 #define EXTENT_BITS_DUMPABLE_SHIFT (EXTENT_BITS_COMMITTED_WIDTH + EXTENT_BITS_COMMITTED_SHIFT) 109 #define EXTENT_BITS_DUMPABLE_MASK MASK(EXTENT_BITS_DUMPABLE_WIDTH, EXTENT_BITS_DUMPABLE_SHIFT) 110 111 #define EXTENT_BITS_ZEROED_WIDTH 1 112 #define EXTENT_BITS_ZEROED_SHIFT (EXTENT_BITS_DUMPABLE_WIDTH + EXTENT_BITS_DUMPABLE_SHIFT) 113 #define EXTENT_BITS_ZEROED_MASK MASK(EXTENT_BITS_ZEROED_WIDTH, EXTENT_BITS_ZEROED_SHIFT) 114 115 #define EXTENT_BITS_STATE_WIDTH 2 116 #define EXTENT_BITS_STATE_SHIFT (EXTENT_BITS_ZEROED_WIDTH + EXTENT_BITS_ZEROED_SHIFT) 117 #define EXTENT_BITS_STATE_MASK MASK(EXTENT_BITS_STATE_WIDTH, EXTENT_BITS_STATE_SHIFT) 118 119 #define EXTENT_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES) 120 #define EXTENT_BITS_SZIND_SHIFT (EXTENT_BITS_STATE_WIDTH + EXTENT_BITS_STATE_SHIFT) 121 #define EXTENT_BITS_SZIND_MASK MASK(EXTENT_BITS_SZIND_WIDTH, EXTENT_BITS_SZIND_SHIFT) 122 123 #define EXTENT_BITS_NFREE_WIDTH (LG_SLAB_MAXREGS + 1) 124 #define EXTENT_BITS_NFREE_SHIFT (EXTENT_BITS_SZIND_WIDTH + EXTENT_BITS_SZIND_SHIFT) 125 #define EXTENT_BITS_NFREE_MASK MASK(EXTENT_BITS_NFREE_WIDTH, EXTENT_BITS_NFREE_SHIFT) 126 127 #define EXTENT_BITS_BINSHARD_WIDTH 6 128 #define EXTENT_BITS_BINSHARD_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT) 129 #define EXTENT_BITS_BINSHARD_MASK MASK(EXTENT_BITS_BINSHARD_WIDTH, EXTENT_BITS_BINSHARD_SHIFT) 130 131 #define EXTENT_BITS_IS_HEAD_WIDTH 1 132 #define EXTENT_BITS_IS_HEAD_SHIFT (EXTENT_BITS_BINSHARD_WIDTH + EXTENT_BITS_BINSHARD_SHIFT) 133 #define EXTENT_BITS_IS_HEAD_MASK MASK(EXTENT_BITS_IS_HEAD_WIDTH, EXTENT_BITS_IS_HEAD_SHIFT) 134 135 #define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_IS_HEAD_WIDTH + EXTENT_BITS_IS_HEAD_SHIFT) 136 #define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT) 137 138 /* Pointer to the extent that this structure is responsible for. */ 139 void *e_addr; 140 141 union { 142 /* 143 * Extent size and serial number associated with the extent 144 * structure (different than the serial number for the extent at 145 * e_addr). 146 * 147 * ssssssss [...] ssssssss ssssnnnn nnnnnnnn 148 */ 149 size_t e_size_esn; 150 #define EXTENT_SIZE_MASK ((size_t)~(PAGE-1)) 151 #define EXTENT_ESN_MASK ((size_t)PAGE-1) 152 /* Base extent size, which may not be a multiple of PAGE. */ 153 size_t e_bsize; 154 }; 155 156 /* 157 * List linkage, used by a variety of lists: 158 * - bin_t's slabs_full 159 * - extents_t's LRU 160 * - stashed dirty extents 161 * - arena's large allocations 162 */ 163 ql_elm(extent_t) ql_link; 164 165 /* 166 * Linkage for per size class sn/address-ordered heaps, and 167 * for extent_avail 168 */ 169 phn(extent_t) ph_link; 170 171 union { 172 /* Small region slab metadata. */ 173 arena_slab_data_t e_slab_data; 174 175 /* Profiling data, used for large objects. */ 176 struct { 177 /* Time when this was allocated. */ 178 nstime_t e_alloc_time; 179 /* Points to a prof_tctx_t. */ 180 atomic_p_t e_prof_tctx; 181 }; 182 }; 183 }; 184 typedef ql_head(extent_t) extent_list_t; 185 typedef ph(extent_t) extent_tree_t; 186 typedef ph(extent_t) extent_heap_t; 187 188 /* Quantized collection of extents, with built-in LRU queue. */ 189 struct extents_s { 190 malloc_mutex_t mtx; 191 192 /* 193 * Quantized per size class heaps of extents. 194 * 195 * Synchronization: mtx. 196 */ 197 extent_heap_t heaps[SC_NPSIZES + 1]; 198 atomic_zu_t nextents[SC_NPSIZES + 1]; 199 atomic_zu_t nbytes[SC_NPSIZES + 1]; 200 201 /* 202 * Bitmap for which set bits correspond to non-empty heaps. 203 * 204 * Synchronization: mtx. 205 */ 206 bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)]; 207 208 /* 209 * LRU of all extents in heaps. 210 * 211 * Synchronization: mtx. 212 */ 213 extent_list_t lru; 214 215 /* 216 * Page sum for all extents in heaps. 217 * 218 * The synchronization here is a little tricky. Modifications to npages 219 * must hold mtx, but reads need not (though, a reader who sees npages 220 * without holding the mutex can't assume anything about the rest of the 221 * state of the extents_t). 222 */ 223 atomic_zu_t npages; 224 225 /* All stored extents must be in the same state. */ 226 extent_state_t state; 227 228 /* 229 * If true, delay coalescing until eviction; otherwise coalesce during 230 * deallocation. 231 */ 232 bool delay_coalesce; 233 }; 234 235 /* 236 * The following two structs are for experimental purposes. See 237 * experimental_utilization_query_ctl and 238 * experimental_utilization_batch_query_ctl in src/ctl.c. 239 */ 240 241 struct extent_util_stats_s { 242 size_t nfree; 243 size_t nregs; 244 size_t size; 245 }; 246 247 struct extent_util_stats_verbose_s { 248 void *slabcur_addr; 249 size_t nfree; 250 size_t nregs; 251 size_t size; 252 size_t bin_nfree; 253 size_t bin_nregs; 254 }; 255 256 #endif /* JEMALLOC_INTERNAL_EXTENT_STRUCTS_H */ 257