1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/assert.h" 5 #include "jemalloc/internal/extent_dss.h" 6 #include "jemalloc/internal/spin.h" 7 8 /******************************************************************************/ 9 /* Data. */ 10 11 const char *opt_dss = DSS_DEFAULT; 12 13 const char *dss_prec_names[] = { 14 "disabled", 15 "primary", 16 "secondary", 17 "N/A" 18 }; 19 20 /* 21 * Current dss precedence default, used when creating new arenas. NB: This is 22 * stored as unsigned rather than dss_prec_t because in principle there's no 23 * guarantee that sizeof(dss_prec_t) is the same as sizeof(unsigned), and we use 24 * atomic operations to synchronize the setting. 25 */ 26 static atomic_u_t dss_prec_default = ATOMIC_INIT( 27 (unsigned)DSS_PREC_DEFAULT); 28 29 /* Base address of the DSS. */ 30 static void *dss_base; 31 /* Atomic boolean indicating whether a thread is currently extending DSS. */ 32 static atomic_b_t dss_extending; 33 /* Atomic boolean indicating whether the DSS is exhausted. */ 34 static atomic_b_t dss_exhausted; 35 /* Atomic current upper limit on DSS addresses. */ 36 static atomic_p_t dss_max; 37 38 /******************************************************************************/ 39 40 static void * 41 extent_dss_sbrk(intptr_t increment) { 42 #ifdef JEMALLOC_DSS 43 return sbrk(increment); 44 #else 45 not_implemented(); 46 return NULL; 47 #endif 48 } 49 50 dss_prec_t 51 extent_dss_prec_get(void) { 52 dss_prec_t ret; 53 54 if (!have_dss) { 55 return dss_prec_disabled; 56 } 57 ret = (dss_prec_t)atomic_load_u(&dss_prec_default, ATOMIC_ACQUIRE); 58 return ret; 59 } 60 61 bool 62 extent_dss_prec_set(dss_prec_t dss_prec) { 63 if (!have_dss) { 64 return (dss_prec != dss_prec_disabled); 65 } 66 atomic_store_u(&dss_prec_default, (unsigned)dss_prec, ATOMIC_RELEASE); 67 return false; 68 } 69 70 static void 71 extent_dss_extending_start(void) { 72 spin_t spinner = SPIN_INITIALIZER; 73 while (true) { 74 bool expected = false; 75 if (atomic_compare_exchange_weak_b(&dss_extending, &expected, 76 true, ATOMIC_ACQ_REL, ATOMIC_RELAXED)) { 77 break; 78 } 79 spin_adaptive(&spinner); 80 } 81 } 82 83 static void 84 extent_dss_extending_finish(void) { 85 assert(atomic_load_b(&dss_extending, ATOMIC_RELAXED)); 86 87 atomic_store_b(&dss_extending, false, ATOMIC_RELEASE); 88 } 89 90 static void * 91 extent_dss_max_update(void *new_addr) { 92 /* 93 * Get the current end of the DSS as max_cur and assure that dss_max is 94 * up to date. 95 */ 96 void *max_cur = extent_dss_sbrk(0); 97 if (max_cur == (void *)-1) { 98 return NULL; 99 } 100 atomic_store_p(&dss_max, max_cur, ATOMIC_RELEASE); 101 /* Fixed new_addr can only be supported if it is at the edge of DSS. */ 102 if (new_addr != NULL && max_cur != new_addr) { 103 return NULL; 104 } 105 return max_cur; 106 } 107 108 void * 109 extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, 110 size_t alignment, bool *zero, bool *commit) { 111 edata_t *gap; 112 113 cassert(have_dss); 114 assert(size > 0); 115 assert(alignment == ALIGNMENT_CEILING(alignment, PAGE)); 116 117 /* 118 * sbrk() uses a signed increment argument, so take care not to 119 * interpret a large allocation request as a negative increment. 120 */ 121 if ((intptr_t)size < 0) { 122 return NULL; 123 } 124 125 gap = edata_cache_get(tsdn, &arena->pa_shard.edata_cache); 126 if (gap == NULL) { 127 return NULL; 128 } 129 130 extent_dss_extending_start(); 131 if (!atomic_load_b(&dss_exhausted, ATOMIC_ACQUIRE)) { 132 /* 133 * The loop is necessary to recover from races with other 134 * threads that are using the DSS for something other than 135 * malloc. 136 */ 137 while (true) { 138 void *max_cur = extent_dss_max_update(new_addr); 139 if (max_cur == NULL) { 140 goto label_oom; 141 } 142 143 bool head_state = opt_retain ? EXTENT_IS_HEAD : 144 EXTENT_NOT_HEAD; 145 /* 146 * Compute how much page-aligned gap space (if any) is 147 * necessary to satisfy alignment. This space can be 148 * recycled for later use. 149 */ 150 void *gap_addr_page = (void *)(PAGE_CEILING( 151 (uintptr_t)max_cur)); 152 void *ret = (void *)ALIGNMENT_CEILING( 153 (uintptr_t)gap_addr_page, alignment); 154 size_t gap_size_page = (uintptr_t)ret - 155 (uintptr_t)gap_addr_page; 156 if (gap_size_page != 0) { 157 edata_init(gap, arena_ind_get(arena), 158 gap_addr_page, gap_size_page, false, 159 SC_NSIZES, extent_sn_next( 160 &arena->pa_shard.pac), 161 extent_state_active, false, true, 162 EXTENT_PAI_PAC, head_state); 163 } 164 /* 165 * Compute the address just past the end of the desired 166 * allocation space. 167 */ 168 void *dss_next = (void *)((uintptr_t)ret + size); 169 if ((uintptr_t)ret < (uintptr_t)max_cur || 170 (uintptr_t)dss_next < (uintptr_t)max_cur) { 171 goto label_oom; /* Wrap-around. */ 172 } 173 /* Compute the increment, including subpage bytes. */ 174 void *gap_addr_subpage = max_cur; 175 size_t gap_size_subpage = (uintptr_t)ret - 176 (uintptr_t)gap_addr_subpage; 177 intptr_t incr = gap_size_subpage + size; 178 179 assert((uintptr_t)max_cur + incr == (uintptr_t)ret + 180 size); 181 182 /* Try to allocate. */ 183 void *dss_prev = extent_dss_sbrk(incr); 184 if (dss_prev == max_cur) { 185 /* Success. */ 186 atomic_store_p(&dss_max, dss_next, 187 ATOMIC_RELEASE); 188 extent_dss_extending_finish(); 189 190 if (gap_size_page != 0) { 191 ehooks_t *ehooks = arena_get_ehooks( 192 arena); 193 extent_dalloc_gap(tsdn, 194 &arena->pa_shard.pac, ehooks, gap); 195 } else { 196 edata_cache_put(tsdn, 197 &arena->pa_shard.edata_cache, gap); 198 } 199 if (!*commit) { 200 *commit = pages_decommit(ret, size); 201 } 202 if (*zero && *commit) { 203 edata_t edata = {0}; 204 ehooks_t *ehooks = arena_get_ehooks( 205 arena); 206 207 edata_init(&edata, 208 arena_ind_get(arena), ret, size, 209 size, false, SC_NSIZES, 210 extent_state_active, false, true, 211 EXTENT_PAI_PAC, head_state); 212 if (extent_purge_forced_wrapper(tsdn, 213 ehooks, &edata, 0, size)) { 214 memset(ret, 0, size); 215 } 216 } 217 return ret; 218 } 219 /* 220 * Failure, whether due to OOM or a race with a raw 221 * sbrk() call from outside the allocator. 222 */ 223 if (dss_prev == (void *)-1) { 224 /* OOM. */ 225 atomic_store_b(&dss_exhausted, true, 226 ATOMIC_RELEASE); 227 goto label_oom; 228 } 229 } 230 } 231 label_oom: 232 extent_dss_extending_finish(); 233 edata_cache_put(tsdn, &arena->pa_shard.edata_cache, gap); 234 return NULL; 235 } 236 237 static bool 238 extent_in_dss_helper(void *addr, void *max) { 239 return ((uintptr_t)addr >= (uintptr_t)dss_base && (uintptr_t)addr < 240 (uintptr_t)max); 241 } 242 243 bool 244 extent_in_dss(void *addr) { 245 cassert(have_dss); 246 247 return extent_in_dss_helper(addr, atomic_load_p(&dss_max, 248 ATOMIC_ACQUIRE)); 249 } 250 251 bool 252 extent_dss_mergeable(void *addr_a, void *addr_b) { 253 void *max; 254 255 cassert(have_dss); 256 257 if ((uintptr_t)addr_a < (uintptr_t)dss_base && (uintptr_t)addr_b < 258 (uintptr_t)dss_base) { 259 return true; 260 } 261 262 max = atomic_load_p(&dss_max, ATOMIC_ACQUIRE); 263 return (extent_in_dss_helper(addr_a, max) == 264 extent_in_dss_helper(addr_b, max)); 265 } 266 267 void 268 extent_dss_boot(void) { 269 cassert(have_dss); 270 271 dss_base = extent_dss_sbrk(0); 272 atomic_store_b(&dss_extending, false, ATOMIC_RELAXED); 273 atomic_store_b(&dss_exhausted, dss_base == (void *)-1, ATOMIC_RELAXED); 274 atomic_store_p(&dss_max, dss_base, ATOMIC_RELAXED); 275 } 276 277 /******************************************************************************/ 278