1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/ehooks.h" 5 #include "jemalloc/internal/extent_mmap.h" 6 7 void 8 ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks, unsigned ind) { 9 /* All other hooks are optional; this one is not. */ 10 assert(extent_hooks->alloc != NULL); 11 ehooks->ind = ind; 12 ehooks_set_extent_hooks_ptr(ehooks, extent_hooks); 13 } 14 15 /* 16 * If the caller specifies (!*zero), it is still possible to receive zeroed 17 * memory, in which case *zero is toggled to true. arena_extent_alloc() takes 18 * advantage of this to avoid demanding zeroed extents, but taking advantage of 19 * them if they are returned. 20 */ 21 static void * 22 extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, 23 size_t alignment, bool *zero, bool *commit, dss_prec_t dss_prec) { 24 void *ret; 25 26 assert(size != 0); 27 assert(alignment != 0); 28 29 /* "primary" dss. */ 30 if (have_dss && dss_prec == dss_prec_primary && (ret = 31 extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, 32 commit)) != NULL) { 33 return ret; 34 } 35 /* mmap. */ 36 if ((ret = extent_alloc_mmap(new_addr, size, alignment, zero, commit)) 37 != NULL) { 38 return ret; 39 } 40 /* "secondary" dss. */ 41 if (have_dss && dss_prec == dss_prec_secondary && (ret = 42 extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, 43 commit)) != NULL) { 44 return ret; 45 } 46 47 /* All strategies for allocation failed. */ 48 return NULL; 49 } 50 51 void * 52 ehooks_default_alloc_impl(tsdn_t *tsdn, void *new_addr, size_t size, 53 size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { 54 arena_t *arena = arena_get(tsdn, arena_ind, false); 55 /* NULL arena indicates arena_create. */ 56 assert(arena != NULL || alignment == HUGEPAGE); 57 dss_prec_t dss = (arena == NULL) ? dss_prec_disabled : 58 (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_RELAXED); 59 void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, 60 zero, commit, dss); 61 if (have_madvise_huge && ret) { 62 pages_set_thp_state(ret, size); 63 } 64 return ret; 65 } 66 67 static void * 68 ehooks_default_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, 69 size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { 70 return ehooks_default_alloc_impl(tsdn_fetch(), new_addr, size, 71 ALIGNMENT_CEILING(alignment, PAGE), zero, commit, arena_ind); 72 } 73 74 bool 75 ehooks_default_dalloc_impl(void *addr, size_t size) { 76 if (!have_dss || !extent_in_dss(addr)) { 77 return extent_dalloc_mmap(addr, size); 78 } 79 return true; 80 } 81 82 static bool 83 ehooks_default_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, 84 bool committed, unsigned arena_ind) { 85 return ehooks_default_dalloc_impl(addr, size); 86 } 87 88 void 89 ehooks_default_destroy_impl(void *addr, size_t size) { 90 if (!have_dss || !extent_in_dss(addr)) { 91 pages_unmap(addr, size); 92 } 93 } 94 95 static void 96 ehooks_default_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, 97 bool committed, unsigned arena_ind) { 98 ehooks_default_destroy_impl(addr, size); 99 } 100 101 bool 102 ehooks_default_commit_impl(void *addr, size_t offset, size_t length) { 103 return pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset), 104 length); 105 } 106 107 static bool 108 ehooks_default_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, 109 size_t offset, size_t length, unsigned arena_ind) { 110 return ehooks_default_commit_impl(addr, offset, length); 111 } 112 113 bool 114 ehooks_default_decommit_impl(void *addr, size_t offset, size_t length) { 115 return pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset), 116 length); 117 } 118 119 static bool 120 ehooks_default_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, 121 size_t offset, size_t length, unsigned arena_ind) { 122 return ehooks_default_decommit_impl(addr, offset, length); 123 } 124 125 #ifdef PAGES_CAN_PURGE_LAZY 126 bool 127 ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length) { 128 return pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset), 129 length); 130 } 131 132 static bool 133 ehooks_default_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size, 134 size_t offset, size_t length, unsigned arena_ind) { 135 assert(addr != NULL); 136 assert((offset & PAGE_MASK) == 0); 137 assert(length != 0); 138 assert((length & PAGE_MASK) == 0); 139 return ehooks_default_purge_lazy_impl(addr, offset, length); 140 } 141 #endif 142 143 #ifdef PAGES_CAN_PURGE_FORCED 144 bool 145 ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length) { 146 return pages_purge_forced((void *)((uintptr_t)addr + 147 (uintptr_t)offset), length); 148 } 149 150 static bool 151 ehooks_default_purge_forced(extent_hooks_t *extent_hooks, void *addr, 152 size_t size, size_t offset, size_t length, unsigned arena_ind) { 153 assert(addr != NULL); 154 assert((offset & PAGE_MASK) == 0); 155 assert(length != 0); 156 assert((length & PAGE_MASK) == 0); 157 return ehooks_default_purge_forced_impl(addr, offset, length); 158 } 159 #endif 160 161 bool 162 ehooks_default_split_impl() { 163 if (!maps_coalesce) { 164 /* 165 * Without retain, only whole regions can be purged (required by 166 * MEM_RELEASE on Windows) -- therefore disallow splitting. See 167 * comments in extent_head_no_merge(). 168 */ 169 return !opt_retain; 170 } 171 172 return false; 173 } 174 175 static bool 176 ehooks_default_split(extent_hooks_t *extent_hooks, void *addr, size_t size, 177 size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { 178 return ehooks_default_split_impl(); 179 } 180 181 bool 182 ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b) { 183 assert(addr_a < addr_b); 184 /* 185 * For non-DSS cases -- 186 * a) W/o maps_coalesce, merge is not always allowed (Windows): 187 * 1) w/o retain, never merge (first branch below). 188 * 2) with retain, only merge extents from the same VirtualAlloc 189 * region (in which case MEM_DECOMMIT is utilized for purging). 190 * 191 * b) With maps_coalesce, it's always possible to merge. 192 * 1) w/o retain, always allow merge (only about dirty / muzzy). 193 * 2) with retain, to preserve the SN / first-fit, merge is still 194 * disallowed if b is a head extent, i.e. no merging across 195 * different mmap regions. 196 * 197 * a2) and b2) are implemented in emap_try_acquire_edata_neighbor, and 198 * sanity checked in the second branch below. 199 */ 200 if (!maps_coalesce && !opt_retain) { 201 return true; 202 } 203 if (config_debug) { 204 edata_t *a = emap_edata_lookup(tsdn, &arena_emap_global, 205 addr_a); 206 bool head_a = edata_is_head_get(a); 207 edata_t *b = emap_edata_lookup(tsdn, &arena_emap_global, 208 addr_b); 209 bool head_b = edata_is_head_get(b); 210 emap_assert_mapped(tsdn, &arena_emap_global, a); 211 emap_assert_mapped(tsdn, &arena_emap_global, b); 212 assert(extent_neighbor_head_state_mergeable(head_a, head_b, 213 /* forward */ true)); 214 } 215 if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) { 216 return true; 217 } 218 219 return false; 220 } 221 222 bool 223 ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, 224 void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { 225 tsdn_t *tsdn = tsdn_fetch(); 226 227 return ehooks_default_merge_impl(tsdn, addr_a, addr_b); 228 } 229 230 void 231 ehooks_default_zero_impl(void *addr, size_t size) { 232 /* 233 * By default, we try to zero out memory using OS-provided demand-zeroed 234 * pages. If the user has specifically requested hugepages, though, we 235 * don't want to purge in the middle of a hugepage (which would break it 236 * up), so we act conservatively and use memset. 237 */ 238 bool needs_memset = true; 239 if (opt_thp != thp_mode_always) { 240 needs_memset = pages_purge_forced(addr, size); 241 } 242 if (needs_memset) { 243 memset(addr, 0, size); 244 } 245 } 246 247 void 248 ehooks_default_guard_impl(void *guard1, void *guard2) { 249 pages_mark_guards(guard1, guard2); 250 } 251 252 void 253 ehooks_default_unguard_impl(void *guard1, void *guard2) { 254 pages_unmark_guards(guard1, guard2); 255 } 256 257 const extent_hooks_t ehooks_default_extent_hooks = { 258 ehooks_default_alloc, 259 ehooks_default_dalloc, 260 ehooks_default_destroy, 261 ehooks_default_commit, 262 ehooks_default_decommit, 263 #ifdef PAGES_CAN_PURGE_LAZY 264 ehooks_default_purge_lazy, 265 #else 266 NULL, 267 #endif 268 #ifdef PAGES_CAN_PURGE_FORCED 269 ehooks_default_purge_forced, 270 #else 271 NULL, 272 #endif 273 ehooks_default_split, 274 ehooks_default_merge 275 }; 276