1 #define JEMALLOC_PAGES_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT 5 #include <sys/sysctl.h> 6 #endif 7 8 /******************************************************************************/ 9 /* Data. */ 10 11 #ifndef _WIN32 12 # define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE) 13 # define PAGES_PROT_DECOMMIT (PROT_NONE) 14 static int mmap_flags; 15 #endif 16 static bool os_overcommits; 17 18 /******************************************************************************/ 19 20 void * 21 pages_map(void *addr, size_t size, bool *commit) 22 { 23 void *ret; 24 25 assert(size != 0); 26 27 if (os_overcommits) 28 *commit = true; 29 30 #ifdef _WIN32 31 /* 32 * If VirtualAlloc can't allocate at the given address when one is 33 * given, it fails and returns NULL. 34 */ 35 ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0), 36 PAGE_READWRITE); 37 #else 38 /* 39 * We don't use MAP_FIXED here, because it can cause the *replacement* 40 * of existing mappings, and we only want to create new mappings. 41 */ 42 { 43 int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; 44 45 ret = mmap(addr, size, prot, mmap_flags, -1, 0); 46 } 47 assert(ret != NULL); 48 49 if (ret == MAP_FAILED) 50 ret = NULL; 51 else if (addr != NULL && ret != addr) { 52 /* 53 * We succeeded in mapping memory, but not in the right place. 54 */ 55 pages_unmap(ret, size); 56 ret = NULL; 57 } 58 #endif 59 assert(ret == NULL || (addr == NULL && ret != addr) 60 || (addr != NULL && ret == addr)); 61 return (ret); 62 } 63 64 void 65 pages_unmap(void *addr, size_t size) 66 { 67 68 #ifdef _WIN32 69 if (VirtualFree(addr, 0, MEM_RELEASE) == 0) 70 #else 71 if (munmap(addr, size) == -1) 72 #endif 73 { 74 char buf[BUFERROR_BUF]; 75 76 buferror(get_errno(), buf, sizeof(buf)); 77 malloc_printf("<jemalloc>: Error in " 78 #ifdef _WIN32 79 "VirtualFree" 80 #else 81 "munmap" 82 #endif 83 "(): %s\n", buf); 84 if (opt_abort) 85 abort(); 86 } 87 } 88 89 void * 90 pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size, 91 bool *commit) 92 { 93 void *ret = (void *)((uintptr_t)addr + leadsize); 94 95 assert(alloc_size >= leadsize + size); 96 #ifdef _WIN32 97 { 98 void *new_addr; 99 100 pages_unmap(addr, alloc_size); 101 new_addr = pages_map(ret, size, commit); 102 if (new_addr == ret) 103 return (ret); 104 if (new_addr) 105 pages_unmap(new_addr, size); 106 return (NULL); 107 } 108 #else 109 { 110 size_t trailsize = alloc_size - leadsize - size; 111 112 if (leadsize != 0) 113 pages_unmap(addr, leadsize); 114 if (trailsize != 0) 115 pages_unmap((void *)((uintptr_t)ret + size), trailsize); 116 return (ret); 117 } 118 #endif 119 } 120 121 static bool 122 pages_commit_impl(void *addr, size_t size, bool commit) 123 { 124 125 if (os_overcommits) 126 return (true); 127 128 #ifdef _WIN32 129 return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, 130 PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT))); 131 #else 132 { 133 int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; 134 void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED, 135 -1, 0); 136 if (result == MAP_FAILED) 137 return (true); 138 if (result != addr) { 139 /* 140 * We succeeded in mapping memory, but not in the right 141 * place. 142 */ 143 pages_unmap(result, size); 144 return (true); 145 } 146 return (false); 147 } 148 #endif 149 } 150 151 bool 152 pages_commit(void *addr, size_t size) 153 { 154 155 return (pages_commit_impl(addr, size, true)); 156 } 157 158 bool 159 pages_decommit(void *addr, size_t size) 160 { 161 162 return (pages_commit_impl(addr, size, false)); 163 } 164 165 bool 166 pages_purge(void *addr, size_t size) 167 { 168 bool unzeroed; 169 170 #ifdef _WIN32 171 VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); 172 unzeroed = true; 173 #elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \ 174 defined(JEMALLOC_PURGE_MADVISE_DONTNEED)) 175 # if defined(JEMALLOC_PURGE_MADVISE_FREE) 176 # define JEMALLOC_MADV_PURGE MADV_FREE 177 # define JEMALLOC_MADV_ZEROS false 178 # elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) 179 # define JEMALLOC_MADV_PURGE MADV_DONTNEED 180 # define JEMALLOC_MADV_ZEROS true 181 # else 182 # error No madvise(2) flag defined for purging unused dirty pages 183 # endif 184 int err = madvise(addr, size, JEMALLOC_MADV_PURGE); 185 unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0); 186 # undef JEMALLOC_MADV_PURGE 187 # undef JEMALLOC_MADV_ZEROS 188 #else 189 /* Last resort no-op. */ 190 unzeroed = true; 191 #endif 192 return (unzeroed); 193 } 194 195 bool 196 pages_huge(void *addr, size_t size) 197 { 198 199 assert(PAGE_ADDR2BASE(addr) == addr); 200 assert(PAGE_CEILING(size) == size); 201 202 #ifdef JEMALLOC_THP 203 return (madvise(addr, size, MADV_HUGEPAGE) != 0); 204 #else 205 return (false); 206 #endif 207 } 208 209 bool 210 pages_nohuge(void *addr, size_t size) 211 { 212 213 assert(PAGE_ADDR2BASE(addr) == addr); 214 assert(PAGE_CEILING(size) == size); 215 216 #ifdef JEMALLOC_THP 217 return (madvise(addr, size, MADV_NOHUGEPAGE) != 0); 218 #else 219 return (false); 220 #endif 221 } 222 223 #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT 224 static bool 225 os_overcommits_sysctl(void) 226 { 227 int vm_overcommit; 228 size_t sz; 229 230 sz = sizeof(vm_overcommit); 231 if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0) 232 return (false); /* Error. */ 233 234 return ((vm_overcommit & 0x3) == 0); 235 } 236 #endif 237 238 #ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY 239 /* 240 * Use syscall(2) rather than {open,read,close}(2) when possible to avoid 241 * reentry during bootstrapping if another library has interposed system call 242 * wrappers. 243 */ 244 static bool 245 os_overcommits_proc(void) 246 { 247 int fd; 248 char buf[1]; 249 ssize_t nread; 250 251 #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open) 252 fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY); 253 #else 254 fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); 255 #endif 256 if (fd == -1) 257 return (false); /* Error. */ 258 259 #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read) 260 nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf)); 261 #else 262 nread = read(fd, &buf, sizeof(buf)); 263 #endif 264 265 #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close) 266 syscall(SYS_close, fd); 267 #else 268 close(fd); 269 #endif 270 271 if (nread < 1) 272 return (false); /* Error. */ 273 /* 274 * /proc/sys/vm/overcommit_memory meanings: 275 * 0: Heuristic overcommit. 276 * 1: Always overcommit. 277 * 2: Never overcommit. 278 */ 279 return (buf[0] == '0' || buf[0] == '1'); 280 } 281 #endif 282 283 void 284 pages_boot(void) 285 { 286 287 #ifndef _WIN32 288 mmap_flags = MAP_PRIVATE | MAP_ANON; 289 #endif 290 291 #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT 292 os_overcommits = os_overcommits_sysctl(); 293 #elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY) 294 os_overcommits = os_overcommits_proc(); 295 # ifdef MAP_NORESERVE 296 if (os_overcommits) 297 mmap_flags |= MAP_NORESERVE; 298 # endif 299 #else 300 os_overcommits = false; 301 #endif 302 } 303