1 //===-- memprof_malloc_linux.cpp -----------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of MemProfiler, a memory profiler. 10 // 11 // Linux-specific malloc interception. 12 // We simply define functions like malloc, free, realloc, etc. 13 // They will replace the corresponding libc functions automagically. 14 //===----------------------------------------------------------------------===// 15 16 #include "sanitizer_common/sanitizer_platform.h" 17 #if !SANITIZER_LINUX 18 #error Unsupported OS 19 #endif 20 21 #include "memprof_allocator.h" 22 #include "memprof_interceptors.h" 23 #include "memprof_internal.h" 24 #include "memprof_stack.h" 25 #include "sanitizer_common/sanitizer_allocator_checks.h" 26 #include "sanitizer_common/sanitizer_errno.h" 27 #include "sanitizer_common/sanitizer_tls_get_addr.h" 28 29 // ---------------------- Replacement functions ---------------- {{{1 30 using namespace __memprof; 31 32 static uptr allocated_for_dlsym; 33 static uptr last_dlsym_alloc_size_in_words; 34 static const uptr kDlsymAllocPoolSize = 1024; 35 static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; 36 37 static inline bool IsInDlsymAllocPool(const void *ptr) { 38 uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; 39 return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]); 40 } 41 42 static void *AllocateFromLocalPool(uptr size_in_bytes) { 43 uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; 44 void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; 45 last_dlsym_alloc_size_in_words = size_in_words; 46 allocated_for_dlsym += size_in_words; 47 CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); 48 return mem; 49 } 50 51 static void DeallocateFromLocalPool(const void *ptr) { 52 // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store 53 // error messages and instead uses malloc followed by free. To avoid pool 54 // exhaustion due to long object filenames, handle that special case here. 55 uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words; 56 void *prev_mem = (void *)&alloc_memory_for_dlsym[prev_offset]; 57 if (prev_mem == ptr) { 58 REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize); 59 allocated_for_dlsym = prev_offset; 60 last_dlsym_alloc_size_in_words = 0; 61 } 62 } 63 64 static int PosixMemalignFromLocalPool(void **memptr, uptr alignment, 65 uptr size_in_bytes) { 66 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) 67 return errno_EINVAL; 68 69 CHECK(alignment >= kWordSize); 70 71 uptr addr = (uptr)&alloc_memory_for_dlsym[allocated_for_dlsym]; 72 uptr aligned_addr = RoundUpTo(addr, alignment); 73 uptr aligned_size = RoundUpTo(size_in_bytes, kWordSize); 74 75 uptr *end_mem = (uptr *)(aligned_addr + aligned_size); 76 uptr allocated = end_mem - alloc_memory_for_dlsym; 77 if (allocated >= kDlsymAllocPoolSize) 78 return errno_ENOMEM; 79 80 allocated_for_dlsym = allocated; 81 *memptr = (void *)aligned_addr; 82 return 0; 83 } 84 85 static inline bool MaybeInDlsym() { return memprof_init_is_running; } 86 87 static inline bool UseLocalPool() { return MaybeInDlsym(); } 88 89 static void *ReallocFromLocalPool(void *ptr, uptr size) { 90 const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; 91 const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); 92 void *new_ptr; 93 if (UNLIKELY(UseLocalPool())) { 94 new_ptr = AllocateFromLocalPool(size); 95 } else { 96 ENSURE_MEMPROF_INITED(); 97 GET_STACK_TRACE_MALLOC; 98 new_ptr = memprof_malloc(size, &stack); 99 } 100 internal_memcpy(new_ptr, ptr, copy_size); 101 return new_ptr; 102 } 103 104 INTERCEPTOR(void, free, void *ptr) { 105 GET_STACK_TRACE_FREE; 106 if (UNLIKELY(IsInDlsymAllocPool(ptr))) { 107 DeallocateFromLocalPool(ptr); 108 return; 109 } 110 memprof_free(ptr, &stack, FROM_MALLOC); 111 } 112 113 #if SANITIZER_INTERCEPT_CFREE 114 INTERCEPTOR(void, cfree, void *ptr) { 115 GET_STACK_TRACE_FREE; 116 if (UNLIKELY(IsInDlsymAllocPool(ptr))) 117 return; 118 memprof_free(ptr, &stack, FROM_MALLOC); 119 } 120 #endif // SANITIZER_INTERCEPT_CFREE 121 122 INTERCEPTOR(void *, malloc, uptr size) { 123 if (UNLIKELY(UseLocalPool())) 124 // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. 125 return AllocateFromLocalPool(size); 126 ENSURE_MEMPROF_INITED(); 127 GET_STACK_TRACE_MALLOC; 128 return memprof_malloc(size, &stack); 129 } 130 131 INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) { 132 if (UNLIKELY(UseLocalPool())) 133 // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. 134 return AllocateFromLocalPool(nmemb * size); 135 ENSURE_MEMPROF_INITED(); 136 GET_STACK_TRACE_MALLOC; 137 return memprof_calloc(nmemb, size, &stack); 138 } 139 140 INTERCEPTOR(void *, realloc, void *ptr, uptr size) { 141 if (UNLIKELY(IsInDlsymAllocPool(ptr))) 142 return ReallocFromLocalPool(ptr, size); 143 if (UNLIKELY(UseLocalPool())) 144 return AllocateFromLocalPool(size); 145 ENSURE_MEMPROF_INITED(); 146 GET_STACK_TRACE_MALLOC; 147 return memprof_realloc(ptr, size, &stack); 148 } 149 150 #if SANITIZER_INTERCEPT_REALLOCARRAY 151 INTERCEPTOR(void *, reallocarray, void *ptr, uptr nmemb, uptr size) { 152 ENSURE_MEMPROF_INITED(); 153 GET_STACK_TRACE_MALLOC; 154 return memprof_reallocarray(ptr, nmemb, size, &stack); 155 } 156 #endif // SANITIZER_INTERCEPT_REALLOCARRAY 157 158 #if SANITIZER_INTERCEPT_MEMALIGN 159 INTERCEPTOR(void *, memalign, uptr boundary, uptr size) { 160 GET_STACK_TRACE_MALLOC; 161 return memprof_memalign(boundary, size, &stack, FROM_MALLOC); 162 } 163 164 INTERCEPTOR(void *, __libc_memalign, uptr boundary, uptr size) { 165 GET_STACK_TRACE_MALLOC; 166 void *res = memprof_memalign(boundary, size, &stack, FROM_MALLOC); 167 DTLS_on_libc_memalign(res, size); 168 return res; 169 } 170 #endif // SANITIZER_INTERCEPT_MEMALIGN 171 172 #if SANITIZER_INTERCEPT_ALIGNED_ALLOC 173 INTERCEPTOR(void *, aligned_alloc, uptr boundary, uptr size) { 174 GET_STACK_TRACE_MALLOC; 175 return memprof_aligned_alloc(boundary, size, &stack); 176 } 177 #endif // SANITIZER_INTERCEPT_ALIGNED_ALLOC 178 179 INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { 180 GET_CURRENT_PC_BP_SP; 181 (void)sp; 182 return memprof_malloc_usable_size(ptr, pc, bp); 183 } 184 185 #if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO 186 // We avoid including malloc.h for portability reasons. 187 // man mallinfo says the fields are "long", but the implementation uses int. 188 // It doesn't matter much -- we just need to make sure that the libc's mallinfo 189 // is not called. 190 struct fake_mallinfo { 191 int x[10]; 192 }; 193 194 INTERCEPTOR(struct fake_mallinfo, mallinfo, void) { 195 struct fake_mallinfo res; 196 REAL(memset)(&res, 0, sizeof(res)); 197 return res; 198 } 199 200 INTERCEPTOR(int, mallopt, int cmd, int value) { return 0; } 201 #endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO 202 203 INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) { 204 if (UNLIKELY(UseLocalPool())) 205 return PosixMemalignFromLocalPool(memptr, alignment, size); 206 GET_STACK_TRACE_MALLOC; 207 return memprof_posix_memalign(memptr, alignment, size, &stack); 208 } 209 210 INTERCEPTOR(void *, valloc, uptr size) { 211 GET_STACK_TRACE_MALLOC; 212 return memprof_valloc(size, &stack); 213 } 214 215 #if SANITIZER_INTERCEPT_PVALLOC 216 INTERCEPTOR(void *, pvalloc, uptr size) { 217 GET_STACK_TRACE_MALLOC; 218 return memprof_pvalloc(size, &stack); 219 } 220 #endif // SANITIZER_INTERCEPT_PVALLOC 221 222 INTERCEPTOR(void, malloc_stats, void) { __memprof_print_accumulated_stats(); } 223 224 namespace __memprof { 225 void ReplaceSystemMalloc() {} 226 } // namespace __memprof 227