1 //=-- lsan_allocator.cpp --------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of LeakSanitizer. 10 // See lsan_allocator.h for details. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "lsan_allocator.h" 15 16 #include "sanitizer_common/sanitizer_allocator.h" 17 #include "sanitizer_common/sanitizer_allocator_checks.h" 18 #include "sanitizer_common/sanitizer_allocator_interface.h" 19 #include "sanitizer_common/sanitizer_allocator_report.h" 20 #include "sanitizer_common/sanitizer_errno.h" 21 #include "sanitizer_common/sanitizer_internal_defs.h" 22 #include "sanitizer_common/sanitizer_stackdepot.h" 23 #include "sanitizer_common/sanitizer_stacktrace.h" 24 #include "lsan_common.h" 25 26 extern "C" void *memset(void *ptr, int value, uptr num); 27 28 namespace __lsan { 29 #if defined(__i386__) || defined(__arm__) 30 static const uptr kMaxAllowedMallocSize = 1ULL << 30; 31 #elif defined(__mips64) || defined(__aarch64__) 32 static const uptr kMaxAllowedMallocSize = 4ULL << 30; 33 #else 34 static const uptr kMaxAllowedMallocSize = 8ULL << 30; 35 #endif 36 37 static Allocator allocator; 38 39 static uptr max_malloc_size; 40 41 void InitializeAllocator() { 42 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); 43 allocator.InitLinkerInitialized( 44 common_flags()->allocator_release_to_os_interval_ms); 45 if (common_flags()->max_allocation_size_mb) 46 max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20, 47 kMaxAllowedMallocSize); 48 else 49 max_malloc_size = kMaxAllowedMallocSize; 50 } 51 52 void AllocatorThreadFinish() { 53 allocator.SwallowCache(GetAllocatorCache()); 54 } 55 56 static ChunkMetadata *Metadata(const void *p) { 57 return reinterpret_cast<ChunkMetadata *>(allocator.GetMetaData(p)); 58 } 59 60 static void RegisterAllocation(const StackTrace &stack, void *p, uptr size) { 61 if (!p) return; 62 ChunkMetadata *m = Metadata(p); 63 CHECK(m); 64 m->tag = DisabledInThisThread() ? kIgnored : kDirectlyLeaked; 65 m->stack_trace_id = StackDepotPut(stack); 66 m->requested_size = size; 67 atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 1, memory_order_relaxed); 68 } 69 70 static void RegisterDeallocation(void *p) { 71 if (!p) return; 72 ChunkMetadata *m = Metadata(p); 73 CHECK(m); 74 atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 0, memory_order_relaxed); 75 } 76 77 static void *ReportAllocationSizeTooBig(uptr size, const StackTrace &stack) { 78 if (AllocatorMayReturnNull()) { 79 Report("WARNING: LeakSanitizer failed to allocate 0x%zx bytes\n", size); 80 return nullptr; 81 } 82 ReportAllocationSizeTooBig(size, max_malloc_size, &stack); 83 } 84 85 void *Allocate(const StackTrace &stack, uptr size, uptr alignment, 86 bool cleared) { 87 if (size == 0) 88 size = 1; 89 if (size > max_malloc_size) 90 return ReportAllocationSizeTooBig(size, stack); 91 if (UNLIKELY(IsRssLimitExceeded())) { 92 if (AllocatorMayReturnNull()) 93 return nullptr; 94 ReportRssLimitExceeded(&stack); 95 } 96 void *p = allocator.Allocate(GetAllocatorCache(), size, alignment); 97 if (UNLIKELY(!p)) { 98 SetAllocatorOutOfMemory(); 99 if (AllocatorMayReturnNull()) 100 return nullptr; 101 ReportOutOfMemory(size, &stack); 102 } 103 // Do not rely on the allocator to clear the memory (it's slow). 104 if (cleared && allocator.FromPrimary(p)) 105 memset(p, 0, size); 106 RegisterAllocation(stack, p, size); 107 if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(p, size); 108 RunMallocHooks(p, size); 109 return p; 110 } 111 112 static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) { 113 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { 114 if (AllocatorMayReturnNull()) 115 return nullptr; 116 ReportCallocOverflow(nmemb, size, &stack); 117 } 118 size *= nmemb; 119 return Allocate(stack, size, 1, true); 120 } 121 122 void Deallocate(void *p) { 123 if (&__sanitizer_free_hook) __sanitizer_free_hook(p); 124 RunFreeHooks(p); 125 RegisterDeallocation(p); 126 allocator.Deallocate(GetAllocatorCache(), p); 127 } 128 129 void *Reallocate(const StackTrace &stack, void *p, uptr new_size, 130 uptr alignment) { 131 if (new_size > max_malloc_size) { 132 ReportAllocationSizeTooBig(new_size, stack); 133 return nullptr; 134 } 135 RegisterDeallocation(p); 136 void *new_p = 137 allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment); 138 if (new_p) 139 RegisterAllocation(stack, new_p, new_size); 140 else if (new_size != 0) 141 RegisterAllocation(stack, p, new_size); 142 return new_p; 143 } 144 145 void GetAllocatorCacheRange(uptr *begin, uptr *end) { 146 *begin = (uptr)GetAllocatorCache(); 147 *end = *begin + sizeof(AllocatorCache); 148 } 149 150 uptr GetMallocUsableSize(const void *p) { 151 ChunkMetadata *m = Metadata(p); 152 if (!m) return 0; 153 return m->requested_size; 154 } 155 156 int lsan_posix_memalign(void **memptr, uptr alignment, uptr size, 157 const StackTrace &stack) { 158 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { 159 if (AllocatorMayReturnNull()) 160 return errno_EINVAL; 161 ReportInvalidPosixMemalignAlignment(alignment, &stack); 162 } 163 void *ptr = Allocate(stack, size, alignment, kAlwaysClearMemory); 164 if (UNLIKELY(!ptr)) 165 // OOM error is already taken care of by Allocate. 166 return errno_ENOMEM; 167 CHECK(IsAligned((uptr)ptr, alignment)); 168 *memptr = ptr; 169 return 0; 170 } 171 172 void *lsan_aligned_alloc(uptr alignment, uptr size, const StackTrace &stack) { 173 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { 174 errno = errno_EINVAL; 175 if (AllocatorMayReturnNull()) 176 return nullptr; 177 ReportInvalidAlignedAllocAlignment(size, alignment, &stack); 178 } 179 return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory)); 180 } 181 182 void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack) { 183 if (UNLIKELY(!IsPowerOfTwo(alignment))) { 184 errno = errno_EINVAL; 185 if (AllocatorMayReturnNull()) 186 return nullptr; 187 ReportInvalidAllocationAlignment(alignment, &stack); 188 } 189 return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory)); 190 } 191 192 void *lsan_malloc(uptr size, const StackTrace &stack) { 193 return SetErrnoOnNull(Allocate(stack, size, 1, kAlwaysClearMemory)); 194 } 195 196 void lsan_free(void *p) { 197 Deallocate(p); 198 } 199 200 void *lsan_realloc(void *p, uptr size, const StackTrace &stack) { 201 return SetErrnoOnNull(Reallocate(stack, p, size, 1)); 202 } 203 204 void *lsan_reallocarray(void *ptr, uptr nmemb, uptr size, 205 const StackTrace &stack) { 206 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { 207 errno = errno_ENOMEM; 208 if (AllocatorMayReturnNull()) 209 return nullptr; 210 ReportReallocArrayOverflow(nmemb, size, &stack); 211 } 212 return lsan_realloc(ptr, nmemb * size, stack); 213 } 214 215 void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) { 216 return SetErrnoOnNull(Calloc(nmemb, size, stack)); 217 } 218 219 void *lsan_valloc(uptr size, const StackTrace &stack) { 220 return SetErrnoOnNull( 221 Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory)); 222 } 223 224 void *lsan_pvalloc(uptr size, const StackTrace &stack) { 225 uptr PageSize = GetPageSizeCached(); 226 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { 227 errno = errno_ENOMEM; 228 if (AllocatorMayReturnNull()) 229 return nullptr; 230 ReportPvallocOverflow(size, &stack); 231 } 232 // pvalloc(0) should allocate one page. 233 size = size ? RoundUpTo(size, PageSize) : PageSize; 234 return SetErrnoOnNull(Allocate(stack, size, PageSize, kAlwaysClearMemory)); 235 } 236 237 uptr lsan_mz_size(const void *p) { 238 return GetMallocUsableSize(p); 239 } 240 241 ///// Interface to the common LSan module. ///// 242 243 void LockAllocator() { 244 allocator.ForceLock(); 245 } 246 247 void UnlockAllocator() { 248 allocator.ForceUnlock(); 249 } 250 251 void GetAllocatorGlobalRange(uptr *begin, uptr *end) { 252 *begin = (uptr)&allocator; 253 *end = *begin + sizeof(allocator); 254 } 255 256 uptr PointsIntoChunk(void* p) { 257 uptr addr = reinterpret_cast<uptr>(p); 258 uptr chunk = reinterpret_cast<uptr>(allocator.GetBlockBeginFastLocked(p)); 259 if (!chunk) return 0; 260 // LargeMmapAllocator considers pointers to the meta-region of a chunk to be 261 // valid, but we don't want that. 262 if (addr < chunk) return 0; 263 ChunkMetadata *m = Metadata(reinterpret_cast<void *>(chunk)); 264 CHECK(m); 265 if (!m->allocated) 266 return 0; 267 if (addr < chunk + m->requested_size) 268 return chunk; 269 if (IsSpecialCaseOfOperatorNew0(chunk, m->requested_size, addr)) 270 return chunk; 271 return 0; 272 } 273 274 uptr GetUserBegin(uptr chunk) { 275 return chunk; 276 } 277 278 LsanMetadata::LsanMetadata(uptr chunk) { 279 metadata_ = Metadata(reinterpret_cast<void *>(chunk)); 280 CHECK(metadata_); 281 } 282 283 bool LsanMetadata::allocated() const { 284 return reinterpret_cast<ChunkMetadata *>(metadata_)->allocated; 285 } 286 287 ChunkTag LsanMetadata::tag() const { 288 return reinterpret_cast<ChunkMetadata *>(metadata_)->tag; 289 } 290 291 void LsanMetadata::set_tag(ChunkTag value) { 292 reinterpret_cast<ChunkMetadata *>(metadata_)->tag = value; 293 } 294 295 uptr LsanMetadata::requested_size() const { 296 return reinterpret_cast<ChunkMetadata *>(metadata_)->requested_size; 297 } 298 299 u32 LsanMetadata::stack_trace_id() const { 300 return reinterpret_cast<ChunkMetadata *>(metadata_)->stack_trace_id; 301 } 302 303 void ForEachChunk(ForEachChunkCallback callback, void *arg) { 304 allocator.ForEachChunk(callback, arg); 305 } 306 307 IgnoreObjectResult IgnoreObjectLocked(const void *p) { 308 void *chunk = allocator.GetBlockBegin(p); 309 if (!chunk || p < chunk) return kIgnoreObjectInvalid; 310 ChunkMetadata *m = Metadata(chunk); 311 CHECK(m); 312 if (m->allocated && (uptr)p < (uptr)chunk + m->requested_size) { 313 if (m->tag == kIgnored) 314 return kIgnoreObjectAlreadyIgnored; 315 m->tag = kIgnored; 316 return kIgnoreObjectSuccess; 317 } else { 318 return kIgnoreObjectInvalid; 319 } 320 } 321 322 void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) { 323 // This function can be used to treat memory reachable from `tctx` as live. 324 // This is useful for threads that have been created but not yet started. 325 326 // This is currently a no-op because the LSan `pthread_create()` interceptor 327 // blocks until the child thread starts which keeps the thread's `arg` pointer 328 // live. 329 } 330 331 } // namespace __lsan 332 333 using namespace __lsan; 334 335 extern "C" { 336 SANITIZER_INTERFACE_ATTRIBUTE 337 uptr __sanitizer_get_current_allocated_bytes() { 338 uptr stats[AllocatorStatCount]; 339 allocator.GetStats(stats); 340 return stats[AllocatorStatAllocated]; 341 } 342 343 SANITIZER_INTERFACE_ATTRIBUTE 344 uptr __sanitizer_get_heap_size() { 345 uptr stats[AllocatorStatCount]; 346 allocator.GetStats(stats); 347 return stats[AllocatorStatMapped]; 348 } 349 350 SANITIZER_INTERFACE_ATTRIBUTE 351 uptr __sanitizer_get_free_bytes() { return 0; } 352 353 SANITIZER_INTERFACE_ATTRIBUTE 354 uptr __sanitizer_get_unmapped_bytes() { return 0; } 355 356 SANITIZER_INTERFACE_ATTRIBUTE 357 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } 358 359 SANITIZER_INTERFACE_ATTRIBUTE 360 int __sanitizer_get_ownership(const void *p) { return Metadata(p) != nullptr; } 361 362 SANITIZER_INTERFACE_ATTRIBUTE 363 uptr __sanitizer_get_allocated_size(const void *p) { 364 return GetMallocUsableSize(p); 365 } 366 367 #if !SANITIZER_SUPPORTS_WEAK_HOOKS 368 // Provide default (no-op) implementation of malloc hooks. 369 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE 370 void __sanitizer_malloc_hook(void *ptr, uptr size) { 371 (void)ptr; 372 (void)size; 373 } 374 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE 375 void __sanitizer_free_hook(void *ptr) { 376 (void)ptr; 377 } 378 #endif 379 } // extern "C" 380