1 //=-- lsan_allocator.cpp --------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of LeakSanitizer. 10 // See lsan_allocator.h for details. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "lsan_allocator.h" 15 16 #include "sanitizer_common/sanitizer_allocator.h" 17 #include "sanitizer_common/sanitizer_allocator_checks.h" 18 #include "sanitizer_common/sanitizer_allocator_interface.h" 19 #include "sanitizer_common/sanitizer_allocator_report.h" 20 #include "sanitizer_common/sanitizer_errno.h" 21 #include "sanitizer_common/sanitizer_internal_defs.h" 22 #include "sanitizer_common/sanitizer_stackdepot.h" 23 #include "sanitizer_common/sanitizer_stacktrace.h" 24 #include "lsan_common.h" 25 26 extern "C" void *memset(void *ptr, int value, uptr num); 27 28 namespace __lsan { 29 #if defined(__i386__) || defined(__arm__) 30 static const uptr kMaxAllowedMallocSize = 1ULL << 30; 31 #elif defined(__mips64) || defined(__aarch64__) 32 static const uptr kMaxAllowedMallocSize = 4ULL << 30; 33 #else 34 static const uptr kMaxAllowedMallocSize = 8ULL << 30; 35 #endif 36 37 static Allocator allocator; 38 39 static uptr max_malloc_size; 40 41 void InitializeAllocator() { 42 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); 43 allocator.InitLinkerInitialized( 44 common_flags()->allocator_release_to_os_interval_ms); 45 if (common_flags()->max_allocation_size_mb) 46 max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20, 47 kMaxAllowedMallocSize); 48 else 49 max_malloc_size = kMaxAllowedMallocSize; 50 } 51 52 void AllocatorThreadStart() { allocator.InitCache(GetAllocatorCache()); } 53 54 void AllocatorThreadFinish() { 55 allocator.SwallowCache(GetAllocatorCache()); 56 allocator.DestroyCache(GetAllocatorCache()); 57 } 58 59 static ChunkMetadata *Metadata(const void *p) { 60 return reinterpret_cast<ChunkMetadata *>(allocator.GetMetaData(p)); 61 } 62 63 static void RegisterAllocation(const StackTrace &stack, void *p, uptr size) { 64 if (!p) return; 65 ChunkMetadata *m = Metadata(p); 66 CHECK(m); 67 m->tag = DisabledInThisThread() ? kIgnored : kDirectlyLeaked; 68 m->stack_trace_id = StackDepotPut(stack); 69 m->requested_size = size; 70 atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 1, memory_order_relaxed); 71 RunMallocHooks(p, size); 72 } 73 74 static void RegisterDeallocation(void *p) { 75 if (!p) return; 76 ChunkMetadata *m = Metadata(p); 77 CHECK(m); 78 RunFreeHooks(p); 79 atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 0, memory_order_relaxed); 80 } 81 82 static void *ReportAllocationSizeTooBig(uptr size, const StackTrace &stack) { 83 if (AllocatorMayReturnNull()) { 84 Report("WARNING: LeakSanitizer failed to allocate 0x%zx bytes\n", size); 85 return nullptr; 86 } 87 ReportAllocationSizeTooBig(size, max_malloc_size, &stack); 88 } 89 90 void *Allocate(const StackTrace &stack, uptr size, uptr alignment, 91 bool cleared) { 92 if (size == 0) 93 size = 1; 94 if (size > max_malloc_size) 95 return ReportAllocationSizeTooBig(size, stack); 96 if (UNLIKELY(IsRssLimitExceeded())) { 97 if (AllocatorMayReturnNull()) 98 return nullptr; 99 ReportRssLimitExceeded(&stack); 100 } 101 void *p = allocator.Allocate(GetAllocatorCache(), size, alignment); 102 if (UNLIKELY(!p)) { 103 SetAllocatorOutOfMemory(); 104 if (AllocatorMayReturnNull()) 105 return nullptr; 106 ReportOutOfMemory(size, &stack); 107 } 108 // Do not rely on the allocator to clear the memory (it's slow). 109 if (cleared && allocator.FromPrimary(p)) 110 memset(p, 0, size); 111 RegisterAllocation(stack, p, size); 112 return p; 113 } 114 115 static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) { 116 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { 117 if (AllocatorMayReturnNull()) 118 return nullptr; 119 ReportCallocOverflow(nmemb, size, &stack); 120 } 121 size *= nmemb; 122 return Allocate(stack, size, 1, true); 123 } 124 125 void Deallocate(void *p) { 126 RegisterDeallocation(p); 127 allocator.Deallocate(GetAllocatorCache(), p); 128 } 129 130 void *Reallocate(const StackTrace &stack, void *p, uptr new_size, 131 uptr alignment) { 132 if (new_size > max_malloc_size) { 133 ReportAllocationSizeTooBig(new_size, stack); 134 return nullptr; 135 } 136 RegisterDeallocation(p); 137 void *new_p = 138 allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment); 139 if (new_p) 140 RegisterAllocation(stack, new_p, new_size); 141 else if (new_size != 0) 142 RegisterAllocation(stack, p, new_size); 143 return new_p; 144 } 145 146 void GetAllocatorCacheRange(uptr *begin, uptr *end) { 147 *begin = (uptr)GetAllocatorCache(); 148 *end = *begin + sizeof(AllocatorCache); 149 } 150 151 static const void *GetMallocBegin(const void *p) { 152 if (!p) 153 return nullptr; 154 void *beg = allocator.GetBlockBegin(p); 155 if (!beg) 156 return nullptr; 157 ChunkMetadata *m = Metadata(beg); 158 if (!m) 159 return nullptr; 160 if (!m->allocated) 161 return nullptr; 162 if (m->requested_size == 0) 163 return nullptr; 164 return (const void *)beg; 165 } 166 167 uptr GetMallocUsableSize(const void *p) { 168 if (!p) 169 return 0; 170 ChunkMetadata *m = Metadata(p); 171 if (!m) return 0; 172 return m->requested_size; 173 } 174 175 uptr GetMallocUsableSizeFast(const void *p) { 176 return Metadata(p)->requested_size; 177 } 178 179 int lsan_posix_memalign(void **memptr, uptr alignment, uptr size, 180 const StackTrace &stack) { 181 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { 182 if (AllocatorMayReturnNull()) 183 return errno_EINVAL; 184 ReportInvalidPosixMemalignAlignment(alignment, &stack); 185 } 186 void *ptr = Allocate(stack, size, alignment, kAlwaysClearMemory); 187 if (UNLIKELY(!ptr)) 188 // OOM error is already taken care of by Allocate. 189 return errno_ENOMEM; 190 CHECK(IsAligned((uptr)ptr, alignment)); 191 *memptr = ptr; 192 return 0; 193 } 194 195 void *lsan_aligned_alloc(uptr alignment, uptr size, const StackTrace &stack) { 196 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { 197 errno = errno_EINVAL; 198 if (AllocatorMayReturnNull()) 199 return nullptr; 200 ReportInvalidAlignedAllocAlignment(size, alignment, &stack); 201 } 202 return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory)); 203 } 204 205 void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack) { 206 if (UNLIKELY(!IsPowerOfTwo(alignment))) { 207 errno = errno_EINVAL; 208 if (AllocatorMayReturnNull()) 209 return nullptr; 210 ReportInvalidAllocationAlignment(alignment, &stack); 211 } 212 return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory)); 213 } 214 215 void *lsan_malloc(uptr size, const StackTrace &stack) { 216 return SetErrnoOnNull(Allocate(stack, size, 1, kAlwaysClearMemory)); 217 } 218 219 void lsan_free(void *p) { 220 Deallocate(p); 221 } 222 223 void *lsan_realloc(void *p, uptr size, const StackTrace &stack) { 224 return SetErrnoOnNull(Reallocate(stack, p, size, 1)); 225 } 226 227 void *lsan_reallocarray(void *ptr, uptr nmemb, uptr size, 228 const StackTrace &stack) { 229 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { 230 errno = errno_ENOMEM; 231 if (AllocatorMayReturnNull()) 232 return nullptr; 233 ReportReallocArrayOverflow(nmemb, size, &stack); 234 } 235 return lsan_realloc(ptr, nmemb * size, stack); 236 } 237 238 void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) { 239 return SetErrnoOnNull(Calloc(nmemb, size, stack)); 240 } 241 242 void *lsan_valloc(uptr size, const StackTrace &stack) { 243 return SetErrnoOnNull( 244 Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory)); 245 } 246 247 void *lsan_pvalloc(uptr size, const StackTrace &stack) { 248 uptr PageSize = GetPageSizeCached(); 249 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { 250 errno = errno_ENOMEM; 251 if (AllocatorMayReturnNull()) 252 return nullptr; 253 ReportPvallocOverflow(size, &stack); 254 } 255 // pvalloc(0) should allocate one page. 256 size = size ? RoundUpTo(size, PageSize) : PageSize; 257 return SetErrnoOnNull(Allocate(stack, size, PageSize, kAlwaysClearMemory)); 258 } 259 260 uptr lsan_mz_size(const void *p) { 261 return GetMallocUsableSize(p); 262 } 263 264 ///// Interface to the common LSan module. ///// 265 266 void LockAllocator() { 267 allocator.ForceLock(); 268 } 269 270 void UnlockAllocator() { 271 allocator.ForceUnlock(); 272 } 273 274 void GetAllocatorGlobalRange(uptr *begin, uptr *end) { 275 *begin = (uptr)&allocator; 276 *end = *begin + sizeof(allocator); 277 } 278 279 uptr PointsIntoChunk(void* p) { 280 uptr addr = reinterpret_cast<uptr>(p); 281 uptr chunk = reinterpret_cast<uptr>(allocator.GetBlockBeginFastLocked(p)); 282 if (!chunk) return 0; 283 // LargeMmapAllocator considers pointers to the meta-region of a chunk to be 284 // valid, but we don't want that. 285 if (addr < chunk) return 0; 286 ChunkMetadata *m = Metadata(reinterpret_cast<void *>(chunk)); 287 CHECK(m); 288 if (!m->allocated) 289 return 0; 290 if (addr < chunk + m->requested_size) 291 return chunk; 292 if (IsSpecialCaseOfOperatorNew0(chunk, m->requested_size, addr)) 293 return chunk; 294 return 0; 295 } 296 297 uptr GetUserBegin(uptr chunk) { 298 return chunk; 299 } 300 301 uptr GetUserAddr(uptr chunk) { 302 return chunk; 303 } 304 305 LsanMetadata::LsanMetadata(uptr chunk) { 306 metadata_ = Metadata(reinterpret_cast<void *>(chunk)); 307 CHECK(metadata_); 308 } 309 310 bool LsanMetadata::allocated() const { 311 return reinterpret_cast<ChunkMetadata *>(metadata_)->allocated; 312 } 313 314 ChunkTag LsanMetadata::tag() const { 315 return reinterpret_cast<ChunkMetadata *>(metadata_)->tag; 316 } 317 318 void LsanMetadata::set_tag(ChunkTag value) { 319 reinterpret_cast<ChunkMetadata *>(metadata_)->tag = value; 320 } 321 322 uptr LsanMetadata::requested_size() const { 323 return reinterpret_cast<ChunkMetadata *>(metadata_)->requested_size; 324 } 325 326 u32 LsanMetadata::stack_trace_id() const { 327 return reinterpret_cast<ChunkMetadata *>(metadata_)->stack_trace_id; 328 } 329 330 void ForEachChunk(ForEachChunkCallback callback, void *arg) { 331 allocator.ForEachChunk(callback, arg); 332 } 333 334 IgnoreObjectResult IgnoreObject(const void *p) { 335 void *chunk = allocator.GetBlockBegin(p); 336 if (!chunk || p < chunk) return kIgnoreObjectInvalid; 337 ChunkMetadata *m = Metadata(chunk); 338 CHECK(m); 339 if (m->allocated && (uptr)p < (uptr)chunk + m->requested_size) { 340 if (m->tag == kIgnored) 341 return kIgnoreObjectAlreadyIgnored; 342 m->tag = kIgnored; 343 return kIgnoreObjectSuccess; 344 } else { 345 return kIgnoreObjectInvalid; 346 } 347 } 348 349 } // namespace __lsan 350 351 using namespace __lsan; 352 353 extern "C" { 354 SANITIZER_INTERFACE_ATTRIBUTE 355 uptr __sanitizer_get_current_allocated_bytes() { 356 uptr stats[AllocatorStatCount]; 357 allocator.GetStats(stats); 358 return stats[AllocatorStatAllocated]; 359 } 360 361 SANITIZER_INTERFACE_ATTRIBUTE 362 uptr __sanitizer_get_heap_size() { 363 uptr stats[AllocatorStatCount]; 364 allocator.GetStats(stats); 365 return stats[AllocatorStatMapped]; 366 } 367 368 SANITIZER_INTERFACE_ATTRIBUTE 369 uptr __sanitizer_get_free_bytes() { return 1; } 370 371 SANITIZER_INTERFACE_ATTRIBUTE 372 uptr __sanitizer_get_unmapped_bytes() { return 0; } 373 374 SANITIZER_INTERFACE_ATTRIBUTE 375 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } 376 377 SANITIZER_INTERFACE_ATTRIBUTE 378 int __sanitizer_get_ownership(const void *p) { 379 return GetMallocBegin(p) != nullptr; 380 } 381 382 SANITIZER_INTERFACE_ATTRIBUTE 383 const void * __sanitizer_get_allocated_begin(const void *p) { 384 return GetMallocBegin(p); 385 } 386 387 SANITIZER_INTERFACE_ATTRIBUTE 388 uptr __sanitizer_get_allocated_size(const void *p) { 389 return GetMallocUsableSize(p); 390 } 391 392 SANITIZER_INTERFACE_ATTRIBUTE 393 uptr __sanitizer_get_allocated_size_fast(const void *p) { 394 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p)); 395 uptr ret = GetMallocUsableSizeFast(p); 396 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p)); 397 return ret; 398 } 399 400 SANITIZER_INTERFACE_ATTRIBUTE 401 void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); } 402 403 } // extern "C" 404