1 //===-- sanitizer_allocator.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is shared between AddressSanitizer and ThreadSanitizer 10 // run-time libraries. 11 // This allocator is used inside run-times. 12 //===----------------------------------------------------------------------===// 13 14 #include "sanitizer_allocator.h" 15 16 #include "sanitizer_allocator_checks.h" 17 #include "sanitizer_allocator_internal.h" 18 #include "sanitizer_atomic.h" 19 #include "sanitizer_common.h" 20 21 namespace __sanitizer { 22 23 // Default allocator names. 24 const char *PrimaryAllocatorName = "SizeClassAllocator"; 25 const char *SecondaryAllocatorName = "LargeMmapAllocator"; 26 27 // ThreadSanitizer for Go uses libc malloc/free. 28 #if defined(SANITIZER_USE_MALLOC) 29 # if SANITIZER_LINUX && !SANITIZER_ANDROID 30 extern "C" void *__libc_malloc(uptr size); 31 # if !SANITIZER_GO 32 extern "C" void *__libc_memalign(uptr alignment, uptr size); 33 # endif 34 extern "C" void *__libc_realloc(void *ptr, uptr size); 35 extern "C" void __libc_free(void *ptr); 36 # else 37 # include <stdlib.h> 38 # define __libc_malloc malloc 39 # if !SANITIZER_GO 40 static void *__libc_memalign(uptr alignment, uptr size) { 41 void *p; 42 uptr error = posix_memalign(&p, alignment, size); 43 if (error) return nullptr; 44 return p; 45 } 46 # endif 47 # define __libc_realloc realloc 48 # define __libc_free free 49 # endif 50 51 static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, 52 uptr alignment) { 53 (void)cache; 54 #if !SANITIZER_GO 55 if (alignment == 0) 56 return __libc_malloc(size); 57 else 58 return __libc_memalign(alignment, size); 59 #else 60 // Windows does not provide __libc_memalign/posix_memalign. It provides 61 // __aligned_malloc, but the allocated blocks can't be passed to free, 62 // they need to be passed to __aligned_free. InternalAlloc interface does 63 // not account for such requirement. Alignemnt does not seem to be used 64 // anywhere in runtime, so just call __libc_malloc for now. 65 DCHECK_EQ(alignment, 0); 66 return __libc_malloc(size); 67 #endif 68 } 69 70 static void *RawInternalRealloc(void *ptr, uptr size, 71 InternalAllocatorCache *cache) { 72 (void)cache; 73 return __libc_realloc(ptr, size); 74 } 75 76 static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { 77 (void)cache; 78 __libc_free(ptr); 79 } 80 81 InternalAllocator *internal_allocator() { 82 return 0; 83 } 84 85 #else // SANITIZER_GO || defined(SANITIZER_USE_MALLOC) 86 87 static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)]; 88 static atomic_uint8_t internal_allocator_initialized; 89 static StaticSpinMutex internal_alloc_init_mu; 90 91 static InternalAllocatorCache internal_allocator_cache; 92 static StaticSpinMutex internal_allocator_cache_mu; 93 94 InternalAllocator *internal_allocator() { 95 InternalAllocator *internal_allocator_instance = 96 reinterpret_cast<InternalAllocator *>(&internal_alloc_placeholder); 97 if (atomic_load(&internal_allocator_initialized, memory_order_acquire) == 0) { 98 SpinMutexLock l(&internal_alloc_init_mu); 99 if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) == 100 0) { 101 internal_allocator_instance->Init(kReleaseToOSIntervalNever); 102 atomic_store(&internal_allocator_initialized, 1, memory_order_release); 103 } 104 } 105 return internal_allocator_instance; 106 } 107 108 static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, 109 uptr alignment) { 110 if (alignment == 0) alignment = 8; 111 if (cache == 0) { 112 SpinMutexLock l(&internal_allocator_cache_mu); 113 return internal_allocator()->Allocate(&internal_allocator_cache, size, 114 alignment); 115 } 116 return internal_allocator()->Allocate(cache, size, alignment); 117 } 118 119 static void *RawInternalRealloc(void *ptr, uptr size, 120 InternalAllocatorCache *cache) { 121 uptr alignment = 8; 122 if (cache == 0) { 123 SpinMutexLock l(&internal_allocator_cache_mu); 124 return internal_allocator()->Reallocate(&internal_allocator_cache, ptr, 125 size, alignment); 126 } 127 return internal_allocator()->Reallocate(cache, ptr, size, alignment); 128 } 129 130 static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { 131 if (!cache) { 132 SpinMutexLock l(&internal_allocator_cache_mu); 133 return internal_allocator()->Deallocate(&internal_allocator_cache, ptr); 134 } 135 internal_allocator()->Deallocate(cache, ptr); 136 } 137 138 #endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC) 139 140 const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull; 141 142 static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) { 143 SetAllocatorOutOfMemory(); 144 Report("FATAL: %s: internal allocator is out of memory trying to allocate " 145 "0x%zx bytes\n", SanitizerToolName, requested_size); 146 Die(); 147 } 148 149 void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) { 150 if (size + sizeof(u64) < size) 151 return nullptr; 152 void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment); 153 if (UNLIKELY(!p)) 154 ReportInternalAllocatorOutOfMemory(size + sizeof(u64)); 155 ((u64*)p)[0] = kBlockMagic; 156 return (char*)p + sizeof(u64); 157 } 158 159 void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) { 160 if (!addr) 161 return InternalAlloc(size, cache); 162 if (size + sizeof(u64) < size) 163 return nullptr; 164 addr = (char*)addr - sizeof(u64); 165 size = size + sizeof(u64); 166 CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); 167 void *p = RawInternalRealloc(addr, size, cache); 168 if (UNLIKELY(!p)) 169 ReportInternalAllocatorOutOfMemory(size); 170 return (char*)p + sizeof(u64); 171 } 172 173 void *InternalReallocArray(void *addr, uptr count, uptr size, 174 InternalAllocatorCache *cache) { 175 if (UNLIKELY(CheckForCallocOverflow(count, size))) { 176 Report( 177 "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) " 178 "cannot be represented in type size_t\n", 179 SanitizerToolName, count, size); 180 Die(); 181 } 182 return InternalRealloc(addr, count * size, cache); 183 } 184 185 void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { 186 if (UNLIKELY(CheckForCallocOverflow(count, size))) { 187 Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) " 188 "cannot be represented in type size_t\n", SanitizerToolName, count, 189 size); 190 Die(); 191 } 192 void *p = InternalAlloc(count * size, cache); 193 if (LIKELY(p)) 194 internal_memset(p, 0, count * size); 195 return p; 196 } 197 198 void InternalFree(void *addr, InternalAllocatorCache *cache) { 199 if (!addr) 200 return; 201 addr = (char*)addr - sizeof(u64); 202 CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); 203 ((u64*)addr)[0] = 0; 204 RawInternalFree(addr, cache); 205 } 206 207 // LowLevelAllocator 208 constexpr uptr kLowLevelAllocatorDefaultAlignment = 8; 209 static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment; 210 static LowLevelAllocateCallback low_level_alloc_callback; 211 212 void *LowLevelAllocator::Allocate(uptr size) { 213 // Align allocation size. 214 size = RoundUpTo(size, low_level_alloc_min_alignment); 215 if (allocated_end_ - allocated_current_ < (sptr)size) { 216 uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached()); 217 allocated_current_ = 218 (char*)MmapOrDie(size_to_allocate, __func__); 219 allocated_end_ = allocated_current_ + size_to_allocate; 220 if (low_level_alloc_callback) { 221 low_level_alloc_callback((uptr)allocated_current_, 222 size_to_allocate); 223 } 224 } 225 CHECK(allocated_end_ - allocated_current_ >= (sptr)size); 226 void *res = allocated_current_; 227 allocated_current_ += size; 228 return res; 229 } 230 231 void SetLowLevelAllocateMinAlignment(uptr alignment) { 232 CHECK(IsPowerOfTwo(alignment)); 233 low_level_alloc_min_alignment = Max(alignment, low_level_alloc_min_alignment); 234 } 235 236 void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) { 237 low_level_alloc_callback = callback; 238 } 239 240 // Allocator's OOM and other errors handling support. 241 242 static atomic_uint8_t allocator_out_of_memory = {0}; 243 static atomic_uint8_t allocator_may_return_null = {0}; 244 245 bool IsAllocatorOutOfMemory() { 246 return atomic_load_relaxed(&allocator_out_of_memory); 247 } 248 249 void SetAllocatorOutOfMemory() { 250 atomic_store_relaxed(&allocator_out_of_memory, 1); 251 } 252 253 bool AllocatorMayReturnNull() { 254 return atomic_load(&allocator_may_return_null, memory_order_relaxed); 255 } 256 257 void SetAllocatorMayReturnNull(bool may_return_null) { 258 atomic_store(&allocator_may_return_null, may_return_null, 259 memory_order_relaxed); 260 } 261 262 void PrintHintAllocatorCannotReturnNull() { 263 Report("HINT: if you don't care about these errors you may set " 264 "allocator_may_return_null=1\n"); 265 } 266 267 } // namespace __sanitizer 268