1 //===-- asan_allocator.h ----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of AddressSanitizer, an address sanity checker. 10 // 11 // ASan-private header for asan_allocator.cpp. 12 //===----------------------------------------------------------------------===// 13 14 #ifndef ASAN_ALLOCATOR_H 15 #define ASAN_ALLOCATOR_H 16 17 #include "asan_flags.h" 18 #include "asan_interceptors.h" 19 #include "asan_internal.h" 20 #include "sanitizer_common/sanitizer_allocator.h" 21 #include "sanitizer_common/sanitizer_list.h" 22 #include "sanitizer_common/sanitizer_platform.h" 23 24 namespace __asan { 25 26 enum AllocType { 27 FROM_MALLOC = 1, // Memory block came from malloc, calloc, realloc, etc. 28 FROM_NEW = 2, // Memory block came from operator new. 29 FROM_NEW_BR = 3 // Memory block came from operator new [ ] 30 }; 31 32 class AsanChunk; 33 34 struct AllocatorOptions { 35 u32 quarantine_size_mb; 36 u32 thread_local_quarantine_size_kb; 37 u16 min_redzone; 38 u16 max_redzone; 39 u8 may_return_null; 40 u8 alloc_dealloc_mismatch; 41 s32 release_to_os_interval_ms; 42 43 void SetFrom(const Flags *f, const CommonFlags *cf); 44 void CopyTo(Flags *f, CommonFlags *cf); 45 }; 46 47 void InitializeAllocator(const AllocatorOptions &options); 48 void ReInitializeAllocator(const AllocatorOptions &options); 49 void GetAllocatorOptions(AllocatorOptions *options); 50 void ApplyAllocatorOptions(const AllocatorOptions &options); 51 52 class AsanChunkView { 53 public: AsanChunkView(AsanChunk * chunk)54 explicit AsanChunkView(AsanChunk *chunk) : chunk_(chunk) {} 55 bool IsValid() const; // Checks if AsanChunkView points to a valid 56 // allocated or quarantined chunk. 57 bool IsAllocated() const; // Checks if the memory is currently allocated. 58 bool IsQuarantined() const; // Checks if the memory is currently quarantined. 59 uptr Beg() const; // First byte of user memory. 60 uptr End() const; // Last byte of user memory. 61 uptr UsedSize() const; // Size requested by the user. 62 u32 UserRequestedAlignment() const; // Originally requested alignment. 63 uptr AllocTid() const; 64 uptr FreeTid() const; Eq(const AsanChunkView & c)65 bool Eq(const AsanChunkView &c) const { return chunk_ == c.chunk_; } 66 u32 GetAllocStackId() const; 67 u32 GetFreeStackId() const; 68 AllocType GetAllocType() const; AddrIsInside(uptr addr,uptr access_size,sptr * offset)69 bool AddrIsInside(uptr addr, uptr access_size, sptr *offset) const { 70 if (addr >= Beg() && (addr + access_size) <= End()) { 71 *offset = addr - Beg(); 72 return true; 73 } 74 return false; 75 } AddrIsAtLeft(uptr addr,uptr access_size,sptr * offset)76 bool AddrIsAtLeft(uptr addr, uptr access_size, sptr *offset) const { 77 (void)access_size; 78 if (addr < Beg()) { 79 *offset = Beg() - addr; 80 return true; 81 } 82 return false; 83 } AddrIsAtRight(uptr addr,uptr access_size,sptr * offset)84 bool AddrIsAtRight(uptr addr, uptr access_size, sptr *offset) const { 85 if (addr + access_size > End()) { 86 *offset = addr - End(); 87 return true; 88 } 89 return false; 90 } 91 92 private: 93 AsanChunk *const chunk_; 94 }; 95 96 AsanChunkView FindHeapChunkByAddress(uptr address); 97 AsanChunkView FindHeapChunkByAllocBeg(uptr address); 98 99 // List of AsanChunks with total size. 100 class AsanChunkFifoList: public IntrusiveList<AsanChunk> { 101 public: AsanChunkFifoList(LinkerInitialized)102 explicit AsanChunkFifoList(LinkerInitialized) { } AsanChunkFifoList()103 AsanChunkFifoList() { clear(); } 104 void Push(AsanChunk *n); 105 void PushList(AsanChunkFifoList *q); 106 AsanChunk *Pop(); size()107 uptr size() { return size_; } clear()108 void clear() { 109 IntrusiveList<AsanChunk>::clear(); 110 size_ = 0; 111 } 112 private: 113 uptr size_; 114 }; 115 116 struct AsanMapUnmapCallback { 117 void OnMap(uptr p, uptr size) const; 118 void OnMapSecondary(uptr p, uptr size, uptr user_begin, uptr user_size) const; 119 void OnUnmap(uptr p, uptr size) const; 120 }; 121 122 #if SANITIZER_CAN_USE_ALLOCATOR64 123 # if SANITIZER_FUCHSIA 124 // This is a sentinel indicating we do not want the primary allocator arena to 125 // be placed at a fixed address. It will be anonymously mmap'd. 126 const uptr kAllocatorSpace = ~(uptr)0; 127 # if SANITIZER_RISCV64 128 129 // These are sanitizer tunings that allow all bringup tests for RISCV-64 Sv39 + 130 // Fuchsia to run with asan-instrumented. That is, we can run bringup, e2e, 131 // libc, and scudo tests with this configuration. 132 // 133 // TODO: This is specifically tuned for Sv39. 48/57 will likely require other 134 // tunings, or possibly use the same tunings Fuchsia uses for other archs. The 135 // VMA size isn't technically tied to the Fuchsia System ABI, so once 48/57 is 136 // supported, we'd need a way of dynamically checking what the VMA size is and 137 // determining optimal configuration. 138 139 // This indicates the total amount of space dedicated for the primary allocator 140 // during initialization. This is roughly proportional to the size set by the 141 // FuchsiaConfig for scudo (~11.25GB == ~2^33.49). Requesting any more could 142 // lead to some failures in sanitized bringup tests where we can't allocate new 143 // vmars because there wouldn't be enough contiguous space. We could try 2^34 if 144 // we re-evaluate the SizeClassMap settings. 145 const uptr kAllocatorSize = UINT64_C(1) << 33; // 8GB 146 147 // This is roughly equivalent to the configuration for the VeryDenseSizeClassMap 148 // but has fewer size classes (ideally at most 32). Fewer class sizes means the 149 // region size for each class is larger, thus less chances of running out of 150 // space for each region. The main differences are the MidSizeLog (which is 151 // smaller) and the MaxSizeLog (which is larger). 152 // 153 // - The MaxSizeLog is higher to allow some of the largest allocations I've 154 // observed to be placed in the primary allocator's arena as opposed to being 155 // mmap'd by the secondary allocator. This helps reduce fragmentation from 156 // large classes. A huge example of this the scudo allocator tests (and its 157 // testing infrastructure) which malloc's/new's objects on the order of 158 // hundreds of kilobytes which normally would not be in the primary allocator 159 // arena with the default VeryDenseSizeClassMap. 160 // - The MidSizeLog is reduced to help shrink the number of size classes and 161 // increase region size. Without this, we'd see ASan complain many times about 162 // a region running out of available space. 163 // 164 // This differs a bit from the fuchsia config in scudo, mainly from the NumBits, 165 // MaxSizeLog, and NumCachedHintT. This should place the number of size classes 166 // for scudo at 45 and some large objects allocated by this config would be 167 // placed in the arena whereas scudo would mmap them. The asan allocator needs 168 // to have a number of classes that are a power of 2 for various internal things 169 // to work, so we can't match the scudo settings to a tee. The sanitizer 170 // allocator is slightly slower than scudo's but this is enough to get 171 // memory-intensive scudo tests to run with asan instrumentation. 172 typedef SizeClassMap</*kNumBits=*/2, 173 /*kMinSizeLog=*/5, 174 /*kMidSizeLog=*/8, 175 /*kMaxSizeLog=*/18, 176 /*kNumCachedHintT=*/8, 177 /*kMaxBytesCachedLog=*/10> 178 SizeClassMap; 179 static_assert(SizeClassMap::kNumClassesRounded <= 32, 180 "The above tunings were specifically selected to ensure there " 181 "would be at most 32 size classes. This restriction could be " 182 "loosened to 64 size classes if we can find a configuration of " 183 "allocator size and SizeClassMap tunings that allows us to " 184 "reliably run all bringup tests in a sanitized environment."); 185 186 # else // SANITIZER_RISCV64 187 // These are the default allocator tunings for non-RISCV environments where the 188 // VMA is usually 48 bits and we have lots of space. 189 const uptr kAllocatorSize = 0x40000000000ULL; // 4T. 190 typedef DefaultSizeClassMap SizeClassMap; 191 # endif // SANITIZER_RISCV64 192 # else // SANITIZER_FUCHSIA 193 194 # if SANITIZER_APPLE 195 const uptr kAllocatorSpace = 0x600000000000ULL; 196 # else // SANITIZER_APPLE 197 const uptr kAllocatorSpace = ~(uptr)0; 198 # endif // SANITIZER_APPLE 199 200 # if defined(__powerpc64__) 201 const uptr kAllocatorSize = 0x20000000000ULL; // 2T. 202 typedef DefaultSizeClassMap SizeClassMap; 203 # elif defined(__aarch64__) && SANITIZER_ANDROID 204 // Android needs to support 39, 42 and 48 bit VMA. 205 const uptr kAllocatorSize = 0x2000000000ULL; // 128G. 206 typedef VeryCompactSizeClassMap SizeClassMap; 207 # elif SANITIZER_RISCV64 208 const uptr kAllocatorSize = 0x2000000000ULL; // 128G. 209 typedef VeryDenseSizeClassMap SizeClassMap; 210 # elif defined(__sparc__) 211 const uptr kAllocatorSize = 0x20000000000ULL; // 2T. 212 typedef DefaultSizeClassMap SizeClassMap; 213 # elif SANITIZER_WINDOWS 214 const uptr kAllocatorSize = 0x8000000000ULL; // 500G 215 typedef DefaultSizeClassMap SizeClassMap; 216 # elif SANITIZER_APPLE 217 const uptr kAllocatorSize = 0x40000000000ULL; // 4T. 218 typedef DefaultSizeClassMap SizeClassMap; 219 # else 220 const uptr kAllocatorSize = 0x40000000000ULL; // 4T. 221 typedef DefaultSizeClassMap SizeClassMap; 222 # endif // defined(__powerpc64__) etc. 223 # endif // SANITIZER_FUCHSIA 224 template <typename AddressSpaceViewTy> 225 struct AP64 { // Allocator64 parameters. Deliberately using a short name. 226 static const uptr kSpaceBeg = kAllocatorSpace; 227 static const uptr kSpaceSize = kAllocatorSize; 228 static const uptr kMetadataSize = 0; 229 typedef __asan::SizeClassMap SizeClassMap; 230 typedef AsanMapUnmapCallback MapUnmapCallback; 231 static const uptr kFlags = 0; 232 using AddressSpaceView = AddressSpaceViewTy; 233 }; 234 235 template <typename AddressSpaceView> 236 using PrimaryAllocatorASVT = SizeClassAllocator64<AP64<AddressSpaceView>>; 237 using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; 238 #else // SANITIZER_CAN_USE_ALLOCATOR64. Fallback to SizeClassAllocator32. 239 typedef CompactSizeClassMap SizeClassMap; 240 template <typename AddressSpaceViewTy> 241 struct AP32 { 242 static const uptr kSpaceBeg = SANITIZER_MMAP_BEGIN; 243 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; 244 static const uptr kMetadataSize = 0; 245 typedef __asan::SizeClassMap SizeClassMap; 246 static const uptr kRegionSizeLog = 20; 247 using AddressSpaceView = AddressSpaceViewTy; 248 typedef AsanMapUnmapCallback MapUnmapCallback; 249 static const uptr kFlags = 0; 250 }; 251 template <typename AddressSpaceView> 252 using PrimaryAllocatorASVT = SizeClassAllocator32<AP32<AddressSpaceView> >; 253 using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; 254 #endif // SANITIZER_CAN_USE_ALLOCATOR64 255 256 static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses; 257 258 template <typename AddressSpaceView> 259 using AsanAllocatorASVT = 260 CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>>; 261 using AsanAllocator = AsanAllocatorASVT<LocalAddressSpaceView>; 262 using AllocatorCache = AsanAllocator::AllocatorCache; 263 264 struct AsanThreadLocalMallocStorage { 265 uptr quarantine_cache[16]; 266 AllocatorCache allocator_cache; 267 void CommitBack(); 268 private: 269 // These objects are allocated via mmap() and are zero-initialized. AsanThreadLocalMallocStorageAsanThreadLocalMallocStorage270 AsanThreadLocalMallocStorage() {} 271 }; 272 273 void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, 274 AllocType alloc_type); 275 void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type); 276 void asan_delete(void *ptr, uptr size, uptr alignment, 277 BufferedStackTrace *stack, AllocType alloc_type); 278 279 void *asan_malloc(uptr size, BufferedStackTrace *stack); 280 void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack); 281 void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack); 282 void *asan_reallocarray(void *p, uptr nmemb, uptr size, 283 BufferedStackTrace *stack); 284 void *asan_valloc(uptr size, BufferedStackTrace *stack); 285 void *asan_pvalloc(uptr size, BufferedStackTrace *stack); 286 287 void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack); 288 int asan_posix_memalign(void **memptr, uptr alignment, uptr size, 289 BufferedStackTrace *stack); 290 uptr asan_malloc_usable_size(const void *ptr, uptr pc, uptr bp); 291 292 uptr asan_mz_size(const void *ptr); 293 void asan_mz_force_lock(); 294 void asan_mz_force_unlock(); 295 296 void PrintInternalAllocatorStats(); 297 void AsanSoftRssLimitExceededCallback(bool exceeded); 298 299 } // namespace __asan 300 #endif // ASAN_ALLOCATOR_H 301