1 //===- nsan_allocator.cpp -------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // NumericalStabilitySanitizer allocator.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "nsan_allocator.h"
14 #include "interception/interception.h"
15 #include "nsan.h"
16 #include "nsan_flags.h"
17 #include "nsan_platform.h"
18 #include "nsan_thread.h"
19 #include "sanitizer_common/sanitizer_allocator.h"
20 #include "sanitizer_common/sanitizer_allocator_checks.h"
21 #include "sanitizer_common/sanitizer_allocator_interface.h"
22 #include "sanitizer_common/sanitizer_allocator_report.h"
23 #include "sanitizer_common/sanitizer_common.h"
24 #include "sanitizer_common/sanitizer_errno.h"
25
26 using namespace __nsan;
27
28 DECLARE_REAL(void *, memcpy, void *dest, const void *src, SIZE_T n)
29 DECLARE_REAL(void *, memset, void *dest, int c, SIZE_T n)
30
31 namespace {
32 struct Metadata {
33 uptr requested_size;
34 };
35
36 struct NsanMapUnmapCallback {
OnMap__anonf933ab720111::NsanMapUnmapCallback37 void OnMap(uptr p, uptr size) const {}
OnMapSecondary__anonf933ab720111::NsanMapUnmapCallback38 void OnMapSecondary(uptr p, uptr size, uptr user_begin,
39 uptr user_size) const {}
OnUnmap__anonf933ab720111::NsanMapUnmapCallback40 void OnUnmap(uptr p, uptr size) const {}
41 };
42
43 const uptr kMaxAllowedMallocSize = 1ULL << 40;
44
45 // Allocator64 parameters. Deliberately using a short name.
46 struct AP64 {
47 static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
48 static const uptr kSpaceSize = 0x40000000000; // 4T.
49 static const uptr kMetadataSize = sizeof(Metadata);
50 using SizeClassMap = DefaultSizeClassMap;
51 using MapUnmapCallback = NsanMapUnmapCallback;
52 static const uptr kFlags = 0;
53 using AddressSpaceView = LocalAddressSpaceView;
54 };
55 } // namespace
56
57 using PrimaryAllocator = SizeClassAllocator64<AP64>;
58 using Allocator = CombinedAllocator<PrimaryAllocator>;
59 using AllocatorCache = Allocator::AllocatorCache;
60
61 static Allocator allocator;
62 static AllocatorCache fallback_allocator_cache;
63 static StaticSpinMutex fallback_mutex;
64
65 static uptr max_malloc_size;
66
NsanAllocatorInit()67 void __nsan::NsanAllocatorInit() {
68 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
69 allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
70 if (common_flags()->max_allocation_size_mb)
71 max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
72 kMaxAllowedMallocSize);
73 else
74 max_malloc_size = kMaxAllowedMallocSize;
75 }
76
GetAllocatorCache(NsanThreadLocalMallocStorage * ms)77 static AllocatorCache *GetAllocatorCache(NsanThreadLocalMallocStorage *ms) {
78 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
79 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
80 }
81
Init()82 void NsanThreadLocalMallocStorage::Init() {
83 allocator.InitCache(GetAllocatorCache(this));
84 }
85
CommitBack()86 void NsanThreadLocalMallocStorage::CommitBack() {
87 allocator.SwallowCache(GetAllocatorCache(this));
88 allocator.DestroyCache(GetAllocatorCache(this));
89 }
90
NsanAllocate(uptr size,uptr alignment,bool zero)91 static void *NsanAllocate(uptr size, uptr alignment, bool zero) {
92 if (UNLIKELY(size > max_malloc_size)) {
93 if (AllocatorMayReturnNull()) {
94 Report("WARNING: NumericalStabilitySanitizer failed to allocate 0x%zx "
95 "bytes\n",
96 size);
97 return nullptr;
98 }
99 BufferedStackTrace stack;
100 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
101 ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
102 }
103 if (UNLIKELY(IsRssLimitExceeded())) {
104 if (AllocatorMayReturnNull())
105 return nullptr;
106 BufferedStackTrace stack;
107 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
108 ReportRssLimitExceeded(&stack);
109 }
110
111 void *allocated;
112 if (NsanThread *t = GetCurrentThread()) {
113 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
114 allocated = allocator.Allocate(cache, size, alignment);
115 } else {
116 SpinMutexLock l(&fallback_mutex);
117 AllocatorCache *cache = &fallback_allocator_cache;
118 allocated = allocator.Allocate(cache, size, alignment);
119 }
120 if (UNLIKELY(!allocated)) {
121 SetAllocatorOutOfMemory();
122 if (AllocatorMayReturnNull())
123 return nullptr;
124 BufferedStackTrace stack;
125 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
126 ReportOutOfMemory(size, &stack);
127 }
128 auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
129 meta->requested_size = size;
130 if (zero && allocator.FromPrimary(allocated))
131 REAL(memset)(allocated, 0, size);
132 __nsan_set_value_unknown(allocated, size);
133 RunMallocHooks(allocated, size);
134 return allocated;
135 }
136
NsanDeallocate(void * p)137 void __nsan::NsanDeallocate(void *p) {
138 DCHECK(p);
139 RunFreeHooks(p);
140 auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
141 uptr size = meta->requested_size;
142 meta->requested_size = 0;
143 if (flags().poison_in_free)
144 __nsan_set_value_unknown(p, size);
145 if (NsanThread *t = GetCurrentThread()) {
146 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
147 allocator.Deallocate(cache, p);
148 } else {
149 // In a just created thread, glibc's _dl_deallocate_tls might reach here
150 // before nsan_current_thread is set.
151 SpinMutexLock l(&fallback_mutex);
152 AllocatorCache *cache = &fallback_allocator_cache;
153 allocator.Deallocate(cache, p);
154 }
155 }
156
NsanReallocate(void * ptr,uptr new_size,uptr alignment)157 static void *NsanReallocate(void *ptr, uptr new_size, uptr alignment) {
158 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(ptr));
159 uptr old_size = meta->requested_size;
160 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(ptr);
161 if (new_size <= actually_allocated_size) {
162 // We are not reallocating here.
163 meta->requested_size = new_size;
164 if (new_size > old_size)
165 __nsan_set_value_unknown((u8 *)ptr + old_size, new_size - old_size);
166 return ptr;
167 }
168 void *new_p = NsanAllocate(new_size, alignment, false);
169 if (new_p) {
170 uptr memcpy_size = Min(new_size, old_size);
171 REAL(memcpy)(new_p, ptr, memcpy_size);
172 __nsan_copy_values(new_p, ptr, memcpy_size);
173 NsanDeallocate(ptr);
174 }
175 return new_p;
176 }
177
NsanCalloc(uptr nmemb,uptr size)178 static void *NsanCalloc(uptr nmemb, uptr size) {
179 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
180 if (AllocatorMayReturnNull())
181 return nullptr;
182 BufferedStackTrace stack;
183 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
184 ReportCallocOverflow(nmemb, size, &stack);
185 }
186 return NsanAllocate(nmemb * size, sizeof(u64), true);
187 }
188
AllocationBegin(const void * p)189 static const void *AllocationBegin(const void *p) {
190 if (!p)
191 return nullptr;
192 void *beg = allocator.GetBlockBegin(p);
193 if (!beg)
194 return nullptr;
195 auto *b = reinterpret_cast<Metadata *>(allocator.GetMetaData(beg));
196 if (!b)
197 return nullptr;
198 if (b->requested_size == 0)
199 return nullptr;
200
201 return beg;
202 }
203
AllocationSizeFast(const void * p)204 static uptr AllocationSizeFast(const void *p) {
205 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;
206 }
207
AllocationSize(const void * p)208 static uptr AllocationSize(const void *p) {
209 if (!p)
210 return 0;
211 if (allocator.GetBlockBegin(p) != p)
212 return 0;
213 return AllocationSizeFast(p);
214 }
215
nsan_malloc(uptr size)216 void *__nsan::nsan_malloc(uptr size) {
217 return SetErrnoOnNull(NsanAllocate(size, sizeof(u64), false));
218 }
219
nsan_calloc(uptr nmemb,uptr size)220 void *__nsan::nsan_calloc(uptr nmemb, uptr size) {
221 return SetErrnoOnNull(NsanCalloc(nmemb, size));
222 }
223
nsan_realloc(void * ptr,uptr size)224 void *__nsan::nsan_realloc(void *ptr, uptr size) {
225 if (!ptr)
226 return SetErrnoOnNull(NsanAllocate(size, sizeof(u64), false));
227 if (size == 0) {
228 NsanDeallocate(ptr);
229 return nullptr;
230 }
231 return SetErrnoOnNull(NsanReallocate(ptr, size, sizeof(u64)));
232 }
233
nsan_reallocarray(void * ptr,uptr nmemb,uptr size)234 void *__nsan::nsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
235 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
236 errno = errno_ENOMEM;
237 if (AllocatorMayReturnNull())
238 return nullptr;
239 BufferedStackTrace stack;
240 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
241 ReportReallocArrayOverflow(nmemb, size, &stack);
242 }
243 return nsan_realloc(ptr, nmemb * size);
244 }
245
nsan_valloc(uptr size)246 void *__nsan::nsan_valloc(uptr size) {
247 return SetErrnoOnNull(NsanAllocate(size, GetPageSizeCached(), false));
248 }
249
nsan_pvalloc(uptr size)250 void *__nsan::nsan_pvalloc(uptr size) {
251 uptr PageSize = GetPageSizeCached();
252 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
253 errno = errno_ENOMEM;
254 if (AllocatorMayReturnNull())
255 return nullptr;
256 BufferedStackTrace stack;
257 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
258 ReportPvallocOverflow(size, &stack);
259 }
260 // pvalloc(0) should allocate one page.
261 size = size ? RoundUpTo(size, PageSize) : PageSize;
262 return SetErrnoOnNull(NsanAllocate(size, PageSize, false));
263 }
264
nsan_aligned_alloc(uptr alignment,uptr size)265 void *__nsan::nsan_aligned_alloc(uptr alignment, uptr size) {
266 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
267 errno = errno_EINVAL;
268 if (AllocatorMayReturnNull())
269 return nullptr;
270 BufferedStackTrace stack;
271 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
272 ReportInvalidAlignedAllocAlignment(size, alignment, &stack);
273 }
274 return SetErrnoOnNull(NsanAllocate(size, alignment, false));
275 }
276
nsan_memalign(uptr alignment,uptr size)277 void *__nsan::nsan_memalign(uptr alignment, uptr size) {
278 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
279 errno = errno_EINVAL;
280 if (AllocatorMayReturnNull())
281 return nullptr;
282 BufferedStackTrace stack;
283 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
284 ReportInvalidAllocationAlignment(alignment, &stack);
285 }
286 return SetErrnoOnNull(NsanAllocate(size, alignment, false));
287 }
288
nsan_posix_memalign(void ** memptr,uptr alignment,uptr size)289 int __nsan::nsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
290 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
291 if (AllocatorMayReturnNull())
292 return errno_EINVAL;
293 BufferedStackTrace stack;
294 ReportInvalidPosixMemalignAlignment(alignment, &stack);
295 }
296 void *ptr = NsanAllocate(size, alignment, false);
297 if (UNLIKELY(!ptr))
298 // OOM error is already taken care of by NsanAllocate.
299 return errno_ENOMEM;
300 DCHECK(IsAligned((uptr)ptr, alignment));
301 *memptr = ptr;
302 return 0;
303 }
304
305 extern "C" {
__sanitizer_get_current_allocated_bytes()306 uptr __sanitizer_get_current_allocated_bytes() {
307 uptr stats[AllocatorStatCount];
308 allocator.GetStats(stats);
309 return stats[AllocatorStatAllocated];
310 }
311
__sanitizer_get_heap_size()312 uptr __sanitizer_get_heap_size() {
313 uptr stats[AllocatorStatCount];
314 allocator.GetStats(stats);
315 return stats[AllocatorStatMapped];
316 }
317
__sanitizer_get_free_bytes()318 uptr __sanitizer_get_free_bytes() { return 1; }
319
__sanitizer_get_unmapped_bytes()320 uptr __sanitizer_get_unmapped_bytes() { return 1; }
321
__sanitizer_get_estimated_allocated_size(uptr size)322 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
323
__sanitizer_get_ownership(const void * p)324 int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
325
__sanitizer_get_allocated_begin(const void * p)326 const void *__sanitizer_get_allocated_begin(const void *p) {
327 return AllocationBegin(p);
328 }
329
__sanitizer_get_allocated_size(const void * p)330 uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
331
__sanitizer_get_allocated_size_fast(const void * p)332 uptr __sanitizer_get_allocated_size_fast(const void *p) {
333 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
334 uptr ret = AllocationSizeFast(p);
335 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
336 return ret;
337 }
338
__sanitizer_purge_allocator()339 void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }
340 }
341