xref: /freebsd/contrib/llvm-project/compiler-rt/lib/memprof/memprof_allocator.cpp (revision 5fb307d29b364982acbde82cbf77db3cae486f8c)
1 //===-- memprof_allocator.cpp --------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of MemProfiler, a memory profiler.
10 //
11 // Implementation of MemProf's memory allocator, which uses the allocator
12 // from sanitizer_common.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "memprof_allocator.h"
17 #include "memprof_mapping.h"
18 #include "memprof_mibmap.h"
19 #include "memprof_rawprofile.h"
20 #include "memprof_stack.h"
21 #include "memprof_thread.h"
22 #include "profile/MemProfData.inc"
23 #include "sanitizer_common/sanitizer_allocator_checks.h"
24 #include "sanitizer_common/sanitizer_allocator_interface.h"
25 #include "sanitizer_common/sanitizer_allocator_report.h"
26 #include "sanitizer_common/sanitizer_array_ref.h"
27 #include "sanitizer_common/sanitizer_common.h"
28 #include "sanitizer_common/sanitizer_errno.h"
29 #include "sanitizer_common/sanitizer_file.h"
30 #include "sanitizer_common/sanitizer_flags.h"
31 #include "sanitizer_common/sanitizer_internal_defs.h"
32 #include "sanitizer_common/sanitizer_stackdepot.h"
33 
34 #include <sched.h>
35 #include <time.h>
36 
37 namespace __memprof {
38 namespace {
39 using ::llvm::memprof::MemInfoBlock;
40 
41 void Print(const MemInfoBlock &M, const u64 id, bool print_terse) {
42   u64 p;
43 
44   if (print_terse) {
45     p = M.TotalSize * 100 / M.AllocCount;
46     Printf("MIB:%llu/%u/%llu.%02llu/%u/%u/", id, M.AllocCount, p / 100, p % 100,
47            M.MinSize, M.MaxSize);
48     p = M.TotalAccessCount * 100 / M.AllocCount;
49     Printf("%llu.%02llu/%llu/%llu/", p / 100, p % 100, M.MinAccessCount,
50            M.MaxAccessCount);
51     p = M.TotalLifetime * 100 / M.AllocCount;
52     Printf("%llu.%02llu/%u/%u/", p / 100, p % 100, M.MinLifetime,
53            M.MaxLifetime);
54     Printf("%u/%u/%u/%u\n", M.NumMigratedCpu, M.NumLifetimeOverlaps,
55            M.NumSameAllocCpu, M.NumSameDeallocCpu);
56   } else {
57     p = M.TotalSize * 100 / M.AllocCount;
58     Printf("Memory allocation stack id = %llu\n", id);
59     Printf("\talloc_count %u, size (ave/min/max) %llu.%02llu / %u / %u\n",
60            M.AllocCount, p / 100, p % 100, M.MinSize, M.MaxSize);
61     p = M.TotalAccessCount * 100 / M.AllocCount;
62     Printf("\taccess_count (ave/min/max): %llu.%02llu / %llu / %llu\n", p / 100,
63            p % 100, M.MinAccessCount, M.MaxAccessCount);
64     p = M.TotalLifetime * 100 / M.AllocCount;
65     Printf("\tlifetime (ave/min/max): %llu.%02llu / %u / %u\n", p / 100,
66            p % 100, M.MinLifetime, M.MaxLifetime);
67     Printf("\tnum migrated: %u, num lifetime overlaps: %u, num same alloc "
68            "cpu: %u, num same dealloc_cpu: %u\n",
69            M.NumMigratedCpu, M.NumLifetimeOverlaps, M.NumSameAllocCpu,
70            M.NumSameDeallocCpu);
71   }
72 }
73 } // namespace
74 
75 static int GetCpuId(void) {
76   // _memprof_preinit is called via the preinit_array, which subsequently calls
77   // malloc. Since this is before _dl_init calls VDSO_SETUP, sched_getcpu
78   // will seg fault as the address of __vdso_getcpu will be null.
79   if (!memprof_inited)
80     return -1;
81   return sched_getcpu();
82 }
83 
84 // Compute the timestamp in ms.
85 static int GetTimestamp(void) {
86   // timespec_get will segfault if called from dl_init
87   if (!memprof_timestamp_inited) {
88     // By returning 0, this will be effectively treated as being
89     // timestamped at memprof init time (when memprof_init_timestamp_s
90     // is initialized).
91     return 0;
92   }
93   timespec ts;
94   clock_gettime(CLOCK_REALTIME, &ts);
95   return (ts.tv_sec - memprof_init_timestamp_s) * 1000 + ts.tv_nsec / 1000000;
96 }
97 
98 static MemprofAllocator &get_allocator();
99 
100 // The memory chunk allocated from the underlying allocator looks like this:
101 // H H U U U U U U
102 //   H -- ChunkHeader (32 bytes)
103 //   U -- user memory.
104 
105 // If there is left padding before the ChunkHeader (due to use of memalign),
106 // we store a magic value in the first uptr word of the memory block and
107 // store the address of ChunkHeader in the next uptr.
108 // M B L L L L L L L L L  H H U U U U U U
109 //   |                    ^
110 //   ---------------------|
111 //   M -- magic value kAllocBegMagic
112 //   B -- address of ChunkHeader pointing to the first 'H'
113 
114 constexpr uptr kMaxAllowedMallocBits = 40;
115 
116 // Should be no more than 32-bytes
117 struct ChunkHeader {
118   // 1-st 4 bytes.
119   u32 alloc_context_id;
120   // 2-nd 4 bytes
121   u32 cpu_id;
122   // 3-rd 4 bytes
123   u32 timestamp_ms;
124   // 4-th 4 bytes
125   // Note only 1 bit is needed for this flag if we need space in the future for
126   // more fields.
127   u32 from_memalign;
128   // 5-th and 6-th 4 bytes
129   // The max size of an allocation is 2^40 (kMaxAllowedMallocSize), so this
130   // could be shrunk to kMaxAllowedMallocBits if we need space in the future for
131   // more fields.
132   atomic_uint64_t user_requested_size;
133   // 23 bits available
134   // 7-th and 8-th 4 bytes
135   u64 data_type_id; // TODO: hash of type name
136 };
137 
138 static const uptr kChunkHeaderSize = sizeof(ChunkHeader);
139 COMPILER_CHECK(kChunkHeaderSize == 32);
140 
141 struct MemprofChunk : ChunkHeader {
142   uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; }
143   uptr UsedSize() {
144     return atomic_load(&user_requested_size, memory_order_relaxed);
145   }
146   void *AllocBeg() {
147     if (from_memalign)
148       return get_allocator().GetBlockBegin(reinterpret_cast<void *>(this));
149     return reinterpret_cast<void *>(this);
150   }
151 };
152 
153 class LargeChunkHeader {
154   static constexpr uptr kAllocBegMagic =
155       FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL);
156   atomic_uintptr_t magic;
157   MemprofChunk *chunk_header;
158 
159 public:
160   MemprofChunk *Get() const {
161     return atomic_load(&magic, memory_order_acquire) == kAllocBegMagic
162                ? chunk_header
163                : nullptr;
164   }
165 
166   void Set(MemprofChunk *p) {
167     if (p) {
168       chunk_header = p;
169       atomic_store(&magic, kAllocBegMagic, memory_order_release);
170       return;
171     }
172 
173     uptr old = kAllocBegMagic;
174     if (!atomic_compare_exchange_strong(&magic, &old, 0,
175                                         memory_order_release)) {
176       CHECK_EQ(old, kAllocBegMagic);
177     }
178   }
179 };
180 
181 void FlushUnneededMemProfShadowMemory(uptr p, uptr size) {
182   // Since memprof's mapping is compacting, the shadow chunk may be
183   // not page-aligned, so we only flush the page-aligned portion.
184   ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
185 }
186 
187 void MemprofMapUnmapCallback::OnMap(uptr p, uptr size) const {
188   // Statistics.
189   MemprofStats &thread_stats = GetCurrentThreadStats();
190   thread_stats.mmaps++;
191   thread_stats.mmaped += size;
192 }
193 
194 void MemprofMapUnmapCallback::OnUnmap(uptr p, uptr size) const {
195   // We are about to unmap a chunk of user memory.
196   // Mark the corresponding shadow memory as not needed.
197   FlushUnneededMemProfShadowMemory(p, size);
198   // Statistics.
199   MemprofStats &thread_stats = GetCurrentThreadStats();
200   thread_stats.munmaps++;
201   thread_stats.munmaped += size;
202 }
203 
204 AllocatorCache *GetAllocatorCache(MemprofThreadLocalMallocStorage *ms) {
205   CHECK(ms);
206   return &ms->allocator_cache;
207 }
208 
209 // Accumulates the access count from the shadow for the given pointer and size.
210 u64 GetShadowCount(uptr p, u32 size) {
211   u64 *shadow = (u64 *)MEM_TO_SHADOW(p);
212   u64 *shadow_end = (u64 *)MEM_TO_SHADOW(p + size);
213   u64 count = 0;
214   for (; shadow <= shadow_end; shadow++)
215     count += *shadow;
216   return count;
217 }
218 
219 // Clears the shadow counters (when memory is allocated).
220 void ClearShadow(uptr addr, uptr size) {
221   CHECK(AddrIsAlignedByGranularity(addr));
222   CHECK(AddrIsInMem(addr));
223   CHECK(AddrIsAlignedByGranularity(addr + size));
224   CHECK(AddrIsInMem(addr + size - SHADOW_GRANULARITY));
225   CHECK(REAL(memset));
226   uptr shadow_beg = MEM_TO_SHADOW(addr);
227   uptr shadow_end = MEM_TO_SHADOW(addr + size - SHADOW_GRANULARITY) + 1;
228   if (shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) {
229     REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg);
230   } else {
231     uptr page_size = GetPageSizeCached();
232     uptr page_beg = RoundUpTo(shadow_beg, page_size);
233     uptr page_end = RoundDownTo(shadow_end, page_size);
234 
235     if (page_beg >= page_end) {
236       REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg);
237     } else {
238       if (page_beg != shadow_beg) {
239         REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg);
240       }
241       if (page_end != shadow_end) {
242         REAL(memset)((void *)page_end, 0, shadow_end - page_end);
243       }
244       ReserveShadowMemoryRange(page_beg, page_end - 1, nullptr);
245     }
246   }
247 }
248 
249 struct Allocator {
250   static const uptr kMaxAllowedMallocSize = 1ULL << kMaxAllowedMallocBits;
251 
252   MemprofAllocator allocator;
253   StaticSpinMutex fallback_mutex;
254   AllocatorCache fallback_allocator_cache;
255 
256   uptr max_user_defined_malloc_size;
257 
258   // Holds the mapping of stack ids to MemInfoBlocks.
259   MIBMapTy MIBMap;
260 
261   atomic_uint8_t destructing;
262   atomic_uint8_t constructed;
263   bool print_text;
264 
265   // ------------------- Initialization ------------------------
266   explicit Allocator(LinkerInitialized) : print_text(flags()->print_text) {
267     atomic_store_relaxed(&destructing, 0);
268     atomic_store_relaxed(&constructed, 1);
269   }
270 
271   ~Allocator() {
272     atomic_store_relaxed(&destructing, 1);
273     FinishAndWrite();
274   }
275 
276   static void PrintCallback(const uptr Key, LockedMemInfoBlock *const &Value,
277                             void *Arg) {
278     SpinMutexLock l(&Value->mutex);
279     Print(Value->mib, Key, bool(Arg));
280   }
281 
282   void FinishAndWrite() {
283     if (print_text && common_flags()->print_module_map)
284       DumpProcessMap();
285 
286     allocator.ForceLock();
287 
288     InsertLiveBlocks();
289     if (print_text) {
290       if (!flags()->print_terse)
291         Printf("Recorded MIBs (incl. live on exit):\n");
292       MIBMap.ForEach(PrintCallback,
293                      reinterpret_cast<void *>(flags()->print_terse));
294       StackDepotPrintAll();
295     } else {
296       // Serialize the contents to a raw profile. Format documented in
297       // memprof_rawprofile.h.
298       char *Buffer = nullptr;
299 
300       __sanitizer::ListOfModules List;
301       List.init();
302       ArrayRef<LoadedModule> Modules(List.begin(), List.end());
303       u64 BytesSerialized = SerializeToRawProfile(MIBMap, Modules, Buffer);
304       CHECK(Buffer && BytesSerialized && "could not serialize to buffer");
305       report_file.Write(Buffer, BytesSerialized);
306     }
307 
308     allocator.ForceUnlock();
309   }
310 
311   // Inserts any blocks which have been allocated but not yet deallocated.
312   void InsertLiveBlocks() {
313     allocator.ForEachChunk(
314         [](uptr chunk, void *alloc) {
315           u64 user_requested_size;
316           Allocator *A = (Allocator *)alloc;
317           MemprofChunk *m =
318               A->GetMemprofChunk((void *)chunk, user_requested_size);
319           if (!m)
320             return;
321           uptr user_beg = ((uptr)m) + kChunkHeaderSize;
322           u64 c = GetShadowCount(user_beg, user_requested_size);
323           long curtime = GetTimestamp();
324           MemInfoBlock newMIB(user_requested_size, c, m->timestamp_ms, curtime,
325                               m->cpu_id, GetCpuId());
326           InsertOrMerge(m->alloc_context_id, newMIB, A->MIBMap);
327         },
328         this);
329   }
330 
331   void InitLinkerInitialized() {
332     SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
333     allocator.InitLinkerInitialized(
334         common_flags()->allocator_release_to_os_interval_ms);
335     max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
336                                        ? common_flags()->max_allocation_size_mb
337                                              << 20
338                                        : kMaxAllowedMallocSize;
339   }
340 
341   // -------------------- Allocation/Deallocation routines ---------------
342   void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
343                  AllocType alloc_type) {
344     if (UNLIKELY(!memprof_inited))
345       MemprofInitFromRtl();
346     if (UNLIKELY(IsRssLimitExceeded())) {
347       if (AllocatorMayReturnNull())
348         return nullptr;
349       ReportRssLimitExceeded(stack);
350     }
351     CHECK(stack);
352     const uptr min_alignment = MEMPROF_ALIGNMENT;
353     if (alignment < min_alignment)
354       alignment = min_alignment;
355     if (size == 0) {
356       // We'd be happy to avoid allocating memory for zero-size requests, but
357       // some programs/tests depend on this behavior and assume that malloc
358       // would not return NULL even for zero-size allocations. Moreover, it
359       // looks like operator new should never return NULL, and results of
360       // consecutive "new" calls must be different even if the allocated size
361       // is zero.
362       size = 1;
363     }
364     CHECK(IsPowerOfTwo(alignment));
365     uptr rounded_size = RoundUpTo(size, alignment);
366     uptr needed_size = rounded_size + kChunkHeaderSize;
367     if (alignment > min_alignment)
368       needed_size += alignment;
369     CHECK(IsAligned(needed_size, min_alignment));
370     if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
371         size > max_user_defined_malloc_size) {
372       if (AllocatorMayReturnNull()) {
373         Report("WARNING: MemProfiler failed to allocate 0x%zx bytes\n", size);
374         return nullptr;
375       }
376       uptr malloc_limit =
377           Min(kMaxAllowedMallocSize, max_user_defined_malloc_size);
378       ReportAllocationSizeTooBig(size, malloc_limit, stack);
379     }
380 
381     MemprofThread *t = GetCurrentThread();
382     void *allocated;
383     if (t) {
384       AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
385       allocated = allocator.Allocate(cache, needed_size, 8);
386     } else {
387       SpinMutexLock l(&fallback_mutex);
388       AllocatorCache *cache = &fallback_allocator_cache;
389       allocated = allocator.Allocate(cache, needed_size, 8);
390     }
391     if (UNLIKELY(!allocated)) {
392       SetAllocatorOutOfMemory();
393       if (AllocatorMayReturnNull())
394         return nullptr;
395       ReportOutOfMemory(size, stack);
396     }
397 
398     uptr alloc_beg = reinterpret_cast<uptr>(allocated);
399     uptr alloc_end = alloc_beg + needed_size;
400     uptr beg_plus_header = alloc_beg + kChunkHeaderSize;
401     uptr user_beg = beg_plus_header;
402     if (!IsAligned(user_beg, alignment))
403       user_beg = RoundUpTo(user_beg, alignment);
404     uptr user_end = user_beg + size;
405     CHECK_LE(user_end, alloc_end);
406     uptr chunk_beg = user_beg - kChunkHeaderSize;
407     MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg);
408     m->from_memalign = alloc_beg != chunk_beg;
409     CHECK(size);
410 
411     m->cpu_id = GetCpuId();
412     m->timestamp_ms = GetTimestamp();
413     m->alloc_context_id = StackDepotPut(*stack);
414 
415     uptr size_rounded_down_to_granularity =
416         RoundDownTo(size, SHADOW_GRANULARITY);
417     if (size_rounded_down_to_granularity)
418       ClearShadow(user_beg, size_rounded_down_to_granularity);
419 
420     MemprofStats &thread_stats = GetCurrentThreadStats();
421     thread_stats.mallocs++;
422     thread_stats.malloced += size;
423     thread_stats.malloced_overhead += needed_size - size;
424     if (needed_size > SizeClassMap::kMaxSize)
425       thread_stats.malloc_large++;
426     else
427       thread_stats.malloced_by_size[SizeClassMap::ClassID(needed_size)]++;
428 
429     void *res = reinterpret_cast<void *>(user_beg);
430     atomic_store(&m->user_requested_size, size, memory_order_release);
431     if (alloc_beg != chunk_beg) {
432       CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg);
433       reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m);
434     }
435     RunMallocHooks(res, size);
436     return res;
437   }
438 
439   void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment,
440                   BufferedStackTrace *stack, AllocType alloc_type) {
441     uptr p = reinterpret_cast<uptr>(ptr);
442     if (p == 0)
443       return;
444 
445     RunFreeHooks(ptr);
446 
447     uptr chunk_beg = p - kChunkHeaderSize;
448     MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg);
449 
450     u64 user_requested_size =
451         atomic_exchange(&m->user_requested_size, 0, memory_order_acquire);
452     if (memprof_inited && atomic_load_relaxed(&constructed) &&
453         !atomic_load_relaxed(&destructing)) {
454       u64 c = GetShadowCount(p, user_requested_size);
455       long curtime = GetTimestamp();
456 
457       MemInfoBlock newMIB(user_requested_size, c, m->timestamp_ms, curtime,
458                           m->cpu_id, GetCpuId());
459       InsertOrMerge(m->alloc_context_id, newMIB, MIBMap);
460     }
461 
462     MemprofStats &thread_stats = GetCurrentThreadStats();
463     thread_stats.frees++;
464     thread_stats.freed += user_requested_size;
465 
466     void *alloc_beg = m->AllocBeg();
467     if (alloc_beg != m) {
468       // Clear the magic value, as allocator internals may overwrite the
469       // contents of deallocated chunk, confusing GetMemprofChunk lookup.
470       reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(nullptr);
471     }
472 
473     MemprofThread *t = GetCurrentThread();
474     if (t) {
475       AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
476       allocator.Deallocate(cache, alloc_beg);
477     } else {
478       SpinMutexLock l(&fallback_mutex);
479       AllocatorCache *cache = &fallback_allocator_cache;
480       allocator.Deallocate(cache, alloc_beg);
481     }
482   }
483 
484   void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) {
485     CHECK(old_ptr && new_size);
486     uptr p = reinterpret_cast<uptr>(old_ptr);
487     uptr chunk_beg = p - kChunkHeaderSize;
488     MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg);
489 
490     MemprofStats &thread_stats = GetCurrentThreadStats();
491     thread_stats.reallocs++;
492     thread_stats.realloced += new_size;
493 
494     void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC);
495     if (new_ptr) {
496       CHECK_NE(REAL(memcpy), nullptr);
497       uptr memcpy_size = Min(new_size, m->UsedSize());
498       REAL(memcpy)(new_ptr, old_ptr, memcpy_size);
499       Deallocate(old_ptr, 0, 0, stack, FROM_MALLOC);
500     }
501     return new_ptr;
502   }
503 
504   void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
505     if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
506       if (AllocatorMayReturnNull())
507         return nullptr;
508       ReportCallocOverflow(nmemb, size, stack);
509     }
510     void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC);
511     // If the memory comes from the secondary allocator no need to clear it
512     // as it comes directly from mmap.
513     if (ptr && allocator.FromPrimary(ptr))
514       REAL(memset)(ptr, 0, nmemb * size);
515     return ptr;
516   }
517 
518   void CommitBack(MemprofThreadLocalMallocStorage *ms,
519                   BufferedStackTrace *stack) {
520     AllocatorCache *ac = GetAllocatorCache(ms);
521     allocator.SwallowCache(ac);
522   }
523 
524   // -------------------------- Chunk lookup ----------------------
525 
526   // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg).
527   MemprofChunk *GetMemprofChunk(void *alloc_beg, u64 &user_requested_size) {
528     if (!alloc_beg)
529       return nullptr;
530     MemprofChunk *p = reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Get();
531     if (!p) {
532       if (!allocator.FromPrimary(alloc_beg))
533         return nullptr;
534       p = reinterpret_cast<MemprofChunk *>(alloc_beg);
535     }
536     // The size is reset to 0 on deallocation (and a min of 1 on
537     // allocation).
538     user_requested_size =
539         atomic_load(&p->user_requested_size, memory_order_acquire);
540     if (user_requested_size)
541       return p;
542     return nullptr;
543   }
544 
545   MemprofChunk *GetMemprofChunkByAddr(uptr p, u64 &user_requested_size) {
546     void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast<void *>(p));
547     return GetMemprofChunk(alloc_beg, user_requested_size);
548   }
549 
550   uptr AllocationSize(uptr p) {
551     u64 user_requested_size;
552     MemprofChunk *m = GetMemprofChunkByAddr(p, user_requested_size);
553     if (!m)
554       return 0;
555     if (m->Beg() != p)
556       return 0;
557     return user_requested_size;
558   }
559 
560   uptr AllocationSizeFast(uptr p) {
561     return reinterpret_cast<MemprofChunk *>(p - kChunkHeaderSize)->UsedSize();
562   }
563 
564   void Purge(BufferedStackTrace *stack) { allocator.ForceReleaseToOS(); }
565 
566   void PrintStats() { allocator.PrintStats(); }
567 
568   void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
569     allocator.ForceLock();
570     fallback_mutex.Lock();
571   }
572 
573   void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
574     fallback_mutex.Unlock();
575     allocator.ForceUnlock();
576   }
577 };
578 
579 static Allocator instance(LINKER_INITIALIZED);
580 
581 static MemprofAllocator &get_allocator() { return instance.allocator; }
582 
583 void InitializeAllocator() { instance.InitLinkerInitialized(); }
584 
585 void MemprofThreadLocalMallocStorage::CommitBack() {
586   GET_STACK_TRACE_MALLOC;
587   instance.CommitBack(this, &stack);
588 }
589 
590 void PrintInternalAllocatorStats() { instance.PrintStats(); }
591 
592 void memprof_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) {
593   instance.Deallocate(ptr, 0, 0, stack, alloc_type);
594 }
595 
596 void memprof_delete(void *ptr, uptr size, uptr alignment,
597                     BufferedStackTrace *stack, AllocType alloc_type) {
598   instance.Deallocate(ptr, size, alignment, stack, alloc_type);
599 }
600 
601 void *memprof_malloc(uptr size, BufferedStackTrace *stack) {
602   return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC));
603 }
604 
605 void *memprof_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
606   return SetErrnoOnNull(instance.Calloc(nmemb, size, stack));
607 }
608 
609 void *memprof_reallocarray(void *p, uptr nmemb, uptr size,
610                            BufferedStackTrace *stack) {
611   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
612     errno = errno_ENOMEM;
613     if (AllocatorMayReturnNull())
614       return nullptr;
615     ReportReallocArrayOverflow(nmemb, size, stack);
616   }
617   return memprof_realloc(p, nmemb * size, stack);
618 }
619 
620 void *memprof_realloc(void *p, uptr size, BufferedStackTrace *stack) {
621   if (!p)
622     return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC));
623   if (size == 0) {
624     if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
625       instance.Deallocate(p, 0, 0, stack, FROM_MALLOC);
626       return nullptr;
627     }
628     // Allocate a size of 1 if we shouldn't free() on Realloc to 0
629     size = 1;
630   }
631   return SetErrnoOnNull(instance.Reallocate(p, size, stack));
632 }
633 
634 void *memprof_valloc(uptr size, BufferedStackTrace *stack) {
635   return SetErrnoOnNull(
636       instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC));
637 }
638 
639 void *memprof_pvalloc(uptr size, BufferedStackTrace *stack) {
640   uptr PageSize = GetPageSizeCached();
641   if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
642     errno = errno_ENOMEM;
643     if (AllocatorMayReturnNull())
644       return nullptr;
645     ReportPvallocOverflow(size, stack);
646   }
647   // pvalloc(0) should allocate one page.
648   size = size ? RoundUpTo(size, PageSize) : PageSize;
649   return SetErrnoOnNull(instance.Allocate(size, PageSize, stack, FROM_MALLOC));
650 }
651 
652 void *memprof_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
653                        AllocType alloc_type) {
654   if (UNLIKELY(!IsPowerOfTwo(alignment))) {
655     errno = errno_EINVAL;
656     if (AllocatorMayReturnNull())
657       return nullptr;
658     ReportInvalidAllocationAlignment(alignment, stack);
659   }
660   return SetErrnoOnNull(instance.Allocate(size, alignment, stack, alloc_type));
661 }
662 
663 void *memprof_aligned_alloc(uptr alignment, uptr size,
664                             BufferedStackTrace *stack) {
665   if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
666     errno = errno_EINVAL;
667     if (AllocatorMayReturnNull())
668       return nullptr;
669     ReportInvalidAlignedAllocAlignment(size, alignment, stack);
670   }
671   return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC));
672 }
673 
674 int memprof_posix_memalign(void **memptr, uptr alignment, uptr size,
675                            BufferedStackTrace *stack) {
676   if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
677     if (AllocatorMayReturnNull())
678       return errno_EINVAL;
679     ReportInvalidPosixMemalignAlignment(alignment, stack);
680   }
681   void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC);
682   if (UNLIKELY(!ptr))
683     // OOM error is already taken care of by Allocate.
684     return errno_ENOMEM;
685   CHECK(IsAligned((uptr)ptr, alignment));
686   *memptr = ptr;
687   return 0;
688 }
689 
690 static const void *memprof_malloc_begin(const void *p) {
691   u64 user_requested_size;
692   MemprofChunk *m =
693       instance.GetMemprofChunkByAddr((uptr)p, user_requested_size);
694   if (!m)
695     return nullptr;
696   if (user_requested_size == 0)
697     return nullptr;
698 
699   return (const void *)m->Beg();
700 }
701 
702 uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp) {
703   if (!ptr)
704     return 0;
705   uptr usable_size = instance.AllocationSize(reinterpret_cast<uptr>(ptr));
706   return usable_size;
707 }
708 
709 } // namespace __memprof
710 
711 // ---------------------- Interface ---------------- {{{1
712 using namespace __memprof;
713 
714 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
715 
716 int __sanitizer_get_ownership(const void *p) {
717   return memprof_malloc_usable_size(p, 0, 0) != 0;
718 }
719 
720 const void *__sanitizer_get_allocated_begin(const void *p) {
721   return memprof_malloc_begin(p);
722 }
723 
724 uptr __sanitizer_get_allocated_size(const void *p) {
725   return memprof_malloc_usable_size(p, 0, 0);
726 }
727 
728 uptr __sanitizer_get_allocated_size_fast(const void *p) {
729   DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
730   uptr ret = instance.AllocationSizeFast(reinterpret_cast<uptr>(p));
731   DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
732   return ret;
733 }
734 
735 int __memprof_profile_dump() {
736   instance.FinishAndWrite();
737   // In the future we may want to return non-zero if there are any errors
738   // detected during the dumping process.
739   return 0;
740 }
741