1 //===-- memprof_rtl.cpp --------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of MemProfiler, a memory profiler. 10 // 11 // Main file of the MemProf run-time library. 12 //===----------------------------------------------------------------------===// 13 14 #include "memprof_allocator.h" 15 #include "memprof_interceptors.h" 16 #include "memprof_interface_internal.h" 17 #include "memprof_internal.h" 18 #include "memprof_mapping.h" 19 #include "memprof_stack.h" 20 #include "memprof_stats.h" 21 #include "memprof_thread.h" 22 #include "sanitizer_common/sanitizer_atomic.h" 23 #include "sanitizer_common/sanitizer_flags.h" 24 #include "sanitizer_common/sanitizer_interface_internal.h" 25 #include "sanitizer_common/sanitizer_libc.h" 26 #include "sanitizer_common/sanitizer_symbolizer.h" 27 28 #include <time.h> 29 30 uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol. 31 32 // Allow the user to specify a profile output file via the binary. 33 SANITIZER_WEAK_ATTRIBUTE char __memprof_profile_filename[1]; 34 35 namespace __memprof { 36 37 static void MemprofDie() { 38 static atomic_uint32_t num_calls; 39 if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) { 40 // Don't die twice - run a busy loop. 41 while (1) { 42 internal_sched_yield(); 43 } 44 } 45 if (common_flags()->print_module_map >= 1) 46 DumpProcessMap(); 47 if (flags()->unmap_shadow_on_exit) { 48 if (kHighShadowEnd) 49 UnmapOrDie((void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg); 50 } 51 } 52 53 static void MemprofOnDeadlySignal(int signo, void *siginfo, void *context) { 54 // We call StartReportDeadlySignal not HandleDeadlySignal so we get the 55 // deadly signal message to stderr but no writing to the profile output file 56 StartReportDeadlySignal(); 57 __memprof_profile_dump(); 58 Die(); 59 } 60 61 static void CheckUnwind() { 62 GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check); 63 stack.Print(); 64 } 65 66 // -------------------------- Globals --------------------- {{{1 67 int memprof_inited; 68 int memprof_init_done; 69 bool memprof_init_is_running; 70 int memprof_timestamp_inited; 71 long memprof_init_timestamp_s; 72 73 uptr kHighMemEnd; 74 75 // -------------------------- Run-time entry ------------------- {{{1 76 // exported functions 77 78 #define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() __memprof::RecordAccess(addr); 79 80 #define MEMPROF_MEMORY_ACCESS_CALLBACK(type) \ 81 extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_##type(uptr addr) { \ 82 MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() \ 83 } 84 85 MEMPROF_MEMORY_ACCESS_CALLBACK(load) 86 MEMPROF_MEMORY_ACCESS_CALLBACK(store) 87 88 // Force the linker to keep the symbols for various MemProf interface 89 // functions. We want to keep those in the executable in order to let the 90 // instrumented dynamic libraries access the symbol even if it is not used by 91 // the executable itself. This should help if the build system is removing dead 92 // code at link time. 93 static NOINLINE void force_interface_symbols() { 94 volatile int fake_condition = 0; // prevent dead condition elimination. 95 // clang-format off 96 switch (fake_condition) { 97 case 1: __memprof_record_access(nullptr); break; 98 case 2: __memprof_record_access_range(nullptr, 0); break; 99 } 100 // clang-format on 101 } 102 103 static void memprof_atexit() { 104 Printf("MemProfiler exit stats:\n"); 105 __memprof_print_accumulated_stats(); 106 } 107 108 static void InitializeHighMemEnd() { 109 kHighMemEnd = GetMaxUserVirtualAddress(); 110 // Increase kHighMemEnd to make sure it's properly 111 // aligned together with kHighMemBeg: 112 kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1; 113 } 114 115 void PrintAddressSpaceLayout() { 116 if (kHighMemBeg) { 117 Printf("|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemBeg, 118 (void *)kHighMemEnd); 119 Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg, 120 (void *)kHighShadowEnd); 121 } 122 Printf("|| `[%p, %p]` || ShadowGap ||\n", (void *)kShadowGapBeg, 123 (void *)kShadowGapEnd); 124 if (kLowShadowBeg) { 125 Printf("|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowBeg, 126 (void *)kLowShadowEnd); 127 Printf("|| `[%p, %p]` || LowMem ||\n", (void *)kLowMemBeg, 128 (void *)kLowMemEnd); 129 } 130 Printf("MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg), 131 (void *)MEM_TO_SHADOW(kLowShadowEnd)); 132 if (kHighMemBeg) { 133 Printf(" %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg), 134 (void *)MEM_TO_SHADOW(kHighShadowEnd)); 135 } 136 Printf("\n"); 137 Printf("malloc_context_size=%zu\n", 138 (uptr)common_flags()->malloc_context_size); 139 140 Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE); 141 Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY); 142 Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET); 143 CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7); 144 } 145 146 static void MemprofInitInternal() { 147 if (LIKELY(memprof_inited)) 148 return; 149 SanitizerToolName = "MemProfiler"; 150 CHECK(!memprof_init_is_running && "MemProf init calls itself!"); 151 memprof_init_is_running = true; 152 153 CacheBinaryName(); 154 155 // Initialize flags. This must be done early, because most of the 156 // initialization steps look at flags(). 157 InitializeFlags(); 158 159 AvoidCVE_2016_2143(); 160 161 SetMallocContextSize(common_flags()->malloc_context_size); 162 163 InitializeHighMemEnd(); 164 165 // Make sure we are not statically linked. 166 MemprofDoesNotSupportStaticLinkage(); 167 168 // Install tool-specific callbacks in sanitizer_common. 169 AddDieCallback(MemprofDie); 170 SetCheckUnwindCallback(CheckUnwind); 171 172 // Use profile name specified via the binary itself if it exists, and hasn't 173 // been overrriden by a flag at runtime. 174 if (__memprof_profile_filename[0] != 0 && !common_flags()->log_path) 175 __sanitizer_set_report_path(__memprof_profile_filename); 176 else 177 __sanitizer_set_report_path(common_flags()->log_path); 178 179 __sanitizer::InitializePlatformEarly(); 180 181 // Setup internal allocator callback. 182 SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY); 183 184 InitializeMemprofInterceptors(); 185 CheckASLR(); 186 187 ReplaceSystemMalloc(); 188 189 DisableCoreDumperIfNecessary(); 190 191 InitializeShadowMemory(); 192 193 TSDInit(PlatformTSDDtor); 194 InstallDeadlySignalHandlers(MemprofOnDeadlySignal); 195 196 InitializeAllocator(); 197 198 // On Linux MemprofThread::ThreadStart() calls malloc() that's why 199 // memprof_inited should be set to 1 prior to initializing the threads. 200 memprof_inited = 1; 201 memprof_init_is_running = false; 202 203 if (flags()->atexit) 204 Atexit(memprof_atexit); 205 206 InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); 207 208 // interceptors 209 InitTlsSize(); 210 211 // Create main thread. 212 MemprofThread *main_thread = CreateMainThread(); 213 CHECK_EQ(0, main_thread->tid()); 214 force_interface_symbols(); // no-op. 215 SanitizerInitializeUnwinder(); 216 217 Symbolizer::LateInitialize(); 218 219 VReport(1, "MemProfiler Init done\n"); 220 221 memprof_init_done = 1; 222 } 223 224 void MemprofInitTime() { 225 if (LIKELY(memprof_timestamp_inited)) 226 return; 227 timespec ts; 228 clock_gettime(CLOCK_REALTIME, &ts); 229 memprof_init_timestamp_s = ts.tv_sec; 230 memprof_timestamp_inited = 1; 231 } 232 233 // Initialize as requested from some part of MemProf runtime library 234 // (interceptors, allocator, etc). 235 void MemprofInitFromRtl() { MemprofInitInternal(); } 236 237 #if MEMPROF_DYNAMIC 238 // Initialize runtime in case it's LD_PRELOAD-ed into uninstrumented executable 239 // (and thus normal initializers from .preinit_array or modules haven't run). 240 241 class MemprofInitializer { 242 public: 243 MemprofInitializer() { MemprofInitFromRtl(); } 244 }; 245 246 static MemprofInitializer memprof_initializer; 247 #endif // MEMPROF_DYNAMIC 248 249 } // namespace __memprof 250 251 // ---------------------- Interface ---------------- {{{1 252 using namespace __memprof; 253 254 // Initialize as requested from instrumented application code. 255 void __memprof_init() { 256 MemprofInitTime(); 257 MemprofInitInternal(); 258 } 259 260 void __memprof_preinit() { MemprofInitInternal(); } 261 262 void __memprof_version_mismatch_check_v1() {} 263 264 void __memprof_record_access(void const volatile *addr) { 265 __memprof::RecordAccess((uptr)addr); 266 } 267 268 void __memprof_record_access_range(void const volatile *addr, uptr size) { 269 for (uptr a = (uptr)addr; a < (uptr)addr + size; a += kWordSize) 270 __memprof::RecordAccess(a); 271 } 272 273 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16 274 __sanitizer_unaligned_load16(const uu16 *p) { 275 __memprof_record_access(p); 276 return *p; 277 } 278 279 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32 280 __sanitizer_unaligned_load32(const uu32 *p) { 281 __memprof_record_access(p); 282 return *p; 283 } 284 285 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64 286 __sanitizer_unaligned_load64(const uu64 *p) { 287 __memprof_record_access(p); 288 return *p; 289 } 290 291 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void 292 __sanitizer_unaligned_store16(uu16 *p, u16 x) { 293 __memprof_record_access(p); 294 *p = x; 295 } 296 297 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void 298 __sanitizer_unaligned_store32(uu32 *p, u32 x) { 299 __memprof_record_access(p); 300 *p = x; 301 } 302 303 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void 304 __sanitizer_unaligned_store64(uu64 *p, u64 x) { 305 __memprof_record_access(p); 306 *p = x; 307 } 308