1 //===-- memprof_rtl.cpp --------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of MemProfiler, a memory profiler. 10 // 11 // Main file of the MemProf run-time library. 12 //===----------------------------------------------------------------------===// 13 14 #include "memprof_allocator.h" 15 #include "memprof_interceptors.h" 16 #include "memprof_interface_internal.h" 17 #include "memprof_internal.h" 18 #include "memprof_mapping.h" 19 #include "memprof_stack.h" 20 #include "memprof_stats.h" 21 #include "memprof_thread.h" 22 #include "sanitizer_common/sanitizer_atomic.h" 23 #include "sanitizer_common/sanitizer_flags.h" 24 #include "sanitizer_common/sanitizer_interface_internal.h" 25 #include "sanitizer_common/sanitizer_libc.h" 26 #include "sanitizer_common/sanitizer_symbolizer.h" 27 28 #include <time.h> 29 30 uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol. 31 32 // Allow the user to specify a profile output file via the binary. 33 SANITIZER_WEAK_ATTRIBUTE char __memprof_profile_filename[1]; 34 35 namespace __memprof { 36 37 static void MemprofDie() { 38 static atomic_uint32_t num_calls; 39 if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) { 40 // Don't die twice - run a busy loop. 41 while (1) { 42 internal_sched_yield(); 43 } 44 } 45 if (common_flags()->print_module_map >= 1) 46 DumpProcessMap(); 47 if (flags()->unmap_shadow_on_exit) { 48 if (kHighShadowEnd) 49 UnmapOrDie((void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg); 50 } 51 } 52 53 static void MemprofOnDeadlySignal(int signo, void *siginfo, void *context) { 54 // We call StartReportDeadlySignal not HandleDeadlySignal so we get the 55 // deadly signal message to stderr but no writing to the profile output file 56 StartReportDeadlySignal(); 57 __memprof_profile_dump(); 58 Die(); 59 } 60 61 static void CheckUnwind() { 62 GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check); 63 stack.Print(); 64 } 65 66 // -------------------------- Globals --------------------- {{{1 67 int memprof_inited; 68 bool memprof_init_is_running; 69 int memprof_timestamp_inited; 70 long memprof_init_timestamp_s; 71 72 uptr kHighMemEnd; 73 74 // -------------------------- Run-time entry ------------------- {{{1 75 // exported functions 76 77 #define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() __memprof::RecordAccess(addr); 78 79 #define MEMPROF_MEMORY_ACCESS_CALLBACK(type) \ 80 extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_##type(uptr addr) { \ 81 MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() \ 82 } 83 84 MEMPROF_MEMORY_ACCESS_CALLBACK(load) 85 MEMPROF_MEMORY_ACCESS_CALLBACK(store) 86 87 // Force the linker to keep the symbols for various MemProf interface 88 // functions. We want to keep those in the executable in order to let the 89 // instrumented dynamic libraries access the symbol even if it is not used by 90 // the executable itself. This should help if the build system is removing dead 91 // code at link time. 92 static NOINLINE void force_interface_symbols() { 93 volatile int fake_condition = 0; // prevent dead condition elimination. 94 // clang-format off 95 switch (fake_condition) { 96 case 1: __memprof_record_access(nullptr); break; 97 case 2: __memprof_record_access_range(nullptr, 0); break; 98 } 99 // clang-format on 100 } 101 102 static void memprof_atexit() { 103 Printf("MemProfiler exit stats:\n"); 104 __memprof_print_accumulated_stats(); 105 } 106 107 static void InitializeHighMemEnd() { 108 kHighMemEnd = GetMaxUserVirtualAddress(); 109 // Increase kHighMemEnd to make sure it's properly 110 // aligned together with kHighMemBeg: 111 kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1; 112 } 113 114 void PrintAddressSpaceLayout() { 115 if (kHighMemBeg) { 116 Printf("|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemBeg, 117 (void *)kHighMemEnd); 118 Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg, 119 (void *)kHighShadowEnd); 120 } 121 Printf("|| `[%p, %p]` || ShadowGap ||\n", (void *)kShadowGapBeg, 122 (void *)kShadowGapEnd); 123 if (kLowShadowBeg) { 124 Printf("|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowBeg, 125 (void *)kLowShadowEnd); 126 Printf("|| `[%p, %p]` || LowMem ||\n", (void *)kLowMemBeg, 127 (void *)kLowMemEnd); 128 } 129 Printf("MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg), 130 (void *)MEM_TO_SHADOW(kLowShadowEnd)); 131 if (kHighMemBeg) { 132 Printf(" %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg), 133 (void *)MEM_TO_SHADOW(kHighShadowEnd)); 134 } 135 Printf("\n"); 136 Printf("malloc_context_size=%zu\n", 137 (uptr)common_flags()->malloc_context_size); 138 139 Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE); 140 Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY); 141 Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET); 142 CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7); 143 } 144 145 static void MemprofInitInternal() { 146 if (LIKELY(memprof_inited)) 147 return; 148 SanitizerToolName = "MemProfiler"; 149 CHECK(!memprof_init_is_running && "MemProf init calls itself!"); 150 memprof_init_is_running = true; 151 152 CacheBinaryName(); 153 154 // Initialize flags. This must be done early, because most of the 155 // initialization steps look at flags(). 156 InitializeFlags(); 157 158 AvoidCVE_2016_2143(); 159 160 SetMallocContextSize(common_flags()->malloc_context_size); 161 162 InitializeHighMemEnd(); 163 164 // Make sure we are not statically linked. 165 MemprofDoesNotSupportStaticLinkage(); 166 167 // Install tool-specific callbacks in sanitizer_common. 168 AddDieCallback(MemprofDie); 169 SetCheckUnwindCallback(CheckUnwind); 170 171 // Use profile name specified via the binary itself if it exists, and hasn't 172 // been overrriden by a flag at runtime. 173 if (__memprof_profile_filename[0] != 0 && !common_flags()->log_path) 174 __sanitizer_set_report_path(__memprof_profile_filename); 175 else 176 __sanitizer_set_report_path(common_flags()->log_path); 177 178 __sanitizer::InitializePlatformEarly(); 179 180 // Setup internal allocator callback. 181 SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY); 182 183 InitializeMemprofInterceptors(); 184 CheckASLR(); 185 186 ReplaceSystemMalloc(); 187 188 DisableCoreDumperIfNecessary(); 189 190 InitializeShadowMemory(); 191 192 TSDInit(PlatformTSDDtor); 193 InstallDeadlySignalHandlers(MemprofOnDeadlySignal); 194 195 InitializeAllocator(); 196 197 if (flags()->atexit) 198 Atexit(memprof_atexit); 199 200 InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); 201 202 // interceptors 203 InitTlsSize(); 204 205 // Create main thread. 206 MemprofThread *main_thread = CreateMainThread(); 207 CHECK_EQ(0, main_thread->tid()); 208 force_interface_symbols(); // no-op. 209 SanitizerInitializeUnwinder(); 210 211 Symbolizer::LateInitialize(); 212 213 VReport(1, "MemProfiler Init done\n"); 214 215 memprof_init_is_running = false; 216 memprof_inited = 1; 217 } 218 219 void MemprofInitTime() { 220 if (LIKELY(memprof_timestamp_inited)) 221 return; 222 timespec ts; 223 clock_gettime(CLOCK_REALTIME, &ts); 224 memprof_init_timestamp_s = ts.tv_sec; 225 memprof_timestamp_inited = 1; 226 } 227 228 // Initialize as requested from some part of MemProf runtime library 229 // (interceptors, allocator, etc). 230 void MemprofInitFromRtl() { MemprofInitInternal(); } 231 232 #if MEMPROF_DYNAMIC 233 // Initialize runtime in case it's LD_PRELOAD-ed into uninstrumented executable 234 // (and thus normal initializers from .preinit_array or modules haven't run). 235 236 class MemprofInitializer { 237 public: 238 MemprofInitializer() { MemprofInitFromRtl(); } 239 }; 240 241 static MemprofInitializer memprof_initializer; 242 #endif // MEMPROF_DYNAMIC 243 244 } // namespace __memprof 245 246 // ---------------------- Interface ---------------- {{{1 247 using namespace __memprof; 248 249 // Initialize as requested from instrumented application code. 250 void __memprof_init() { 251 MemprofInitTime(); 252 MemprofInitInternal(); 253 } 254 255 void __memprof_preinit() { MemprofInitInternal(); } 256 257 void __memprof_version_mismatch_check_v1() {} 258 259 void __memprof_record_access(void const volatile *addr) { 260 __memprof::RecordAccess((uptr)addr); 261 } 262 263 void __memprof_record_access_range(void const volatile *addr, uptr size) { 264 for (uptr a = (uptr)addr; a < (uptr)addr + size; a += kWordSize) 265 __memprof::RecordAccess(a); 266 } 267 268 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16 269 __sanitizer_unaligned_load16(const uu16 *p) { 270 __memprof_record_access(p); 271 return *p; 272 } 273 274 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32 275 __sanitizer_unaligned_load32(const uu32 *p) { 276 __memprof_record_access(p); 277 return *p; 278 } 279 280 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64 281 __sanitizer_unaligned_load64(const uu64 *p) { 282 __memprof_record_access(p); 283 return *p; 284 } 285 286 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void 287 __sanitizer_unaligned_store16(uu16 *p, u16 x) { 288 __memprof_record_access(p); 289 *p = x; 290 } 291 292 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void 293 __sanitizer_unaligned_store32(uu32 *p, u32 x) { 294 __memprof_record_access(p); 295 *p = x; 296 } 297 298 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void 299 __sanitizer_unaligned_store64(uu64 *p, u64 x) { 300 __memprof_record_access(p); 301 *p = x; 302 } 303