1 //===-- memprof_rtl.cpp --------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of MemProfiler, a memory profiler. 10 // 11 // Main file of the MemProf run-time library. 12 //===----------------------------------------------------------------------===// 13 14 #include "memprof_allocator.h" 15 #include "memprof_interceptors.h" 16 #include "memprof_interface_internal.h" 17 #include "memprof_internal.h" 18 #include "memprof_mapping.h" 19 #include "memprof_stack.h" 20 #include "memprof_stats.h" 21 #include "memprof_thread.h" 22 #include "sanitizer_common/sanitizer_atomic.h" 23 #include "sanitizer_common/sanitizer_flags.h" 24 #include "sanitizer_common/sanitizer_libc.h" 25 #include "sanitizer_common/sanitizer_symbolizer.h" 26 27 #include <time.h> 28 29 uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol. 30 31 // Allow the user to specify a profile output file via the binary. 32 SANITIZER_WEAK_ATTRIBUTE char __memprof_profile_filename[1]; 33 34 namespace __memprof { 35 36 static void MemprofDie() { 37 static atomic_uint32_t num_calls; 38 if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) { 39 // Don't die twice - run a busy loop. 40 while (1) { 41 } 42 } 43 if (common_flags()->print_module_map >= 1) 44 DumpProcessMap(); 45 if (flags()->unmap_shadow_on_exit) { 46 if (kHighShadowEnd) 47 UnmapOrDie((void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg); 48 } 49 } 50 51 static void CheckUnwind() { 52 GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check); 53 stack.Print(); 54 } 55 56 // -------------------------- Globals --------------------- {{{1 57 int memprof_inited; 58 int memprof_init_done; 59 bool memprof_init_is_running; 60 int memprof_timestamp_inited; 61 long memprof_init_timestamp_s; 62 63 uptr kHighMemEnd; 64 65 // -------------------------- Run-time entry ------------------- {{{1 66 // exported functions 67 68 #define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() __memprof::RecordAccess(addr); 69 70 #define MEMPROF_MEMORY_ACCESS_CALLBACK(type) \ 71 extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_##type(uptr addr) { \ 72 MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() \ 73 } 74 75 MEMPROF_MEMORY_ACCESS_CALLBACK(load) 76 MEMPROF_MEMORY_ACCESS_CALLBACK(store) 77 78 // Force the linker to keep the symbols for various MemProf interface 79 // functions. We want to keep those in the executable in order to let the 80 // instrumented dynamic libraries access the symbol even if it is not used by 81 // the executable itself. This should help if the build system is removing dead 82 // code at link time. 83 static NOINLINE void force_interface_symbols() { 84 volatile int fake_condition = 0; // prevent dead condition elimination. 85 // clang-format off 86 switch (fake_condition) { 87 case 1: __memprof_record_access(nullptr); break; 88 case 2: __memprof_record_access_range(nullptr, 0); break; 89 } 90 // clang-format on 91 } 92 93 static void memprof_atexit() { 94 Printf("MemProfiler exit stats:\n"); 95 __memprof_print_accumulated_stats(); 96 } 97 98 static void InitializeHighMemEnd() { 99 kHighMemEnd = GetMaxUserVirtualAddress(); 100 // Increase kHighMemEnd to make sure it's properly 101 // aligned together with kHighMemBeg: 102 kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1; 103 } 104 105 void PrintAddressSpaceLayout() { 106 if (kHighMemBeg) { 107 Printf("|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemBeg, 108 (void *)kHighMemEnd); 109 Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg, 110 (void *)kHighShadowEnd); 111 } 112 Printf("|| `[%p, %p]` || ShadowGap ||\n", (void *)kShadowGapBeg, 113 (void *)kShadowGapEnd); 114 if (kLowShadowBeg) { 115 Printf("|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowBeg, 116 (void *)kLowShadowEnd); 117 Printf("|| `[%p, %p]` || LowMem ||\n", (void *)kLowMemBeg, 118 (void *)kLowMemEnd); 119 } 120 Printf("MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg), 121 (void *)MEM_TO_SHADOW(kLowShadowEnd)); 122 if (kHighMemBeg) { 123 Printf(" %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg), 124 (void *)MEM_TO_SHADOW(kHighShadowEnd)); 125 } 126 Printf("\n"); 127 Printf("malloc_context_size=%zu\n", 128 (uptr)common_flags()->malloc_context_size); 129 130 Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE); 131 Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY); 132 Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET); 133 CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7); 134 } 135 136 static void MemprofInitInternal() { 137 if (LIKELY(memprof_inited)) 138 return; 139 SanitizerToolName = "MemProfiler"; 140 CHECK(!memprof_init_is_running && "MemProf init calls itself!"); 141 memprof_init_is_running = true; 142 143 CacheBinaryName(); 144 145 // Initialize flags. This must be done early, because most of the 146 // initialization steps look at flags(). 147 InitializeFlags(); 148 149 AvoidCVE_2016_2143(); 150 151 SetMallocContextSize(common_flags()->malloc_context_size); 152 153 InitializeHighMemEnd(); 154 155 // Make sure we are not statically linked. 156 MemprofDoesNotSupportStaticLinkage(); 157 158 // Install tool-specific callbacks in sanitizer_common. 159 AddDieCallback(MemprofDie); 160 SetCheckUnwindCallback(CheckUnwind); 161 162 // Use profile name specified via the binary itself if it exists, and hasn't 163 // been overrriden by a flag at runtime. 164 if (__memprof_profile_filename[0] != 0 && !common_flags()->log_path) 165 __sanitizer_set_report_path(__memprof_profile_filename); 166 else 167 __sanitizer_set_report_path(common_flags()->log_path); 168 169 __sanitizer::InitializePlatformEarly(); 170 171 // Re-exec ourselves if we need to set additional env or command line args. 172 MaybeReexec(); 173 174 // Setup internal allocator callback. 175 SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY); 176 177 InitializeMemprofInterceptors(); 178 CheckASLR(); 179 180 ReplaceSystemMalloc(); 181 182 DisableCoreDumperIfNecessary(); 183 184 InitializeShadowMemory(); 185 186 TSDInit(PlatformTSDDtor); 187 188 InitializeAllocator(); 189 190 // On Linux MemprofThread::ThreadStart() calls malloc() that's why 191 // memprof_inited should be set to 1 prior to initializing the threads. 192 memprof_inited = 1; 193 memprof_init_is_running = false; 194 195 if (flags()->atexit) 196 Atexit(memprof_atexit); 197 198 InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); 199 200 // interceptors 201 InitTlsSize(); 202 203 // Create main thread. 204 MemprofThread *main_thread = CreateMainThread(); 205 CHECK_EQ(0, main_thread->tid()); 206 force_interface_symbols(); // no-op. 207 SanitizerInitializeUnwinder(); 208 209 Symbolizer::LateInitialize(); 210 211 VReport(1, "MemProfiler Init done\n"); 212 213 memprof_init_done = 1; 214 } 215 216 void MemprofInitTime() { 217 if (LIKELY(memprof_timestamp_inited)) 218 return; 219 timespec ts; 220 clock_gettime(CLOCK_REALTIME, &ts); 221 memprof_init_timestamp_s = ts.tv_sec; 222 memprof_timestamp_inited = 1; 223 } 224 225 // Initialize as requested from some part of MemProf runtime library 226 // (interceptors, allocator, etc). 227 void MemprofInitFromRtl() { MemprofInitInternal(); } 228 229 #if MEMPROF_DYNAMIC 230 // Initialize runtime in case it's LD_PRELOAD-ed into uninstrumented executable 231 // (and thus normal initializers from .preinit_array or modules haven't run). 232 233 class MemprofInitializer { 234 public: 235 MemprofInitializer() { MemprofInitFromRtl(); } 236 }; 237 238 static MemprofInitializer memprof_initializer; 239 #endif // MEMPROF_DYNAMIC 240 241 } // namespace __memprof 242 243 // ---------------------- Interface ---------------- {{{1 244 using namespace __memprof; 245 246 // Initialize as requested from instrumented application code. 247 void __memprof_init() { 248 MemprofInitTime(); 249 MemprofInitInternal(); 250 } 251 252 void __memprof_preinit() { MemprofInitInternal(); } 253 254 void __memprof_version_mismatch_check_v1() {} 255 256 void __memprof_record_access(void const volatile *addr) { 257 __memprof::RecordAccess((uptr)addr); 258 } 259 260 void __memprof_record_access_range(void const volatile *addr, uptr size) { 261 for (uptr a = (uptr)addr; a < (uptr)addr + size; a += kWordSize) 262 __memprof::RecordAccess(a); 263 } 264 265 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16 266 __sanitizer_unaligned_load16(const uu16 *p) { 267 __memprof_record_access(p); 268 return *p; 269 } 270 271 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32 272 __sanitizer_unaligned_load32(const uu32 *p) { 273 __memprof_record_access(p); 274 return *p; 275 } 276 277 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64 278 __sanitizer_unaligned_load64(const uu64 *p) { 279 __memprof_record_access(p); 280 return *p; 281 } 282 283 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void 284 __sanitizer_unaligned_store16(uu16 *p, u16 x) { 285 __memprof_record_access(p); 286 *p = x; 287 } 288 289 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void 290 __sanitizer_unaligned_store32(uu32 *p, u32 x) { 291 __memprof_record_access(p); 292 *p = x; 293 } 294 295 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void 296 __sanitizer_unaligned_store64(uu64 *p, u64 x) { 297 __memprof_record_access(p); 298 *p = x; 299 } 300