1 //===-- hwasan.cpp --------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of HWAddressSanitizer. 10 // 11 // HWAddressSanitizer runtime. 12 //===----------------------------------------------------------------------===// 13 14 #include "hwasan.h" 15 16 #include "hwasan_checks.h" 17 #include "hwasan_dynamic_shadow.h" 18 #include "hwasan_globals.h" 19 #include "hwasan_mapping.h" 20 #include "hwasan_poisoning.h" 21 #include "hwasan_report.h" 22 #include "hwasan_thread.h" 23 #include "hwasan_thread_list.h" 24 #include "sanitizer_common/sanitizer_atomic.h" 25 #include "sanitizer_common/sanitizer_common.h" 26 #include "sanitizer_common/sanitizer_flag_parser.h" 27 #include "sanitizer_common/sanitizer_flags.h" 28 #include "sanitizer_common/sanitizer_libc.h" 29 #include "sanitizer_common/sanitizer_procmaps.h" 30 #include "sanitizer_common/sanitizer_stackdepot.h" 31 #include "sanitizer_common/sanitizer_stacktrace.h" 32 #include "sanitizer_common/sanitizer_symbolizer.h" 33 #include "ubsan/ubsan_flags.h" 34 #include "ubsan/ubsan_init.h" 35 36 // ACHTUNG! No system header includes in this file. 37 38 using namespace __sanitizer; 39 40 namespace __hwasan { 41 42 static Flags hwasan_flags; 43 44 Flags *flags() { 45 return &hwasan_flags; 46 } 47 48 int hwasan_inited = 0; 49 int hwasan_instrumentation_inited = 0; 50 bool hwasan_init_is_running; 51 52 int hwasan_report_count = 0; 53 54 uptr kLowShadowStart; 55 uptr kLowShadowEnd; 56 uptr kHighShadowStart; 57 uptr kHighShadowEnd; 58 59 void Flags::SetDefaults() { 60 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; 61 #include "hwasan_flags.inc" 62 #undef HWASAN_FLAG 63 } 64 65 static void RegisterHwasanFlags(FlagParser *parser, Flags *f) { 66 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) \ 67 RegisterFlag(parser, #Name, Description, &f->Name); 68 #include "hwasan_flags.inc" 69 #undef HWASAN_FLAG 70 } 71 72 static void InitializeFlags() { 73 SetCommonFlagsDefaults(); 74 { 75 CommonFlags cf; 76 cf.CopyFrom(*common_flags()); 77 cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH"); 78 cf.malloc_context_size = 20; 79 cf.handle_ioctl = true; 80 // FIXME: test and enable. 81 cf.check_printf = false; 82 cf.intercept_tls_get_addr = true; 83 cf.exitcode = 99; 84 // 8 shadow pages ~512kB, small enough to cover common stack sizes. 85 cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8); 86 // Sigtrap is used in error reporting. 87 cf.handle_sigtrap = kHandleSignalExclusive; 88 89 #if SANITIZER_ANDROID 90 // Let platform handle other signals. It is better at reporting them then we 91 // are. 92 cf.handle_segv = kHandleSignalNo; 93 cf.handle_sigbus = kHandleSignalNo; 94 cf.handle_abort = kHandleSignalNo; 95 cf.handle_sigill = kHandleSignalNo; 96 cf.handle_sigfpe = kHandleSignalNo; 97 #endif 98 OverrideCommonFlags(cf); 99 } 100 101 Flags *f = flags(); 102 f->SetDefaults(); 103 104 FlagParser parser; 105 RegisterHwasanFlags(&parser, f); 106 RegisterCommonFlags(&parser); 107 108 #if HWASAN_CONTAINS_UBSAN 109 __ubsan::Flags *uf = __ubsan::flags(); 110 uf->SetDefaults(); 111 112 FlagParser ubsan_parser; 113 __ubsan::RegisterUbsanFlags(&ubsan_parser, uf); 114 RegisterCommonFlags(&ubsan_parser); 115 #endif 116 117 // Override from user-specified string. 118 if (__hwasan_default_options) 119 parser.ParseString(__hwasan_default_options()); 120 #if HWASAN_CONTAINS_UBSAN 121 const char *ubsan_default_options = __ubsan_default_options(); 122 ubsan_parser.ParseString(ubsan_default_options); 123 #endif 124 125 parser.ParseStringFromEnv("HWASAN_OPTIONS"); 126 #if HWASAN_CONTAINS_UBSAN 127 ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); 128 #endif 129 130 InitializeCommonFlags(); 131 132 if (Verbosity()) ReportUnrecognizedFlags(); 133 134 if (common_flags()->help) parser.PrintFlagDescriptions(); 135 } 136 137 static void CheckUnwind() { 138 GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); 139 stack.Print(); 140 } 141 142 static void HwasanFormatMemoryUsage(InternalScopedString &s) { 143 HwasanThreadList &thread_list = hwasanThreadList(); 144 auto thread_stats = thread_list.GetThreadStats(); 145 auto sds = StackDepotGetStats(); 146 AllocatorStatCounters asc; 147 GetAllocatorStats(asc); 148 s.append( 149 "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd" 150 " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd" 151 " heap: %zd", 152 internal_getpid(), GetRSS(), thread_stats.n_live_threads, 153 thread_stats.total_stack_size, 154 thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(), 155 sds.allocated, sds.n_uniq_ids, asc[AllocatorStatMapped]); 156 } 157 158 #if SANITIZER_ANDROID 159 static constexpr uptr kMemoryUsageBufferSize = 4096; 160 161 static char *memory_usage_buffer = nullptr; 162 163 static void InitMemoryUsage() { 164 memory_usage_buffer = 165 (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string"); 166 CHECK(memory_usage_buffer); 167 memory_usage_buffer[0] = '\0'; 168 DecorateMapping((uptr)memory_usage_buffer, kMemoryUsageBufferSize, 169 memory_usage_buffer); 170 } 171 172 void UpdateMemoryUsage() { 173 if (!flags()->export_memory_stats) 174 return; 175 if (!memory_usage_buffer) 176 InitMemoryUsage(); 177 InternalScopedString s; 178 HwasanFormatMemoryUsage(s); 179 internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1); 180 memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0'; 181 } 182 #else 183 void UpdateMemoryUsage() {} 184 #endif 185 186 void HwasanAtExit() { 187 if (common_flags()->print_module_map) 188 DumpProcessMap(); 189 if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) 190 ReportStats(); 191 if (hwasan_report_count > 0) { 192 // ReportAtExitStatistics(); 193 if (common_flags()->exitcode) 194 internal__exit(common_flags()->exitcode); 195 } 196 } 197 198 void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc, 199 uptr *registers_frame) { 200 InternalMmapVector<BufferedStackTrace> stack_buffer(1); 201 BufferedStackTrace *stack = stack_buffer.data(); 202 stack->Reset(); 203 stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); 204 205 // The second stack frame contains the failure __hwasan_check function, as 206 // we have a stack frame for the registers saved in __hwasan_tag_mismatch that 207 // we wish to ignore. This (currently) only occurs on AArch64, as x64 208 // implementations use SIGTRAP to implement the failure, and thus do not go 209 // through the stack saver. 210 if (registers_frame && stack->trace && stack->size > 0) { 211 stack->trace++; 212 stack->size--; 213 } 214 215 bool fatal = flags()->halt_on_error || !ai.recover; 216 ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, 217 registers_frame); 218 } 219 220 void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame, 221 size_t outsize) { 222 __hwasan::AccessInfo ai; 223 ai.is_store = access_info & 0x10; 224 ai.is_load = !ai.is_store; 225 ai.recover = access_info & 0x20; 226 ai.addr = addr; 227 if ((access_info & 0xf) == 0xf) 228 ai.size = outsize; 229 else 230 ai.size = 1 << (access_info & 0xf); 231 232 HandleTagMismatch(ai, (uptr)__builtin_return_address(0), 233 (uptr)__builtin_frame_address(0), nullptr, registers_frame); 234 __builtin_unreachable(); 235 } 236 237 Thread *GetCurrentThread() { 238 uptr *ThreadLongPtr = GetCurrentThreadLongPtr(); 239 if (UNLIKELY(*ThreadLongPtr == 0)) 240 return nullptr; 241 auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr; 242 return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next()); 243 } 244 245 } // namespace __hwasan 246 247 using namespace __hwasan; 248 249 void __sanitizer::BufferedStackTrace::UnwindImpl( 250 uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { 251 Thread *t = GetCurrentThread(); 252 if (!t) { 253 // The thread is still being created, or has already been destroyed. 254 size = 0; 255 return; 256 } 257 Unwind(max_depth, pc, bp, context, t->stack_top(), t->stack_bottom(), 258 request_fast); 259 } 260 261 static bool InitializeSingleGlobal(const hwasan_global &global) { 262 uptr full_granule_size = RoundDownTo(global.size(), 16); 263 TagMemoryAligned(global.addr(), full_granule_size, global.tag()); 264 if (global.size() % 16) 265 TagMemoryAligned(global.addr() + full_granule_size, 16, global.size() % 16); 266 return false; 267 } 268 269 static void InitLoadedGlobals() { 270 dl_iterate_phdr( 271 [](dl_phdr_info *info, size_t /* size */, void * /* data */) -> int { 272 for (const hwasan_global &global : HwasanGlobalsFor( 273 info->dlpi_addr, info->dlpi_phdr, info->dlpi_phnum)) 274 InitializeSingleGlobal(global); 275 return 0; 276 }, 277 nullptr); 278 } 279 280 // Prepare to run instrumented code on the main thread. 281 static void InitInstrumentation() { 282 if (hwasan_instrumentation_inited) return; 283 284 InitializeOsSupport(); 285 286 if (!InitShadow()) { 287 Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); 288 DumpProcessMap(); 289 Die(); 290 } 291 292 InitThreads(); 293 294 hwasan_instrumentation_inited = 1; 295 } 296 297 // Interface. 298 299 uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. 300 301 // This function was used by the old frame descriptor mechanism. We keep it 302 // around to avoid breaking ABI. 303 void __hwasan_init_frames(uptr beg, uptr end) {} 304 305 void __hwasan_init_static() { 306 InitShadowGOT(); 307 InitInstrumentation(); 308 309 // In the non-static code path we call dl_iterate_phdr here. But at this point 310 // libc might not have been initialized enough for dl_iterate_phdr to work. 311 // Fortunately, since this is a statically linked executable we can use the 312 // linker-defined symbol __ehdr_start to find the only relevant set of phdrs. 313 extern ElfW(Ehdr) __ehdr_start; 314 for (const hwasan_global &global : HwasanGlobalsFor( 315 /* base */ 0, 316 reinterpret_cast<const ElfW(Phdr) *>( 317 reinterpret_cast<const char *>(&__ehdr_start) + 318 __ehdr_start.e_phoff), 319 __ehdr_start.e_phnum)) 320 InitializeSingleGlobal(global); 321 } 322 323 __attribute__((constructor(0))) void __hwasan_init() { 324 CHECK(!hwasan_init_is_running); 325 if (hwasan_inited) return; 326 hwasan_init_is_running = 1; 327 SanitizerToolName = "HWAddressSanitizer"; 328 329 InitTlsSize(); 330 331 CacheBinaryName(); 332 InitializeFlags(); 333 334 // Install tool-specific callbacks in sanitizer_common. 335 SetCheckUnwindCallback(CheckUnwind); 336 337 __sanitizer_set_report_path(common_flags()->log_path); 338 339 AndroidTestTlsSlot(); 340 341 DisableCoreDumperIfNecessary(); 342 343 InitInstrumentation(); 344 InitLoadedGlobals(); 345 346 // Needs to be called here because flags()->random_tags might not have been 347 // initialized when InitInstrumentation() was called. 348 GetCurrentThread()->EnsureRandomStateInited(); 349 350 SetPrintfAndReportCallback(AppendToErrorMessageBuffer); 351 // This may call libc -> needs initialized shadow. 352 AndroidLogInit(); 353 354 InitializeInterceptors(); 355 InstallDeadlySignalHandlers(HwasanOnDeadlySignal); 356 InstallAtExitHandler(); // Needs __cxa_atexit interceptor. 357 358 InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); 359 360 HwasanTSDInit(); 361 HwasanTSDThreadInit(); 362 363 HwasanAllocatorInit(); 364 HwasanInstallAtForkHandler(); 365 366 #if HWASAN_CONTAINS_UBSAN 367 __ubsan::InitAsPlugin(); 368 #endif 369 370 VPrintf(1, "HWAddressSanitizer init done\n"); 371 372 hwasan_init_is_running = 0; 373 hwasan_inited = 1; 374 } 375 376 void __hwasan_library_loaded(ElfW(Addr) base, const ElfW(Phdr) * phdr, 377 ElfW(Half) phnum) { 378 for (const hwasan_global &global : HwasanGlobalsFor(base, phdr, phnum)) 379 InitializeSingleGlobal(global); 380 } 381 382 void __hwasan_library_unloaded(ElfW(Addr) base, const ElfW(Phdr) * phdr, 383 ElfW(Half) phnum) { 384 for (; phnum != 0; ++phdr, --phnum) 385 if (phdr->p_type == PT_LOAD) 386 TagMemory(base + phdr->p_vaddr, phdr->p_memsz, 0); 387 } 388 389 void __hwasan_print_shadow(const void *p, uptr sz) { 390 uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); 391 uptr shadow_first = MemToShadow(ptr_raw); 392 uptr shadow_last = MemToShadow(ptr_raw + sz - 1); 393 Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw, 394 ptr_raw + sz, GetTagFromPointer((uptr)p)); 395 for (uptr s = shadow_first; s <= shadow_last; ++s) { 396 tag_t mem_tag = *reinterpret_cast<tag_t *>(s); 397 uptr granule_addr = ShadowToMem(s); 398 if (mem_tag && mem_tag < kShadowAlignment) 399 Printf(" %zx: %02x(%02x)\n", granule_addr, mem_tag, 400 *reinterpret_cast<tag_t *>(granule_addr + kShadowAlignment - 1)); 401 else 402 Printf(" %zx: %02x\n", granule_addr, mem_tag); 403 } 404 } 405 406 sptr __hwasan_test_shadow(const void *p, uptr sz) { 407 if (sz == 0) 408 return -1; 409 tag_t ptr_tag = GetTagFromPointer((uptr)p); 410 uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); 411 uptr shadow_first = MemToShadow(ptr_raw); 412 uptr shadow_last = MemToShadow(ptr_raw + sz - 1); 413 for (uptr s = shadow_first; s <= shadow_last; ++s) 414 if (*(tag_t *)s != ptr_tag) { 415 sptr offset = ShadowToMem(s) - ptr_raw; 416 return offset < 0 ? 0 : offset; 417 } 418 return -1; 419 } 420 421 u16 __sanitizer_unaligned_load16(const uu16 *p) { 422 return *p; 423 } 424 u32 __sanitizer_unaligned_load32(const uu32 *p) { 425 return *p; 426 } 427 u64 __sanitizer_unaligned_load64(const uu64 *p) { 428 return *p; 429 } 430 void __sanitizer_unaligned_store16(uu16 *p, u16 x) { 431 *p = x; 432 } 433 void __sanitizer_unaligned_store32(uu32 *p, u32 x) { 434 *p = x; 435 } 436 void __sanitizer_unaligned_store64(uu64 *p, u64 x) { 437 *p = x; 438 } 439 440 void __hwasan_loadN(uptr p, uptr sz) { 441 CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz); 442 } 443 void __hwasan_load1(uptr p) { 444 CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p); 445 } 446 void __hwasan_load2(uptr p) { 447 CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p); 448 } 449 void __hwasan_load4(uptr p) { 450 CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p); 451 } 452 void __hwasan_load8(uptr p) { 453 CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p); 454 } 455 void __hwasan_load16(uptr p) { 456 CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p); 457 } 458 459 void __hwasan_loadN_noabort(uptr p, uptr sz) { 460 CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz); 461 } 462 void __hwasan_load1_noabort(uptr p) { 463 CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p); 464 } 465 void __hwasan_load2_noabort(uptr p) { 466 CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p); 467 } 468 void __hwasan_load4_noabort(uptr p) { 469 CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p); 470 } 471 void __hwasan_load8_noabort(uptr p) { 472 CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p); 473 } 474 void __hwasan_load16_noabort(uptr p) { 475 CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p); 476 } 477 478 void __hwasan_storeN(uptr p, uptr sz) { 479 CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz); 480 } 481 void __hwasan_store1(uptr p) { 482 CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p); 483 } 484 void __hwasan_store2(uptr p) { 485 CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p); 486 } 487 void __hwasan_store4(uptr p) { 488 CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p); 489 } 490 void __hwasan_store8(uptr p) { 491 CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p); 492 } 493 void __hwasan_store16(uptr p) { 494 CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p); 495 } 496 497 void __hwasan_storeN_noabort(uptr p, uptr sz) { 498 CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz); 499 } 500 void __hwasan_store1_noabort(uptr p) { 501 CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p); 502 } 503 void __hwasan_store2_noabort(uptr p) { 504 CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p); 505 } 506 void __hwasan_store4_noabort(uptr p) { 507 CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p); 508 } 509 void __hwasan_store8_noabort(uptr p) { 510 CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p); 511 } 512 void __hwasan_store16_noabort(uptr p) { 513 CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p); 514 } 515 516 void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) { 517 TagMemoryAligned(p, sz, tag); 518 } 519 520 uptr __hwasan_tag_pointer(uptr p, u8 tag) { 521 return AddTagToPointer(p, tag); 522 } 523 524 void __hwasan_handle_longjmp(const void *sp_dst) { 525 uptr dst = (uptr)sp_dst; 526 // HWASan does not support tagged SP. 527 CHECK(GetTagFromPointer(dst) == 0); 528 529 uptr sp = (uptr)__builtin_frame_address(0); 530 static const uptr kMaxExpectedCleanupSize = 64 << 20; // 64M 531 if (dst < sp || dst - sp > kMaxExpectedCleanupSize) { 532 Report( 533 "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: " 534 "stack top: %p; target %p; distance: %p (%zd)\n" 535 "False positive error reports may follow\n", 536 (void *)sp, (void *)dst, dst - sp); 537 return; 538 } 539 TagMemory(sp, dst - sp, 0); 540 } 541 542 void __hwasan_handle_vfork(const void *sp_dst) { 543 uptr sp = (uptr)sp_dst; 544 Thread *t = GetCurrentThread(); 545 CHECK(t); 546 uptr top = t->stack_top(); 547 uptr bottom = t->stack_bottom(); 548 if (top == 0 || bottom == 0 || sp < bottom || sp >= top) { 549 Report( 550 "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: " 551 "stack top: %zx; current %zx; bottom: %zx \n" 552 "False positive error reports may follow\n", 553 top, sp, bottom); 554 return; 555 } 556 TagMemory(bottom, sp - bottom, 0); 557 } 558 559 extern "C" void *__hwasan_extra_spill_area() { 560 Thread *t = GetCurrentThread(); 561 return &t->vfork_spill(); 562 } 563 564 void __hwasan_print_memory_usage() { 565 InternalScopedString s; 566 HwasanFormatMemoryUsage(s); 567 Printf("%s\n", s.data()); 568 } 569 570 static const u8 kFallbackTag = 0xBB & kTagMask; 571 572 u8 __hwasan_generate_tag() { 573 Thread *t = GetCurrentThread(); 574 if (!t) return kFallbackTag; 575 return t->GenerateRandomTag(); 576 } 577 578 #if !SANITIZER_SUPPORTS_WEAK_HOOKS 579 extern "C" { 580 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE 581 const char* __hwasan_default_options() { return ""; } 582 } // extern "C" 583 #endif 584 585 extern "C" { 586 SANITIZER_INTERFACE_ATTRIBUTE 587 void __sanitizer_print_stack_trace() { 588 GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); 589 stack.Print(); 590 } 591 592 // Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the 593 // rest of the mismatch handling code (C++). 594 void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame, 595 size_t outsize) { 596 __hwasan::HwasanTagMismatch(addr, access_info, registers_frame, outsize); 597 } 598 599 } // extern "C" 600