xref: /freebsd/contrib/llvm-project/compiler-rt/lib/hwasan/hwasan.cpp (revision ec0ea6efa1ad229d75c394c1a9b9cac33af2b1d3)
1 //===-- hwasan.cpp --------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of HWAddressSanitizer.
10 //
11 // HWAddressSanitizer runtime.
12 //===----------------------------------------------------------------------===//
13 
14 #include "hwasan.h"
15 
16 #include "hwasan_checks.h"
17 #include "hwasan_dynamic_shadow.h"
18 #include "hwasan_globals.h"
19 #include "hwasan_poisoning.h"
20 #include "hwasan_report.h"
21 #include "hwasan_thread.h"
22 #include "hwasan_thread_list.h"
23 #include "sanitizer_common/sanitizer_atomic.h"
24 #include "sanitizer_common/sanitizer_common.h"
25 #include "sanitizer_common/sanitizer_flag_parser.h"
26 #include "sanitizer_common/sanitizer_flags.h"
27 #include "sanitizer_common/sanitizer_libc.h"
28 #include "sanitizer_common/sanitizer_procmaps.h"
29 #include "sanitizer_common/sanitizer_stackdepot.h"
30 #include "sanitizer_common/sanitizer_stacktrace.h"
31 #include "sanitizer_common/sanitizer_symbolizer.h"
32 #include "ubsan/ubsan_flags.h"
33 #include "ubsan/ubsan_init.h"
34 
35 // ACHTUNG! No system header includes in this file.
36 
37 using namespace __sanitizer;
38 
39 namespace __hwasan {
40 
41 static Flags hwasan_flags;
42 
43 Flags *flags() {
44   return &hwasan_flags;
45 }
46 
47 int hwasan_inited = 0;
48 int hwasan_instrumentation_inited = 0;
49 bool hwasan_init_is_running;
50 
51 int hwasan_report_count = 0;
52 
53 uptr kLowShadowStart;
54 uptr kLowShadowEnd;
55 uptr kHighShadowStart;
56 uptr kHighShadowEnd;
57 
58 void Flags::SetDefaults() {
59 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
60 #include "hwasan_flags.inc"
61 #undef HWASAN_FLAG
62 }
63 
64 static void RegisterHwasanFlags(FlagParser *parser, Flags *f) {
65 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) \
66   RegisterFlag(parser, #Name, Description, &f->Name);
67 #include "hwasan_flags.inc"
68 #undef HWASAN_FLAG
69 }
70 
71 static void InitializeFlags() {
72   SetCommonFlagsDefaults();
73   {
74     CommonFlags cf;
75     cf.CopyFrom(*common_flags());
76     cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH");
77     cf.malloc_context_size = 20;
78     cf.handle_ioctl = true;
79     // FIXME: test and enable.
80     cf.check_printf = false;
81     cf.intercept_tls_get_addr = true;
82     cf.exitcode = 99;
83     // 8 shadow pages ~512kB, small enough to cover common stack sizes.
84     cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8);
85     // Sigtrap is used in error reporting.
86     cf.handle_sigtrap = kHandleSignalExclusive;
87 
88 #if SANITIZER_ANDROID
89     // Let platform handle other signals. It is better at reporting them then we
90     // are.
91     cf.handle_segv = kHandleSignalNo;
92     cf.handle_sigbus = kHandleSignalNo;
93     cf.handle_abort = kHandleSignalNo;
94     cf.handle_sigill = kHandleSignalNo;
95     cf.handle_sigfpe = kHandleSignalNo;
96 #endif
97     OverrideCommonFlags(cf);
98   }
99 
100   Flags *f = flags();
101   f->SetDefaults();
102 
103   FlagParser parser;
104   RegisterHwasanFlags(&parser, f);
105   RegisterCommonFlags(&parser);
106 
107 #if HWASAN_CONTAINS_UBSAN
108   __ubsan::Flags *uf = __ubsan::flags();
109   uf->SetDefaults();
110 
111   FlagParser ubsan_parser;
112   __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
113   RegisterCommonFlags(&ubsan_parser);
114 #endif
115 
116   // Override from user-specified string.
117   if (__hwasan_default_options)
118     parser.ParseString(__hwasan_default_options());
119 #if HWASAN_CONTAINS_UBSAN
120   const char *ubsan_default_options = __ubsan_default_options();
121   ubsan_parser.ParseString(ubsan_default_options);
122 #endif
123 
124   parser.ParseStringFromEnv("HWASAN_OPTIONS");
125 #if HWASAN_CONTAINS_UBSAN
126   ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
127 #endif
128 
129   InitializeCommonFlags();
130 
131   if (Verbosity()) ReportUnrecognizedFlags();
132 
133   if (common_flags()->help) parser.PrintFlagDescriptions();
134 }
135 
136 static void CheckUnwind() {
137   GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
138   stack.Print();
139 }
140 
141 static void HwasanFormatMemoryUsage(InternalScopedString &s) {
142   HwasanThreadList &thread_list = hwasanThreadList();
143   auto thread_stats = thread_list.GetThreadStats();
144   auto *sds = StackDepotGetStats();
145   AllocatorStatCounters asc;
146   GetAllocatorStats(asc);
147   s.append(
148       "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd"
149       " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd"
150       " heap: %zd",
151       internal_getpid(), GetRSS(), thread_stats.n_live_threads,
152       thread_stats.total_stack_size,
153       thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(),
154       sds->allocated, sds->n_uniq_ids, asc[AllocatorStatMapped]);
155 }
156 
157 #if SANITIZER_ANDROID
158 static constexpr uptr kMemoryUsageBufferSize = 4096;
159 
160 static char *memory_usage_buffer = nullptr;
161 
162 static void InitMemoryUsage() {
163   memory_usage_buffer =
164       (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string");
165   CHECK(memory_usage_buffer);
166   memory_usage_buffer[0] = '\0';
167   DecorateMapping((uptr)memory_usage_buffer, kMemoryUsageBufferSize,
168                   memory_usage_buffer);
169 }
170 
171 void UpdateMemoryUsage() {
172   if (!flags()->export_memory_stats)
173     return;
174   if (!memory_usage_buffer)
175     InitMemoryUsage();
176   InternalScopedString s;
177   HwasanFormatMemoryUsage(s);
178   internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1);
179   memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0';
180 }
181 #else
182 void UpdateMemoryUsage() {}
183 #endif
184 
185 void HwasanAtExit() {
186   if (common_flags()->print_module_map)
187     DumpProcessMap();
188   if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
189     ReportStats();
190   if (hwasan_report_count > 0) {
191     // ReportAtExitStatistics();
192     if (common_flags()->exitcode)
193       internal__exit(common_flags()->exitcode);
194   }
195 }
196 
197 void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc,
198                        uptr *registers_frame) {
199   InternalMmapVector<BufferedStackTrace> stack_buffer(1);
200   BufferedStackTrace *stack = stack_buffer.data();
201   stack->Reset();
202   stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal);
203 
204   // The second stack frame contains the failure __hwasan_check function, as
205   // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
206   // we wish to ignore. This (currently) only occurs on AArch64, as x64
207   // implementations use SIGTRAP to implement the failure, and thus do not go
208   // through the stack saver.
209   if (registers_frame && stack->trace && stack->size > 0) {
210     stack->trace++;
211     stack->size--;
212   }
213 
214   bool fatal = flags()->halt_on_error || !ai.recover;
215   ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal,
216                     registers_frame);
217 }
218 
219 void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame,
220                        size_t outsize) {
221   __hwasan::AccessInfo ai;
222   ai.is_store = access_info & 0x10;
223   ai.is_load = !ai.is_store;
224   ai.recover = access_info & 0x20;
225   ai.addr = addr;
226   if ((access_info & 0xf) == 0xf)
227     ai.size = outsize;
228   else
229     ai.size = 1 << (access_info & 0xf);
230 
231   HandleTagMismatch(ai, (uptr)__builtin_return_address(0),
232                     (uptr)__builtin_frame_address(0), nullptr, registers_frame);
233   __builtin_unreachable();
234 }
235 
236 Thread *GetCurrentThread() {
237   uptr *ThreadLongPtr = GetCurrentThreadLongPtr();
238   if (UNLIKELY(*ThreadLongPtr == 0))
239     return nullptr;
240   auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr;
241   return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next());
242 }
243 
244 } // namespace __hwasan
245 
246 using namespace __hwasan;
247 
248 void __sanitizer::BufferedStackTrace::UnwindImpl(
249     uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
250   Thread *t = GetCurrentThread();
251   if (!t) {
252     // The thread is still being created, or has already been destroyed.
253     size = 0;
254     return;
255   }
256   Unwind(max_depth, pc, bp, context, t->stack_top(), t->stack_bottom(),
257          request_fast);
258 }
259 
260 static bool InitializeSingleGlobal(const hwasan_global &global) {
261   uptr full_granule_size = RoundDownTo(global.size(), 16);
262   TagMemoryAligned(global.addr(), full_granule_size, global.tag());
263   if (global.size() % 16)
264     TagMemoryAligned(global.addr() + full_granule_size, 16, global.size() % 16);
265   return false;
266 }
267 
268 static void InitLoadedGlobals() {
269   dl_iterate_phdr(
270       [](dl_phdr_info *info, size_t /* size */, void * /* data */) -> int {
271         for (const hwasan_global &global : HwasanGlobalsFor(
272                  info->dlpi_addr, info->dlpi_phdr, info->dlpi_phnum))
273           InitializeSingleGlobal(global);
274         return 0;
275       },
276       nullptr);
277 }
278 
279 // Prepare to run instrumented code on the main thread.
280 static void InitInstrumentation() {
281   if (hwasan_instrumentation_inited) return;
282 
283   InitializeOsSupport();
284 
285   if (!InitShadow()) {
286     Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
287     DumpProcessMap();
288     Die();
289   }
290 
291   InitThreads();
292 
293   hwasan_instrumentation_inited = 1;
294 }
295 
296 // Interface.
297 
298 uptr __hwasan_shadow_memory_dynamic_address;  // Global interface symbol.
299 
300 // This function was used by the old frame descriptor mechanism. We keep it
301 // around to avoid breaking ABI.
302 void __hwasan_init_frames(uptr beg, uptr end) {}
303 
304 void __hwasan_init_static() {
305   InitShadowGOT();
306   InitInstrumentation();
307 
308   // In the non-static code path we call dl_iterate_phdr here. But at this point
309   // libc might not have been initialized enough for dl_iterate_phdr to work.
310   // Fortunately, since this is a statically linked executable we can use the
311   // linker-defined symbol __ehdr_start to find the only relevant set of phdrs.
312   extern ElfW(Ehdr) __ehdr_start;
313   for (const hwasan_global &global : HwasanGlobalsFor(
314            /* base */ 0,
315            reinterpret_cast<const ElfW(Phdr) *>(
316                reinterpret_cast<const char *>(&__ehdr_start) +
317                __ehdr_start.e_phoff),
318            __ehdr_start.e_phnum))
319     InitializeSingleGlobal(global);
320 }
321 
322 void __hwasan_init() {
323   CHECK(!hwasan_init_is_running);
324   if (hwasan_inited) return;
325   hwasan_init_is_running = 1;
326   SanitizerToolName = "HWAddressSanitizer";
327 
328   InitTlsSize();
329 
330   CacheBinaryName();
331   InitializeFlags();
332 
333   // Install tool-specific callbacks in sanitizer_common.
334   SetCheckUnwindCallback(CheckUnwind);
335 
336   __sanitizer_set_report_path(common_flags()->log_path);
337 
338   AndroidTestTlsSlot();
339 
340   DisableCoreDumperIfNecessary();
341 
342   InitInstrumentation();
343   InitLoadedGlobals();
344 
345   // Needs to be called here because flags()->random_tags might not have been
346   // initialized when InitInstrumentation() was called.
347   GetCurrentThread()->InitRandomState();
348 
349   SetPrintfAndReportCallback(AppendToErrorMessageBuffer);
350   // This may call libc -> needs initialized shadow.
351   AndroidLogInit();
352 
353   InitializeInterceptors();
354   InstallDeadlySignalHandlers(HwasanOnDeadlySignal);
355   InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
356 
357   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
358 
359   HwasanTSDInit();
360   HwasanTSDThreadInit();
361 
362   HwasanAllocatorInit();
363 
364 #if HWASAN_CONTAINS_UBSAN
365   __ubsan::InitAsPlugin();
366 #endif
367 
368   VPrintf(1, "HWAddressSanitizer init done\n");
369 
370   hwasan_init_is_running = 0;
371   hwasan_inited = 1;
372 }
373 
374 void __hwasan_library_loaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
375                              ElfW(Half) phnum) {
376   for (const hwasan_global &global : HwasanGlobalsFor(base, phdr, phnum))
377     InitializeSingleGlobal(global);
378 }
379 
380 void __hwasan_library_unloaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
381                                ElfW(Half) phnum) {
382   for (; phnum != 0; ++phdr, --phnum)
383     if (phdr->p_type == PT_LOAD)
384       TagMemory(base + phdr->p_vaddr, phdr->p_memsz, 0);
385 }
386 
387 void __hwasan_print_shadow(const void *p, uptr sz) {
388   uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p));
389   uptr shadow_first = MemToShadow(ptr_raw);
390   uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
391   Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw,
392          ptr_raw + sz, GetTagFromPointer((uptr)p));
393   for (uptr s = shadow_first; s <= shadow_last; ++s)
394     Printf("  %zx: %x\n", ShadowToMem(s), *(tag_t *)s);
395 }
396 
397 sptr __hwasan_test_shadow(const void *p, uptr sz) {
398   if (sz == 0)
399     return -1;
400   tag_t ptr_tag = GetTagFromPointer((uptr)p);
401   uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p));
402   uptr shadow_first = MemToShadow(ptr_raw);
403   uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
404   for (uptr s = shadow_first; s <= shadow_last; ++s)
405     if (*(tag_t *)s != ptr_tag) {
406       sptr offset = ShadowToMem(s) - ptr_raw;
407       return offset < 0 ? 0 : offset;
408     }
409   return -1;
410 }
411 
412 u16 __sanitizer_unaligned_load16(const uu16 *p) {
413   return *p;
414 }
415 u32 __sanitizer_unaligned_load32(const uu32 *p) {
416   return *p;
417 }
418 u64 __sanitizer_unaligned_load64(const uu64 *p) {
419   return *p;
420 }
421 void __sanitizer_unaligned_store16(uu16 *p, u16 x) {
422   *p = x;
423 }
424 void __sanitizer_unaligned_store32(uu32 *p, u32 x) {
425   *p = x;
426 }
427 void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
428   *p = x;
429 }
430 
431 void __hwasan_loadN(uptr p, uptr sz) {
432   CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz);
433 }
434 void __hwasan_load1(uptr p) {
435   CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p);
436 }
437 void __hwasan_load2(uptr p) {
438   CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p);
439 }
440 void __hwasan_load4(uptr p) {
441   CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p);
442 }
443 void __hwasan_load8(uptr p) {
444   CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p);
445 }
446 void __hwasan_load16(uptr p) {
447   CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p);
448 }
449 
450 void __hwasan_loadN_noabort(uptr p, uptr sz) {
451   CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz);
452 }
453 void __hwasan_load1_noabort(uptr p) {
454   CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p);
455 }
456 void __hwasan_load2_noabort(uptr p) {
457   CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p);
458 }
459 void __hwasan_load4_noabort(uptr p) {
460   CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p);
461 }
462 void __hwasan_load8_noabort(uptr p) {
463   CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p);
464 }
465 void __hwasan_load16_noabort(uptr p) {
466   CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p);
467 }
468 
469 void __hwasan_storeN(uptr p, uptr sz) {
470   CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz);
471 }
472 void __hwasan_store1(uptr p) {
473   CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p);
474 }
475 void __hwasan_store2(uptr p) {
476   CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p);
477 }
478 void __hwasan_store4(uptr p) {
479   CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p);
480 }
481 void __hwasan_store8(uptr p) {
482   CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p);
483 }
484 void __hwasan_store16(uptr p) {
485   CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p);
486 }
487 
488 void __hwasan_storeN_noabort(uptr p, uptr sz) {
489   CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz);
490 }
491 void __hwasan_store1_noabort(uptr p) {
492   CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p);
493 }
494 void __hwasan_store2_noabort(uptr p) {
495   CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p);
496 }
497 void __hwasan_store4_noabort(uptr p) {
498   CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p);
499 }
500 void __hwasan_store8_noabort(uptr p) {
501   CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p);
502 }
503 void __hwasan_store16_noabort(uptr p) {
504   CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p);
505 }
506 
507 void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) {
508   TagMemoryAligned(p, sz, tag);
509 }
510 
511 uptr __hwasan_tag_pointer(uptr p, u8 tag) {
512   return AddTagToPointer(p, tag);
513 }
514 
515 void __hwasan_handle_longjmp(const void *sp_dst) {
516   uptr dst = (uptr)sp_dst;
517   // HWASan does not support tagged SP.
518   CHECK(GetTagFromPointer(dst) == 0);
519 
520   uptr sp = (uptr)__builtin_frame_address(0);
521   static const uptr kMaxExpectedCleanupSize = 64 << 20;  // 64M
522   if (dst < sp || dst - sp > kMaxExpectedCleanupSize) {
523     Report(
524         "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: "
525         "stack top: %p; target %p; distance: %p (%zd)\n"
526         "False positive error reports may follow\n",
527         (void *)sp, (void *)dst, dst - sp);
528     return;
529   }
530   TagMemory(sp, dst - sp, 0);
531 }
532 
533 void __hwasan_handle_vfork(const void *sp_dst) {
534   uptr sp = (uptr)sp_dst;
535   Thread *t = GetCurrentThread();
536   CHECK(t);
537   uptr top = t->stack_top();
538   uptr bottom = t->stack_bottom();
539   if (top == 0 || bottom == 0 || sp < bottom || sp >= top) {
540     Report(
541         "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: "
542         "stack top: %zx; current %zx; bottom: %zx \n"
543         "False positive error reports may follow\n",
544         top, sp, bottom);
545     return;
546   }
547   TagMemory(bottom, sp - bottom, 0);
548 }
549 
550 extern "C" void *__hwasan_extra_spill_area() {
551   Thread *t = GetCurrentThread();
552   return &t->vfork_spill();
553 }
554 
555 void __hwasan_print_memory_usage() {
556   InternalScopedString s;
557   HwasanFormatMemoryUsage(s);
558   Printf("%s\n", s.data());
559 }
560 
561 static const u8 kFallbackTag = 0xBB & kTagMask;
562 
563 u8 __hwasan_generate_tag() {
564   Thread *t = GetCurrentThread();
565   if (!t) return kFallbackTag;
566   return t->GenerateRandomTag();
567 }
568 
569 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
570 extern "C" {
571 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
572 const char* __hwasan_default_options() { return ""; }
573 }  // extern "C"
574 #endif
575 
576 extern "C" {
577 SANITIZER_INTERFACE_ATTRIBUTE
578 void __sanitizer_print_stack_trace() {
579   GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
580   stack.Print();
581 }
582 
583 // Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
584 // rest of the mismatch handling code (C++).
585 void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
586                             size_t outsize) {
587   __hwasan::HwasanTagMismatch(addr, access_info, registers_frame, outsize);
588 }
589 
590 } // extern "C"
591