1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Main internal TSan header file. 12 // 13 // Ground rules: 14 // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static 15 // function-scope locals) 16 // - All functions/classes/etc reside in namespace __tsan, except for those 17 // declared in tsan_interface.h. 18 // - Platform-specific files should be used instead of ifdefs (*). 19 // - No system headers included in header files (*). 20 // - Platform specific headres included only into platform-specific files (*). 21 // 22 // (*) Except when inlining is critical for performance. 23 //===----------------------------------------------------------------------===// 24 25 #ifndef TSAN_RTL_H 26 #define TSAN_RTL_H 27 28 #include "sanitizer_common/sanitizer_allocator.h" 29 #include "sanitizer_common/sanitizer_allocator_internal.h" 30 #include "sanitizer_common/sanitizer_asm.h" 31 #include "sanitizer_common/sanitizer_common.h" 32 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h" 33 #include "sanitizer_common/sanitizer_libignore.h" 34 #include "sanitizer_common/sanitizer_suppressions.h" 35 #include "sanitizer_common/sanitizer_thread_registry.h" 36 #include "sanitizer_common/sanitizer_vector.h" 37 #include "tsan_defs.h" 38 #include "tsan_flags.h" 39 #include "tsan_ignoreset.h" 40 #include "tsan_ilist.h" 41 #include "tsan_mman.h" 42 #include "tsan_mutexset.h" 43 #include "tsan_platform.h" 44 #include "tsan_report.h" 45 #include "tsan_shadow.h" 46 #include "tsan_stack_trace.h" 47 #include "tsan_sync.h" 48 #include "tsan_trace.h" 49 #include "tsan_vector_clock.h" 50 51 #if SANITIZER_WORDSIZE != 64 52 # error "ThreadSanitizer is supported only on 64-bit platforms" 53 #endif 54 55 namespace __tsan { 56 57 #if !SANITIZER_GO 58 struct MapUnmapCallback; 59 # if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \ 60 defined(__powerpc__) || SANITIZER_RISCV64 61 62 struct AP32 { 63 static const uptr kSpaceBeg = 0; 64 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; 65 static const uptr kMetadataSize = 0; 66 typedef __sanitizer::CompactSizeClassMap SizeClassMap; 67 static const uptr kRegionSizeLog = 20; 68 using AddressSpaceView = LocalAddressSpaceView; 69 typedef __tsan::MapUnmapCallback MapUnmapCallback; 70 static const uptr kFlags = 0; 71 }; 72 typedef SizeClassAllocator32<AP32> PrimaryAllocator; 73 #else 74 struct AP64 { // Allocator64 parameters. Deliberately using a short name. 75 # if defined(__s390x__) 76 typedef MappingS390x Mapping; 77 # else 78 typedef Mapping48AddressSpace Mapping; 79 # endif 80 static const uptr kSpaceBeg = Mapping::kHeapMemBeg; 81 static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg; 82 static const uptr kMetadataSize = 0; 83 typedef DefaultSizeClassMap SizeClassMap; 84 typedef __tsan::MapUnmapCallback MapUnmapCallback; 85 static const uptr kFlags = 0; 86 using AddressSpaceView = LocalAddressSpaceView; 87 }; 88 typedef SizeClassAllocator64<AP64> PrimaryAllocator; 89 #endif 90 typedef CombinedAllocator<PrimaryAllocator> Allocator; 91 typedef Allocator::AllocatorCache AllocatorCache; 92 Allocator *allocator(); 93 #endif 94 95 struct ThreadSignalContext; 96 97 struct JmpBuf { 98 uptr sp; 99 int int_signal_send; 100 bool in_blocking_func; 101 uptr in_signal_handler; 102 uptr *shadow_stack_pos; 103 }; 104 105 // A Processor represents a physical thread, or a P for Go. 106 // It is used to store internal resources like allocate cache, and does not 107 // participate in race-detection logic (invisible to end user). 108 // In C++ it is tied to an OS thread just like ThreadState, however ideally 109 // it should be tied to a CPU (this way we will have fewer allocator caches). 110 // In Go it is tied to a P, so there are significantly fewer Processor's than 111 // ThreadState's (which are tied to Gs). 112 // A ThreadState must be wired with a Processor to handle events. 113 struct Processor { 114 ThreadState *thr; // currently wired thread, or nullptr 115 #if !SANITIZER_GO 116 AllocatorCache alloc_cache; 117 InternalAllocatorCache internal_alloc_cache; 118 #endif 119 DenseSlabAllocCache block_cache; 120 DenseSlabAllocCache sync_cache; 121 DDPhysicalThread *dd_pt; 122 }; 123 124 #if !SANITIZER_GO 125 // ScopedGlobalProcessor temporary setups a global processor for the current 126 // thread, if it does not have one. Intended for interceptors that can run 127 // at the very thread end, when we already destroyed the thread processor. 128 struct ScopedGlobalProcessor { 129 ScopedGlobalProcessor(); 130 ~ScopedGlobalProcessor(); 131 }; 132 #endif 133 134 struct TidEpoch { 135 Tid tid; 136 Epoch epoch; 137 }; 138 139 struct TidSlot { 140 Mutex mtx; 141 Sid sid; 142 atomic_uint32_t raw_epoch; 143 ThreadState *thr; 144 Vector<TidEpoch> journal; 145 INode node; 146 147 Epoch epoch() const { 148 return static_cast<Epoch>(atomic_load(&raw_epoch, memory_order_relaxed)); 149 } 150 151 void SetEpoch(Epoch v) { 152 atomic_store(&raw_epoch, static_cast<u32>(v), memory_order_relaxed); 153 } 154 155 TidSlot(); 156 } ALIGNED(SANITIZER_CACHE_LINE_SIZE); 157 158 // This struct is stored in TLS. 159 struct ThreadState { 160 FastState fast_state; 161 int ignore_sync; 162 #if !SANITIZER_GO 163 int ignore_interceptors; 164 #endif 165 uptr *shadow_stack_pos; 166 167 // Current position in tctx->trace.Back()->events (Event*). 168 atomic_uintptr_t trace_pos; 169 // PC of the last memory access, used to compute PC deltas in the trace. 170 uptr trace_prev_pc; 171 172 // Technically `current` should be a separate THREADLOCAL variable; 173 // but it is placed here in order to share cache line with previous fields. 174 ThreadState* current; 175 176 atomic_sint32_t pending_signals; 177 178 VectorClock clock; 179 180 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. 181 // We do not distinguish beteween ignoring reads and writes 182 // for better performance. 183 int ignore_reads_and_writes; 184 int suppress_reports; 185 // Go does not support ignores. 186 #if !SANITIZER_GO 187 IgnoreSet mop_ignore_set; 188 IgnoreSet sync_ignore_set; 189 #endif 190 uptr *shadow_stack; 191 uptr *shadow_stack_end; 192 #if !SANITIZER_GO 193 Vector<JmpBuf> jmp_bufs; 194 int in_symbolizer; 195 atomic_uintptr_t in_blocking_func; 196 bool in_ignored_lib; 197 bool is_inited; 198 #endif 199 MutexSet mset; 200 bool is_dead; 201 const Tid tid; 202 uptr stk_addr; 203 uptr stk_size; 204 uptr tls_addr; 205 uptr tls_size; 206 ThreadContext *tctx; 207 208 DDLogicalThread *dd_lt; 209 210 TidSlot *slot; 211 uptr slot_epoch; 212 bool slot_locked; 213 214 // Current wired Processor, or nullptr. Required to handle any events. 215 Processor *proc1; 216 #if !SANITIZER_GO 217 Processor *proc() { return proc1; } 218 #else 219 Processor *proc(); 220 #endif 221 222 atomic_uintptr_t in_signal_handler; 223 atomic_uintptr_t signal_ctx; 224 225 #if !SANITIZER_GO 226 StackID last_sleep_stack_id; 227 VectorClock last_sleep_clock; 228 #endif 229 230 // Set in regions of runtime that must be signal-safe and fork-safe. 231 // If set, malloc must not be called. 232 int nomalloc; 233 234 const ReportDesc *current_report; 235 236 explicit ThreadState(Tid tid); 237 } ALIGNED(SANITIZER_CACHE_LINE_SIZE); 238 239 #if !SANITIZER_GO 240 #if SANITIZER_APPLE || SANITIZER_ANDROID 241 ThreadState *cur_thread(); 242 void set_cur_thread(ThreadState *thr); 243 void cur_thread_finalize(); 244 inline ThreadState *cur_thread_init() { return cur_thread(); } 245 # else 246 __attribute__((tls_model("initial-exec"))) 247 extern THREADLOCAL char cur_thread_placeholder[]; 248 inline ThreadState *cur_thread() { 249 return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current; 250 } 251 inline ThreadState *cur_thread_init() { 252 ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder); 253 if (UNLIKELY(!thr->current)) 254 thr->current = thr; 255 return thr->current; 256 } 257 inline void set_cur_thread(ThreadState *thr) { 258 reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr; 259 } 260 inline void cur_thread_finalize() { } 261 # endif // SANITIZER_APPLE || SANITIZER_ANDROID 262 #endif // SANITIZER_GO 263 264 class ThreadContext final : public ThreadContextBase { 265 public: 266 explicit ThreadContext(Tid tid); 267 ~ThreadContext(); 268 ThreadState *thr; 269 StackID creation_stack_id; 270 VectorClock *sync; 271 uptr sync_epoch; 272 Trace trace; 273 274 // Override superclass callbacks. 275 void OnDead() override; 276 void OnJoined(void *arg) override; 277 void OnFinished() override; 278 void OnStarted(void *arg) override; 279 void OnCreated(void *arg) override; 280 void OnReset() override; 281 void OnDetached(void *arg) override; 282 }; 283 284 struct RacyStacks { 285 MD5Hash hash[2]; 286 bool operator==(const RacyStacks &other) const; 287 }; 288 289 struct RacyAddress { 290 uptr addr_min; 291 uptr addr_max; 292 }; 293 294 struct FiredSuppression { 295 ReportType type; 296 uptr pc_or_addr; 297 Suppression *supp; 298 }; 299 300 struct Context { 301 Context(); 302 303 bool initialized; 304 #if !SANITIZER_GO 305 bool after_multithreaded_fork; 306 #endif 307 308 MetaMap metamap; 309 310 Mutex report_mtx; 311 int nreported; 312 atomic_uint64_t last_symbolize_time_ns; 313 314 void *background_thread; 315 atomic_uint32_t stop_background_thread; 316 317 ThreadRegistry thread_registry; 318 319 // This is used to prevent a very unlikely but very pathological behavior. 320 // Since memory access handling is not synchronized with DoReset, 321 // a thread running concurrently with DoReset can leave a bogus shadow value 322 // that will be later falsely detected as a race. For such false races 323 // RestoreStack will return false and we will not report it. 324 // However, consider that a thread leaves a whole lot of such bogus values 325 // and these values are later read by a whole lot of threads. 326 // This will cause massive amounts of ReportRace calls and lots of 327 // serialization. In very pathological cases the resulting slowdown 328 // can be >100x. This is very unlikely, but it was presumably observed 329 // in practice: https://github.com/google/sanitizers/issues/1552 330 // If this happens, previous access sid+epoch will be the same for all of 331 // these false races b/c if the thread will try to increment epoch, it will 332 // notice that DoReset has happened and will stop producing bogus shadow 333 // values. So, last_spurious_race is used to remember the last sid+epoch 334 // for which RestoreStack returned false. Then it is used to filter out 335 // races with the same sid+epoch very early and quickly. 336 // It is of course possible that multiple threads left multiple bogus shadow 337 // values and all of them are read by lots of threads at the same time. 338 // In such case last_spurious_race will only be able to deduplicate a few 339 // races from one thread, then few from another and so on. An alternative 340 // would be to hold an array of such sid+epoch, but we consider such scenario 341 // as even less likely. 342 // Note: this can lead to some rare false negatives as well: 343 // 1. When a legit access with the same sid+epoch participates in a race 344 // as the "previous" memory access, it will be wrongly filtered out. 345 // 2. When RestoreStack returns false for a legit memory access because it 346 // was already evicted from the thread trace, we will still remember it in 347 // last_spurious_race. Then if there is another racing memory access from 348 // the same thread that happened in the same epoch, but was stored in the 349 // next thread trace part (which is still preserved in the thread trace), 350 // we will also wrongly filter it out while RestoreStack would actually 351 // succeed for that second memory access. 352 RawShadow last_spurious_race; 353 354 Mutex racy_mtx; 355 Vector<RacyStacks> racy_stacks; 356 // Number of fired suppressions may be large enough. 357 Mutex fired_suppressions_mtx; 358 InternalMmapVector<FiredSuppression> fired_suppressions; 359 DDetector *dd; 360 361 Flags flags; 362 fd_t memprof_fd; 363 364 // The last slot index (kFreeSid) is used to denote freed memory. 365 TidSlot slots[kThreadSlotCount - 1]; 366 367 // Protects global_epoch, slot_queue, trace_part_recycle. 368 Mutex slot_mtx; 369 uptr global_epoch; // guarded by slot_mtx and by all slot mutexes 370 bool resetting; // global reset is in progress 371 IList<TidSlot, &TidSlot::node> slot_queue SANITIZER_GUARDED_BY(slot_mtx); 372 IList<TraceHeader, &TraceHeader::global, TracePart> trace_part_recycle 373 SANITIZER_GUARDED_BY(slot_mtx); 374 uptr trace_part_total_allocated SANITIZER_GUARDED_BY(slot_mtx); 375 uptr trace_part_recycle_finished SANITIZER_GUARDED_BY(slot_mtx); 376 uptr trace_part_finished_excess SANITIZER_GUARDED_BY(slot_mtx); 377 #if SANITIZER_GO 378 uptr mapped_shadow_begin; 379 uptr mapped_shadow_end; 380 #endif 381 }; 382 383 extern Context *ctx; // The one and the only global runtime context. 384 385 ALWAYS_INLINE Flags *flags() { 386 return &ctx->flags; 387 } 388 389 struct ScopedIgnoreInterceptors { 390 ScopedIgnoreInterceptors() { 391 #if !SANITIZER_GO 392 cur_thread()->ignore_interceptors++; 393 #endif 394 } 395 396 ~ScopedIgnoreInterceptors() { 397 #if !SANITIZER_GO 398 cur_thread()->ignore_interceptors--; 399 #endif 400 } 401 }; 402 403 const char *GetObjectTypeFromTag(uptr tag); 404 const char *GetReportHeaderFromTag(uptr tag); 405 uptr TagFromShadowStackFrame(uptr pc); 406 407 class ScopedReportBase { 408 public: 409 void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid, 410 StackTrace stack, const MutexSet *mset); 411 void AddStack(StackTrace stack, bool suppressable = false); 412 void AddThread(const ThreadContext *tctx, bool suppressable = false); 413 void AddThread(Tid tid, bool suppressable = false); 414 void AddUniqueTid(Tid unique_tid); 415 int AddMutex(uptr addr, StackID creation_stack_id); 416 void AddLocation(uptr addr, uptr size); 417 void AddSleep(StackID stack_id); 418 void SetCount(int count); 419 void SetSigNum(int sig); 420 421 const ReportDesc *GetReport() const; 422 423 protected: 424 ScopedReportBase(ReportType typ, uptr tag); 425 ~ScopedReportBase(); 426 427 private: 428 ReportDesc *rep_; 429 // Symbolizer makes lots of intercepted calls. If we try to process them, 430 // at best it will cause deadlocks on internal mutexes. 431 ScopedIgnoreInterceptors ignore_interceptors_; 432 433 ScopedReportBase(const ScopedReportBase &) = delete; 434 void operator=(const ScopedReportBase &) = delete; 435 }; 436 437 class ScopedReport : public ScopedReportBase { 438 public: 439 explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone); 440 ~ScopedReport(); 441 442 private: 443 ScopedErrorReportLock lock_; 444 }; 445 446 bool ShouldReport(ThreadState *thr, ReportType typ); 447 ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack); 448 449 // The stack could look like: 450 // <start> | <main> | <foo> | tag | <bar> 451 // This will extract the tag and keep: 452 // <start> | <main> | <foo> | <bar> 453 template<typename StackTraceTy> 454 void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) { 455 if (stack->size < 2) return; 456 uptr possible_tag_pc = stack->trace[stack->size - 2]; 457 uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc); 458 if (possible_tag == kExternalTagNone) return; 459 stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1]; 460 stack->size -= 1; 461 if (tag) *tag = possible_tag; 462 } 463 464 template<typename StackTraceTy> 465 void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack, 466 uptr *tag = nullptr) { 467 uptr size = thr->shadow_stack_pos - thr->shadow_stack; 468 uptr start = 0; 469 if (size + !!toppc > kStackTraceMax) { 470 start = size + !!toppc - kStackTraceMax; 471 size = kStackTraceMax - !!toppc; 472 } 473 stack->Init(&thr->shadow_stack[start], size, toppc); 474 ExtractTagFromStack(stack, tag); 475 } 476 477 #define GET_STACK_TRACE_FATAL(thr, pc) \ 478 VarSizeStackTrace stack; \ 479 ObtainCurrentStack(thr, pc, &stack); \ 480 stack.ReverseOrder(); 481 482 void MapShadow(uptr addr, uptr size); 483 void MapThreadTrace(uptr addr, uptr size, const char *name); 484 void DontNeedShadowFor(uptr addr, uptr size); 485 void UnmapShadow(ThreadState *thr, uptr addr, uptr size); 486 void InitializeShadowMemory(); 487 void DontDumpShadow(uptr addr, uptr size); 488 void InitializeInterceptors(); 489 void InitializeLibIgnore(); 490 void InitializeDynamicAnnotations(); 491 492 void ForkBefore(ThreadState *thr, uptr pc); 493 void ForkParentAfter(ThreadState *thr, uptr pc); 494 void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread); 495 496 void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old, 497 AccessType typ); 498 bool OutputReport(ThreadState *thr, const ScopedReport &srep); 499 bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace); 500 bool IsExpectedReport(uptr addr, uptr size); 501 502 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1 503 # define DPrintf Printf 504 #else 505 # define DPrintf(...) 506 #endif 507 508 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2 509 # define DPrintf2 Printf 510 #else 511 # define DPrintf2(...) 512 #endif 513 514 StackID CurrentStackId(ThreadState *thr, uptr pc); 515 ReportStack *SymbolizeStackId(StackID stack_id); 516 void PrintCurrentStack(ThreadState *thr, uptr pc); 517 void PrintCurrentStackSlow(uptr pc); // uses libunwind 518 MBlock *JavaHeapBlock(uptr addr, uptr *start); 519 520 void Initialize(ThreadState *thr); 521 void MaybeSpawnBackgroundThread(); 522 int Finalize(ThreadState *thr); 523 524 void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write); 525 void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write); 526 527 void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, 528 AccessType typ); 529 void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, 530 AccessType typ); 531 // This creates 2 non-inlined specialized versions of MemoryAccessRange. 532 template <bool is_read> 533 void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size); 534 535 ALWAYS_INLINE 536 void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, 537 bool is_write) { 538 if (size == 0) 539 return; 540 if (is_write) 541 MemoryAccessRangeT<false>(thr, pc, addr, size); 542 else 543 MemoryAccessRangeT<true>(thr, pc, addr, size); 544 } 545 546 void ShadowSet(RawShadow *p, RawShadow *end, RawShadow v); 547 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size); 548 void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); 549 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size); 550 void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr, 551 uptr size); 552 553 void ThreadIgnoreBegin(ThreadState *thr, uptr pc); 554 void ThreadIgnoreEnd(ThreadState *thr); 555 void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc); 556 void ThreadIgnoreSyncEnd(ThreadState *thr); 557 558 Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); 559 void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, 560 ThreadType thread_type); 561 void ThreadFinish(ThreadState *thr); 562 Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid); 563 void ThreadJoin(ThreadState *thr, uptr pc, Tid tid); 564 void ThreadDetach(ThreadState *thr, uptr pc, Tid tid); 565 void ThreadFinalize(ThreadState *thr); 566 void ThreadSetName(ThreadState *thr, const char *name); 567 int ThreadCount(ThreadState *thr); 568 void ProcessPendingSignalsImpl(ThreadState *thr); 569 void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid); 570 571 Processor *ProcCreate(); 572 void ProcDestroy(Processor *proc); 573 void ProcWire(Processor *proc, ThreadState *thr); 574 void ProcUnwire(Processor *proc, ThreadState *thr); 575 576 // Note: the parameter is called flagz, because flags is already taken 577 // by the global function that returns flags. 578 void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 579 void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 580 void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 581 void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0, 582 int rec = 1); 583 int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 584 void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 585 void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 586 void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr); 587 void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr); 588 void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD 589 void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr); 590 591 void Acquire(ThreadState *thr, uptr pc, uptr addr); 592 // AcquireGlobal synchronizes the current thread with all other threads. 593 // In terms of happens-before relation, it draws a HB edge from all threads 594 // (where they happen to execute right now) to the current thread. We use it to 595 // handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal 596 // right before executing finalizers. This provides a coarse, but simple 597 // approximation of the actual required synchronization. 598 void AcquireGlobal(ThreadState *thr); 599 void Release(ThreadState *thr, uptr pc, uptr addr); 600 void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr); 601 void ReleaseStore(ThreadState *thr, uptr pc, uptr addr); 602 void AfterSleep(ThreadState *thr, uptr pc); 603 void IncrementEpoch(ThreadState *thr); 604 605 #if !SANITIZER_GO 606 uptr ALWAYS_INLINE HeapEnd() { 607 return HeapMemEnd() + PrimaryAllocator::AdditionalSize(); 608 } 609 #endif 610 611 void SlotAttachAndLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx); 612 void SlotDetach(ThreadState *thr); 613 void SlotLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx); 614 void SlotUnlock(ThreadState *thr) SANITIZER_RELEASE(thr->slot->mtx); 615 void DoReset(ThreadState *thr, uptr epoch); 616 void FlushShadowMemory(); 617 618 ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags); 619 void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber); 620 void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags); 621 622 // These need to match __tsan_switch_to_fiber_* flags defined in 623 // tsan_interface.h. See documentation there as well. 624 enum FiberSwitchFlags { 625 FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync 626 }; 627 628 class SlotLocker { 629 public: 630 ALWAYS_INLINE 631 SlotLocker(ThreadState *thr, bool recursive = false) 632 : thr_(thr), locked_(recursive ? thr->slot_locked : false) { 633 #if !SANITIZER_GO 634 // We are in trouble if we are here with in_blocking_func set. 635 // If in_blocking_func is set, all signals will be delivered synchronously, 636 // which means we can't lock slots since the signal handler will try 637 // to lock it recursively and deadlock. 638 DCHECK(!atomic_load(&thr->in_blocking_func, memory_order_relaxed)); 639 #endif 640 if (!locked_) 641 SlotLock(thr_); 642 } 643 644 ALWAYS_INLINE 645 ~SlotLocker() { 646 if (!locked_) 647 SlotUnlock(thr_); 648 } 649 650 private: 651 ThreadState *thr_; 652 bool locked_; 653 }; 654 655 class SlotUnlocker { 656 public: 657 SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) { 658 if (locked_) 659 SlotUnlock(thr_); 660 } 661 662 ~SlotUnlocker() { 663 if (locked_) 664 SlotLock(thr_); 665 } 666 667 private: 668 ThreadState *thr_; 669 bool locked_; 670 }; 671 672 ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) { 673 if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals))) 674 ProcessPendingSignalsImpl(thr); 675 } 676 677 extern bool is_initialized; 678 679 ALWAYS_INLINE 680 void LazyInitialize(ThreadState *thr) { 681 // If we can use .preinit_array, assume that __tsan_init 682 // called from .preinit_array initializes runtime before 683 // any instrumented code except when tsan is used as a 684 // shared library. 685 #if (!SANITIZER_CAN_USE_PREINIT_ARRAY || defined(SANITIZER_SHARED)) 686 if (UNLIKELY(!is_initialized)) 687 Initialize(thr); 688 #endif 689 } 690 691 void TraceResetForTesting(); 692 void TraceSwitchPart(ThreadState *thr); 693 void TraceSwitchPartImpl(ThreadState *thr); 694 bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size, 695 AccessType typ, Tid *ptid, VarSizeStackTrace *pstk, 696 MutexSet *pmset, uptr *ptag); 697 698 template <typename EventT> 699 ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr, 700 EventT **ev) { 701 // TraceSwitchPart accesses shadow_stack, but it's called infrequently, 702 // so we check it here proactively. 703 DCHECK(thr->shadow_stack); 704 Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos)); 705 #if SANITIZER_DEBUG 706 // TraceSwitch acquires these mutexes, 707 // so we lock them here to detect deadlocks more reliably. 708 { Lock lock(&ctx->slot_mtx); } 709 { Lock lock(&thr->tctx->trace.mtx); } 710 TracePart *current = thr->tctx->trace.parts.Back(); 711 if (current) { 712 DCHECK_GE(pos, ¤t->events[0]); 713 DCHECK_LE(pos, ¤t->events[TracePart::kSize]); 714 } else { 715 DCHECK_EQ(pos, nullptr); 716 } 717 #endif 718 // TracePart is allocated with mmap and is at least 4K aligned. 719 // So the following check is a faster way to check for part end. 720 // It may have false positives in the middle of the trace, 721 // they are filtered out in TraceSwitch. 722 if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0)) 723 return false; 724 *ev = reinterpret_cast<EventT *>(pos); 725 return true; 726 } 727 728 template <typename EventT> 729 ALWAYS_INLINE void TraceRelease(ThreadState *thr, EventT *evp) { 730 DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]); 731 atomic_store_relaxed(&thr->trace_pos, (uptr)(evp + 1)); 732 } 733 734 template <typename EventT> 735 void TraceEvent(ThreadState *thr, EventT ev) { 736 EventT *evp; 737 if (!TraceAcquire(thr, &evp)) { 738 TraceSwitchPart(thr); 739 UNUSED bool res = TraceAcquire(thr, &evp); 740 DCHECK(res); 741 } 742 *evp = ev; 743 TraceRelease(thr, evp); 744 } 745 746 ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr, 747 uptr pc = 0) { 748 if (!kCollectHistory) 749 return true; 750 EventFunc *ev; 751 if (UNLIKELY(!TraceAcquire(thr, &ev))) 752 return false; 753 ev->is_access = 0; 754 ev->is_func = 1; 755 ev->pc = pc; 756 TraceRelease(thr, ev); 757 return true; 758 } 759 760 WARN_UNUSED_RESULT 761 bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, 762 AccessType typ); 763 WARN_UNUSED_RESULT 764 bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, 765 AccessType typ); 766 void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, 767 AccessType typ); 768 void TraceFunc(ThreadState *thr, uptr pc = 0); 769 void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr, 770 StackID stk); 771 void TraceMutexUnlock(ThreadState *thr, uptr addr); 772 void TraceTime(ThreadState *thr); 773 774 void TraceRestartFuncExit(ThreadState *thr); 775 void TraceRestartFuncEntry(ThreadState *thr, uptr pc); 776 777 void GrowShadowStack(ThreadState *thr); 778 779 ALWAYS_INLINE 780 void FuncEntry(ThreadState *thr, uptr pc) { 781 DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc); 782 if (UNLIKELY(!TryTraceFunc(thr, pc))) 783 return TraceRestartFuncEntry(thr, pc); 784 DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack); 785 #if !SANITIZER_GO 786 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); 787 #else 788 if (thr->shadow_stack_pos == thr->shadow_stack_end) 789 GrowShadowStack(thr); 790 #endif 791 thr->shadow_stack_pos[0] = pc; 792 thr->shadow_stack_pos++; 793 } 794 795 ALWAYS_INLINE 796 void FuncExit(ThreadState *thr) { 797 DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid()); 798 if (UNLIKELY(!TryTraceFunc(thr, 0))) 799 return TraceRestartFuncExit(thr); 800 DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack); 801 #if !SANITIZER_GO 802 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); 803 #endif 804 thr->shadow_stack_pos--; 805 } 806 807 #if !SANITIZER_GO 808 extern void (*on_initialize)(void); 809 extern int (*on_finalize)(int); 810 #endif 811 } // namespace __tsan 812 813 #endif // TSAN_RTL_H 814