1 //===-- ThreadSanitizer.cpp - race detector -------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer, a race detector. 10 // 11 // The tool is under development, for the details about previous versions see 12 // http://code.google.com/p/data-race-test 13 // 14 // The instrumentation phase is quite simple: 15 // - Insert calls to run-time library before every memory access. 16 // - Optimizations may apply to avoid instrumenting some of the accesses. 17 // - Insert calls at function entry/exit. 18 // The rest is handled by the run-time library. 19 //===----------------------------------------------------------------------===// 20 21 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" 22 #include "llvm/ADT/SmallPtrSet.h" 23 #include "llvm/ADT/SmallString.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/Statistic.h" 26 #include "llvm/ADT/StringExtras.h" 27 #include "llvm/Analysis/CaptureTracking.h" 28 #include "llvm/Analysis/TargetLibraryInfo.h" 29 #include "llvm/Transforms/Utils/Local.h" 30 #include "llvm/Analysis/ValueTracking.h" 31 #include "llvm/IR/DataLayout.h" 32 #include "llvm/IR/Function.h" 33 #include "llvm/IR/IRBuilder.h" 34 #include "llvm/IR/IntrinsicInst.h" 35 #include "llvm/IR/Intrinsics.h" 36 #include "llvm/IR/LLVMContext.h" 37 #include "llvm/IR/Metadata.h" 38 #include "llvm/IR/Module.h" 39 #include "llvm/IR/Type.h" 40 #include "llvm/ProfileData/InstrProf.h" 41 #include "llvm/Support/CommandLine.h" 42 #include "llvm/Support/Debug.h" 43 #include "llvm/Support/MathExtras.h" 44 #include "llvm/Support/raw_ostream.h" 45 #include "llvm/Transforms/Instrumentation.h" 46 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 47 #include "llvm/Transforms/Utils/EscapeEnumerator.h" 48 #include "llvm/Transforms/Utils/ModuleUtils.h" 49 50 using namespace llvm; 51 52 #define DEBUG_TYPE "tsan" 53 54 static cl::opt<bool> ClInstrumentMemoryAccesses( 55 "tsan-instrument-memory-accesses", cl::init(true), 56 cl::desc("Instrument memory accesses"), cl::Hidden); 57 static cl::opt<bool> ClInstrumentFuncEntryExit( 58 "tsan-instrument-func-entry-exit", cl::init(true), 59 cl::desc("Instrument function entry and exit"), cl::Hidden); 60 static cl::opt<bool> ClHandleCxxExceptions( 61 "tsan-handle-cxx-exceptions", cl::init(true), 62 cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"), 63 cl::Hidden); 64 static cl::opt<bool> ClInstrumentAtomics( 65 "tsan-instrument-atomics", cl::init(true), 66 cl::desc("Instrument atomics"), cl::Hidden); 67 static cl::opt<bool> ClInstrumentMemIntrinsics( 68 "tsan-instrument-memintrinsics", cl::init(true), 69 cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); 70 71 STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); 72 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); 73 STATISTIC(NumOmittedReadsBeforeWrite, 74 "Number of reads ignored due to following writes"); 75 STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); 76 STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); 77 STATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads"); 78 STATISTIC(NumOmittedReadsFromConstantGlobals, 79 "Number of reads from constant globals"); 80 STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); 81 STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing"); 82 83 static const char *const kTsanModuleCtorName = "tsan.module_ctor"; 84 static const char *const kTsanInitName = "__tsan_init"; 85 86 namespace { 87 88 /// ThreadSanitizer: instrument the code in module to find races. 89 /// 90 /// Instantiating ThreadSanitizer inserts the tsan runtime library API function 91 /// declarations into the module if they don't exist already. Instantiating 92 /// ensures the __tsan_init function is in the list of global constructors for 93 /// the module. 94 struct ThreadSanitizer { 95 ThreadSanitizer(Module &M); 96 bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI); 97 98 private: 99 void initializeCallbacks(Module &M); 100 bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); 101 bool instrumentAtomic(Instruction *I, const DataLayout &DL); 102 bool instrumentMemIntrinsic(Instruction *I); 103 void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local, 104 SmallVectorImpl<Instruction *> &All, 105 const DataLayout &DL); 106 bool addrPointsToConstantData(Value *Addr); 107 int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL); 108 void InsertRuntimeIgnores(Function &F); 109 110 Type *IntptrTy; 111 IntegerType *OrdTy; 112 // Callbacks to run-time library are computed in doInitialization. 113 FunctionCallee TsanFuncEntry; 114 FunctionCallee TsanFuncExit; 115 FunctionCallee TsanIgnoreBegin; 116 FunctionCallee TsanIgnoreEnd; 117 // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 118 static const size_t kNumberOfAccessSizes = 5; 119 FunctionCallee TsanRead[kNumberOfAccessSizes]; 120 FunctionCallee TsanWrite[kNumberOfAccessSizes]; 121 FunctionCallee TsanUnalignedRead[kNumberOfAccessSizes]; 122 FunctionCallee TsanUnalignedWrite[kNumberOfAccessSizes]; 123 FunctionCallee TsanAtomicLoad[kNumberOfAccessSizes]; 124 FunctionCallee TsanAtomicStore[kNumberOfAccessSizes]; 125 FunctionCallee TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1] 126 [kNumberOfAccessSizes]; 127 FunctionCallee TsanAtomicCAS[kNumberOfAccessSizes]; 128 FunctionCallee TsanAtomicThreadFence; 129 FunctionCallee TsanAtomicSignalFence; 130 FunctionCallee TsanVptrUpdate; 131 FunctionCallee TsanVptrLoad; 132 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn; 133 Function *TsanCtorFunction; 134 }; 135 136 struct ThreadSanitizerLegacyPass : FunctionPass { 137 ThreadSanitizerLegacyPass() : FunctionPass(ID) {} 138 StringRef getPassName() const override; 139 void getAnalysisUsage(AnalysisUsage &AU) const override; 140 bool runOnFunction(Function &F) override; 141 bool doInitialization(Module &M) override; 142 static char ID; // Pass identification, replacement for typeid. 143 private: 144 Optional<ThreadSanitizer> TSan; 145 }; 146 } // namespace 147 148 PreservedAnalyses ThreadSanitizerPass::run(Function &F, 149 FunctionAnalysisManager &FAM) { 150 ThreadSanitizer TSan(*F.getParent()); 151 if (TSan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F))) 152 return PreservedAnalyses::none(); 153 return PreservedAnalyses::all(); 154 } 155 156 char ThreadSanitizerLegacyPass::ID = 0; 157 INITIALIZE_PASS_BEGIN(ThreadSanitizerLegacyPass, "tsan", 158 "ThreadSanitizer: detects data races.", false, false) 159 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 160 INITIALIZE_PASS_END(ThreadSanitizerLegacyPass, "tsan", 161 "ThreadSanitizer: detects data races.", false, false) 162 163 StringRef ThreadSanitizerLegacyPass::getPassName() const { 164 return "ThreadSanitizerLegacyPass"; 165 } 166 167 void ThreadSanitizerLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const { 168 AU.addRequired<TargetLibraryInfoWrapperPass>(); 169 } 170 171 bool ThreadSanitizerLegacyPass::doInitialization(Module &M) { 172 TSan.emplace(M); 173 return true; 174 } 175 176 bool ThreadSanitizerLegacyPass::runOnFunction(Function &F) { 177 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(); 178 TSan->sanitizeFunction(F, TLI); 179 return true; 180 } 181 182 FunctionPass *llvm::createThreadSanitizerLegacyPassPass() { 183 return new ThreadSanitizerLegacyPass(); 184 } 185 186 void ThreadSanitizer::initializeCallbacks(Module &M) { 187 IRBuilder<> IRB(M.getContext()); 188 AttributeList Attr; 189 Attr = Attr.addAttribute(M.getContext(), AttributeList::FunctionIndex, 190 Attribute::NoUnwind); 191 // Initialize the callbacks. 192 TsanFuncEntry = M.getOrInsertFunction("__tsan_func_entry", Attr, 193 IRB.getVoidTy(), IRB.getInt8PtrTy()); 194 TsanFuncExit = 195 M.getOrInsertFunction("__tsan_func_exit", Attr, IRB.getVoidTy()); 196 TsanIgnoreBegin = M.getOrInsertFunction("__tsan_ignore_thread_begin", Attr, 197 IRB.getVoidTy()); 198 TsanIgnoreEnd = 199 M.getOrInsertFunction("__tsan_ignore_thread_end", Attr, IRB.getVoidTy()); 200 OrdTy = IRB.getInt32Ty(); 201 for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { 202 const unsigned ByteSize = 1U << i; 203 const unsigned BitSize = ByteSize * 8; 204 std::string ByteSizeStr = utostr(ByteSize); 205 std::string BitSizeStr = utostr(BitSize); 206 SmallString<32> ReadName("__tsan_read" + ByteSizeStr); 207 TsanRead[i] = M.getOrInsertFunction(ReadName, Attr, IRB.getVoidTy(), 208 IRB.getInt8PtrTy()); 209 210 SmallString<32> WriteName("__tsan_write" + ByteSizeStr); 211 TsanWrite[i] = M.getOrInsertFunction(WriteName, Attr, IRB.getVoidTy(), 212 IRB.getInt8PtrTy()); 213 214 SmallString<64> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr); 215 TsanUnalignedRead[i] = M.getOrInsertFunction( 216 UnalignedReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()); 217 218 SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + ByteSizeStr); 219 TsanUnalignedWrite[i] = M.getOrInsertFunction( 220 UnalignedWriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()); 221 222 Type *Ty = Type::getIntNTy(M.getContext(), BitSize); 223 Type *PtrTy = Ty->getPointerTo(); 224 SmallString<32> AtomicLoadName("__tsan_atomic" + BitSizeStr + "_load"); 225 TsanAtomicLoad[i] = 226 M.getOrInsertFunction(AtomicLoadName, Attr, Ty, PtrTy, OrdTy); 227 228 SmallString<32> AtomicStoreName("__tsan_atomic" + BitSizeStr + "_store"); 229 TsanAtomicStore[i] = M.getOrInsertFunction( 230 AtomicStoreName, Attr, IRB.getVoidTy(), PtrTy, Ty, OrdTy); 231 232 for (int op = AtomicRMWInst::FIRST_BINOP; 233 op <= AtomicRMWInst::LAST_BINOP; ++op) { 234 TsanAtomicRMW[op][i] = nullptr; 235 const char *NamePart = nullptr; 236 if (op == AtomicRMWInst::Xchg) 237 NamePart = "_exchange"; 238 else if (op == AtomicRMWInst::Add) 239 NamePart = "_fetch_add"; 240 else if (op == AtomicRMWInst::Sub) 241 NamePart = "_fetch_sub"; 242 else if (op == AtomicRMWInst::And) 243 NamePart = "_fetch_and"; 244 else if (op == AtomicRMWInst::Or) 245 NamePart = "_fetch_or"; 246 else if (op == AtomicRMWInst::Xor) 247 NamePart = "_fetch_xor"; 248 else if (op == AtomicRMWInst::Nand) 249 NamePart = "_fetch_nand"; 250 else 251 continue; 252 SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); 253 TsanAtomicRMW[op][i] = 254 M.getOrInsertFunction(RMWName, Attr, Ty, PtrTy, Ty, OrdTy); 255 } 256 257 SmallString<32> AtomicCASName("__tsan_atomic" + BitSizeStr + 258 "_compare_exchange_val"); 259 TsanAtomicCAS[i] = M.getOrInsertFunction(AtomicCASName, Attr, Ty, PtrTy, Ty, 260 Ty, OrdTy, OrdTy); 261 } 262 TsanVptrUpdate = 263 M.getOrInsertFunction("__tsan_vptr_update", Attr, IRB.getVoidTy(), 264 IRB.getInt8PtrTy(), IRB.getInt8PtrTy()); 265 TsanVptrLoad = M.getOrInsertFunction("__tsan_vptr_read", Attr, 266 IRB.getVoidTy(), IRB.getInt8PtrTy()); 267 TsanAtomicThreadFence = M.getOrInsertFunction("__tsan_atomic_thread_fence", 268 Attr, IRB.getVoidTy(), OrdTy); 269 TsanAtomicSignalFence = M.getOrInsertFunction("__tsan_atomic_signal_fence", 270 Attr, IRB.getVoidTy(), OrdTy); 271 272 MemmoveFn = 273 M.getOrInsertFunction("memmove", Attr, IRB.getInt8PtrTy(), 274 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); 275 MemcpyFn = 276 M.getOrInsertFunction("memcpy", Attr, IRB.getInt8PtrTy(), 277 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); 278 MemsetFn = 279 M.getOrInsertFunction("memset", Attr, IRB.getInt8PtrTy(), 280 IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy); 281 } 282 283 ThreadSanitizer::ThreadSanitizer(Module &M) { 284 const DataLayout &DL = M.getDataLayout(); 285 IntptrTy = DL.getIntPtrType(M.getContext()); 286 std::tie(TsanCtorFunction, std::ignore) = 287 getOrCreateSanitizerCtorAndInitFunctions( 288 M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{}, 289 /*InitArgs=*/{}, 290 // This callback is invoked when the functions are created the first 291 // time. Hook them into the global ctors list in that case: 292 [&](Function *Ctor, FunctionCallee) { 293 appendToGlobalCtors(M, Ctor, 0); 294 }); 295 } 296 297 static bool isVtableAccess(Instruction *I) { 298 if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) 299 return Tag->isTBAAVtableAccess(); 300 return false; 301 } 302 303 // Do not instrument known races/"benign races" that come from compiler 304 // instrumentatin. The user has no way of suppressing them. 305 static bool shouldInstrumentReadWriteFromAddress(const Module *M, Value *Addr) { 306 // Peel off GEPs and BitCasts. 307 Addr = Addr->stripInBoundsOffsets(); 308 309 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 310 if (GV->hasSection()) { 311 StringRef SectionName = GV->getSection(); 312 // Check if the global is in the PGO counters section. 313 auto OF = Triple(M->getTargetTriple()).getObjectFormat(); 314 if (SectionName.endswith( 315 getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false))) 316 return false; 317 } 318 319 // Check if the global is private gcov data. 320 if (GV->getName().startswith("__llvm_gcov") || 321 GV->getName().startswith("__llvm_gcda")) 322 return false; 323 } 324 325 // Do not instrument acesses from different address spaces; we cannot deal 326 // with them. 327 if (Addr) { 328 Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType()); 329 if (PtrTy->getPointerAddressSpace() != 0) 330 return false; 331 } 332 333 return true; 334 } 335 336 bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { 337 // If this is a GEP, just analyze its pointer operand. 338 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 339 Addr = GEP->getPointerOperand(); 340 341 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 342 if (GV->isConstant()) { 343 // Reads from constant globals can not race with any writes. 344 NumOmittedReadsFromConstantGlobals++; 345 return true; 346 } 347 } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) { 348 if (isVtableAccess(L)) { 349 // Reads from a vtable pointer can not race with any writes. 350 NumOmittedReadsFromVtable++; 351 return true; 352 } 353 } 354 return false; 355 } 356 357 // Instrumenting some of the accesses may be proven redundant. 358 // Currently handled: 359 // - read-before-write (within same BB, no calls between) 360 // - not captured variables 361 // 362 // We do not handle some of the patterns that should not survive 363 // after the classic compiler optimizations. 364 // E.g. two reads from the same temp should be eliminated by CSE, 365 // two writes should be eliminated by DSE, etc. 366 // 367 // 'Local' is a vector of insns within the same BB (no calls between). 368 // 'All' is a vector of insns that will be instrumented. 369 void ThreadSanitizer::chooseInstructionsToInstrument( 370 SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All, 371 const DataLayout &DL) { 372 SmallPtrSet<Value*, 8> WriteTargets; 373 // Iterate from the end. 374 for (Instruction *I : reverse(Local)) { 375 if (StoreInst *Store = dyn_cast<StoreInst>(I)) { 376 Value *Addr = Store->getPointerOperand(); 377 if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr)) 378 continue; 379 WriteTargets.insert(Addr); 380 } else { 381 LoadInst *Load = cast<LoadInst>(I); 382 Value *Addr = Load->getPointerOperand(); 383 if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr)) 384 continue; 385 if (WriteTargets.count(Addr)) { 386 // We will write to this temp, so no reason to analyze the read. 387 NumOmittedReadsBeforeWrite++; 388 continue; 389 } 390 if (addrPointsToConstantData(Addr)) { 391 // Addr points to some constant data -- it can not race with any writes. 392 continue; 393 } 394 } 395 Value *Addr = isa<StoreInst>(*I) 396 ? cast<StoreInst>(I)->getPointerOperand() 397 : cast<LoadInst>(I)->getPointerOperand(); 398 if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) && 399 !PointerMayBeCaptured(Addr, true, true)) { 400 // The variable is addressable but not captured, so it cannot be 401 // referenced from a different thread and participate in a data race 402 // (see llvm/Analysis/CaptureTracking.h for details). 403 NumOmittedNonCaptured++; 404 continue; 405 } 406 All.push_back(I); 407 } 408 Local.clear(); 409 } 410 411 static bool isAtomic(Instruction *I) { 412 // TODO: Ask TTI whether synchronization scope is between threads. 413 if (LoadInst *LI = dyn_cast<LoadInst>(I)) 414 return LI->isAtomic() && LI->getSyncScopeID() != SyncScope::SingleThread; 415 if (StoreInst *SI = dyn_cast<StoreInst>(I)) 416 return SI->isAtomic() && SI->getSyncScopeID() != SyncScope::SingleThread; 417 if (isa<AtomicRMWInst>(I)) 418 return true; 419 if (isa<AtomicCmpXchgInst>(I)) 420 return true; 421 if (isa<FenceInst>(I)) 422 return true; 423 return false; 424 } 425 426 void ThreadSanitizer::InsertRuntimeIgnores(Function &F) { 427 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 428 IRB.CreateCall(TsanIgnoreBegin); 429 EscapeEnumerator EE(F, "tsan_ignore_cleanup", ClHandleCxxExceptions); 430 while (IRBuilder<> *AtExit = EE.Next()) { 431 AtExit->CreateCall(TsanIgnoreEnd); 432 } 433 } 434 435 bool ThreadSanitizer::sanitizeFunction(Function &F, 436 const TargetLibraryInfo &TLI) { 437 // This is required to prevent instrumenting call to __tsan_init from within 438 // the module constructor. 439 if (&F == TsanCtorFunction) 440 return false; 441 initializeCallbacks(*F.getParent()); 442 SmallVector<Instruction*, 8> AllLoadsAndStores; 443 SmallVector<Instruction*, 8> LocalLoadsAndStores; 444 SmallVector<Instruction*, 8> AtomicAccesses; 445 SmallVector<Instruction*, 8> MemIntrinCalls; 446 bool Res = false; 447 bool HasCalls = false; 448 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeThread); 449 const DataLayout &DL = F.getParent()->getDataLayout(); 450 451 // Traverse all instructions, collect loads/stores/returns, check for calls. 452 for (auto &BB : F) { 453 for (auto &Inst : BB) { 454 if (isAtomic(&Inst)) 455 AtomicAccesses.push_back(&Inst); 456 else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) 457 LocalLoadsAndStores.push_back(&Inst); 458 else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) { 459 if (CallInst *CI = dyn_cast<CallInst>(&Inst)) 460 maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI); 461 if (isa<MemIntrinsic>(Inst)) 462 MemIntrinCalls.push_back(&Inst); 463 HasCalls = true; 464 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, 465 DL); 466 } 467 } 468 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL); 469 } 470 471 // We have collected all loads and stores. 472 // FIXME: many of these accesses do not need to be checked for races 473 // (e.g. variables that do not escape, etc). 474 475 // Instrument memory accesses only if we want to report bugs in the function. 476 if (ClInstrumentMemoryAccesses && SanitizeFunction) 477 for (auto Inst : AllLoadsAndStores) { 478 Res |= instrumentLoadOrStore(Inst, DL); 479 } 480 481 // Instrument atomic memory accesses in any case (they can be used to 482 // implement synchronization). 483 if (ClInstrumentAtomics) 484 for (auto Inst : AtomicAccesses) { 485 Res |= instrumentAtomic(Inst, DL); 486 } 487 488 if (ClInstrumentMemIntrinsics && SanitizeFunction) 489 for (auto Inst : MemIntrinCalls) { 490 Res |= instrumentMemIntrinsic(Inst); 491 } 492 493 if (F.hasFnAttribute("sanitize_thread_no_checking_at_run_time")) { 494 assert(!F.hasFnAttribute(Attribute::SanitizeThread)); 495 if (HasCalls) 496 InsertRuntimeIgnores(F); 497 } 498 499 // Instrument function entry/exit points if there were instrumented accesses. 500 if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { 501 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 502 Value *ReturnAddress = IRB.CreateCall( 503 Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), 504 IRB.getInt32(0)); 505 IRB.CreateCall(TsanFuncEntry, ReturnAddress); 506 507 EscapeEnumerator EE(F, "tsan_cleanup", ClHandleCxxExceptions); 508 while (IRBuilder<> *AtExit = EE.Next()) { 509 AtExit->CreateCall(TsanFuncExit, {}); 510 } 511 Res = true; 512 } 513 return Res; 514 } 515 516 bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I, 517 const DataLayout &DL) { 518 IRBuilder<> IRB(I); 519 bool IsWrite = isa<StoreInst>(*I); 520 Value *Addr = IsWrite 521 ? cast<StoreInst>(I)->getPointerOperand() 522 : cast<LoadInst>(I)->getPointerOperand(); 523 524 // swifterror memory addresses are mem2reg promoted by instruction selection. 525 // As such they cannot have regular uses like an instrumentation function and 526 // it makes no sense to track them as memory. 527 if (Addr->isSwiftError()) 528 return false; 529 530 int Idx = getMemoryAccessFuncIndex(Addr, DL); 531 if (Idx < 0) 532 return false; 533 if (IsWrite && isVtableAccess(I)) { 534 LLVM_DEBUG(dbgs() << " VPTR : " << *I << "\n"); 535 Value *StoredValue = cast<StoreInst>(I)->getValueOperand(); 536 // StoredValue may be a vector type if we are storing several vptrs at once. 537 // In this case, just take the first element of the vector since this is 538 // enough to find vptr races. 539 if (isa<VectorType>(StoredValue->getType())) 540 StoredValue = IRB.CreateExtractElement( 541 StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0)); 542 if (StoredValue->getType()->isIntegerTy()) 543 StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy()); 544 // Call TsanVptrUpdate. 545 IRB.CreateCall(TsanVptrUpdate, 546 {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), 547 IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())}); 548 NumInstrumentedVtableWrites++; 549 return true; 550 } 551 if (!IsWrite && isVtableAccess(I)) { 552 IRB.CreateCall(TsanVptrLoad, 553 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 554 NumInstrumentedVtableReads++; 555 return true; 556 } 557 const unsigned Alignment = IsWrite 558 ? cast<StoreInst>(I)->getAlignment() 559 : cast<LoadInst>(I)->getAlignment(); 560 Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 561 const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 562 FunctionCallee OnAccessFunc = nullptr; 563 if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0) 564 OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; 565 else 566 OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx]; 567 IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 568 if (IsWrite) NumInstrumentedWrites++; 569 else NumInstrumentedReads++; 570 return true; 571 } 572 573 static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 574 uint32_t v = 0; 575 switch (ord) { 576 case AtomicOrdering::NotAtomic: 577 llvm_unreachable("unexpected atomic ordering!"); 578 case AtomicOrdering::Unordered: LLVM_FALLTHROUGH; 579 case AtomicOrdering::Monotonic: v = 0; break; 580 // Not specified yet: 581 // case AtomicOrdering::Consume: v = 1; break; 582 case AtomicOrdering::Acquire: v = 2; break; 583 case AtomicOrdering::Release: v = 3; break; 584 case AtomicOrdering::AcquireRelease: v = 4; break; 585 case AtomicOrdering::SequentiallyConsistent: v = 5; break; 586 } 587 return IRB->getInt32(v); 588 } 589 590 // If a memset intrinsic gets inlined by the code gen, we will miss races on it. 591 // So, we either need to ensure the intrinsic is not inlined, or instrument it. 592 // We do not instrument memset/memmove/memcpy intrinsics (too complicated), 593 // instead we simply replace them with regular function calls, which are then 594 // intercepted by the run-time. 595 // Since tsan is running after everyone else, the calls should not be 596 // replaced back with intrinsics. If that becomes wrong at some point, 597 // we will need to call e.g. __tsan_memset to avoid the intrinsics. 598 bool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) { 599 IRBuilder<> IRB(I); 600 if (MemSetInst *M = dyn_cast<MemSetInst>(I)) { 601 IRB.CreateCall( 602 MemsetFn, 603 {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 604 IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false), 605 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 606 I->eraseFromParent(); 607 } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) { 608 IRB.CreateCall( 609 isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn, 610 {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 611 IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()), 612 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 613 I->eraseFromParent(); 614 } 615 return false; 616 } 617 618 // Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x 619 // standards. For background see C++11 standard. A slightly older, publicly 620 // available draft of the standard (not entirely up-to-date, but close enough 621 // for casual browsing) is available here: 622 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 623 // The following page contains more background information: 624 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 625 626 bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { 627 IRBuilder<> IRB(I); 628 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 629 Value *Addr = LI->getPointerOperand(); 630 int Idx = getMemoryAccessFuncIndex(Addr, DL); 631 if (Idx < 0) 632 return false; 633 const unsigned ByteSize = 1U << Idx; 634 const unsigned BitSize = ByteSize * 8; 635 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 636 Type *PtrTy = Ty->getPointerTo(); 637 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 638 createOrdering(&IRB, LI->getOrdering())}; 639 Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 640 Value *C = IRB.CreateCall(TsanAtomicLoad[Idx], Args); 641 Value *Cast = IRB.CreateBitOrPointerCast(C, OrigTy); 642 I->replaceAllUsesWith(Cast); 643 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 644 Value *Addr = SI->getPointerOperand(); 645 int Idx = getMemoryAccessFuncIndex(Addr, DL); 646 if (Idx < 0) 647 return false; 648 const unsigned ByteSize = 1U << Idx; 649 const unsigned BitSize = ByteSize * 8; 650 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 651 Type *PtrTy = Ty->getPointerTo(); 652 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 653 IRB.CreateBitOrPointerCast(SI->getValueOperand(), Ty), 654 createOrdering(&IRB, SI->getOrdering())}; 655 CallInst *C = CallInst::Create(TsanAtomicStore[Idx], Args); 656 ReplaceInstWithInst(I, C); 657 } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) { 658 Value *Addr = RMWI->getPointerOperand(); 659 int Idx = getMemoryAccessFuncIndex(Addr, DL); 660 if (Idx < 0) 661 return false; 662 FunctionCallee F = TsanAtomicRMW[RMWI->getOperation()][Idx]; 663 if (!F) 664 return false; 665 const unsigned ByteSize = 1U << Idx; 666 const unsigned BitSize = ByteSize * 8; 667 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 668 Type *PtrTy = Ty->getPointerTo(); 669 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 670 IRB.CreateIntCast(RMWI->getValOperand(), Ty, false), 671 createOrdering(&IRB, RMWI->getOrdering())}; 672 CallInst *C = CallInst::Create(F, Args); 673 ReplaceInstWithInst(I, C); 674 } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) { 675 Value *Addr = CASI->getPointerOperand(); 676 int Idx = getMemoryAccessFuncIndex(Addr, DL); 677 if (Idx < 0) 678 return false; 679 const unsigned ByteSize = 1U << Idx; 680 const unsigned BitSize = ByteSize * 8; 681 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 682 Type *PtrTy = Ty->getPointerTo(); 683 Value *CmpOperand = 684 IRB.CreateBitOrPointerCast(CASI->getCompareOperand(), Ty); 685 Value *NewOperand = 686 IRB.CreateBitOrPointerCast(CASI->getNewValOperand(), Ty); 687 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 688 CmpOperand, 689 NewOperand, 690 createOrdering(&IRB, CASI->getSuccessOrdering()), 691 createOrdering(&IRB, CASI->getFailureOrdering())}; 692 CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args); 693 Value *Success = IRB.CreateICmpEQ(C, CmpOperand); 694 Value *OldVal = C; 695 Type *OrigOldValTy = CASI->getNewValOperand()->getType(); 696 if (Ty != OrigOldValTy) { 697 // The value is a pointer, so we need to cast the return value. 698 OldVal = IRB.CreateIntToPtr(C, OrigOldValTy); 699 } 700 701 Value *Res = 702 IRB.CreateInsertValue(UndefValue::get(CASI->getType()), OldVal, 0); 703 Res = IRB.CreateInsertValue(Res, Success, 1); 704 705 I->replaceAllUsesWith(Res); 706 I->eraseFromParent(); 707 } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { 708 Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; 709 FunctionCallee F = FI->getSyncScopeID() == SyncScope::SingleThread 710 ? TsanAtomicSignalFence 711 : TsanAtomicThreadFence; 712 CallInst *C = CallInst::Create(F, Args); 713 ReplaceInstWithInst(I, C); 714 } 715 return true; 716 } 717 718 int ThreadSanitizer::getMemoryAccessFuncIndex(Value *Addr, 719 const DataLayout &DL) { 720 Type *OrigPtrTy = Addr->getType(); 721 Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); 722 assert(OrigTy->isSized()); 723 uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 724 if (TypeSize != 8 && TypeSize != 16 && 725 TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { 726 NumAccessesWithBadSize++; 727 // Ignore all unusual sizes. 728 return -1; 729 } 730 size_t Idx = countTrailingZeros(TypeSize / 8); 731 assert(Idx < kNumberOfAccessSizes); 732 return Idx; 733 } 734