1 //===-- ThreadSanitizer.cpp - race detector -------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer, a race detector. 10 // 11 // The tool is under development, for the details about previous versions see 12 // http://code.google.com/p/data-race-test 13 // 14 // The instrumentation phase is quite simple: 15 // - Insert calls to run-time library before every memory access. 16 // - Optimizations may apply to avoid instrumenting some of the accesses. 17 // - Insert calls at function entry/exit. 18 // The rest is handled by the run-time library. 19 //===----------------------------------------------------------------------===// 20 21 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" 22 #include "llvm/ADT/SmallPtrSet.h" 23 #include "llvm/ADT/SmallString.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/Statistic.h" 26 #include "llvm/ADT/StringExtras.h" 27 #include "llvm/Analysis/CaptureTracking.h" 28 #include "llvm/Analysis/TargetLibraryInfo.h" 29 #include "llvm/Analysis/ValueTracking.h" 30 #include "llvm/IR/DataLayout.h" 31 #include "llvm/IR/Function.h" 32 #include "llvm/IR/IRBuilder.h" 33 #include "llvm/IR/IntrinsicInst.h" 34 #include "llvm/IR/Intrinsics.h" 35 #include "llvm/IR/LLVMContext.h" 36 #include "llvm/IR/Metadata.h" 37 #include "llvm/IR/Module.h" 38 #include "llvm/IR/Type.h" 39 #include "llvm/InitializePasses.h" 40 #include "llvm/ProfileData/InstrProf.h" 41 #include "llvm/Support/CommandLine.h" 42 #include "llvm/Support/Debug.h" 43 #include "llvm/Support/MathExtras.h" 44 #include "llvm/Support/raw_ostream.h" 45 #include "llvm/Transforms/Instrumentation.h" 46 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 47 #include "llvm/Transforms/Utils/EscapeEnumerator.h" 48 #include "llvm/Transforms/Utils/Local.h" 49 #include "llvm/Transforms/Utils/ModuleUtils.h" 50 51 using namespace llvm; 52 53 #define DEBUG_TYPE "tsan" 54 55 static cl::opt<bool> ClInstrumentMemoryAccesses( 56 "tsan-instrument-memory-accesses", cl::init(true), 57 cl::desc("Instrument memory accesses"), cl::Hidden); 58 static cl::opt<bool> ClInstrumentFuncEntryExit( 59 "tsan-instrument-func-entry-exit", cl::init(true), 60 cl::desc("Instrument function entry and exit"), cl::Hidden); 61 static cl::opt<bool> ClHandleCxxExceptions( 62 "tsan-handle-cxx-exceptions", cl::init(true), 63 cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"), 64 cl::Hidden); 65 static cl::opt<bool> ClInstrumentAtomics( 66 "tsan-instrument-atomics", cl::init(true), 67 cl::desc("Instrument atomics"), cl::Hidden); 68 static cl::opt<bool> ClInstrumentMemIntrinsics( 69 "tsan-instrument-memintrinsics", cl::init(true), 70 cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); 71 72 STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); 73 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); 74 STATISTIC(NumOmittedReadsBeforeWrite, 75 "Number of reads ignored due to following writes"); 76 STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); 77 STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); 78 STATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads"); 79 STATISTIC(NumOmittedReadsFromConstantGlobals, 80 "Number of reads from constant globals"); 81 STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); 82 STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing"); 83 84 static const char *const kTsanModuleCtorName = "tsan.module_ctor"; 85 static const char *const kTsanInitName = "__tsan_init"; 86 87 namespace { 88 89 /// ThreadSanitizer: instrument the code in module to find races. 90 /// 91 /// Instantiating ThreadSanitizer inserts the tsan runtime library API function 92 /// declarations into the module if they don't exist already. Instantiating 93 /// ensures the __tsan_init function is in the list of global constructors for 94 /// the module. 95 struct ThreadSanitizer { 96 bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI); 97 98 private: 99 void initialize(Module &M); 100 bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); 101 bool instrumentAtomic(Instruction *I, const DataLayout &DL); 102 bool instrumentMemIntrinsic(Instruction *I); 103 void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local, 104 SmallVectorImpl<Instruction *> &All, 105 const DataLayout &DL); 106 bool addrPointsToConstantData(Value *Addr); 107 int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL); 108 void InsertRuntimeIgnores(Function &F); 109 110 Type *IntptrTy; 111 FunctionCallee TsanFuncEntry; 112 FunctionCallee TsanFuncExit; 113 FunctionCallee TsanIgnoreBegin; 114 FunctionCallee TsanIgnoreEnd; 115 // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 116 static const size_t kNumberOfAccessSizes = 5; 117 FunctionCallee TsanRead[kNumberOfAccessSizes]; 118 FunctionCallee TsanWrite[kNumberOfAccessSizes]; 119 FunctionCallee TsanUnalignedRead[kNumberOfAccessSizes]; 120 FunctionCallee TsanUnalignedWrite[kNumberOfAccessSizes]; 121 FunctionCallee TsanAtomicLoad[kNumberOfAccessSizes]; 122 FunctionCallee TsanAtomicStore[kNumberOfAccessSizes]; 123 FunctionCallee TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1] 124 [kNumberOfAccessSizes]; 125 FunctionCallee TsanAtomicCAS[kNumberOfAccessSizes]; 126 FunctionCallee TsanAtomicThreadFence; 127 FunctionCallee TsanAtomicSignalFence; 128 FunctionCallee TsanVptrUpdate; 129 FunctionCallee TsanVptrLoad; 130 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn; 131 }; 132 133 struct ThreadSanitizerLegacyPass : FunctionPass { 134 ThreadSanitizerLegacyPass() : FunctionPass(ID) {} 135 StringRef getPassName() const override; 136 void getAnalysisUsage(AnalysisUsage &AU) const override; 137 bool runOnFunction(Function &F) override; 138 bool doInitialization(Module &M) override; 139 static char ID; // Pass identification, replacement for typeid. 140 private: 141 Optional<ThreadSanitizer> TSan; 142 }; 143 144 void insertModuleCtor(Module &M) { 145 getOrCreateSanitizerCtorAndInitFunctions( 146 M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{}, 147 /*InitArgs=*/{}, 148 // This callback is invoked when the functions are created the first 149 // time. Hook them into the global ctors list in that case: 150 [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); }); 151 } 152 153 } // namespace 154 155 PreservedAnalyses ThreadSanitizerPass::run(Function &F, 156 FunctionAnalysisManager &FAM) { 157 ThreadSanitizer TSan; 158 if (TSan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F))) 159 return PreservedAnalyses::none(); 160 return PreservedAnalyses::all(); 161 } 162 163 PreservedAnalyses ThreadSanitizerPass::run(Module &M, 164 ModuleAnalysisManager &MAM) { 165 insertModuleCtor(M); 166 return PreservedAnalyses::none(); 167 } 168 169 char ThreadSanitizerLegacyPass::ID = 0; 170 INITIALIZE_PASS_BEGIN(ThreadSanitizerLegacyPass, "tsan", 171 "ThreadSanitizer: detects data races.", false, false) 172 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 173 INITIALIZE_PASS_END(ThreadSanitizerLegacyPass, "tsan", 174 "ThreadSanitizer: detects data races.", false, false) 175 176 StringRef ThreadSanitizerLegacyPass::getPassName() const { 177 return "ThreadSanitizerLegacyPass"; 178 } 179 180 void ThreadSanitizerLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const { 181 AU.addRequired<TargetLibraryInfoWrapperPass>(); 182 } 183 184 bool ThreadSanitizerLegacyPass::doInitialization(Module &M) { 185 insertModuleCtor(M); 186 TSan.emplace(); 187 return true; 188 } 189 190 bool ThreadSanitizerLegacyPass::runOnFunction(Function &F) { 191 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 192 TSan->sanitizeFunction(F, TLI); 193 return true; 194 } 195 196 FunctionPass *llvm::createThreadSanitizerLegacyPassPass() { 197 return new ThreadSanitizerLegacyPass(); 198 } 199 200 void ThreadSanitizer::initialize(Module &M) { 201 const DataLayout &DL = M.getDataLayout(); 202 IntptrTy = DL.getIntPtrType(M.getContext()); 203 204 IRBuilder<> IRB(M.getContext()); 205 AttributeList Attr; 206 Attr = Attr.addAttribute(M.getContext(), AttributeList::FunctionIndex, 207 Attribute::NoUnwind); 208 // Initialize the callbacks. 209 TsanFuncEntry = M.getOrInsertFunction("__tsan_func_entry", Attr, 210 IRB.getVoidTy(), IRB.getInt8PtrTy()); 211 TsanFuncExit = 212 M.getOrInsertFunction("__tsan_func_exit", Attr, IRB.getVoidTy()); 213 TsanIgnoreBegin = M.getOrInsertFunction("__tsan_ignore_thread_begin", Attr, 214 IRB.getVoidTy()); 215 TsanIgnoreEnd = 216 M.getOrInsertFunction("__tsan_ignore_thread_end", Attr, IRB.getVoidTy()); 217 IntegerType *OrdTy = IRB.getInt32Ty(); 218 for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { 219 const unsigned ByteSize = 1U << i; 220 const unsigned BitSize = ByteSize * 8; 221 std::string ByteSizeStr = utostr(ByteSize); 222 std::string BitSizeStr = utostr(BitSize); 223 SmallString<32> ReadName("__tsan_read" + ByteSizeStr); 224 TsanRead[i] = M.getOrInsertFunction(ReadName, Attr, IRB.getVoidTy(), 225 IRB.getInt8PtrTy()); 226 227 SmallString<32> WriteName("__tsan_write" + ByteSizeStr); 228 TsanWrite[i] = M.getOrInsertFunction(WriteName, Attr, IRB.getVoidTy(), 229 IRB.getInt8PtrTy()); 230 231 SmallString<64> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr); 232 TsanUnalignedRead[i] = M.getOrInsertFunction( 233 UnalignedReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()); 234 235 SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + ByteSizeStr); 236 TsanUnalignedWrite[i] = M.getOrInsertFunction( 237 UnalignedWriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()); 238 239 Type *Ty = Type::getIntNTy(M.getContext(), BitSize); 240 Type *PtrTy = Ty->getPointerTo(); 241 SmallString<32> AtomicLoadName("__tsan_atomic" + BitSizeStr + "_load"); 242 TsanAtomicLoad[i] = 243 M.getOrInsertFunction(AtomicLoadName, Attr, Ty, PtrTy, OrdTy); 244 245 SmallString<32> AtomicStoreName("__tsan_atomic" + BitSizeStr + "_store"); 246 TsanAtomicStore[i] = M.getOrInsertFunction( 247 AtomicStoreName, Attr, IRB.getVoidTy(), PtrTy, Ty, OrdTy); 248 249 for (int op = AtomicRMWInst::FIRST_BINOP; 250 op <= AtomicRMWInst::LAST_BINOP; ++op) { 251 TsanAtomicRMW[op][i] = nullptr; 252 const char *NamePart = nullptr; 253 if (op == AtomicRMWInst::Xchg) 254 NamePart = "_exchange"; 255 else if (op == AtomicRMWInst::Add) 256 NamePart = "_fetch_add"; 257 else if (op == AtomicRMWInst::Sub) 258 NamePart = "_fetch_sub"; 259 else if (op == AtomicRMWInst::And) 260 NamePart = "_fetch_and"; 261 else if (op == AtomicRMWInst::Or) 262 NamePart = "_fetch_or"; 263 else if (op == AtomicRMWInst::Xor) 264 NamePart = "_fetch_xor"; 265 else if (op == AtomicRMWInst::Nand) 266 NamePart = "_fetch_nand"; 267 else 268 continue; 269 SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); 270 TsanAtomicRMW[op][i] = 271 M.getOrInsertFunction(RMWName, Attr, Ty, PtrTy, Ty, OrdTy); 272 } 273 274 SmallString<32> AtomicCASName("__tsan_atomic" + BitSizeStr + 275 "_compare_exchange_val"); 276 TsanAtomicCAS[i] = M.getOrInsertFunction(AtomicCASName, Attr, Ty, PtrTy, Ty, 277 Ty, OrdTy, OrdTy); 278 } 279 TsanVptrUpdate = 280 M.getOrInsertFunction("__tsan_vptr_update", Attr, IRB.getVoidTy(), 281 IRB.getInt8PtrTy(), IRB.getInt8PtrTy()); 282 TsanVptrLoad = M.getOrInsertFunction("__tsan_vptr_read", Attr, 283 IRB.getVoidTy(), IRB.getInt8PtrTy()); 284 TsanAtomicThreadFence = M.getOrInsertFunction("__tsan_atomic_thread_fence", 285 Attr, IRB.getVoidTy(), OrdTy); 286 TsanAtomicSignalFence = M.getOrInsertFunction("__tsan_atomic_signal_fence", 287 Attr, IRB.getVoidTy(), OrdTy); 288 289 MemmoveFn = 290 M.getOrInsertFunction("memmove", Attr, IRB.getInt8PtrTy(), 291 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); 292 MemcpyFn = 293 M.getOrInsertFunction("memcpy", Attr, IRB.getInt8PtrTy(), 294 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); 295 MemsetFn = 296 M.getOrInsertFunction("memset", Attr, IRB.getInt8PtrTy(), 297 IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy); 298 } 299 300 static bool isVtableAccess(Instruction *I) { 301 if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) 302 return Tag->isTBAAVtableAccess(); 303 return false; 304 } 305 306 // Do not instrument known races/"benign races" that come from compiler 307 // instrumentatin. The user has no way of suppressing them. 308 static bool shouldInstrumentReadWriteFromAddress(const Module *M, Value *Addr) { 309 // Peel off GEPs and BitCasts. 310 Addr = Addr->stripInBoundsOffsets(); 311 312 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 313 if (GV->hasSection()) { 314 StringRef SectionName = GV->getSection(); 315 // Check if the global is in the PGO counters section. 316 auto OF = Triple(M->getTargetTriple()).getObjectFormat(); 317 if (SectionName.endswith( 318 getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false))) 319 return false; 320 } 321 322 // Check if the global is private gcov data. 323 if (GV->getName().startswith("__llvm_gcov") || 324 GV->getName().startswith("__llvm_gcda")) 325 return false; 326 } 327 328 // Do not instrument acesses from different address spaces; we cannot deal 329 // with them. 330 if (Addr) { 331 Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType()); 332 if (PtrTy->getPointerAddressSpace() != 0) 333 return false; 334 } 335 336 return true; 337 } 338 339 bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { 340 // If this is a GEP, just analyze its pointer operand. 341 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 342 Addr = GEP->getPointerOperand(); 343 344 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 345 if (GV->isConstant()) { 346 // Reads from constant globals can not race with any writes. 347 NumOmittedReadsFromConstantGlobals++; 348 return true; 349 } 350 } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) { 351 if (isVtableAccess(L)) { 352 // Reads from a vtable pointer can not race with any writes. 353 NumOmittedReadsFromVtable++; 354 return true; 355 } 356 } 357 return false; 358 } 359 360 // Instrumenting some of the accesses may be proven redundant. 361 // Currently handled: 362 // - read-before-write (within same BB, no calls between) 363 // - not captured variables 364 // 365 // We do not handle some of the patterns that should not survive 366 // after the classic compiler optimizations. 367 // E.g. two reads from the same temp should be eliminated by CSE, 368 // two writes should be eliminated by DSE, etc. 369 // 370 // 'Local' is a vector of insns within the same BB (no calls between). 371 // 'All' is a vector of insns that will be instrumented. 372 void ThreadSanitizer::chooseInstructionsToInstrument( 373 SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All, 374 const DataLayout &DL) { 375 SmallPtrSet<Value*, 8> WriteTargets; 376 // Iterate from the end. 377 for (Instruction *I : reverse(Local)) { 378 if (StoreInst *Store = dyn_cast<StoreInst>(I)) { 379 Value *Addr = Store->getPointerOperand(); 380 if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr)) 381 continue; 382 WriteTargets.insert(Addr); 383 } else { 384 LoadInst *Load = cast<LoadInst>(I); 385 Value *Addr = Load->getPointerOperand(); 386 if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr)) 387 continue; 388 if (WriteTargets.count(Addr)) { 389 // We will write to this temp, so no reason to analyze the read. 390 NumOmittedReadsBeforeWrite++; 391 continue; 392 } 393 if (addrPointsToConstantData(Addr)) { 394 // Addr points to some constant data -- it can not race with any writes. 395 continue; 396 } 397 } 398 Value *Addr = isa<StoreInst>(*I) 399 ? cast<StoreInst>(I)->getPointerOperand() 400 : cast<LoadInst>(I)->getPointerOperand(); 401 if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) && 402 !PointerMayBeCaptured(Addr, true, true)) { 403 // The variable is addressable but not captured, so it cannot be 404 // referenced from a different thread and participate in a data race 405 // (see llvm/Analysis/CaptureTracking.h for details). 406 NumOmittedNonCaptured++; 407 continue; 408 } 409 All.push_back(I); 410 } 411 Local.clear(); 412 } 413 414 static bool isAtomic(Instruction *I) { 415 // TODO: Ask TTI whether synchronization scope is between threads. 416 if (LoadInst *LI = dyn_cast<LoadInst>(I)) 417 return LI->isAtomic() && LI->getSyncScopeID() != SyncScope::SingleThread; 418 if (StoreInst *SI = dyn_cast<StoreInst>(I)) 419 return SI->isAtomic() && SI->getSyncScopeID() != SyncScope::SingleThread; 420 if (isa<AtomicRMWInst>(I)) 421 return true; 422 if (isa<AtomicCmpXchgInst>(I)) 423 return true; 424 if (isa<FenceInst>(I)) 425 return true; 426 return false; 427 } 428 429 void ThreadSanitizer::InsertRuntimeIgnores(Function &F) { 430 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 431 IRB.CreateCall(TsanIgnoreBegin); 432 EscapeEnumerator EE(F, "tsan_ignore_cleanup", ClHandleCxxExceptions); 433 while (IRBuilder<> *AtExit = EE.Next()) { 434 AtExit->CreateCall(TsanIgnoreEnd); 435 } 436 } 437 438 bool ThreadSanitizer::sanitizeFunction(Function &F, 439 const TargetLibraryInfo &TLI) { 440 // This is required to prevent instrumenting call to __tsan_init from within 441 // the module constructor. 442 if (F.getName() == kTsanModuleCtorName) 443 return false; 444 initialize(*F.getParent()); 445 SmallVector<Instruction*, 8> AllLoadsAndStores; 446 SmallVector<Instruction*, 8> LocalLoadsAndStores; 447 SmallVector<Instruction*, 8> AtomicAccesses; 448 SmallVector<Instruction*, 8> MemIntrinCalls; 449 bool Res = false; 450 bool HasCalls = false; 451 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeThread); 452 const DataLayout &DL = F.getParent()->getDataLayout(); 453 454 // Traverse all instructions, collect loads/stores/returns, check for calls. 455 for (auto &BB : F) { 456 for (auto &Inst : BB) { 457 if (isAtomic(&Inst)) 458 AtomicAccesses.push_back(&Inst); 459 else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) 460 LocalLoadsAndStores.push_back(&Inst); 461 else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) { 462 if (CallInst *CI = dyn_cast<CallInst>(&Inst)) 463 maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI); 464 if (isa<MemIntrinsic>(Inst)) 465 MemIntrinCalls.push_back(&Inst); 466 HasCalls = true; 467 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, 468 DL); 469 } 470 } 471 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL); 472 } 473 474 // We have collected all loads and stores. 475 // FIXME: many of these accesses do not need to be checked for races 476 // (e.g. variables that do not escape, etc). 477 478 // Instrument memory accesses only if we want to report bugs in the function. 479 if (ClInstrumentMemoryAccesses && SanitizeFunction) 480 for (auto Inst : AllLoadsAndStores) { 481 Res |= instrumentLoadOrStore(Inst, DL); 482 } 483 484 // Instrument atomic memory accesses in any case (they can be used to 485 // implement synchronization). 486 if (ClInstrumentAtomics) 487 for (auto Inst : AtomicAccesses) { 488 Res |= instrumentAtomic(Inst, DL); 489 } 490 491 if (ClInstrumentMemIntrinsics && SanitizeFunction) 492 for (auto Inst : MemIntrinCalls) { 493 Res |= instrumentMemIntrinsic(Inst); 494 } 495 496 if (F.hasFnAttribute("sanitize_thread_no_checking_at_run_time")) { 497 assert(!F.hasFnAttribute(Attribute::SanitizeThread)); 498 if (HasCalls) 499 InsertRuntimeIgnores(F); 500 } 501 502 // Instrument function entry/exit points if there were instrumented accesses. 503 if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { 504 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 505 Value *ReturnAddress = IRB.CreateCall( 506 Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), 507 IRB.getInt32(0)); 508 IRB.CreateCall(TsanFuncEntry, ReturnAddress); 509 510 EscapeEnumerator EE(F, "tsan_cleanup", ClHandleCxxExceptions); 511 while (IRBuilder<> *AtExit = EE.Next()) { 512 AtExit->CreateCall(TsanFuncExit, {}); 513 } 514 Res = true; 515 } 516 return Res; 517 } 518 519 bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I, 520 const DataLayout &DL) { 521 IRBuilder<> IRB(I); 522 bool IsWrite = isa<StoreInst>(*I); 523 Value *Addr = IsWrite 524 ? cast<StoreInst>(I)->getPointerOperand() 525 : cast<LoadInst>(I)->getPointerOperand(); 526 527 // swifterror memory addresses are mem2reg promoted by instruction selection. 528 // As such they cannot have regular uses like an instrumentation function and 529 // it makes no sense to track them as memory. 530 if (Addr->isSwiftError()) 531 return false; 532 533 int Idx = getMemoryAccessFuncIndex(Addr, DL); 534 if (Idx < 0) 535 return false; 536 if (IsWrite && isVtableAccess(I)) { 537 LLVM_DEBUG(dbgs() << " VPTR : " << *I << "\n"); 538 Value *StoredValue = cast<StoreInst>(I)->getValueOperand(); 539 // StoredValue may be a vector type if we are storing several vptrs at once. 540 // In this case, just take the first element of the vector since this is 541 // enough to find vptr races. 542 if (isa<VectorType>(StoredValue->getType())) 543 StoredValue = IRB.CreateExtractElement( 544 StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0)); 545 if (StoredValue->getType()->isIntegerTy()) 546 StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy()); 547 // Call TsanVptrUpdate. 548 IRB.CreateCall(TsanVptrUpdate, 549 {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), 550 IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())}); 551 NumInstrumentedVtableWrites++; 552 return true; 553 } 554 if (!IsWrite && isVtableAccess(I)) { 555 IRB.CreateCall(TsanVptrLoad, 556 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 557 NumInstrumentedVtableReads++; 558 return true; 559 } 560 const unsigned Alignment = IsWrite 561 ? cast<StoreInst>(I)->getAlignment() 562 : cast<LoadInst>(I)->getAlignment(); 563 Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 564 const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 565 FunctionCallee OnAccessFunc = nullptr; 566 if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0) 567 OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; 568 else 569 OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx]; 570 IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 571 if (IsWrite) NumInstrumentedWrites++; 572 else NumInstrumentedReads++; 573 return true; 574 } 575 576 static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 577 uint32_t v = 0; 578 switch (ord) { 579 case AtomicOrdering::NotAtomic: 580 llvm_unreachable("unexpected atomic ordering!"); 581 case AtomicOrdering::Unordered: LLVM_FALLTHROUGH; 582 case AtomicOrdering::Monotonic: v = 0; break; 583 // Not specified yet: 584 // case AtomicOrdering::Consume: v = 1; break; 585 case AtomicOrdering::Acquire: v = 2; break; 586 case AtomicOrdering::Release: v = 3; break; 587 case AtomicOrdering::AcquireRelease: v = 4; break; 588 case AtomicOrdering::SequentiallyConsistent: v = 5; break; 589 } 590 return IRB->getInt32(v); 591 } 592 593 // If a memset intrinsic gets inlined by the code gen, we will miss races on it. 594 // So, we either need to ensure the intrinsic is not inlined, or instrument it. 595 // We do not instrument memset/memmove/memcpy intrinsics (too complicated), 596 // instead we simply replace them with regular function calls, which are then 597 // intercepted by the run-time. 598 // Since tsan is running after everyone else, the calls should not be 599 // replaced back with intrinsics. If that becomes wrong at some point, 600 // we will need to call e.g. __tsan_memset to avoid the intrinsics. 601 bool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) { 602 IRBuilder<> IRB(I); 603 if (MemSetInst *M = dyn_cast<MemSetInst>(I)) { 604 IRB.CreateCall( 605 MemsetFn, 606 {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 607 IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false), 608 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 609 I->eraseFromParent(); 610 } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) { 611 IRB.CreateCall( 612 isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn, 613 {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 614 IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()), 615 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 616 I->eraseFromParent(); 617 } 618 return false; 619 } 620 621 // Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x 622 // standards. For background see C++11 standard. A slightly older, publicly 623 // available draft of the standard (not entirely up-to-date, but close enough 624 // for casual browsing) is available here: 625 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 626 // The following page contains more background information: 627 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 628 629 bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { 630 IRBuilder<> IRB(I); 631 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 632 Value *Addr = LI->getPointerOperand(); 633 int Idx = getMemoryAccessFuncIndex(Addr, DL); 634 if (Idx < 0) 635 return false; 636 const unsigned ByteSize = 1U << Idx; 637 const unsigned BitSize = ByteSize * 8; 638 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 639 Type *PtrTy = Ty->getPointerTo(); 640 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 641 createOrdering(&IRB, LI->getOrdering())}; 642 Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 643 Value *C = IRB.CreateCall(TsanAtomicLoad[Idx], Args); 644 Value *Cast = IRB.CreateBitOrPointerCast(C, OrigTy); 645 I->replaceAllUsesWith(Cast); 646 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 647 Value *Addr = SI->getPointerOperand(); 648 int Idx = getMemoryAccessFuncIndex(Addr, DL); 649 if (Idx < 0) 650 return false; 651 const unsigned ByteSize = 1U << Idx; 652 const unsigned BitSize = ByteSize * 8; 653 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 654 Type *PtrTy = Ty->getPointerTo(); 655 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 656 IRB.CreateBitOrPointerCast(SI->getValueOperand(), Ty), 657 createOrdering(&IRB, SI->getOrdering())}; 658 CallInst *C = CallInst::Create(TsanAtomicStore[Idx], Args); 659 ReplaceInstWithInst(I, C); 660 } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) { 661 Value *Addr = RMWI->getPointerOperand(); 662 int Idx = getMemoryAccessFuncIndex(Addr, DL); 663 if (Idx < 0) 664 return false; 665 FunctionCallee F = TsanAtomicRMW[RMWI->getOperation()][Idx]; 666 if (!F) 667 return false; 668 const unsigned ByteSize = 1U << Idx; 669 const unsigned BitSize = ByteSize * 8; 670 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 671 Type *PtrTy = Ty->getPointerTo(); 672 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 673 IRB.CreateIntCast(RMWI->getValOperand(), Ty, false), 674 createOrdering(&IRB, RMWI->getOrdering())}; 675 CallInst *C = CallInst::Create(F, Args); 676 ReplaceInstWithInst(I, C); 677 } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) { 678 Value *Addr = CASI->getPointerOperand(); 679 int Idx = getMemoryAccessFuncIndex(Addr, DL); 680 if (Idx < 0) 681 return false; 682 const unsigned ByteSize = 1U << Idx; 683 const unsigned BitSize = ByteSize * 8; 684 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 685 Type *PtrTy = Ty->getPointerTo(); 686 Value *CmpOperand = 687 IRB.CreateBitOrPointerCast(CASI->getCompareOperand(), Ty); 688 Value *NewOperand = 689 IRB.CreateBitOrPointerCast(CASI->getNewValOperand(), Ty); 690 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 691 CmpOperand, 692 NewOperand, 693 createOrdering(&IRB, CASI->getSuccessOrdering()), 694 createOrdering(&IRB, CASI->getFailureOrdering())}; 695 CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args); 696 Value *Success = IRB.CreateICmpEQ(C, CmpOperand); 697 Value *OldVal = C; 698 Type *OrigOldValTy = CASI->getNewValOperand()->getType(); 699 if (Ty != OrigOldValTy) { 700 // The value is a pointer, so we need to cast the return value. 701 OldVal = IRB.CreateIntToPtr(C, OrigOldValTy); 702 } 703 704 Value *Res = 705 IRB.CreateInsertValue(UndefValue::get(CASI->getType()), OldVal, 0); 706 Res = IRB.CreateInsertValue(Res, Success, 1); 707 708 I->replaceAllUsesWith(Res); 709 I->eraseFromParent(); 710 } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { 711 Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; 712 FunctionCallee F = FI->getSyncScopeID() == SyncScope::SingleThread 713 ? TsanAtomicSignalFence 714 : TsanAtomicThreadFence; 715 CallInst *C = CallInst::Create(F, Args); 716 ReplaceInstWithInst(I, C); 717 } 718 return true; 719 } 720 721 int ThreadSanitizer::getMemoryAccessFuncIndex(Value *Addr, 722 const DataLayout &DL) { 723 Type *OrigPtrTy = Addr->getType(); 724 Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); 725 assert(OrigTy->isSized()); 726 uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 727 if (TypeSize != 8 && TypeSize != 16 && 728 TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { 729 NumAccessesWithBadSize++; 730 // Ignore all unusual sizes. 731 return -1; 732 } 733 size_t Idx = countTrailingZeros(TypeSize / 8); 734 assert(Idx < kNumberOfAccessSizes); 735 return Idx; 736 } 737