1 //===-- ThreadSanitizer.cpp - race detector -------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer, a race detector. 10 // 11 // The tool is under development, for the details about previous versions see 12 // http://code.google.com/p/data-race-test 13 // 14 // The instrumentation phase is quite simple: 15 // - Insert calls to run-time library before every memory access. 16 // - Optimizations may apply to avoid instrumenting some of the accesses. 17 // - Insert calls at function entry/exit. 18 // The rest is handled by the run-time library. 19 //===----------------------------------------------------------------------===// 20 21 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" 22 #include "llvm/ADT/SmallPtrSet.h" 23 #include "llvm/ADT/SmallString.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/Statistic.h" 26 #include "llvm/ADT/StringExtras.h" 27 #include "llvm/Analysis/CaptureTracking.h" 28 #include "llvm/Analysis/TargetLibraryInfo.h" 29 #include "llvm/Transforms/Utils/Local.h" 30 #include "llvm/Analysis/ValueTracking.h" 31 #include "llvm/IR/DataLayout.h" 32 #include "llvm/IR/Function.h" 33 #include "llvm/IR/IRBuilder.h" 34 #include "llvm/IR/IntrinsicInst.h" 35 #include "llvm/IR/Intrinsics.h" 36 #include "llvm/IR/LLVMContext.h" 37 #include "llvm/IR/Metadata.h" 38 #include "llvm/IR/Module.h" 39 #include "llvm/IR/Type.h" 40 #include "llvm/ProfileData/InstrProf.h" 41 #include "llvm/Support/CommandLine.h" 42 #include "llvm/Support/Debug.h" 43 #include "llvm/Support/MathExtras.h" 44 #include "llvm/Support/raw_ostream.h" 45 #include "llvm/Transforms/Instrumentation.h" 46 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 47 #include "llvm/Transforms/Utils/EscapeEnumerator.h" 48 #include "llvm/Transforms/Utils/ModuleUtils.h" 49 50 using namespace llvm; 51 52 #define DEBUG_TYPE "tsan" 53 54 static cl::opt<bool> ClInstrumentMemoryAccesses( 55 "tsan-instrument-memory-accesses", cl::init(true), 56 cl::desc("Instrument memory accesses"), cl::Hidden); 57 static cl::opt<bool> ClInstrumentFuncEntryExit( 58 "tsan-instrument-func-entry-exit", cl::init(true), 59 cl::desc("Instrument function entry and exit"), cl::Hidden); 60 static cl::opt<bool> ClHandleCxxExceptions( 61 "tsan-handle-cxx-exceptions", cl::init(true), 62 cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"), 63 cl::Hidden); 64 static cl::opt<bool> ClInstrumentAtomics( 65 "tsan-instrument-atomics", cl::init(true), 66 cl::desc("Instrument atomics"), cl::Hidden); 67 static cl::opt<bool> ClInstrumentMemIntrinsics( 68 "tsan-instrument-memintrinsics", cl::init(true), 69 cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); 70 71 STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); 72 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); 73 STATISTIC(NumOmittedReadsBeforeWrite, 74 "Number of reads ignored due to following writes"); 75 STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size"); 76 STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes"); 77 STATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads"); 78 STATISTIC(NumOmittedReadsFromConstantGlobals, 79 "Number of reads from constant globals"); 80 STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads"); 81 STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing"); 82 83 static const char *const kTsanModuleCtorName = "tsan.module_ctor"; 84 static const char *const kTsanInitName = "__tsan_init"; 85 86 namespace { 87 88 /// ThreadSanitizer: instrument the code in module to find races. 89 /// 90 /// Instantiating ThreadSanitizer inserts the tsan runtime library API function 91 /// declarations into the module if they don't exist already. Instantiating 92 /// ensures the __tsan_init function is in the list of global constructors for 93 /// the module. 94 struct ThreadSanitizer { 95 bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI); 96 97 private: 98 void initialize(Module &M); 99 bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); 100 bool instrumentAtomic(Instruction *I, const DataLayout &DL); 101 bool instrumentMemIntrinsic(Instruction *I); 102 void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local, 103 SmallVectorImpl<Instruction *> &All, 104 const DataLayout &DL); 105 bool addrPointsToConstantData(Value *Addr); 106 int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL); 107 void InsertRuntimeIgnores(Function &F); 108 109 Type *IntptrTy; 110 FunctionCallee TsanFuncEntry; 111 FunctionCallee TsanFuncExit; 112 FunctionCallee TsanIgnoreBegin; 113 FunctionCallee TsanIgnoreEnd; 114 // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 115 static const size_t kNumberOfAccessSizes = 5; 116 FunctionCallee TsanRead[kNumberOfAccessSizes]; 117 FunctionCallee TsanWrite[kNumberOfAccessSizes]; 118 FunctionCallee TsanUnalignedRead[kNumberOfAccessSizes]; 119 FunctionCallee TsanUnalignedWrite[kNumberOfAccessSizes]; 120 FunctionCallee TsanAtomicLoad[kNumberOfAccessSizes]; 121 FunctionCallee TsanAtomicStore[kNumberOfAccessSizes]; 122 FunctionCallee TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1] 123 [kNumberOfAccessSizes]; 124 FunctionCallee TsanAtomicCAS[kNumberOfAccessSizes]; 125 FunctionCallee TsanAtomicThreadFence; 126 FunctionCallee TsanAtomicSignalFence; 127 FunctionCallee TsanVptrUpdate; 128 FunctionCallee TsanVptrLoad; 129 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn; 130 }; 131 132 struct ThreadSanitizerLegacyPass : FunctionPass { 133 ThreadSanitizerLegacyPass() : FunctionPass(ID) {} 134 StringRef getPassName() const override; 135 void getAnalysisUsage(AnalysisUsage &AU) const override; 136 bool runOnFunction(Function &F) override; 137 bool doInitialization(Module &M) override; 138 static char ID; // Pass identification, replacement for typeid. 139 private: 140 Optional<ThreadSanitizer> TSan; 141 }; 142 143 void insertModuleCtor(Module &M) { 144 getOrCreateSanitizerCtorAndInitFunctions( 145 M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{}, 146 /*InitArgs=*/{}, 147 // This callback is invoked when the functions are created the first 148 // time. Hook them into the global ctors list in that case: 149 [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); }); 150 } 151 152 } // namespace 153 154 PreservedAnalyses ThreadSanitizerPass::run(Function &F, 155 FunctionAnalysisManager &FAM) { 156 ThreadSanitizer TSan; 157 if (TSan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F))) 158 return PreservedAnalyses::none(); 159 return PreservedAnalyses::all(); 160 } 161 162 PreservedAnalyses ThreadSanitizerPass::run(Module &M, 163 ModuleAnalysisManager &MAM) { 164 insertModuleCtor(M); 165 return PreservedAnalyses::none(); 166 } 167 168 char ThreadSanitizerLegacyPass::ID = 0; 169 INITIALIZE_PASS_BEGIN(ThreadSanitizerLegacyPass, "tsan", 170 "ThreadSanitizer: detects data races.", false, false) 171 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 172 INITIALIZE_PASS_END(ThreadSanitizerLegacyPass, "tsan", 173 "ThreadSanitizer: detects data races.", false, false) 174 175 StringRef ThreadSanitizerLegacyPass::getPassName() const { 176 return "ThreadSanitizerLegacyPass"; 177 } 178 179 void ThreadSanitizerLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const { 180 AU.addRequired<TargetLibraryInfoWrapperPass>(); 181 } 182 183 bool ThreadSanitizerLegacyPass::doInitialization(Module &M) { 184 insertModuleCtor(M); 185 TSan.emplace(); 186 return true; 187 } 188 189 bool ThreadSanitizerLegacyPass::runOnFunction(Function &F) { 190 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 191 TSan->sanitizeFunction(F, TLI); 192 return true; 193 } 194 195 FunctionPass *llvm::createThreadSanitizerLegacyPassPass() { 196 return new ThreadSanitizerLegacyPass(); 197 } 198 199 void ThreadSanitizer::initialize(Module &M) { 200 const DataLayout &DL = M.getDataLayout(); 201 IntptrTy = DL.getIntPtrType(M.getContext()); 202 203 IRBuilder<> IRB(M.getContext()); 204 AttributeList Attr; 205 Attr = Attr.addAttribute(M.getContext(), AttributeList::FunctionIndex, 206 Attribute::NoUnwind); 207 // Initialize the callbacks. 208 TsanFuncEntry = M.getOrInsertFunction("__tsan_func_entry", Attr, 209 IRB.getVoidTy(), IRB.getInt8PtrTy()); 210 TsanFuncExit = 211 M.getOrInsertFunction("__tsan_func_exit", Attr, IRB.getVoidTy()); 212 TsanIgnoreBegin = M.getOrInsertFunction("__tsan_ignore_thread_begin", Attr, 213 IRB.getVoidTy()); 214 TsanIgnoreEnd = 215 M.getOrInsertFunction("__tsan_ignore_thread_end", Attr, IRB.getVoidTy()); 216 IntegerType *OrdTy = IRB.getInt32Ty(); 217 for (size_t i = 0; i < kNumberOfAccessSizes; ++i) { 218 const unsigned ByteSize = 1U << i; 219 const unsigned BitSize = ByteSize * 8; 220 std::string ByteSizeStr = utostr(ByteSize); 221 std::string BitSizeStr = utostr(BitSize); 222 SmallString<32> ReadName("__tsan_read" + ByteSizeStr); 223 TsanRead[i] = M.getOrInsertFunction(ReadName, Attr, IRB.getVoidTy(), 224 IRB.getInt8PtrTy()); 225 226 SmallString<32> WriteName("__tsan_write" + ByteSizeStr); 227 TsanWrite[i] = M.getOrInsertFunction(WriteName, Attr, IRB.getVoidTy(), 228 IRB.getInt8PtrTy()); 229 230 SmallString<64> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr); 231 TsanUnalignedRead[i] = M.getOrInsertFunction( 232 UnalignedReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()); 233 234 SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + ByteSizeStr); 235 TsanUnalignedWrite[i] = M.getOrInsertFunction( 236 UnalignedWriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()); 237 238 Type *Ty = Type::getIntNTy(M.getContext(), BitSize); 239 Type *PtrTy = Ty->getPointerTo(); 240 SmallString<32> AtomicLoadName("__tsan_atomic" + BitSizeStr + "_load"); 241 TsanAtomicLoad[i] = 242 M.getOrInsertFunction(AtomicLoadName, Attr, Ty, PtrTy, OrdTy); 243 244 SmallString<32> AtomicStoreName("__tsan_atomic" + BitSizeStr + "_store"); 245 TsanAtomicStore[i] = M.getOrInsertFunction( 246 AtomicStoreName, Attr, IRB.getVoidTy(), PtrTy, Ty, OrdTy); 247 248 for (int op = AtomicRMWInst::FIRST_BINOP; 249 op <= AtomicRMWInst::LAST_BINOP; ++op) { 250 TsanAtomicRMW[op][i] = nullptr; 251 const char *NamePart = nullptr; 252 if (op == AtomicRMWInst::Xchg) 253 NamePart = "_exchange"; 254 else if (op == AtomicRMWInst::Add) 255 NamePart = "_fetch_add"; 256 else if (op == AtomicRMWInst::Sub) 257 NamePart = "_fetch_sub"; 258 else if (op == AtomicRMWInst::And) 259 NamePart = "_fetch_and"; 260 else if (op == AtomicRMWInst::Or) 261 NamePart = "_fetch_or"; 262 else if (op == AtomicRMWInst::Xor) 263 NamePart = "_fetch_xor"; 264 else if (op == AtomicRMWInst::Nand) 265 NamePart = "_fetch_nand"; 266 else 267 continue; 268 SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart); 269 TsanAtomicRMW[op][i] = 270 M.getOrInsertFunction(RMWName, Attr, Ty, PtrTy, Ty, OrdTy); 271 } 272 273 SmallString<32> AtomicCASName("__tsan_atomic" + BitSizeStr + 274 "_compare_exchange_val"); 275 TsanAtomicCAS[i] = M.getOrInsertFunction(AtomicCASName, Attr, Ty, PtrTy, Ty, 276 Ty, OrdTy, OrdTy); 277 } 278 TsanVptrUpdate = 279 M.getOrInsertFunction("__tsan_vptr_update", Attr, IRB.getVoidTy(), 280 IRB.getInt8PtrTy(), IRB.getInt8PtrTy()); 281 TsanVptrLoad = M.getOrInsertFunction("__tsan_vptr_read", Attr, 282 IRB.getVoidTy(), IRB.getInt8PtrTy()); 283 TsanAtomicThreadFence = M.getOrInsertFunction("__tsan_atomic_thread_fence", 284 Attr, IRB.getVoidTy(), OrdTy); 285 TsanAtomicSignalFence = M.getOrInsertFunction("__tsan_atomic_signal_fence", 286 Attr, IRB.getVoidTy(), OrdTy); 287 288 MemmoveFn = 289 M.getOrInsertFunction("memmove", Attr, IRB.getInt8PtrTy(), 290 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); 291 MemcpyFn = 292 M.getOrInsertFunction("memcpy", Attr, IRB.getInt8PtrTy(), 293 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy); 294 MemsetFn = 295 M.getOrInsertFunction("memset", Attr, IRB.getInt8PtrTy(), 296 IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy); 297 } 298 299 static bool isVtableAccess(Instruction *I) { 300 if (MDNode *Tag = I->getMetadata(LLVMContext::MD_tbaa)) 301 return Tag->isTBAAVtableAccess(); 302 return false; 303 } 304 305 // Do not instrument known races/"benign races" that come from compiler 306 // instrumentatin. The user has no way of suppressing them. 307 static bool shouldInstrumentReadWriteFromAddress(const Module *M, Value *Addr) { 308 // Peel off GEPs and BitCasts. 309 Addr = Addr->stripInBoundsOffsets(); 310 311 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 312 if (GV->hasSection()) { 313 StringRef SectionName = GV->getSection(); 314 // Check if the global is in the PGO counters section. 315 auto OF = Triple(M->getTargetTriple()).getObjectFormat(); 316 if (SectionName.endswith( 317 getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false))) 318 return false; 319 } 320 321 // Check if the global is private gcov data. 322 if (GV->getName().startswith("__llvm_gcov") || 323 GV->getName().startswith("__llvm_gcda")) 324 return false; 325 } 326 327 // Do not instrument acesses from different address spaces; we cannot deal 328 // with them. 329 if (Addr) { 330 Type *PtrTy = cast<PointerType>(Addr->getType()->getScalarType()); 331 if (PtrTy->getPointerAddressSpace() != 0) 332 return false; 333 } 334 335 return true; 336 } 337 338 bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) { 339 // If this is a GEP, just analyze its pointer operand. 340 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr)) 341 Addr = GEP->getPointerOperand(); 342 343 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) { 344 if (GV->isConstant()) { 345 // Reads from constant globals can not race with any writes. 346 NumOmittedReadsFromConstantGlobals++; 347 return true; 348 } 349 } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) { 350 if (isVtableAccess(L)) { 351 // Reads from a vtable pointer can not race with any writes. 352 NumOmittedReadsFromVtable++; 353 return true; 354 } 355 } 356 return false; 357 } 358 359 // Instrumenting some of the accesses may be proven redundant. 360 // Currently handled: 361 // - read-before-write (within same BB, no calls between) 362 // - not captured variables 363 // 364 // We do not handle some of the patterns that should not survive 365 // after the classic compiler optimizations. 366 // E.g. two reads from the same temp should be eliminated by CSE, 367 // two writes should be eliminated by DSE, etc. 368 // 369 // 'Local' is a vector of insns within the same BB (no calls between). 370 // 'All' is a vector of insns that will be instrumented. 371 void ThreadSanitizer::chooseInstructionsToInstrument( 372 SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All, 373 const DataLayout &DL) { 374 SmallPtrSet<Value*, 8> WriteTargets; 375 // Iterate from the end. 376 for (Instruction *I : reverse(Local)) { 377 if (StoreInst *Store = dyn_cast<StoreInst>(I)) { 378 Value *Addr = Store->getPointerOperand(); 379 if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr)) 380 continue; 381 WriteTargets.insert(Addr); 382 } else { 383 LoadInst *Load = cast<LoadInst>(I); 384 Value *Addr = Load->getPointerOperand(); 385 if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr)) 386 continue; 387 if (WriteTargets.count(Addr)) { 388 // We will write to this temp, so no reason to analyze the read. 389 NumOmittedReadsBeforeWrite++; 390 continue; 391 } 392 if (addrPointsToConstantData(Addr)) { 393 // Addr points to some constant data -- it can not race with any writes. 394 continue; 395 } 396 } 397 Value *Addr = isa<StoreInst>(*I) 398 ? cast<StoreInst>(I)->getPointerOperand() 399 : cast<LoadInst>(I)->getPointerOperand(); 400 if (isa<AllocaInst>(GetUnderlyingObject(Addr, DL)) && 401 !PointerMayBeCaptured(Addr, true, true)) { 402 // The variable is addressable but not captured, so it cannot be 403 // referenced from a different thread and participate in a data race 404 // (see llvm/Analysis/CaptureTracking.h for details). 405 NumOmittedNonCaptured++; 406 continue; 407 } 408 All.push_back(I); 409 } 410 Local.clear(); 411 } 412 413 static bool isAtomic(Instruction *I) { 414 // TODO: Ask TTI whether synchronization scope is between threads. 415 if (LoadInst *LI = dyn_cast<LoadInst>(I)) 416 return LI->isAtomic() && LI->getSyncScopeID() != SyncScope::SingleThread; 417 if (StoreInst *SI = dyn_cast<StoreInst>(I)) 418 return SI->isAtomic() && SI->getSyncScopeID() != SyncScope::SingleThread; 419 if (isa<AtomicRMWInst>(I)) 420 return true; 421 if (isa<AtomicCmpXchgInst>(I)) 422 return true; 423 if (isa<FenceInst>(I)) 424 return true; 425 return false; 426 } 427 428 void ThreadSanitizer::InsertRuntimeIgnores(Function &F) { 429 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 430 IRB.CreateCall(TsanIgnoreBegin); 431 EscapeEnumerator EE(F, "tsan_ignore_cleanup", ClHandleCxxExceptions); 432 while (IRBuilder<> *AtExit = EE.Next()) { 433 AtExit->CreateCall(TsanIgnoreEnd); 434 } 435 } 436 437 bool ThreadSanitizer::sanitizeFunction(Function &F, 438 const TargetLibraryInfo &TLI) { 439 // This is required to prevent instrumenting call to __tsan_init from within 440 // the module constructor. 441 if (F.getName() == kTsanModuleCtorName) 442 return false; 443 initialize(*F.getParent()); 444 SmallVector<Instruction*, 8> AllLoadsAndStores; 445 SmallVector<Instruction*, 8> LocalLoadsAndStores; 446 SmallVector<Instruction*, 8> AtomicAccesses; 447 SmallVector<Instruction*, 8> MemIntrinCalls; 448 bool Res = false; 449 bool HasCalls = false; 450 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeThread); 451 const DataLayout &DL = F.getParent()->getDataLayout(); 452 453 // Traverse all instructions, collect loads/stores/returns, check for calls. 454 for (auto &BB : F) { 455 for (auto &Inst : BB) { 456 if (isAtomic(&Inst)) 457 AtomicAccesses.push_back(&Inst); 458 else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) 459 LocalLoadsAndStores.push_back(&Inst); 460 else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) { 461 if (CallInst *CI = dyn_cast<CallInst>(&Inst)) 462 maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI); 463 if (isa<MemIntrinsic>(Inst)) 464 MemIntrinCalls.push_back(&Inst); 465 HasCalls = true; 466 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, 467 DL); 468 } 469 } 470 chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores, DL); 471 } 472 473 // We have collected all loads and stores. 474 // FIXME: many of these accesses do not need to be checked for races 475 // (e.g. variables that do not escape, etc). 476 477 // Instrument memory accesses only if we want to report bugs in the function. 478 if (ClInstrumentMemoryAccesses && SanitizeFunction) 479 for (auto Inst : AllLoadsAndStores) { 480 Res |= instrumentLoadOrStore(Inst, DL); 481 } 482 483 // Instrument atomic memory accesses in any case (they can be used to 484 // implement synchronization). 485 if (ClInstrumentAtomics) 486 for (auto Inst : AtomicAccesses) { 487 Res |= instrumentAtomic(Inst, DL); 488 } 489 490 if (ClInstrumentMemIntrinsics && SanitizeFunction) 491 for (auto Inst : MemIntrinCalls) { 492 Res |= instrumentMemIntrinsic(Inst); 493 } 494 495 if (F.hasFnAttribute("sanitize_thread_no_checking_at_run_time")) { 496 assert(!F.hasFnAttribute(Attribute::SanitizeThread)); 497 if (HasCalls) 498 InsertRuntimeIgnores(F); 499 } 500 501 // Instrument function entry/exit points if there were instrumented accesses. 502 if ((Res || HasCalls) && ClInstrumentFuncEntryExit) { 503 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); 504 Value *ReturnAddress = IRB.CreateCall( 505 Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress), 506 IRB.getInt32(0)); 507 IRB.CreateCall(TsanFuncEntry, ReturnAddress); 508 509 EscapeEnumerator EE(F, "tsan_cleanup", ClHandleCxxExceptions); 510 while (IRBuilder<> *AtExit = EE.Next()) { 511 AtExit->CreateCall(TsanFuncExit, {}); 512 } 513 Res = true; 514 } 515 return Res; 516 } 517 518 bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I, 519 const DataLayout &DL) { 520 IRBuilder<> IRB(I); 521 bool IsWrite = isa<StoreInst>(*I); 522 Value *Addr = IsWrite 523 ? cast<StoreInst>(I)->getPointerOperand() 524 : cast<LoadInst>(I)->getPointerOperand(); 525 526 // swifterror memory addresses are mem2reg promoted by instruction selection. 527 // As such they cannot have regular uses like an instrumentation function and 528 // it makes no sense to track them as memory. 529 if (Addr->isSwiftError()) 530 return false; 531 532 int Idx = getMemoryAccessFuncIndex(Addr, DL); 533 if (Idx < 0) 534 return false; 535 if (IsWrite && isVtableAccess(I)) { 536 LLVM_DEBUG(dbgs() << " VPTR : " << *I << "\n"); 537 Value *StoredValue = cast<StoreInst>(I)->getValueOperand(); 538 // StoredValue may be a vector type if we are storing several vptrs at once. 539 // In this case, just take the first element of the vector since this is 540 // enough to find vptr races. 541 if (isa<VectorType>(StoredValue->getType())) 542 StoredValue = IRB.CreateExtractElement( 543 StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0)); 544 if (StoredValue->getType()->isIntegerTy()) 545 StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy()); 546 // Call TsanVptrUpdate. 547 IRB.CreateCall(TsanVptrUpdate, 548 {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), 549 IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())}); 550 NumInstrumentedVtableWrites++; 551 return true; 552 } 553 if (!IsWrite && isVtableAccess(I)) { 554 IRB.CreateCall(TsanVptrLoad, 555 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 556 NumInstrumentedVtableReads++; 557 return true; 558 } 559 const unsigned Alignment = IsWrite 560 ? cast<StoreInst>(I)->getAlignment() 561 : cast<LoadInst>(I)->getAlignment(); 562 Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 563 const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 564 FunctionCallee OnAccessFunc = nullptr; 565 if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0) 566 OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx]; 567 else 568 OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx]; 569 IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); 570 if (IsWrite) NumInstrumentedWrites++; 571 else NumInstrumentedReads++; 572 return true; 573 } 574 575 static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) { 576 uint32_t v = 0; 577 switch (ord) { 578 case AtomicOrdering::NotAtomic: 579 llvm_unreachable("unexpected atomic ordering!"); 580 case AtomicOrdering::Unordered: LLVM_FALLTHROUGH; 581 case AtomicOrdering::Monotonic: v = 0; break; 582 // Not specified yet: 583 // case AtomicOrdering::Consume: v = 1; break; 584 case AtomicOrdering::Acquire: v = 2; break; 585 case AtomicOrdering::Release: v = 3; break; 586 case AtomicOrdering::AcquireRelease: v = 4; break; 587 case AtomicOrdering::SequentiallyConsistent: v = 5; break; 588 } 589 return IRB->getInt32(v); 590 } 591 592 // If a memset intrinsic gets inlined by the code gen, we will miss races on it. 593 // So, we either need to ensure the intrinsic is not inlined, or instrument it. 594 // We do not instrument memset/memmove/memcpy intrinsics (too complicated), 595 // instead we simply replace them with regular function calls, which are then 596 // intercepted by the run-time. 597 // Since tsan is running after everyone else, the calls should not be 598 // replaced back with intrinsics. If that becomes wrong at some point, 599 // we will need to call e.g. __tsan_memset to avoid the intrinsics. 600 bool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) { 601 IRBuilder<> IRB(I); 602 if (MemSetInst *M = dyn_cast<MemSetInst>(I)) { 603 IRB.CreateCall( 604 MemsetFn, 605 {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 606 IRB.CreateIntCast(M->getArgOperand(1), IRB.getInt32Ty(), false), 607 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 608 I->eraseFromParent(); 609 } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(I)) { 610 IRB.CreateCall( 611 isa<MemCpyInst>(M) ? MemcpyFn : MemmoveFn, 612 {IRB.CreatePointerCast(M->getArgOperand(0), IRB.getInt8PtrTy()), 613 IRB.CreatePointerCast(M->getArgOperand(1), IRB.getInt8PtrTy()), 614 IRB.CreateIntCast(M->getArgOperand(2), IntptrTy, false)}); 615 I->eraseFromParent(); 616 } 617 return false; 618 } 619 620 // Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x 621 // standards. For background see C++11 standard. A slightly older, publicly 622 // available draft of the standard (not entirely up-to-date, but close enough 623 // for casual browsing) is available here: 624 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf 625 // The following page contains more background information: 626 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ 627 628 bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) { 629 IRBuilder<> IRB(I); 630 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 631 Value *Addr = LI->getPointerOperand(); 632 int Idx = getMemoryAccessFuncIndex(Addr, DL); 633 if (Idx < 0) 634 return false; 635 const unsigned ByteSize = 1U << Idx; 636 const unsigned BitSize = ByteSize * 8; 637 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 638 Type *PtrTy = Ty->getPointerTo(); 639 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 640 createOrdering(&IRB, LI->getOrdering())}; 641 Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); 642 Value *C = IRB.CreateCall(TsanAtomicLoad[Idx], Args); 643 Value *Cast = IRB.CreateBitOrPointerCast(C, OrigTy); 644 I->replaceAllUsesWith(Cast); 645 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 646 Value *Addr = SI->getPointerOperand(); 647 int Idx = getMemoryAccessFuncIndex(Addr, DL); 648 if (Idx < 0) 649 return false; 650 const unsigned ByteSize = 1U << Idx; 651 const unsigned BitSize = ByteSize * 8; 652 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 653 Type *PtrTy = Ty->getPointerTo(); 654 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 655 IRB.CreateBitOrPointerCast(SI->getValueOperand(), Ty), 656 createOrdering(&IRB, SI->getOrdering())}; 657 CallInst *C = CallInst::Create(TsanAtomicStore[Idx], Args); 658 ReplaceInstWithInst(I, C); 659 } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) { 660 Value *Addr = RMWI->getPointerOperand(); 661 int Idx = getMemoryAccessFuncIndex(Addr, DL); 662 if (Idx < 0) 663 return false; 664 FunctionCallee F = TsanAtomicRMW[RMWI->getOperation()][Idx]; 665 if (!F) 666 return false; 667 const unsigned ByteSize = 1U << Idx; 668 const unsigned BitSize = ByteSize * 8; 669 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 670 Type *PtrTy = Ty->getPointerTo(); 671 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 672 IRB.CreateIntCast(RMWI->getValOperand(), Ty, false), 673 createOrdering(&IRB, RMWI->getOrdering())}; 674 CallInst *C = CallInst::Create(F, Args); 675 ReplaceInstWithInst(I, C); 676 } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) { 677 Value *Addr = CASI->getPointerOperand(); 678 int Idx = getMemoryAccessFuncIndex(Addr, DL); 679 if (Idx < 0) 680 return false; 681 const unsigned ByteSize = 1U << Idx; 682 const unsigned BitSize = ByteSize * 8; 683 Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize); 684 Type *PtrTy = Ty->getPointerTo(); 685 Value *CmpOperand = 686 IRB.CreateBitOrPointerCast(CASI->getCompareOperand(), Ty); 687 Value *NewOperand = 688 IRB.CreateBitOrPointerCast(CASI->getNewValOperand(), Ty); 689 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), 690 CmpOperand, 691 NewOperand, 692 createOrdering(&IRB, CASI->getSuccessOrdering()), 693 createOrdering(&IRB, CASI->getFailureOrdering())}; 694 CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args); 695 Value *Success = IRB.CreateICmpEQ(C, CmpOperand); 696 Value *OldVal = C; 697 Type *OrigOldValTy = CASI->getNewValOperand()->getType(); 698 if (Ty != OrigOldValTy) { 699 // The value is a pointer, so we need to cast the return value. 700 OldVal = IRB.CreateIntToPtr(C, OrigOldValTy); 701 } 702 703 Value *Res = 704 IRB.CreateInsertValue(UndefValue::get(CASI->getType()), OldVal, 0); 705 Res = IRB.CreateInsertValue(Res, Success, 1); 706 707 I->replaceAllUsesWith(Res); 708 I->eraseFromParent(); 709 } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) { 710 Value *Args[] = {createOrdering(&IRB, FI->getOrdering())}; 711 FunctionCallee F = FI->getSyncScopeID() == SyncScope::SingleThread 712 ? TsanAtomicSignalFence 713 : TsanAtomicThreadFence; 714 CallInst *C = CallInst::Create(F, Args); 715 ReplaceInstWithInst(I, C); 716 } 717 return true; 718 } 719 720 int ThreadSanitizer::getMemoryAccessFuncIndex(Value *Addr, 721 const DataLayout &DL) { 722 Type *OrigPtrTy = Addr->getType(); 723 Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); 724 assert(OrigTy->isSized()); 725 uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); 726 if (TypeSize != 8 && TypeSize != 16 && 727 TypeSize != 32 && TypeSize != 64 && TypeSize != 128) { 728 NumAccessesWithBadSize++; 729 // Ignore all unusual sizes. 730 return -1; 731 } 732 size_t Idx = countTrailingZeros(TypeSize / 8); 733 assert(Idx < kNumberOfAccessSizes); 734 return Idx; 735 } 736