1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling. 10 // It also builds the data structures and initialization code needed for 11 // updating execution counts and emitting the profile at runtime. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "llvm/Transforms/Instrumentation/InstrProfiling.h" 16 #include "llvm/ADT/ArrayRef.h" 17 #include "llvm/ADT/SmallVector.h" 18 #include "llvm/ADT/StringRef.h" 19 #include "llvm/ADT/Triple.h" 20 #include "llvm/ADT/Twine.h" 21 #include "llvm/Analysis/BlockFrequencyInfo.h" 22 #include "llvm/Analysis/BranchProbabilityInfo.h" 23 #include "llvm/Analysis/LoopInfo.h" 24 #include "llvm/Analysis/TargetLibraryInfo.h" 25 #include "llvm/IR/Attributes.h" 26 #include "llvm/IR/BasicBlock.h" 27 #include "llvm/IR/Constant.h" 28 #include "llvm/IR/Constants.h" 29 #include "llvm/IR/DerivedTypes.h" 30 #include "llvm/IR/Dominators.h" 31 #include "llvm/IR/Function.h" 32 #include "llvm/IR/GlobalValue.h" 33 #include "llvm/IR/GlobalVariable.h" 34 #include "llvm/IR/IRBuilder.h" 35 #include "llvm/IR/Instruction.h" 36 #include "llvm/IR/Instructions.h" 37 #include "llvm/IR/IntrinsicInst.h" 38 #include "llvm/IR/Module.h" 39 #include "llvm/IR/Type.h" 40 #include "llvm/InitializePasses.h" 41 #include "llvm/Pass.h" 42 #include "llvm/ProfileData/InstrProf.h" 43 #include "llvm/Support/Casting.h" 44 #include "llvm/Support/CommandLine.h" 45 #include "llvm/Support/Error.h" 46 #include "llvm/Support/ErrorHandling.h" 47 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 48 #include "llvm/Transforms/Utils/ModuleUtils.h" 49 #include "llvm/Transforms/Utils/SSAUpdater.h" 50 #include <algorithm> 51 #include <cassert> 52 #include <cstddef> 53 #include <cstdint> 54 #include <string> 55 56 using namespace llvm; 57 58 #define DEBUG_TYPE "instrprof" 59 60 namespace { 61 62 cl::opt<bool> DoHashBasedCounterSplit( 63 "hash-based-counter-split", 64 cl::desc("Rename counter variable of a comdat function based on cfg hash"), 65 cl::init(true)); 66 67 cl::opt<bool> RuntimeCounterRelocation( 68 "runtime-counter-relocation", 69 cl::desc("Enable relocating counters at runtime."), 70 cl::init(false)); 71 72 cl::opt<bool> ValueProfileStaticAlloc( 73 "vp-static-alloc", 74 cl::desc("Do static counter allocation for value profiler"), 75 cl::init(true)); 76 77 cl::opt<double> NumCountersPerValueSite( 78 "vp-counters-per-site", 79 cl::desc("The average number of profile counters allocated " 80 "per value profiling site."), 81 // This is set to a very small value because in real programs, only 82 // a very small percentage of value sites have non-zero targets, e.g, 1/30. 83 // For those sites with non-zero profile, the average number of targets 84 // is usually smaller than 2. 85 cl::init(1.0)); 86 87 cl::opt<bool> AtomicCounterUpdateAll( 88 "instrprof-atomic-counter-update-all", cl::ZeroOrMore, 89 cl::desc("Make all profile counter updates atomic (for testing only)"), 90 cl::init(false)); 91 92 cl::opt<bool> AtomicCounterUpdatePromoted( 93 "atomic-counter-update-promoted", cl::ZeroOrMore, 94 cl::desc("Do counter update using atomic fetch add " 95 " for promoted counters only"), 96 cl::init(false)); 97 98 cl::opt<bool> AtomicFirstCounter( 99 "atomic-first-counter", cl::ZeroOrMore, 100 cl::desc("Use atomic fetch add for first counter in a function (usually " 101 "the entry counter)"), 102 cl::init(false)); 103 104 // If the option is not specified, the default behavior about whether 105 // counter promotion is done depends on how instrumentaiton lowering 106 // pipeline is setup, i.e., the default value of true of this option 107 // does not mean the promotion will be done by default. Explicitly 108 // setting this option can override the default behavior. 109 cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore, 110 cl::desc("Do counter register promotion"), 111 cl::init(false)); 112 cl::opt<unsigned> MaxNumOfPromotionsPerLoop( 113 cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20), 114 cl::desc("Max number counter promotions per loop to avoid" 115 " increasing register pressure too much")); 116 117 // A debug option 118 cl::opt<int> 119 MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1), 120 cl::desc("Max number of allowed counter promotions")); 121 122 cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting( 123 cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3), 124 cl::desc("The max number of exiting blocks of a loop to allow " 125 " speculative counter promotion")); 126 127 cl::opt<bool> SpeculativeCounterPromotionToLoop( 128 cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false), 129 cl::desc("When the option is false, if the target block is in a loop, " 130 "the promotion will be disallowed unless the promoted counter " 131 " update can be further/iteratively promoted into an acyclic " 132 " region.")); 133 134 cl::opt<bool> IterativeCounterPromotion( 135 cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true), 136 cl::desc("Allow counter promotion across the whole loop nest.")); 137 138 cl::opt<bool> SkipRetExitBlock( 139 cl::ZeroOrMore, "skip-ret-exit-block", cl::init(true), 140 cl::desc("Suppress counter promotion if exit blocks contain ret.")); 141 142 class InstrProfilingLegacyPass : public ModulePass { 143 InstrProfiling InstrProf; 144 145 public: 146 static char ID; 147 148 InstrProfilingLegacyPass() : ModulePass(ID) {} 149 InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false) 150 : ModulePass(ID), InstrProf(Options, IsCS) { 151 initializeInstrProfilingLegacyPassPass(*PassRegistry::getPassRegistry()); 152 } 153 154 StringRef getPassName() const override { 155 return "Frontend instrumentation-based coverage lowering"; 156 } 157 158 bool runOnModule(Module &M) override { 159 auto GetTLI = [this](Function &F) -> TargetLibraryInfo & { 160 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 161 }; 162 return InstrProf.run(M, GetTLI); 163 } 164 165 void getAnalysisUsage(AnalysisUsage &AU) const override { 166 AU.setPreservesCFG(); 167 AU.addRequired<TargetLibraryInfoWrapperPass>(); 168 } 169 }; 170 171 /// 172 /// A helper class to promote one counter RMW operation in the loop 173 /// into register update. 174 /// 175 /// RWM update for the counter will be sinked out of the loop after 176 /// the transformation. 177 /// 178 class PGOCounterPromoterHelper : public LoadAndStorePromoter { 179 public: 180 PGOCounterPromoterHelper( 181 Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init, 182 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks, 183 ArrayRef<Instruction *> InsertPts, 184 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands, 185 LoopInfo &LI) 186 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks), 187 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) { 188 assert(isa<LoadInst>(L)); 189 assert(isa<StoreInst>(S)); 190 SSA.AddAvailableValue(PH, Init); 191 } 192 193 void doExtraRewritesBeforeFinalDeletion() override { 194 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) { 195 BasicBlock *ExitBlock = ExitBlocks[i]; 196 Instruction *InsertPos = InsertPts[i]; 197 // Get LiveIn value into the ExitBlock. If there are multiple 198 // predecessors, the value is defined by a PHI node in this 199 // block. 200 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock); 201 Value *Addr = cast<StoreInst>(Store)->getPointerOperand(); 202 Type *Ty = LiveInValue->getType(); 203 IRBuilder<> Builder(InsertPos); 204 if (AtomicCounterUpdatePromoted) 205 // automic update currently can only be promoted across the current 206 // loop, not the whole loop nest. 207 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue, 208 MaybeAlign(), 209 AtomicOrdering::SequentiallyConsistent); 210 else { 211 LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted"); 212 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue); 213 auto *NewStore = Builder.CreateStore(NewVal, Addr); 214 215 // Now update the parent loop's candidate list: 216 if (IterativeCounterPromotion) { 217 auto *TargetLoop = LI.getLoopFor(ExitBlock); 218 if (TargetLoop) 219 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore); 220 } 221 } 222 } 223 } 224 225 private: 226 Instruction *Store; 227 ArrayRef<BasicBlock *> ExitBlocks; 228 ArrayRef<Instruction *> InsertPts; 229 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates; 230 LoopInfo &LI; 231 }; 232 233 /// A helper class to do register promotion for all profile counter 234 /// updates in a loop. 235 /// 236 class PGOCounterPromoter { 237 public: 238 PGOCounterPromoter( 239 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands, 240 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI) 241 : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop), 242 LI(LI), BFI(BFI) { 243 244 // Skip collection of ExitBlocks and InsertPts for loops that will not be 245 // able to have counters promoted. 246 SmallVector<BasicBlock *, 8> LoopExitBlocks; 247 SmallPtrSet<BasicBlock *, 8> BlockSet; 248 249 L.getExitBlocks(LoopExitBlocks); 250 if (!isPromotionPossible(&L, LoopExitBlocks)) 251 return; 252 253 for (BasicBlock *ExitBlock : LoopExitBlocks) { 254 if (BlockSet.insert(ExitBlock).second) { 255 ExitBlocks.push_back(ExitBlock); 256 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt()); 257 } 258 } 259 } 260 261 bool run(int64_t *NumPromoted) { 262 // Skip 'infinite' loops: 263 if (ExitBlocks.size() == 0) 264 return false; 265 266 // Skip if any of the ExitBlocks contains a ret instruction. 267 // This is to prevent dumping of incomplete profile -- if the 268 // the loop is a long running loop and dump is called in the middle 269 // of the loop, the result profile is incomplete. 270 // FIXME: add other heuristics to detect long running loops. 271 if (SkipRetExitBlock) { 272 for (auto BB : ExitBlocks) 273 if (isa<ReturnInst>(BB->getTerminator())) 274 return false; 275 } 276 277 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L); 278 if (MaxProm == 0) 279 return false; 280 281 unsigned Promoted = 0; 282 for (auto &Cand : LoopToCandidates[&L]) { 283 284 SmallVector<PHINode *, 4> NewPHIs; 285 SSAUpdater SSA(&NewPHIs); 286 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0); 287 288 // If BFI is set, we will use it to guide the promotions. 289 if (BFI) { 290 auto *BB = Cand.first->getParent(); 291 auto InstrCount = BFI->getBlockProfileCount(BB); 292 if (!InstrCount) 293 continue; 294 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader()); 295 // If the average loop trip count is not greater than 1.5, we skip 296 // promotion. 297 if (PreheaderCount && 298 (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2)) 299 continue; 300 } 301 302 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal, 303 L.getLoopPreheader(), ExitBlocks, 304 InsertPts, LoopToCandidates, LI); 305 Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second})); 306 Promoted++; 307 if (Promoted >= MaxProm) 308 break; 309 310 (*NumPromoted)++; 311 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions) 312 break; 313 } 314 315 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth=" 316 << L.getLoopDepth() << ")\n"); 317 return Promoted != 0; 318 } 319 320 private: 321 bool allowSpeculativeCounterPromotion(Loop *LP) { 322 SmallVector<BasicBlock *, 8> ExitingBlocks; 323 L.getExitingBlocks(ExitingBlocks); 324 // Not considierered speculative. 325 if (ExitingBlocks.size() == 1) 326 return true; 327 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting) 328 return false; 329 return true; 330 } 331 332 // Check whether the loop satisfies the basic conditions needed to perform 333 // Counter Promotions. 334 bool isPromotionPossible(Loop *LP, 335 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) { 336 // We can't insert into a catchswitch. 337 if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) { 338 return isa<CatchSwitchInst>(Exit->getTerminator()); 339 })) 340 return false; 341 342 if (!LP->hasDedicatedExits()) 343 return false; 344 345 BasicBlock *PH = LP->getLoopPreheader(); 346 if (!PH) 347 return false; 348 349 return true; 350 } 351 352 // Returns the max number of Counter Promotions for LP. 353 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) { 354 SmallVector<BasicBlock *, 8> LoopExitBlocks; 355 LP->getExitBlocks(LoopExitBlocks); 356 if (!isPromotionPossible(LP, LoopExitBlocks)) 357 return 0; 358 359 SmallVector<BasicBlock *, 8> ExitingBlocks; 360 LP->getExitingBlocks(ExitingBlocks); 361 362 // If BFI is set, we do more aggressive promotions based on BFI. 363 if (BFI) 364 return (unsigned)-1; 365 366 // Not considierered speculative. 367 if (ExitingBlocks.size() == 1) 368 return MaxNumOfPromotionsPerLoop; 369 370 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting) 371 return 0; 372 373 // Whether the target block is in a loop does not matter: 374 if (SpeculativeCounterPromotionToLoop) 375 return MaxNumOfPromotionsPerLoop; 376 377 // Now check the target block: 378 unsigned MaxProm = MaxNumOfPromotionsPerLoop; 379 for (auto *TargetBlock : LoopExitBlocks) { 380 auto *TargetLoop = LI.getLoopFor(TargetBlock); 381 if (!TargetLoop) 382 continue; 383 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop); 384 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size(); 385 MaxProm = 386 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) - 387 PendingCandsInTarget); 388 } 389 return MaxProm; 390 } 391 392 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates; 393 SmallVector<BasicBlock *, 8> ExitBlocks; 394 SmallVector<Instruction *, 8> InsertPts; 395 Loop &L; 396 LoopInfo &LI; 397 BlockFrequencyInfo *BFI; 398 }; 399 400 enum class ValueProfilingCallType { 401 // Individual values are tracked. Currently used for indiret call target 402 // profiling. 403 Default, 404 405 // MemOp: the memop size value profiling. 406 MemOp 407 }; 408 409 } // end anonymous namespace 410 411 PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) { 412 FunctionAnalysisManager &FAM = 413 AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 414 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & { 415 return FAM.getResult<TargetLibraryAnalysis>(F); 416 }; 417 if (!run(M, GetTLI)) 418 return PreservedAnalyses::all(); 419 420 return PreservedAnalyses::none(); 421 } 422 423 char InstrProfilingLegacyPass::ID = 0; 424 INITIALIZE_PASS_BEGIN( 425 InstrProfilingLegacyPass, "instrprof", 426 "Frontend instrumentation-based coverage lowering.", false, false) 427 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 428 INITIALIZE_PASS_END( 429 InstrProfilingLegacyPass, "instrprof", 430 "Frontend instrumentation-based coverage lowering.", false, false) 431 432 ModulePass * 433 llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options, 434 bool IsCS) { 435 return new InstrProfilingLegacyPass(Options, IsCS); 436 } 437 438 static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) { 439 InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr); 440 if (Inc) 441 return Inc; 442 return dyn_cast<InstrProfIncrementInst>(Instr); 443 } 444 445 bool InstrProfiling::lowerIntrinsics(Function *F) { 446 bool MadeChange = false; 447 PromotionCandidates.clear(); 448 for (BasicBlock &BB : *F) { 449 for (auto I = BB.begin(), E = BB.end(); I != E;) { 450 auto Instr = I++; 451 InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr); 452 if (Inc) { 453 lowerIncrement(Inc); 454 MadeChange = true; 455 } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) { 456 lowerValueProfileInst(Ind); 457 MadeChange = true; 458 } 459 } 460 } 461 462 if (!MadeChange) 463 return false; 464 465 promoteCounterLoadStores(F); 466 return true; 467 } 468 469 bool InstrProfiling::isRuntimeCounterRelocationEnabled() const { 470 // Mach-O don't support weak external references. 471 if (TT.isOSBinFormatMachO()) 472 return false; 473 474 if (RuntimeCounterRelocation.getNumOccurrences() > 0) 475 return RuntimeCounterRelocation; 476 477 // Fuchsia uses runtime counter relocation by default. 478 return TT.isOSFuchsia(); 479 } 480 481 bool InstrProfiling::isCounterPromotionEnabled() const { 482 if (DoCounterPromotion.getNumOccurrences() > 0) 483 return DoCounterPromotion; 484 485 return Options.DoCounterPromotion; 486 } 487 488 void InstrProfiling::promoteCounterLoadStores(Function *F) { 489 if (!isCounterPromotionEnabled()) 490 return; 491 492 DominatorTree DT(*F); 493 LoopInfo LI(DT); 494 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates; 495 496 std::unique_ptr<BlockFrequencyInfo> BFI; 497 if (Options.UseBFIInPromotion) { 498 std::unique_ptr<BranchProbabilityInfo> BPI; 499 BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F))); 500 BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI)); 501 } 502 503 for (const auto &LoadStore : PromotionCandidates) { 504 auto *CounterLoad = LoadStore.first; 505 auto *CounterStore = LoadStore.second; 506 BasicBlock *BB = CounterLoad->getParent(); 507 Loop *ParentLoop = LI.getLoopFor(BB); 508 if (!ParentLoop) 509 continue; 510 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore); 511 } 512 513 SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder(); 514 515 // Do a post-order traversal of the loops so that counter updates can be 516 // iteratively hoisted outside the loop nest. 517 for (auto *Loop : llvm::reverse(Loops)) { 518 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get()); 519 Promoter.run(&TotalCountersPromoted); 520 } 521 } 522 523 /// Check if the module contains uses of any profiling intrinsics. 524 static bool containsProfilingIntrinsics(Module &M) { 525 if (auto *F = M.getFunction( 526 Intrinsic::getName(llvm::Intrinsic::instrprof_increment))) 527 if (!F->use_empty()) 528 return true; 529 if (auto *F = M.getFunction( 530 Intrinsic::getName(llvm::Intrinsic::instrprof_increment_step))) 531 if (!F->use_empty()) 532 return true; 533 if (auto *F = M.getFunction( 534 Intrinsic::getName(llvm::Intrinsic::instrprof_value_profile))) 535 if (!F->use_empty()) 536 return true; 537 return false; 538 } 539 540 bool InstrProfiling::run( 541 Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) { 542 this->M = &M; 543 this->GetTLI = std::move(GetTLI); 544 NamesVar = nullptr; 545 NamesSize = 0; 546 ProfileDataMap.clear(); 547 CompilerUsedVars.clear(); 548 UsedVars.clear(); 549 TT = Triple(M.getTargetTriple()); 550 551 // Emit the runtime hook even if no counters are present. 552 bool MadeChange = emitRuntimeHook(); 553 554 // Improve compile time by avoiding linear scans when there is no work. 555 GlobalVariable *CoverageNamesVar = 556 M.getNamedGlobal(getCoverageUnusedNamesVarName()); 557 if (!containsProfilingIntrinsics(M) && !CoverageNamesVar) 558 return MadeChange; 559 560 // We did not know how many value sites there would be inside 561 // the instrumented function. This is counting the number of instrumented 562 // target value sites to enter it as field in the profile data variable. 563 for (Function &F : M) { 564 InstrProfIncrementInst *FirstProfIncInst = nullptr; 565 for (BasicBlock &BB : F) 566 for (auto I = BB.begin(), E = BB.end(); I != E; I++) 567 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I)) 568 computeNumValueSiteCounts(Ind); 569 else if (FirstProfIncInst == nullptr) 570 FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I); 571 572 // Value profiling intrinsic lowering requires per-function profile data 573 // variable to be created first. 574 if (FirstProfIncInst != nullptr) 575 static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst)); 576 } 577 578 for (Function &F : M) 579 MadeChange |= lowerIntrinsics(&F); 580 581 if (CoverageNamesVar) { 582 lowerCoverageData(CoverageNamesVar); 583 MadeChange = true; 584 } 585 586 if (!MadeChange) 587 return false; 588 589 emitVNodes(); 590 emitNameData(); 591 emitRegistration(); 592 emitUses(); 593 emitInitialization(); 594 return true; 595 } 596 597 static FunctionCallee getOrInsertValueProfilingCall( 598 Module &M, const TargetLibraryInfo &TLI, 599 ValueProfilingCallType CallType = ValueProfilingCallType::Default) { 600 LLVMContext &Ctx = M.getContext(); 601 auto *ReturnTy = Type::getVoidTy(M.getContext()); 602 603 AttributeList AL; 604 if (auto AK = TLI.getExtAttrForI32Param(false)) 605 AL = AL.addParamAttribute(M.getContext(), 2, AK); 606 607 assert((CallType == ValueProfilingCallType::Default || 608 CallType == ValueProfilingCallType::MemOp) && 609 "Must be Default or MemOp"); 610 Type *ParamTypes[] = { 611 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType 612 #include "llvm/ProfileData/InstrProfData.inc" 613 }; 614 auto *ValueProfilingCallTy = 615 FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false); 616 StringRef FuncName = CallType == ValueProfilingCallType::Default 617 ? getInstrProfValueProfFuncName() 618 : getInstrProfValueProfMemOpFuncName(); 619 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL); 620 } 621 622 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) { 623 GlobalVariable *Name = Ind->getName(); 624 uint64_t ValueKind = Ind->getValueKind()->getZExtValue(); 625 uint64_t Index = Ind->getIndex()->getZExtValue(); 626 auto It = ProfileDataMap.find(Name); 627 if (It == ProfileDataMap.end()) { 628 PerFunctionProfileData PD; 629 PD.NumValueSites[ValueKind] = Index + 1; 630 ProfileDataMap[Name] = PD; 631 } else if (It->second.NumValueSites[ValueKind] <= Index) 632 It->second.NumValueSites[ValueKind] = Index + 1; 633 } 634 635 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { 636 GlobalVariable *Name = Ind->getName(); 637 auto It = ProfileDataMap.find(Name); 638 assert(It != ProfileDataMap.end() && It->second.DataVar && 639 "value profiling detected in function with no counter incerement"); 640 641 GlobalVariable *DataVar = It->second.DataVar; 642 uint64_t ValueKind = Ind->getValueKind()->getZExtValue(); 643 uint64_t Index = Ind->getIndex()->getZExtValue(); 644 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind) 645 Index += It->second.NumValueSites[Kind]; 646 647 IRBuilder<> Builder(Ind); 648 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() == 649 llvm::InstrProfValueKind::IPVK_MemOPSize); 650 CallInst *Call = nullptr; 651 auto *TLI = &GetTLI(*Ind->getFunction()); 652 653 // To support value profiling calls within Windows exception handlers, funclet 654 // information contained within operand bundles needs to be copied over to 655 // the library call. This is required for the IR to be processed by the 656 // WinEHPrepare pass. 657 SmallVector<OperandBundleDef, 1> OpBundles; 658 Ind->getOperandBundlesAsDefs(OpBundles); 659 if (!IsMemOpSize) { 660 Value *Args[3] = {Ind->getTargetValue(), 661 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()), 662 Builder.getInt32(Index)}; 663 Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args, 664 OpBundles); 665 } else { 666 Value *Args[3] = {Ind->getTargetValue(), 667 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()), 668 Builder.getInt32(Index)}; 669 Call = Builder.CreateCall( 670 getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp), 671 Args, OpBundles); 672 } 673 if (auto AK = TLI->getExtAttrForI32Param(false)) 674 Call->addParamAttr(2, AK); 675 Ind->replaceAllUsesWith(Call); 676 Ind->eraseFromParent(); 677 } 678 679 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) { 680 GlobalVariable *Counters = getOrCreateRegionCounters(Inc); 681 682 IRBuilder<> Builder(Inc); 683 uint64_t Index = Inc->getIndex()->getZExtValue(); 684 Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(), 685 Counters, 0, Index); 686 687 if (isRuntimeCounterRelocationEnabled()) { 688 Type *Int64Ty = Type::getInt64Ty(M->getContext()); 689 Type *Int64PtrTy = Type::getInt64PtrTy(M->getContext()); 690 Function *Fn = Inc->getParent()->getParent(); 691 Instruction &I = Fn->getEntryBlock().front(); 692 LoadInst *LI = dyn_cast<LoadInst>(&I); 693 if (!LI) { 694 IRBuilder<> Builder(&I); 695 Type *Int64Ty = Type::getInt64Ty(M->getContext()); 696 GlobalVariable *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName()); 697 if (!Bias) { 698 // Compiler must define this variable when runtime counter relocation 699 // is being used. Runtime has a weak external reference that is used 700 // to check whether that's the case or not. 701 Bias = new GlobalVariable(*M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage, 702 Constant::getNullValue(Int64Ty), 703 getInstrProfCounterBiasVarName()); 704 Bias->setVisibility(GlobalVariable::HiddenVisibility); 705 // A definition that's weak (linkonce_odr) without being in a COMDAT 706 // section wouldn't lead to link errors, but it would lead to a dead 707 // data word from every TU but one. Putting it in COMDAT ensures there 708 // will be exactly one data slot in the link. 709 if (TT.supportsCOMDAT()) 710 Bias->setComdat(M->getOrInsertComdat(Bias->getName())); 711 } 712 LI = Builder.CreateLoad(Int64Ty, Bias); 713 } 714 auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), LI); 715 Addr = Builder.CreateIntToPtr(Add, Int64PtrTy); 716 } 717 718 if (Options.Atomic || AtomicCounterUpdateAll || 719 (Index == 0 && AtomicFirstCounter)) { 720 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(), 721 MaybeAlign(), AtomicOrdering::Monotonic); 722 } else { 723 Value *IncStep = Inc->getStep(); 724 Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount"); 725 auto *Count = Builder.CreateAdd(Load, Inc->getStep()); 726 auto *Store = Builder.CreateStore(Count, Addr); 727 if (isCounterPromotionEnabled()) 728 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store); 729 } 730 Inc->eraseFromParent(); 731 } 732 733 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) { 734 ConstantArray *Names = 735 cast<ConstantArray>(CoverageNamesVar->getInitializer()); 736 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) { 737 Constant *NC = Names->getOperand(I); 738 Value *V = NC->stripPointerCasts(); 739 assert(isa<GlobalVariable>(V) && "Missing reference to function name"); 740 GlobalVariable *Name = cast<GlobalVariable>(V); 741 742 Name->setLinkage(GlobalValue::PrivateLinkage); 743 ReferencedNames.push_back(Name); 744 NC->dropAllReferences(); 745 } 746 CoverageNamesVar->eraseFromParent(); 747 } 748 749 /// Get the name of a profiling variable for a particular function. 750 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) { 751 StringRef NamePrefix = getInstrProfNameVarPrefix(); 752 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size()); 753 Function *F = Inc->getParent()->getParent(); 754 Module *M = F->getParent(); 755 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) || 756 !canRenameComdatFunc(*F)) 757 return (Prefix + Name).str(); 758 uint64_t FuncHash = Inc->getHash()->getZExtValue(); 759 SmallVector<char, 24> HashPostfix; 760 if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix))) 761 return (Prefix + Name).str(); 762 return (Prefix + Name + "." + Twine(FuncHash)).str(); 763 } 764 765 static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) { 766 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag)); 767 if (!MD) 768 return 0; 769 770 // If the flag is a ConstantAsMetadata, it should be an integer representable 771 // in 64-bits. 772 return cast<ConstantInt>(MD->getValue())->getZExtValue(); 773 } 774 775 static bool enablesValueProfiling(const Module &M) { 776 return isIRPGOFlagSet(&M) || 777 getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0; 778 } 779 780 // Conservatively returns true if data variables may be referenced by code. 781 static bool profDataReferencedByCode(const Module &M) { 782 return enablesValueProfiling(M); 783 } 784 785 static inline bool shouldRecordFunctionAddr(Function *F) { 786 // Only record function addresses if IR PGO is enabled or if clang value 787 // profiling is enabled. Recording function addresses greatly increases object 788 // file size, because it prevents the inliner from deleting functions that 789 // have been inlined everywhere. 790 if (!profDataReferencedByCode(*F->getParent())) 791 return false; 792 793 // Check the linkage 794 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage(); 795 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() && 796 !HasAvailableExternallyLinkage) 797 return true; 798 799 // A function marked 'alwaysinline' with available_externally linkage can't 800 // have its address taken. Doing so would create an undefined external ref to 801 // the function, which would fail to link. 802 if (HasAvailableExternallyLinkage && 803 F->hasFnAttribute(Attribute::AlwaysInline)) 804 return false; 805 806 // Prohibit function address recording if the function is both internal and 807 // COMDAT. This avoids the profile data variable referencing internal symbols 808 // in COMDAT. 809 if (F->hasLocalLinkage() && F->hasComdat()) 810 return false; 811 812 // Check uses of this function for other than direct calls or invokes to it. 813 // Inline virtual functions have linkeOnceODR linkage. When a key method 814 // exists, the vtable will only be emitted in the TU where the key method 815 // is defined. In a TU where vtable is not available, the function won't 816 // be 'addresstaken'. If its address is not recorded here, the profile data 817 // with missing address may be picked by the linker leading to missing 818 // indirect call target info. 819 return F->hasAddressTaken() || F->hasLinkOnceLinkage(); 820 } 821 822 static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { 823 // Don't do this for Darwin. compiler-rt uses linker magic. 824 if (TT.isOSDarwin()) 825 return false; 826 // Use linker script magic to get data/cnts/name start/end. 827 if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() || 828 TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS4CPU() || 829 TT.isOSWindows()) 830 return false; 831 832 return true; 833 } 834 835 GlobalVariable * 836 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) { 837 GlobalVariable *NamePtr = Inc->getName(); 838 auto It = ProfileDataMap.find(NamePtr); 839 PerFunctionProfileData PD; 840 if (It != ProfileDataMap.end()) { 841 if (It->second.RegionCounters) 842 return It->second.RegionCounters; 843 PD = It->second; 844 } 845 846 // Match the linkage and visibility of the name global. 847 Function *Fn = Inc->getParent()->getParent(); 848 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); 849 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility(); 850 851 // Move the name variable to the right section. Place them in a COMDAT group 852 // if the associated function is a COMDAT. This will make sure that only one 853 // copy of counters of the COMDAT function will be emitted after linking. Keep 854 // in mind that this pass may run before the inliner, so we need to create a 855 // new comdat group for the counters and profiling data. If we use the comdat 856 // of the parent function, that will result in relocations against discarded 857 // sections. 858 // 859 // If the data variable is referenced by code, counters and data have to be 860 // in different comdats for COFF because the Visual C++ linker will report 861 // duplicate symbol errors if there are multiple external symbols with the 862 // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE. 863 // 864 // For ELF, when not using COMDAT, put counters, data and values into a 865 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This 866 // allows -z start-stop-gc to discard the entire group when the function is 867 // discarded. 868 bool DataReferencedByCode = profDataReferencedByCode(*M); 869 bool NeedComdat = needsComdatForCounter(*Fn, *M); 870 std::string CntsVarName = getVarName(Inc, getInstrProfCountersVarPrefix()); 871 std::string DataVarName = getVarName(Inc, getInstrProfDataVarPrefix()); 872 auto MaybeSetComdat = [&](GlobalVariable *GV) { 873 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); 874 if (UseComdat) { 875 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode 876 ? GV->getName() 877 : CntsVarName; 878 Comdat *C = M->getOrInsertComdat(GroupName); 879 if (!NeedComdat) 880 C->setSelectionKind(Comdat::NoDeduplicate); 881 GV->setComdat(C); 882 } 883 }; 884 885 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); 886 LLVMContext &Ctx = M->getContext(); 887 ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); 888 889 // Create the counters variable. 890 auto *CounterPtr = 891 new GlobalVariable(*M, CounterTy, false, Linkage, 892 Constant::getNullValue(CounterTy), CntsVarName); 893 CounterPtr->setVisibility(Visibility); 894 CounterPtr->setSection( 895 getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat())); 896 CounterPtr->setAlignment(Align(8)); 897 MaybeSetComdat(CounterPtr); 898 CounterPtr->setLinkage(Linkage); 899 900 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); 901 // Allocate statically the array of pointers to value profile nodes for 902 // the current function. 903 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy); 904 uint64_t NS = 0; 905 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 906 NS += PD.NumValueSites[Kind]; 907 if (NS > 0 && ValueProfileStaticAlloc && 908 !needsRuntimeRegistrationOfSectionRange(TT)) { 909 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS); 910 auto *ValuesVar = new GlobalVariable( 911 *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy), 912 getVarName(Inc, getInstrProfValuesVarPrefix())); 913 ValuesVar->setVisibility(Visibility); 914 ValuesVar->setSection( 915 getInstrProfSectionName(IPSK_vals, TT.getObjectFormat())); 916 ValuesVar->setAlignment(Align(8)); 917 MaybeSetComdat(ValuesVar); 918 ValuesPtrExpr = 919 ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx)); 920 } 921 922 // Create data variable. 923 auto *Int16Ty = Type::getInt16Ty(Ctx); 924 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1); 925 Type *DataTypes[] = { 926 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType, 927 #include "llvm/ProfileData/InstrProfData.inc" 928 }; 929 auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes)); 930 931 Constant *FunctionAddr = shouldRecordFunctionAddr(Fn) 932 ? ConstantExpr::getBitCast(Fn, Int8PtrTy) 933 : ConstantPointerNull::get(Int8PtrTy); 934 935 Constant *Int16ArrayVals[IPVK_Last + 1]; 936 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 937 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]); 938 939 Constant *DataVals[] = { 940 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init, 941 #include "llvm/ProfileData/InstrProfData.inc" 942 }; 943 // If the data variable is not referenced by code (if we don't emit 944 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the 945 // data variable live under linker GC, the data variable can be private. This 946 // optimization applies to ELF. 947 // 948 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode 949 // to be false. 950 if (NS == 0 && (TT.isOSBinFormatELF() || 951 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) { 952 Linkage = GlobalValue::PrivateLinkage; 953 Visibility = GlobalValue::DefaultVisibility; 954 } 955 auto *Data = 956 new GlobalVariable(*M, DataTy, false, Linkage, 957 ConstantStruct::get(DataTy, DataVals), DataVarName); 958 Data->setVisibility(Visibility); 959 Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat())); 960 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT)); 961 MaybeSetComdat(Data); 962 Data->setLinkage(Linkage); 963 964 PD.RegionCounters = CounterPtr; 965 PD.DataVar = Data; 966 ProfileDataMap[NamePtr] = PD; 967 968 // Mark the data variable as used so that it isn't stripped out. 969 CompilerUsedVars.push_back(Data); 970 // Now that the linkage set by the FE has been passed to the data and counter 971 // variables, reset Name variable's linkage and visibility to private so that 972 // it can be removed later by the compiler. 973 NamePtr->setLinkage(GlobalValue::PrivateLinkage); 974 // Collect the referenced names to be used by emitNameData. 975 ReferencedNames.push_back(NamePtr); 976 977 return CounterPtr; 978 } 979 980 void InstrProfiling::emitVNodes() { 981 if (!ValueProfileStaticAlloc) 982 return; 983 984 // For now only support this on platforms that do 985 // not require runtime registration to discover 986 // named section start/end. 987 if (needsRuntimeRegistrationOfSectionRange(TT)) 988 return; 989 990 size_t TotalNS = 0; 991 for (auto &PD : ProfileDataMap) { 992 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 993 TotalNS += PD.second.NumValueSites[Kind]; 994 } 995 996 if (!TotalNS) 997 return; 998 999 uint64_t NumCounters = TotalNS * NumCountersPerValueSite; 1000 // Heuristic for small programs with very few total value sites. 1001 // The default value of vp-counters-per-site is chosen based on 1002 // the observation that large apps usually have a low percentage 1003 // of value sites that actually have any profile data, and thus 1004 // the average number of counters per site is low. For small 1005 // apps with very few sites, this may not be true. Bump up the 1006 // number of counters in this case. 1007 #define INSTR_PROF_MIN_VAL_COUNTS 10 1008 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS) 1009 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2); 1010 1011 auto &Ctx = M->getContext(); 1012 Type *VNodeTypes[] = { 1013 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType, 1014 #include "llvm/ProfileData/InstrProfData.inc" 1015 }; 1016 auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes)); 1017 1018 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters); 1019 auto *VNodesVar = new GlobalVariable( 1020 *M, VNodesTy, false, GlobalValue::PrivateLinkage, 1021 Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName()); 1022 VNodesVar->setSection( 1023 getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat())); 1024 // VNodesVar is used by runtime but not referenced via relocation by other 1025 // sections. Conservatively make it linker retained. 1026 UsedVars.push_back(VNodesVar); 1027 } 1028 1029 void InstrProfiling::emitNameData() { 1030 std::string UncompressedData; 1031 1032 if (ReferencedNames.empty()) 1033 return; 1034 1035 std::string CompressedNameStr; 1036 if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr, 1037 DoInstrProfNameCompression)) { 1038 report_fatal_error(toString(std::move(E)), false); 1039 } 1040 1041 auto &Ctx = M->getContext(); 1042 auto *NamesVal = ConstantDataArray::getString( 1043 Ctx, StringRef(CompressedNameStr), false); 1044 NamesVar = new GlobalVariable(*M, NamesVal->getType(), true, 1045 GlobalValue::PrivateLinkage, NamesVal, 1046 getInstrProfNamesVarName()); 1047 NamesSize = CompressedNameStr.size(); 1048 NamesVar->setSection( 1049 getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); 1050 // On COFF, it's important to reduce the alignment down to 1 to prevent the 1051 // linker from inserting padding before the start of the names section or 1052 // between names entries. 1053 NamesVar->setAlignment(Align(1)); 1054 // NamesVar is used by runtime but not referenced via relocation by other 1055 // sections. Conservatively make it linker retained. 1056 UsedVars.push_back(NamesVar); 1057 1058 for (auto *NamePtr : ReferencedNames) 1059 NamePtr->eraseFromParent(); 1060 } 1061 1062 void InstrProfiling::emitRegistration() { 1063 if (!needsRuntimeRegistrationOfSectionRange(TT)) 1064 return; 1065 1066 // Construct the function. 1067 auto *VoidTy = Type::getVoidTy(M->getContext()); 1068 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext()); 1069 auto *Int64Ty = Type::getInt64Ty(M->getContext()); 1070 auto *RegisterFTy = FunctionType::get(VoidTy, false); 1071 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage, 1072 getInstrProfRegFuncsName(), M); 1073 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); 1074 if (Options.NoRedZone) 1075 RegisterF->addFnAttr(Attribute::NoRedZone); 1076 1077 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false); 1078 auto *RuntimeRegisterF = 1079 Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage, 1080 getInstrProfRegFuncName(), M); 1081 1082 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF)); 1083 for (Value *Data : CompilerUsedVars) 1084 if (!isa<Function>(Data)) 1085 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy)); 1086 for (Value *Data : UsedVars) 1087 if (Data != NamesVar && !isa<Function>(Data)) 1088 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy)); 1089 1090 if (NamesVar) { 1091 Type *ParamTypes[] = {VoidPtrTy, Int64Ty}; 1092 auto *NamesRegisterTy = 1093 FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false); 1094 auto *NamesRegisterF = 1095 Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage, 1096 getInstrProfNamesRegFuncName(), M); 1097 IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy), 1098 IRB.getInt64(NamesSize)}); 1099 } 1100 1101 IRB.CreateRetVoid(); 1102 } 1103 1104 bool InstrProfiling::emitRuntimeHook() { 1105 // We expect the linker to be invoked with -u<hook_var> flag for Linux or 1106 // Fuchsia, in which case there is no need to emit the user function. 1107 if (TT.isOSLinux() || TT.isOSFuchsia()) 1108 return false; 1109 1110 // If the module's provided its own runtime, we don't need to do anything. 1111 if (M->getGlobalVariable(getInstrProfRuntimeHookVarName())) 1112 return false; 1113 1114 // Declare an external variable that will pull in the runtime initialization. 1115 auto *Int32Ty = Type::getInt32Ty(M->getContext()); 1116 auto *Var = 1117 new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage, 1118 nullptr, getInstrProfRuntimeHookVarName()); 1119 1120 // Make a function that uses it. 1121 auto *User = Function::Create(FunctionType::get(Int32Ty, false), 1122 GlobalValue::LinkOnceODRLinkage, 1123 getInstrProfRuntimeHookVarUseFuncName(), M); 1124 User->addFnAttr(Attribute::NoInline); 1125 if (Options.NoRedZone) 1126 User->addFnAttr(Attribute::NoRedZone); 1127 User->setVisibility(GlobalValue::HiddenVisibility); 1128 if (TT.supportsCOMDAT()) 1129 User->setComdat(M->getOrInsertComdat(User->getName())); 1130 1131 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User)); 1132 auto *Load = IRB.CreateLoad(Int32Ty, Var); 1133 IRB.CreateRet(Load); 1134 1135 // Mark the user variable as used so that it isn't stripped out. 1136 CompilerUsedVars.push_back(User); 1137 return true; 1138 } 1139 1140 void InstrProfiling::emitUses() { 1141 // The metadata sections are parallel arrays. Optimizers (e.g. 1142 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so 1143 // we conservatively retain all unconditionally in the compiler. 1144 // 1145 // On ELF, the linker can guarantee the associated sections will be retained 1146 // or discarded as a unit, so llvm.compiler.used is sufficient. Similarly on 1147 // COFF, if prof data is not referenced by code we use one comdat and ensure 1148 // this GC property as well. Otherwise, we have to conservatively make all of 1149 // the sections retained by the linker. 1150 if (TT.isOSBinFormatELF() || 1151 (TT.isOSBinFormatCOFF() && !profDataReferencedByCode(*M))) 1152 appendToCompilerUsed(*M, CompilerUsedVars); 1153 else 1154 appendToUsed(*M, CompilerUsedVars); 1155 1156 // We do not add proper references from used metadata sections to NamesVar and 1157 // VNodesVar, so we have to be conservative and place them in llvm.used 1158 // regardless of the target, 1159 appendToUsed(*M, UsedVars); 1160 } 1161 1162 void InstrProfiling::emitInitialization() { 1163 // Create ProfileFileName variable. Don't don't this for the 1164 // context-sensitive instrumentation lowering: This lowering is after 1165 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should 1166 // have already create the variable before LTO/ThinLTO linking. 1167 if (!IsCS) 1168 createProfileFileNameVar(*M, Options.InstrProfileOutput); 1169 Function *RegisterF = M->getFunction(getInstrProfRegFuncsName()); 1170 if (!RegisterF) 1171 return; 1172 1173 // Create the initialization function. 1174 auto *VoidTy = Type::getVoidTy(M->getContext()); 1175 auto *F = Function::Create(FunctionType::get(VoidTy, false), 1176 GlobalValue::InternalLinkage, 1177 getInstrProfInitFuncName(), M); 1178 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); 1179 F->addFnAttr(Attribute::NoInline); 1180 if (Options.NoRedZone) 1181 F->addFnAttr(Attribute::NoRedZone); 1182 1183 // Add the basic block and the necessary calls. 1184 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F)); 1185 IRB.CreateCall(RegisterF, {}); 1186 IRB.CreateRetVoid(); 1187 1188 appendToGlobalCtors(*M, F, 0); 1189 } 1190