xref: /freebsd/contrib/llvm-project/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp (revision cfd6422a5217410fbd66f7a7a8a64d9d85e61229)
1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10 // It also builds the data structures and initialization code needed for
11 // updating execution counts and emitting the profile at runtime.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/ADT/Twine.h"
21 #include "llvm/Analysis/BlockFrequencyInfo.h"
22 #include "llvm/Analysis/BranchProbabilityInfo.h"
23 #include "llvm/Analysis/LoopInfo.h"
24 #include "llvm/Analysis/TargetLibraryInfo.h"
25 #include "llvm/IR/Attributes.h"
26 #include "llvm/IR/BasicBlock.h"
27 #include "llvm/IR/Constant.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DerivedTypes.h"
30 #include "llvm/IR/Dominators.h"
31 #include "llvm/IR/Function.h"
32 #include "llvm/IR/GlobalValue.h"
33 #include "llvm/IR/GlobalVariable.h"
34 #include "llvm/IR/IRBuilder.h"
35 #include "llvm/IR/Instruction.h"
36 #include "llvm/IR/Instructions.h"
37 #include "llvm/IR/IntrinsicInst.h"
38 #include "llvm/IR/Module.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/InitializePasses.h"
41 #include "llvm/Pass.h"
42 #include "llvm/ProfileData/InstrProf.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/CommandLine.h"
45 #include "llvm/Support/Error.h"
46 #include "llvm/Support/ErrorHandling.h"
47 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
48 #include "llvm/Transforms/Utils/ModuleUtils.h"
49 #include "llvm/Transforms/Utils/SSAUpdater.h"
50 #include <algorithm>
51 #include <cassert>
52 #include <cstddef>
53 #include <cstdint>
54 #include <string>
55 
56 using namespace llvm;
57 
58 #define DEBUG_TYPE "instrprof"
59 
60 // The start and end values of precise value profile range for memory
61 // intrinsic sizes
62 cl::opt<std::string> MemOPSizeRange(
63     "memop-size-range",
64     cl::desc("Set the range of size in memory intrinsic calls to be profiled "
65              "precisely, in a format of <start_val>:<end_val>"),
66     cl::init(""));
67 
68 // The value that considered to be large value in  memory intrinsic.
69 cl::opt<unsigned> MemOPSizeLarge(
70     "memop-size-large",
71     cl::desc("Set large value thresthold in memory intrinsic size profiling. "
72              "Value of 0 disables the large value profiling."),
73     cl::init(8192));
74 
75 namespace {
76 
77 cl::opt<bool> DoHashBasedCounterSplit(
78     "hash-based-counter-split",
79     cl::desc("Rename counter variable of a comdat function based on cfg hash"),
80     cl::init(true));
81 
82 cl::opt<bool> RuntimeCounterRelocation(
83     "runtime-counter-relocation",
84     cl::desc("Enable relocating counters at runtime."),
85     cl::init(false));
86 
87 cl::opt<bool> ValueProfileStaticAlloc(
88     "vp-static-alloc",
89     cl::desc("Do static counter allocation for value profiler"),
90     cl::init(true));
91 
92 cl::opt<double> NumCountersPerValueSite(
93     "vp-counters-per-site",
94     cl::desc("The average number of profile counters allocated "
95              "per value profiling site."),
96     // This is set to a very small value because in real programs, only
97     // a very small percentage of value sites have non-zero targets, e.g, 1/30.
98     // For those sites with non-zero profile, the average number of targets
99     // is usually smaller than 2.
100     cl::init(1.0));
101 
102 cl::opt<bool> AtomicCounterUpdateAll(
103     "instrprof-atomic-counter-update-all", cl::ZeroOrMore,
104     cl::desc("Make all profile counter updates atomic (for testing only)"),
105     cl::init(false));
106 
107 cl::opt<bool> AtomicCounterUpdatePromoted(
108     "atomic-counter-update-promoted", cl::ZeroOrMore,
109     cl::desc("Do counter update using atomic fetch add "
110              " for promoted counters only"),
111     cl::init(false));
112 
113 cl::opt<bool> AtomicFirstCounter(
114     "atomic-first-counter", cl::ZeroOrMore,
115     cl::desc("Use atomic fetch add for first counter in a function (usually "
116              "the entry counter)"),
117     cl::init(false));
118 
119 // If the option is not specified, the default behavior about whether
120 // counter promotion is done depends on how instrumentaiton lowering
121 // pipeline is setup, i.e., the default value of true of this option
122 // does not mean the promotion will be done by default. Explicitly
123 // setting this option can override the default behavior.
124 cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
125                                  cl::desc("Do counter register promotion"),
126                                  cl::init(false));
127 cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
128     cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20),
129     cl::desc("Max number counter promotions per loop to avoid"
130              " increasing register pressure too much"));
131 
132 // A debug option
133 cl::opt<int>
134     MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
135                        cl::desc("Max number of allowed counter promotions"));
136 
137 cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
138     cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3),
139     cl::desc("The max number of exiting blocks of a loop to allow "
140              " speculative counter promotion"));
141 
142 cl::opt<bool> SpeculativeCounterPromotionToLoop(
143     cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false),
144     cl::desc("When the option is false, if the target block is in a loop, "
145              "the promotion will be disallowed unless the promoted counter "
146              " update can be further/iteratively promoted into an acyclic "
147              " region."));
148 
149 cl::opt<bool> IterativeCounterPromotion(
150     cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
151     cl::desc("Allow counter promotion across the whole loop nest."));
152 
153 class InstrProfilingLegacyPass : public ModulePass {
154   InstrProfiling InstrProf;
155 
156 public:
157   static char ID;
158 
159   InstrProfilingLegacyPass() : ModulePass(ID) {}
160   InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false)
161       : ModulePass(ID), InstrProf(Options, IsCS) {
162     initializeInstrProfilingLegacyPassPass(*PassRegistry::getPassRegistry());
163   }
164 
165   StringRef getPassName() const override {
166     return "Frontend instrumentation-based coverage lowering";
167   }
168 
169   bool runOnModule(Module &M) override {
170     auto GetTLI = [this](Function &F) -> TargetLibraryInfo & {
171       return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
172     };
173     return InstrProf.run(M, GetTLI);
174   }
175 
176   void getAnalysisUsage(AnalysisUsage &AU) const override {
177     AU.setPreservesCFG();
178     AU.addRequired<TargetLibraryInfoWrapperPass>();
179   }
180 };
181 
182 ///
183 /// A helper class to promote one counter RMW operation in the loop
184 /// into register update.
185 ///
186 /// RWM update for the counter will be sinked out of the loop after
187 /// the transformation.
188 ///
189 class PGOCounterPromoterHelper : public LoadAndStorePromoter {
190 public:
191   PGOCounterPromoterHelper(
192       Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,
193       BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
194       ArrayRef<Instruction *> InsertPts,
195       DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
196       LoopInfo &LI)
197       : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
198         InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
199     assert(isa<LoadInst>(L));
200     assert(isa<StoreInst>(S));
201     SSA.AddAvailableValue(PH, Init);
202   }
203 
204   void doExtraRewritesBeforeFinalDeletion() override {
205     for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
206       BasicBlock *ExitBlock = ExitBlocks[i];
207       Instruction *InsertPos = InsertPts[i];
208       // Get LiveIn value into the ExitBlock. If there are multiple
209       // predecessors, the value is defined by a PHI node in this
210       // block.
211       Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
212       Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
213       Type *Ty = LiveInValue->getType();
214       IRBuilder<> Builder(InsertPos);
215       if (AtomicCounterUpdatePromoted)
216         // automic update currently can only be promoted across the current
217         // loop, not the whole loop nest.
218         Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
219                                 AtomicOrdering::SequentiallyConsistent);
220       else {
221         LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
222         auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
223         auto *NewStore = Builder.CreateStore(NewVal, Addr);
224 
225         // Now update the parent loop's candidate list:
226         if (IterativeCounterPromotion) {
227           auto *TargetLoop = LI.getLoopFor(ExitBlock);
228           if (TargetLoop)
229             LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
230         }
231       }
232     }
233   }
234 
235 private:
236   Instruction *Store;
237   ArrayRef<BasicBlock *> ExitBlocks;
238   ArrayRef<Instruction *> InsertPts;
239   DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
240   LoopInfo &LI;
241 };
242 
243 /// A helper class to do register promotion for all profile counter
244 /// updates in a loop.
245 ///
246 class PGOCounterPromoter {
247 public:
248   PGOCounterPromoter(
249       DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
250       Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
251       : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop),
252         LI(LI), BFI(BFI) {
253 
254     // Skip collection of ExitBlocks and InsertPts for loops that will not be
255     // able to have counters promoted.
256     SmallVector<BasicBlock *, 8> LoopExitBlocks;
257     SmallPtrSet<BasicBlock *, 8> BlockSet;
258 
259     L.getExitBlocks(LoopExitBlocks);
260     if (!isPromotionPossible(&L, LoopExitBlocks))
261       return;
262 
263     for (BasicBlock *ExitBlock : LoopExitBlocks) {
264       if (BlockSet.insert(ExitBlock).second) {
265         ExitBlocks.push_back(ExitBlock);
266         InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
267       }
268     }
269   }
270 
271   bool run(int64_t *NumPromoted) {
272     // Skip 'infinite' loops:
273     if (ExitBlocks.size() == 0)
274       return false;
275     unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
276     if (MaxProm == 0)
277       return false;
278 
279     unsigned Promoted = 0;
280     for (auto &Cand : LoopToCandidates[&L]) {
281 
282       SmallVector<PHINode *, 4> NewPHIs;
283       SSAUpdater SSA(&NewPHIs);
284       Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
285 
286       // If BFI is set, we will use it to guide the promotions.
287       if (BFI) {
288         auto *BB = Cand.first->getParent();
289         auto InstrCount = BFI->getBlockProfileCount(BB);
290         if (!InstrCount)
291           continue;
292         auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
293         // If the average loop trip count is not greater than 1.5, we skip
294         // promotion.
295         if (PreheaderCount &&
296             (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2))
297           continue;
298       }
299 
300       PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
301                                         L.getLoopPreheader(), ExitBlocks,
302                                         InsertPts, LoopToCandidates, LI);
303       Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
304       Promoted++;
305       if (Promoted >= MaxProm)
306         break;
307 
308       (*NumPromoted)++;
309       if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
310         break;
311     }
312 
313     LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
314                       << L.getLoopDepth() << ")\n");
315     return Promoted != 0;
316   }
317 
318 private:
319   bool allowSpeculativeCounterPromotion(Loop *LP) {
320     SmallVector<BasicBlock *, 8> ExitingBlocks;
321     L.getExitingBlocks(ExitingBlocks);
322     // Not considierered speculative.
323     if (ExitingBlocks.size() == 1)
324       return true;
325     if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
326       return false;
327     return true;
328   }
329 
330   // Check whether the loop satisfies the basic conditions needed to perform
331   // Counter Promotions.
332   bool isPromotionPossible(Loop *LP,
333                            const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
334     // We can't insert into a catchswitch.
335     if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
336           return isa<CatchSwitchInst>(Exit->getTerminator());
337         }))
338       return false;
339 
340     if (!LP->hasDedicatedExits())
341       return false;
342 
343     BasicBlock *PH = LP->getLoopPreheader();
344     if (!PH)
345       return false;
346 
347     return true;
348   }
349 
350   // Returns the max number of Counter Promotions for LP.
351   unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
352     SmallVector<BasicBlock *, 8> LoopExitBlocks;
353     LP->getExitBlocks(LoopExitBlocks);
354     if (!isPromotionPossible(LP, LoopExitBlocks))
355       return 0;
356 
357     SmallVector<BasicBlock *, 8> ExitingBlocks;
358     LP->getExitingBlocks(ExitingBlocks);
359 
360     // If BFI is set, we do more aggressive promotions based on BFI.
361     if (BFI)
362       return (unsigned)-1;
363 
364     // Not considierered speculative.
365     if (ExitingBlocks.size() == 1)
366       return MaxNumOfPromotionsPerLoop;
367 
368     if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
369       return 0;
370 
371     // Whether the target block is in a loop does not matter:
372     if (SpeculativeCounterPromotionToLoop)
373       return MaxNumOfPromotionsPerLoop;
374 
375     // Now check the target block:
376     unsigned MaxProm = MaxNumOfPromotionsPerLoop;
377     for (auto *TargetBlock : LoopExitBlocks) {
378       auto *TargetLoop = LI.getLoopFor(TargetBlock);
379       if (!TargetLoop)
380         continue;
381       unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
382       unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
383       MaxProm =
384           std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
385                                 PendingCandsInTarget);
386     }
387     return MaxProm;
388   }
389 
390   DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
391   SmallVector<BasicBlock *, 8> ExitBlocks;
392   SmallVector<Instruction *, 8> InsertPts;
393   Loop &L;
394   LoopInfo &LI;
395   BlockFrequencyInfo *BFI;
396 };
397 
398 } // end anonymous namespace
399 
400 PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
401   FunctionAnalysisManager &FAM =
402       AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
403   auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
404     return FAM.getResult<TargetLibraryAnalysis>(F);
405   };
406   if (!run(M, GetTLI))
407     return PreservedAnalyses::all();
408 
409   return PreservedAnalyses::none();
410 }
411 
412 char InstrProfilingLegacyPass::ID = 0;
413 INITIALIZE_PASS_BEGIN(
414     InstrProfilingLegacyPass, "instrprof",
415     "Frontend instrumentation-based coverage lowering.", false, false)
416 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
417 INITIALIZE_PASS_END(
418     InstrProfilingLegacyPass, "instrprof",
419     "Frontend instrumentation-based coverage lowering.", false, false)
420 
421 ModulePass *
422 llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options,
423                                      bool IsCS) {
424   return new InstrProfilingLegacyPass(Options, IsCS);
425 }
426 
427 static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
428   InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
429   if (Inc)
430     return Inc;
431   return dyn_cast<InstrProfIncrementInst>(Instr);
432 }
433 
434 bool InstrProfiling::lowerIntrinsics(Function *F) {
435   bool MadeChange = false;
436   PromotionCandidates.clear();
437   for (BasicBlock &BB : *F) {
438     for (auto I = BB.begin(), E = BB.end(); I != E;) {
439       auto Instr = I++;
440       InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
441       if (Inc) {
442         lowerIncrement(Inc);
443         MadeChange = true;
444       } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
445         lowerValueProfileInst(Ind);
446         MadeChange = true;
447       }
448     }
449   }
450 
451   if (!MadeChange)
452     return false;
453 
454   promoteCounterLoadStores(F);
455   return true;
456 }
457 
458 bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
459   if (RuntimeCounterRelocation.getNumOccurrences() > 0)
460     return RuntimeCounterRelocation;
461 
462   return TT.isOSFuchsia();
463 }
464 
465 bool InstrProfiling::isCounterPromotionEnabled() const {
466   if (DoCounterPromotion.getNumOccurrences() > 0)
467     return DoCounterPromotion;
468 
469   return Options.DoCounterPromotion;
470 }
471 
472 void InstrProfiling::promoteCounterLoadStores(Function *F) {
473   if (!isCounterPromotionEnabled())
474     return;
475 
476   DominatorTree DT(*F);
477   LoopInfo LI(DT);
478   DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
479 
480   std::unique_ptr<BlockFrequencyInfo> BFI;
481   if (Options.UseBFIInPromotion) {
482     std::unique_ptr<BranchProbabilityInfo> BPI;
483     BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
484     BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
485   }
486 
487   for (const auto &LoadStore : PromotionCandidates) {
488     auto *CounterLoad = LoadStore.first;
489     auto *CounterStore = LoadStore.second;
490     BasicBlock *BB = CounterLoad->getParent();
491     Loop *ParentLoop = LI.getLoopFor(BB);
492     if (!ParentLoop)
493       continue;
494     LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
495   }
496 
497   SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
498 
499   // Do a post-order traversal of the loops so that counter updates can be
500   // iteratively hoisted outside the loop nest.
501   for (auto *Loop : llvm::reverse(Loops)) {
502     PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
503     Promoter.run(&TotalCountersPromoted);
504   }
505 }
506 
507 /// Check if the module contains uses of any profiling intrinsics.
508 static bool containsProfilingIntrinsics(Module &M) {
509   if (auto *F = M.getFunction(
510           Intrinsic::getName(llvm::Intrinsic::instrprof_increment)))
511     if (!F->use_empty())
512       return true;
513   if (auto *F = M.getFunction(
514           Intrinsic::getName(llvm::Intrinsic::instrprof_increment_step)))
515     if (!F->use_empty())
516       return true;
517   if (auto *F = M.getFunction(
518           Intrinsic::getName(llvm::Intrinsic::instrprof_value_profile)))
519     if (!F->use_empty())
520       return true;
521   return false;
522 }
523 
524 bool InstrProfiling::run(
525     Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
526   this->M = &M;
527   this->GetTLI = std::move(GetTLI);
528   NamesVar = nullptr;
529   NamesSize = 0;
530   ProfileDataMap.clear();
531   UsedVars.clear();
532   getMemOPSizeRangeFromOption(MemOPSizeRange, MemOPSizeRangeStart,
533                               MemOPSizeRangeLast);
534   TT = Triple(M.getTargetTriple());
535 
536   // Emit the runtime hook even if no counters are present.
537   bool MadeChange = emitRuntimeHook();
538 
539   // Improve compile time by avoiding linear scans when there is no work.
540   GlobalVariable *CoverageNamesVar =
541       M.getNamedGlobal(getCoverageUnusedNamesVarName());
542   if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
543     return MadeChange;
544 
545   // We did not know how many value sites there would be inside
546   // the instrumented function. This is counting the number of instrumented
547   // target value sites to enter it as field in the profile data variable.
548   for (Function &F : M) {
549     InstrProfIncrementInst *FirstProfIncInst = nullptr;
550     for (BasicBlock &BB : F)
551       for (auto I = BB.begin(), E = BB.end(); I != E; I++)
552         if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
553           computeNumValueSiteCounts(Ind);
554         else if (FirstProfIncInst == nullptr)
555           FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
556 
557     // Value profiling intrinsic lowering requires per-function profile data
558     // variable to be created first.
559     if (FirstProfIncInst != nullptr)
560       static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
561   }
562 
563   for (Function &F : M)
564     MadeChange |= lowerIntrinsics(&F);
565 
566   if (CoverageNamesVar) {
567     lowerCoverageData(CoverageNamesVar);
568     MadeChange = true;
569   }
570 
571   if (!MadeChange)
572     return false;
573 
574   emitVNodes();
575   emitNameData();
576   emitRegistration();
577   emitUses();
578   emitInitialization();
579   return true;
580 }
581 
582 static FunctionCallee
583 getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
584                               bool IsRange = false) {
585   LLVMContext &Ctx = M.getContext();
586   auto *ReturnTy = Type::getVoidTy(M.getContext());
587 
588   AttributeList AL;
589   if (auto AK = TLI.getExtAttrForI32Param(false))
590     AL = AL.addParamAttribute(M.getContext(), 2, AK);
591 
592   if (!IsRange) {
593     Type *ParamTypes[] = {
594 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
595 #include "llvm/ProfileData/InstrProfData.inc"
596     };
597     auto *ValueProfilingCallTy =
598         FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
599     return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
600                                  ValueProfilingCallTy, AL);
601   } else {
602     Type *RangeParamTypes[] = {
603 #define VALUE_RANGE_PROF 1
604 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
605 #include "llvm/ProfileData/InstrProfData.inc"
606 #undef VALUE_RANGE_PROF
607     };
608     auto *ValueRangeProfilingCallTy =
609         FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
610     return M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(),
611                                  ValueRangeProfilingCallTy, AL);
612   }
613 }
614 
615 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
616   GlobalVariable *Name = Ind->getName();
617   uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
618   uint64_t Index = Ind->getIndex()->getZExtValue();
619   auto It = ProfileDataMap.find(Name);
620   if (It == ProfileDataMap.end()) {
621     PerFunctionProfileData PD;
622     PD.NumValueSites[ValueKind] = Index + 1;
623     ProfileDataMap[Name] = PD;
624   } else if (It->second.NumValueSites[ValueKind] <= Index)
625     It->second.NumValueSites[ValueKind] = Index + 1;
626 }
627 
628 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
629   GlobalVariable *Name = Ind->getName();
630   auto It = ProfileDataMap.find(Name);
631   assert(It != ProfileDataMap.end() && It->second.DataVar &&
632          "value profiling detected in function with no counter incerement");
633 
634   GlobalVariable *DataVar = It->second.DataVar;
635   uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
636   uint64_t Index = Ind->getIndex()->getZExtValue();
637   for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
638     Index += It->second.NumValueSites[Kind];
639 
640   IRBuilder<> Builder(Ind);
641   bool IsRange = (Ind->getValueKind()->getZExtValue() ==
642                   llvm::InstrProfValueKind::IPVK_MemOPSize);
643   CallInst *Call = nullptr;
644   auto *TLI = &GetTLI(*Ind->getFunction());
645 
646   // To support value profiling calls within Windows exception handlers, funclet
647   // information contained within operand bundles needs to be copied over to
648   // the library call. This is required for the IR to be processed by the
649   // WinEHPrepare pass.
650   SmallVector<OperandBundleDef, 1> OpBundles;
651   Ind->getOperandBundlesAsDefs(OpBundles);
652   if (!IsRange) {
653     Value *Args[3] = {Ind->getTargetValue(),
654                       Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
655                       Builder.getInt32(Index)};
656     Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
657                               OpBundles);
658   } else {
659     Value *Args[6] = {
660         Ind->getTargetValue(),
661         Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
662         Builder.getInt32(Index),
663         Builder.getInt64(MemOPSizeRangeStart),
664         Builder.getInt64(MemOPSizeRangeLast),
665         Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
666     Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true),
667                               Args, OpBundles);
668   }
669   if (auto AK = TLI->getExtAttrForI32Param(false))
670     Call->addParamAttr(2, AK);
671   Ind->replaceAllUsesWith(Call);
672   Ind->eraseFromParent();
673 }
674 
675 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
676   GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
677 
678   IRBuilder<> Builder(Inc);
679   uint64_t Index = Inc->getIndex()->getZExtValue();
680   Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(),
681                                                    Counters, 0, Index);
682 
683   if (isRuntimeCounterRelocationEnabled()) {
684     Type *Int64Ty = Type::getInt64Ty(M->getContext());
685     Type *Int64PtrTy = Type::getInt64PtrTy(M->getContext());
686     Function *Fn = Inc->getParent()->getParent();
687     Instruction &I = Fn->getEntryBlock().front();
688     LoadInst *LI = dyn_cast<LoadInst>(&I);
689     if (!LI) {
690       IRBuilder<> Builder(&I);
691       Type *Int64Ty = Type::getInt64Ty(M->getContext());
692       GlobalVariable *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
693       if (!Bias) {
694         Bias = new GlobalVariable(*M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
695                                   Constant::getNullValue(Int64Ty),
696                                   getInstrProfCounterBiasVarName());
697         Bias->setVisibility(GlobalVariable::HiddenVisibility);
698       }
699       LI = Builder.CreateLoad(Int64Ty, Bias);
700     }
701     auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), LI);
702     Addr = Builder.CreateIntToPtr(Add, Int64PtrTy);
703   }
704 
705   if (Options.Atomic || AtomicCounterUpdateAll ||
706       (Index == 0 && AtomicFirstCounter)) {
707     Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
708                             AtomicOrdering::Monotonic);
709   } else {
710     Value *IncStep = Inc->getStep();
711     Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
712     auto *Count = Builder.CreateAdd(Load, Inc->getStep());
713     auto *Store = Builder.CreateStore(Count, Addr);
714     if (isCounterPromotionEnabled())
715       PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
716   }
717   Inc->eraseFromParent();
718 }
719 
720 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
721   ConstantArray *Names =
722       cast<ConstantArray>(CoverageNamesVar->getInitializer());
723   for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
724     Constant *NC = Names->getOperand(I);
725     Value *V = NC->stripPointerCasts();
726     assert(isa<GlobalVariable>(V) && "Missing reference to function name");
727     GlobalVariable *Name = cast<GlobalVariable>(V);
728 
729     Name->setLinkage(GlobalValue::PrivateLinkage);
730     ReferencedNames.push_back(Name);
731     NC->dropAllReferences();
732   }
733   CoverageNamesVar->eraseFromParent();
734 }
735 
736 /// Get the name of a profiling variable for a particular function.
737 static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
738   StringRef NamePrefix = getInstrProfNameVarPrefix();
739   StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
740   Function *F = Inc->getParent()->getParent();
741   Module *M = F->getParent();
742   if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
743       !canRenameComdatFunc(*F))
744     return (Prefix + Name).str();
745   uint64_t FuncHash = Inc->getHash()->getZExtValue();
746   SmallVector<char, 24> HashPostfix;
747   if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
748     return (Prefix + Name).str();
749   return (Prefix + Name + "." + Twine(FuncHash)).str();
750 }
751 
752 static inline bool shouldRecordFunctionAddr(Function *F) {
753   // Check the linkage
754   bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
755   if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
756       !HasAvailableExternallyLinkage)
757     return true;
758 
759   // A function marked 'alwaysinline' with available_externally linkage can't
760   // have its address taken. Doing so would create an undefined external ref to
761   // the function, which would fail to link.
762   if (HasAvailableExternallyLinkage &&
763       F->hasFnAttribute(Attribute::AlwaysInline))
764     return false;
765 
766   // Prohibit function address recording if the function is both internal and
767   // COMDAT. This avoids the profile data variable referencing internal symbols
768   // in COMDAT.
769   if (F->hasLocalLinkage() && F->hasComdat())
770     return false;
771 
772   // Check uses of this function for other than direct calls or invokes to it.
773   // Inline virtual functions have linkeOnceODR linkage. When a key method
774   // exists, the vtable will only be emitted in the TU where the key method
775   // is defined. In a TU where vtable is not available, the function won't
776   // be 'addresstaken'. If its address is not recorded here, the profile data
777   // with missing address may be picked by the linker leading  to missing
778   // indirect call target info.
779   return F->hasAddressTaken() || F->hasLinkOnceLinkage();
780 }
781 
782 static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
783   // Don't do this for Darwin.  compiler-rt uses linker magic.
784   if (TT.isOSDarwin())
785     return false;
786   // Use linker script magic to get data/cnts/name start/end.
787   if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
788       TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS4CPU() ||
789       TT.isOSWindows())
790     return false;
791 
792   return true;
793 }
794 
795 GlobalVariable *
796 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
797   GlobalVariable *NamePtr = Inc->getName();
798   auto It = ProfileDataMap.find(NamePtr);
799   PerFunctionProfileData PD;
800   if (It != ProfileDataMap.end()) {
801     if (It->second.RegionCounters)
802       return It->second.RegionCounters;
803     PD = It->second;
804   }
805 
806   // Match the linkage and visibility of the name global. COFF supports using
807   // comdats with internal symbols, so do that if we can.
808   Function *Fn = Inc->getParent()->getParent();
809   GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
810   GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
811   if (TT.isOSBinFormatCOFF()) {
812     Linkage = GlobalValue::InternalLinkage;
813     Visibility = GlobalValue::DefaultVisibility;
814   }
815 
816   // Move the name variable to the right section. Place them in a COMDAT group
817   // if the associated function is a COMDAT. This will make sure that only one
818   // copy of counters of the COMDAT function will be emitted after linking. Keep
819   // in mind that this pass may run before the inliner, so we need to create a
820   // new comdat group for the counters and profiling data. If we use the comdat
821   // of the parent function, that will result in relocations against discarded
822   // sections.
823   bool NeedComdat = needsComdatForCounter(*Fn, *M);
824   if (NeedComdat) {
825     if (TT.isOSBinFormatCOFF()) {
826       // For COFF, put the counters, data, and values each into their own
827       // comdats. We can't use a group because the Visual C++ linker will
828       // report duplicate symbol errors if there are multiple external symbols
829       // with the same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
830       Linkage = GlobalValue::LinkOnceODRLinkage;
831       Visibility = GlobalValue::HiddenVisibility;
832     }
833   }
834   auto MaybeSetComdat = [=](GlobalVariable *GV) {
835     if (NeedComdat)
836       GV->setComdat(M->getOrInsertComdat(GV->getName()));
837   };
838 
839   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
840   LLVMContext &Ctx = M->getContext();
841   ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
842 
843   // Create the counters variable.
844   auto *CounterPtr =
845       new GlobalVariable(*M, CounterTy, false, Linkage,
846                          Constant::getNullValue(CounterTy),
847                          getVarName(Inc, getInstrProfCountersVarPrefix()));
848   CounterPtr->setVisibility(Visibility);
849   CounterPtr->setSection(
850       getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
851   CounterPtr->setAlignment(Align(8));
852   MaybeSetComdat(CounterPtr);
853   CounterPtr->setLinkage(Linkage);
854 
855   auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
856   // Allocate statically the array of pointers to value profile nodes for
857   // the current function.
858   Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
859   if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(TT)) {
860     uint64_t NS = 0;
861     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
862       NS += PD.NumValueSites[Kind];
863     if (NS) {
864       ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
865 
866       auto *ValuesVar =
867           new GlobalVariable(*M, ValuesTy, false, Linkage,
868                              Constant::getNullValue(ValuesTy),
869                              getVarName(Inc, getInstrProfValuesVarPrefix()));
870       ValuesVar->setVisibility(Visibility);
871       ValuesVar->setSection(
872           getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
873       ValuesVar->setAlignment(Align(8));
874       MaybeSetComdat(ValuesVar);
875       ValuesPtrExpr =
876           ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
877     }
878   }
879 
880   // Create data variable.
881   auto *Int16Ty = Type::getInt16Ty(Ctx);
882   auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
883   Type *DataTypes[] = {
884 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
885 #include "llvm/ProfileData/InstrProfData.inc"
886   };
887   auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
888 
889   Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
890                                ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
891                                : ConstantPointerNull::get(Int8PtrTy);
892 
893   Constant *Int16ArrayVals[IPVK_Last + 1];
894   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
895     Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
896 
897   Constant *DataVals[] = {
898 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
899 #include "llvm/ProfileData/InstrProfData.inc"
900   };
901   auto *Data = new GlobalVariable(*M, DataTy, false, Linkage,
902                                   ConstantStruct::get(DataTy, DataVals),
903                                   getVarName(Inc, getInstrProfDataVarPrefix()));
904   Data->setVisibility(Visibility);
905   Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
906   Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
907   MaybeSetComdat(Data);
908   Data->setLinkage(Linkage);
909 
910   PD.RegionCounters = CounterPtr;
911   PD.DataVar = Data;
912   ProfileDataMap[NamePtr] = PD;
913 
914   // Mark the data variable as used so that it isn't stripped out.
915   UsedVars.push_back(Data);
916   // Now that the linkage set by the FE has been passed to the data and counter
917   // variables, reset Name variable's linkage and visibility to private so that
918   // it can be removed later by the compiler.
919   NamePtr->setLinkage(GlobalValue::PrivateLinkage);
920   // Collect the referenced names to be used by emitNameData.
921   ReferencedNames.push_back(NamePtr);
922 
923   return CounterPtr;
924 }
925 
926 void InstrProfiling::emitVNodes() {
927   if (!ValueProfileStaticAlloc)
928     return;
929 
930   // For now only support this on platforms that do
931   // not require runtime registration to discover
932   // named section start/end.
933   if (needsRuntimeRegistrationOfSectionRange(TT))
934     return;
935 
936   size_t TotalNS = 0;
937   for (auto &PD : ProfileDataMap) {
938     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
939       TotalNS += PD.second.NumValueSites[Kind];
940   }
941 
942   if (!TotalNS)
943     return;
944 
945   uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
946 // Heuristic for small programs with very few total value sites.
947 // The default value of vp-counters-per-site is chosen based on
948 // the observation that large apps usually have a low percentage
949 // of value sites that actually have any profile data, and thus
950 // the average number of counters per site is low. For small
951 // apps with very few sites, this may not be true. Bump up the
952 // number of counters in this case.
953 #define INSTR_PROF_MIN_VAL_COUNTS 10
954   if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
955     NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
956 
957   auto &Ctx = M->getContext();
958   Type *VNodeTypes[] = {
959 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
960 #include "llvm/ProfileData/InstrProfData.inc"
961   };
962   auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
963 
964   ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
965   auto *VNodesVar = new GlobalVariable(
966       *M, VNodesTy, false, GlobalValue::PrivateLinkage,
967       Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
968   VNodesVar->setSection(
969       getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
970   UsedVars.push_back(VNodesVar);
971 }
972 
973 void InstrProfiling::emitNameData() {
974   std::string UncompressedData;
975 
976   if (ReferencedNames.empty())
977     return;
978 
979   std::string CompressedNameStr;
980   if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
981                                           DoInstrProfNameCompression)) {
982     report_fatal_error(toString(std::move(E)), false);
983   }
984 
985   auto &Ctx = M->getContext();
986   auto *NamesVal = ConstantDataArray::getString(
987       Ctx, StringRef(CompressedNameStr), false);
988   NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
989                                 GlobalValue::PrivateLinkage, NamesVal,
990                                 getInstrProfNamesVarName());
991   NamesSize = CompressedNameStr.size();
992   NamesVar->setSection(
993       getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
994   // On COFF, it's important to reduce the alignment down to 1 to prevent the
995   // linker from inserting padding before the start of the names section or
996   // between names entries.
997   NamesVar->setAlignment(Align(1));
998   UsedVars.push_back(NamesVar);
999 
1000   for (auto *NamePtr : ReferencedNames)
1001     NamePtr->eraseFromParent();
1002 }
1003 
1004 void InstrProfiling::emitRegistration() {
1005   if (!needsRuntimeRegistrationOfSectionRange(TT))
1006     return;
1007 
1008   // Construct the function.
1009   auto *VoidTy = Type::getVoidTy(M->getContext());
1010   auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
1011   auto *Int64Ty = Type::getInt64Ty(M->getContext());
1012   auto *RegisterFTy = FunctionType::get(VoidTy, false);
1013   auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1014                                      getInstrProfRegFuncsName(), M);
1015   RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1016   if (Options.NoRedZone)
1017     RegisterF->addFnAttr(Attribute::NoRedZone);
1018 
1019   auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1020   auto *RuntimeRegisterF =
1021       Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
1022                        getInstrProfRegFuncName(), M);
1023 
1024   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1025   for (Value *Data : UsedVars)
1026     if (Data != NamesVar && !isa<Function>(Data))
1027       IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1028 
1029   if (NamesVar) {
1030     Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1031     auto *NamesRegisterTy =
1032         FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
1033     auto *NamesRegisterF =
1034         Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
1035                          getInstrProfNamesRegFuncName(), M);
1036     IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
1037                                     IRB.getInt64(NamesSize)});
1038   }
1039 
1040   IRB.CreateRetVoid();
1041 }
1042 
1043 bool InstrProfiling::emitRuntimeHook() {
1044   // We expect the linker to be invoked with -u<hook_var> flag for Linux or
1045   // Fuchsia, in which case there is no need to emit the user function.
1046   if (TT.isOSLinux() || TT.isOSFuchsia())
1047     return false;
1048 
1049   // If the module's provided its own runtime, we don't need to do anything.
1050   if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1051     return false;
1052 
1053   // Declare an external variable that will pull in the runtime initialization.
1054   auto *Int32Ty = Type::getInt32Ty(M->getContext());
1055   auto *Var =
1056       new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
1057                          nullptr, getInstrProfRuntimeHookVarName());
1058 
1059   // Make a function that uses it.
1060   auto *User = Function::Create(FunctionType::get(Int32Ty, false),
1061                                 GlobalValue::LinkOnceODRLinkage,
1062                                 getInstrProfRuntimeHookVarUseFuncName(), M);
1063   User->addFnAttr(Attribute::NoInline);
1064   if (Options.NoRedZone)
1065     User->addFnAttr(Attribute::NoRedZone);
1066   User->setVisibility(GlobalValue::HiddenVisibility);
1067   if (TT.supportsCOMDAT())
1068     User->setComdat(M->getOrInsertComdat(User->getName()));
1069 
1070   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1071   auto *Load = IRB.CreateLoad(Int32Ty, Var);
1072   IRB.CreateRet(Load);
1073 
1074   // Mark the user variable as used so that it isn't stripped out.
1075   UsedVars.push_back(User);
1076   return true;
1077 }
1078 
1079 void InstrProfiling::emitUses() {
1080   if (!UsedVars.empty())
1081     appendToUsed(*M, UsedVars);
1082 }
1083 
1084 void InstrProfiling::emitInitialization() {
1085   // Create ProfileFileName variable. Don't don't this for the
1086   // context-sensitive instrumentation lowering: This lowering is after
1087   // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1088   // have already create the variable before LTO/ThinLTO linking.
1089   if (!IsCS)
1090     createProfileFileNameVar(*M, Options.InstrProfileOutput);
1091   Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1092   if (!RegisterF)
1093     return;
1094 
1095   // Create the initialization function.
1096   auto *VoidTy = Type::getVoidTy(M->getContext());
1097   auto *F = Function::Create(FunctionType::get(VoidTy, false),
1098                              GlobalValue::InternalLinkage,
1099                              getInstrProfInitFuncName(), M);
1100   F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1101   F->addFnAttr(Attribute::NoInline);
1102   if (Options.NoRedZone)
1103     F->addFnAttr(Attribute::NoRedZone);
1104 
1105   // Add the basic block and the necessary calls.
1106   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1107   IRB.CreateCall(RegisterF, {});
1108   IRB.CreateRetVoid();
1109 
1110   appendToGlobalCtors(*M, F, 0);
1111 }
1112