1 //===- SampleProfileLoaderBaseUtil.cpp - Profile loader Util func ---------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the SampleProfileLoader base utility functions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h" 14 #include "llvm/Analysis/ProfileSummaryInfo.h" 15 #include "llvm/IR/Constants.h" 16 #include "llvm/IR/Module.h" 17 #include "llvm/Transforms/Utils/ModuleUtils.h" 18 19 namespace llvm { 20 21 cl::opt<unsigned> SampleProfileMaxPropagateIterations( 22 "sample-profile-max-propagate-iterations", cl::init(100), 23 cl::desc("Maximum number of iterations to go through when propagating " 24 "sample block/edge weights through the CFG.")); 25 26 cl::opt<unsigned> SampleProfileRecordCoverage( 27 "sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"), 28 cl::desc("Emit a warning if less than N% of records in the input profile " 29 "are matched to the IR.")); 30 31 cl::opt<unsigned> SampleProfileSampleCoverage( 32 "sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"), 33 cl::desc("Emit a warning if less than N% of samples in the input profile " 34 "are matched to the IR.")); 35 36 cl::opt<bool> NoWarnSampleUnused( 37 "no-warn-sample-unused", cl::init(false), cl::Hidden, 38 cl::desc("Use this option to turn off/on warnings about function with " 39 "samples but without debug information to use those samples. ")); 40 41 cl::opt<bool> SampleProfileUseProfi( 42 "sample-profile-use-profi", cl::Hidden, 43 cl::desc("Use profi to infer block and edge counts.")); 44 45 cl::opt<bool> SampleProfileInferEntryCount( 46 "sample-profile-infer-entry-count", cl::init(true), cl::Hidden, 47 cl::desc("Use profi to infer function entry count.")); 48 49 namespace sampleprofutil { 50 51 /// Return true if the given callsite is hot wrt to hot cutoff threshold. 52 /// 53 /// Functions that were inlined in the original binary will be represented 54 /// in the inline stack in the sample profile. If the profile shows that 55 /// the original inline decision was "good" (i.e., the callsite is executed 56 /// frequently), then we will recreate the inline decision and apply the 57 /// profile from the inlined callsite. 58 /// 59 /// To decide whether an inlined callsite is hot, we compare the callsite 60 /// sample count with the hot cutoff computed by ProfileSummaryInfo, it is 61 /// regarded as hot if the count is above the cutoff value. 62 /// 63 /// When ProfileAccurateForSymsInList is enabled and profile symbol list 64 /// is present, functions in the profile symbol list but without profile will 65 /// be regarded as cold and much less inlining will happen in CGSCC inlining 66 /// pass, so we tend to lower the hot criteria here to allow more early 67 /// inlining to happen for warm callsites and it is helpful for performance. 68 bool callsiteIsHot(const FunctionSamples *CallsiteFS, ProfileSummaryInfo *PSI, 69 bool ProfAccForSymsInList) { 70 if (!CallsiteFS) 71 return false; // The callsite was not inlined in the original binary. 72 73 assert(PSI && "PSI is expected to be non null"); 74 uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples(); 75 if (ProfAccForSymsInList) 76 return !PSI->isColdCount(CallsiteTotalSamples); 77 else 78 return PSI->isHotCount(CallsiteTotalSamples); 79 } 80 81 /// Mark as used the sample record for the given function samples at 82 /// (LineOffset, Discriminator). 83 /// 84 /// \returns true if this is the first time we mark the given record. 85 bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS, 86 uint32_t LineOffset, 87 uint32_t Discriminator, 88 uint64_t Samples) { 89 LineLocation Loc(LineOffset, Discriminator); 90 unsigned &Count = SampleCoverage[FS][Loc]; 91 bool FirstTime = (++Count == 1); 92 if (FirstTime) 93 TotalUsedSamples += Samples; 94 return FirstTime; 95 } 96 97 /// Return the number of sample records that were applied from this profile. 98 /// 99 /// This count does not include records from cold inlined callsites. 100 unsigned 101 SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS, 102 ProfileSummaryInfo *PSI) const { 103 auto I = SampleCoverage.find(FS); 104 105 // The size of the coverage map for FS represents the number of records 106 // that were marked used at least once. 107 unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0; 108 109 // If there are inlined callsites in this function, count the samples found 110 // in the respective bodies. However, do not bother counting callees with 0 111 // total samples, these are callees that were never invoked at runtime. 112 for (const auto &I : FS->getCallsiteSamples()) 113 for (const auto &J : I.second) { 114 const FunctionSamples *CalleeSamples = &J.second; 115 if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList)) 116 Count += countUsedRecords(CalleeSamples, PSI); 117 } 118 119 return Count; 120 } 121 122 /// Return the number of sample records in the body of this profile. 123 /// 124 /// This count does not include records from cold inlined callsites. 125 unsigned 126 SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS, 127 ProfileSummaryInfo *PSI) const { 128 unsigned Count = FS->getBodySamples().size(); 129 130 // Only count records in hot callsites. 131 for (const auto &I : FS->getCallsiteSamples()) 132 for (const auto &J : I.second) { 133 const FunctionSamples *CalleeSamples = &J.second; 134 if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList)) 135 Count += countBodyRecords(CalleeSamples, PSI); 136 } 137 138 return Count; 139 } 140 141 /// Return the number of samples collected in the body of this profile. 142 /// 143 /// This count does not include samples from cold inlined callsites. 144 uint64_t 145 SampleCoverageTracker::countBodySamples(const FunctionSamples *FS, 146 ProfileSummaryInfo *PSI) const { 147 uint64_t Total = 0; 148 for (const auto &I : FS->getBodySamples()) 149 Total += I.second.getSamples(); 150 151 // Only count samples in hot callsites. 152 for (const auto &I : FS->getCallsiteSamples()) 153 for (const auto &J : I.second) { 154 const FunctionSamples *CalleeSamples = &J.second; 155 if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList)) 156 Total += countBodySamples(CalleeSamples, PSI); 157 } 158 159 return Total; 160 } 161 162 /// Return the fraction of sample records used in this profile. 163 /// 164 /// The returned value is an unsigned integer in the range 0-100 indicating 165 /// the percentage of sample records that were used while applying this 166 /// profile to the associated function. 167 unsigned SampleCoverageTracker::computeCoverage(unsigned Used, 168 unsigned Total) const { 169 assert(Used <= Total && 170 "number of used records cannot exceed the total number of records"); 171 return Total > 0 ? Used * 100 / Total : 100; 172 } 173 174 /// Create a global variable to flag FSDiscriminators are used. 175 void createFSDiscriminatorVariable(Module *M) { 176 const char *FSDiscriminatorVar = "__llvm_fs_discriminator__"; 177 if (M->getGlobalVariable(FSDiscriminatorVar)) 178 return; 179 180 auto &Context = M->getContext(); 181 // Place this variable to llvm.used so it won't be GC'ed. 182 appendToUsed(*M, {new GlobalVariable(*M, Type::getInt1Ty(Context), true, 183 GlobalValue::WeakODRLinkage, 184 ConstantInt::getTrue(Context), 185 FSDiscriminatorVar)}); 186 } 187 188 } // end of namespace sampleprofutil 189 } // end of namespace llvm 190