1 //===- SampleProfileLoaderBaseUtil.cpp - Profile loader Util func ---------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the SampleProfileLoader base utility functions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h" 14 15 namespace llvm { 16 17 cl::opt<unsigned> SampleProfileMaxPropagateIterations( 18 "sample-profile-max-propagate-iterations", cl::init(100), 19 cl::desc("Maximum number of iterations to go through when propagating " 20 "sample block/edge weights through the CFG.")); 21 22 cl::opt<unsigned> SampleProfileRecordCoverage( 23 "sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"), 24 cl::desc("Emit a warning if less than N% of records in the input profile " 25 "are matched to the IR.")); 26 27 cl::opt<unsigned> SampleProfileSampleCoverage( 28 "sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"), 29 cl::desc("Emit a warning if less than N% of samples in the input profile " 30 "are matched to the IR.")); 31 32 cl::opt<bool> NoWarnSampleUnused( 33 "no-warn-sample-unused", cl::init(false), cl::Hidden, 34 cl::desc("Use this option to turn off/on warnings about function with " 35 "samples but without debug information to use those samples. ")); 36 37 cl::opt<bool> SampleProfileUseProfi( 38 "sample-profile-use-profi", cl::init(false), cl::Hidden, cl::ZeroOrMore, 39 cl::desc("Use profi to infer block and edge counts.")); 40 41 namespace sampleprofutil { 42 43 /// Return true if the given callsite is hot wrt to hot cutoff threshold. 44 /// 45 /// Functions that were inlined in the original binary will be represented 46 /// in the inline stack in the sample profile. If the profile shows that 47 /// the original inline decision was "good" (i.e., the callsite is executed 48 /// frequently), then we will recreate the inline decision and apply the 49 /// profile from the inlined callsite. 50 /// 51 /// To decide whether an inlined callsite is hot, we compare the callsite 52 /// sample count with the hot cutoff computed by ProfileSummaryInfo, it is 53 /// regarded as hot if the count is above the cutoff value. 54 /// 55 /// When ProfileAccurateForSymsInList is enabled and profile symbol list 56 /// is present, functions in the profile symbol list but without profile will 57 /// be regarded as cold and much less inlining will happen in CGSCC inlining 58 /// pass, so we tend to lower the hot criteria here to allow more early 59 /// inlining to happen for warm callsites and it is helpful for performance. 60 bool callsiteIsHot(const FunctionSamples *CallsiteFS, ProfileSummaryInfo *PSI, 61 bool ProfAccForSymsInList) { 62 if (!CallsiteFS) 63 return false; // The callsite was not inlined in the original binary. 64 65 assert(PSI && "PSI is expected to be non null"); 66 uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples(); 67 if (ProfAccForSymsInList) 68 return !PSI->isColdCount(CallsiteTotalSamples); 69 else 70 return PSI->isHotCount(CallsiteTotalSamples); 71 } 72 73 /// Mark as used the sample record for the given function samples at 74 /// (LineOffset, Discriminator). 75 /// 76 /// \returns true if this is the first time we mark the given record. 77 bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS, 78 uint32_t LineOffset, 79 uint32_t Discriminator, 80 uint64_t Samples) { 81 LineLocation Loc(LineOffset, Discriminator); 82 unsigned &Count = SampleCoverage[FS][Loc]; 83 bool FirstTime = (++Count == 1); 84 if (FirstTime) 85 TotalUsedSamples += Samples; 86 return FirstTime; 87 } 88 89 /// Return the number of sample records that were applied from this profile. 90 /// 91 /// This count does not include records from cold inlined callsites. 92 unsigned 93 SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS, 94 ProfileSummaryInfo *PSI) const { 95 auto I = SampleCoverage.find(FS); 96 97 // The size of the coverage map for FS represents the number of records 98 // that were marked used at least once. 99 unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0; 100 101 // If there are inlined callsites in this function, count the samples found 102 // in the respective bodies. However, do not bother counting callees with 0 103 // total samples, these are callees that were never invoked at runtime. 104 for (const auto &I : FS->getCallsiteSamples()) 105 for (const auto &J : I.second) { 106 const FunctionSamples *CalleeSamples = &J.second; 107 if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList)) 108 Count += countUsedRecords(CalleeSamples, PSI); 109 } 110 111 return Count; 112 } 113 114 /// Return the number of sample records in the body of this profile. 115 /// 116 /// This count does not include records from cold inlined callsites. 117 unsigned 118 SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS, 119 ProfileSummaryInfo *PSI) const { 120 unsigned Count = FS->getBodySamples().size(); 121 122 // Only count records in hot callsites. 123 for (const auto &I : FS->getCallsiteSamples()) 124 for (const auto &J : I.second) { 125 const FunctionSamples *CalleeSamples = &J.second; 126 if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList)) 127 Count += countBodyRecords(CalleeSamples, PSI); 128 } 129 130 return Count; 131 } 132 133 /// Return the number of samples collected in the body of this profile. 134 /// 135 /// This count does not include samples from cold inlined callsites. 136 uint64_t 137 SampleCoverageTracker::countBodySamples(const FunctionSamples *FS, 138 ProfileSummaryInfo *PSI) const { 139 uint64_t Total = 0; 140 for (const auto &I : FS->getBodySamples()) 141 Total += I.second.getSamples(); 142 143 // Only count samples in hot callsites. 144 for (const auto &I : FS->getCallsiteSamples()) 145 for (const auto &J : I.second) { 146 const FunctionSamples *CalleeSamples = &J.second; 147 if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList)) 148 Total += countBodySamples(CalleeSamples, PSI); 149 } 150 151 return Total; 152 } 153 154 /// Return the fraction of sample records used in this profile. 155 /// 156 /// The returned value is an unsigned integer in the range 0-100 indicating 157 /// the percentage of sample records that were used while applying this 158 /// profile to the associated function. 159 unsigned SampleCoverageTracker::computeCoverage(unsigned Used, 160 unsigned Total) const { 161 assert(Used <= Total && 162 "number of used records cannot exceed the total number of records"); 163 return Total > 0 ? Used * 100 / Total : 100; 164 } 165 166 /// Create a global variable to flag FSDiscriminators are used. 167 void createFSDiscriminatorVariable(Module *M) { 168 const char *FSDiscriminatorVar = "__llvm_fs_discriminator__"; 169 if (M->getGlobalVariable(FSDiscriminatorVar)) 170 return; 171 172 auto &Context = M->getContext(); 173 // Place this variable to llvm.used so it won't be GC'ed. 174 appendToUsed(*M, {new GlobalVariable(*M, Type::getInt1Ty(Context), true, 175 GlobalValue::WeakODRLinkage, 176 ConstantInt::getTrue(Context), 177 FSDiscriminatorVar)}); 178 } 179 180 } // end of namespace sampleprofutil 181 } // end of namespace llvm 182