1 //=-- ProfilesummaryBuilder.cpp - Profile summary computation ---------------=// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains support for computing profile summary data. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/IR/ProfileSummary.h" 14 #include "llvm/ProfileData/InstrProf.h" 15 #include "llvm/ProfileData/ProfileCommon.h" 16 #include "llvm/ProfileData/SampleProf.h" 17 #include "llvm/Support/CommandLine.h" 18 19 using namespace llvm; 20 21 namespace llvm { 22 cl::opt<bool> UseContextLessSummary( 23 "profile-summary-contextless", cl::Hidden, 24 cl::desc("Merge context profiles before calculating thresholds.")); 25 26 // The following two parameters determine the threshold for a count to be 27 // considered hot/cold. These two parameters are percentile values (multiplied 28 // by 10000). If the counts are sorted in descending order, the minimum count to 29 // reach ProfileSummaryCutoffHot gives the threshold to determine a hot count. 30 // Similarly, the minimum count to reach ProfileSummaryCutoffCold gives the 31 // threshold for determining cold count (everything <= this threshold is 32 // considered cold). 33 cl::opt<int> ProfileSummaryCutoffHot( 34 "profile-summary-cutoff-hot", cl::Hidden, cl::init(990000), 35 cl::desc("A count is hot if it exceeds the minimum count to" 36 " reach this percentile of total counts.")); 37 38 cl::opt<int> ProfileSummaryCutoffCold( 39 "profile-summary-cutoff-cold", cl::Hidden, cl::init(999999), 40 cl::desc("A count is cold if it is below the minimum count" 41 " to reach this percentile of total counts.")); 42 43 cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold( 44 "profile-summary-huge-working-set-size-threshold", cl::Hidden, 45 cl::init(15000), 46 cl::desc("The code working set size is considered huge if the number of" 47 " blocks required to reach the -profile-summary-cutoff-hot" 48 " percentile exceeds this count.")); 49 50 cl::opt<unsigned> ProfileSummaryLargeWorkingSetSizeThreshold( 51 "profile-summary-large-working-set-size-threshold", cl::Hidden, 52 cl::init(12500), 53 cl::desc("The code working set size is considered large if the number of" 54 " blocks required to reach the -profile-summary-cutoff-hot" 55 " percentile exceeds this count.")); 56 57 // The next two options override the counts derived from summary computation and 58 // are useful for debugging purposes. 59 cl::opt<uint64_t> ProfileSummaryHotCount( 60 "profile-summary-hot-count", cl::ReallyHidden, 61 cl::desc("A fixed hot count that overrides the count derived from" 62 " profile-summary-cutoff-hot")); 63 64 cl::opt<uint64_t> ProfileSummaryColdCount( 65 "profile-summary-cold-count", cl::ReallyHidden, 66 cl::desc("A fixed cold count that overrides the count derived from" 67 " profile-summary-cutoff-cold")); 68 } // namespace llvm 69 70 // A set of cutoff values. Each value, when divided by ProfileSummary::Scale 71 // (which is 1000000) is a desired percentile of total counts. 72 static const uint32_t DefaultCutoffsData[] = { 73 10000, /* 1% */ 74 100000, /* 10% */ 75 200000, 300000, 400000, 500000, 600000, 700000, 800000, 76 900000, 950000, 990000, 999000, 999900, 999990, 999999}; 77 const ArrayRef<uint32_t> ProfileSummaryBuilder::DefaultCutoffs = 78 DefaultCutoffsData; 79 80 const ProfileSummaryEntry & 81 ProfileSummaryBuilder::getEntryForPercentile(const SummaryEntryVector &DS, 82 uint64_t Percentile) { 83 auto It = partition_point(DS, [=](const ProfileSummaryEntry &Entry) { 84 return Entry.Cutoff < Percentile; 85 }); 86 // The required percentile has to be <= one of the percentiles in the 87 // detailed summary. 88 if (It == DS.end()) 89 report_fatal_error("Desired percentile exceeds the maximum cutoff"); 90 return *It; 91 } 92 93 void InstrProfSummaryBuilder::addRecord(const InstrProfRecord &R) { 94 // The first counter is not necessarily an entry count for IR 95 // instrumentation profiles. 96 // Eventually MaxFunctionCount will become obsolete and this can be 97 // removed. 98 99 if (R.getCountPseudoKind() != InstrProfRecord::NotPseudo) 100 return; 101 102 addEntryCount(R.Counts[0]); 103 for (size_t I = 1, E = R.Counts.size(); I < E; ++I) 104 addInternalCount(R.Counts[I]); 105 } 106 107 // To compute the detailed summary, we consider each line containing samples as 108 // equivalent to a block with a count in the instrumented profile. 109 void SampleProfileSummaryBuilder::addRecord( 110 const sampleprof::FunctionSamples &FS, bool isCallsiteSample) { 111 if (!isCallsiteSample) { 112 NumFunctions++; 113 if (FS.getHeadSamples() > MaxFunctionCount) 114 MaxFunctionCount = FS.getHeadSamples(); 115 } else if (FS.getContext().hasAttribute( 116 sampleprof::ContextDuplicatedIntoBase)) { 117 // Do not recount callee samples if they are already merged into their base 118 // profiles. This can happen to CS nested profile. 119 return; 120 } 121 122 for (const auto &I : FS.getBodySamples()) { 123 uint64_t Count = I.second.getSamples(); 124 addCount(Count); 125 } 126 for (const auto &I : FS.getCallsiteSamples()) 127 for (const auto &CS : I.second) 128 addRecord(CS.second, true); 129 } 130 131 // The argument to this method is a vector of cutoff percentages and the return 132 // value is a vector of (Cutoff, MinCount, NumCounts) triplets. 133 void ProfileSummaryBuilder::computeDetailedSummary() { 134 if (DetailedSummaryCutoffs.empty()) 135 return; 136 llvm::sort(DetailedSummaryCutoffs); 137 auto Iter = CountFrequencies.begin(); 138 const auto End = CountFrequencies.end(); 139 140 uint32_t CountsSeen = 0; 141 uint64_t CurrSum = 0, Count = 0; 142 143 for (const uint32_t Cutoff : DetailedSummaryCutoffs) { 144 assert(Cutoff <= 999999); 145 APInt Temp(128, TotalCount); 146 APInt N(128, Cutoff); 147 APInt D(128, ProfileSummary::Scale); 148 Temp *= N; 149 Temp = Temp.sdiv(D); 150 uint64_t DesiredCount = Temp.getZExtValue(); 151 assert(DesiredCount <= TotalCount); 152 while (CurrSum < DesiredCount && Iter != End) { 153 Count = Iter->first; 154 uint32_t Freq = Iter->second; 155 CurrSum += (Count * Freq); 156 CountsSeen += Freq; 157 Iter++; 158 } 159 assert(CurrSum >= DesiredCount); 160 ProfileSummaryEntry PSE = {Cutoff, Count, CountsSeen}; 161 DetailedSummary.push_back(PSE); 162 } 163 } 164 165 uint64_t 166 ProfileSummaryBuilder::getHotCountThreshold(const SummaryEntryVector &DS) { 167 auto &HotEntry = 168 ProfileSummaryBuilder::getEntryForPercentile(DS, ProfileSummaryCutoffHot); 169 uint64_t HotCountThreshold = HotEntry.MinCount; 170 if (ProfileSummaryHotCount.getNumOccurrences() > 0) 171 HotCountThreshold = ProfileSummaryHotCount; 172 return HotCountThreshold; 173 } 174 175 uint64_t 176 ProfileSummaryBuilder::getColdCountThreshold(const SummaryEntryVector &DS) { 177 auto &ColdEntry = ProfileSummaryBuilder::getEntryForPercentile( 178 DS, ProfileSummaryCutoffCold); 179 uint64_t ColdCountThreshold = ColdEntry.MinCount; 180 if (ProfileSummaryColdCount.getNumOccurrences() > 0) 181 ColdCountThreshold = ProfileSummaryColdCount; 182 return ColdCountThreshold; 183 } 184 185 std::unique_ptr<ProfileSummary> SampleProfileSummaryBuilder::getSummary() { 186 computeDetailedSummary(); 187 return std::make_unique<ProfileSummary>( 188 ProfileSummary::PSK_Sample, DetailedSummary, TotalCount, MaxCount, 0, 189 MaxFunctionCount, NumCounts, NumFunctions); 190 } 191 192 std::unique_ptr<ProfileSummary> 193 SampleProfileSummaryBuilder::computeSummaryForProfiles( 194 const SampleProfileMap &Profiles) { 195 assert(NumFunctions == 0 && 196 "This can only be called on an empty summary builder"); 197 sampleprof::SampleProfileMap ContextLessProfiles; 198 const sampleprof::SampleProfileMap *ProfilesToUse = &Profiles; 199 // For CSSPGO, context-sensitive profile effectively split a function profile 200 // into many copies each representing the CFG profile of a particular calling 201 // context. That makes the count distribution looks more flat as we now have 202 // more function profiles each with lower counts, which in turn leads to lower 203 // hot thresholds. To compensate for that, by default we merge context 204 // profiles before computing profile summary. 205 if (UseContextLessSummary || (sampleprof::FunctionSamples::ProfileIsCS && 206 !UseContextLessSummary.getNumOccurrences())) { 207 ProfileConverter::flattenProfile(Profiles, ContextLessProfiles, true); 208 ProfilesToUse = &ContextLessProfiles; 209 } 210 211 for (const auto &I : *ProfilesToUse) { 212 const sampleprof::FunctionSamples &Profile = I.second; 213 addRecord(Profile); 214 } 215 216 return getSummary(); 217 } 218 219 std::unique_ptr<ProfileSummary> InstrProfSummaryBuilder::getSummary() { 220 computeDetailedSummary(); 221 return std::make_unique<ProfileSummary>( 222 ProfileSummary::PSK_Instr, DetailedSummary, TotalCount, MaxCount, 223 MaxInternalBlockCount, MaxFunctionCount, NumCounts, NumFunctions); 224 } 225 226 void InstrProfSummaryBuilder::addEntryCount(uint64_t Count) { 227 assert(Count <= getInstrMaxCountValue() && 228 "Count value should be less than the max count value."); 229 NumFunctions++; 230 addCount(Count); 231 if (Count > MaxFunctionCount) 232 MaxFunctionCount = Count; 233 } 234 235 void InstrProfSummaryBuilder::addInternalCount(uint64_t Count) { 236 assert(Count <= getInstrMaxCountValue() && 237 "Count value should be less than the max count value."); 238 addCount(Count); 239 if (Count > MaxInternalBlockCount) 240 MaxInternalBlockCount = Count; 241 } 242