1 //===- llvm-profdata.cpp - LLVM profile data tool -------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // llvm-profdata merges .profdata files. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ADT/SmallSet.h" 14 #include "llvm/ADT/SmallVector.h" 15 #include "llvm/ADT/StringRef.h" 16 #include "llvm/IR/LLVMContext.h" 17 #include "llvm/ProfileData/InstrProfReader.h" 18 #include "llvm/ProfileData/InstrProfWriter.h" 19 #include "llvm/ProfileData/ProfileCommon.h" 20 #include "llvm/ProfileData/SampleProfReader.h" 21 #include "llvm/ProfileData/SampleProfWriter.h" 22 #include "llvm/Support/CommandLine.h" 23 #include "llvm/Support/Errc.h" 24 #include "llvm/Support/FileSystem.h" 25 #include "llvm/Support/Format.h" 26 #include "llvm/Support/FormattedStream.h" 27 #include "llvm/Support/InitLLVM.h" 28 #include "llvm/Support/MemoryBuffer.h" 29 #include "llvm/Support/Path.h" 30 #include "llvm/Support/ThreadPool.h" 31 #include "llvm/Support/Threading.h" 32 #include "llvm/Support/WithColor.h" 33 #include "llvm/Support/raw_ostream.h" 34 #include <algorithm> 35 36 using namespace llvm; 37 38 enum ProfileFormat { 39 PF_None = 0, 40 PF_Text, 41 PF_Compact_Binary, 42 PF_Ext_Binary, 43 PF_GCC, 44 PF_Binary 45 }; 46 47 static void warn(Twine Message, std::string Whence = "", 48 std::string Hint = "") { 49 WithColor::warning(); 50 if (!Whence.empty()) 51 errs() << Whence << ": "; 52 errs() << Message << "\n"; 53 if (!Hint.empty()) 54 WithColor::note() << Hint << "\n"; 55 } 56 57 static void exitWithError(Twine Message, std::string Whence = "", 58 std::string Hint = "") { 59 WithColor::error(); 60 if (!Whence.empty()) 61 errs() << Whence << ": "; 62 errs() << Message << "\n"; 63 if (!Hint.empty()) 64 WithColor::note() << Hint << "\n"; 65 ::exit(1); 66 } 67 68 static void exitWithError(Error E, StringRef Whence = "") { 69 if (E.isA<InstrProfError>()) { 70 handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { 71 instrprof_error instrError = IPE.get(); 72 StringRef Hint = ""; 73 if (instrError == instrprof_error::unrecognized_format) { 74 // Hint for common error of forgetting --sample for sample profiles. 75 Hint = "Perhaps you forgot to use the --sample option?"; 76 } 77 exitWithError(IPE.message(), std::string(Whence), std::string(Hint)); 78 }); 79 } 80 81 exitWithError(toString(std::move(E)), std::string(Whence)); 82 } 83 84 static void exitWithErrorCode(std::error_code EC, StringRef Whence = "") { 85 exitWithError(EC.message(), std::string(Whence)); 86 } 87 88 namespace { 89 enum ProfileKinds { instr, sample }; 90 enum FailureMode { failIfAnyAreInvalid, failIfAllAreInvalid }; 91 } 92 93 static void warnOrExitGivenError(FailureMode FailMode, std::error_code EC, 94 StringRef Whence = "") { 95 if (FailMode == failIfAnyAreInvalid) 96 exitWithErrorCode(EC, Whence); 97 else 98 warn(EC.message(), std::string(Whence)); 99 } 100 101 static void handleMergeWriterError(Error E, StringRef WhenceFile = "", 102 StringRef WhenceFunction = "", 103 bool ShowHint = true) { 104 if (!WhenceFile.empty()) 105 errs() << WhenceFile << ": "; 106 if (!WhenceFunction.empty()) 107 errs() << WhenceFunction << ": "; 108 109 auto IPE = instrprof_error::success; 110 E = handleErrors(std::move(E), 111 [&IPE](std::unique_ptr<InstrProfError> E) -> Error { 112 IPE = E->get(); 113 return Error(std::move(E)); 114 }); 115 errs() << toString(std::move(E)) << "\n"; 116 117 if (ShowHint) { 118 StringRef Hint = ""; 119 if (IPE != instrprof_error::success) { 120 switch (IPE) { 121 case instrprof_error::hash_mismatch: 122 case instrprof_error::count_mismatch: 123 case instrprof_error::value_site_count_mismatch: 124 Hint = "Make sure that all profile data to be merged is generated " 125 "from the same binary."; 126 break; 127 default: 128 break; 129 } 130 } 131 132 if (!Hint.empty()) 133 errs() << Hint << "\n"; 134 } 135 } 136 137 namespace { 138 /// A remapper from original symbol names to new symbol names based on a file 139 /// containing a list of mappings from old name to new name. 140 class SymbolRemapper { 141 std::unique_ptr<MemoryBuffer> File; 142 DenseMap<StringRef, StringRef> RemappingTable; 143 144 public: 145 /// Build a SymbolRemapper from a file containing a list of old/new symbols. 146 static std::unique_ptr<SymbolRemapper> create(StringRef InputFile) { 147 auto BufOrError = MemoryBuffer::getFileOrSTDIN(InputFile); 148 if (!BufOrError) 149 exitWithErrorCode(BufOrError.getError(), InputFile); 150 151 auto Remapper = std::make_unique<SymbolRemapper>(); 152 Remapper->File = std::move(BufOrError.get()); 153 154 for (line_iterator LineIt(*Remapper->File, /*SkipBlanks=*/true, '#'); 155 !LineIt.is_at_eof(); ++LineIt) { 156 std::pair<StringRef, StringRef> Parts = LineIt->split(' '); 157 if (Parts.first.empty() || Parts.second.empty() || 158 Parts.second.count(' ')) { 159 exitWithError("unexpected line in remapping file", 160 (InputFile + ":" + Twine(LineIt.line_number())).str(), 161 "expected 'old_symbol new_symbol'"); 162 } 163 Remapper->RemappingTable.insert(Parts); 164 } 165 return Remapper; 166 } 167 168 /// Attempt to map the given old symbol into a new symbol. 169 /// 170 /// \return The new symbol, or \p Name if no such symbol was found. 171 StringRef operator()(StringRef Name) { 172 StringRef New = RemappingTable.lookup(Name); 173 return New.empty() ? Name : New; 174 } 175 }; 176 } 177 178 struct WeightedFile { 179 std::string Filename; 180 uint64_t Weight; 181 }; 182 typedef SmallVector<WeightedFile, 5> WeightedFileVector; 183 184 /// Keep track of merged data and reported errors. 185 struct WriterContext { 186 std::mutex Lock; 187 InstrProfWriter Writer; 188 std::vector<std::pair<Error, std::string>> Errors; 189 std::mutex &ErrLock; 190 SmallSet<instrprof_error, 4> &WriterErrorCodes; 191 192 WriterContext(bool IsSparse, std::mutex &ErrLock, 193 SmallSet<instrprof_error, 4> &WriterErrorCodes) 194 : Lock(), Writer(IsSparse), Errors(), ErrLock(ErrLock), 195 WriterErrorCodes(WriterErrorCodes) {} 196 }; 197 198 /// Computer the overlap b/w profile BaseFilename and TestFileName, 199 /// and store the program level result to Overlap. 200 static void overlapInput(const std::string &BaseFilename, 201 const std::string &TestFilename, WriterContext *WC, 202 OverlapStats &Overlap, 203 const OverlapFuncFilters &FuncFilter, 204 raw_fd_ostream &OS, bool IsCS) { 205 auto ReaderOrErr = InstrProfReader::create(TestFilename); 206 if (Error E = ReaderOrErr.takeError()) { 207 // Skip the empty profiles by returning sliently. 208 instrprof_error IPE = InstrProfError::take(std::move(E)); 209 if (IPE != instrprof_error::empty_raw_profile) 210 WC->Errors.emplace_back(make_error<InstrProfError>(IPE), TestFilename); 211 return; 212 } 213 214 auto Reader = std::move(ReaderOrErr.get()); 215 for (auto &I : *Reader) { 216 OverlapStats FuncOverlap(OverlapStats::FunctionLevel); 217 FuncOverlap.setFuncInfo(I.Name, I.Hash); 218 219 WC->Writer.overlapRecord(std::move(I), Overlap, FuncOverlap, FuncFilter); 220 FuncOverlap.dump(OS); 221 } 222 } 223 224 /// Load an input into a writer context. 225 static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, 226 WriterContext *WC) { 227 std::unique_lock<std::mutex> CtxGuard{WC->Lock}; 228 229 // Copy the filename, because llvm::ThreadPool copied the input "const 230 // WeightedFile &" by value, making a reference to the filename within it 231 // invalid outside of this packaged task. 232 std::string Filename = Input.Filename; 233 234 auto ReaderOrErr = InstrProfReader::create(Input.Filename); 235 if (Error E = ReaderOrErr.takeError()) { 236 // Skip the empty profiles by returning sliently. 237 instrprof_error IPE = InstrProfError::take(std::move(E)); 238 if (IPE != instrprof_error::empty_raw_profile) 239 WC->Errors.emplace_back(make_error<InstrProfError>(IPE), Filename); 240 return; 241 } 242 243 auto Reader = std::move(ReaderOrErr.get()); 244 bool IsIRProfile = Reader->isIRLevelProfile(); 245 bool HasCSIRProfile = Reader->hasCSIRLevelProfile(); 246 if (WC->Writer.setIsIRLevelProfile(IsIRProfile, HasCSIRProfile)) { 247 WC->Errors.emplace_back( 248 make_error<StringError>( 249 "Merge IR generated profile with Clang generated profile.", 250 std::error_code()), 251 Filename); 252 return; 253 } 254 WC->Writer.setInstrEntryBBEnabled(Reader->instrEntryBBEnabled()); 255 256 for (auto &I : *Reader) { 257 if (Remapper) 258 I.Name = (*Remapper)(I.Name); 259 const StringRef FuncName = I.Name; 260 bool Reported = false; 261 WC->Writer.addRecord(std::move(I), Input.Weight, [&](Error E) { 262 if (Reported) { 263 consumeError(std::move(E)); 264 return; 265 } 266 Reported = true; 267 // Only show hint the first time an error occurs. 268 instrprof_error IPE = InstrProfError::take(std::move(E)); 269 std::unique_lock<std::mutex> ErrGuard{WC->ErrLock}; 270 bool firstTime = WC->WriterErrorCodes.insert(IPE).second; 271 handleMergeWriterError(make_error<InstrProfError>(IPE), Input.Filename, 272 FuncName, firstTime); 273 }); 274 } 275 if (Reader->hasError()) 276 if (Error E = Reader->getError()) 277 WC->Errors.emplace_back(std::move(E), Filename); 278 } 279 280 /// Merge the \p Src writer context into \p Dst. 281 static void mergeWriterContexts(WriterContext *Dst, WriterContext *Src) { 282 for (auto &ErrorPair : Src->Errors) 283 Dst->Errors.push_back(std::move(ErrorPair)); 284 Src->Errors.clear(); 285 286 Dst->Writer.mergeRecordsFromWriter(std::move(Src->Writer), [&](Error E) { 287 instrprof_error IPE = InstrProfError::take(std::move(E)); 288 std::unique_lock<std::mutex> ErrGuard{Dst->ErrLock}; 289 bool firstTime = Dst->WriterErrorCodes.insert(IPE).second; 290 if (firstTime) 291 warn(toString(make_error<InstrProfError>(IPE))); 292 }); 293 } 294 295 static void writeInstrProfile(StringRef OutputFilename, 296 ProfileFormat OutputFormat, 297 InstrProfWriter &Writer) { 298 std::error_code EC; 299 raw_fd_ostream Output(OutputFilename.data(), EC, 300 OutputFormat == PF_Text ? sys::fs::OF_Text 301 : sys::fs::OF_None); 302 if (EC) 303 exitWithErrorCode(EC, OutputFilename); 304 305 if (OutputFormat == PF_Text) { 306 if (Error E = Writer.writeText(Output)) 307 exitWithError(std::move(E)); 308 } else { 309 Writer.write(Output); 310 } 311 } 312 313 static void mergeInstrProfile(const WeightedFileVector &Inputs, 314 SymbolRemapper *Remapper, 315 StringRef OutputFilename, 316 ProfileFormat OutputFormat, bool OutputSparse, 317 unsigned NumThreads, FailureMode FailMode) { 318 if (OutputFilename.compare("-") == 0) 319 exitWithError("Cannot write indexed profdata format to stdout."); 320 321 if (OutputFormat != PF_Binary && OutputFormat != PF_Compact_Binary && 322 OutputFormat != PF_Ext_Binary && OutputFormat != PF_Text) 323 exitWithError("Unknown format is specified."); 324 325 std::mutex ErrorLock; 326 SmallSet<instrprof_error, 4> WriterErrorCodes; 327 328 // If NumThreads is not specified, auto-detect a good default. 329 if (NumThreads == 0) 330 NumThreads = std::min(hardware_concurrency().compute_thread_count(), 331 unsigned((Inputs.size() + 1) / 2)); 332 // FIXME: There's a bug here, where setting NumThreads = Inputs.size() fails 333 // the merge_empty_profile.test because the InstrProfWriter.ProfileKind isn't 334 // merged, thus the emitted file ends up with a PF_Unknown kind. 335 336 // Initialize the writer contexts. 337 SmallVector<std::unique_ptr<WriterContext>, 4> Contexts; 338 for (unsigned I = 0; I < NumThreads; ++I) 339 Contexts.emplace_back(std::make_unique<WriterContext>( 340 OutputSparse, ErrorLock, WriterErrorCodes)); 341 342 if (NumThreads == 1) { 343 for (const auto &Input : Inputs) 344 loadInput(Input, Remapper, Contexts[0].get()); 345 } else { 346 ThreadPool Pool(hardware_concurrency(NumThreads)); 347 348 // Load the inputs in parallel (N/NumThreads serial steps). 349 unsigned Ctx = 0; 350 for (const auto &Input : Inputs) { 351 Pool.async(loadInput, Input, Remapper, Contexts[Ctx].get()); 352 Ctx = (Ctx + 1) % NumThreads; 353 } 354 Pool.wait(); 355 356 // Merge the writer contexts together (~ lg(NumThreads) serial steps). 357 unsigned Mid = Contexts.size() / 2; 358 unsigned End = Contexts.size(); 359 assert(Mid > 0 && "Expected more than one context"); 360 do { 361 for (unsigned I = 0; I < Mid; ++I) 362 Pool.async(mergeWriterContexts, Contexts[I].get(), 363 Contexts[I + Mid].get()); 364 Pool.wait(); 365 if (End & 1) { 366 Pool.async(mergeWriterContexts, Contexts[0].get(), 367 Contexts[End - 1].get()); 368 Pool.wait(); 369 } 370 End = Mid; 371 Mid /= 2; 372 } while (Mid > 0); 373 } 374 375 // Handle deferred errors encountered during merging. If the number of errors 376 // is equal to the number of inputs the merge failed. 377 unsigned NumErrors = 0; 378 for (std::unique_ptr<WriterContext> &WC : Contexts) { 379 for (auto &ErrorPair : WC->Errors) { 380 ++NumErrors; 381 warn(toString(std::move(ErrorPair.first)), ErrorPair.second); 382 } 383 } 384 if (NumErrors == Inputs.size() || 385 (NumErrors > 0 && FailMode == failIfAnyAreInvalid)) 386 exitWithError("No profiles could be merged."); 387 388 writeInstrProfile(OutputFilename, OutputFormat, Contexts[0]->Writer); 389 } 390 391 /// The profile entry for a function in instrumentation profile. 392 struct InstrProfileEntry { 393 uint64_t MaxCount = 0; 394 float ZeroCounterRatio = 0.0; 395 InstrProfRecord *ProfRecord; 396 InstrProfileEntry(InstrProfRecord *Record); 397 InstrProfileEntry() = default; 398 }; 399 400 InstrProfileEntry::InstrProfileEntry(InstrProfRecord *Record) { 401 ProfRecord = Record; 402 uint64_t CntNum = Record->Counts.size(); 403 uint64_t ZeroCntNum = 0; 404 for (size_t I = 0; I < CntNum; ++I) { 405 MaxCount = std::max(MaxCount, Record->Counts[I]); 406 ZeroCntNum += !Record->Counts[I]; 407 } 408 ZeroCounterRatio = (float)ZeroCntNum / CntNum; 409 } 410 411 /// Either set all the counters in the instr profile entry \p IFE to -1 412 /// in order to drop the profile or scale up the counters in \p IFP to 413 /// be above hot threshold. We use the ratio of zero counters in the 414 /// profile of a function to decide the profile is helpful or harmful 415 /// for performance, and to choose whether to scale up or drop it. 416 static void updateInstrProfileEntry(InstrProfileEntry &IFE, 417 uint64_t HotInstrThreshold, 418 float ZeroCounterThreshold) { 419 InstrProfRecord *ProfRecord = IFE.ProfRecord; 420 if (!IFE.MaxCount || IFE.ZeroCounterRatio > ZeroCounterThreshold) { 421 // If all or most of the counters of the function are zero, the 422 // profile is unaccountable and shuld be dropped. Reset all the 423 // counters to be -1 and PGO profile-use will drop the profile. 424 // All counters being -1 also implies that the function is hot so 425 // PGO profile-use will also set the entry count metadata to be 426 // above hot threshold. 427 for (size_t I = 0; I < ProfRecord->Counts.size(); ++I) 428 ProfRecord->Counts[I] = -1; 429 return; 430 } 431 432 // Scale up the MaxCount to be multiple times above hot threshold. 433 const unsigned MultiplyFactor = 3; 434 uint64_t Numerator = HotInstrThreshold * MultiplyFactor; 435 uint64_t Denominator = IFE.MaxCount; 436 ProfRecord->scale(Numerator, Denominator, [&](instrprof_error E) { 437 warn(toString(make_error<InstrProfError>(E))); 438 }); 439 } 440 441 const uint64_t ColdPercentileIdx = 15; 442 const uint64_t HotPercentileIdx = 11; 443 444 /// Adjust the instr profile in \p WC based on the sample profile in 445 /// \p Reader. 446 static void 447 adjustInstrProfile(std::unique_ptr<WriterContext> &WC, 448 std::unique_ptr<sampleprof::SampleProfileReader> &Reader, 449 unsigned SupplMinSizeThreshold, float ZeroCounterThreshold, 450 unsigned InstrProfColdThreshold) { 451 // Function to its entry in instr profile. 452 StringMap<InstrProfileEntry> InstrProfileMap; 453 InstrProfSummaryBuilder IPBuilder(ProfileSummaryBuilder::DefaultCutoffs); 454 for (auto &PD : WC->Writer.getProfileData()) { 455 // Populate IPBuilder. 456 for (const auto &PDV : PD.getValue()) { 457 InstrProfRecord Record = PDV.second; 458 IPBuilder.addRecord(Record); 459 } 460 461 // If a function has multiple entries in instr profile, skip it. 462 if (PD.getValue().size() != 1) 463 continue; 464 465 // Initialize InstrProfileMap. 466 InstrProfRecord *R = &PD.getValue().begin()->second; 467 InstrProfileMap[PD.getKey()] = InstrProfileEntry(R); 468 } 469 470 ProfileSummary InstrPS = *IPBuilder.getSummary(); 471 ProfileSummary SamplePS = Reader->getSummary(); 472 473 // Compute cold thresholds for instr profile and sample profile. 474 uint64_t ColdSampleThreshold = 475 ProfileSummaryBuilder::getEntryForPercentile( 476 SamplePS.getDetailedSummary(), 477 ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx]) 478 .MinCount; 479 uint64_t HotInstrThreshold = 480 ProfileSummaryBuilder::getEntryForPercentile( 481 InstrPS.getDetailedSummary(), 482 ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx]) 483 .MinCount; 484 uint64_t ColdInstrThreshold = 485 InstrProfColdThreshold 486 ? InstrProfColdThreshold 487 : ProfileSummaryBuilder::getEntryForPercentile( 488 InstrPS.getDetailedSummary(), 489 ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx]) 490 .MinCount; 491 492 // Find hot/warm functions in sample profile which is cold in instr profile 493 // and adjust the profiles of those functions in the instr profile. 494 for (const auto &PD : Reader->getProfiles()) { 495 StringRef FName = PD.getKey(); 496 const sampleprof::FunctionSamples &FS = PD.getValue(); 497 auto It = InstrProfileMap.find(FName); 498 if (FS.getHeadSamples() > ColdSampleThreshold && 499 It != InstrProfileMap.end() && 500 It->second.MaxCount <= ColdInstrThreshold && 501 FS.getBodySamples().size() >= SupplMinSizeThreshold) { 502 updateInstrProfileEntry(It->second, HotInstrThreshold, 503 ZeroCounterThreshold); 504 } 505 } 506 } 507 508 /// The main function to supplement instr profile with sample profile. 509 /// \Inputs contains the instr profile. \p SampleFilename specifies the 510 /// sample profile. \p OutputFilename specifies the output profile name. 511 /// \p OutputFormat specifies the output profile format. \p OutputSparse 512 /// specifies whether to generate sparse profile. \p SupplMinSizeThreshold 513 /// specifies the minimal size for the functions whose profile will be 514 /// adjusted. \p ZeroCounterThreshold is the threshold to check whether 515 /// a function contains too many zero counters and whether its profile 516 /// should be dropped. \p InstrProfColdThreshold is the user specified 517 /// cold threshold which will override the cold threshold got from the 518 /// instr profile summary. 519 static void supplementInstrProfile( 520 const WeightedFileVector &Inputs, StringRef SampleFilename, 521 StringRef OutputFilename, ProfileFormat OutputFormat, bool OutputSparse, 522 unsigned SupplMinSizeThreshold, float ZeroCounterThreshold, 523 unsigned InstrProfColdThreshold) { 524 if (OutputFilename.compare("-") == 0) 525 exitWithError("Cannot write indexed profdata format to stdout."); 526 if (Inputs.size() != 1) 527 exitWithError("Expect one input to be an instr profile."); 528 if (Inputs[0].Weight != 1) 529 exitWithError("Expect instr profile doesn't have weight."); 530 531 StringRef InstrFilename = Inputs[0].Filename; 532 533 // Read sample profile. 534 LLVMContext Context; 535 auto ReaderOrErr = 536 sampleprof::SampleProfileReader::create(SampleFilename.str(), Context); 537 if (std::error_code EC = ReaderOrErr.getError()) 538 exitWithErrorCode(EC, SampleFilename); 539 auto Reader = std::move(ReaderOrErr.get()); 540 if (std::error_code EC = Reader->read()) 541 exitWithErrorCode(EC, SampleFilename); 542 543 // Read instr profile. 544 std::mutex ErrorLock; 545 SmallSet<instrprof_error, 4> WriterErrorCodes; 546 auto WC = std::make_unique<WriterContext>(OutputSparse, ErrorLock, 547 WriterErrorCodes); 548 loadInput(Inputs[0], nullptr, WC.get()); 549 if (WC->Errors.size() > 0) 550 exitWithError(std::move(WC->Errors[0].first), InstrFilename); 551 552 adjustInstrProfile(WC, Reader, SupplMinSizeThreshold, ZeroCounterThreshold, 553 InstrProfColdThreshold); 554 writeInstrProfile(OutputFilename, OutputFormat, WC->Writer); 555 } 556 557 /// Make a copy of the given function samples with all symbol names remapped 558 /// by the provided symbol remapper. 559 static sampleprof::FunctionSamples 560 remapSamples(const sampleprof::FunctionSamples &Samples, 561 SymbolRemapper &Remapper, sampleprof_error &Error) { 562 sampleprof::FunctionSamples Result; 563 Result.setName(Remapper(Samples.getName())); 564 Result.addTotalSamples(Samples.getTotalSamples()); 565 Result.addHeadSamples(Samples.getHeadSamples()); 566 for (const auto &BodySample : Samples.getBodySamples()) { 567 Result.addBodySamples(BodySample.first.LineOffset, 568 BodySample.first.Discriminator, 569 BodySample.second.getSamples()); 570 for (const auto &Target : BodySample.second.getCallTargets()) { 571 Result.addCalledTargetSamples(BodySample.first.LineOffset, 572 BodySample.first.Discriminator, 573 Remapper(Target.first()), Target.second); 574 } 575 } 576 for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) { 577 sampleprof::FunctionSamplesMap &Target = 578 Result.functionSamplesAt(CallsiteSamples.first); 579 for (const auto &Callsite : CallsiteSamples.second) { 580 sampleprof::FunctionSamples Remapped = 581 remapSamples(Callsite.second, Remapper, Error); 582 MergeResult(Error, 583 Target[std::string(Remapped.getName())].merge(Remapped)); 584 } 585 } 586 return Result; 587 } 588 589 static sampleprof::SampleProfileFormat FormatMap[] = { 590 sampleprof::SPF_None, 591 sampleprof::SPF_Text, 592 sampleprof::SPF_Compact_Binary, 593 sampleprof::SPF_Ext_Binary, 594 sampleprof::SPF_GCC, 595 sampleprof::SPF_Binary}; 596 597 static std::unique_ptr<MemoryBuffer> 598 getInputFileBuf(const StringRef &InputFile) { 599 if (InputFile == "") 600 return {}; 601 602 auto BufOrError = MemoryBuffer::getFileOrSTDIN(InputFile); 603 if (!BufOrError) 604 exitWithErrorCode(BufOrError.getError(), InputFile); 605 606 return std::move(*BufOrError); 607 } 608 609 static void populateProfileSymbolList(MemoryBuffer *Buffer, 610 sampleprof::ProfileSymbolList &PSL) { 611 if (!Buffer) 612 return; 613 614 SmallVector<StringRef, 32> SymbolVec; 615 StringRef Data = Buffer->getBuffer(); 616 Data.split(SymbolVec, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false); 617 618 for (StringRef symbol : SymbolVec) 619 PSL.add(symbol); 620 } 621 622 static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer, 623 ProfileFormat OutputFormat, 624 MemoryBuffer *Buffer, 625 sampleprof::ProfileSymbolList &WriterList, 626 bool CompressAllSections, bool UseMD5, 627 bool GenPartialProfile) { 628 populateProfileSymbolList(Buffer, WriterList); 629 if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary) 630 warn("Profile Symbol list is not empty but the output format is not " 631 "ExtBinary format. The list will be lost in the output. "); 632 633 Writer.setProfileSymbolList(&WriterList); 634 635 if (CompressAllSections) { 636 if (OutputFormat != PF_Ext_Binary) 637 warn("-compress-all-section is ignored. Specify -extbinary to enable it"); 638 else 639 Writer.setToCompressAllSections(); 640 } 641 if (UseMD5) { 642 if (OutputFormat != PF_Ext_Binary) 643 warn("-use-md5 is ignored. Specify -extbinary to enable it"); 644 else 645 Writer.setUseMD5(); 646 } 647 if (GenPartialProfile) { 648 if (OutputFormat != PF_Ext_Binary) 649 warn("-gen-partial-profile is ignored. Specify -extbinary to enable it"); 650 else 651 Writer.setPartialProfile(); 652 } 653 } 654 655 static void 656 mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper, 657 StringRef OutputFilename, ProfileFormat OutputFormat, 658 StringRef ProfileSymbolListFile, bool CompressAllSections, 659 bool UseMD5, bool GenPartialProfile, FailureMode FailMode) { 660 using namespace sampleprof; 661 StringMap<FunctionSamples> ProfileMap; 662 SmallVector<std::unique_ptr<sampleprof::SampleProfileReader>, 5> Readers; 663 LLVMContext Context; 664 sampleprof::ProfileSymbolList WriterList; 665 Optional<bool> ProfileIsProbeBased; 666 for (const auto &Input : Inputs) { 667 auto ReaderOrErr = SampleProfileReader::create(Input.Filename, Context); 668 if (std::error_code EC = ReaderOrErr.getError()) { 669 warnOrExitGivenError(FailMode, EC, Input.Filename); 670 continue; 671 } 672 673 // We need to keep the readers around until after all the files are 674 // read so that we do not lose the function names stored in each 675 // reader's memory. The function names are needed to write out the 676 // merged profile map. 677 Readers.push_back(std::move(ReaderOrErr.get())); 678 const auto Reader = Readers.back().get(); 679 if (std::error_code EC = Reader->read()) { 680 warnOrExitGivenError(FailMode, EC, Input.Filename); 681 Readers.pop_back(); 682 continue; 683 } 684 685 StringMap<FunctionSamples> &Profiles = Reader->getProfiles(); 686 if (ProfileIsProbeBased && 687 ProfileIsProbeBased != FunctionSamples::ProfileIsProbeBased) 688 exitWithError( 689 "cannot merge probe-based profile with non-probe-based profile"); 690 ProfileIsProbeBased = FunctionSamples::ProfileIsProbeBased; 691 for (StringMap<FunctionSamples>::iterator I = Profiles.begin(), 692 E = Profiles.end(); 693 I != E; ++I) { 694 sampleprof_error Result = sampleprof_error::success; 695 FunctionSamples Remapped = 696 Remapper ? remapSamples(I->second, *Remapper, Result) 697 : FunctionSamples(); 698 FunctionSamples &Samples = Remapper ? Remapped : I->second; 699 StringRef FName = Samples.getNameWithContext(true); 700 MergeResult(Result, ProfileMap[FName].merge(Samples, Input.Weight)); 701 if (Result != sampleprof_error::success) { 702 std::error_code EC = make_error_code(Result); 703 handleMergeWriterError(errorCodeToError(EC), Input.Filename, FName); 704 } 705 } 706 707 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList = 708 Reader->getProfileSymbolList(); 709 if (ReaderList) 710 WriterList.merge(*ReaderList); 711 } 712 auto WriterOrErr = 713 SampleProfileWriter::create(OutputFilename, FormatMap[OutputFormat]); 714 if (std::error_code EC = WriterOrErr.getError()) 715 exitWithErrorCode(EC, OutputFilename); 716 717 auto Writer = std::move(WriterOrErr.get()); 718 // WriterList will have StringRef refering to string in Buffer. 719 // Make sure Buffer lives as long as WriterList. 720 auto Buffer = getInputFileBuf(ProfileSymbolListFile); 721 handleExtBinaryWriter(*Writer, OutputFormat, Buffer.get(), WriterList, 722 CompressAllSections, UseMD5, GenPartialProfile); 723 Writer->write(ProfileMap); 724 } 725 726 static WeightedFile parseWeightedFile(const StringRef &WeightedFilename) { 727 StringRef WeightStr, FileName; 728 std::tie(WeightStr, FileName) = WeightedFilename.split(','); 729 730 uint64_t Weight; 731 if (WeightStr.getAsInteger(10, Weight) || Weight < 1) 732 exitWithError("Input weight must be a positive integer."); 733 734 return {std::string(FileName), Weight}; 735 } 736 737 static void addWeightedInput(WeightedFileVector &WNI, const WeightedFile &WF) { 738 StringRef Filename = WF.Filename; 739 uint64_t Weight = WF.Weight; 740 741 // If it's STDIN just pass it on. 742 if (Filename == "-") { 743 WNI.push_back({std::string(Filename), Weight}); 744 return; 745 } 746 747 llvm::sys::fs::file_status Status; 748 llvm::sys::fs::status(Filename, Status); 749 if (!llvm::sys::fs::exists(Status)) 750 exitWithErrorCode(make_error_code(errc::no_such_file_or_directory), 751 Filename); 752 // If it's a source file, collect it. 753 if (llvm::sys::fs::is_regular_file(Status)) { 754 WNI.push_back({std::string(Filename), Weight}); 755 return; 756 } 757 758 if (llvm::sys::fs::is_directory(Status)) { 759 std::error_code EC; 760 for (llvm::sys::fs::recursive_directory_iterator F(Filename, EC), E; 761 F != E && !EC; F.increment(EC)) { 762 if (llvm::sys::fs::is_regular_file(F->path())) { 763 addWeightedInput(WNI, {F->path(), Weight}); 764 } 765 } 766 if (EC) 767 exitWithErrorCode(EC, Filename); 768 } 769 } 770 771 static void parseInputFilenamesFile(MemoryBuffer *Buffer, 772 WeightedFileVector &WFV) { 773 if (!Buffer) 774 return; 775 776 SmallVector<StringRef, 8> Entries; 777 StringRef Data = Buffer->getBuffer(); 778 Data.split(Entries, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false); 779 for (const StringRef &FileWeightEntry : Entries) { 780 StringRef SanitizedEntry = FileWeightEntry.trim(" \t\v\f\r"); 781 // Skip comments. 782 if (SanitizedEntry.startswith("#")) 783 continue; 784 // If there's no comma, it's an unweighted profile. 785 else if (SanitizedEntry.find(',') == StringRef::npos) 786 addWeightedInput(WFV, {std::string(SanitizedEntry), 1}); 787 else 788 addWeightedInput(WFV, parseWeightedFile(SanitizedEntry)); 789 } 790 } 791 792 static int merge_main(int argc, const char *argv[]) { 793 cl::list<std::string> InputFilenames(cl::Positional, 794 cl::desc("<filename...>")); 795 cl::list<std::string> WeightedInputFilenames("weighted-input", 796 cl::desc("<weight>,<filename>")); 797 cl::opt<std::string> InputFilenamesFile( 798 "input-files", cl::init(""), 799 cl::desc("Path to file containing newline-separated " 800 "[<weight>,]<filename> entries")); 801 cl::alias InputFilenamesFileA("f", cl::desc("Alias for --input-files"), 802 cl::aliasopt(InputFilenamesFile)); 803 cl::opt<bool> DumpInputFileList( 804 "dump-input-file-list", cl::init(false), cl::Hidden, 805 cl::desc("Dump the list of input files and their weights, then exit")); 806 cl::opt<std::string> RemappingFile("remapping-file", cl::value_desc("file"), 807 cl::desc("Symbol remapping file")); 808 cl::alias RemappingFileA("r", cl::desc("Alias for --remapping-file"), 809 cl::aliasopt(RemappingFile)); 810 cl::opt<std::string> OutputFilename("output", cl::value_desc("output"), 811 cl::init("-"), cl::Required, 812 cl::desc("Output file")); 813 cl::alias OutputFilenameA("o", cl::desc("Alias for --output"), 814 cl::aliasopt(OutputFilename)); 815 cl::opt<ProfileKinds> ProfileKind( 816 cl::desc("Profile kind:"), cl::init(instr), 817 cl::values(clEnumVal(instr, "Instrumentation profile (default)"), 818 clEnumVal(sample, "Sample profile"))); 819 cl::opt<ProfileFormat> OutputFormat( 820 cl::desc("Format of output profile"), cl::init(PF_Binary), 821 cl::values( 822 clEnumValN(PF_Binary, "binary", "Binary encoding (default)"), 823 clEnumValN(PF_Compact_Binary, "compbinary", 824 "Compact binary encoding"), 825 clEnumValN(PF_Ext_Binary, "extbinary", "Extensible binary encoding"), 826 clEnumValN(PF_Text, "text", "Text encoding"), 827 clEnumValN(PF_GCC, "gcc", 828 "GCC encoding (only meaningful for -sample)"))); 829 cl::opt<FailureMode> FailureMode( 830 "failure-mode", cl::init(failIfAnyAreInvalid), cl::desc("Failure mode:"), 831 cl::values(clEnumValN(failIfAnyAreInvalid, "any", 832 "Fail if any profile is invalid."), 833 clEnumValN(failIfAllAreInvalid, "all", 834 "Fail only if all profiles are invalid."))); 835 cl::opt<bool> OutputSparse("sparse", cl::init(false), 836 cl::desc("Generate a sparse profile (only meaningful for -instr)")); 837 cl::opt<unsigned> NumThreads( 838 "num-threads", cl::init(0), 839 cl::desc("Number of merge threads to use (default: autodetect)")); 840 cl::alias NumThreadsA("j", cl::desc("Alias for --num-threads"), 841 cl::aliasopt(NumThreads)); 842 cl::opt<std::string> ProfileSymbolListFile( 843 "prof-sym-list", cl::init(""), 844 cl::desc("Path to file containing the list of function symbols " 845 "used to populate profile symbol list")); 846 cl::opt<bool> CompressAllSections( 847 "compress-all-sections", cl::init(false), cl::Hidden, 848 cl::desc("Compress all sections when writing the profile (only " 849 "meaningful for -extbinary)")); 850 cl::opt<bool> UseMD5( 851 "use-md5", cl::init(false), cl::Hidden, 852 cl::desc("Choose to use MD5 to represent string in name table (only " 853 "meaningful for -extbinary)")); 854 cl::opt<bool> GenPartialProfile( 855 "gen-partial-profile", cl::init(false), cl::Hidden, 856 cl::desc("Generate a partial profile (only meaningful for -extbinary)")); 857 cl::opt<std::string> SupplInstrWithSample( 858 "supplement-instr-with-sample", cl::init(""), cl::Hidden, 859 cl::desc("Supplement an instr profile with sample profile, to correct " 860 "the profile unrepresentativeness issue. The sample " 861 "profile is the input of the flag. Output will be in instr " 862 "format (The flag only works with -instr)")); 863 cl::opt<float> ZeroCounterThreshold( 864 "zero-counter-threshold", cl::init(0.7), cl::Hidden, 865 cl::desc("For the function which is cold in instr profile but hot in " 866 "sample profile, if the ratio of the number of zero counters " 867 "divided by the the total number of counters is above the " 868 "threshold, the profile of the function will be regarded as " 869 "being harmful for performance and will be dropped. ")); 870 cl::opt<unsigned> SupplMinSizeThreshold( 871 "suppl-min-size-threshold", cl::init(10), cl::Hidden, 872 cl::desc("If the size of a function is smaller than the threshold, " 873 "assume it can be inlined by PGO early inliner and it won't " 874 "be adjusted based on sample profile. ")); 875 cl::opt<unsigned> InstrProfColdThreshold( 876 "instr-prof-cold-threshold", cl::init(0), cl::Hidden, 877 cl::desc("User specified cold threshold for instr profile which will " 878 "override the cold threshold got from profile summary. ")); 879 880 cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n"); 881 882 WeightedFileVector WeightedInputs; 883 for (StringRef Filename : InputFilenames) 884 addWeightedInput(WeightedInputs, {std::string(Filename), 1}); 885 for (StringRef WeightedFilename : WeightedInputFilenames) 886 addWeightedInput(WeightedInputs, parseWeightedFile(WeightedFilename)); 887 888 // Make sure that the file buffer stays alive for the duration of the 889 // weighted input vector's lifetime. 890 auto Buffer = getInputFileBuf(InputFilenamesFile); 891 parseInputFilenamesFile(Buffer.get(), WeightedInputs); 892 893 if (WeightedInputs.empty()) 894 exitWithError("No input files specified. See " + 895 sys::path::filename(argv[0]) + " -help"); 896 897 if (DumpInputFileList) { 898 for (auto &WF : WeightedInputs) 899 outs() << WF.Weight << "," << WF.Filename << "\n"; 900 return 0; 901 } 902 903 std::unique_ptr<SymbolRemapper> Remapper; 904 if (!RemappingFile.empty()) 905 Remapper = SymbolRemapper::create(RemappingFile); 906 907 if (!SupplInstrWithSample.empty()) { 908 if (ProfileKind != instr) 909 exitWithError( 910 "-supplement-instr-with-sample can only work with -instr. "); 911 912 supplementInstrProfile(WeightedInputs, SupplInstrWithSample, OutputFilename, 913 OutputFormat, OutputSparse, SupplMinSizeThreshold, 914 ZeroCounterThreshold, InstrProfColdThreshold); 915 return 0; 916 } 917 918 if (ProfileKind == instr) 919 mergeInstrProfile(WeightedInputs, Remapper.get(), OutputFilename, 920 OutputFormat, OutputSparse, NumThreads, FailureMode); 921 else 922 mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename, 923 OutputFormat, ProfileSymbolListFile, CompressAllSections, 924 UseMD5, GenPartialProfile, FailureMode); 925 926 return 0; 927 } 928 929 /// Computer the overlap b/w profile BaseFilename and profile TestFilename. 930 static void overlapInstrProfile(const std::string &BaseFilename, 931 const std::string &TestFilename, 932 const OverlapFuncFilters &FuncFilter, 933 raw_fd_ostream &OS, bool IsCS) { 934 std::mutex ErrorLock; 935 SmallSet<instrprof_error, 4> WriterErrorCodes; 936 WriterContext Context(false, ErrorLock, WriterErrorCodes); 937 WeightedFile WeightedInput{BaseFilename, 1}; 938 OverlapStats Overlap; 939 Error E = Overlap.accumulateCounts(BaseFilename, TestFilename, IsCS); 940 if (E) 941 exitWithError(std::move(E), "Error in getting profile count sums"); 942 if (Overlap.Base.CountSum < 1.0f) { 943 OS << "Sum of edge counts for profile " << BaseFilename << " is 0.\n"; 944 exit(0); 945 } 946 if (Overlap.Test.CountSum < 1.0f) { 947 OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n"; 948 exit(0); 949 } 950 loadInput(WeightedInput, nullptr, &Context); 951 overlapInput(BaseFilename, TestFilename, &Context, Overlap, FuncFilter, OS, 952 IsCS); 953 Overlap.dump(OS); 954 } 955 956 namespace { 957 struct SampleOverlapStats { 958 StringRef BaseName; 959 StringRef TestName; 960 // Number of overlap units 961 uint64_t OverlapCount; 962 // Total samples of overlap units 963 uint64_t OverlapSample; 964 // Number of and total samples of units that only present in base or test 965 // profile 966 uint64_t BaseUniqueCount; 967 uint64_t BaseUniqueSample; 968 uint64_t TestUniqueCount; 969 uint64_t TestUniqueSample; 970 // Number of units and total samples in base or test profile 971 uint64_t BaseCount; 972 uint64_t BaseSample; 973 uint64_t TestCount; 974 uint64_t TestSample; 975 // Number of and total samples of units that present in at least one profile 976 uint64_t UnionCount; 977 uint64_t UnionSample; 978 // Weighted similarity 979 double Similarity; 980 // For SampleOverlapStats instances representing functions, weights of the 981 // function in base and test profiles 982 double BaseWeight; 983 double TestWeight; 984 985 SampleOverlapStats() 986 : OverlapCount(0), OverlapSample(0), BaseUniqueCount(0), 987 BaseUniqueSample(0), TestUniqueCount(0), TestUniqueSample(0), 988 BaseCount(0), BaseSample(0), TestCount(0), TestSample(0), UnionCount(0), 989 UnionSample(0), Similarity(0.0), BaseWeight(0.0), TestWeight(0.0) {} 990 }; 991 } // end anonymous namespace 992 993 namespace { 994 struct FuncSampleStats { 995 uint64_t SampleSum; 996 uint64_t MaxSample; 997 uint64_t HotBlockCount; 998 FuncSampleStats() : SampleSum(0), MaxSample(0), HotBlockCount(0) {} 999 FuncSampleStats(uint64_t SampleSum, uint64_t MaxSample, 1000 uint64_t HotBlockCount) 1001 : SampleSum(SampleSum), MaxSample(MaxSample), 1002 HotBlockCount(HotBlockCount) {} 1003 }; 1004 } // end anonymous namespace 1005 1006 namespace { 1007 enum MatchStatus { MS_Match, MS_FirstUnique, MS_SecondUnique, MS_None }; 1008 1009 // Class for updating merging steps for two sorted maps. The class should be 1010 // instantiated with a map iterator type. 1011 template <class T> class MatchStep { 1012 public: 1013 MatchStep() = delete; 1014 1015 MatchStep(T FirstIter, T FirstEnd, T SecondIter, T SecondEnd) 1016 : FirstIter(FirstIter), FirstEnd(FirstEnd), SecondIter(SecondIter), 1017 SecondEnd(SecondEnd), Status(MS_None) {} 1018 1019 bool areBothFinished() const { 1020 return (FirstIter == FirstEnd && SecondIter == SecondEnd); 1021 } 1022 1023 bool isFirstFinished() const { return FirstIter == FirstEnd; } 1024 1025 bool isSecondFinished() const { return SecondIter == SecondEnd; } 1026 1027 /// Advance one step based on the previous match status unless the previous 1028 /// status is MS_None. Then update Status based on the comparison between two 1029 /// container iterators at the current step. If the previous status is 1030 /// MS_None, it means two iterators are at the beginning and no comparison has 1031 /// been made, so we simply update Status without advancing the iterators. 1032 void updateOneStep(); 1033 1034 T getFirstIter() const { return FirstIter; } 1035 1036 T getSecondIter() const { return SecondIter; } 1037 1038 MatchStatus getMatchStatus() const { return Status; } 1039 1040 private: 1041 // Current iterator and end iterator of the first container. 1042 T FirstIter; 1043 T FirstEnd; 1044 // Current iterator and end iterator of the second container. 1045 T SecondIter; 1046 T SecondEnd; 1047 // Match status of the current step. 1048 MatchStatus Status; 1049 }; 1050 } // end anonymous namespace 1051 1052 template <class T> void MatchStep<T>::updateOneStep() { 1053 switch (Status) { 1054 case MS_Match: 1055 ++FirstIter; 1056 ++SecondIter; 1057 break; 1058 case MS_FirstUnique: 1059 ++FirstIter; 1060 break; 1061 case MS_SecondUnique: 1062 ++SecondIter; 1063 break; 1064 case MS_None: 1065 break; 1066 } 1067 1068 // Update Status according to iterators at the current step. 1069 if (areBothFinished()) 1070 return; 1071 if (FirstIter != FirstEnd && 1072 (SecondIter == SecondEnd || FirstIter->first < SecondIter->first)) 1073 Status = MS_FirstUnique; 1074 else if (SecondIter != SecondEnd && 1075 (FirstIter == FirstEnd || SecondIter->first < FirstIter->first)) 1076 Status = MS_SecondUnique; 1077 else 1078 Status = MS_Match; 1079 } 1080 1081 // Return the sum of line/block samples, the max line/block sample, and the 1082 // number of line/block samples above the given threshold in a function 1083 // including its inlinees. 1084 static void getFuncSampleStats(const sampleprof::FunctionSamples &Func, 1085 FuncSampleStats &FuncStats, 1086 uint64_t HotThreshold) { 1087 for (const auto &L : Func.getBodySamples()) { 1088 uint64_t Sample = L.second.getSamples(); 1089 FuncStats.SampleSum += Sample; 1090 FuncStats.MaxSample = std::max(FuncStats.MaxSample, Sample); 1091 if (Sample >= HotThreshold) 1092 ++FuncStats.HotBlockCount; 1093 } 1094 1095 for (const auto &C : Func.getCallsiteSamples()) { 1096 for (const auto &F : C.second) 1097 getFuncSampleStats(F.second, FuncStats, HotThreshold); 1098 } 1099 } 1100 1101 /// Predicate that determines if a function is hot with a given threshold. We 1102 /// keep it separate from its callsites for possible extension in the future. 1103 static bool isFunctionHot(const FuncSampleStats &FuncStats, 1104 uint64_t HotThreshold) { 1105 // We intentionally compare the maximum sample count in a function with the 1106 // HotThreshold to get an approximate determination on hot functions. 1107 return (FuncStats.MaxSample >= HotThreshold); 1108 } 1109 1110 namespace { 1111 class SampleOverlapAggregator { 1112 public: 1113 SampleOverlapAggregator(const std::string &BaseFilename, 1114 const std::string &TestFilename, 1115 double LowSimilarityThreshold, double Epsilon, 1116 const OverlapFuncFilters &FuncFilter) 1117 : BaseFilename(BaseFilename), TestFilename(TestFilename), 1118 LowSimilarityThreshold(LowSimilarityThreshold), Epsilon(Epsilon), 1119 FuncFilter(FuncFilter) {} 1120 1121 /// Detect 0-sample input profile and report to output stream. This interface 1122 /// should be called after loadProfiles(). 1123 bool detectZeroSampleProfile(raw_fd_ostream &OS) const; 1124 1125 /// Write out function-level similarity statistics for functions specified by 1126 /// options --function, --value-cutoff, and --similarity-cutoff. 1127 void dumpFuncSimilarity(raw_fd_ostream &OS) const; 1128 1129 /// Write out program-level similarity and overlap statistics. 1130 void dumpProgramSummary(raw_fd_ostream &OS) const; 1131 1132 /// Write out hot-function and hot-block statistics for base_profile, 1133 /// test_profile, and their overlap. For both cases, the overlap HO is 1134 /// calculated as follows: 1135 /// Given the number of functions (or blocks) that are hot in both profiles 1136 /// HCommon and the number of functions (or blocks) that are hot in at 1137 /// least one profile HUnion, HO = HCommon / HUnion. 1138 void dumpHotFuncAndBlockOverlap(raw_fd_ostream &OS) const; 1139 1140 /// This function tries matching functions in base and test profiles. For each 1141 /// pair of matched functions, it aggregates the function-level 1142 /// similarity into a profile-level similarity. It also dump function-level 1143 /// similarity information of functions specified by --function, 1144 /// --value-cutoff, and --similarity-cutoff options. The program-level 1145 /// similarity PS is computed as follows: 1146 /// Given function-level similarity FS(A) for all function A, the 1147 /// weight of function A in base profile WB(A), and the weight of function 1148 /// A in test profile WT(A), compute PS(base_profile, test_profile) = 1149 /// sum_A(FS(A) * avg(WB(A), WT(A))) ranging in [0.0f to 1.0f] with 0.0 1150 /// meaning no-overlap. 1151 void computeSampleProfileOverlap(raw_fd_ostream &OS); 1152 1153 /// Initialize ProfOverlap with the sum of samples in base and test 1154 /// profiles. This function also computes and keeps the sum of samples and 1155 /// max sample counts of each function in BaseStats and TestStats for later 1156 /// use to avoid re-computations. 1157 void initializeSampleProfileOverlap(); 1158 1159 /// Load profiles specified by BaseFilename and TestFilename. 1160 std::error_code loadProfiles(); 1161 1162 private: 1163 SampleOverlapStats ProfOverlap; 1164 SampleOverlapStats HotFuncOverlap; 1165 SampleOverlapStats HotBlockOverlap; 1166 std::string BaseFilename; 1167 std::string TestFilename; 1168 std::unique_ptr<sampleprof::SampleProfileReader> BaseReader; 1169 std::unique_ptr<sampleprof::SampleProfileReader> TestReader; 1170 // BaseStats and TestStats hold FuncSampleStats for each function, with 1171 // function name as the key. 1172 StringMap<FuncSampleStats> BaseStats; 1173 StringMap<FuncSampleStats> TestStats; 1174 // Low similarity threshold in floating point number 1175 double LowSimilarityThreshold; 1176 // Block samples above BaseHotThreshold or TestHotThreshold are considered hot 1177 // for tracking hot blocks. 1178 uint64_t BaseHotThreshold; 1179 uint64_t TestHotThreshold; 1180 // A small threshold used to round the results of floating point accumulations 1181 // to resolve imprecision. 1182 const double Epsilon; 1183 std::multimap<double, SampleOverlapStats, std::greater<double>> 1184 FuncSimilarityDump; 1185 // FuncFilter carries specifications in options --value-cutoff and 1186 // --function. 1187 OverlapFuncFilters FuncFilter; 1188 // Column offsets for printing the function-level details table. 1189 static const unsigned int TestWeightCol = 15; 1190 static const unsigned int SimilarityCol = 30; 1191 static const unsigned int OverlapCol = 43; 1192 static const unsigned int BaseUniqueCol = 53; 1193 static const unsigned int TestUniqueCol = 67; 1194 static const unsigned int BaseSampleCol = 81; 1195 static const unsigned int TestSampleCol = 96; 1196 static const unsigned int FuncNameCol = 111; 1197 1198 /// Return a similarity of two line/block sample counters in the same 1199 /// function in base and test profiles. The line/block-similarity BS(i) is 1200 /// computed as follows: 1201 /// For an offsets i, given the sample count at i in base profile BB(i), 1202 /// the sample count at i in test profile BT(i), the sum of sample counts 1203 /// in this function in base profile SB, and the sum of sample counts in 1204 /// this function in test profile ST, compute BS(i) = 1.0 - fabs(BB(i)/SB - 1205 /// BT(i)/ST), ranging in [0.0f to 1.0f] with 0.0 meaning no-overlap. 1206 double computeBlockSimilarity(uint64_t BaseSample, uint64_t TestSample, 1207 const SampleOverlapStats &FuncOverlap) const; 1208 1209 void updateHotBlockOverlap(uint64_t BaseSample, uint64_t TestSample, 1210 uint64_t HotBlockCount); 1211 1212 void getHotFunctions(const StringMap<FuncSampleStats> &ProfStats, 1213 StringMap<FuncSampleStats> &HotFunc, 1214 uint64_t HotThreshold) const; 1215 1216 void computeHotFuncOverlap(); 1217 1218 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and 1219 /// Difference for two sample units in a matched function according to the 1220 /// given match status. 1221 void updateOverlapStatsForFunction(uint64_t BaseSample, uint64_t TestSample, 1222 uint64_t HotBlockCount, 1223 SampleOverlapStats &FuncOverlap, 1224 double &Difference, MatchStatus Status); 1225 1226 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and 1227 /// Difference for unmatched callees that only present in one profile in a 1228 /// matched caller function. 1229 void updateForUnmatchedCallee(const sampleprof::FunctionSamples &Func, 1230 SampleOverlapStats &FuncOverlap, 1231 double &Difference, MatchStatus Status); 1232 1233 /// This function updates sample overlap statistics of an overlap function in 1234 /// base and test profile. It also calculates a function-internal similarity 1235 /// FIS as follows: 1236 /// For offsets i that have samples in at least one profile in this 1237 /// function A, given BS(i) returned by computeBlockSimilarity(), compute 1238 /// FIS(A) = (2.0 - sum_i(1.0 - BS(i))) / 2, ranging in [0.0f to 1.0f] with 1239 /// 0.0 meaning no overlap. 1240 double computeSampleFunctionInternalOverlap( 1241 const sampleprof::FunctionSamples &BaseFunc, 1242 const sampleprof::FunctionSamples &TestFunc, 1243 SampleOverlapStats &FuncOverlap); 1244 1245 /// Function-level similarity (FS) is a weighted value over function internal 1246 /// similarity (FIS). This function computes a function's FS from its FIS by 1247 /// applying the weight. 1248 double weightForFuncSimilarity(double FuncSimilarity, uint64_t BaseFuncSample, 1249 uint64_t TestFuncSample) const; 1250 1251 /// The function-level similarity FS(A) for a function A is computed as 1252 /// follows: 1253 /// Compute a function-internal similarity FIS(A) by 1254 /// computeSampleFunctionInternalOverlap(). Then, with the weight of 1255 /// function A in base profile WB(A), and the weight of function A in test 1256 /// profile WT(A), compute FS(A) = FIS(A) * (1.0 - fabs(WB(A) - WT(A))) 1257 /// ranging in [0.0f to 1.0f] with 0.0 meaning no overlap. 1258 double 1259 computeSampleFunctionOverlap(const sampleprof::FunctionSamples *BaseFunc, 1260 const sampleprof::FunctionSamples *TestFunc, 1261 SampleOverlapStats *FuncOverlap, 1262 uint64_t BaseFuncSample, 1263 uint64_t TestFuncSample); 1264 1265 /// Profile-level similarity (PS) is a weighted aggregate over function-level 1266 /// similarities (FS). This method weights the FS value by the function 1267 /// weights in the base and test profiles for the aggregation. 1268 double weightByImportance(double FuncSimilarity, uint64_t BaseFuncSample, 1269 uint64_t TestFuncSample) const; 1270 }; 1271 } // end anonymous namespace 1272 1273 bool SampleOverlapAggregator::detectZeroSampleProfile( 1274 raw_fd_ostream &OS) const { 1275 bool HaveZeroSample = false; 1276 if (ProfOverlap.BaseSample == 0) { 1277 OS << "Sum of sample counts for profile " << BaseFilename << " is 0.\n"; 1278 HaveZeroSample = true; 1279 } 1280 if (ProfOverlap.TestSample == 0) { 1281 OS << "Sum of sample counts for profile " << TestFilename << " is 0.\n"; 1282 HaveZeroSample = true; 1283 } 1284 return HaveZeroSample; 1285 } 1286 1287 double SampleOverlapAggregator::computeBlockSimilarity( 1288 uint64_t BaseSample, uint64_t TestSample, 1289 const SampleOverlapStats &FuncOverlap) const { 1290 double BaseFrac = 0.0; 1291 double TestFrac = 0.0; 1292 if (FuncOverlap.BaseSample > 0) 1293 BaseFrac = static_cast<double>(BaseSample) / FuncOverlap.BaseSample; 1294 if (FuncOverlap.TestSample > 0) 1295 TestFrac = static_cast<double>(TestSample) / FuncOverlap.TestSample; 1296 return 1.0 - std::fabs(BaseFrac - TestFrac); 1297 } 1298 1299 void SampleOverlapAggregator::updateHotBlockOverlap(uint64_t BaseSample, 1300 uint64_t TestSample, 1301 uint64_t HotBlockCount) { 1302 bool IsBaseHot = (BaseSample >= BaseHotThreshold); 1303 bool IsTestHot = (TestSample >= TestHotThreshold); 1304 if (!IsBaseHot && !IsTestHot) 1305 return; 1306 1307 HotBlockOverlap.UnionCount += HotBlockCount; 1308 if (IsBaseHot) 1309 HotBlockOverlap.BaseCount += HotBlockCount; 1310 if (IsTestHot) 1311 HotBlockOverlap.TestCount += HotBlockCount; 1312 if (IsBaseHot && IsTestHot) 1313 HotBlockOverlap.OverlapCount += HotBlockCount; 1314 } 1315 1316 void SampleOverlapAggregator::getHotFunctions( 1317 const StringMap<FuncSampleStats> &ProfStats, 1318 StringMap<FuncSampleStats> &HotFunc, uint64_t HotThreshold) const { 1319 for (const auto &F : ProfStats) { 1320 if (isFunctionHot(F.second, HotThreshold)) 1321 HotFunc.try_emplace(F.first(), F.second); 1322 } 1323 } 1324 1325 void SampleOverlapAggregator::computeHotFuncOverlap() { 1326 StringMap<FuncSampleStats> BaseHotFunc; 1327 getHotFunctions(BaseStats, BaseHotFunc, BaseHotThreshold); 1328 HotFuncOverlap.BaseCount = BaseHotFunc.size(); 1329 1330 StringMap<FuncSampleStats> TestHotFunc; 1331 getHotFunctions(TestStats, TestHotFunc, TestHotThreshold); 1332 HotFuncOverlap.TestCount = TestHotFunc.size(); 1333 HotFuncOverlap.UnionCount = HotFuncOverlap.TestCount; 1334 1335 for (const auto &F : BaseHotFunc) { 1336 if (TestHotFunc.count(F.first())) 1337 ++HotFuncOverlap.OverlapCount; 1338 else 1339 ++HotFuncOverlap.UnionCount; 1340 } 1341 } 1342 1343 void SampleOverlapAggregator::updateOverlapStatsForFunction( 1344 uint64_t BaseSample, uint64_t TestSample, uint64_t HotBlockCount, 1345 SampleOverlapStats &FuncOverlap, double &Difference, MatchStatus Status) { 1346 assert(Status != MS_None && 1347 "Match status should be updated before updating overlap statistics"); 1348 if (Status == MS_FirstUnique) { 1349 TestSample = 0; 1350 FuncOverlap.BaseUniqueSample += BaseSample; 1351 } else if (Status == MS_SecondUnique) { 1352 BaseSample = 0; 1353 FuncOverlap.TestUniqueSample += TestSample; 1354 } else { 1355 ++FuncOverlap.OverlapCount; 1356 } 1357 1358 FuncOverlap.UnionSample += std::max(BaseSample, TestSample); 1359 FuncOverlap.OverlapSample += std::min(BaseSample, TestSample); 1360 Difference += 1361 1.0 - computeBlockSimilarity(BaseSample, TestSample, FuncOverlap); 1362 updateHotBlockOverlap(BaseSample, TestSample, HotBlockCount); 1363 } 1364 1365 void SampleOverlapAggregator::updateForUnmatchedCallee( 1366 const sampleprof::FunctionSamples &Func, SampleOverlapStats &FuncOverlap, 1367 double &Difference, MatchStatus Status) { 1368 assert((Status == MS_FirstUnique || Status == MS_SecondUnique) && 1369 "Status must be either of the two unmatched cases"); 1370 FuncSampleStats FuncStats; 1371 if (Status == MS_FirstUnique) { 1372 getFuncSampleStats(Func, FuncStats, BaseHotThreshold); 1373 updateOverlapStatsForFunction(FuncStats.SampleSum, 0, 1374 FuncStats.HotBlockCount, FuncOverlap, 1375 Difference, Status); 1376 } else { 1377 getFuncSampleStats(Func, FuncStats, TestHotThreshold); 1378 updateOverlapStatsForFunction(0, FuncStats.SampleSum, 1379 FuncStats.HotBlockCount, FuncOverlap, 1380 Difference, Status); 1381 } 1382 } 1383 1384 double SampleOverlapAggregator::computeSampleFunctionInternalOverlap( 1385 const sampleprof::FunctionSamples &BaseFunc, 1386 const sampleprof::FunctionSamples &TestFunc, 1387 SampleOverlapStats &FuncOverlap) { 1388 1389 using namespace sampleprof; 1390 1391 double Difference = 0; 1392 1393 // Accumulate Difference for regular line/block samples in the function. 1394 // We match them through sort-merge join algorithm because 1395 // FunctionSamples::getBodySamples() returns a map of sample counters ordered 1396 // by their offsets. 1397 MatchStep<BodySampleMap::const_iterator> BlockIterStep( 1398 BaseFunc.getBodySamples().cbegin(), BaseFunc.getBodySamples().cend(), 1399 TestFunc.getBodySamples().cbegin(), TestFunc.getBodySamples().cend()); 1400 BlockIterStep.updateOneStep(); 1401 while (!BlockIterStep.areBothFinished()) { 1402 uint64_t BaseSample = 1403 BlockIterStep.isFirstFinished() 1404 ? 0 1405 : BlockIterStep.getFirstIter()->second.getSamples(); 1406 uint64_t TestSample = 1407 BlockIterStep.isSecondFinished() 1408 ? 0 1409 : BlockIterStep.getSecondIter()->second.getSamples(); 1410 updateOverlapStatsForFunction(BaseSample, TestSample, 1, FuncOverlap, 1411 Difference, BlockIterStep.getMatchStatus()); 1412 1413 BlockIterStep.updateOneStep(); 1414 } 1415 1416 // Accumulate Difference for callsite lines in the function. We match 1417 // them through sort-merge algorithm because 1418 // FunctionSamples::getCallsiteSamples() returns a map of callsite records 1419 // ordered by their offsets. 1420 MatchStep<CallsiteSampleMap::const_iterator> CallsiteIterStep( 1421 BaseFunc.getCallsiteSamples().cbegin(), 1422 BaseFunc.getCallsiteSamples().cend(), 1423 TestFunc.getCallsiteSamples().cbegin(), 1424 TestFunc.getCallsiteSamples().cend()); 1425 CallsiteIterStep.updateOneStep(); 1426 while (!CallsiteIterStep.areBothFinished()) { 1427 MatchStatus CallsiteStepStatus = CallsiteIterStep.getMatchStatus(); 1428 assert(CallsiteStepStatus != MS_None && 1429 "Match status should be updated before entering loop body"); 1430 1431 if (CallsiteStepStatus != MS_Match) { 1432 auto Callsite = (CallsiteStepStatus == MS_FirstUnique) 1433 ? CallsiteIterStep.getFirstIter() 1434 : CallsiteIterStep.getSecondIter(); 1435 for (const auto &F : Callsite->second) 1436 updateForUnmatchedCallee(F.second, FuncOverlap, Difference, 1437 CallsiteStepStatus); 1438 } else { 1439 // There may be multiple inlinees at the same offset, so we need to try 1440 // matching all of them. This match is implemented through sort-merge 1441 // algorithm because callsite records at the same offset are ordered by 1442 // function names. 1443 MatchStep<FunctionSamplesMap::const_iterator> CalleeIterStep( 1444 CallsiteIterStep.getFirstIter()->second.cbegin(), 1445 CallsiteIterStep.getFirstIter()->second.cend(), 1446 CallsiteIterStep.getSecondIter()->second.cbegin(), 1447 CallsiteIterStep.getSecondIter()->second.cend()); 1448 CalleeIterStep.updateOneStep(); 1449 while (!CalleeIterStep.areBothFinished()) { 1450 MatchStatus CalleeStepStatus = CalleeIterStep.getMatchStatus(); 1451 if (CalleeStepStatus != MS_Match) { 1452 auto Callee = (CalleeStepStatus == MS_FirstUnique) 1453 ? CalleeIterStep.getFirstIter() 1454 : CalleeIterStep.getSecondIter(); 1455 updateForUnmatchedCallee(Callee->second, FuncOverlap, Difference, 1456 CalleeStepStatus); 1457 } else { 1458 // An inlined function can contain other inlinees inside, so compute 1459 // the Difference recursively. 1460 Difference += 2.0 - 2 * computeSampleFunctionInternalOverlap( 1461 CalleeIterStep.getFirstIter()->second, 1462 CalleeIterStep.getSecondIter()->second, 1463 FuncOverlap); 1464 } 1465 CalleeIterStep.updateOneStep(); 1466 } 1467 } 1468 CallsiteIterStep.updateOneStep(); 1469 } 1470 1471 // Difference reflects the total differences of line/block samples in this 1472 // function and ranges in [0.0f to 2.0f]. Take (2.0 - Difference) / 2 to 1473 // reflect the similarity between function profiles in [0.0f to 1.0f]. 1474 return (2.0 - Difference) / 2; 1475 } 1476 1477 double SampleOverlapAggregator::weightForFuncSimilarity( 1478 double FuncInternalSimilarity, uint64_t BaseFuncSample, 1479 uint64_t TestFuncSample) const { 1480 // Compute the weight as the distance between the function weights in two 1481 // profiles. 1482 double BaseFrac = 0.0; 1483 double TestFrac = 0.0; 1484 assert(ProfOverlap.BaseSample > 0 && 1485 "Total samples in base profile should be greater than 0"); 1486 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample; 1487 assert(ProfOverlap.TestSample > 0 && 1488 "Total samples in test profile should be greater than 0"); 1489 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample; 1490 double WeightDistance = std::fabs(BaseFrac - TestFrac); 1491 1492 // Take WeightDistance into the similarity. 1493 return FuncInternalSimilarity * (1 - WeightDistance); 1494 } 1495 1496 double 1497 SampleOverlapAggregator::weightByImportance(double FuncSimilarity, 1498 uint64_t BaseFuncSample, 1499 uint64_t TestFuncSample) const { 1500 1501 double BaseFrac = 0.0; 1502 double TestFrac = 0.0; 1503 assert(ProfOverlap.BaseSample > 0 && 1504 "Total samples in base profile should be greater than 0"); 1505 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample / 2.0; 1506 assert(ProfOverlap.TestSample > 0 && 1507 "Total samples in test profile should be greater than 0"); 1508 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample / 2.0; 1509 return FuncSimilarity * (BaseFrac + TestFrac); 1510 } 1511 1512 double SampleOverlapAggregator::computeSampleFunctionOverlap( 1513 const sampleprof::FunctionSamples *BaseFunc, 1514 const sampleprof::FunctionSamples *TestFunc, 1515 SampleOverlapStats *FuncOverlap, uint64_t BaseFuncSample, 1516 uint64_t TestFuncSample) { 1517 // Default function internal similarity before weighted, meaning two functions 1518 // has no overlap. 1519 const double DefaultFuncInternalSimilarity = 0; 1520 double FuncSimilarity; 1521 double FuncInternalSimilarity; 1522 1523 // If BaseFunc or TestFunc is nullptr, it means the functions do not overlap. 1524 // In this case, we use DefaultFuncInternalSimilarity as the function internal 1525 // similarity. 1526 if (!BaseFunc || !TestFunc) { 1527 FuncInternalSimilarity = DefaultFuncInternalSimilarity; 1528 } else { 1529 assert(FuncOverlap != nullptr && 1530 "FuncOverlap should be provided in this case"); 1531 FuncInternalSimilarity = computeSampleFunctionInternalOverlap( 1532 *BaseFunc, *TestFunc, *FuncOverlap); 1533 // Now, FuncInternalSimilarity may be a little less than 0 due to 1534 // imprecision of floating point accumulations. Make it zero if the 1535 // difference is below Epsilon. 1536 FuncInternalSimilarity = (std::fabs(FuncInternalSimilarity - 0) < Epsilon) 1537 ? 0 1538 : FuncInternalSimilarity; 1539 } 1540 FuncSimilarity = weightForFuncSimilarity(FuncInternalSimilarity, 1541 BaseFuncSample, TestFuncSample); 1542 return FuncSimilarity; 1543 } 1544 1545 void SampleOverlapAggregator::computeSampleProfileOverlap(raw_fd_ostream &OS) { 1546 using namespace sampleprof; 1547 1548 StringMap<const FunctionSamples *> BaseFuncProf; 1549 const auto &BaseProfiles = BaseReader->getProfiles(); 1550 for (const auto &BaseFunc : BaseProfiles) { 1551 BaseFuncProf.try_emplace(BaseFunc.second.getName(), &(BaseFunc.second)); 1552 } 1553 ProfOverlap.UnionCount = BaseFuncProf.size(); 1554 1555 const auto &TestProfiles = TestReader->getProfiles(); 1556 for (const auto &TestFunc : TestProfiles) { 1557 SampleOverlapStats FuncOverlap; 1558 FuncOverlap.TestName = TestFunc.second.getName(); 1559 assert(TestStats.count(FuncOverlap.TestName) && 1560 "TestStats should have records for all functions in test profile " 1561 "except inlinees"); 1562 FuncOverlap.TestSample = TestStats[FuncOverlap.TestName].SampleSum; 1563 1564 const auto Match = BaseFuncProf.find(FuncOverlap.TestName); 1565 if (Match == BaseFuncProf.end()) { 1566 const FuncSampleStats &FuncStats = TestStats[FuncOverlap.TestName]; 1567 ++ProfOverlap.TestUniqueCount; 1568 ProfOverlap.TestUniqueSample += FuncStats.SampleSum; 1569 FuncOverlap.TestUniqueSample = FuncStats.SampleSum; 1570 1571 updateHotBlockOverlap(0, FuncStats.SampleSum, FuncStats.HotBlockCount); 1572 1573 double FuncSimilarity = computeSampleFunctionOverlap( 1574 nullptr, nullptr, nullptr, 0, FuncStats.SampleSum); 1575 ProfOverlap.Similarity += 1576 weightByImportance(FuncSimilarity, 0, FuncStats.SampleSum); 1577 1578 ++ProfOverlap.UnionCount; 1579 ProfOverlap.UnionSample += FuncStats.SampleSum; 1580 } else { 1581 ++ProfOverlap.OverlapCount; 1582 1583 // Two functions match with each other. Compute function-level overlap and 1584 // aggregate them into profile-level overlap. 1585 FuncOverlap.BaseName = Match->second->getName(); 1586 assert(BaseStats.count(FuncOverlap.BaseName) && 1587 "BaseStats should have records for all functions in base profile " 1588 "except inlinees"); 1589 FuncOverlap.BaseSample = BaseStats[FuncOverlap.BaseName].SampleSum; 1590 1591 FuncOverlap.Similarity = computeSampleFunctionOverlap( 1592 Match->second, &TestFunc.second, &FuncOverlap, FuncOverlap.BaseSample, 1593 FuncOverlap.TestSample); 1594 ProfOverlap.Similarity += 1595 weightByImportance(FuncOverlap.Similarity, FuncOverlap.BaseSample, 1596 FuncOverlap.TestSample); 1597 ProfOverlap.OverlapSample += FuncOverlap.OverlapSample; 1598 ProfOverlap.UnionSample += FuncOverlap.UnionSample; 1599 1600 // Accumulate the percentage of base unique and test unique samples into 1601 // ProfOverlap. 1602 ProfOverlap.BaseUniqueSample += FuncOverlap.BaseUniqueSample; 1603 ProfOverlap.TestUniqueSample += FuncOverlap.TestUniqueSample; 1604 1605 // Remove matched base functions for later reporting functions not found 1606 // in test profile. 1607 BaseFuncProf.erase(Match); 1608 } 1609 1610 // Print function-level similarity information if specified by options. 1611 assert(TestStats.count(FuncOverlap.TestName) && 1612 "TestStats should have records for all functions in test profile " 1613 "except inlinees"); 1614 if (TestStats[FuncOverlap.TestName].MaxSample >= FuncFilter.ValueCutoff || 1615 (Match != BaseFuncProf.end() && 1616 FuncOverlap.Similarity < LowSimilarityThreshold) || 1617 (Match != BaseFuncProf.end() && !FuncFilter.NameFilter.empty() && 1618 FuncOverlap.BaseName.find(FuncFilter.NameFilter) != 1619 FuncOverlap.BaseName.npos)) { 1620 assert(ProfOverlap.BaseSample > 0 && 1621 "Total samples in base profile should be greater than 0"); 1622 FuncOverlap.BaseWeight = 1623 static_cast<double>(FuncOverlap.BaseSample) / ProfOverlap.BaseSample; 1624 assert(ProfOverlap.TestSample > 0 && 1625 "Total samples in test profile should be greater than 0"); 1626 FuncOverlap.TestWeight = 1627 static_cast<double>(FuncOverlap.TestSample) / ProfOverlap.TestSample; 1628 FuncSimilarityDump.emplace(FuncOverlap.BaseWeight, FuncOverlap); 1629 } 1630 } 1631 1632 // Traverse through functions in base profile but not in test profile. 1633 for (const auto &F : BaseFuncProf) { 1634 assert(BaseStats.count(F.second->getName()) && 1635 "BaseStats should have records for all functions in base profile " 1636 "except inlinees"); 1637 const FuncSampleStats &FuncStats = BaseStats[F.second->getName()]; 1638 ++ProfOverlap.BaseUniqueCount; 1639 ProfOverlap.BaseUniqueSample += FuncStats.SampleSum; 1640 1641 updateHotBlockOverlap(FuncStats.SampleSum, 0, FuncStats.HotBlockCount); 1642 1643 double FuncSimilarity = computeSampleFunctionOverlap( 1644 nullptr, nullptr, nullptr, FuncStats.SampleSum, 0); 1645 ProfOverlap.Similarity += 1646 weightByImportance(FuncSimilarity, FuncStats.SampleSum, 0); 1647 1648 ProfOverlap.UnionSample += FuncStats.SampleSum; 1649 } 1650 1651 // Now, ProfSimilarity may be a little greater than 1 due to imprecision 1652 // of floating point accumulations. Make it 1.0 if the difference is below 1653 // Epsilon. 1654 ProfOverlap.Similarity = (std::fabs(ProfOverlap.Similarity - 1) < Epsilon) 1655 ? 1 1656 : ProfOverlap.Similarity; 1657 1658 computeHotFuncOverlap(); 1659 } 1660 1661 void SampleOverlapAggregator::initializeSampleProfileOverlap() { 1662 const auto &BaseProf = BaseReader->getProfiles(); 1663 for (const auto &I : BaseProf) { 1664 ++ProfOverlap.BaseCount; 1665 FuncSampleStats FuncStats; 1666 getFuncSampleStats(I.second, FuncStats, BaseHotThreshold); 1667 ProfOverlap.BaseSample += FuncStats.SampleSum; 1668 BaseStats.try_emplace(I.second.getName(), FuncStats); 1669 } 1670 1671 const auto &TestProf = TestReader->getProfiles(); 1672 for (const auto &I : TestProf) { 1673 ++ProfOverlap.TestCount; 1674 FuncSampleStats FuncStats; 1675 getFuncSampleStats(I.second, FuncStats, TestHotThreshold); 1676 ProfOverlap.TestSample += FuncStats.SampleSum; 1677 TestStats.try_emplace(I.second.getName(), FuncStats); 1678 } 1679 1680 ProfOverlap.BaseName = StringRef(BaseFilename); 1681 ProfOverlap.TestName = StringRef(TestFilename); 1682 } 1683 1684 void SampleOverlapAggregator::dumpFuncSimilarity(raw_fd_ostream &OS) const { 1685 using namespace sampleprof; 1686 1687 if (FuncSimilarityDump.empty()) 1688 return; 1689 1690 formatted_raw_ostream FOS(OS); 1691 FOS << "Function-level details:\n"; 1692 FOS << "Base weight"; 1693 FOS.PadToColumn(TestWeightCol); 1694 FOS << "Test weight"; 1695 FOS.PadToColumn(SimilarityCol); 1696 FOS << "Similarity"; 1697 FOS.PadToColumn(OverlapCol); 1698 FOS << "Overlap"; 1699 FOS.PadToColumn(BaseUniqueCol); 1700 FOS << "Base unique"; 1701 FOS.PadToColumn(TestUniqueCol); 1702 FOS << "Test unique"; 1703 FOS.PadToColumn(BaseSampleCol); 1704 FOS << "Base samples"; 1705 FOS.PadToColumn(TestSampleCol); 1706 FOS << "Test samples"; 1707 FOS.PadToColumn(FuncNameCol); 1708 FOS << "Function name\n"; 1709 for (const auto &F : FuncSimilarityDump) { 1710 double OverlapPercent = 1711 F.second.UnionSample > 0 1712 ? static_cast<double>(F.second.OverlapSample) / F.second.UnionSample 1713 : 0; 1714 double BaseUniquePercent = 1715 F.second.BaseSample > 0 1716 ? static_cast<double>(F.second.BaseUniqueSample) / 1717 F.second.BaseSample 1718 : 0; 1719 double TestUniquePercent = 1720 F.second.TestSample > 0 1721 ? static_cast<double>(F.second.TestUniqueSample) / 1722 F.second.TestSample 1723 : 0; 1724 1725 FOS << format("%.2f%%", F.second.BaseWeight * 100); 1726 FOS.PadToColumn(TestWeightCol); 1727 FOS << format("%.2f%%", F.second.TestWeight * 100); 1728 FOS.PadToColumn(SimilarityCol); 1729 FOS << format("%.2f%%", F.second.Similarity * 100); 1730 FOS.PadToColumn(OverlapCol); 1731 FOS << format("%.2f%%", OverlapPercent * 100); 1732 FOS.PadToColumn(BaseUniqueCol); 1733 FOS << format("%.2f%%", BaseUniquePercent * 100); 1734 FOS.PadToColumn(TestUniqueCol); 1735 FOS << format("%.2f%%", TestUniquePercent * 100); 1736 FOS.PadToColumn(BaseSampleCol); 1737 FOS << F.second.BaseSample; 1738 FOS.PadToColumn(TestSampleCol); 1739 FOS << F.second.TestSample; 1740 FOS.PadToColumn(FuncNameCol); 1741 FOS << F.second.TestName << "\n"; 1742 } 1743 } 1744 1745 void SampleOverlapAggregator::dumpProgramSummary(raw_fd_ostream &OS) const { 1746 OS << "Profile overlap infomation for base_profile: " << ProfOverlap.BaseName 1747 << " and test_profile: " << ProfOverlap.TestName << "\nProgram level:\n"; 1748 1749 OS << " Whole program profile similarity: " 1750 << format("%.3f%%", ProfOverlap.Similarity * 100) << "\n"; 1751 1752 assert(ProfOverlap.UnionSample > 0 && 1753 "Total samples in two profile should be greater than 0"); 1754 double OverlapPercent = 1755 static_cast<double>(ProfOverlap.OverlapSample) / ProfOverlap.UnionSample; 1756 assert(ProfOverlap.BaseSample > 0 && 1757 "Total samples in base profile should be greater than 0"); 1758 double BaseUniquePercent = static_cast<double>(ProfOverlap.BaseUniqueSample) / 1759 ProfOverlap.BaseSample; 1760 assert(ProfOverlap.TestSample > 0 && 1761 "Total samples in test profile should be greater than 0"); 1762 double TestUniquePercent = static_cast<double>(ProfOverlap.TestUniqueSample) / 1763 ProfOverlap.TestSample; 1764 1765 OS << " Whole program sample overlap: " 1766 << format("%.3f%%", OverlapPercent * 100) << "\n"; 1767 OS << " percentage of samples unique in base profile: " 1768 << format("%.3f%%", BaseUniquePercent * 100) << "\n"; 1769 OS << " percentage of samples unique in test profile: " 1770 << format("%.3f%%", TestUniquePercent * 100) << "\n"; 1771 OS << " total samples in base profile: " << ProfOverlap.BaseSample << "\n" 1772 << " total samples in test profile: " << ProfOverlap.TestSample << "\n"; 1773 1774 assert(ProfOverlap.UnionCount > 0 && 1775 "There should be at least one function in two input profiles"); 1776 double FuncOverlapPercent = 1777 static_cast<double>(ProfOverlap.OverlapCount) / ProfOverlap.UnionCount; 1778 OS << " Function overlap: " << format("%.3f%%", FuncOverlapPercent * 100) 1779 << "\n"; 1780 OS << " overlap functions: " << ProfOverlap.OverlapCount << "\n"; 1781 OS << " functions unique in base profile: " << ProfOverlap.BaseUniqueCount 1782 << "\n"; 1783 OS << " functions unique in test profile: " << ProfOverlap.TestUniqueCount 1784 << "\n"; 1785 } 1786 1787 void SampleOverlapAggregator::dumpHotFuncAndBlockOverlap( 1788 raw_fd_ostream &OS) const { 1789 assert(HotFuncOverlap.UnionCount > 0 && 1790 "There should be at least one hot function in two input profiles"); 1791 OS << " Hot-function overlap: " 1792 << format("%.3f%%", static_cast<double>(HotFuncOverlap.OverlapCount) / 1793 HotFuncOverlap.UnionCount * 100) 1794 << "\n"; 1795 OS << " overlap hot functions: " << HotFuncOverlap.OverlapCount << "\n"; 1796 OS << " hot functions unique in base profile: " 1797 << HotFuncOverlap.BaseCount - HotFuncOverlap.OverlapCount << "\n"; 1798 OS << " hot functions unique in test profile: " 1799 << HotFuncOverlap.TestCount - HotFuncOverlap.OverlapCount << "\n"; 1800 1801 assert(HotBlockOverlap.UnionCount > 0 && 1802 "There should be at least one hot block in two input profiles"); 1803 OS << " Hot-block overlap: " 1804 << format("%.3f%%", static_cast<double>(HotBlockOverlap.OverlapCount) / 1805 HotBlockOverlap.UnionCount * 100) 1806 << "\n"; 1807 OS << " overlap hot blocks: " << HotBlockOverlap.OverlapCount << "\n"; 1808 OS << " hot blocks unique in base profile: " 1809 << HotBlockOverlap.BaseCount - HotBlockOverlap.OverlapCount << "\n"; 1810 OS << " hot blocks unique in test profile: " 1811 << HotBlockOverlap.TestCount - HotBlockOverlap.OverlapCount << "\n"; 1812 } 1813 1814 std::error_code SampleOverlapAggregator::loadProfiles() { 1815 using namespace sampleprof; 1816 1817 LLVMContext Context; 1818 auto BaseReaderOrErr = SampleProfileReader::create(BaseFilename, Context); 1819 if (std::error_code EC = BaseReaderOrErr.getError()) 1820 exitWithErrorCode(EC, BaseFilename); 1821 1822 auto TestReaderOrErr = SampleProfileReader::create(TestFilename, Context); 1823 if (std::error_code EC = TestReaderOrErr.getError()) 1824 exitWithErrorCode(EC, TestFilename); 1825 1826 BaseReader = std::move(BaseReaderOrErr.get()); 1827 TestReader = std::move(TestReaderOrErr.get()); 1828 1829 if (std::error_code EC = BaseReader->read()) 1830 exitWithErrorCode(EC, BaseFilename); 1831 if (std::error_code EC = TestReader->read()) 1832 exitWithErrorCode(EC, TestFilename); 1833 if (BaseReader->profileIsProbeBased() != TestReader->profileIsProbeBased()) 1834 exitWithError( 1835 "cannot compare probe-based profile with non-probe-based profile"); 1836 1837 // Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in 1838 // profile summary. 1839 const uint64_t HotCutoff = 990000; 1840 ProfileSummary &BasePS = BaseReader->getSummary(); 1841 for (const auto &SummaryEntry : BasePS.getDetailedSummary()) { 1842 if (SummaryEntry.Cutoff == HotCutoff) { 1843 BaseHotThreshold = SummaryEntry.MinCount; 1844 break; 1845 } 1846 } 1847 1848 ProfileSummary &TestPS = TestReader->getSummary(); 1849 for (const auto &SummaryEntry : TestPS.getDetailedSummary()) { 1850 if (SummaryEntry.Cutoff == HotCutoff) { 1851 TestHotThreshold = SummaryEntry.MinCount; 1852 break; 1853 } 1854 } 1855 return std::error_code(); 1856 } 1857 1858 void overlapSampleProfile(const std::string &BaseFilename, 1859 const std::string &TestFilename, 1860 const OverlapFuncFilters &FuncFilter, 1861 uint64_t SimilarityCutoff, raw_fd_ostream &OS) { 1862 using namespace sampleprof; 1863 1864 // We use 0.000005 to initialize OverlapAggr.Epsilon because the final metrics 1865 // report 2--3 places after decimal point in percentage numbers. 1866 SampleOverlapAggregator OverlapAggr( 1867 BaseFilename, TestFilename, 1868 static_cast<double>(SimilarityCutoff) / 1000000, 0.000005, FuncFilter); 1869 if (std::error_code EC = OverlapAggr.loadProfiles()) 1870 exitWithErrorCode(EC); 1871 1872 OverlapAggr.initializeSampleProfileOverlap(); 1873 if (OverlapAggr.detectZeroSampleProfile(OS)) 1874 return; 1875 1876 OverlapAggr.computeSampleProfileOverlap(OS); 1877 1878 OverlapAggr.dumpProgramSummary(OS); 1879 OverlapAggr.dumpHotFuncAndBlockOverlap(OS); 1880 OverlapAggr.dumpFuncSimilarity(OS); 1881 } 1882 1883 static int overlap_main(int argc, const char *argv[]) { 1884 cl::opt<std::string> BaseFilename(cl::Positional, cl::Required, 1885 cl::desc("<base profile file>")); 1886 cl::opt<std::string> TestFilename(cl::Positional, cl::Required, 1887 cl::desc("<test profile file>")); 1888 cl::opt<std::string> Output("output", cl::value_desc("output"), cl::init("-"), 1889 cl::desc("Output file")); 1890 cl::alias OutputA("o", cl::desc("Alias for --output"), cl::aliasopt(Output)); 1891 cl::opt<bool> IsCS("cs", cl::init(false), 1892 cl::desc("For context sensitive counts")); 1893 cl::opt<unsigned long long> ValueCutoff( 1894 "value-cutoff", cl::init(-1), 1895 cl::desc( 1896 "Function level overlap information for every function in test " 1897 "profile with max count value greater then the parameter value")); 1898 cl::opt<std::string> FuncNameFilter( 1899 "function", 1900 cl::desc("Function level overlap information for matching functions")); 1901 cl::opt<unsigned long long> SimilarityCutoff( 1902 "similarity-cutoff", cl::init(0), 1903 cl::desc( 1904 "For sample profiles, list function names for overlapped functions " 1905 "with similarities below the cutoff (percentage times 10000).")); 1906 cl::opt<ProfileKinds> ProfileKind( 1907 cl::desc("Profile kind:"), cl::init(instr), 1908 cl::values(clEnumVal(instr, "Instrumentation profile (default)"), 1909 clEnumVal(sample, "Sample profile"))); 1910 cl::ParseCommandLineOptions(argc, argv, "LLVM profile data overlap tool\n"); 1911 1912 std::error_code EC; 1913 raw_fd_ostream OS(Output.data(), EC, sys::fs::OF_Text); 1914 if (EC) 1915 exitWithErrorCode(EC, Output); 1916 1917 if (ProfileKind == instr) 1918 overlapInstrProfile(BaseFilename, TestFilename, 1919 OverlapFuncFilters{ValueCutoff, FuncNameFilter}, OS, 1920 IsCS); 1921 else 1922 overlapSampleProfile(BaseFilename, TestFilename, 1923 OverlapFuncFilters{ValueCutoff, FuncNameFilter}, 1924 SimilarityCutoff, OS); 1925 1926 return 0; 1927 } 1928 1929 typedef struct ValueSitesStats { 1930 ValueSitesStats() 1931 : TotalNumValueSites(0), TotalNumValueSitesWithValueProfile(0), 1932 TotalNumValues(0) {} 1933 uint64_t TotalNumValueSites; 1934 uint64_t TotalNumValueSitesWithValueProfile; 1935 uint64_t TotalNumValues; 1936 std::vector<unsigned> ValueSitesHistogram; 1937 } ValueSitesStats; 1938 1939 static void traverseAllValueSites(const InstrProfRecord &Func, uint32_t VK, 1940 ValueSitesStats &Stats, raw_fd_ostream &OS, 1941 InstrProfSymtab *Symtab) { 1942 uint32_t NS = Func.getNumValueSites(VK); 1943 Stats.TotalNumValueSites += NS; 1944 for (size_t I = 0; I < NS; ++I) { 1945 uint32_t NV = Func.getNumValueDataForSite(VK, I); 1946 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, I); 1947 Stats.TotalNumValues += NV; 1948 if (NV) { 1949 Stats.TotalNumValueSitesWithValueProfile++; 1950 if (NV > Stats.ValueSitesHistogram.size()) 1951 Stats.ValueSitesHistogram.resize(NV, 0); 1952 Stats.ValueSitesHistogram[NV - 1]++; 1953 } 1954 1955 uint64_t SiteSum = 0; 1956 for (uint32_t V = 0; V < NV; V++) 1957 SiteSum += VD[V].Count; 1958 if (SiteSum == 0) 1959 SiteSum = 1; 1960 1961 for (uint32_t V = 0; V < NV; V++) { 1962 OS << "\t[ " << format("%2u", I) << ", "; 1963 if (Symtab == nullptr) 1964 OS << format("%4" PRIu64, VD[V].Value); 1965 else 1966 OS << Symtab->getFuncName(VD[V].Value); 1967 OS << ", " << format("%10" PRId64, VD[V].Count) << " ] (" 1968 << format("%.2f%%", (VD[V].Count * 100.0 / SiteSum)) << ")\n"; 1969 } 1970 } 1971 } 1972 1973 static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK, 1974 ValueSitesStats &Stats) { 1975 OS << " Total number of sites: " << Stats.TotalNumValueSites << "\n"; 1976 OS << " Total number of sites with values: " 1977 << Stats.TotalNumValueSitesWithValueProfile << "\n"; 1978 OS << " Total number of profiled values: " << Stats.TotalNumValues << "\n"; 1979 1980 OS << " Value sites histogram:\n\tNumTargets, SiteCount\n"; 1981 for (unsigned I = 0; I < Stats.ValueSitesHistogram.size(); I++) { 1982 if (Stats.ValueSitesHistogram[I] > 0) 1983 OS << "\t" << I + 1 << ", " << Stats.ValueSitesHistogram[I] << "\n"; 1984 } 1985 } 1986 1987 static int showInstrProfile(const std::string &Filename, bool ShowCounts, 1988 uint32_t TopN, bool ShowIndirectCallTargets, 1989 bool ShowMemOPSizes, bool ShowDetailedSummary, 1990 std::vector<uint32_t> DetailedSummaryCutoffs, 1991 bool ShowAllFunctions, bool ShowCS, 1992 uint64_t ValueCutoff, bool OnlyListBelow, 1993 const std::string &ShowFunction, bool TextFormat, 1994 raw_fd_ostream &OS) { 1995 auto ReaderOrErr = InstrProfReader::create(Filename); 1996 std::vector<uint32_t> Cutoffs = std::move(DetailedSummaryCutoffs); 1997 if (ShowDetailedSummary && Cutoffs.empty()) { 1998 Cutoffs = {800000, 900000, 950000, 990000, 999000, 999900, 999990}; 1999 } 2000 InstrProfSummaryBuilder Builder(std::move(Cutoffs)); 2001 if (Error E = ReaderOrErr.takeError()) 2002 exitWithError(std::move(E), Filename); 2003 2004 auto Reader = std::move(ReaderOrErr.get()); 2005 bool IsIRInstr = Reader->isIRLevelProfile(); 2006 size_t ShownFunctions = 0; 2007 size_t BelowCutoffFunctions = 0; 2008 int NumVPKind = IPVK_Last - IPVK_First + 1; 2009 std::vector<ValueSitesStats> VPStats(NumVPKind); 2010 2011 auto MinCmp = [](const std::pair<std::string, uint64_t> &v1, 2012 const std::pair<std::string, uint64_t> &v2) { 2013 return v1.second > v2.second; 2014 }; 2015 2016 std::priority_queue<std::pair<std::string, uint64_t>, 2017 std::vector<std::pair<std::string, uint64_t>>, 2018 decltype(MinCmp)> 2019 HottestFuncs(MinCmp); 2020 2021 if (!TextFormat && OnlyListBelow) { 2022 OS << "The list of functions with the maximum counter less than " 2023 << ValueCutoff << ":\n"; 2024 } 2025 2026 // Add marker so that IR-level instrumentation round-trips properly. 2027 if (TextFormat && IsIRInstr) 2028 OS << ":ir\n"; 2029 2030 for (const auto &Func : *Reader) { 2031 if (Reader->isIRLevelProfile()) { 2032 bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(Func.Hash); 2033 if (FuncIsCS != ShowCS) 2034 continue; 2035 } 2036 bool Show = 2037 ShowAllFunctions || (!ShowFunction.empty() && 2038 Func.Name.find(ShowFunction) != Func.Name.npos); 2039 2040 bool doTextFormatDump = (Show && TextFormat); 2041 2042 if (doTextFormatDump) { 2043 InstrProfSymtab &Symtab = Reader->getSymtab(); 2044 InstrProfWriter::writeRecordInText(Func.Name, Func.Hash, Func, Symtab, 2045 OS); 2046 continue; 2047 } 2048 2049 assert(Func.Counts.size() > 0 && "function missing entry counter"); 2050 Builder.addRecord(Func); 2051 2052 uint64_t FuncMax = 0; 2053 uint64_t FuncSum = 0; 2054 for (size_t I = 0, E = Func.Counts.size(); I < E; ++I) { 2055 if (Func.Counts[I] == (uint64_t)-1) 2056 continue; 2057 FuncMax = std::max(FuncMax, Func.Counts[I]); 2058 FuncSum += Func.Counts[I]; 2059 } 2060 2061 if (FuncMax < ValueCutoff) { 2062 ++BelowCutoffFunctions; 2063 if (OnlyListBelow) { 2064 OS << " " << Func.Name << ": (Max = " << FuncMax 2065 << " Sum = " << FuncSum << ")\n"; 2066 } 2067 continue; 2068 } else if (OnlyListBelow) 2069 continue; 2070 2071 if (TopN) { 2072 if (HottestFuncs.size() == TopN) { 2073 if (HottestFuncs.top().second < FuncMax) { 2074 HottestFuncs.pop(); 2075 HottestFuncs.emplace(std::make_pair(std::string(Func.Name), FuncMax)); 2076 } 2077 } else 2078 HottestFuncs.emplace(std::make_pair(std::string(Func.Name), FuncMax)); 2079 } 2080 2081 if (Show) { 2082 if (!ShownFunctions) 2083 OS << "Counters:\n"; 2084 2085 ++ShownFunctions; 2086 2087 OS << " " << Func.Name << ":\n" 2088 << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n" 2089 << " Counters: " << Func.Counts.size() << "\n"; 2090 if (!IsIRInstr) 2091 OS << " Function count: " << Func.Counts[0] << "\n"; 2092 2093 if (ShowIndirectCallTargets) 2094 OS << " Indirect Call Site Count: " 2095 << Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n"; 2096 2097 uint32_t NumMemOPCalls = Func.getNumValueSites(IPVK_MemOPSize); 2098 if (ShowMemOPSizes && NumMemOPCalls > 0) 2099 OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls 2100 << "\n"; 2101 2102 if (ShowCounts) { 2103 OS << " Block counts: ["; 2104 size_t Start = (IsIRInstr ? 0 : 1); 2105 for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) { 2106 OS << (I == Start ? "" : ", ") << Func.Counts[I]; 2107 } 2108 OS << "]\n"; 2109 } 2110 2111 if (ShowIndirectCallTargets) { 2112 OS << " Indirect Target Results:\n"; 2113 traverseAllValueSites(Func, IPVK_IndirectCallTarget, 2114 VPStats[IPVK_IndirectCallTarget], OS, 2115 &(Reader->getSymtab())); 2116 } 2117 2118 if (ShowMemOPSizes && NumMemOPCalls > 0) { 2119 OS << " Memory Intrinsic Size Results:\n"; 2120 traverseAllValueSites(Func, IPVK_MemOPSize, VPStats[IPVK_MemOPSize], OS, 2121 nullptr); 2122 } 2123 } 2124 } 2125 if (Reader->hasError()) 2126 exitWithError(Reader->getError(), Filename); 2127 2128 if (TextFormat) 2129 return 0; 2130 std::unique_ptr<ProfileSummary> PS(Builder.getSummary()); 2131 bool IsIR = Reader->isIRLevelProfile(); 2132 OS << "Instrumentation level: " << (IsIR ? "IR" : "Front-end"); 2133 if (IsIR) 2134 OS << " entry_first = " << Reader->instrEntryBBEnabled(); 2135 OS << "\n"; 2136 if (ShowAllFunctions || !ShowFunction.empty()) 2137 OS << "Functions shown: " << ShownFunctions << "\n"; 2138 OS << "Total functions: " << PS->getNumFunctions() << "\n"; 2139 if (ValueCutoff > 0) { 2140 OS << "Number of functions with maximum count (< " << ValueCutoff 2141 << "): " << BelowCutoffFunctions << "\n"; 2142 OS << "Number of functions with maximum count (>= " << ValueCutoff 2143 << "): " << PS->getNumFunctions() - BelowCutoffFunctions << "\n"; 2144 } 2145 OS << "Maximum function count: " << PS->getMaxFunctionCount() << "\n"; 2146 OS << "Maximum internal block count: " << PS->getMaxInternalCount() << "\n"; 2147 2148 if (TopN) { 2149 std::vector<std::pair<std::string, uint64_t>> SortedHottestFuncs; 2150 while (!HottestFuncs.empty()) { 2151 SortedHottestFuncs.emplace_back(HottestFuncs.top()); 2152 HottestFuncs.pop(); 2153 } 2154 OS << "Top " << TopN 2155 << " functions with the largest internal block counts: \n"; 2156 for (auto &hotfunc : llvm::reverse(SortedHottestFuncs)) 2157 OS << " " << hotfunc.first << ", max count = " << hotfunc.second << "\n"; 2158 } 2159 2160 if (ShownFunctions && ShowIndirectCallTargets) { 2161 OS << "Statistics for indirect call sites profile:\n"; 2162 showValueSitesStats(OS, IPVK_IndirectCallTarget, 2163 VPStats[IPVK_IndirectCallTarget]); 2164 } 2165 2166 if (ShownFunctions && ShowMemOPSizes) { 2167 OS << "Statistics for memory intrinsic calls sizes profile:\n"; 2168 showValueSitesStats(OS, IPVK_MemOPSize, VPStats[IPVK_MemOPSize]); 2169 } 2170 2171 if (ShowDetailedSummary) { 2172 OS << "Total number of blocks: " << PS->getNumCounts() << "\n"; 2173 OS << "Total count: " << PS->getTotalCount() << "\n"; 2174 PS->printDetailedSummary(OS); 2175 } 2176 return 0; 2177 } 2178 2179 static void showSectionInfo(sampleprof::SampleProfileReader *Reader, 2180 raw_fd_ostream &OS) { 2181 if (!Reader->dumpSectionInfo(OS)) { 2182 WithColor::warning() << "-show-sec-info-only is only supported for " 2183 << "sample profile in extbinary format and is " 2184 << "ignored for other formats.\n"; 2185 return; 2186 } 2187 } 2188 2189 namespace { 2190 struct HotFuncInfo { 2191 StringRef FuncName; 2192 uint64_t TotalCount; 2193 double TotalCountPercent; 2194 uint64_t MaxCount; 2195 uint64_t EntryCount; 2196 2197 HotFuncInfo() 2198 : FuncName(), TotalCount(0), TotalCountPercent(0.0f), MaxCount(0), 2199 EntryCount(0) {} 2200 2201 HotFuncInfo(StringRef FN, uint64_t TS, double TSP, uint64_t MS, uint64_t ES) 2202 : FuncName(FN), TotalCount(TS), TotalCountPercent(TSP), MaxCount(MS), 2203 EntryCount(ES) {} 2204 }; 2205 } // namespace 2206 2207 // Print out detailed information about hot functions in PrintValues vector. 2208 // Users specify titles and offset of every columns through ColumnTitle and 2209 // ColumnOffset. The size of ColumnTitle and ColumnOffset need to be the same 2210 // and at least 4. Besides, users can optionally give a HotFuncMetric string to 2211 // print out or let it be an empty string. 2212 static void dumpHotFunctionList(const std::vector<std::string> &ColumnTitle, 2213 const std::vector<int> &ColumnOffset, 2214 const std::vector<HotFuncInfo> &PrintValues, 2215 uint64_t HotFuncCount, uint64_t TotalFuncCount, 2216 uint64_t HotProfCount, uint64_t TotalProfCount, 2217 const std::string &HotFuncMetric, 2218 raw_fd_ostream &OS) { 2219 assert(ColumnOffset.size() == ColumnTitle.size() && 2220 "ColumnOffset and ColumnTitle should have the same size"); 2221 assert(ColumnTitle.size() >= 4 && 2222 "ColumnTitle should have at least 4 elements"); 2223 assert(TotalFuncCount > 0 && 2224 "There should be at least one function in the profile"); 2225 double TotalProfPercent = 0; 2226 if (TotalProfCount > 0) 2227 TotalProfPercent = static_cast<double>(HotProfCount) / TotalProfCount * 100; 2228 2229 formatted_raw_ostream FOS(OS); 2230 FOS << HotFuncCount << " out of " << TotalFuncCount 2231 << " functions with profile (" 2232 << format("%.2f%%", 2233 (static_cast<double>(HotFuncCount) / TotalFuncCount * 100)) 2234 << ") are considered hot functions"; 2235 if (!HotFuncMetric.empty()) 2236 FOS << " (" << HotFuncMetric << ")"; 2237 FOS << ".\n"; 2238 FOS << HotProfCount << " out of " << TotalProfCount << " profile counts (" 2239 << format("%.2f%%", TotalProfPercent) << ") are from hot functions.\n"; 2240 2241 for (size_t I = 0; I < ColumnTitle.size(); ++I) { 2242 FOS.PadToColumn(ColumnOffset[I]); 2243 FOS << ColumnTitle[I]; 2244 } 2245 FOS << "\n"; 2246 2247 for (const HotFuncInfo &R : PrintValues) { 2248 FOS.PadToColumn(ColumnOffset[0]); 2249 FOS << R.TotalCount << " (" << format("%.2f%%", R.TotalCountPercent) << ")"; 2250 FOS.PadToColumn(ColumnOffset[1]); 2251 FOS << R.MaxCount; 2252 FOS.PadToColumn(ColumnOffset[2]); 2253 FOS << R.EntryCount; 2254 FOS.PadToColumn(ColumnOffset[3]); 2255 FOS << R.FuncName << "\n"; 2256 } 2257 } 2258 2259 static int 2260 showHotFunctionList(const StringMap<sampleprof::FunctionSamples> &Profiles, 2261 ProfileSummary &PS, raw_fd_ostream &OS) { 2262 using namespace sampleprof; 2263 2264 const uint32_t HotFuncCutoff = 990000; 2265 auto &SummaryVector = PS.getDetailedSummary(); 2266 uint64_t MinCountThreshold = 0; 2267 for (const ProfileSummaryEntry &SummaryEntry : SummaryVector) { 2268 if (SummaryEntry.Cutoff == HotFuncCutoff) { 2269 MinCountThreshold = SummaryEntry.MinCount; 2270 break; 2271 } 2272 } 2273 2274 // Traverse all functions in the profile and keep only hot functions. 2275 // The following loop also calculates the sum of total samples of all 2276 // functions. 2277 std::multimap<uint64_t, std::pair<const FunctionSamples *, const uint64_t>, 2278 std::greater<uint64_t>> 2279 HotFunc; 2280 uint64_t ProfileTotalSample = 0; 2281 uint64_t HotFuncSample = 0; 2282 uint64_t HotFuncCount = 0; 2283 2284 for (const auto &I : Profiles) { 2285 FuncSampleStats FuncStats; 2286 const FunctionSamples &FuncProf = I.second; 2287 ProfileTotalSample += FuncProf.getTotalSamples(); 2288 getFuncSampleStats(FuncProf, FuncStats, MinCountThreshold); 2289 2290 if (isFunctionHot(FuncStats, MinCountThreshold)) { 2291 HotFunc.emplace(FuncProf.getTotalSamples(), 2292 std::make_pair(&(I.second), FuncStats.MaxSample)); 2293 HotFuncSample += FuncProf.getTotalSamples(); 2294 ++HotFuncCount; 2295 } 2296 } 2297 2298 std::vector<std::string> ColumnTitle{"Total sample (%)", "Max sample", 2299 "Entry sample", "Function name"}; 2300 std::vector<int> ColumnOffset{0, 24, 42, 58}; 2301 std::string Metric = 2302 std::string("max sample >= ") + std::to_string(MinCountThreshold); 2303 std::vector<HotFuncInfo> PrintValues; 2304 for (const auto &FuncPair : HotFunc) { 2305 const FunctionSamples &Func = *FuncPair.second.first; 2306 double TotalSamplePercent = 2307 (ProfileTotalSample > 0) 2308 ? (Func.getTotalSamples() * 100.0) / ProfileTotalSample 2309 : 0; 2310 PrintValues.emplace_back( 2311 HotFuncInfo(Func.getName(), Func.getTotalSamples(), TotalSamplePercent, 2312 FuncPair.second.second, Func.getEntrySamples())); 2313 } 2314 dumpHotFunctionList(ColumnTitle, ColumnOffset, PrintValues, HotFuncCount, 2315 Profiles.size(), HotFuncSample, ProfileTotalSample, 2316 Metric, OS); 2317 2318 return 0; 2319 } 2320 2321 static int showSampleProfile(const std::string &Filename, bool ShowCounts, 2322 bool ShowAllFunctions, bool ShowDetailedSummary, 2323 const std::string &ShowFunction, 2324 bool ShowProfileSymbolList, 2325 bool ShowSectionInfoOnly, bool ShowHotFuncList, 2326 raw_fd_ostream &OS) { 2327 using namespace sampleprof; 2328 LLVMContext Context; 2329 auto ReaderOrErr = SampleProfileReader::create(Filename, Context); 2330 if (std::error_code EC = ReaderOrErr.getError()) 2331 exitWithErrorCode(EC, Filename); 2332 2333 auto Reader = std::move(ReaderOrErr.get()); 2334 2335 if (ShowSectionInfoOnly) { 2336 showSectionInfo(Reader.get(), OS); 2337 return 0; 2338 } 2339 2340 if (std::error_code EC = Reader->read()) 2341 exitWithErrorCode(EC, Filename); 2342 2343 if (ShowAllFunctions || ShowFunction.empty()) 2344 Reader->dump(OS); 2345 else 2346 Reader->dumpFunctionProfile(ShowFunction, OS); 2347 2348 if (ShowProfileSymbolList) { 2349 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList = 2350 Reader->getProfileSymbolList(); 2351 ReaderList->dump(OS); 2352 } 2353 2354 if (ShowDetailedSummary) { 2355 auto &PS = Reader->getSummary(); 2356 PS.printSummary(OS); 2357 PS.printDetailedSummary(OS); 2358 } 2359 2360 if (ShowHotFuncList) 2361 showHotFunctionList(Reader->getProfiles(), Reader->getSummary(), OS); 2362 2363 return 0; 2364 } 2365 2366 static int show_main(int argc, const char *argv[]) { 2367 cl::opt<std::string> Filename(cl::Positional, cl::Required, 2368 cl::desc("<profdata-file>")); 2369 2370 cl::opt<bool> ShowCounts("counts", cl::init(false), 2371 cl::desc("Show counter values for shown functions")); 2372 cl::opt<bool> TextFormat( 2373 "text", cl::init(false), 2374 cl::desc("Show instr profile data in text dump format")); 2375 cl::opt<bool> ShowIndirectCallTargets( 2376 "ic-targets", cl::init(false), 2377 cl::desc("Show indirect call site target values for shown functions")); 2378 cl::opt<bool> ShowMemOPSizes( 2379 "memop-sizes", cl::init(false), 2380 cl::desc("Show the profiled sizes of the memory intrinsic calls " 2381 "for shown functions")); 2382 cl::opt<bool> ShowDetailedSummary("detailed-summary", cl::init(false), 2383 cl::desc("Show detailed profile summary")); 2384 cl::list<uint32_t> DetailedSummaryCutoffs( 2385 cl::CommaSeparated, "detailed-summary-cutoffs", 2386 cl::desc( 2387 "Cutoff percentages (times 10000) for generating detailed summary"), 2388 cl::value_desc("800000,901000,999999")); 2389 cl::opt<bool> ShowHotFuncList( 2390 "hot-func-list", cl::init(false), 2391 cl::desc("Show profile summary of a list of hot functions")); 2392 cl::opt<bool> ShowAllFunctions("all-functions", cl::init(false), 2393 cl::desc("Details for every function")); 2394 cl::opt<bool> ShowCS("showcs", cl::init(false), 2395 cl::desc("Show context sensitive counts")); 2396 cl::opt<std::string> ShowFunction("function", 2397 cl::desc("Details for matching functions")); 2398 2399 cl::opt<std::string> OutputFilename("output", cl::value_desc("output"), 2400 cl::init("-"), cl::desc("Output file")); 2401 cl::alias OutputFilenameA("o", cl::desc("Alias for --output"), 2402 cl::aliasopt(OutputFilename)); 2403 cl::opt<ProfileKinds> ProfileKind( 2404 cl::desc("Profile kind:"), cl::init(instr), 2405 cl::values(clEnumVal(instr, "Instrumentation profile (default)"), 2406 clEnumVal(sample, "Sample profile"))); 2407 cl::opt<uint32_t> TopNFunctions( 2408 "topn", cl::init(0), 2409 cl::desc("Show the list of functions with the largest internal counts")); 2410 cl::opt<uint32_t> ValueCutoff( 2411 "value-cutoff", cl::init(0), 2412 cl::desc("Set the count value cutoff. Functions with the maximum count " 2413 "less than this value will not be printed out. (Default is 0)")); 2414 cl::opt<bool> OnlyListBelow( 2415 "list-below-cutoff", cl::init(false), 2416 cl::desc("Only output names of functions whose max count values are " 2417 "below the cutoff value")); 2418 cl::opt<bool> ShowProfileSymbolList( 2419 "show-prof-sym-list", cl::init(false), 2420 cl::desc("Show profile symbol list if it exists in the profile. ")); 2421 cl::opt<bool> ShowSectionInfoOnly( 2422 "show-sec-info-only", cl::init(false), 2423 cl::desc("Show the information of each section in the sample profile. " 2424 "The flag is only usable when the sample profile is in " 2425 "extbinary format")); 2426 2427 cl::ParseCommandLineOptions(argc, argv, "LLVM profile data summary\n"); 2428 2429 if (OutputFilename.empty()) 2430 OutputFilename = "-"; 2431 2432 if (Filename == OutputFilename) { 2433 errs() << sys::path::filename(argv[0]) 2434 << ": Input file name cannot be the same as the output file name!\n"; 2435 return 1; 2436 } 2437 2438 std::error_code EC; 2439 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_Text); 2440 if (EC) 2441 exitWithErrorCode(EC, OutputFilename); 2442 2443 if (ShowAllFunctions && !ShowFunction.empty()) 2444 WithColor::warning() << "-function argument ignored: showing all functions\n"; 2445 2446 if (ProfileKind == instr) 2447 return showInstrProfile(Filename, ShowCounts, TopNFunctions, 2448 ShowIndirectCallTargets, ShowMemOPSizes, 2449 ShowDetailedSummary, DetailedSummaryCutoffs, 2450 ShowAllFunctions, ShowCS, ValueCutoff, 2451 OnlyListBelow, ShowFunction, TextFormat, OS); 2452 else 2453 return showSampleProfile(Filename, ShowCounts, ShowAllFunctions, 2454 ShowDetailedSummary, ShowFunction, 2455 ShowProfileSymbolList, ShowSectionInfoOnly, 2456 ShowHotFuncList, OS); 2457 } 2458 2459 int main(int argc, const char *argv[]) { 2460 InitLLVM X(argc, argv); 2461 2462 StringRef ProgName(sys::path::filename(argv[0])); 2463 if (argc > 1) { 2464 int (*func)(int, const char *[]) = nullptr; 2465 2466 if (strcmp(argv[1], "merge") == 0) 2467 func = merge_main; 2468 else if (strcmp(argv[1], "show") == 0) 2469 func = show_main; 2470 else if (strcmp(argv[1], "overlap") == 0) 2471 func = overlap_main; 2472 2473 if (func) { 2474 std::string Invocation(ProgName.str() + " " + argv[1]); 2475 argv[1] = Invocation.c_str(); 2476 return func(argc - 1, argv + 1); 2477 } 2478 2479 if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "-help") == 0 || 2480 strcmp(argv[1], "--help") == 0) { 2481 2482 errs() << "OVERVIEW: LLVM profile data tools\n\n" 2483 << "USAGE: " << ProgName << " <command> [args...]\n" 2484 << "USAGE: " << ProgName << " <command> -help\n\n" 2485 << "See each individual command --help for more details.\n" 2486 << "Available commands: merge, show, overlap\n"; 2487 return 0; 2488 } 2489 } 2490 2491 if (argc < 2) 2492 errs() << ProgName << ": No command specified!\n"; 2493 else 2494 errs() << ProgName << ": Unknown command!\n"; 2495 2496 errs() << "USAGE: " << ProgName << " <merge|show|overlap> [args...]\n"; 2497 return 1; 2498 } 2499