1 //===- llvm-profdata.cpp - LLVM profile data tool -------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // llvm-profdata merges .profdata files. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ADT/SmallSet.h" 14 #include "llvm/ADT/SmallVector.h" 15 #include "llvm/ADT/StringRef.h" 16 #include "llvm/IR/LLVMContext.h" 17 #include "llvm/Object/Binary.h" 18 #include "llvm/ProfileData/InstrProfCorrelator.h" 19 #include "llvm/ProfileData/InstrProfReader.h" 20 #include "llvm/ProfileData/InstrProfWriter.h" 21 #include "llvm/ProfileData/MemProf.h" 22 #include "llvm/ProfileData/ProfileCommon.h" 23 #include "llvm/ProfileData/RawMemProfReader.h" 24 #include "llvm/ProfileData/SampleProfReader.h" 25 #include "llvm/ProfileData/SampleProfWriter.h" 26 #include "llvm/Support/CommandLine.h" 27 #include "llvm/Support/Discriminator.h" 28 #include "llvm/Support/Errc.h" 29 #include "llvm/Support/FileSystem.h" 30 #include "llvm/Support/Format.h" 31 #include "llvm/Support/FormattedStream.h" 32 #include "llvm/Support/InitLLVM.h" 33 #include "llvm/Support/MD5.h" 34 #include "llvm/Support/MemoryBuffer.h" 35 #include "llvm/Support/Path.h" 36 #include "llvm/Support/ThreadPool.h" 37 #include "llvm/Support/Threading.h" 38 #include "llvm/Support/WithColor.h" 39 #include "llvm/Support/raw_ostream.h" 40 #include <algorithm> 41 #include <cmath> 42 #include <optional> 43 #include <queue> 44 45 using namespace llvm; 46 47 // We use this string to indicate that there are 48 // multiple static functions map to the same name. 49 const std::string DuplicateNameStr = "----"; 50 51 enum ProfileFormat { 52 PF_None = 0, 53 PF_Text, 54 PF_Compact_Binary, 55 PF_Ext_Binary, 56 PF_GCC, 57 PF_Binary 58 }; 59 60 enum class ShowFormat { Text, Json, Yaml }; 61 62 static void warn(Twine Message, std::string Whence = "", 63 std::string Hint = "") { 64 WithColor::warning(); 65 if (!Whence.empty()) 66 errs() << Whence << ": "; 67 errs() << Message << "\n"; 68 if (!Hint.empty()) 69 WithColor::note() << Hint << "\n"; 70 } 71 72 static void warn(Error E, StringRef Whence = "") { 73 if (E.isA<InstrProfError>()) { 74 handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { 75 warn(IPE.message(), std::string(Whence), std::string("")); 76 }); 77 } 78 } 79 80 static void exitWithError(Twine Message, std::string Whence = "", 81 std::string Hint = "") { 82 WithColor::error(); 83 if (!Whence.empty()) 84 errs() << Whence << ": "; 85 errs() << Message << "\n"; 86 if (!Hint.empty()) 87 WithColor::note() << Hint << "\n"; 88 ::exit(1); 89 } 90 91 static void exitWithError(Error E, StringRef Whence = "") { 92 if (E.isA<InstrProfError>()) { 93 handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { 94 instrprof_error instrError = IPE.get(); 95 StringRef Hint = ""; 96 if (instrError == instrprof_error::unrecognized_format) { 97 // Hint in case user missed specifying the profile type. 98 Hint = "Perhaps you forgot to use the --sample or --memory option?"; 99 } 100 exitWithError(IPE.message(), std::string(Whence), std::string(Hint)); 101 }); 102 return; 103 } 104 105 exitWithError(toString(std::move(E)), std::string(Whence)); 106 } 107 108 static void exitWithErrorCode(std::error_code EC, StringRef Whence = "") { 109 exitWithError(EC.message(), std::string(Whence)); 110 } 111 112 namespace { 113 enum ProfileKinds { instr, sample, memory }; 114 enum FailureMode { failIfAnyAreInvalid, failIfAllAreInvalid }; 115 } 116 117 static void warnOrExitGivenError(FailureMode FailMode, std::error_code EC, 118 StringRef Whence = "") { 119 if (FailMode == failIfAnyAreInvalid) 120 exitWithErrorCode(EC, Whence); 121 else 122 warn(EC.message(), std::string(Whence)); 123 } 124 125 static void handleMergeWriterError(Error E, StringRef WhenceFile = "", 126 StringRef WhenceFunction = "", 127 bool ShowHint = true) { 128 if (!WhenceFile.empty()) 129 errs() << WhenceFile << ": "; 130 if (!WhenceFunction.empty()) 131 errs() << WhenceFunction << ": "; 132 133 auto IPE = instrprof_error::success; 134 E = handleErrors(std::move(E), 135 [&IPE](std::unique_ptr<InstrProfError> E) -> Error { 136 IPE = E->get(); 137 return Error(std::move(E)); 138 }); 139 errs() << toString(std::move(E)) << "\n"; 140 141 if (ShowHint) { 142 StringRef Hint = ""; 143 if (IPE != instrprof_error::success) { 144 switch (IPE) { 145 case instrprof_error::hash_mismatch: 146 case instrprof_error::count_mismatch: 147 case instrprof_error::value_site_count_mismatch: 148 Hint = "Make sure that all profile data to be merged is generated " 149 "from the same binary."; 150 break; 151 default: 152 break; 153 } 154 } 155 156 if (!Hint.empty()) 157 errs() << Hint << "\n"; 158 } 159 } 160 161 namespace { 162 /// A remapper from original symbol names to new symbol names based on a file 163 /// containing a list of mappings from old name to new name. 164 class SymbolRemapper { 165 std::unique_ptr<MemoryBuffer> File; 166 DenseMap<StringRef, StringRef> RemappingTable; 167 168 public: 169 /// Build a SymbolRemapper from a file containing a list of old/new symbols. 170 static std::unique_ptr<SymbolRemapper> create(StringRef InputFile) { 171 auto BufOrError = MemoryBuffer::getFileOrSTDIN(InputFile); 172 if (!BufOrError) 173 exitWithErrorCode(BufOrError.getError(), InputFile); 174 175 auto Remapper = std::make_unique<SymbolRemapper>(); 176 Remapper->File = std::move(BufOrError.get()); 177 178 for (line_iterator LineIt(*Remapper->File, /*SkipBlanks=*/true, '#'); 179 !LineIt.is_at_eof(); ++LineIt) { 180 std::pair<StringRef, StringRef> Parts = LineIt->split(' '); 181 if (Parts.first.empty() || Parts.second.empty() || 182 Parts.second.count(' ')) { 183 exitWithError("unexpected line in remapping file", 184 (InputFile + ":" + Twine(LineIt.line_number())).str(), 185 "expected 'old_symbol new_symbol'"); 186 } 187 Remapper->RemappingTable.insert(Parts); 188 } 189 return Remapper; 190 } 191 192 /// Attempt to map the given old symbol into a new symbol. 193 /// 194 /// \return The new symbol, or \p Name if no such symbol was found. 195 StringRef operator()(StringRef Name) { 196 StringRef New = RemappingTable.lookup(Name); 197 return New.empty() ? Name : New; 198 } 199 }; 200 } 201 202 struct WeightedFile { 203 std::string Filename; 204 uint64_t Weight; 205 }; 206 typedef SmallVector<WeightedFile, 5> WeightedFileVector; 207 208 /// Keep track of merged data and reported errors. 209 struct WriterContext { 210 std::mutex Lock; 211 InstrProfWriter Writer; 212 std::vector<std::pair<Error, std::string>> Errors; 213 std::mutex &ErrLock; 214 SmallSet<instrprof_error, 4> &WriterErrorCodes; 215 216 WriterContext(bool IsSparse, std::mutex &ErrLock, 217 SmallSet<instrprof_error, 4> &WriterErrorCodes) 218 : Writer(IsSparse), ErrLock(ErrLock), WriterErrorCodes(WriterErrorCodes) { 219 } 220 }; 221 222 /// Computer the overlap b/w profile BaseFilename and TestFileName, 223 /// and store the program level result to Overlap. 224 static void overlapInput(const std::string &BaseFilename, 225 const std::string &TestFilename, WriterContext *WC, 226 OverlapStats &Overlap, 227 const OverlapFuncFilters &FuncFilter, 228 raw_fd_ostream &OS, bool IsCS) { 229 auto ReaderOrErr = InstrProfReader::create(TestFilename); 230 if (Error E = ReaderOrErr.takeError()) { 231 // Skip the empty profiles by returning sliently. 232 instrprof_error IPE = InstrProfError::take(std::move(E)); 233 if (IPE != instrprof_error::empty_raw_profile) 234 WC->Errors.emplace_back(make_error<InstrProfError>(IPE), TestFilename); 235 return; 236 } 237 238 auto Reader = std::move(ReaderOrErr.get()); 239 for (auto &I : *Reader) { 240 OverlapStats FuncOverlap(OverlapStats::FunctionLevel); 241 FuncOverlap.setFuncInfo(I.Name, I.Hash); 242 243 WC->Writer.overlapRecord(std::move(I), Overlap, FuncOverlap, FuncFilter); 244 FuncOverlap.dump(OS); 245 } 246 } 247 248 /// Load an input into a writer context. 249 static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, 250 const InstrProfCorrelator *Correlator, 251 const StringRef ProfiledBinary, WriterContext *WC) { 252 std::unique_lock<std::mutex> CtxGuard{WC->Lock}; 253 254 // Copy the filename, because llvm::ThreadPool copied the input "const 255 // WeightedFile &" by value, making a reference to the filename within it 256 // invalid outside of this packaged task. 257 std::string Filename = Input.Filename; 258 259 using ::llvm::memprof::RawMemProfReader; 260 if (RawMemProfReader::hasFormat(Input.Filename)) { 261 auto ReaderOrErr = RawMemProfReader::create(Input.Filename, ProfiledBinary); 262 if (!ReaderOrErr) { 263 exitWithError(ReaderOrErr.takeError(), Input.Filename); 264 } 265 std::unique_ptr<RawMemProfReader> Reader = std::move(ReaderOrErr.get()); 266 // Check if the profile types can be merged, e.g. clang frontend profiles 267 // should not be merged with memprof profiles. 268 if (Error E = WC->Writer.mergeProfileKind(Reader->getProfileKind())) { 269 consumeError(std::move(E)); 270 WC->Errors.emplace_back( 271 make_error<StringError>( 272 "Cannot merge MemProf profile with Clang generated profile.", 273 std::error_code()), 274 Filename); 275 return; 276 } 277 278 auto MemProfError = [&](Error E) { 279 instrprof_error IPE = InstrProfError::take(std::move(E)); 280 WC->Errors.emplace_back(make_error<InstrProfError>(IPE), Filename); 281 }; 282 283 // Add the frame mappings into the writer context. 284 const auto &IdToFrame = Reader->getFrameMapping(); 285 for (const auto &I : IdToFrame) { 286 bool Succeeded = WC->Writer.addMemProfFrame( 287 /*Id=*/I.first, /*Frame=*/I.getSecond(), MemProfError); 288 // If we weren't able to add the frame mappings then it doesn't make sense 289 // to try to add the records from this profile. 290 if (!Succeeded) 291 return; 292 } 293 const auto &FunctionProfileData = Reader->getProfileData(); 294 // Add the memprof records into the writer context. 295 for (const auto &I : FunctionProfileData) { 296 WC->Writer.addMemProfRecord(/*Id=*/I.first, /*Record=*/I.second); 297 } 298 return; 299 } 300 301 auto ReaderOrErr = InstrProfReader::create(Input.Filename, Correlator); 302 if (Error E = ReaderOrErr.takeError()) { 303 // Skip the empty profiles by returning sliently. 304 instrprof_error IPE = InstrProfError::take(std::move(E)); 305 if (IPE != instrprof_error::empty_raw_profile) 306 WC->Errors.emplace_back(make_error<InstrProfError>(IPE), Filename); 307 return; 308 } 309 310 auto Reader = std::move(ReaderOrErr.get()); 311 if (Error E = WC->Writer.mergeProfileKind(Reader->getProfileKind())) { 312 consumeError(std::move(E)); 313 WC->Errors.emplace_back( 314 make_error<StringError>( 315 "Merge IR generated profile with Clang generated profile.", 316 std::error_code()), 317 Filename); 318 return; 319 } 320 321 for (auto &I : *Reader) { 322 if (Remapper) 323 I.Name = (*Remapper)(I.Name); 324 const StringRef FuncName = I.Name; 325 bool Reported = false; 326 WC->Writer.addRecord(std::move(I), Input.Weight, [&](Error E) { 327 if (Reported) { 328 consumeError(std::move(E)); 329 return; 330 } 331 Reported = true; 332 // Only show hint the first time an error occurs. 333 instrprof_error IPE = InstrProfError::take(std::move(E)); 334 std::unique_lock<std::mutex> ErrGuard{WC->ErrLock}; 335 bool firstTime = WC->WriterErrorCodes.insert(IPE).second; 336 handleMergeWriterError(make_error<InstrProfError>(IPE), Input.Filename, 337 FuncName, firstTime); 338 }); 339 } 340 341 if (Reader->hasError()) { 342 if (Error E = Reader->getError()) 343 WC->Errors.emplace_back(std::move(E), Filename); 344 } 345 346 std::vector<llvm::object::BuildID> BinaryIds; 347 if (Error E = Reader->readBinaryIds(BinaryIds)) 348 WC->Errors.emplace_back(std::move(E), Filename); 349 WC->Writer.addBinaryIds(BinaryIds); 350 } 351 352 /// Merge the \p Src writer context into \p Dst. 353 static void mergeWriterContexts(WriterContext *Dst, WriterContext *Src) { 354 for (auto &ErrorPair : Src->Errors) 355 Dst->Errors.push_back(std::move(ErrorPair)); 356 Src->Errors.clear(); 357 358 if (Error E = Dst->Writer.mergeProfileKind(Src->Writer.getProfileKind())) 359 exitWithError(std::move(E)); 360 361 Dst->Writer.mergeRecordsFromWriter(std::move(Src->Writer), [&](Error E) { 362 instrprof_error IPE = InstrProfError::take(std::move(E)); 363 std::unique_lock<std::mutex> ErrGuard{Dst->ErrLock}; 364 bool firstTime = Dst->WriterErrorCodes.insert(IPE).second; 365 if (firstTime) 366 warn(toString(make_error<InstrProfError>(IPE))); 367 }); 368 } 369 370 static void writeInstrProfile(StringRef OutputFilename, 371 ProfileFormat OutputFormat, 372 InstrProfWriter &Writer) { 373 std::error_code EC; 374 raw_fd_ostream Output(OutputFilename.data(), EC, 375 OutputFormat == PF_Text ? sys::fs::OF_TextWithCRLF 376 : sys::fs::OF_None); 377 if (EC) 378 exitWithErrorCode(EC, OutputFilename); 379 380 if (OutputFormat == PF_Text) { 381 if (Error E = Writer.writeText(Output)) 382 warn(std::move(E)); 383 } else { 384 if (Output.is_displayed()) 385 exitWithError("cannot write a non-text format profile to the terminal"); 386 if (Error E = Writer.write(Output)) 387 warn(std::move(E)); 388 } 389 } 390 391 static void mergeInstrProfile(const WeightedFileVector &Inputs, 392 StringRef DebugInfoFilename, 393 SymbolRemapper *Remapper, 394 StringRef OutputFilename, 395 ProfileFormat OutputFormat, bool OutputSparse, 396 unsigned NumThreads, FailureMode FailMode, 397 const StringRef ProfiledBinary) { 398 if (OutputFormat != PF_Binary && OutputFormat != PF_Compact_Binary && 399 OutputFormat != PF_Ext_Binary && OutputFormat != PF_Text) 400 exitWithError("unknown format is specified"); 401 402 std::unique_ptr<InstrProfCorrelator> Correlator; 403 if (!DebugInfoFilename.empty()) { 404 if (auto Err = 405 InstrProfCorrelator::get(DebugInfoFilename).moveInto(Correlator)) 406 exitWithError(std::move(Err), DebugInfoFilename); 407 if (auto Err = Correlator->correlateProfileData()) 408 exitWithError(std::move(Err), DebugInfoFilename); 409 } 410 411 std::mutex ErrorLock; 412 SmallSet<instrprof_error, 4> WriterErrorCodes; 413 414 // If NumThreads is not specified, auto-detect a good default. 415 if (NumThreads == 0) 416 NumThreads = std::min(hardware_concurrency().compute_thread_count(), 417 unsigned((Inputs.size() + 1) / 2)); 418 419 // Initialize the writer contexts. 420 SmallVector<std::unique_ptr<WriterContext>, 4> Contexts; 421 for (unsigned I = 0; I < NumThreads; ++I) 422 Contexts.emplace_back(std::make_unique<WriterContext>( 423 OutputSparse, ErrorLock, WriterErrorCodes)); 424 425 if (NumThreads == 1) { 426 for (const auto &Input : Inputs) 427 loadInput(Input, Remapper, Correlator.get(), ProfiledBinary, 428 Contexts[0].get()); 429 } else { 430 ThreadPool Pool(hardware_concurrency(NumThreads)); 431 432 // Load the inputs in parallel (N/NumThreads serial steps). 433 unsigned Ctx = 0; 434 for (const auto &Input : Inputs) { 435 Pool.async(loadInput, Input, Remapper, Correlator.get(), ProfiledBinary, 436 Contexts[Ctx].get()); 437 Ctx = (Ctx + 1) % NumThreads; 438 } 439 Pool.wait(); 440 441 // Merge the writer contexts together (~ lg(NumThreads) serial steps). 442 unsigned Mid = Contexts.size() / 2; 443 unsigned End = Contexts.size(); 444 assert(Mid > 0 && "Expected more than one context"); 445 do { 446 for (unsigned I = 0; I < Mid; ++I) 447 Pool.async(mergeWriterContexts, Contexts[I].get(), 448 Contexts[I + Mid].get()); 449 Pool.wait(); 450 if (End & 1) { 451 Pool.async(mergeWriterContexts, Contexts[0].get(), 452 Contexts[End - 1].get()); 453 Pool.wait(); 454 } 455 End = Mid; 456 Mid /= 2; 457 } while (Mid > 0); 458 } 459 460 // Handle deferred errors encountered during merging. If the number of errors 461 // is equal to the number of inputs the merge failed. 462 unsigned NumErrors = 0; 463 for (std::unique_ptr<WriterContext> &WC : Contexts) { 464 for (auto &ErrorPair : WC->Errors) { 465 ++NumErrors; 466 warn(toString(std::move(ErrorPair.first)), ErrorPair.second); 467 } 468 } 469 if (NumErrors == Inputs.size() || 470 (NumErrors > 0 && FailMode == failIfAnyAreInvalid)) 471 exitWithError("no profile can be merged"); 472 473 writeInstrProfile(OutputFilename, OutputFormat, Contexts[0]->Writer); 474 } 475 476 /// The profile entry for a function in instrumentation profile. 477 struct InstrProfileEntry { 478 uint64_t MaxCount = 0; 479 uint64_t NumEdgeCounters = 0; 480 float ZeroCounterRatio = 0.0; 481 InstrProfRecord *ProfRecord; 482 InstrProfileEntry(InstrProfRecord *Record); 483 InstrProfileEntry() = default; 484 }; 485 486 InstrProfileEntry::InstrProfileEntry(InstrProfRecord *Record) { 487 ProfRecord = Record; 488 uint64_t CntNum = Record->Counts.size(); 489 uint64_t ZeroCntNum = 0; 490 for (size_t I = 0; I < CntNum; ++I) { 491 MaxCount = std::max(MaxCount, Record->Counts[I]); 492 ZeroCntNum += !Record->Counts[I]; 493 } 494 ZeroCounterRatio = (float)ZeroCntNum / CntNum; 495 NumEdgeCounters = CntNum; 496 } 497 498 /// Either set all the counters in the instr profile entry \p IFE to 499 /// -1 / -2 /in order to drop the profile or scale up the 500 /// counters in \p IFP to be above hot / cold threshold. We use 501 /// the ratio of zero counters in the profile of a function to 502 /// decide the profile is helpful or harmful for performance, 503 /// and to choose whether to scale up or drop it. 504 static void updateInstrProfileEntry(InstrProfileEntry &IFE, bool SetToHot, 505 uint64_t HotInstrThreshold, 506 uint64_t ColdInstrThreshold, 507 float ZeroCounterThreshold) { 508 InstrProfRecord *ProfRecord = IFE.ProfRecord; 509 if (!IFE.MaxCount || IFE.ZeroCounterRatio > ZeroCounterThreshold) { 510 // If all or most of the counters of the function are zero, the 511 // profile is unaccountable and should be dropped. Reset all the 512 // counters to be -1 / -2 and PGO profile-use will drop the profile. 513 // All counters being -1 also implies that the function is hot so 514 // PGO profile-use will also set the entry count metadata to be 515 // above hot threshold. 516 // All counters being -2 implies that the function is warm so 517 // PGO profile-use will also set the entry count metadata to be 518 // above cold threshold. 519 auto Kind = 520 (SetToHot ? InstrProfRecord::PseudoHot : InstrProfRecord::PseudoWarm); 521 ProfRecord->setPseudoCount(Kind); 522 return; 523 } 524 525 // Scale up the MaxCount to be multiple times above hot / cold threshold. 526 const unsigned MultiplyFactor = 3; 527 uint64_t Threshold = (SetToHot ? HotInstrThreshold : ColdInstrThreshold); 528 uint64_t Numerator = Threshold * MultiplyFactor; 529 530 // Make sure Threshold for warm counters is below the HotInstrThreshold. 531 if (!SetToHot && Threshold >= HotInstrThreshold) { 532 Threshold = (HotInstrThreshold + ColdInstrThreshold) / 2; 533 } 534 535 uint64_t Denominator = IFE.MaxCount; 536 if (Numerator <= Denominator) 537 return; 538 ProfRecord->scale(Numerator, Denominator, [&](instrprof_error E) { 539 warn(toString(make_error<InstrProfError>(E))); 540 }); 541 } 542 543 const uint64_t ColdPercentileIdx = 15; 544 const uint64_t HotPercentileIdx = 11; 545 546 using sampleprof::FSDiscriminatorPass; 547 548 // Internal options to set FSDiscriminatorPass. Used in merge and show 549 // commands. 550 static cl::opt<FSDiscriminatorPass> FSDiscriminatorPassOption( 551 "fs-discriminator-pass", cl::init(PassLast), cl::Hidden, 552 cl::desc("Zero out the discriminator bits for the FS discrimiantor " 553 "pass beyond this value. The enum values are defined in " 554 "Support/Discriminator.h"), 555 cl::values(clEnumVal(Base, "Use base discriminators only"), 556 clEnumVal(Pass1, "Use base and pass 1 discriminators"), 557 clEnumVal(Pass2, "Use base and pass 1-2 discriminators"), 558 clEnumVal(Pass3, "Use base and pass 1-3 discriminators"), 559 clEnumVal(PassLast, "Use all discriminator bits (default)"))); 560 561 static unsigned getDiscriminatorMask() { 562 return getN1Bits(getFSPassBitEnd(FSDiscriminatorPassOption.getValue())); 563 } 564 565 /// Adjust the instr profile in \p WC based on the sample profile in 566 /// \p Reader. 567 static void 568 adjustInstrProfile(std::unique_ptr<WriterContext> &WC, 569 std::unique_ptr<sampleprof::SampleProfileReader> &Reader, 570 unsigned SupplMinSizeThreshold, float ZeroCounterThreshold, 571 unsigned InstrProfColdThreshold) { 572 // Function to its entry in instr profile. 573 StringMap<InstrProfileEntry> InstrProfileMap; 574 StringMap<StringRef> StaticFuncMap; 575 InstrProfSummaryBuilder IPBuilder(ProfileSummaryBuilder::DefaultCutoffs); 576 577 auto checkSampleProfileHasFUnique = [&Reader]() { 578 for (const auto &PD : Reader->getProfiles()) { 579 auto &FContext = PD.first; 580 if (FContext.toString().find(FunctionSamples::UniqSuffix) != 581 std::string::npos) { 582 return true; 583 } 584 } 585 return false; 586 }; 587 588 bool SampleProfileHasFUnique = checkSampleProfileHasFUnique(); 589 590 auto buildStaticFuncMap = [&StaticFuncMap, 591 SampleProfileHasFUnique](const StringRef Name) { 592 std::string Prefixes[] = {".cpp:", "cc:", ".c:", ".hpp:", ".h:"}; 593 size_t PrefixPos = StringRef::npos; 594 for (auto &Prefix : Prefixes) { 595 PrefixPos = Name.find_insensitive(Prefix); 596 if (PrefixPos == StringRef::npos) 597 continue; 598 PrefixPos += Prefix.size(); 599 break; 600 } 601 602 if (PrefixPos == StringRef::npos) { 603 return; 604 } 605 606 StringRef NewName = Name.drop_front(PrefixPos); 607 StringRef FName = Name.substr(0, PrefixPos - 1); 608 if (NewName.size() == 0) { 609 return; 610 } 611 612 // This name should have a static linkage. 613 size_t PostfixPos = NewName.find(FunctionSamples::UniqSuffix); 614 bool ProfileHasFUnique = (PostfixPos != StringRef::npos); 615 616 // If sample profile and instrumented profile do not agree on symbol 617 // uniqification. 618 if (SampleProfileHasFUnique != ProfileHasFUnique) { 619 // If instrumented profile uses -funique-internal-linakge-symbols, 620 // we need to trim the name. 621 if (ProfileHasFUnique) { 622 NewName = NewName.substr(0, PostfixPos); 623 } else { 624 // If sample profile uses -funique-internal-linakge-symbols, 625 // we build the map. 626 std::string NStr = 627 NewName.str() + getUniqueInternalLinkagePostfix(FName); 628 NewName = StringRef(NStr); 629 StaticFuncMap[NewName] = Name; 630 return; 631 } 632 } 633 634 if (StaticFuncMap.find(NewName) == StaticFuncMap.end()) { 635 StaticFuncMap[NewName] = Name; 636 } else { 637 StaticFuncMap[NewName] = DuplicateNameStr; 638 } 639 }; 640 641 // We need to flatten the SampleFDO profile as the InstrFDO 642 // profile does not have inlined callsite profiles. 643 // One caveat is the pre-inlined function -- their samples 644 // should be collapsed into the caller function. 645 // Here we do a DFS traversal to get the flatten profile 646 // info: the sum of entrycount and the max of maxcount. 647 // Here is the algorithm: 648 // recursive (FS, root_name) { 649 // name = FS->getName(); 650 // get samples for FS; 651 // if (InstrProf.find(name) { 652 // root_name = name; 653 // } else { 654 // if (name is in static_func map) { 655 // root_name = static_name; 656 // } 657 // } 658 // update the Map entry for root_name; 659 // for (subfs: FS) { 660 // recursive(subfs, root_name); 661 // } 662 // } 663 // 664 // Here is an example. 665 // 666 // SampleProfile: 667 // foo:12345:1000 668 // 1: 1000 669 // 2.1: 1000 670 // 15: 5000 671 // 4: bar:1000 672 // 1: 1000 673 // 2: goo:3000 674 // 1: 3000 675 // 8: bar:40000 676 // 1: 10000 677 // 2: goo:30000 678 // 1: 30000 679 // 680 // InstrProfile has two entries: 681 // foo 682 // bar.cc:bar 683 // 684 // After BuildMaxSampleMap, we should have the following in FlattenSampleMap: 685 // {"foo", {1000, 5000}} 686 // {"bar.cc:bar", {11000, 30000}} 687 // 688 // foo's has an entry count of 1000, and max body count of 5000. 689 // bar.cc:bar has an entry count of 11000 (sum two callsites of 1000 and 690 // 10000), and max count of 30000 (from the callsite in line 8). 691 // 692 // Note that goo's count will remain in bar.cc:bar() as it does not have an 693 // entry in InstrProfile. 694 DenseMap<StringRef, std::pair<uint64_t, uint64_t>> FlattenSampleMap; 695 auto BuildMaxSampleMap = [&FlattenSampleMap, &StaticFuncMap, 696 &InstrProfileMap](const FunctionSamples &FS, 697 const StringRef &RootName) { 698 auto BuildMaxSampleMapImpl = [&](const FunctionSamples &FS, 699 const StringRef &RootName, 700 auto &BuildImpl) -> void { 701 const StringRef &Name = FS.getName(); 702 const StringRef *NewRootName = &RootName; 703 uint64_t EntrySample = FS.getHeadSamplesEstimate(); 704 uint64_t MaxBodySample = FS.getMaxCountInside(/* SkipCallSite*/ true); 705 706 auto It = InstrProfileMap.find(Name); 707 if (It != InstrProfileMap.end()) { 708 NewRootName = &Name; 709 } else { 710 auto NewName = StaticFuncMap.find(Name); 711 if (NewName != StaticFuncMap.end()) { 712 It = InstrProfileMap.find(NewName->second.str()); 713 if (NewName->second != DuplicateNameStr) { 714 NewRootName = &NewName->second; 715 } 716 } else { 717 // Here the EntrySample is of an inlined function, so we should not 718 // update the EntrySample in the map. 719 EntrySample = 0; 720 } 721 } 722 EntrySample += FlattenSampleMap[*NewRootName].first; 723 MaxBodySample = 724 std::max(FlattenSampleMap[*NewRootName].second, MaxBodySample); 725 FlattenSampleMap[*NewRootName] = 726 std::make_pair(EntrySample, MaxBodySample); 727 728 for (const auto &C : FS.getCallsiteSamples()) 729 for (const auto &F : C.second) 730 BuildImpl(F.second, *NewRootName, BuildImpl); 731 }; 732 BuildMaxSampleMapImpl(FS, RootName, BuildMaxSampleMapImpl); 733 }; 734 735 for (auto &PD : WC->Writer.getProfileData()) { 736 // Populate IPBuilder. 737 for (const auto &PDV : PD.getValue()) { 738 InstrProfRecord Record = PDV.second; 739 IPBuilder.addRecord(Record); 740 } 741 742 // If a function has multiple entries in instr profile, skip it. 743 if (PD.getValue().size() != 1) 744 continue; 745 746 // Initialize InstrProfileMap. 747 InstrProfRecord *R = &PD.getValue().begin()->second; 748 StringRef FullName = PD.getKey(); 749 InstrProfileMap[FullName] = InstrProfileEntry(R); 750 buildStaticFuncMap(FullName); 751 } 752 753 for (auto &PD : Reader->getProfiles()) { 754 sampleprof::FunctionSamples &FS = PD.second; 755 BuildMaxSampleMap(FS, FS.getName()); 756 } 757 758 ProfileSummary InstrPS = *IPBuilder.getSummary(); 759 ProfileSummary SamplePS = Reader->getSummary(); 760 761 // Compute cold thresholds for instr profile and sample profile. 762 uint64_t HotSampleThreshold = 763 ProfileSummaryBuilder::getEntryForPercentile( 764 SamplePS.getDetailedSummary(), 765 ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx]) 766 .MinCount; 767 uint64_t ColdSampleThreshold = 768 ProfileSummaryBuilder::getEntryForPercentile( 769 SamplePS.getDetailedSummary(), 770 ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx]) 771 .MinCount; 772 uint64_t HotInstrThreshold = 773 ProfileSummaryBuilder::getEntryForPercentile( 774 InstrPS.getDetailedSummary(), 775 ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx]) 776 .MinCount; 777 uint64_t ColdInstrThreshold = 778 InstrProfColdThreshold 779 ? InstrProfColdThreshold 780 : ProfileSummaryBuilder::getEntryForPercentile( 781 InstrPS.getDetailedSummary(), 782 ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx]) 783 .MinCount; 784 785 // Find hot/warm functions in sample profile which is cold in instr profile 786 // and adjust the profiles of those functions in the instr profile. 787 for (const auto &E : FlattenSampleMap) { 788 uint64_t SampleMaxCount = std::max(E.second.first, E.second.second); 789 if (SampleMaxCount < ColdSampleThreshold) 790 continue; 791 const StringRef &Name = E.first; 792 auto It = InstrProfileMap.find(Name); 793 if (It == InstrProfileMap.end()) { 794 auto NewName = StaticFuncMap.find(Name); 795 if (NewName != StaticFuncMap.end()) { 796 It = InstrProfileMap.find(NewName->second.str()); 797 if (NewName->second == DuplicateNameStr) { 798 WithColor::warning() 799 << "Static function " << Name 800 << " has multiple promoted names, cannot adjust profile.\n"; 801 } 802 } 803 } 804 if (It == InstrProfileMap.end() || 805 It->second.MaxCount > ColdInstrThreshold || 806 It->second.NumEdgeCounters < SupplMinSizeThreshold) 807 continue; 808 bool SetToHot = SampleMaxCount >= HotSampleThreshold; 809 updateInstrProfileEntry(It->second, SetToHot, HotInstrThreshold, 810 ColdInstrThreshold, ZeroCounterThreshold); 811 } 812 } 813 814 /// The main function to supplement instr profile with sample profile. 815 /// \Inputs contains the instr profile. \p SampleFilename specifies the 816 /// sample profile. \p OutputFilename specifies the output profile name. 817 /// \p OutputFormat specifies the output profile format. \p OutputSparse 818 /// specifies whether to generate sparse profile. \p SupplMinSizeThreshold 819 /// specifies the minimal size for the functions whose profile will be 820 /// adjusted. \p ZeroCounterThreshold is the threshold to check whether 821 /// a function contains too many zero counters and whether its profile 822 /// should be dropped. \p InstrProfColdThreshold is the user specified 823 /// cold threshold which will override the cold threshold got from the 824 /// instr profile summary. 825 static void supplementInstrProfile( 826 const WeightedFileVector &Inputs, StringRef SampleFilename, 827 StringRef OutputFilename, ProfileFormat OutputFormat, bool OutputSparse, 828 unsigned SupplMinSizeThreshold, float ZeroCounterThreshold, 829 unsigned InstrProfColdThreshold) { 830 if (OutputFilename.compare("-") == 0) 831 exitWithError("cannot write indexed profdata format to stdout"); 832 if (Inputs.size() != 1) 833 exitWithError("expect one input to be an instr profile"); 834 if (Inputs[0].Weight != 1) 835 exitWithError("expect instr profile doesn't have weight"); 836 837 StringRef InstrFilename = Inputs[0].Filename; 838 839 // Read sample profile. 840 LLVMContext Context; 841 auto ReaderOrErr = sampleprof::SampleProfileReader::create( 842 SampleFilename.str(), Context, FSDiscriminatorPassOption); 843 if (std::error_code EC = ReaderOrErr.getError()) 844 exitWithErrorCode(EC, SampleFilename); 845 auto Reader = std::move(ReaderOrErr.get()); 846 if (std::error_code EC = Reader->read()) 847 exitWithErrorCode(EC, SampleFilename); 848 849 // Read instr profile. 850 std::mutex ErrorLock; 851 SmallSet<instrprof_error, 4> WriterErrorCodes; 852 auto WC = std::make_unique<WriterContext>(OutputSparse, ErrorLock, 853 WriterErrorCodes); 854 loadInput(Inputs[0], nullptr, nullptr, /*ProfiledBinary=*/"", WC.get()); 855 if (WC->Errors.size() > 0) 856 exitWithError(std::move(WC->Errors[0].first), InstrFilename); 857 858 adjustInstrProfile(WC, Reader, SupplMinSizeThreshold, ZeroCounterThreshold, 859 InstrProfColdThreshold); 860 writeInstrProfile(OutputFilename, OutputFormat, WC->Writer); 861 } 862 863 /// Make a copy of the given function samples with all symbol names remapped 864 /// by the provided symbol remapper. 865 static sampleprof::FunctionSamples 866 remapSamples(const sampleprof::FunctionSamples &Samples, 867 SymbolRemapper &Remapper, sampleprof_error &Error) { 868 sampleprof::FunctionSamples Result; 869 Result.setName(Remapper(Samples.getName())); 870 Result.addTotalSamples(Samples.getTotalSamples()); 871 Result.addHeadSamples(Samples.getHeadSamples()); 872 for (const auto &BodySample : Samples.getBodySamples()) { 873 uint32_t MaskedDiscriminator = 874 BodySample.first.Discriminator & getDiscriminatorMask(); 875 Result.addBodySamples(BodySample.first.LineOffset, MaskedDiscriminator, 876 BodySample.second.getSamples()); 877 for (const auto &Target : BodySample.second.getCallTargets()) { 878 Result.addCalledTargetSamples(BodySample.first.LineOffset, 879 MaskedDiscriminator, 880 Remapper(Target.first()), Target.second); 881 } 882 } 883 for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) { 884 sampleprof::FunctionSamplesMap &Target = 885 Result.functionSamplesAt(CallsiteSamples.first); 886 for (const auto &Callsite : CallsiteSamples.second) { 887 sampleprof::FunctionSamples Remapped = 888 remapSamples(Callsite.second, Remapper, Error); 889 MergeResult(Error, 890 Target[std::string(Remapped.getName())].merge(Remapped)); 891 } 892 } 893 return Result; 894 } 895 896 static sampleprof::SampleProfileFormat FormatMap[] = { 897 sampleprof::SPF_None, 898 sampleprof::SPF_Text, 899 sampleprof::SPF_Compact_Binary, 900 sampleprof::SPF_Ext_Binary, 901 sampleprof::SPF_GCC, 902 sampleprof::SPF_Binary}; 903 904 static std::unique_ptr<MemoryBuffer> 905 getInputFileBuf(const StringRef &InputFile) { 906 if (InputFile == "") 907 return {}; 908 909 auto BufOrError = MemoryBuffer::getFileOrSTDIN(InputFile); 910 if (!BufOrError) 911 exitWithErrorCode(BufOrError.getError(), InputFile); 912 913 return std::move(*BufOrError); 914 } 915 916 static void populateProfileSymbolList(MemoryBuffer *Buffer, 917 sampleprof::ProfileSymbolList &PSL) { 918 if (!Buffer) 919 return; 920 921 SmallVector<StringRef, 32> SymbolVec; 922 StringRef Data = Buffer->getBuffer(); 923 Data.split(SymbolVec, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false); 924 925 for (StringRef SymbolStr : SymbolVec) 926 PSL.add(SymbolStr.trim()); 927 } 928 929 static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer, 930 ProfileFormat OutputFormat, 931 MemoryBuffer *Buffer, 932 sampleprof::ProfileSymbolList &WriterList, 933 bool CompressAllSections, bool UseMD5, 934 bool GenPartialProfile) { 935 populateProfileSymbolList(Buffer, WriterList); 936 if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary) 937 warn("Profile Symbol list is not empty but the output format is not " 938 "ExtBinary format. The list will be lost in the output. "); 939 940 Writer.setProfileSymbolList(&WriterList); 941 942 if (CompressAllSections) { 943 if (OutputFormat != PF_Ext_Binary) 944 warn("-compress-all-section is ignored. Specify -extbinary to enable it"); 945 else 946 Writer.setToCompressAllSections(); 947 } 948 if (UseMD5) { 949 if (OutputFormat != PF_Ext_Binary) 950 warn("-use-md5 is ignored. Specify -extbinary to enable it"); 951 else 952 Writer.setUseMD5(); 953 } 954 if (GenPartialProfile) { 955 if (OutputFormat != PF_Ext_Binary) 956 warn("-gen-partial-profile is ignored. Specify -extbinary to enable it"); 957 else 958 Writer.setPartialProfile(); 959 } 960 } 961 962 static void 963 mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper, 964 StringRef OutputFilename, ProfileFormat OutputFormat, 965 StringRef ProfileSymbolListFile, bool CompressAllSections, 966 bool UseMD5, bool GenPartialProfile, bool GenCSNestedProfile, 967 bool SampleMergeColdContext, bool SampleTrimColdContext, 968 bool SampleColdContextFrameDepth, FailureMode FailMode, 969 bool DropProfileSymbolList) { 970 using namespace sampleprof; 971 SampleProfileMap ProfileMap; 972 SmallVector<std::unique_ptr<sampleprof::SampleProfileReader>, 5> Readers; 973 LLVMContext Context; 974 sampleprof::ProfileSymbolList WriterList; 975 std::optional<bool> ProfileIsProbeBased; 976 std::optional<bool> ProfileIsCS; 977 for (const auto &Input : Inputs) { 978 auto ReaderOrErr = SampleProfileReader::create(Input.Filename, Context, 979 FSDiscriminatorPassOption); 980 if (std::error_code EC = ReaderOrErr.getError()) { 981 warnOrExitGivenError(FailMode, EC, Input.Filename); 982 continue; 983 } 984 985 // We need to keep the readers around until after all the files are 986 // read so that we do not lose the function names stored in each 987 // reader's memory. The function names are needed to write out the 988 // merged profile map. 989 Readers.push_back(std::move(ReaderOrErr.get())); 990 const auto Reader = Readers.back().get(); 991 if (std::error_code EC = Reader->read()) { 992 warnOrExitGivenError(FailMode, EC, Input.Filename); 993 Readers.pop_back(); 994 continue; 995 } 996 997 SampleProfileMap &Profiles = Reader->getProfiles(); 998 if (ProfileIsProbeBased && 999 ProfileIsProbeBased != FunctionSamples::ProfileIsProbeBased) 1000 exitWithError( 1001 "cannot merge probe-based profile with non-probe-based profile"); 1002 ProfileIsProbeBased = FunctionSamples::ProfileIsProbeBased; 1003 if (ProfileIsCS && ProfileIsCS != FunctionSamples::ProfileIsCS) 1004 exitWithError("cannot merge CS profile with non-CS profile"); 1005 ProfileIsCS = FunctionSamples::ProfileIsCS; 1006 for (SampleProfileMap::iterator I = Profiles.begin(), E = Profiles.end(); 1007 I != E; ++I) { 1008 sampleprof_error Result = sampleprof_error::success; 1009 FunctionSamples Remapped = 1010 Remapper ? remapSamples(I->second, *Remapper, Result) 1011 : FunctionSamples(); 1012 FunctionSamples &Samples = Remapper ? Remapped : I->second; 1013 SampleContext FContext = Samples.getContext(); 1014 MergeResult(Result, ProfileMap[FContext].merge(Samples, Input.Weight)); 1015 if (Result != sampleprof_error::success) { 1016 std::error_code EC = make_error_code(Result); 1017 handleMergeWriterError(errorCodeToError(EC), Input.Filename, 1018 FContext.toString()); 1019 } 1020 } 1021 1022 if (!DropProfileSymbolList) { 1023 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList = 1024 Reader->getProfileSymbolList(); 1025 if (ReaderList) 1026 WriterList.merge(*ReaderList); 1027 } 1028 } 1029 1030 if (ProfileIsCS && (SampleMergeColdContext || SampleTrimColdContext)) { 1031 // Use threshold calculated from profile summary unless specified. 1032 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); 1033 auto Summary = Builder.computeSummaryForProfiles(ProfileMap); 1034 uint64_t SampleProfColdThreshold = 1035 ProfileSummaryBuilder::getColdCountThreshold( 1036 (Summary->getDetailedSummary())); 1037 1038 // Trim and merge cold context profile using cold threshold above; 1039 SampleContextTrimmer(ProfileMap) 1040 .trimAndMergeColdContextProfiles( 1041 SampleProfColdThreshold, SampleTrimColdContext, 1042 SampleMergeColdContext, SampleColdContextFrameDepth, false); 1043 } 1044 1045 if (ProfileIsCS && GenCSNestedProfile) { 1046 CSProfileConverter CSConverter(ProfileMap); 1047 CSConverter.convertProfiles(); 1048 ProfileIsCS = FunctionSamples::ProfileIsCS = false; 1049 } 1050 1051 auto WriterOrErr = 1052 SampleProfileWriter::create(OutputFilename, FormatMap[OutputFormat]); 1053 if (std::error_code EC = WriterOrErr.getError()) 1054 exitWithErrorCode(EC, OutputFilename); 1055 1056 auto Writer = std::move(WriterOrErr.get()); 1057 // WriterList will have StringRef refering to string in Buffer. 1058 // Make sure Buffer lives as long as WriterList. 1059 auto Buffer = getInputFileBuf(ProfileSymbolListFile); 1060 handleExtBinaryWriter(*Writer, OutputFormat, Buffer.get(), WriterList, 1061 CompressAllSections, UseMD5, GenPartialProfile); 1062 if (std::error_code EC = Writer->write(ProfileMap)) 1063 exitWithErrorCode(std::move(EC)); 1064 } 1065 1066 static WeightedFile parseWeightedFile(const StringRef &WeightedFilename) { 1067 StringRef WeightStr, FileName; 1068 std::tie(WeightStr, FileName) = WeightedFilename.split(','); 1069 1070 uint64_t Weight; 1071 if (WeightStr.getAsInteger(10, Weight) || Weight < 1) 1072 exitWithError("input weight must be a positive integer"); 1073 1074 return {std::string(FileName), Weight}; 1075 } 1076 1077 static void addWeightedInput(WeightedFileVector &WNI, const WeightedFile &WF) { 1078 StringRef Filename = WF.Filename; 1079 uint64_t Weight = WF.Weight; 1080 1081 // If it's STDIN just pass it on. 1082 if (Filename == "-") { 1083 WNI.push_back({std::string(Filename), Weight}); 1084 return; 1085 } 1086 1087 llvm::sys::fs::file_status Status; 1088 llvm::sys::fs::status(Filename, Status); 1089 if (!llvm::sys::fs::exists(Status)) 1090 exitWithErrorCode(make_error_code(errc::no_such_file_or_directory), 1091 Filename); 1092 // If it's a source file, collect it. 1093 if (llvm::sys::fs::is_regular_file(Status)) { 1094 WNI.push_back({std::string(Filename), Weight}); 1095 return; 1096 } 1097 1098 if (llvm::sys::fs::is_directory(Status)) { 1099 std::error_code EC; 1100 for (llvm::sys::fs::recursive_directory_iterator F(Filename, EC), E; 1101 F != E && !EC; F.increment(EC)) { 1102 if (llvm::sys::fs::is_regular_file(F->path())) { 1103 addWeightedInput(WNI, {F->path(), Weight}); 1104 } 1105 } 1106 if (EC) 1107 exitWithErrorCode(EC, Filename); 1108 } 1109 } 1110 1111 static void parseInputFilenamesFile(MemoryBuffer *Buffer, 1112 WeightedFileVector &WFV) { 1113 if (!Buffer) 1114 return; 1115 1116 SmallVector<StringRef, 8> Entries; 1117 StringRef Data = Buffer->getBuffer(); 1118 Data.split(Entries, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false); 1119 for (const StringRef &FileWeightEntry : Entries) { 1120 StringRef SanitizedEntry = FileWeightEntry.trim(" \t\v\f\r"); 1121 // Skip comments. 1122 if (SanitizedEntry.startswith("#")) 1123 continue; 1124 // If there's no comma, it's an unweighted profile. 1125 else if (!SanitizedEntry.contains(',')) 1126 addWeightedInput(WFV, {std::string(SanitizedEntry), 1}); 1127 else 1128 addWeightedInput(WFV, parseWeightedFile(SanitizedEntry)); 1129 } 1130 } 1131 1132 static int merge_main(int argc, const char *argv[]) { 1133 cl::list<std::string> InputFilenames(cl::Positional, 1134 cl::desc("<filename...>")); 1135 cl::list<std::string> WeightedInputFilenames("weighted-input", 1136 cl::desc("<weight>,<filename>")); 1137 cl::opt<std::string> InputFilenamesFile( 1138 "input-files", cl::init(""), 1139 cl::desc("Path to file containing newline-separated " 1140 "[<weight>,]<filename> entries")); 1141 cl::alias InputFilenamesFileA("f", cl::desc("Alias for --input-files"), 1142 cl::aliasopt(InputFilenamesFile)); 1143 cl::opt<bool> DumpInputFileList( 1144 "dump-input-file-list", cl::init(false), cl::Hidden, 1145 cl::desc("Dump the list of input files and their weights, then exit")); 1146 cl::opt<std::string> RemappingFile("remapping-file", cl::value_desc("file"), 1147 cl::desc("Symbol remapping file")); 1148 cl::alias RemappingFileA("r", cl::desc("Alias for --remapping-file"), 1149 cl::aliasopt(RemappingFile)); 1150 cl::opt<std::string> OutputFilename("output", cl::value_desc("output"), 1151 cl::init("-"), cl::desc("Output file")); 1152 cl::alias OutputFilenameA("o", cl::desc("Alias for --output"), 1153 cl::aliasopt(OutputFilename)); 1154 cl::opt<ProfileKinds> ProfileKind( 1155 cl::desc("Profile kind:"), cl::init(instr), 1156 cl::values(clEnumVal(instr, "Instrumentation profile (default)"), 1157 clEnumVal(sample, "Sample profile"))); 1158 cl::opt<ProfileFormat> OutputFormat( 1159 cl::desc("Format of output profile"), cl::init(PF_Binary), 1160 cl::values( 1161 clEnumValN(PF_Binary, "binary", "Binary encoding (default)"), 1162 clEnumValN(PF_Compact_Binary, "compbinary", 1163 "Compact binary encoding"), 1164 clEnumValN(PF_Ext_Binary, "extbinary", "Extensible binary encoding"), 1165 clEnumValN(PF_Text, "text", "Text encoding"), 1166 clEnumValN(PF_GCC, "gcc", 1167 "GCC encoding (only meaningful for -sample)"))); 1168 cl::opt<FailureMode> FailureMode( 1169 "failure-mode", cl::init(failIfAnyAreInvalid), cl::desc("Failure mode:"), 1170 cl::values(clEnumValN(failIfAnyAreInvalid, "any", 1171 "Fail if any profile is invalid."), 1172 clEnumValN(failIfAllAreInvalid, "all", 1173 "Fail only if all profiles are invalid."))); 1174 cl::opt<bool> OutputSparse("sparse", cl::init(false), 1175 cl::desc("Generate a sparse profile (only meaningful for -instr)")); 1176 cl::opt<unsigned> NumThreads( 1177 "num-threads", cl::init(0), 1178 cl::desc("Number of merge threads to use (default: autodetect)")); 1179 cl::alias NumThreadsA("j", cl::desc("Alias for --num-threads"), 1180 cl::aliasopt(NumThreads)); 1181 cl::opt<std::string> ProfileSymbolListFile( 1182 "prof-sym-list", cl::init(""), 1183 cl::desc("Path to file containing the list of function symbols " 1184 "used to populate profile symbol list")); 1185 cl::opt<bool> CompressAllSections( 1186 "compress-all-sections", cl::init(false), cl::Hidden, 1187 cl::desc("Compress all sections when writing the profile (only " 1188 "meaningful for -extbinary)")); 1189 cl::opt<bool> UseMD5( 1190 "use-md5", cl::init(false), cl::Hidden, 1191 cl::desc("Choose to use MD5 to represent string in name table (only " 1192 "meaningful for -extbinary)")); 1193 cl::opt<bool> SampleMergeColdContext( 1194 "sample-merge-cold-context", cl::init(false), cl::Hidden, 1195 cl::desc( 1196 "Merge context sample profiles whose count is below cold threshold")); 1197 cl::opt<bool> SampleTrimColdContext( 1198 "sample-trim-cold-context", cl::init(false), cl::Hidden, 1199 cl::desc( 1200 "Trim context sample profiles whose count is below cold threshold")); 1201 cl::opt<uint32_t> SampleColdContextFrameDepth( 1202 "sample-frame-depth-for-cold-context", cl::init(1), 1203 cl::desc("Keep the last K frames while merging cold profile. 1 means the " 1204 "context-less base profile")); 1205 cl::opt<bool> GenPartialProfile( 1206 "gen-partial-profile", cl::init(false), cl::Hidden, 1207 cl::desc("Generate a partial profile (only meaningful for -extbinary)")); 1208 cl::opt<std::string> SupplInstrWithSample( 1209 "supplement-instr-with-sample", cl::init(""), cl::Hidden, 1210 cl::desc("Supplement an instr profile with sample profile, to correct " 1211 "the profile unrepresentativeness issue. The sample " 1212 "profile is the input of the flag. Output will be in instr " 1213 "format (The flag only works with -instr)")); 1214 cl::opt<float> ZeroCounterThreshold( 1215 "zero-counter-threshold", cl::init(0.7), cl::Hidden, 1216 cl::desc("For the function which is cold in instr profile but hot in " 1217 "sample profile, if the ratio of the number of zero counters " 1218 "divided by the total number of counters is above the " 1219 "threshold, the profile of the function will be regarded as " 1220 "being harmful for performance and will be dropped.")); 1221 cl::opt<unsigned> SupplMinSizeThreshold( 1222 "suppl-min-size-threshold", cl::init(10), cl::Hidden, 1223 cl::desc("If the size of a function is smaller than the threshold, " 1224 "assume it can be inlined by PGO early inliner and it won't " 1225 "be adjusted based on sample profile.")); 1226 cl::opt<unsigned> InstrProfColdThreshold( 1227 "instr-prof-cold-threshold", cl::init(0), cl::Hidden, 1228 cl::desc("User specified cold threshold for instr profile which will " 1229 "override the cold threshold got from profile summary. ")); 1230 cl::opt<bool> GenCSNestedProfile( 1231 "gen-cs-nested-profile", cl::Hidden, cl::init(false), 1232 cl::desc("Generate nested function profiles for CSSPGO")); 1233 cl::opt<std::string> DebugInfoFilename( 1234 "debug-info", cl::init(""), 1235 cl::desc("Use the provided debug info to correlate the raw profile.")); 1236 cl::opt<std::string> ProfiledBinary( 1237 "profiled-binary", cl::init(""), 1238 cl::desc("Path to binary from which the profile was collected.")); 1239 cl::opt<bool> DropProfileSymbolList( 1240 "drop-profile-symbol-list", cl::init(false), cl::Hidden, 1241 cl::desc("Drop the profile symbol list when merging AutoFDO profiles " 1242 "(only meaningful for -sample)")); 1243 1244 cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n"); 1245 1246 WeightedFileVector WeightedInputs; 1247 for (StringRef Filename : InputFilenames) 1248 addWeightedInput(WeightedInputs, {std::string(Filename), 1}); 1249 for (StringRef WeightedFilename : WeightedInputFilenames) 1250 addWeightedInput(WeightedInputs, parseWeightedFile(WeightedFilename)); 1251 1252 // Make sure that the file buffer stays alive for the duration of the 1253 // weighted input vector's lifetime. 1254 auto Buffer = getInputFileBuf(InputFilenamesFile); 1255 parseInputFilenamesFile(Buffer.get(), WeightedInputs); 1256 1257 if (WeightedInputs.empty()) 1258 exitWithError("no input files specified. See " + 1259 sys::path::filename(argv[0]) + " -help"); 1260 1261 if (DumpInputFileList) { 1262 for (auto &WF : WeightedInputs) 1263 outs() << WF.Weight << "," << WF.Filename << "\n"; 1264 return 0; 1265 } 1266 1267 std::unique_ptr<SymbolRemapper> Remapper; 1268 if (!RemappingFile.empty()) 1269 Remapper = SymbolRemapper::create(RemappingFile); 1270 1271 if (!SupplInstrWithSample.empty()) { 1272 if (ProfileKind != instr) 1273 exitWithError( 1274 "-supplement-instr-with-sample can only work with -instr. "); 1275 1276 supplementInstrProfile(WeightedInputs, SupplInstrWithSample, OutputFilename, 1277 OutputFormat, OutputSparse, SupplMinSizeThreshold, 1278 ZeroCounterThreshold, InstrProfColdThreshold); 1279 return 0; 1280 } 1281 1282 if (ProfileKind == instr) 1283 mergeInstrProfile(WeightedInputs, DebugInfoFilename, Remapper.get(), 1284 OutputFilename, OutputFormat, OutputSparse, NumThreads, 1285 FailureMode, ProfiledBinary); 1286 else 1287 mergeSampleProfile( 1288 WeightedInputs, Remapper.get(), OutputFilename, OutputFormat, 1289 ProfileSymbolListFile, CompressAllSections, UseMD5, GenPartialProfile, 1290 GenCSNestedProfile, SampleMergeColdContext, SampleTrimColdContext, 1291 SampleColdContextFrameDepth, FailureMode, DropProfileSymbolList); 1292 return 0; 1293 } 1294 1295 /// Computer the overlap b/w profile BaseFilename and profile TestFilename. 1296 static void overlapInstrProfile(const std::string &BaseFilename, 1297 const std::string &TestFilename, 1298 const OverlapFuncFilters &FuncFilter, 1299 raw_fd_ostream &OS, bool IsCS) { 1300 std::mutex ErrorLock; 1301 SmallSet<instrprof_error, 4> WriterErrorCodes; 1302 WriterContext Context(false, ErrorLock, WriterErrorCodes); 1303 WeightedFile WeightedInput{BaseFilename, 1}; 1304 OverlapStats Overlap; 1305 Error E = Overlap.accumulateCounts(BaseFilename, TestFilename, IsCS); 1306 if (E) 1307 exitWithError(std::move(E), "error in getting profile count sums"); 1308 if (Overlap.Base.CountSum < 1.0f) { 1309 OS << "Sum of edge counts for profile " << BaseFilename << " is 0.\n"; 1310 exit(0); 1311 } 1312 if (Overlap.Test.CountSum < 1.0f) { 1313 OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n"; 1314 exit(0); 1315 } 1316 loadInput(WeightedInput, nullptr, nullptr, /*ProfiledBinary=*/"", &Context); 1317 overlapInput(BaseFilename, TestFilename, &Context, Overlap, FuncFilter, OS, 1318 IsCS); 1319 Overlap.dump(OS); 1320 } 1321 1322 namespace { 1323 struct SampleOverlapStats { 1324 SampleContext BaseName; 1325 SampleContext TestName; 1326 // Number of overlap units 1327 uint64_t OverlapCount; 1328 // Total samples of overlap units 1329 uint64_t OverlapSample; 1330 // Number of and total samples of units that only present in base or test 1331 // profile 1332 uint64_t BaseUniqueCount; 1333 uint64_t BaseUniqueSample; 1334 uint64_t TestUniqueCount; 1335 uint64_t TestUniqueSample; 1336 // Number of units and total samples in base or test profile 1337 uint64_t BaseCount; 1338 uint64_t BaseSample; 1339 uint64_t TestCount; 1340 uint64_t TestSample; 1341 // Number of and total samples of units that present in at least one profile 1342 uint64_t UnionCount; 1343 uint64_t UnionSample; 1344 // Weighted similarity 1345 double Similarity; 1346 // For SampleOverlapStats instances representing functions, weights of the 1347 // function in base and test profiles 1348 double BaseWeight; 1349 double TestWeight; 1350 1351 SampleOverlapStats() 1352 : OverlapCount(0), OverlapSample(0), BaseUniqueCount(0), 1353 BaseUniqueSample(0), TestUniqueCount(0), TestUniqueSample(0), 1354 BaseCount(0), BaseSample(0), TestCount(0), TestSample(0), UnionCount(0), 1355 UnionSample(0), Similarity(0.0), BaseWeight(0.0), TestWeight(0.0) {} 1356 }; 1357 } // end anonymous namespace 1358 1359 namespace { 1360 struct FuncSampleStats { 1361 uint64_t SampleSum; 1362 uint64_t MaxSample; 1363 uint64_t HotBlockCount; 1364 FuncSampleStats() : SampleSum(0), MaxSample(0), HotBlockCount(0) {} 1365 FuncSampleStats(uint64_t SampleSum, uint64_t MaxSample, 1366 uint64_t HotBlockCount) 1367 : SampleSum(SampleSum), MaxSample(MaxSample), 1368 HotBlockCount(HotBlockCount) {} 1369 }; 1370 } // end anonymous namespace 1371 1372 namespace { 1373 enum MatchStatus { MS_Match, MS_FirstUnique, MS_SecondUnique, MS_None }; 1374 1375 // Class for updating merging steps for two sorted maps. The class should be 1376 // instantiated with a map iterator type. 1377 template <class T> class MatchStep { 1378 public: 1379 MatchStep() = delete; 1380 1381 MatchStep(T FirstIter, T FirstEnd, T SecondIter, T SecondEnd) 1382 : FirstIter(FirstIter), FirstEnd(FirstEnd), SecondIter(SecondIter), 1383 SecondEnd(SecondEnd), Status(MS_None) {} 1384 1385 bool areBothFinished() const { 1386 return (FirstIter == FirstEnd && SecondIter == SecondEnd); 1387 } 1388 1389 bool isFirstFinished() const { return FirstIter == FirstEnd; } 1390 1391 bool isSecondFinished() const { return SecondIter == SecondEnd; } 1392 1393 /// Advance one step based on the previous match status unless the previous 1394 /// status is MS_None. Then update Status based on the comparison between two 1395 /// container iterators at the current step. If the previous status is 1396 /// MS_None, it means two iterators are at the beginning and no comparison has 1397 /// been made, so we simply update Status without advancing the iterators. 1398 void updateOneStep(); 1399 1400 T getFirstIter() const { return FirstIter; } 1401 1402 T getSecondIter() const { return SecondIter; } 1403 1404 MatchStatus getMatchStatus() const { return Status; } 1405 1406 private: 1407 // Current iterator and end iterator of the first container. 1408 T FirstIter; 1409 T FirstEnd; 1410 // Current iterator and end iterator of the second container. 1411 T SecondIter; 1412 T SecondEnd; 1413 // Match status of the current step. 1414 MatchStatus Status; 1415 }; 1416 } // end anonymous namespace 1417 1418 template <class T> void MatchStep<T>::updateOneStep() { 1419 switch (Status) { 1420 case MS_Match: 1421 ++FirstIter; 1422 ++SecondIter; 1423 break; 1424 case MS_FirstUnique: 1425 ++FirstIter; 1426 break; 1427 case MS_SecondUnique: 1428 ++SecondIter; 1429 break; 1430 case MS_None: 1431 break; 1432 } 1433 1434 // Update Status according to iterators at the current step. 1435 if (areBothFinished()) 1436 return; 1437 if (FirstIter != FirstEnd && 1438 (SecondIter == SecondEnd || FirstIter->first < SecondIter->first)) 1439 Status = MS_FirstUnique; 1440 else if (SecondIter != SecondEnd && 1441 (FirstIter == FirstEnd || SecondIter->first < FirstIter->first)) 1442 Status = MS_SecondUnique; 1443 else 1444 Status = MS_Match; 1445 } 1446 1447 // Return the sum of line/block samples, the max line/block sample, and the 1448 // number of line/block samples above the given threshold in a function 1449 // including its inlinees. 1450 static void getFuncSampleStats(const sampleprof::FunctionSamples &Func, 1451 FuncSampleStats &FuncStats, 1452 uint64_t HotThreshold) { 1453 for (const auto &L : Func.getBodySamples()) { 1454 uint64_t Sample = L.second.getSamples(); 1455 FuncStats.SampleSum += Sample; 1456 FuncStats.MaxSample = std::max(FuncStats.MaxSample, Sample); 1457 if (Sample >= HotThreshold) 1458 ++FuncStats.HotBlockCount; 1459 } 1460 1461 for (const auto &C : Func.getCallsiteSamples()) { 1462 for (const auto &F : C.second) 1463 getFuncSampleStats(F.second, FuncStats, HotThreshold); 1464 } 1465 } 1466 1467 /// Predicate that determines if a function is hot with a given threshold. We 1468 /// keep it separate from its callsites for possible extension in the future. 1469 static bool isFunctionHot(const FuncSampleStats &FuncStats, 1470 uint64_t HotThreshold) { 1471 // We intentionally compare the maximum sample count in a function with the 1472 // HotThreshold to get an approximate determination on hot functions. 1473 return (FuncStats.MaxSample >= HotThreshold); 1474 } 1475 1476 namespace { 1477 class SampleOverlapAggregator { 1478 public: 1479 SampleOverlapAggregator(const std::string &BaseFilename, 1480 const std::string &TestFilename, 1481 double LowSimilarityThreshold, double Epsilon, 1482 const OverlapFuncFilters &FuncFilter) 1483 : BaseFilename(BaseFilename), TestFilename(TestFilename), 1484 LowSimilarityThreshold(LowSimilarityThreshold), Epsilon(Epsilon), 1485 FuncFilter(FuncFilter) {} 1486 1487 /// Detect 0-sample input profile and report to output stream. This interface 1488 /// should be called after loadProfiles(). 1489 bool detectZeroSampleProfile(raw_fd_ostream &OS) const; 1490 1491 /// Write out function-level similarity statistics for functions specified by 1492 /// options --function, --value-cutoff, and --similarity-cutoff. 1493 void dumpFuncSimilarity(raw_fd_ostream &OS) const; 1494 1495 /// Write out program-level similarity and overlap statistics. 1496 void dumpProgramSummary(raw_fd_ostream &OS) const; 1497 1498 /// Write out hot-function and hot-block statistics for base_profile, 1499 /// test_profile, and their overlap. For both cases, the overlap HO is 1500 /// calculated as follows: 1501 /// Given the number of functions (or blocks) that are hot in both profiles 1502 /// HCommon and the number of functions (or blocks) that are hot in at 1503 /// least one profile HUnion, HO = HCommon / HUnion. 1504 void dumpHotFuncAndBlockOverlap(raw_fd_ostream &OS) const; 1505 1506 /// This function tries matching functions in base and test profiles. For each 1507 /// pair of matched functions, it aggregates the function-level 1508 /// similarity into a profile-level similarity. It also dump function-level 1509 /// similarity information of functions specified by --function, 1510 /// --value-cutoff, and --similarity-cutoff options. The program-level 1511 /// similarity PS is computed as follows: 1512 /// Given function-level similarity FS(A) for all function A, the 1513 /// weight of function A in base profile WB(A), and the weight of function 1514 /// A in test profile WT(A), compute PS(base_profile, test_profile) = 1515 /// sum_A(FS(A) * avg(WB(A), WT(A))) ranging in [0.0f to 1.0f] with 0.0 1516 /// meaning no-overlap. 1517 void computeSampleProfileOverlap(raw_fd_ostream &OS); 1518 1519 /// Initialize ProfOverlap with the sum of samples in base and test 1520 /// profiles. This function also computes and keeps the sum of samples and 1521 /// max sample counts of each function in BaseStats and TestStats for later 1522 /// use to avoid re-computations. 1523 void initializeSampleProfileOverlap(); 1524 1525 /// Load profiles specified by BaseFilename and TestFilename. 1526 std::error_code loadProfiles(); 1527 1528 using FuncSampleStatsMap = 1529 std::unordered_map<SampleContext, FuncSampleStats, SampleContext::Hash>; 1530 1531 private: 1532 SampleOverlapStats ProfOverlap; 1533 SampleOverlapStats HotFuncOverlap; 1534 SampleOverlapStats HotBlockOverlap; 1535 std::string BaseFilename; 1536 std::string TestFilename; 1537 std::unique_ptr<sampleprof::SampleProfileReader> BaseReader; 1538 std::unique_ptr<sampleprof::SampleProfileReader> TestReader; 1539 // BaseStats and TestStats hold FuncSampleStats for each function, with 1540 // function name as the key. 1541 FuncSampleStatsMap BaseStats; 1542 FuncSampleStatsMap TestStats; 1543 // Low similarity threshold in floating point number 1544 double LowSimilarityThreshold; 1545 // Block samples above BaseHotThreshold or TestHotThreshold are considered hot 1546 // for tracking hot blocks. 1547 uint64_t BaseHotThreshold; 1548 uint64_t TestHotThreshold; 1549 // A small threshold used to round the results of floating point accumulations 1550 // to resolve imprecision. 1551 const double Epsilon; 1552 std::multimap<double, SampleOverlapStats, std::greater<double>> 1553 FuncSimilarityDump; 1554 // FuncFilter carries specifications in options --value-cutoff and 1555 // --function. 1556 OverlapFuncFilters FuncFilter; 1557 // Column offsets for printing the function-level details table. 1558 static const unsigned int TestWeightCol = 15; 1559 static const unsigned int SimilarityCol = 30; 1560 static const unsigned int OverlapCol = 43; 1561 static const unsigned int BaseUniqueCol = 53; 1562 static const unsigned int TestUniqueCol = 67; 1563 static const unsigned int BaseSampleCol = 81; 1564 static const unsigned int TestSampleCol = 96; 1565 static const unsigned int FuncNameCol = 111; 1566 1567 /// Return a similarity of two line/block sample counters in the same 1568 /// function in base and test profiles. The line/block-similarity BS(i) is 1569 /// computed as follows: 1570 /// For an offsets i, given the sample count at i in base profile BB(i), 1571 /// the sample count at i in test profile BT(i), the sum of sample counts 1572 /// in this function in base profile SB, and the sum of sample counts in 1573 /// this function in test profile ST, compute BS(i) = 1.0 - fabs(BB(i)/SB - 1574 /// BT(i)/ST), ranging in [0.0f to 1.0f] with 0.0 meaning no-overlap. 1575 double computeBlockSimilarity(uint64_t BaseSample, uint64_t TestSample, 1576 const SampleOverlapStats &FuncOverlap) const; 1577 1578 void updateHotBlockOverlap(uint64_t BaseSample, uint64_t TestSample, 1579 uint64_t HotBlockCount); 1580 1581 void getHotFunctions(const FuncSampleStatsMap &ProfStats, 1582 FuncSampleStatsMap &HotFunc, 1583 uint64_t HotThreshold) const; 1584 1585 void computeHotFuncOverlap(); 1586 1587 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and 1588 /// Difference for two sample units in a matched function according to the 1589 /// given match status. 1590 void updateOverlapStatsForFunction(uint64_t BaseSample, uint64_t TestSample, 1591 uint64_t HotBlockCount, 1592 SampleOverlapStats &FuncOverlap, 1593 double &Difference, MatchStatus Status); 1594 1595 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and 1596 /// Difference for unmatched callees that only present in one profile in a 1597 /// matched caller function. 1598 void updateForUnmatchedCallee(const sampleprof::FunctionSamples &Func, 1599 SampleOverlapStats &FuncOverlap, 1600 double &Difference, MatchStatus Status); 1601 1602 /// This function updates sample overlap statistics of an overlap function in 1603 /// base and test profile. It also calculates a function-internal similarity 1604 /// FIS as follows: 1605 /// For offsets i that have samples in at least one profile in this 1606 /// function A, given BS(i) returned by computeBlockSimilarity(), compute 1607 /// FIS(A) = (2.0 - sum_i(1.0 - BS(i))) / 2, ranging in [0.0f to 1.0f] with 1608 /// 0.0 meaning no overlap. 1609 double computeSampleFunctionInternalOverlap( 1610 const sampleprof::FunctionSamples &BaseFunc, 1611 const sampleprof::FunctionSamples &TestFunc, 1612 SampleOverlapStats &FuncOverlap); 1613 1614 /// Function-level similarity (FS) is a weighted value over function internal 1615 /// similarity (FIS). This function computes a function's FS from its FIS by 1616 /// applying the weight. 1617 double weightForFuncSimilarity(double FuncSimilarity, uint64_t BaseFuncSample, 1618 uint64_t TestFuncSample) const; 1619 1620 /// The function-level similarity FS(A) for a function A is computed as 1621 /// follows: 1622 /// Compute a function-internal similarity FIS(A) by 1623 /// computeSampleFunctionInternalOverlap(). Then, with the weight of 1624 /// function A in base profile WB(A), and the weight of function A in test 1625 /// profile WT(A), compute FS(A) = FIS(A) * (1.0 - fabs(WB(A) - WT(A))) 1626 /// ranging in [0.0f to 1.0f] with 0.0 meaning no overlap. 1627 double 1628 computeSampleFunctionOverlap(const sampleprof::FunctionSamples *BaseFunc, 1629 const sampleprof::FunctionSamples *TestFunc, 1630 SampleOverlapStats *FuncOverlap, 1631 uint64_t BaseFuncSample, 1632 uint64_t TestFuncSample); 1633 1634 /// Profile-level similarity (PS) is a weighted aggregate over function-level 1635 /// similarities (FS). This method weights the FS value by the function 1636 /// weights in the base and test profiles for the aggregation. 1637 double weightByImportance(double FuncSimilarity, uint64_t BaseFuncSample, 1638 uint64_t TestFuncSample) const; 1639 }; 1640 } // end anonymous namespace 1641 1642 bool SampleOverlapAggregator::detectZeroSampleProfile( 1643 raw_fd_ostream &OS) const { 1644 bool HaveZeroSample = false; 1645 if (ProfOverlap.BaseSample == 0) { 1646 OS << "Sum of sample counts for profile " << BaseFilename << " is 0.\n"; 1647 HaveZeroSample = true; 1648 } 1649 if (ProfOverlap.TestSample == 0) { 1650 OS << "Sum of sample counts for profile " << TestFilename << " is 0.\n"; 1651 HaveZeroSample = true; 1652 } 1653 return HaveZeroSample; 1654 } 1655 1656 double SampleOverlapAggregator::computeBlockSimilarity( 1657 uint64_t BaseSample, uint64_t TestSample, 1658 const SampleOverlapStats &FuncOverlap) const { 1659 double BaseFrac = 0.0; 1660 double TestFrac = 0.0; 1661 if (FuncOverlap.BaseSample > 0) 1662 BaseFrac = static_cast<double>(BaseSample) / FuncOverlap.BaseSample; 1663 if (FuncOverlap.TestSample > 0) 1664 TestFrac = static_cast<double>(TestSample) / FuncOverlap.TestSample; 1665 return 1.0 - std::fabs(BaseFrac - TestFrac); 1666 } 1667 1668 void SampleOverlapAggregator::updateHotBlockOverlap(uint64_t BaseSample, 1669 uint64_t TestSample, 1670 uint64_t HotBlockCount) { 1671 bool IsBaseHot = (BaseSample >= BaseHotThreshold); 1672 bool IsTestHot = (TestSample >= TestHotThreshold); 1673 if (!IsBaseHot && !IsTestHot) 1674 return; 1675 1676 HotBlockOverlap.UnionCount += HotBlockCount; 1677 if (IsBaseHot) 1678 HotBlockOverlap.BaseCount += HotBlockCount; 1679 if (IsTestHot) 1680 HotBlockOverlap.TestCount += HotBlockCount; 1681 if (IsBaseHot && IsTestHot) 1682 HotBlockOverlap.OverlapCount += HotBlockCount; 1683 } 1684 1685 void SampleOverlapAggregator::getHotFunctions( 1686 const FuncSampleStatsMap &ProfStats, FuncSampleStatsMap &HotFunc, 1687 uint64_t HotThreshold) const { 1688 for (const auto &F : ProfStats) { 1689 if (isFunctionHot(F.second, HotThreshold)) 1690 HotFunc.emplace(F.first, F.second); 1691 } 1692 } 1693 1694 void SampleOverlapAggregator::computeHotFuncOverlap() { 1695 FuncSampleStatsMap BaseHotFunc; 1696 getHotFunctions(BaseStats, BaseHotFunc, BaseHotThreshold); 1697 HotFuncOverlap.BaseCount = BaseHotFunc.size(); 1698 1699 FuncSampleStatsMap TestHotFunc; 1700 getHotFunctions(TestStats, TestHotFunc, TestHotThreshold); 1701 HotFuncOverlap.TestCount = TestHotFunc.size(); 1702 HotFuncOverlap.UnionCount = HotFuncOverlap.TestCount; 1703 1704 for (const auto &F : BaseHotFunc) { 1705 if (TestHotFunc.count(F.first)) 1706 ++HotFuncOverlap.OverlapCount; 1707 else 1708 ++HotFuncOverlap.UnionCount; 1709 } 1710 } 1711 1712 void SampleOverlapAggregator::updateOverlapStatsForFunction( 1713 uint64_t BaseSample, uint64_t TestSample, uint64_t HotBlockCount, 1714 SampleOverlapStats &FuncOverlap, double &Difference, MatchStatus Status) { 1715 assert(Status != MS_None && 1716 "Match status should be updated before updating overlap statistics"); 1717 if (Status == MS_FirstUnique) { 1718 TestSample = 0; 1719 FuncOverlap.BaseUniqueSample += BaseSample; 1720 } else if (Status == MS_SecondUnique) { 1721 BaseSample = 0; 1722 FuncOverlap.TestUniqueSample += TestSample; 1723 } else { 1724 ++FuncOverlap.OverlapCount; 1725 } 1726 1727 FuncOverlap.UnionSample += std::max(BaseSample, TestSample); 1728 FuncOverlap.OverlapSample += std::min(BaseSample, TestSample); 1729 Difference += 1730 1.0 - computeBlockSimilarity(BaseSample, TestSample, FuncOverlap); 1731 updateHotBlockOverlap(BaseSample, TestSample, HotBlockCount); 1732 } 1733 1734 void SampleOverlapAggregator::updateForUnmatchedCallee( 1735 const sampleprof::FunctionSamples &Func, SampleOverlapStats &FuncOverlap, 1736 double &Difference, MatchStatus Status) { 1737 assert((Status == MS_FirstUnique || Status == MS_SecondUnique) && 1738 "Status must be either of the two unmatched cases"); 1739 FuncSampleStats FuncStats; 1740 if (Status == MS_FirstUnique) { 1741 getFuncSampleStats(Func, FuncStats, BaseHotThreshold); 1742 updateOverlapStatsForFunction(FuncStats.SampleSum, 0, 1743 FuncStats.HotBlockCount, FuncOverlap, 1744 Difference, Status); 1745 } else { 1746 getFuncSampleStats(Func, FuncStats, TestHotThreshold); 1747 updateOverlapStatsForFunction(0, FuncStats.SampleSum, 1748 FuncStats.HotBlockCount, FuncOverlap, 1749 Difference, Status); 1750 } 1751 } 1752 1753 double SampleOverlapAggregator::computeSampleFunctionInternalOverlap( 1754 const sampleprof::FunctionSamples &BaseFunc, 1755 const sampleprof::FunctionSamples &TestFunc, 1756 SampleOverlapStats &FuncOverlap) { 1757 1758 using namespace sampleprof; 1759 1760 double Difference = 0; 1761 1762 // Accumulate Difference for regular line/block samples in the function. 1763 // We match them through sort-merge join algorithm because 1764 // FunctionSamples::getBodySamples() returns a map of sample counters ordered 1765 // by their offsets. 1766 MatchStep<BodySampleMap::const_iterator> BlockIterStep( 1767 BaseFunc.getBodySamples().cbegin(), BaseFunc.getBodySamples().cend(), 1768 TestFunc.getBodySamples().cbegin(), TestFunc.getBodySamples().cend()); 1769 BlockIterStep.updateOneStep(); 1770 while (!BlockIterStep.areBothFinished()) { 1771 uint64_t BaseSample = 1772 BlockIterStep.isFirstFinished() 1773 ? 0 1774 : BlockIterStep.getFirstIter()->second.getSamples(); 1775 uint64_t TestSample = 1776 BlockIterStep.isSecondFinished() 1777 ? 0 1778 : BlockIterStep.getSecondIter()->second.getSamples(); 1779 updateOverlapStatsForFunction(BaseSample, TestSample, 1, FuncOverlap, 1780 Difference, BlockIterStep.getMatchStatus()); 1781 1782 BlockIterStep.updateOneStep(); 1783 } 1784 1785 // Accumulate Difference for callsite lines in the function. We match 1786 // them through sort-merge algorithm because 1787 // FunctionSamples::getCallsiteSamples() returns a map of callsite records 1788 // ordered by their offsets. 1789 MatchStep<CallsiteSampleMap::const_iterator> CallsiteIterStep( 1790 BaseFunc.getCallsiteSamples().cbegin(), 1791 BaseFunc.getCallsiteSamples().cend(), 1792 TestFunc.getCallsiteSamples().cbegin(), 1793 TestFunc.getCallsiteSamples().cend()); 1794 CallsiteIterStep.updateOneStep(); 1795 while (!CallsiteIterStep.areBothFinished()) { 1796 MatchStatus CallsiteStepStatus = CallsiteIterStep.getMatchStatus(); 1797 assert(CallsiteStepStatus != MS_None && 1798 "Match status should be updated before entering loop body"); 1799 1800 if (CallsiteStepStatus != MS_Match) { 1801 auto Callsite = (CallsiteStepStatus == MS_FirstUnique) 1802 ? CallsiteIterStep.getFirstIter() 1803 : CallsiteIterStep.getSecondIter(); 1804 for (const auto &F : Callsite->second) 1805 updateForUnmatchedCallee(F.second, FuncOverlap, Difference, 1806 CallsiteStepStatus); 1807 } else { 1808 // There may be multiple inlinees at the same offset, so we need to try 1809 // matching all of them. This match is implemented through sort-merge 1810 // algorithm because callsite records at the same offset are ordered by 1811 // function names. 1812 MatchStep<FunctionSamplesMap::const_iterator> CalleeIterStep( 1813 CallsiteIterStep.getFirstIter()->second.cbegin(), 1814 CallsiteIterStep.getFirstIter()->second.cend(), 1815 CallsiteIterStep.getSecondIter()->second.cbegin(), 1816 CallsiteIterStep.getSecondIter()->second.cend()); 1817 CalleeIterStep.updateOneStep(); 1818 while (!CalleeIterStep.areBothFinished()) { 1819 MatchStatus CalleeStepStatus = CalleeIterStep.getMatchStatus(); 1820 if (CalleeStepStatus != MS_Match) { 1821 auto Callee = (CalleeStepStatus == MS_FirstUnique) 1822 ? CalleeIterStep.getFirstIter() 1823 : CalleeIterStep.getSecondIter(); 1824 updateForUnmatchedCallee(Callee->second, FuncOverlap, Difference, 1825 CalleeStepStatus); 1826 } else { 1827 // An inlined function can contain other inlinees inside, so compute 1828 // the Difference recursively. 1829 Difference += 2.0 - 2 * computeSampleFunctionInternalOverlap( 1830 CalleeIterStep.getFirstIter()->second, 1831 CalleeIterStep.getSecondIter()->second, 1832 FuncOverlap); 1833 } 1834 CalleeIterStep.updateOneStep(); 1835 } 1836 } 1837 CallsiteIterStep.updateOneStep(); 1838 } 1839 1840 // Difference reflects the total differences of line/block samples in this 1841 // function and ranges in [0.0f to 2.0f]. Take (2.0 - Difference) / 2 to 1842 // reflect the similarity between function profiles in [0.0f to 1.0f]. 1843 return (2.0 - Difference) / 2; 1844 } 1845 1846 double SampleOverlapAggregator::weightForFuncSimilarity( 1847 double FuncInternalSimilarity, uint64_t BaseFuncSample, 1848 uint64_t TestFuncSample) const { 1849 // Compute the weight as the distance between the function weights in two 1850 // profiles. 1851 double BaseFrac = 0.0; 1852 double TestFrac = 0.0; 1853 assert(ProfOverlap.BaseSample > 0 && 1854 "Total samples in base profile should be greater than 0"); 1855 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample; 1856 assert(ProfOverlap.TestSample > 0 && 1857 "Total samples in test profile should be greater than 0"); 1858 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample; 1859 double WeightDistance = std::fabs(BaseFrac - TestFrac); 1860 1861 // Take WeightDistance into the similarity. 1862 return FuncInternalSimilarity * (1 - WeightDistance); 1863 } 1864 1865 double 1866 SampleOverlapAggregator::weightByImportance(double FuncSimilarity, 1867 uint64_t BaseFuncSample, 1868 uint64_t TestFuncSample) const { 1869 1870 double BaseFrac = 0.0; 1871 double TestFrac = 0.0; 1872 assert(ProfOverlap.BaseSample > 0 && 1873 "Total samples in base profile should be greater than 0"); 1874 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample / 2.0; 1875 assert(ProfOverlap.TestSample > 0 && 1876 "Total samples in test profile should be greater than 0"); 1877 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample / 2.0; 1878 return FuncSimilarity * (BaseFrac + TestFrac); 1879 } 1880 1881 double SampleOverlapAggregator::computeSampleFunctionOverlap( 1882 const sampleprof::FunctionSamples *BaseFunc, 1883 const sampleprof::FunctionSamples *TestFunc, 1884 SampleOverlapStats *FuncOverlap, uint64_t BaseFuncSample, 1885 uint64_t TestFuncSample) { 1886 // Default function internal similarity before weighted, meaning two functions 1887 // has no overlap. 1888 const double DefaultFuncInternalSimilarity = 0; 1889 double FuncSimilarity; 1890 double FuncInternalSimilarity; 1891 1892 // If BaseFunc or TestFunc is nullptr, it means the functions do not overlap. 1893 // In this case, we use DefaultFuncInternalSimilarity as the function internal 1894 // similarity. 1895 if (!BaseFunc || !TestFunc) { 1896 FuncInternalSimilarity = DefaultFuncInternalSimilarity; 1897 } else { 1898 assert(FuncOverlap != nullptr && 1899 "FuncOverlap should be provided in this case"); 1900 FuncInternalSimilarity = computeSampleFunctionInternalOverlap( 1901 *BaseFunc, *TestFunc, *FuncOverlap); 1902 // Now, FuncInternalSimilarity may be a little less than 0 due to 1903 // imprecision of floating point accumulations. Make it zero if the 1904 // difference is below Epsilon. 1905 FuncInternalSimilarity = (std::fabs(FuncInternalSimilarity - 0) < Epsilon) 1906 ? 0 1907 : FuncInternalSimilarity; 1908 } 1909 FuncSimilarity = weightForFuncSimilarity(FuncInternalSimilarity, 1910 BaseFuncSample, TestFuncSample); 1911 return FuncSimilarity; 1912 } 1913 1914 void SampleOverlapAggregator::computeSampleProfileOverlap(raw_fd_ostream &OS) { 1915 using namespace sampleprof; 1916 1917 std::unordered_map<SampleContext, const FunctionSamples *, 1918 SampleContext::Hash> 1919 BaseFuncProf; 1920 const auto &BaseProfiles = BaseReader->getProfiles(); 1921 for (const auto &BaseFunc : BaseProfiles) { 1922 BaseFuncProf.emplace(BaseFunc.second.getContext(), &(BaseFunc.second)); 1923 } 1924 ProfOverlap.UnionCount = BaseFuncProf.size(); 1925 1926 const auto &TestProfiles = TestReader->getProfiles(); 1927 for (const auto &TestFunc : TestProfiles) { 1928 SampleOverlapStats FuncOverlap; 1929 FuncOverlap.TestName = TestFunc.second.getContext(); 1930 assert(TestStats.count(FuncOverlap.TestName) && 1931 "TestStats should have records for all functions in test profile " 1932 "except inlinees"); 1933 FuncOverlap.TestSample = TestStats[FuncOverlap.TestName].SampleSum; 1934 1935 bool Matched = false; 1936 const auto Match = BaseFuncProf.find(FuncOverlap.TestName); 1937 if (Match == BaseFuncProf.end()) { 1938 const FuncSampleStats &FuncStats = TestStats[FuncOverlap.TestName]; 1939 ++ProfOverlap.TestUniqueCount; 1940 ProfOverlap.TestUniqueSample += FuncStats.SampleSum; 1941 FuncOverlap.TestUniqueSample = FuncStats.SampleSum; 1942 1943 updateHotBlockOverlap(0, FuncStats.SampleSum, FuncStats.HotBlockCount); 1944 1945 double FuncSimilarity = computeSampleFunctionOverlap( 1946 nullptr, nullptr, nullptr, 0, FuncStats.SampleSum); 1947 ProfOverlap.Similarity += 1948 weightByImportance(FuncSimilarity, 0, FuncStats.SampleSum); 1949 1950 ++ProfOverlap.UnionCount; 1951 ProfOverlap.UnionSample += FuncStats.SampleSum; 1952 } else { 1953 ++ProfOverlap.OverlapCount; 1954 1955 // Two functions match with each other. Compute function-level overlap and 1956 // aggregate them into profile-level overlap. 1957 FuncOverlap.BaseName = Match->second->getContext(); 1958 assert(BaseStats.count(FuncOverlap.BaseName) && 1959 "BaseStats should have records for all functions in base profile " 1960 "except inlinees"); 1961 FuncOverlap.BaseSample = BaseStats[FuncOverlap.BaseName].SampleSum; 1962 1963 FuncOverlap.Similarity = computeSampleFunctionOverlap( 1964 Match->second, &TestFunc.second, &FuncOverlap, FuncOverlap.BaseSample, 1965 FuncOverlap.TestSample); 1966 ProfOverlap.Similarity += 1967 weightByImportance(FuncOverlap.Similarity, FuncOverlap.BaseSample, 1968 FuncOverlap.TestSample); 1969 ProfOverlap.OverlapSample += FuncOverlap.OverlapSample; 1970 ProfOverlap.UnionSample += FuncOverlap.UnionSample; 1971 1972 // Accumulate the percentage of base unique and test unique samples into 1973 // ProfOverlap. 1974 ProfOverlap.BaseUniqueSample += FuncOverlap.BaseUniqueSample; 1975 ProfOverlap.TestUniqueSample += FuncOverlap.TestUniqueSample; 1976 1977 // Remove matched base functions for later reporting functions not found 1978 // in test profile. 1979 BaseFuncProf.erase(Match); 1980 Matched = true; 1981 } 1982 1983 // Print function-level similarity information if specified by options. 1984 assert(TestStats.count(FuncOverlap.TestName) && 1985 "TestStats should have records for all functions in test profile " 1986 "except inlinees"); 1987 if (TestStats[FuncOverlap.TestName].MaxSample >= FuncFilter.ValueCutoff || 1988 (Matched && FuncOverlap.Similarity < LowSimilarityThreshold) || 1989 (Matched && !FuncFilter.NameFilter.empty() && 1990 FuncOverlap.BaseName.toString().find(FuncFilter.NameFilter) != 1991 std::string::npos)) { 1992 assert(ProfOverlap.BaseSample > 0 && 1993 "Total samples in base profile should be greater than 0"); 1994 FuncOverlap.BaseWeight = 1995 static_cast<double>(FuncOverlap.BaseSample) / ProfOverlap.BaseSample; 1996 assert(ProfOverlap.TestSample > 0 && 1997 "Total samples in test profile should be greater than 0"); 1998 FuncOverlap.TestWeight = 1999 static_cast<double>(FuncOverlap.TestSample) / ProfOverlap.TestSample; 2000 FuncSimilarityDump.emplace(FuncOverlap.BaseWeight, FuncOverlap); 2001 } 2002 } 2003 2004 // Traverse through functions in base profile but not in test profile. 2005 for (const auto &F : BaseFuncProf) { 2006 assert(BaseStats.count(F.second->getContext()) && 2007 "BaseStats should have records for all functions in base profile " 2008 "except inlinees"); 2009 const FuncSampleStats &FuncStats = BaseStats[F.second->getContext()]; 2010 ++ProfOverlap.BaseUniqueCount; 2011 ProfOverlap.BaseUniqueSample += FuncStats.SampleSum; 2012 2013 updateHotBlockOverlap(FuncStats.SampleSum, 0, FuncStats.HotBlockCount); 2014 2015 double FuncSimilarity = computeSampleFunctionOverlap( 2016 nullptr, nullptr, nullptr, FuncStats.SampleSum, 0); 2017 ProfOverlap.Similarity += 2018 weightByImportance(FuncSimilarity, FuncStats.SampleSum, 0); 2019 2020 ProfOverlap.UnionSample += FuncStats.SampleSum; 2021 } 2022 2023 // Now, ProfSimilarity may be a little greater than 1 due to imprecision 2024 // of floating point accumulations. Make it 1.0 if the difference is below 2025 // Epsilon. 2026 ProfOverlap.Similarity = (std::fabs(ProfOverlap.Similarity - 1) < Epsilon) 2027 ? 1 2028 : ProfOverlap.Similarity; 2029 2030 computeHotFuncOverlap(); 2031 } 2032 2033 void SampleOverlapAggregator::initializeSampleProfileOverlap() { 2034 const auto &BaseProf = BaseReader->getProfiles(); 2035 for (const auto &I : BaseProf) { 2036 ++ProfOverlap.BaseCount; 2037 FuncSampleStats FuncStats; 2038 getFuncSampleStats(I.second, FuncStats, BaseHotThreshold); 2039 ProfOverlap.BaseSample += FuncStats.SampleSum; 2040 BaseStats.emplace(I.second.getContext(), FuncStats); 2041 } 2042 2043 const auto &TestProf = TestReader->getProfiles(); 2044 for (const auto &I : TestProf) { 2045 ++ProfOverlap.TestCount; 2046 FuncSampleStats FuncStats; 2047 getFuncSampleStats(I.second, FuncStats, TestHotThreshold); 2048 ProfOverlap.TestSample += FuncStats.SampleSum; 2049 TestStats.emplace(I.second.getContext(), FuncStats); 2050 } 2051 2052 ProfOverlap.BaseName = StringRef(BaseFilename); 2053 ProfOverlap.TestName = StringRef(TestFilename); 2054 } 2055 2056 void SampleOverlapAggregator::dumpFuncSimilarity(raw_fd_ostream &OS) const { 2057 using namespace sampleprof; 2058 2059 if (FuncSimilarityDump.empty()) 2060 return; 2061 2062 formatted_raw_ostream FOS(OS); 2063 FOS << "Function-level details:\n"; 2064 FOS << "Base weight"; 2065 FOS.PadToColumn(TestWeightCol); 2066 FOS << "Test weight"; 2067 FOS.PadToColumn(SimilarityCol); 2068 FOS << "Similarity"; 2069 FOS.PadToColumn(OverlapCol); 2070 FOS << "Overlap"; 2071 FOS.PadToColumn(BaseUniqueCol); 2072 FOS << "Base unique"; 2073 FOS.PadToColumn(TestUniqueCol); 2074 FOS << "Test unique"; 2075 FOS.PadToColumn(BaseSampleCol); 2076 FOS << "Base samples"; 2077 FOS.PadToColumn(TestSampleCol); 2078 FOS << "Test samples"; 2079 FOS.PadToColumn(FuncNameCol); 2080 FOS << "Function name\n"; 2081 for (const auto &F : FuncSimilarityDump) { 2082 double OverlapPercent = 2083 F.second.UnionSample > 0 2084 ? static_cast<double>(F.second.OverlapSample) / F.second.UnionSample 2085 : 0; 2086 double BaseUniquePercent = 2087 F.second.BaseSample > 0 2088 ? static_cast<double>(F.second.BaseUniqueSample) / 2089 F.second.BaseSample 2090 : 0; 2091 double TestUniquePercent = 2092 F.second.TestSample > 0 2093 ? static_cast<double>(F.second.TestUniqueSample) / 2094 F.second.TestSample 2095 : 0; 2096 2097 FOS << format("%.2f%%", F.second.BaseWeight * 100); 2098 FOS.PadToColumn(TestWeightCol); 2099 FOS << format("%.2f%%", F.second.TestWeight * 100); 2100 FOS.PadToColumn(SimilarityCol); 2101 FOS << format("%.2f%%", F.second.Similarity * 100); 2102 FOS.PadToColumn(OverlapCol); 2103 FOS << format("%.2f%%", OverlapPercent * 100); 2104 FOS.PadToColumn(BaseUniqueCol); 2105 FOS << format("%.2f%%", BaseUniquePercent * 100); 2106 FOS.PadToColumn(TestUniqueCol); 2107 FOS << format("%.2f%%", TestUniquePercent * 100); 2108 FOS.PadToColumn(BaseSampleCol); 2109 FOS << F.second.BaseSample; 2110 FOS.PadToColumn(TestSampleCol); 2111 FOS << F.second.TestSample; 2112 FOS.PadToColumn(FuncNameCol); 2113 FOS << F.second.TestName.toString() << "\n"; 2114 } 2115 } 2116 2117 void SampleOverlapAggregator::dumpProgramSummary(raw_fd_ostream &OS) const { 2118 OS << "Profile overlap infomation for base_profile: " 2119 << ProfOverlap.BaseName.toString() 2120 << " and test_profile: " << ProfOverlap.TestName.toString() 2121 << "\nProgram level:\n"; 2122 2123 OS << " Whole program profile similarity: " 2124 << format("%.3f%%", ProfOverlap.Similarity * 100) << "\n"; 2125 2126 assert(ProfOverlap.UnionSample > 0 && 2127 "Total samples in two profile should be greater than 0"); 2128 double OverlapPercent = 2129 static_cast<double>(ProfOverlap.OverlapSample) / ProfOverlap.UnionSample; 2130 assert(ProfOverlap.BaseSample > 0 && 2131 "Total samples in base profile should be greater than 0"); 2132 double BaseUniquePercent = static_cast<double>(ProfOverlap.BaseUniqueSample) / 2133 ProfOverlap.BaseSample; 2134 assert(ProfOverlap.TestSample > 0 && 2135 "Total samples in test profile should be greater than 0"); 2136 double TestUniquePercent = static_cast<double>(ProfOverlap.TestUniqueSample) / 2137 ProfOverlap.TestSample; 2138 2139 OS << " Whole program sample overlap: " 2140 << format("%.3f%%", OverlapPercent * 100) << "\n"; 2141 OS << " percentage of samples unique in base profile: " 2142 << format("%.3f%%", BaseUniquePercent * 100) << "\n"; 2143 OS << " percentage of samples unique in test profile: " 2144 << format("%.3f%%", TestUniquePercent * 100) << "\n"; 2145 OS << " total samples in base profile: " << ProfOverlap.BaseSample << "\n" 2146 << " total samples in test profile: " << ProfOverlap.TestSample << "\n"; 2147 2148 assert(ProfOverlap.UnionCount > 0 && 2149 "There should be at least one function in two input profiles"); 2150 double FuncOverlapPercent = 2151 static_cast<double>(ProfOverlap.OverlapCount) / ProfOverlap.UnionCount; 2152 OS << " Function overlap: " << format("%.3f%%", FuncOverlapPercent * 100) 2153 << "\n"; 2154 OS << " overlap functions: " << ProfOverlap.OverlapCount << "\n"; 2155 OS << " functions unique in base profile: " << ProfOverlap.BaseUniqueCount 2156 << "\n"; 2157 OS << " functions unique in test profile: " << ProfOverlap.TestUniqueCount 2158 << "\n"; 2159 } 2160 2161 void SampleOverlapAggregator::dumpHotFuncAndBlockOverlap( 2162 raw_fd_ostream &OS) const { 2163 assert(HotFuncOverlap.UnionCount > 0 && 2164 "There should be at least one hot function in two input profiles"); 2165 OS << " Hot-function overlap: " 2166 << format("%.3f%%", static_cast<double>(HotFuncOverlap.OverlapCount) / 2167 HotFuncOverlap.UnionCount * 100) 2168 << "\n"; 2169 OS << " overlap hot functions: " << HotFuncOverlap.OverlapCount << "\n"; 2170 OS << " hot functions unique in base profile: " 2171 << HotFuncOverlap.BaseCount - HotFuncOverlap.OverlapCount << "\n"; 2172 OS << " hot functions unique in test profile: " 2173 << HotFuncOverlap.TestCount - HotFuncOverlap.OverlapCount << "\n"; 2174 2175 assert(HotBlockOverlap.UnionCount > 0 && 2176 "There should be at least one hot block in two input profiles"); 2177 OS << " Hot-block overlap: " 2178 << format("%.3f%%", static_cast<double>(HotBlockOverlap.OverlapCount) / 2179 HotBlockOverlap.UnionCount * 100) 2180 << "\n"; 2181 OS << " overlap hot blocks: " << HotBlockOverlap.OverlapCount << "\n"; 2182 OS << " hot blocks unique in base profile: " 2183 << HotBlockOverlap.BaseCount - HotBlockOverlap.OverlapCount << "\n"; 2184 OS << " hot blocks unique in test profile: " 2185 << HotBlockOverlap.TestCount - HotBlockOverlap.OverlapCount << "\n"; 2186 } 2187 2188 std::error_code SampleOverlapAggregator::loadProfiles() { 2189 using namespace sampleprof; 2190 2191 LLVMContext Context; 2192 auto BaseReaderOrErr = SampleProfileReader::create(BaseFilename, Context, 2193 FSDiscriminatorPassOption); 2194 if (std::error_code EC = BaseReaderOrErr.getError()) 2195 exitWithErrorCode(EC, BaseFilename); 2196 2197 auto TestReaderOrErr = SampleProfileReader::create(TestFilename, Context, 2198 FSDiscriminatorPassOption); 2199 if (std::error_code EC = TestReaderOrErr.getError()) 2200 exitWithErrorCode(EC, TestFilename); 2201 2202 BaseReader = std::move(BaseReaderOrErr.get()); 2203 TestReader = std::move(TestReaderOrErr.get()); 2204 2205 if (std::error_code EC = BaseReader->read()) 2206 exitWithErrorCode(EC, BaseFilename); 2207 if (std::error_code EC = TestReader->read()) 2208 exitWithErrorCode(EC, TestFilename); 2209 if (BaseReader->profileIsProbeBased() != TestReader->profileIsProbeBased()) 2210 exitWithError( 2211 "cannot compare probe-based profile with non-probe-based profile"); 2212 if (BaseReader->profileIsCS() != TestReader->profileIsCS()) 2213 exitWithError("cannot compare CS profile with non-CS profile"); 2214 2215 // Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in 2216 // profile summary. 2217 ProfileSummary &BasePS = BaseReader->getSummary(); 2218 ProfileSummary &TestPS = TestReader->getSummary(); 2219 BaseHotThreshold = 2220 ProfileSummaryBuilder::getHotCountThreshold(BasePS.getDetailedSummary()); 2221 TestHotThreshold = 2222 ProfileSummaryBuilder::getHotCountThreshold(TestPS.getDetailedSummary()); 2223 2224 return std::error_code(); 2225 } 2226 2227 void overlapSampleProfile(const std::string &BaseFilename, 2228 const std::string &TestFilename, 2229 const OverlapFuncFilters &FuncFilter, 2230 uint64_t SimilarityCutoff, raw_fd_ostream &OS) { 2231 using namespace sampleprof; 2232 2233 // We use 0.000005 to initialize OverlapAggr.Epsilon because the final metrics 2234 // report 2--3 places after decimal point in percentage numbers. 2235 SampleOverlapAggregator OverlapAggr( 2236 BaseFilename, TestFilename, 2237 static_cast<double>(SimilarityCutoff) / 1000000, 0.000005, FuncFilter); 2238 if (std::error_code EC = OverlapAggr.loadProfiles()) 2239 exitWithErrorCode(EC); 2240 2241 OverlapAggr.initializeSampleProfileOverlap(); 2242 if (OverlapAggr.detectZeroSampleProfile(OS)) 2243 return; 2244 2245 OverlapAggr.computeSampleProfileOverlap(OS); 2246 2247 OverlapAggr.dumpProgramSummary(OS); 2248 OverlapAggr.dumpHotFuncAndBlockOverlap(OS); 2249 OverlapAggr.dumpFuncSimilarity(OS); 2250 } 2251 2252 static int overlap_main(int argc, const char *argv[]) { 2253 cl::opt<std::string> BaseFilename(cl::Positional, cl::Required, 2254 cl::desc("<base profile file>")); 2255 cl::opt<std::string> TestFilename(cl::Positional, cl::Required, 2256 cl::desc("<test profile file>")); 2257 cl::opt<std::string> Output("output", cl::value_desc("output"), cl::init("-"), 2258 cl::desc("Output file")); 2259 cl::alias OutputA("o", cl::desc("Alias for --output"), cl::aliasopt(Output)); 2260 cl::opt<bool> IsCS( 2261 "cs", cl::init(false), 2262 cl::desc("For context sensitive PGO counts. Does not work with CSSPGO.")); 2263 cl::opt<unsigned long long> ValueCutoff( 2264 "value-cutoff", cl::init(-1), 2265 cl::desc( 2266 "Function level overlap information for every function (with calling " 2267 "context for csspgo) in test " 2268 "profile with max count value greater then the parameter value")); 2269 cl::opt<std::string> FuncNameFilter( 2270 "function", 2271 cl::desc("Function level overlap information for matching functions. For " 2272 "CSSPGO this takes a a function name with calling context")); 2273 cl::opt<unsigned long long> SimilarityCutoff( 2274 "similarity-cutoff", cl::init(0), 2275 cl::desc("For sample profiles, list function names (with calling context " 2276 "for csspgo) for overlapped functions " 2277 "with similarities below the cutoff (percentage times 10000).")); 2278 cl::opt<ProfileKinds> ProfileKind( 2279 cl::desc("Profile kind:"), cl::init(instr), 2280 cl::values(clEnumVal(instr, "Instrumentation profile (default)"), 2281 clEnumVal(sample, "Sample profile"))); 2282 cl::ParseCommandLineOptions(argc, argv, "LLVM profile data overlap tool\n"); 2283 2284 std::error_code EC; 2285 raw_fd_ostream OS(Output.data(), EC, sys::fs::OF_TextWithCRLF); 2286 if (EC) 2287 exitWithErrorCode(EC, Output); 2288 2289 if (ProfileKind == instr) 2290 overlapInstrProfile(BaseFilename, TestFilename, 2291 OverlapFuncFilters{ValueCutoff, FuncNameFilter}, OS, 2292 IsCS); 2293 else 2294 overlapSampleProfile(BaseFilename, TestFilename, 2295 OverlapFuncFilters{ValueCutoff, FuncNameFilter}, 2296 SimilarityCutoff, OS); 2297 2298 return 0; 2299 } 2300 2301 namespace { 2302 struct ValueSitesStats { 2303 ValueSitesStats() 2304 : TotalNumValueSites(0), TotalNumValueSitesWithValueProfile(0), 2305 TotalNumValues(0) {} 2306 uint64_t TotalNumValueSites; 2307 uint64_t TotalNumValueSitesWithValueProfile; 2308 uint64_t TotalNumValues; 2309 std::vector<unsigned> ValueSitesHistogram; 2310 }; 2311 } // namespace 2312 2313 static void traverseAllValueSites(const InstrProfRecord &Func, uint32_t VK, 2314 ValueSitesStats &Stats, raw_fd_ostream &OS, 2315 InstrProfSymtab *Symtab) { 2316 uint32_t NS = Func.getNumValueSites(VK); 2317 Stats.TotalNumValueSites += NS; 2318 for (size_t I = 0; I < NS; ++I) { 2319 uint32_t NV = Func.getNumValueDataForSite(VK, I); 2320 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, I); 2321 Stats.TotalNumValues += NV; 2322 if (NV) { 2323 Stats.TotalNumValueSitesWithValueProfile++; 2324 if (NV > Stats.ValueSitesHistogram.size()) 2325 Stats.ValueSitesHistogram.resize(NV, 0); 2326 Stats.ValueSitesHistogram[NV - 1]++; 2327 } 2328 2329 uint64_t SiteSum = 0; 2330 for (uint32_t V = 0; V < NV; V++) 2331 SiteSum += VD[V].Count; 2332 if (SiteSum == 0) 2333 SiteSum = 1; 2334 2335 for (uint32_t V = 0; V < NV; V++) { 2336 OS << "\t[ " << format("%2u", I) << ", "; 2337 if (Symtab == nullptr) 2338 OS << format("%4" PRIu64, VD[V].Value); 2339 else 2340 OS << Symtab->getFuncName(VD[V].Value); 2341 OS << ", " << format("%10" PRId64, VD[V].Count) << " ] (" 2342 << format("%.2f%%", (VD[V].Count * 100.0 / SiteSum)) << ")\n"; 2343 } 2344 } 2345 } 2346 2347 static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK, 2348 ValueSitesStats &Stats) { 2349 OS << " Total number of sites: " << Stats.TotalNumValueSites << "\n"; 2350 OS << " Total number of sites with values: " 2351 << Stats.TotalNumValueSitesWithValueProfile << "\n"; 2352 OS << " Total number of profiled values: " << Stats.TotalNumValues << "\n"; 2353 2354 OS << " Value sites histogram:\n\tNumTargets, SiteCount\n"; 2355 for (unsigned I = 0; I < Stats.ValueSitesHistogram.size(); I++) { 2356 if (Stats.ValueSitesHistogram[I] > 0) 2357 OS << "\t" << I + 1 << ", " << Stats.ValueSitesHistogram[I] << "\n"; 2358 } 2359 } 2360 2361 static int showInstrProfile(const std::string &Filename, bool ShowCounts, 2362 uint32_t TopN, bool ShowIndirectCallTargets, 2363 bool ShowMemOPSizes, bool ShowDetailedSummary, 2364 std::vector<uint32_t> DetailedSummaryCutoffs, 2365 bool ShowAllFunctions, bool ShowCS, 2366 uint64_t ValueCutoff, bool OnlyListBelow, 2367 const std::string &ShowFunction, bool TextFormat, 2368 bool ShowBinaryIds, bool ShowCovered, 2369 bool ShowProfileVersion, ShowFormat SFormat, 2370 raw_fd_ostream &OS) { 2371 if (SFormat == ShowFormat::Json) 2372 exitWithError("JSON output is not supported for instr profiles"); 2373 if (SFormat == ShowFormat::Yaml) 2374 exitWithError("YAML output is not supported for instr profiles"); 2375 auto ReaderOrErr = InstrProfReader::create(Filename); 2376 std::vector<uint32_t> Cutoffs = std::move(DetailedSummaryCutoffs); 2377 if (ShowDetailedSummary && Cutoffs.empty()) { 2378 Cutoffs = ProfileSummaryBuilder::DefaultCutoffs; 2379 } 2380 InstrProfSummaryBuilder Builder(std::move(Cutoffs)); 2381 if (Error E = ReaderOrErr.takeError()) 2382 exitWithError(std::move(E), Filename); 2383 2384 auto Reader = std::move(ReaderOrErr.get()); 2385 bool IsIRInstr = Reader->isIRLevelProfile(); 2386 size_t ShownFunctions = 0; 2387 size_t BelowCutoffFunctions = 0; 2388 int NumVPKind = IPVK_Last - IPVK_First + 1; 2389 std::vector<ValueSitesStats> VPStats(NumVPKind); 2390 2391 auto MinCmp = [](const std::pair<std::string, uint64_t> &v1, 2392 const std::pair<std::string, uint64_t> &v2) { 2393 return v1.second > v2.second; 2394 }; 2395 2396 std::priority_queue<std::pair<std::string, uint64_t>, 2397 std::vector<std::pair<std::string, uint64_t>>, 2398 decltype(MinCmp)> 2399 HottestFuncs(MinCmp); 2400 2401 if (!TextFormat && OnlyListBelow) { 2402 OS << "The list of functions with the maximum counter less than " 2403 << ValueCutoff << ":\n"; 2404 } 2405 2406 // Add marker so that IR-level instrumentation round-trips properly. 2407 if (TextFormat && IsIRInstr) 2408 OS << ":ir\n"; 2409 2410 for (const auto &Func : *Reader) { 2411 if (Reader->isIRLevelProfile()) { 2412 bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(Func.Hash); 2413 if (FuncIsCS != ShowCS) 2414 continue; 2415 } 2416 bool Show = ShowAllFunctions || 2417 (!ShowFunction.empty() && Func.Name.contains(ShowFunction)); 2418 2419 bool doTextFormatDump = (Show && TextFormat); 2420 2421 if (doTextFormatDump) { 2422 InstrProfSymtab &Symtab = Reader->getSymtab(); 2423 InstrProfWriter::writeRecordInText(Func.Name, Func.Hash, Func, Symtab, 2424 OS); 2425 continue; 2426 } 2427 2428 assert(Func.Counts.size() > 0 && "function missing entry counter"); 2429 Builder.addRecord(Func); 2430 2431 if (ShowCovered) { 2432 if (llvm::any_of(Func.Counts, [](uint64_t C) { return C; })) 2433 OS << Func.Name << "\n"; 2434 continue; 2435 } 2436 2437 uint64_t FuncMax = 0; 2438 uint64_t FuncSum = 0; 2439 2440 auto PseudoKind = Func.getCountPseudoKind(); 2441 if (PseudoKind != InstrProfRecord::NotPseudo) { 2442 if (Show) { 2443 if (!ShownFunctions) 2444 OS << "Counters:\n"; 2445 ++ShownFunctions; 2446 OS << " " << Func.Name << ":\n" 2447 << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n" 2448 << " Counters: " << Func.Counts.size(); 2449 if (PseudoKind == InstrProfRecord::PseudoHot) 2450 OS << " <PseudoHot>\n"; 2451 else if (PseudoKind == InstrProfRecord::PseudoWarm) 2452 OS << " <PseudoWarm>\n"; 2453 else 2454 llvm_unreachable("Unknown PseudoKind"); 2455 } 2456 continue; 2457 } 2458 2459 for (size_t I = 0, E = Func.Counts.size(); I < E; ++I) { 2460 FuncMax = std::max(FuncMax, Func.Counts[I]); 2461 FuncSum += Func.Counts[I]; 2462 } 2463 2464 if (FuncMax < ValueCutoff) { 2465 ++BelowCutoffFunctions; 2466 if (OnlyListBelow) { 2467 OS << " " << Func.Name << ": (Max = " << FuncMax 2468 << " Sum = " << FuncSum << ")\n"; 2469 } 2470 continue; 2471 } else if (OnlyListBelow) 2472 continue; 2473 2474 if (TopN) { 2475 if (HottestFuncs.size() == TopN) { 2476 if (HottestFuncs.top().second < FuncMax) { 2477 HottestFuncs.pop(); 2478 HottestFuncs.emplace(std::make_pair(std::string(Func.Name), FuncMax)); 2479 } 2480 } else 2481 HottestFuncs.emplace(std::make_pair(std::string(Func.Name), FuncMax)); 2482 } 2483 2484 if (Show) { 2485 if (!ShownFunctions) 2486 OS << "Counters:\n"; 2487 2488 ++ShownFunctions; 2489 2490 OS << " " << Func.Name << ":\n" 2491 << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n" 2492 << " Counters: " << Func.Counts.size() << "\n"; 2493 if (!IsIRInstr) 2494 OS << " Function count: " << Func.Counts[0] << "\n"; 2495 2496 if (ShowIndirectCallTargets) 2497 OS << " Indirect Call Site Count: " 2498 << Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n"; 2499 2500 uint32_t NumMemOPCalls = Func.getNumValueSites(IPVK_MemOPSize); 2501 if (ShowMemOPSizes && NumMemOPCalls > 0) 2502 OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls 2503 << "\n"; 2504 2505 if (ShowCounts) { 2506 OS << " Block counts: ["; 2507 size_t Start = (IsIRInstr ? 0 : 1); 2508 for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) { 2509 OS << (I == Start ? "" : ", ") << Func.Counts[I]; 2510 } 2511 OS << "]\n"; 2512 } 2513 2514 if (ShowIndirectCallTargets) { 2515 OS << " Indirect Target Results:\n"; 2516 traverseAllValueSites(Func, IPVK_IndirectCallTarget, 2517 VPStats[IPVK_IndirectCallTarget], OS, 2518 &(Reader->getSymtab())); 2519 } 2520 2521 if (ShowMemOPSizes && NumMemOPCalls > 0) { 2522 OS << " Memory Intrinsic Size Results:\n"; 2523 traverseAllValueSites(Func, IPVK_MemOPSize, VPStats[IPVK_MemOPSize], OS, 2524 nullptr); 2525 } 2526 } 2527 } 2528 if (Reader->hasError()) 2529 exitWithError(Reader->getError(), Filename); 2530 2531 if (TextFormat || ShowCovered) 2532 return 0; 2533 std::unique_ptr<ProfileSummary> PS(Builder.getSummary()); 2534 bool IsIR = Reader->isIRLevelProfile(); 2535 OS << "Instrumentation level: " << (IsIR ? "IR" : "Front-end"); 2536 if (IsIR) 2537 OS << " entry_first = " << Reader->instrEntryBBEnabled(); 2538 OS << "\n"; 2539 if (ShowAllFunctions || !ShowFunction.empty()) 2540 OS << "Functions shown: " << ShownFunctions << "\n"; 2541 OS << "Total functions: " << PS->getNumFunctions() << "\n"; 2542 if (ValueCutoff > 0) { 2543 OS << "Number of functions with maximum count (< " << ValueCutoff 2544 << "): " << BelowCutoffFunctions << "\n"; 2545 OS << "Number of functions with maximum count (>= " << ValueCutoff 2546 << "): " << PS->getNumFunctions() - BelowCutoffFunctions << "\n"; 2547 } 2548 OS << "Maximum function count: " << PS->getMaxFunctionCount() << "\n"; 2549 OS << "Maximum internal block count: " << PS->getMaxInternalCount() << "\n"; 2550 2551 if (TopN) { 2552 std::vector<std::pair<std::string, uint64_t>> SortedHottestFuncs; 2553 while (!HottestFuncs.empty()) { 2554 SortedHottestFuncs.emplace_back(HottestFuncs.top()); 2555 HottestFuncs.pop(); 2556 } 2557 OS << "Top " << TopN 2558 << " functions with the largest internal block counts: \n"; 2559 for (auto &hotfunc : llvm::reverse(SortedHottestFuncs)) 2560 OS << " " << hotfunc.first << ", max count = " << hotfunc.second << "\n"; 2561 } 2562 2563 if (ShownFunctions && ShowIndirectCallTargets) { 2564 OS << "Statistics for indirect call sites profile:\n"; 2565 showValueSitesStats(OS, IPVK_IndirectCallTarget, 2566 VPStats[IPVK_IndirectCallTarget]); 2567 } 2568 2569 if (ShownFunctions && ShowMemOPSizes) { 2570 OS << "Statistics for memory intrinsic calls sizes profile:\n"; 2571 showValueSitesStats(OS, IPVK_MemOPSize, VPStats[IPVK_MemOPSize]); 2572 } 2573 2574 if (ShowDetailedSummary) { 2575 OS << "Total number of blocks: " << PS->getNumCounts() << "\n"; 2576 OS << "Total count: " << PS->getTotalCount() << "\n"; 2577 PS->printDetailedSummary(OS); 2578 } 2579 2580 if (ShowBinaryIds) 2581 if (Error E = Reader->printBinaryIds(OS)) 2582 exitWithError(std::move(E), Filename); 2583 2584 if (ShowProfileVersion) 2585 OS << "Profile version: " << Reader->getVersion() << "\n"; 2586 return 0; 2587 } 2588 2589 static void showSectionInfo(sampleprof::SampleProfileReader *Reader, 2590 raw_fd_ostream &OS) { 2591 if (!Reader->dumpSectionInfo(OS)) { 2592 WithColor::warning() << "-show-sec-info-only is only supported for " 2593 << "sample profile in extbinary format and is " 2594 << "ignored for other formats.\n"; 2595 return; 2596 } 2597 } 2598 2599 namespace { 2600 struct HotFuncInfo { 2601 std::string FuncName; 2602 uint64_t TotalCount; 2603 double TotalCountPercent; 2604 uint64_t MaxCount; 2605 uint64_t EntryCount; 2606 2607 HotFuncInfo() 2608 : TotalCount(0), TotalCountPercent(0.0f), MaxCount(0), EntryCount(0) {} 2609 2610 HotFuncInfo(StringRef FN, uint64_t TS, double TSP, uint64_t MS, uint64_t ES) 2611 : FuncName(FN.begin(), FN.end()), TotalCount(TS), TotalCountPercent(TSP), 2612 MaxCount(MS), EntryCount(ES) {} 2613 }; 2614 } // namespace 2615 2616 // Print out detailed information about hot functions in PrintValues vector. 2617 // Users specify titles and offset of every columns through ColumnTitle and 2618 // ColumnOffset. The size of ColumnTitle and ColumnOffset need to be the same 2619 // and at least 4. Besides, users can optionally give a HotFuncMetric string to 2620 // print out or let it be an empty string. 2621 static void dumpHotFunctionList(const std::vector<std::string> &ColumnTitle, 2622 const std::vector<int> &ColumnOffset, 2623 const std::vector<HotFuncInfo> &PrintValues, 2624 uint64_t HotFuncCount, uint64_t TotalFuncCount, 2625 uint64_t HotProfCount, uint64_t TotalProfCount, 2626 const std::string &HotFuncMetric, 2627 uint32_t TopNFunctions, raw_fd_ostream &OS) { 2628 assert(ColumnOffset.size() == ColumnTitle.size() && 2629 "ColumnOffset and ColumnTitle should have the same size"); 2630 assert(ColumnTitle.size() >= 4 && 2631 "ColumnTitle should have at least 4 elements"); 2632 assert(TotalFuncCount > 0 && 2633 "There should be at least one function in the profile"); 2634 double TotalProfPercent = 0; 2635 if (TotalProfCount > 0) 2636 TotalProfPercent = static_cast<double>(HotProfCount) / TotalProfCount * 100; 2637 2638 formatted_raw_ostream FOS(OS); 2639 FOS << HotFuncCount << " out of " << TotalFuncCount 2640 << " functions with profile (" 2641 << format("%.2f%%", 2642 (static_cast<double>(HotFuncCount) / TotalFuncCount * 100)) 2643 << ") are considered hot functions"; 2644 if (!HotFuncMetric.empty()) 2645 FOS << " (" << HotFuncMetric << ")"; 2646 FOS << ".\n"; 2647 FOS << HotProfCount << " out of " << TotalProfCount << " profile counts (" 2648 << format("%.2f%%", TotalProfPercent) << ") are from hot functions.\n"; 2649 2650 for (size_t I = 0; I < ColumnTitle.size(); ++I) { 2651 FOS.PadToColumn(ColumnOffset[I]); 2652 FOS << ColumnTitle[I]; 2653 } 2654 FOS << "\n"; 2655 2656 uint32_t Count = 0; 2657 for (const auto &R : PrintValues) { 2658 if (TopNFunctions && (Count++ == TopNFunctions)) 2659 break; 2660 FOS.PadToColumn(ColumnOffset[0]); 2661 FOS << R.TotalCount << " (" << format("%.2f%%", R.TotalCountPercent) << ")"; 2662 FOS.PadToColumn(ColumnOffset[1]); 2663 FOS << R.MaxCount; 2664 FOS.PadToColumn(ColumnOffset[2]); 2665 FOS << R.EntryCount; 2666 FOS.PadToColumn(ColumnOffset[3]); 2667 FOS << R.FuncName << "\n"; 2668 } 2669 } 2670 2671 static int showHotFunctionList(const sampleprof::SampleProfileMap &Profiles, 2672 ProfileSummary &PS, uint32_t TopN, 2673 raw_fd_ostream &OS) { 2674 using namespace sampleprof; 2675 2676 const uint32_t HotFuncCutoff = 990000; 2677 auto &SummaryVector = PS.getDetailedSummary(); 2678 uint64_t MinCountThreshold = 0; 2679 for (const ProfileSummaryEntry &SummaryEntry : SummaryVector) { 2680 if (SummaryEntry.Cutoff == HotFuncCutoff) { 2681 MinCountThreshold = SummaryEntry.MinCount; 2682 break; 2683 } 2684 } 2685 2686 // Traverse all functions in the profile and keep only hot functions. 2687 // The following loop also calculates the sum of total samples of all 2688 // functions. 2689 std::multimap<uint64_t, std::pair<const FunctionSamples *, const uint64_t>, 2690 std::greater<uint64_t>> 2691 HotFunc; 2692 uint64_t ProfileTotalSample = 0; 2693 uint64_t HotFuncSample = 0; 2694 uint64_t HotFuncCount = 0; 2695 2696 for (const auto &I : Profiles) { 2697 FuncSampleStats FuncStats; 2698 const FunctionSamples &FuncProf = I.second; 2699 ProfileTotalSample += FuncProf.getTotalSamples(); 2700 getFuncSampleStats(FuncProf, FuncStats, MinCountThreshold); 2701 2702 if (isFunctionHot(FuncStats, MinCountThreshold)) { 2703 HotFunc.emplace(FuncProf.getTotalSamples(), 2704 std::make_pair(&(I.second), FuncStats.MaxSample)); 2705 HotFuncSample += FuncProf.getTotalSamples(); 2706 ++HotFuncCount; 2707 } 2708 } 2709 2710 std::vector<std::string> ColumnTitle{"Total sample (%)", "Max sample", 2711 "Entry sample", "Function name"}; 2712 std::vector<int> ColumnOffset{0, 24, 42, 58}; 2713 std::string Metric = 2714 std::string("max sample >= ") + std::to_string(MinCountThreshold); 2715 std::vector<HotFuncInfo> PrintValues; 2716 for (const auto &FuncPair : HotFunc) { 2717 const FunctionSamples &Func = *FuncPair.second.first; 2718 double TotalSamplePercent = 2719 (ProfileTotalSample > 0) 2720 ? (Func.getTotalSamples() * 100.0) / ProfileTotalSample 2721 : 0; 2722 PrintValues.emplace_back( 2723 HotFuncInfo(Func.getContext().toString(), Func.getTotalSamples(), 2724 TotalSamplePercent, FuncPair.second.second, 2725 Func.getHeadSamplesEstimate())); 2726 } 2727 dumpHotFunctionList(ColumnTitle, ColumnOffset, PrintValues, HotFuncCount, 2728 Profiles.size(), HotFuncSample, ProfileTotalSample, 2729 Metric, TopN, OS); 2730 2731 return 0; 2732 } 2733 2734 static int showSampleProfile(const std::string &Filename, bool ShowCounts, 2735 uint32_t TopN, bool ShowAllFunctions, 2736 bool ShowDetailedSummary, 2737 const std::string &ShowFunction, 2738 bool ShowProfileSymbolList, 2739 bool ShowSectionInfoOnly, bool ShowHotFuncList, 2740 ShowFormat SFormat, raw_fd_ostream &OS) { 2741 if (SFormat == ShowFormat::Yaml) 2742 exitWithError("YAML output is not supported for sample profiles"); 2743 using namespace sampleprof; 2744 LLVMContext Context; 2745 auto ReaderOrErr = 2746 SampleProfileReader::create(Filename, Context, FSDiscriminatorPassOption); 2747 if (std::error_code EC = ReaderOrErr.getError()) 2748 exitWithErrorCode(EC, Filename); 2749 2750 auto Reader = std::move(ReaderOrErr.get()); 2751 if (ShowSectionInfoOnly) { 2752 showSectionInfo(Reader.get(), OS); 2753 return 0; 2754 } 2755 2756 if (std::error_code EC = Reader->read()) 2757 exitWithErrorCode(EC, Filename); 2758 2759 if (ShowAllFunctions || ShowFunction.empty()) { 2760 if (SFormat == ShowFormat::Json) 2761 Reader->dumpJson(OS); 2762 else 2763 Reader->dump(OS); 2764 } else { 2765 if (SFormat == ShowFormat::Json) 2766 exitWithError( 2767 "the JSON format is supported only when all functions are to " 2768 "be printed"); 2769 2770 // TODO: parse context string to support filtering by contexts. 2771 Reader->dumpFunctionProfile(StringRef(ShowFunction), OS); 2772 } 2773 2774 if (ShowProfileSymbolList) { 2775 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList = 2776 Reader->getProfileSymbolList(); 2777 ReaderList->dump(OS); 2778 } 2779 2780 if (ShowDetailedSummary) { 2781 auto &PS = Reader->getSummary(); 2782 PS.printSummary(OS); 2783 PS.printDetailedSummary(OS); 2784 } 2785 2786 if (ShowHotFuncList || TopN) 2787 showHotFunctionList(Reader->getProfiles(), Reader->getSummary(), TopN, OS); 2788 2789 return 0; 2790 } 2791 2792 static int showMemProfProfile(const std::string &Filename, 2793 const std::string &ProfiledBinary, 2794 ShowFormat SFormat, raw_fd_ostream &OS) { 2795 if (SFormat == ShowFormat::Json) 2796 exitWithError("JSON output is not supported for MemProf"); 2797 auto ReaderOr = llvm::memprof::RawMemProfReader::create( 2798 Filename, ProfiledBinary, /*KeepNames=*/true); 2799 if (Error E = ReaderOr.takeError()) 2800 // Since the error can be related to the profile or the binary we do not 2801 // pass whence. Instead additional context is provided where necessary in 2802 // the error message. 2803 exitWithError(std::move(E), /*Whence*/ ""); 2804 2805 std::unique_ptr<llvm::memprof::RawMemProfReader> Reader( 2806 ReaderOr.get().release()); 2807 2808 Reader->printYAML(OS); 2809 return 0; 2810 } 2811 2812 static int showDebugInfoCorrelation(const std::string &Filename, 2813 bool ShowDetailedSummary, 2814 bool ShowProfileSymbolList, 2815 ShowFormat SFormat, raw_fd_ostream &OS) { 2816 if (SFormat == ShowFormat::Json) 2817 exitWithError("JSON output is not supported for debug info correlation"); 2818 std::unique_ptr<InstrProfCorrelator> Correlator; 2819 if (auto Err = InstrProfCorrelator::get(Filename).moveInto(Correlator)) 2820 exitWithError(std::move(Err), Filename); 2821 if (SFormat == ShowFormat::Yaml) { 2822 if (auto Err = Correlator->dumpYaml(OS)) 2823 exitWithError(std::move(Err), Filename); 2824 return 0; 2825 } 2826 2827 if (auto Err = Correlator->correlateProfileData()) 2828 exitWithError(std::move(Err), Filename); 2829 2830 InstrProfSymtab Symtab; 2831 if (auto Err = Symtab.create( 2832 StringRef(Correlator->getNamesPointer(), Correlator->getNamesSize()))) 2833 exitWithError(std::move(Err), Filename); 2834 2835 if (ShowProfileSymbolList) 2836 Symtab.dumpNames(OS); 2837 // TODO: Read "Profile Data Type" from debug info to compute and show how many 2838 // counters the section holds. 2839 if (ShowDetailedSummary) 2840 OS << "Counters section size: 0x" 2841 << Twine::utohexstr(Correlator->getCountersSectionSize()) << " bytes\n"; 2842 OS << "Found " << Correlator->getDataSize() << " functions\n"; 2843 2844 return 0; 2845 } 2846 2847 static int show_main(int argc, const char *argv[]) { 2848 cl::opt<std::string> Filename(cl::Positional, cl::desc("<profdata-file>")); 2849 2850 cl::opt<bool> ShowCounts("counts", cl::init(false), 2851 cl::desc("Show counter values for shown functions")); 2852 cl::opt<ShowFormat> SFormat( 2853 "show-format", cl::init(ShowFormat::Text), 2854 cl::desc("Emit output in the selected format if supported"), 2855 cl::values(clEnumValN(ShowFormat::Text, "text", 2856 "emit normal text output (default)"), 2857 clEnumValN(ShowFormat::Json, "json", "emit JSON"), 2858 clEnumValN(ShowFormat::Yaml, "yaml", "emit YAML"))); 2859 // TODO: Consider replacing this with `--show-format=text-encoding`. 2860 cl::opt<bool> TextFormat( 2861 "text", cl::init(false), 2862 cl::desc("Show instr profile data in text dump format")); 2863 cl::opt<bool> JsonFormat( 2864 "json", cl::desc("Show sample profile data in the JSON format " 2865 "(deprecated, please use --show-format=json)")); 2866 cl::opt<bool> ShowIndirectCallTargets( 2867 "ic-targets", cl::init(false), 2868 cl::desc("Show indirect call site target values for shown functions")); 2869 cl::opt<bool> ShowMemOPSizes( 2870 "memop-sizes", cl::init(false), 2871 cl::desc("Show the profiled sizes of the memory intrinsic calls " 2872 "for shown functions")); 2873 cl::opt<bool> ShowDetailedSummary("detailed-summary", cl::init(false), 2874 cl::desc("Show detailed profile summary")); 2875 cl::list<uint32_t> DetailedSummaryCutoffs( 2876 cl::CommaSeparated, "detailed-summary-cutoffs", 2877 cl::desc( 2878 "Cutoff percentages (times 10000) for generating detailed summary"), 2879 cl::value_desc("800000,901000,999999")); 2880 cl::opt<bool> ShowHotFuncList( 2881 "hot-func-list", cl::init(false), 2882 cl::desc("Show profile summary of a list of hot functions")); 2883 cl::opt<bool> ShowAllFunctions("all-functions", cl::init(false), 2884 cl::desc("Details for every function")); 2885 cl::opt<bool> ShowCS("showcs", cl::init(false), 2886 cl::desc("Show context sensitive counts")); 2887 cl::opt<std::string> ShowFunction("function", 2888 cl::desc("Details for matching functions")); 2889 2890 cl::opt<std::string> OutputFilename("output", cl::value_desc("output"), 2891 cl::init("-"), cl::desc("Output file")); 2892 cl::alias OutputFilenameA("o", cl::desc("Alias for --output"), 2893 cl::aliasopt(OutputFilename)); 2894 cl::opt<ProfileKinds> ProfileKind( 2895 cl::desc("Profile kind:"), cl::init(instr), 2896 cl::values(clEnumVal(instr, "Instrumentation profile (default)"), 2897 clEnumVal(sample, "Sample profile"), 2898 clEnumVal(memory, "MemProf memory access profile"))); 2899 cl::opt<uint32_t> TopNFunctions( 2900 "topn", cl::init(0), 2901 cl::desc("Show the list of functions with the largest internal counts")); 2902 cl::opt<uint32_t> ValueCutoff( 2903 "value-cutoff", cl::init(0), 2904 cl::desc("Set the count value cutoff. Functions with the maximum count " 2905 "less than this value will not be printed out. (Default is 0)")); 2906 cl::opt<bool> OnlyListBelow( 2907 "list-below-cutoff", cl::init(false), 2908 cl::desc("Only output names of functions whose max count values are " 2909 "below the cutoff value")); 2910 cl::opt<bool> ShowProfileSymbolList( 2911 "show-prof-sym-list", cl::init(false), 2912 cl::desc("Show profile symbol list if it exists in the profile. ")); 2913 cl::opt<bool> ShowSectionInfoOnly( 2914 "show-sec-info-only", cl::init(false), 2915 cl::desc("Show the information of each section in the sample profile. " 2916 "The flag is only usable when the sample profile is in " 2917 "extbinary format")); 2918 cl::opt<bool> ShowBinaryIds("binary-ids", cl::init(false), 2919 cl::desc("Show binary ids in the profile. ")); 2920 cl::opt<std::string> DebugInfoFilename( 2921 "debug-info", cl::init(""), 2922 cl::desc("Read and extract profile metadata from debug info and show " 2923 "the functions it found.")); 2924 cl::opt<bool> ShowCovered( 2925 "covered", cl::init(false), 2926 cl::desc("Show only the functions that have been executed.")); 2927 cl::opt<std::string> ProfiledBinary( 2928 "profiled-binary", cl::init(""), 2929 cl::desc("Path to binary from which the profile was collected.")); 2930 cl::opt<bool> ShowProfileVersion("profile-version", cl::init(false), 2931 cl::desc("Show profile version. ")); 2932 cl::ParseCommandLineOptions(argc, argv, "LLVM profile data summary\n"); 2933 2934 if (Filename.empty() && DebugInfoFilename.empty()) 2935 exitWithError( 2936 "the positional argument '<profdata-file>' is required unless '--" + 2937 DebugInfoFilename.ArgStr + "' is provided"); 2938 2939 if (Filename == OutputFilename) { 2940 errs() << sys::path::filename(argv[0]) 2941 << ": Input file name cannot be the same as the output file name!\n"; 2942 return 1; 2943 } 2944 if (JsonFormat) 2945 SFormat = ShowFormat::Json; 2946 2947 std::error_code EC; 2948 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF); 2949 if (EC) 2950 exitWithErrorCode(EC, OutputFilename); 2951 2952 if (ShowAllFunctions && !ShowFunction.empty()) 2953 WithColor::warning() << "-function argument ignored: showing all functions\n"; 2954 2955 if (!DebugInfoFilename.empty()) 2956 return showDebugInfoCorrelation(DebugInfoFilename, ShowDetailedSummary, 2957 ShowProfileSymbolList, SFormat, OS); 2958 2959 if (ProfileKind == instr) 2960 return showInstrProfile( 2961 Filename, ShowCounts, TopNFunctions, ShowIndirectCallTargets, 2962 ShowMemOPSizes, ShowDetailedSummary, DetailedSummaryCutoffs, 2963 ShowAllFunctions, ShowCS, ValueCutoff, OnlyListBelow, ShowFunction, 2964 TextFormat, ShowBinaryIds, ShowCovered, ShowProfileVersion, SFormat, 2965 OS); 2966 if (ProfileKind == sample) 2967 return showSampleProfile(Filename, ShowCounts, TopNFunctions, 2968 ShowAllFunctions, ShowDetailedSummary, 2969 ShowFunction, ShowProfileSymbolList, 2970 ShowSectionInfoOnly, ShowHotFuncList, SFormat, OS); 2971 return showMemProfProfile(Filename, ProfiledBinary, SFormat, OS); 2972 } 2973 2974 int llvm_profdata_main(int argc, char **argvNonConst) { 2975 const char **argv = const_cast<const char **>(argvNonConst); 2976 InitLLVM X(argc, argv); 2977 2978 StringRef ProgName(sys::path::filename(argv[0])); 2979 if (argc > 1) { 2980 int (*func)(int, const char *[]) = nullptr; 2981 2982 if (strcmp(argv[1], "merge") == 0) 2983 func = merge_main; 2984 else if (strcmp(argv[1], "show") == 0) 2985 func = show_main; 2986 else if (strcmp(argv[1], "overlap") == 0) 2987 func = overlap_main; 2988 2989 if (func) { 2990 std::string Invocation(ProgName.str() + " " + argv[1]); 2991 argv[1] = Invocation.c_str(); 2992 return func(argc - 1, argv + 1); 2993 } 2994 2995 if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "-help") == 0 || 2996 strcmp(argv[1], "--help") == 0) { 2997 2998 errs() << "OVERVIEW: LLVM profile data tools\n\n" 2999 << "USAGE: " << ProgName << " <command> [args...]\n" 3000 << "USAGE: " << ProgName << " <command> -help\n\n" 3001 << "See each individual command --help for more details.\n" 3002 << "Available commands: merge, show, overlap\n"; 3003 return 0; 3004 } 3005 } 3006 3007 if (argc < 2) 3008 errs() << ProgName << ": No command specified!\n"; 3009 else 3010 errs() << ProgName << ": Unknown command!\n"; 3011 3012 errs() << "USAGE: " << ProgName << " <merge|show|overlap> [args...]\n"; 3013 return 1; 3014 } 3015