xref: /freebsd/contrib/llvm-project/llvm/lib/ProfileData/SampleProfWriter.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
1 //===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class that writes LLVM sample profiles. It
10 // supports two file formats: text and binary. The textual representation
11 // is useful for debugging and testing purposes. The binary representation
12 // is more compact, resulting in smaller file sizes. However, they can
13 // both be used interchangeably.
14 //
15 // See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16 // supported formats.
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "llvm/ProfileData/SampleProfWriter.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/ProfileData/ProfileCommon.h"
23 #include "llvm/ProfileData/SampleProf.h"
24 #include "llvm/Support/Compression.h"
25 #include "llvm/Support/Endian.h"
26 #include "llvm/Support/EndianStream.h"
27 #include "llvm/Support/ErrorOr.h"
28 #include "llvm/Support/FileSystem.h"
29 #include "llvm/Support/LEB128.h"
30 #include "llvm/Support/MD5.h"
31 #include "llvm/Support/raw_ostream.h"
32 #include <algorithm>
33 #include <cmath>
34 #include <cstdint>
35 #include <memory>
36 #include <set>
37 #include <system_error>
38 #include <utility>
39 #include <vector>
40 
41 #define DEBUG_TYPE "llvm-profdata"
42 
43 using namespace llvm;
44 using namespace sampleprof;
45 
46 namespace llvm {
47 namespace support {
48 namespace endian {
49 namespace {
50 
51 // Adapter class to llvm::support::endian::Writer for pwrite().
52 struct SeekableWriter {
53   raw_pwrite_stream &OS;
54   endianness Endian;
SeekableWriterllvm::support::endian::__anonbc33b9800111::SeekableWriter55   SeekableWriter(raw_pwrite_stream &OS, endianness Endian)
56       : OS(OS), Endian(Endian) {}
57 
58   template <typename ValueType>
pwritellvm::support::endian::__anonbc33b9800111::SeekableWriter59   void pwrite(ValueType Val, size_t Offset) {
60     std::string StringBuf;
61     raw_string_ostream SStream(StringBuf);
62     Writer(SStream, Endian).write(Val);
63     OS.pwrite(StringBuf.data(), StringBuf.size(), Offset);
64   }
65 };
66 
67 } // namespace
68 } // namespace endian
69 } // namespace support
70 } // namespace llvm
71 
DefaultFunctionPruningStrategy(SampleProfileMap & ProfileMap,size_t OutputSizeLimit)72 DefaultFunctionPruningStrategy::DefaultFunctionPruningStrategy(
73     SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
74     : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {
75   sortFuncProfiles(ProfileMap, SortedFunctions);
76 }
77 
Erase(size_t CurrentOutputSize)78 void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {
79   double D = (double)OutputSizeLimit / CurrentOutputSize;
80   size_t NewSize = (size_t)round(ProfileMap.size() * D * D);
81   size_t NumToRemove = ProfileMap.size() - NewSize;
82   if (NumToRemove < 1)
83     NumToRemove = 1;
84 
85   assert(NumToRemove <= SortedFunctions.size());
86   for (const NameFunctionSamples &E :
87        llvm::drop_begin(SortedFunctions, SortedFunctions.size() - NumToRemove))
88     ProfileMap.erase(E.first);
89   SortedFunctions.resize(SortedFunctions.size() - NumToRemove);
90 }
91 
writeWithSizeLimitInternal(SampleProfileMap & ProfileMap,size_t OutputSizeLimit,FunctionPruningStrategy * Strategy)92 std::error_code SampleProfileWriter::writeWithSizeLimitInternal(
93     SampleProfileMap &ProfileMap, size_t OutputSizeLimit,
94     FunctionPruningStrategy *Strategy) {
95   if (OutputSizeLimit == 0)
96     return write(ProfileMap);
97 
98   size_t OriginalFunctionCount = ProfileMap.size();
99 
100   std::unique_ptr<raw_ostream> OriginalOutputStream;
101   OutputStream.swap(OriginalOutputStream);
102 
103   size_t IterationCount = 0;
104   size_t TotalSize;
105 
106   SmallVector<char> StringBuffer;
107   do {
108     StringBuffer.clear();
109     OutputStream.reset(new raw_svector_ostream(StringBuffer));
110     if (std::error_code EC = write(ProfileMap))
111       return EC;
112 
113     TotalSize = StringBuffer.size();
114     // On Windows every "\n" is actually written as "\r\n" to disk but not to
115     // memory buffer, this difference should be added when considering the total
116     // output size.
117 #ifdef _WIN32
118     if (Format == SPF_Text)
119       TotalSize += LineCount;
120 #endif
121     if (TotalSize <= OutputSizeLimit)
122       break;
123 
124     Strategy->Erase(TotalSize);
125     IterationCount++;
126   } while (ProfileMap.size() != 0);
127 
128   if (ProfileMap.size() == 0)
129     return sampleprof_error::too_large;
130 
131   OutputStream.swap(OriginalOutputStream);
132   OutputStream->write(StringBuffer.data(), StringBuffer.size());
133   LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount
134                     << " functions, reduced to " << ProfileMap.size() << " in "
135                     << IterationCount << " iterations\n");
136   // Silence warning on Release build.
137   (void)OriginalFunctionCount;
138   (void)IterationCount;
139   return sampleprof_error::success;
140 }
141 
142 std::error_code
writeFuncProfiles(const SampleProfileMap & ProfileMap)143 SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) {
144   std::vector<NameFunctionSamples> V;
145   sortFuncProfiles(ProfileMap, V);
146   for (const auto &I : V) {
147     if (std::error_code EC = writeSample(*I.second))
148       return EC;
149   }
150   return sampleprof_error::success;
151 }
152 
write(const SampleProfileMap & ProfileMap)153 std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {
154   if (std::error_code EC = writeHeader(ProfileMap))
155     return EC;
156 
157   if (std::error_code EC = writeFuncProfiles(ProfileMap))
158     return EC;
159 
160   return sampleprof_error::success;
161 }
162 
163 /// Return the current position and prepare to use it as the start
164 /// position of a section given the section type \p Type and its position
165 /// \p LayoutIdx in SectionHdrLayout.
166 uint64_t
markSectionStart(SecType Type,uint32_t LayoutIdx)167 SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,
168                                                    uint32_t LayoutIdx) {
169   uint64_t SectionStart = OutputStream->tell();
170   assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
171   const auto &Entry = SectionHdrLayout[LayoutIdx];
172   assert(Entry.Type == Type && "Unexpected section type");
173   // Use LocalBuf as a temporary output for writting data.
174   if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress))
175     LocalBufStream.swap(OutputStream);
176   return SectionStart;
177 }
178 
compressAndOutput()179 std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
180   if (!llvm::compression::zlib::isAvailable())
181     return sampleprof_error::zlib_unavailable;
182   std::string &UncompressedStrings =
183       static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
184   if (UncompressedStrings.size() == 0)
185     return sampleprof_error::success;
186   auto &OS = *OutputStream;
187   SmallVector<uint8_t, 128> CompressedStrings;
188   compression::zlib::compress(arrayRefFromStringRef(UncompressedStrings),
189                               CompressedStrings,
190                               compression::zlib::BestSizeCompression);
191   encodeULEB128(UncompressedStrings.size(), OS);
192   encodeULEB128(CompressedStrings.size(), OS);
193   OS << toStringRef(CompressedStrings);
194   UncompressedStrings.clear();
195   return sampleprof_error::success;
196 }
197 
198 /// Add a new section into section header table given the section type
199 /// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
200 /// location \p SectionStart where the section should be written to.
addNewSection(SecType Type,uint32_t LayoutIdx,uint64_t SectionStart)201 std::error_code SampleProfileWriterExtBinaryBase::addNewSection(
202     SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
203   assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
204   const auto &Entry = SectionHdrLayout[LayoutIdx];
205   assert(Entry.Type == Type && "Unexpected section type");
206   if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress)) {
207     LocalBufStream.swap(OutputStream);
208     if (std::error_code EC = compressAndOutput())
209       return EC;
210   }
211   SecHdrTable.push_back({Type, Entry.Flags, SectionStart - FileStart,
212                          OutputStream->tell() - SectionStart, LayoutIdx});
213   return sampleprof_error::success;
214 }
215 
216 std::error_code
write(const SampleProfileMap & ProfileMap)217 SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) {
218   // When calling write on a different profile map, existing states should be
219   // cleared.
220   NameTable.clear();
221   CSNameTable.clear();
222   SecHdrTable.clear();
223 
224   if (std::error_code EC = writeHeader(ProfileMap))
225     return EC;
226 
227   std::string LocalBuf;
228   LocalBufStream = std::make_unique<raw_string_ostream>(LocalBuf);
229   if (std::error_code EC = writeSections(ProfileMap))
230     return EC;
231 
232   if (std::error_code EC = writeSecHdrTable())
233     return EC;
234 
235   return sampleprof_error::success;
236 }
237 
writeContextIdx(const SampleContext & Context)238 std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
239     const SampleContext &Context) {
240   if (Context.hasContext())
241     return writeCSNameIdx(Context);
242   else
243     return SampleProfileWriterBinary::writeNameIdx(Context.getFunction());
244 }
245 
246 std::error_code
writeCSNameIdx(const SampleContext & Context)247 SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) {
248   const auto &Ret = CSNameTable.find(Context);
249   if (Ret == CSNameTable.end())
250     return sampleprof_error::truncated_name_table;
251   encodeULEB128(Ret->second, *OutputStream);
252   return sampleprof_error::success;
253 }
254 
255 std::error_code
writeSample(const FunctionSamples & S)256 SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
257   uint64_t Offset = OutputStream->tell();
258   auto &Context = S.getContext();
259   FuncOffsetTable[Context] = Offset - SecLBRProfileStart;
260   encodeULEB128(S.getHeadSamples(), *OutputStream);
261   return writeBody(S);
262 }
263 
writeFuncOffsetTable()264 std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
265   auto &OS = *OutputStream;
266 
267   // Write out the table size.
268   encodeULEB128(FuncOffsetTable.size(), OS);
269 
270   // Write out FuncOffsetTable.
271   auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {
272     if (std::error_code EC = writeContextIdx(Context))
273       return EC;
274     encodeULEB128(Offset, OS);
275     return (std::error_code)sampleprof_error::success;
276   };
277 
278   if (FunctionSamples::ProfileIsCS) {
279     // Sort the contexts before writing them out. This is to help fast load all
280     // context profiles for a function as well as their callee contexts which
281     // can help profile-guided importing for ThinLTO.
282     std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(
283         FuncOffsetTable.begin(), FuncOffsetTable.end());
284     for (const auto &Entry : OrderedFuncOffsetTable) {
285       if (std::error_code EC = WriteItem(Entry.first, Entry.second))
286         return EC;
287     }
288     addSectionFlag(SecFuncOffsetTable, SecFuncOffsetFlags::SecFlagOrdered);
289   } else {
290     for (const auto &Entry : FuncOffsetTable) {
291       if (std::error_code EC = WriteItem(Entry.first, Entry.second))
292         return EC;
293     }
294   }
295 
296   FuncOffsetTable.clear();
297   return sampleprof_error::success;
298 }
299 
writeFuncMetadata(const FunctionSamples & FunctionProfile)300 std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
301     const FunctionSamples &FunctionProfile) {
302   auto &OS = *OutputStream;
303   if (std::error_code EC = writeContextIdx(FunctionProfile.getContext()))
304     return EC;
305 
306   if (FunctionSamples::ProfileIsProbeBased)
307     encodeULEB128(FunctionProfile.getFunctionHash(), OS);
308   if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) {
309     encodeULEB128(FunctionProfile.getContext().getAllAttributes(), OS);
310   }
311 
312   if (!FunctionSamples::ProfileIsCS) {
313     // Recursively emit attributes for all callee samples.
314     uint64_t NumCallsites = 0;
315     for (const auto &J : FunctionProfile.getCallsiteSamples())
316       NumCallsites += J.second.size();
317     encodeULEB128(NumCallsites, OS);
318     for (const auto &J : FunctionProfile.getCallsiteSamples()) {
319       for (const auto &FS : J.second) {
320         LineLocation Loc = J.first;
321         encodeULEB128(Loc.LineOffset, OS);
322         encodeULEB128(Loc.Discriminator, OS);
323         if (std::error_code EC = writeFuncMetadata(FS.second))
324           return EC;
325       }
326     }
327   }
328 
329   return sampleprof_error::success;
330 }
331 
writeFuncMetadata(const SampleProfileMap & Profiles)332 std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
333     const SampleProfileMap &Profiles) {
334   if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS &&
335       !FunctionSamples::ProfileIsPreInlined)
336     return sampleprof_error::success;
337   for (const auto &Entry : Profiles) {
338     if (std::error_code EC = writeFuncMetadata(Entry.second))
339       return EC;
340   }
341   return sampleprof_error::success;
342 }
343 
writeNameTable()344 std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
345   if (!UseMD5)
346     return SampleProfileWriterBinary::writeNameTable();
347 
348   auto &OS = *OutputStream;
349   std::set<FunctionId> V;
350   stablizeNameTable(NameTable, V);
351 
352   // Write out the MD5 name table. We wrote unencoded MD5 so reader can
353   // retrieve the name using the name index without having to read the
354   // whole name table.
355   encodeULEB128(NameTable.size(), OS);
356   support::endian::Writer Writer(OS, llvm::endianness::little);
357   for (auto N : V)
358     Writer.write(N.getHashCode());
359   return sampleprof_error::success;
360 }
361 
writeNameTableSection(const SampleProfileMap & ProfileMap)362 std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
363     const SampleProfileMap &ProfileMap) {
364   for (const auto &I : ProfileMap) {
365     addContext(I.second.getContext());
366     addNames(I.second);
367   }
368 
369   // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
370   // so compiler won't strip the suffix during profile matching after
371   // seeing the flag in the profile.
372   // Original names are unavailable if using MD5, so this option has no use.
373   if (!UseMD5) {
374     for (const auto &I : NameTable) {
375       if (I.first.stringRef().contains(FunctionSamples::UniqSuffix)) {
376         addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix);
377         break;
378       }
379     }
380   }
381 
382   if (auto EC = writeNameTable())
383     return EC;
384   return sampleprof_error::success;
385 }
386 
writeCSNameTableSection()387 std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
388   // Sort the names to make CSNameTable deterministic.
389   std::set<SampleContext> OrderedContexts;
390   for (const auto &I : CSNameTable)
391     OrderedContexts.insert(I.first);
392   assert(OrderedContexts.size() == CSNameTable.size() &&
393          "Unmatched ordered and unordered contexts");
394   uint64_t I = 0;
395   for (auto &Context : OrderedContexts)
396     CSNameTable[Context] = I++;
397 
398   auto &OS = *OutputStream;
399   encodeULEB128(OrderedContexts.size(), OS);
400   support::endian::Writer Writer(OS, llvm::endianness::little);
401   for (auto Context : OrderedContexts) {
402     auto Frames = Context.getContextFrames();
403     encodeULEB128(Frames.size(), OS);
404     for (auto &Callsite : Frames) {
405       if (std::error_code EC = writeNameIdx(Callsite.Func))
406         return EC;
407       encodeULEB128(Callsite.Location.LineOffset, OS);
408       encodeULEB128(Callsite.Location.Discriminator, OS);
409     }
410   }
411 
412   return sampleprof_error::success;
413 }
414 
415 std::error_code
writeProfileSymbolListSection()416 SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {
417   if (ProfSymList && ProfSymList->size() > 0)
418     if (std::error_code EC = ProfSymList->write(*OutputStream))
419       return EC;
420 
421   return sampleprof_error::success;
422 }
423 
writeOneSection(SecType Type,uint32_t LayoutIdx,const SampleProfileMap & ProfileMap)424 std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
425     SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) {
426   // The setting of SecFlagCompress should happen before markSectionStart.
427   if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())
428     setToCompressSection(SecProfileSymbolList);
429   if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
430     addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased);
431   if (Type == SecFuncMetadata &&
432       (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined))
433     addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagHasAttribute);
434   if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
435     addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFullContext);
436   if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined)
437     addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagIsPreInlined);
438   if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
439     addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFSDiscriminator);
440 
441   uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
442   switch (Type) {
443   case SecProfSummary:
444     computeSummary(ProfileMap);
445     if (auto EC = writeSummary())
446       return EC;
447     break;
448   case SecNameTable:
449     if (auto EC = writeNameTableSection(ProfileMap))
450       return EC;
451     break;
452   case SecCSNameTable:
453     if (auto EC = writeCSNameTableSection())
454       return EC;
455     break;
456   case SecLBRProfile:
457     SecLBRProfileStart = OutputStream->tell();
458     if (std::error_code EC = writeFuncProfiles(ProfileMap))
459       return EC;
460     break;
461   case SecFuncOffsetTable:
462     if (auto EC = writeFuncOffsetTable())
463       return EC;
464     break;
465   case SecFuncMetadata:
466     if (std::error_code EC = writeFuncMetadata(ProfileMap))
467       return EC;
468     break;
469   case SecProfileSymbolList:
470     if (auto EC = writeProfileSymbolListSection())
471       return EC;
472     break;
473   default:
474     if (auto EC = writeCustomSection(Type))
475       return EC;
476     break;
477   }
478   if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
479     return EC;
480   return sampleprof_error::success;
481 }
482 
writeDefaultLayout(const SampleProfileMap & ProfileMap)483 std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
484     const SampleProfileMap &ProfileMap) {
485   // The const indices passed to writeOneSection below are specifying the
486   // positions of the sections in SectionHdrLayout. Look at
487   // initSectionHdrLayout to find out where each section is located in
488   // SectionHdrLayout.
489   if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))
490     return EC;
491   if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))
492     return EC;
493   if (auto EC = writeOneSection(SecCSNameTable, 2, ProfileMap))
494     return EC;
495   if (auto EC = writeOneSection(SecLBRProfile, 4, ProfileMap))
496     return EC;
497   if (auto EC = writeOneSection(SecProfileSymbolList, 5, ProfileMap))
498     return EC;
499   if (auto EC = writeOneSection(SecFuncOffsetTable, 3, ProfileMap))
500     return EC;
501   if (auto EC = writeOneSection(SecFuncMetadata, 6, ProfileMap))
502     return EC;
503   return sampleprof_error::success;
504 }
505 
splitProfileMapToTwo(const SampleProfileMap & ProfileMap,SampleProfileMap & ContextProfileMap,SampleProfileMap & NoContextProfileMap)506 static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,
507                                  SampleProfileMap &ContextProfileMap,
508                                  SampleProfileMap &NoContextProfileMap) {
509   for (const auto &I : ProfileMap) {
510     if (I.second.getCallsiteSamples().size())
511       ContextProfileMap.insert({I.first, I.second});
512     else
513       NoContextProfileMap.insert({I.first, I.second});
514   }
515 }
516 
writeCtxSplitLayout(const SampleProfileMap & ProfileMap)517 std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
518     const SampleProfileMap &ProfileMap) {
519   SampleProfileMap ContextProfileMap, NoContextProfileMap;
520   splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap);
521 
522   if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))
523     return EC;
524   if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))
525     return EC;
526   if (auto EC = writeOneSection(SecLBRProfile, 3, ContextProfileMap))
527     return EC;
528   if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ContextProfileMap))
529     return EC;
530   // Mark the section to have no context. Note section flag needs to be set
531   // before writing the section.
532   addSectionFlag(5, SecCommonFlags::SecFlagFlat);
533   if (auto EC = writeOneSection(SecLBRProfile, 5, NoContextProfileMap))
534     return EC;
535   // Mark the section to have no context. Note section flag needs to be set
536   // before writing the section.
537   addSectionFlag(4, SecCommonFlags::SecFlagFlat);
538   if (auto EC = writeOneSection(SecFuncOffsetTable, 4, NoContextProfileMap))
539     return EC;
540   if (auto EC = writeOneSection(SecProfileSymbolList, 6, ProfileMap))
541     return EC;
542   if (auto EC = writeOneSection(SecFuncMetadata, 7, ProfileMap))
543     return EC;
544 
545   return sampleprof_error::success;
546 }
547 
writeSections(const SampleProfileMap & ProfileMap)548 std::error_code SampleProfileWriterExtBinary::writeSections(
549     const SampleProfileMap &ProfileMap) {
550   std::error_code EC;
551   if (SecLayout == DefaultLayout)
552     EC = writeDefaultLayout(ProfileMap);
553   else if (SecLayout == CtxSplitLayout)
554     EC = writeCtxSplitLayout(ProfileMap);
555   else
556     llvm_unreachable("Unsupported layout");
557   return EC;
558 }
559 
560 /// Write samples to a text file.
561 ///
562 /// Note: it may be tempting to implement this in terms of
563 /// FunctionSamples::print().  Please don't.  The dump functionality is intended
564 /// for debugging and has no specified form.
565 ///
566 /// The format used here is more structured and deliberate because
567 /// it needs to be parsed by the SampleProfileReaderText class.
writeSample(const FunctionSamples & S)568 std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
569   auto &OS = *OutputStream;
570   if (FunctionSamples::ProfileIsCS)
571     OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
572   else
573     OS << S.getFunction() << ":" << S.getTotalSamples();
574 
575   if (Indent == 0)
576     OS << ":" << S.getHeadSamples();
577   OS << "\n";
578   LineCount++;
579 
580   SampleSorter<LineLocation, SampleRecord> SortedSamples(S.getBodySamples());
581   for (const auto &I : SortedSamples.get()) {
582     LineLocation Loc = I->first;
583     const SampleRecord &Sample = I->second;
584     OS.indent(Indent + 1);
585     if (Loc.Discriminator == 0)
586       OS << Loc.LineOffset << ": ";
587     else
588       OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
589 
590     OS << Sample.getSamples();
591 
592     for (const auto &J : Sample.getSortedCallTargets())
593       OS << " " << J.first << ":" << J.second;
594     OS << "\n";
595     LineCount++;
596   }
597 
598   SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
599       S.getCallsiteSamples());
600   Indent += 1;
601   for (const auto &I : SortedCallsiteSamples.get())
602     for (const auto &FS : I->second) {
603       LineLocation Loc = I->first;
604       const FunctionSamples &CalleeSamples = FS.second;
605       OS.indent(Indent);
606       if (Loc.Discriminator == 0)
607         OS << Loc.LineOffset << ": ";
608       else
609         OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
610       if (std::error_code EC = writeSample(CalleeSamples))
611         return EC;
612     }
613   Indent -= 1;
614 
615   if (FunctionSamples::ProfileIsProbeBased) {
616     OS.indent(Indent + 1);
617     OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
618     LineCount++;
619   }
620 
621   if (S.getContext().getAllAttributes()) {
622     OS.indent(Indent + 1);
623     OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
624     LineCount++;
625   }
626 
627   return sampleprof_error::success;
628 }
629 
630 std::error_code
writeContextIdx(const SampleContext & Context)631 SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
632   assert(!Context.hasContext() && "cs profile is not supported");
633   return writeNameIdx(Context.getFunction());
634 }
635 
writeNameIdx(FunctionId FName)636 std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
637   auto &NTable = getNameTable();
638   const auto &Ret = NTable.find(FName);
639   if (Ret == NTable.end())
640     return sampleprof_error::truncated_name_table;
641   encodeULEB128(Ret->second, *OutputStream);
642   return sampleprof_error::success;
643 }
644 
addName(FunctionId FName)645 void SampleProfileWriterBinary::addName(FunctionId FName) {
646   auto &NTable = getNameTable();
647   NTable.insert(std::make_pair(FName, 0));
648 }
649 
addContext(const SampleContext & Context)650 void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
651   addName(Context.getFunction());
652 }
653 
addNames(const FunctionSamples & S)654 void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
655   // Add all the names in indirect call targets.
656   for (const auto &I : S.getBodySamples()) {
657     const SampleRecord &Sample = I.second;
658     for (const auto &J : Sample.getCallTargets())
659       addName(J.first);
660   }
661 
662   // Recursively add all the names for inlined callsites.
663   for (const auto &J : S.getCallsiteSamples())
664     for (const auto &FS : J.second) {
665       const FunctionSamples &CalleeSamples = FS.second;
666       addName(CalleeSamples.getFunction());
667       addNames(CalleeSamples);
668     }
669 }
670 
addContext(const SampleContext & Context)671 void SampleProfileWriterExtBinaryBase::addContext(
672     const SampleContext &Context) {
673   if (Context.hasContext()) {
674     for (auto &Callsite : Context.getContextFrames())
675       SampleProfileWriterBinary::addName(Callsite.Func);
676     CSNameTable.insert(std::make_pair(Context, 0));
677   } else {
678     SampleProfileWriterBinary::addName(Context.getFunction());
679   }
680 }
681 
stablizeNameTable(MapVector<FunctionId,uint32_t> & NameTable,std::set<FunctionId> & V)682 void SampleProfileWriterBinary::stablizeNameTable(
683     MapVector<FunctionId, uint32_t> &NameTable, std::set<FunctionId> &V) {
684   // Sort the names to make NameTable deterministic.
685   for (const auto &I : NameTable)
686     V.insert(I.first);
687   int i = 0;
688   for (const FunctionId &N : V)
689     NameTable[N] = i++;
690 }
691 
writeNameTable()692 std::error_code SampleProfileWriterBinary::writeNameTable() {
693   auto &OS = *OutputStream;
694   std::set<FunctionId> V;
695   stablizeNameTable(NameTable, V);
696 
697   // Write out the name table.
698   encodeULEB128(NameTable.size(), OS);
699   for (auto N : V) {
700     OS << N;
701     encodeULEB128(0, OS);
702   }
703   return sampleprof_error::success;
704 }
705 
706 std::error_code
writeMagicIdent(SampleProfileFormat Format)707 SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
708   auto &OS = *OutputStream;
709   // Write file magic identifier.
710   encodeULEB128(SPMagic(Format), OS);
711   encodeULEB128(SPVersion(), OS);
712   return sampleprof_error::success;
713 }
714 
715 std::error_code
writeHeader(const SampleProfileMap & ProfileMap)716 SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) {
717   // When calling write on a different profile map, existing names should be
718   // cleared.
719   NameTable.clear();
720 
721   writeMagicIdent(Format);
722 
723   computeSummary(ProfileMap);
724   if (auto EC = writeSummary())
725     return EC;
726 
727   // Generate the name table for all the functions referenced in the profile.
728   for (const auto &I : ProfileMap) {
729     addContext(I.second.getContext());
730     addNames(I.second);
731   }
732 
733   writeNameTable();
734   return sampleprof_error::success;
735 }
736 
setToCompressAllSections()737 void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
738   for (auto &Entry : SectionHdrLayout)
739     addSecFlag(Entry, SecCommonFlags::SecFlagCompress);
740 }
741 
setToCompressSection(SecType Type)742 void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
743   addSectionFlag(Type, SecCommonFlags::SecFlagCompress);
744 }
745 
allocSecHdrTable()746 void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
747   support::endian::Writer Writer(*OutputStream, llvm::endianness::little);
748 
749   Writer.write(static_cast<uint64_t>(SectionHdrLayout.size()));
750   SecHdrTableOffset = OutputStream->tell();
751   for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
752     Writer.write(static_cast<uint64_t>(-1));
753     Writer.write(static_cast<uint64_t>(-1));
754     Writer.write(static_cast<uint64_t>(-1));
755     Writer.write(static_cast<uint64_t>(-1));
756   }
757 }
758 
writeSecHdrTable()759 std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
760   assert(SecHdrTable.size() == SectionHdrLayout.size() &&
761          "SecHdrTable entries doesn't match SectionHdrLayout");
762   SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
763   for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
764     IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
765   }
766 
767   // Write the section header table in the order specified in
768   // SectionHdrLayout. SectionHdrLayout specifies the sections
769   // order in which profile reader expect to read, so the section
770   // header table should be written in the order in SectionHdrLayout.
771   // Note that the section order in SecHdrTable may be different
772   // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
773   // needs to be computed after SecLBRProfile (the order in SecHdrTable),
774   // but it needs to be read before SecLBRProfile (the order in
775   // SectionHdrLayout). So we use IndexMap above to switch the order.
776   support::endian::SeekableWriter Writer(
777       static_cast<raw_pwrite_stream &>(*OutputStream),
778       llvm::endianness::little);
779   for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
780        LayoutIdx++) {
781     assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
782            "Incorrect LayoutIdx in SecHdrTable");
783     auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
784     Writer.pwrite(static_cast<uint64_t>(Entry.Type),
785                   SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));
786     Writer.pwrite(static_cast<uint64_t>(Entry.Flags),
787                   SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));
788     Writer.pwrite(static_cast<uint64_t>(Entry.Offset),
789                   SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));
790     Writer.pwrite(static_cast<uint64_t>(Entry.Size),
791                   SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));
792   }
793 
794   return sampleprof_error::success;
795 }
796 
writeHeader(const SampleProfileMap & ProfileMap)797 std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
798     const SampleProfileMap &ProfileMap) {
799   auto &OS = *OutputStream;
800   FileStart = OS.tell();
801   writeMagicIdent(Format);
802 
803   allocSecHdrTable();
804   return sampleprof_error::success;
805 }
806 
writeSummary()807 std::error_code SampleProfileWriterBinary::writeSummary() {
808   auto &OS = *OutputStream;
809   encodeULEB128(Summary->getTotalCount(), OS);
810   encodeULEB128(Summary->getMaxCount(), OS);
811   encodeULEB128(Summary->getMaxFunctionCount(), OS);
812   encodeULEB128(Summary->getNumCounts(), OS);
813   encodeULEB128(Summary->getNumFunctions(), OS);
814   ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary();
815   encodeULEB128(Entries.size(), OS);
816   for (auto Entry : Entries) {
817     encodeULEB128(Entry.Cutoff, OS);
818     encodeULEB128(Entry.MinCount, OS);
819     encodeULEB128(Entry.NumCounts, OS);
820   }
821   return sampleprof_error::success;
822 }
writeBody(const FunctionSamples & S)823 std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
824   auto &OS = *OutputStream;
825   if (std::error_code EC = writeContextIdx(S.getContext()))
826     return EC;
827 
828   encodeULEB128(S.getTotalSamples(), OS);
829 
830   // Emit all the body samples.
831   encodeULEB128(S.getBodySamples().size(), OS);
832   for (const auto &I : S.getBodySamples()) {
833     LineLocation Loc = I.first;
834     const SampleRecord &Sample = I.second;
835     encodeULEB128(Loc.LineOffset, OS);
836     encodeULEB128(Loc.Discriminator, OS);
837     encodeULEB128(Sample.getSamples(), OS);
838     encodeULEB128(Sample.getCallTargets().size(), OS);
839     for (const auto &J : Sample.getSortedCallTargets()) {
840       FunctionId Callee = J.first;
841       uint64_t CalleeSamples = J.second;
842       if (std::error_code EC = writeNameIdx(Callee))
843         return EC;
844       encodeULEB128(CalleeSamples, OS);
845     }
846   }
847 
848   // Recursively emit all the callsite samples.
849   uint64_t NumCallsites = 0;
850   for (const auto &J : S.getCallsiteSamples())
851     NumCallsites += J.second.size();
852   encodeULEB128(NumCallsites, OS);
853   for (const auto &J : S.getCallsiteSamples())
854     for (const auto &FS : J.second) {
855       LineLocation Loc = J.first;
856       const FunctionSamples &CalleeSamples = FS.second;
857       encodeULEB128(Loc.LineOffset, OS);
858       encodeULEB128(Loc.Discriminator, OS);
859       if (std::error_code EC = writeBody(CalleeSamples))
860         return EC;
861     }
862 
863   return sampleprof_error::success;
864 }
865 
866 /// Write samples of a top-level function to a binary file.
867 ///
868 /// \returns true if the samples were written successfully, false otherwise.
869 std::error_code
writeSample(const FunctionSamples & S)870 SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
871   encodeULEB128(S.getHeadSamples(), *OutputStream);
872   return writeBody(S);
873 }
874 
875 /// Create a sample profile file writer based on the specified format.
876 ///
877 /// \param Filename The file to create.
878 ///
879 /// \param Format Encoding format for the profile file.
880 ///
881 /// \returns an error code indicating the status of the created writer.
882 ErrorOr<std::unique_ptr<SampleProfileWriter>>
create(StringRef Filename,SampleProfileFormat Format)883 SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
884   std::error_code EC;
885   std::unique_ptr<raw_ostream> OS;
886   if (Format == SPF_Binary || Format == SPF_Ext_Binary)
887     OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
888   else
889     OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
890   if (EC)
891     return EC;
892 
893   return create(OS, Format);
894 }
895 
896 /// Create a sample profile stream writer based on the specified format.
897 ///
898 /// \param OS The output stream to store the profile data to.
899 ///
900 /// \param Format Encoding format for the profile file.
901 ///
902 /// \returns an error code indicating the status of the created writer.
903 ErrorOr<std::unique_ptr<SampleProfileWriter>>
create(std::unique_ptr<raw_ostream> & OS,SampleProfileFormat Format)904 SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
905                             SampleProfileFormat Format) {
906   std::error_code EC;
907   std::unique_ptr<SampleProfileWriter> Writer;
908 
909   // Currently only Text and Extended Binary format are supported for CSSPGO.
910   if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
911       Format == SPF_Binary)
912     return sampleprof_error::unsupported_writing_format;
913 
914   if (Format == SPF_Binary)
915     Writer.reset(new SampleProfileWriterRawBinary(OS));
916   else if (Format == SPF_Ext_Binary)
917     Writer.reset(new SampleProfileWriterExtBinary(OS));
918   else if (Format == SPF_Text)
919     Writer.reset(new SampleProfileWriterText(OS));
920   else if (Format == SPF_GCC)
921     EC = sampleprof_error::unsupported_writing_format;
922   else
923     EC = sampleprof_error::unrecognized_format;
924 
925   if (EC)
926     return EC;
927 
928   Writer->Format = Format;
929   return std::move(Writer);
930 }
931 
computeSummary(const SampleProfileMap & ProfileMap)932 void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) {
933   SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
934   Summary = Builder.computeSummaryForProfiles(ProfileMap);
935 }
936