1 //===- InstrProfWriter.h - Instrumented profiling writer --------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains support for writing profiling data for instrumentation 10 // based PGO and coverage. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_PROFILEDATA_INSTRPROFWRITER_H 15 #define LLVM_PROFILEDATA_INSTRPROFWRITER_H 16 17 #include "llvm/ADT/DenseMap.h" 18 #include "llvm/ADT/MapVector.h" 19 #include "llvm/ADT/StringMap.h" 20 #include "llvm/IR/GlobalValue.h" 21 #include "llvm/Object/BuildID.h" 22 #include "llvm/ProfileData/DataAccessProf.h" 23 #include "llvm/ProfileData/IndexedMemProfData.h" 24 #include "llvm/ProfileData/InstrProf.h" 25 #include "llvm/ProfileData/MemProfSummaryBuilder.h" 26 #include "llvm/Support/Compiler.h" 27 #include "llvm/Support/Error.h" 28 #include <cstdint> 29 #include <memory> 30 #include <random> 31 32 namespace llvm { 33 34 /// Writer for instrumentation based profile data. 35 class InstrProfRecordWriterTrait; 36 class ProfOStream; 37 class MemoryBuffer; 38 class raw_fd_ostream; 39 40 class InstrProfWriter { 41 public: 42 using ProfilingData = SmallDenseMap<uint64_t, InstrProfRecord>; 43 44 private: 45 bool Sparse; 46 StringMap<ProfilingData> FunctionData; 47 /// The maximum length of a single temporal profile trace. 48 uint64_t MaxTemporalProfTraceLength; 49 /// The maximum number of stored temporal profile traces. 50 uint64_t TemporalProfTraceReservoirSize; 51 /// The total number of temporal profile traces seen. 52 uint64_t TemporalProfTraceStreamSize = 0; 53 /// The list of temporal profile traces. 54 SmallVector<TemporalProfTraceTy> TemporalProfTraces; 55 std::mt19937 RNG; 56 57 // The MemProf data. 58 memprof::IndexedMemProfData MemProfData; 59 60 // List of binary ids. 61 std::vector<llvm::object::BuildID> BinaryIds; 62 63 // Read the vtable names from raw instr profile reader. 64 StringSet<> VTableNames; 65 66 // An enum describing the attributes of the profile. 67 InstrProfKind ProfileKind = InstrProfKind::Unknown; 68 // Use raw pointer here for the incomplete type object. 69 InstrProfRecordWriterTrait *InfoObj; 70 71 // Temporary support for writing the previous version of the format, to enable 72 // some forward compatibility. Currently this suppresses the writing of the 73 // new vtable names section and header fields. 74 // TODO: Consider enabling this with future version changes as well, to ease 75 // deployment of newer versions of llvm-profdata. 76 bool WritePrevVersion = false; 77 78 // The MemProf version we should write. 79 memprof::IndexedVersion MemProfVersionRequested; 80 81 // Whether to serialize the full schema. 82 bool MemProfFullSchema; 83 84 // Whether to generated random memprof hotness for testing. 85 bool MemprofGenerateRandomHotness; 86 87 std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData; 88 89 // MemProf summary builder to which records are added as MemProf data is added 90 // to the writer. 91 memprof::MemProfSummaryBuilder MemProfSumBuilder; 92 93 public: 94 // For memprof testing, random hotness can be assigned to the contexts if 95 // MemprofGenerateRandomHotness is enabled. The random seed can be either 96 // provided by MemprofGenerateRandomHotnessSeed, or if that is 0, one will be 97 // generated in the writer using the current time. 98 LLVM_ABI InstrProfWriter(bool Sparse = false, 99 uint64_t TemporalProfTraceReservoirSize = 0, 100 uint64_t MaxTemporalProfTraceLength = 0, 101 bool WritePrevVersion = false, 102 memprof::IndexedVersion MemProfVersionRequested = 103 static_cast<memprof::IndexedVersion>( 104 memprof::MinimumSupportedVersion), 105 bool MemProfFullSchema = false, 106 bool MemprofGenerateRandomHotness = false, 107 unsigned MemprofGenerateRandomHotnessSeed = 0); 108 LLVM_ABI ~InstrProfWriter(); 109 getProfileData()110 StringMap<ProfilingData> &getProfileData() { return FunctionData; } 111 112 /// Add function counts for the given function. If there are already counts 113 /// for this function and the hash and number of counts match, each counter is 114 /// summed. Optionally scale counts by \p Weight. 115 LLVM_ABI void addRecord(NamedInstrProfRecord &&I, uint64_t Weight, 116 function_ref<void(Error)> Warn); addRecord(NamedInstrProfRecord && I,function_ref<void (Error)> Warn)117 void addRecord(NamedInstrProfRecord &&I, function_ref<void(Error)> Warn) { 118 addRecord(std::move(I), 1, Warn); 119 } addVTableName(StringRef VTableName)120 void addVTableName(StringRef VTableName) { VTableNames.insert(VTableName); } 121 122 /// Add \p SrcTraces using reservoir sampling where \p SrcStreamSize is the 123 /// total number of temporal profiling traces the source has seen. 124 LLVM_ABI void 125 addTemporalProfileTraces(SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, 126 uint64_t SrcStreamSize); 127 128 /// Add the entire MemProfData \p Incoming to the writer context. 129 LLVM_ABI bool addMemProfData(memprof::IndexedMemProfData Incoming, 130 function_ref<void(Error)> Warn); 131 132 // Add a binary id to the binary ids list. 133 LLVM_ABI void addBinaryIds(ArrayRef<llvm::object::BuildID> BIs); 134 135 LLVM_ABI void addDataAccessProfData( 136 std::unique_ptr<memprof::DataAccessProfData> DataAccessProfile); 137 138 /// Merge existing function counts from the given writer. 139 LLVM_ABI void mergeRecordsFromWriter(InstrProfWriter &&IPW, 140 function_ref<void(Error)> Warn); 141 142 /// Write the profile to \c OS 143 LLVM_ABI Error write(raw_fd_ostream &OS); 144 145 /// Write the profile to a string output stream \c OS 146 LLVM_ABI Error write(raw_string_ostream &OS); 147 148 /// Write the profile in text format to \c OS 149 LLVM_ABI Error writeText(raw_fd_ostream &OS); 150 151 /// Write temporal profile trace data to the header in text format to \c OS 152 LLVM_ABI void writeTextTemporalProfTraceData(raw_fd_ostream &OS, 153 InstrProfSymtab &Symtab); 154 155 LLVM_ABI Error validateRecord(const InstrProfRecord &Func); 156 157 /// Write \c Record in text format to \c OS 158 LLVM_ABI static void writeRecordInText(StringRef Name, uint64_t Hash, 159 const InstrProfRecord &Counters, 160 InstrProfSymtab &Symtab, 161 raw_fd_ostream &OS); 162 163 /// Write the profile, returning the raw data. For testing. 164 LLVM_ABI std::unique_ptr<MemoryBuffer> writeBuffer(); 165 166 /// Update the attributes of the current profile from the attributes 167 /// specified. An error is returned if IR and FE profiles are mixed. mergeProfileKind(const InstrProfKind Other)168 Error mergeProfileKind(const InstrProfKind Other) { 169 // If the kind is unset, this is the first profile we are merging so just 170 // set it to the given type. 171 if (ProfileKind == InstrProfKind::Unknown) { 172 ProfileKind = Other; 173 return Error::success(); 174 } 175 176 // Returns true if merging is should fail assuming A and B are incompatible. 177 auto testIncompatible = [&](InstrProfKind A, InstrProfKind B) { 178 return (static_cast<bool>(ProfileKind & A) && 179 static_cast<bool>(Other & B)) || 180 (static_cast<bool>(ProfileKind & B) && 181 static_cast<bool>(Other & A)); 182 }; 183 184 // Check if the profiles are in-compatible. Clang frontend profiles can't be 185 // merged with other profile types. 186 if (static_cast<bool>( 187 (ProfileKind & InstrProfKind::FrontendInstrumentation) ^ 188 (Other & InstrProfKind::FrontendInstrumentation))) { 189 return make_error<InstrProfError>(instrprof_error::unsupported_version); 190 } 191 if (testIncompatible(InstrProfKind::FunctionEntryOnly, 192 InstrProfKind::FunctionEntryInstrumentation) || 193 testIncompatible(InstrProfKind::FunctionEntryOnly, 194 InstrProfKind::LoopEntriesInstrumentation)) { 195 return make_error<InstrProfError>( 196 instrprof_error::unsupported_version, 197 "cannot merge FunctionEntryOnly profiles and BB profiles together"); 198 } 199 200 // Now we update the profile type with the bits that are set. 201 ProfileKind |= Other; 202 return Error::success(); 203 } 204 getProfileKind()205 InstrProfKind getProfileKind() const { return ProfileKind; } 206 hasSingleByteCoverage()207 bool hasSingleByteCoverage() const { 208 return static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage); 209 } 210 211 // Internal interfaces for testing purpose only. 212 LLVM_ABI void setValueProfDataEndianness(llvm::endianness Endianness); 213 LLVM_ABI void setOutputSparse(bool Sparse); setMemProfVersionRequested(memprof::IndexedVersion Version)214 void setMemProfVersionRequested(memprof::IndexedVersion Version) { 215 MemProfVersionRequested = Version; 216 } setMemProfFullSchema(bool Full)217 void setMemProfFullSchema(bool Full) { MemProfFullSchema = Full; } 218 // Compute the overlap b/w this object and Other. Program level result is 219 // stored in Overlap and function level result is stored in FuncLevelOverlap. 220 LLVM_ABI void overlapRecord(NamedInstrProfRecord &&Other, 221 OverlapStats &Overlap, 222 OverlapStats &FuncLevelOverlap, 223 const OverlapFuncFilters &FuncFilter); 224 225 private: 226 void addRecord(StringRef Name, uint64_t Hash, InstrProfRecord &&I, 227 uint64_t Weight, function_ref<void(Error)> Warn); 228 bool shouldEncodeData(const ProfilingData &PD); 229 /// Add \p Trace using reservoir sampling. 230 void addTemporalProfileTrace(TemporalProfTraceTy Trace); 231 232 /// Add a memprof record for a function identified by its \p Id. 233 void addMemProfRecord(const GlobalValue::GUID Id, 234 const memprof::IndexedMemProfRecord &Record); 235 236 /// Add a memprof frame identified by the hash of the contents of the frame in 237 /// \p FrameId. 238 bool addMemProfFrame(const memprof::FrameId, const memprof::Frame &F, 239 function_ref<void(Error)> Warn); 240 241 /// Add a call stack identified by the hash of the contents of the call stack 242 /// in \p CallStack. 243 bool addMemProfCallStack(const memprof::CallStackId CSId, 244 const llvm::SmallVector<memprof::FrameId> &CallStack, 245 function_ref<void(Error)> Warn); 246 247 Error writeImpl(ProfOStream &OS); 248 249 // Writes known header fields and reserves space for fields whose value are 250 // known only after payloads are written. Returns the start byte offset for 251 // back patching. 252 uint64_t writeHeader(const IndexedInstrProf::Header &header, 253 const bool WritePrevVersion, ProfOStream &OS); 254 255 // Writes binary IDs. 256 Error writeBinaryIds(ProfOStream &OS); 257 258 // Writes compressed vtable names to profiles. 259 Error writeVTableNames(ProfOStream &OS); 260 }; 261 262 } // end namespace llvm 263 264 #endif // LLVM_PROFILEDATA_INSTRPROFWRITER_H 265