1 //===- PGOCtxProfWriter.cpp - Contextual Instrumentation profile writer ---===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Write a contextual profile to bitstream. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ProfileData/PGOCtxProfWriter.h" 14 #include "llvm/Bitstream/BitCodeEnums.h" 15 #include "llvm/ProfileData/CtxInstrContextNode.h" 16 #include "llvm/Support/CommandLine.h" 17 #include "llvm/Support/Error.h" 18 #include "llvm/Support/YAMLTraits.h" 19 #include "llvm/Support/raw_ostream.h" 20 21 using namespace llvm; 22 using namespace llvm::ctx_profile; 23 24 static cl::opt<bool> 25 IncludeEmptyOpt("ctx-prof-include-empty", cl::init(false), 26 cl::desc("Also write profiles with all-zero counters. " 27 "Intended for testing/debugging.")); 28 29 PGOCtxProfileWriter::PGOCtxProfileWriter( 30 raw_ostream &Out, std::optional<unsigned> VersionOverride, 31 bool IncludeEmpty) 32 : Writer(Out, 0), 33 IncludeEmpty(IncludeEmptyOpt.getNumOccurrences() > 0 ? IncludeEmptyOpt 34 : IncludeEmpty) { 35 static_assert(ContainerMagic.size() == 4); 36 Out.write(ContainerMagic.data(), ContainerMagic.size()); 37 Writer.EnterBlockInfoBlock(); 38 { 39 auto DescribeBlock = [&](unsigned ID, StringRef Name) { 40 Writer.EmitRecord(bitc::BLOCKINFO_CODE_SETBID, 41 SmallVector<unsigned, 1>{ID}); 42 Writer.EmitRecord(bitc::BLOCKINFO_CODE_BLOCKNAME, 43 llvm::arrayRefFromStringRef(Name)); 44 }; 45 SmallVector<uint64_t, 16> Data; 46 auto DescribeRecord = [&](unsigned RecordID, StringRef Name) { 47 Data.clear(); 48 Data.push_back(RecordID); 49 llvm::append_range(Data, Name); 50 Writer.EmitRecord(bitc::BLOCKINFO_CODE_SETRECORDNAME, Data); 51 }; 52 DescribeBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID, "Metadata"); 53 DescribeRecord(PGOCtxProfileRecords::Version, "Version"); 54 DescribeBlock(PGOCtxProfileBlockIDs::ContextsSectionBlockID, "Contexts"); 55 DescribeBlock(PGOCtxProfileBlockIDs::ContextRootBlockID, "Root"); 56 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID"); 57 DescribeRecord(PGOCtxProfileRecords::TotalRootEntryCount, 58 "TotalRootEntryCount"); 59 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters"); 60 DescribeBlock(PGOCtxProfileBlockIDs::UnhandledBlockID, "Unhandled"); 61 DescribeBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID, "Context"); 62 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID"); 63 DescribeRecord(PGOCtxProfileRecords::CallsiteIndex, "CalleeIndex"); 64 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters"); 65 DescribeBlock(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID, 66 "FlatProfiles"); 67 DescribeBlock(PGOCtxProfileBlockIDs::FlatProfileBlockID, "Flat"); 68 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID"); 69 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters"); 70 } 71 Writer.ExitBlock(); 72 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID, CodeLen); 73 const auto Version = VersionOverride.value_or(CurrentVersion); 74 Writer.EmitRecord(PGOCtxProfileRecords::Version, 75 SmallVector<unsigned, 1>({Version})); 76 } 77 78 void PGOCtxProfileWriter::writeCounters(ArrayRef<uint64_t> Counters) { 79 Writer.EmitCode(bitc::UNABBREV_RECORD); 80 Writer.EmitVBR(PGOCtxProfileRecords::Counters, VBREncodingBits); 81 Writer.EmitVBR(Counters.size(), VBREncodingBits); 82 for (uint64_t C : Counters) 83 Writer.EmitVBR64(C, VBREncodingBits); 84 } 85 86 void PGOCtxProfileWriter::writeGuid(ctx_profile::GUID Guid) { 87 Writer.EmitRecord(PGOCtxProfileRecords::Guid, SmallVector<uint64_t, 1>{Guid}); 88 } 89 90 void PGOCtxProfileWriter::writeCallsiteIndex(uint32_t CallsiteIndex) { 91 Writer.EmitRecord(PGOCtxProfileRecords::CallsiteIndex, 92 SmallVector<uint64_t, 1>{CallsiteIndex}); 93 } 94 95 void PGOCtxProfileWriter::writeRootEntryCount(uint64_t TotalRootEntryCount) { 96 Writer.EmitRecord(PGOCtxProfileRecords::TotalRootEntryCount, 97 SmallVector<uint64_t, 1>{TotalRootEntryCount}); 98 } 99 100 // recursively write all the subcontexts. We do need to traverse depth first to 101 // model the context->subcontext implicitly, and since this captures call 102 // stacks, we don't really need to be worried about stack overflow and we can 103 // keep the implementation simple. 104 void PGOCtxProfileWriter::writeNode(uint32_t CallsiteIndex, 105 const ContextNode &Node) { 106 // A node with no counters is an error. We don't expect this to happen from 107 // the runtime, rather, this is interesting for testing the reader. 108 if (!IncludeEmpty && (Node.counters_size() > 0 && Node.entrycount() == 0)) 109 return; 110 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextNodeBlockID, CodeLen); 111 writeGuid(Node.guid()); 112 writeCallsiteIndex(CallsiteIndex); 113 writeCounters({Node.counters(), Node.counters_size()}); 114 writeSubcontexts(Node); 115 Writer.ExitBlock(); 116 } 117 118 void PGOCtxProfileWriter::writeSubcontexts(const ContextNode &Node) { 119 for (uint32_t I = 0U; I < Node.callsites_size(); ++I) 120 for (const auto *Subcontext = Node.subContexts()[I]; Subcontext; 121 Subcontext = Subcontext->next()) 122 writeNode(I, *Subcontext); 123 } 124 125 void PGOCtxProfileWriter::startContextSection() { 126 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextsSectionBlockID, CodeLen); 127 } 128 129 void PGOCtxProfileWriter::startFlatSection() { 130 Writer.EnterSubblock(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID, 131 CodeLen); 132 } 133 134 void PGOCtxProfileWriter::endContextSection() { Writer.ExitBlock(); } 135 void PGOCtxProfileWriter::endFlatSection() { Writer.ExitBlock(); } 136 137 void PGOCtxProfileWriter::writeContextual(const ContextNode &RootNode, 138 const ContextNode *Unhandled, 139 uint64_t TotalRootEntryCount) { 140 if (!IncludeEmpty && (!TotalRootEntryCount || (RootNode.counters_size() > 0 && 141 RootNode.entrycount() == 0))) 142 return; 143 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextRootBlockID, CodeLen); 144 writeGuid(RootNode.guid()); 145 writeRootEntryCount(TotalRootEntryCount); 146 writeCounters({RootNode.counters(), RootNode.counters_size()}); 147 148 Writer.EnterSubblock(PGOCtxProfileBlockIDs::UnhandledBlockID, CodeLen); 149 for (const auto *P = Unhandled; P; P = P->next()) 150 writeFlat(P->guid(), P->counters(), P->counters_size()); 151 Writer.ExitBlock(); 152 153 writeSubcontexts(RootNode); 154 Writer.ExitBlock(); 155 } 156 157 void PGOCtxProfileWriter::writeFlat(ctx_profile::GUID Guid, 158 const uint64_t *Buffer, size_t Size) { 159 Writer.EnterSubblock(PGOCtxProfileBlockIDs::FlatProfileBlockID, CodeLen); 160 writeGuid(Guid); 161 writeCounters({Buffer, Size}); 162 Writer.ExitBlock(); 163 } 164 165 namespace { 166 167 /// Representation of the context node suitable for yaml serialization / 168 /// deserialization. 169 using SerializableFlatProfileRepresentation = 170 std::pair<ctx_profile::GUID, std::vector<uint64_t>>; 171 172 struct SerializableCtxRepresentation { 173 ctx_profile::GUID Guid = 0; 174 std::vector<uint64_t> Counters; 175 std::vector<std::vector<SerializableCtxRepresentation>> Callsites; 176 }; 177 178 struct SerializableRootRepresentation : public SerializableCtxRepresentation { 179 uint64_t TotalRootEntryCount = 0; 180 std::vector<SerializableFlatProfileRepresentation> Unhandled; 181 }; 182 183 struct SerializableProfileRepresentation { 184 std::vector<SerializableRootRepresentation> Contexts; 185 std::vector<SerializableFlatProfileRepresentation> FlatProfiles; 186 }; 187 188 ctx_profile::ContextNode * 189 createNode(std::vector<std::unique_ptr<char[]>> &Nodes, 190 const std::vector<SerializableCtxRepresentation> &DCList); 191 192 // Convert a DeserializableCtx into a ContextNode, potentially linking it to 193 // its sibling (e.g. callee at same callsite) "Next". 194 ctx_profile::ContextNode * 195 createNode(std::vector<std::unique_ptr<char[]>> &Nodes, 196 const SerializableCtxRepresentation &DC, 197 ctx_profile::ContextNode *Next = nullptr) { 198 auto AllocSize = ctx_profile::ContextNode::getAllocSize(DC.Counters.size(), 199 DC.Callsites.size()); 200 auto *Mem = Nodes.emplace_back(std::make_unique<char[]>(AllocSize)).get(); 201 std::memset(Mem, 0, AllocSize); 202 auto *Ret = new (Mem) ctx_profile::ContextNode(DC.Guid, DC.Counters.size(), 203 DC.Callsites.size(), Next); 204 std::memcpy(Ret->counters(), DC.Counters.data(), 205 sizeof(uint64_t) * DC.Counters.size()); 206 for (const auto &[I, DCList] : llvm::enumerate(DC.Callsites)) 207 Ret->subContexts()[I] = createNode(Nodes, DCList); 208 return Ret; 209 } 210 211 // Convert a list of SerializableCtxRepresentation into a linked list of 212 // ContextNodes. 213 ctx_profile::ContextNode * 214 createNode(std::vector<std::unique_ptr<char[]>> &Nodes, 215 const std::vector<SerializableCtxRepresentation> &DCList) { 216 ctx_profile::ContextNode *List = nullptr; 217 for (const auto &DC : DCList) 218 List = createNode(Nodes, DC, List); 219 return List; 220 } 221 } // namespace 222 223 LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableCtxRepresentation) 224 LLVM_YAML_IS_SEQUENCE_VECTOR(std::vector<SerializableCtxRepresentation>) 225 LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableRootRepresentation) 226 LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableFlatProfileRepresentation) 227 template <> struct yaml::MappingTraits<SerializableCtxRepresentation> { 228 static void mapping(yaml::IO &IO, SerializableCtxRepresentation &SCR) { 229 IO.mapRequired("Guid", SCR.Guid); 230 IO.mapRequired("Counters", SCR.Counters); 231 IO.mapOptional("Callsites", SCR.Callsites); 232 } 233 }; 234 235 template <> struct yaml::MappingTraits<SerializableRootRepresentation> { 236 static void mapping(yaml::IO &IO, SerializableRootRepresentation &R) { 237 yaml::MappingTraits<SerializableCtxRepresentation>::mapping(IO, R); 238 IO.mapRequired("TotalRootEntryCount", R.TotalRootEntryCount); 239 IO.mapOptional("Unhandled", R.Unhandled); 240 } 241 }; 242 243 template <> struct yaml::MappingTraits<SerializableProfileRepresentation> { 244 static void mapping(yaml::IO &IO, SerializableProfileRepresentation &SPR) { 245 IO.mapOptional("Contexts", SPR.Contexts); 246 IO.mapOptional("FlatProfiles", SPR.FlatProfiles); 247 } 248 }; 249 250 template <> struct yaml::MappingTraits<SerializableFlatProfileRepresentation> { 251 static void mapping(yaml::IO &IO, 252 SerializableFlatProfileRepresentation &SFPR) { 253 IO.mapRequired("Guid", SFPR.first); 254 IO.mapRequired("Counters", SFPR.second); 255 } 256 }; 257 258 Error llvm::createCtxProfFromYAML(StringRef Profile, raw_ostream &Out) { 259 yaml::Input In(Profile); 260 SerializableProfileRepresentation SPR; 261 In >> SPR; 262 if (In.error()) 263 return createStringError(In.error(), "incorrect yaml content"); 264 std::vector<std::unique_ptr<char[]>> Nodes; 265 std::error_code EC; 266 if (EC) 267 return createStringError(EC, "failed to open output"); 268 PGOCtxProfileWriter Writer(Out); 269 270 if (!SPR.Contexts.empty()) { 271 Writer.startContextSection(); 272 for (const auto &DC : SPR.Contexts) { 273 auto *TopList = createNode(Nodes, DC); 274 if (!TopList) 275 return createStringError( 276 "Unexpected error converting internal structure to ctx profile"); 277 278 ctx_profile::ContextNode *FirstUnhandled = nullptr; 279 for (const auto &U : DC.Unhandled) { 280 SerializableCtxRepresentation Unhandled; 281 Unhandled.Guid = U.first; 282 Unhandled.Counters = U.second; 283 FirstUnhandled = createNode(Nodes, Unhandled, FirstUnhandled); 284 } 285 Writer.writeContextual(*TopList, FirstUnhandled, DC.TotalRootEntryCount); 286 } 287 Writer.endContextSection(); 288 } 289 if (!SPR.FlatProfiles.empty()) { 290 Writer.startFlatSection(); 291 for (const auto &[Guid, Counters] : SPR.FlatProfiles) 292 Writer.writeFlat(Guid, Counters.data(), Counters.size()); 293 Writer.endFlatSection(); 294 } 295 if (EC) 296 return createStringError(EC, "failed to write output"); 297 return Error::success(); 298 } 299