1 //===- PGOCtxProfWriter.cpp - Contextual Instrumentation profile writer ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Write a contextual profile to bitstream.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/ProfileData/PGOCtxProfWriter.h"
14 #include "llvm/Bitstream/BitCodeEnums.h"
15 #include "llvm/ProfileData/CtxInstrContextNode.h"
16 #include "llvm/Support/CommandLine.h"
17 #include "llvm/Support/Error.h"
18 #include "llvm/Support/YAMLTraits.h"
19 #include "llvm/Support/raw_ostream.h"
20
21 using namespace llvm;
22 using namespace llvm::ctx_profile;
23
24 static cl::opt<bool>
25 IncludeEmptyOpt("ctx-prof-include-empty", cl::init(false),
26 cl::desc("Also write profiles with all-zero counters. "
27 "Intended for testing/debugging."));
28
PGOCtxProfileWriter(raw_ostream & Out,std::optional<unsigned> VersionOverride,bool IncludeEmpty)29 PGOCtxProfileWriter::PGOCtxProfileWriter(
30 raw_ostream &Out, std::optional<unsigned> VersionOverride,
31 bool IncludeEmpty)
32 : Writer(Out, 0),
33 IncludeEmpty(IncludeEmptyOpt.getNumOccurrences() > 0 ? IncludeEmptyOpt
34 : IncludeEmpty) {
35 static_assert(ContainerMagic.size() == 4);
36 Out.write(ContainerMagic.data(), ContainerMagic.size());
37 Writer.EnterBlockInfoBlock();
38 {
39 auto DescribeBlock = [&](unsigned ID, StringRef Name) {
40 Writer.EmitRecord(bitc::BLOCKINFO_CODE_SETBID,
41 SmallVector<unsigned, 1>{ID});
42 Writer.EmitRecord(bitc::BLOCKINFO_CODE_BLOCKNAME,
43 llvm::arrayRefFromStringRef(Name));
44 };
45 SmallVector<uint64_t, 16> Data;
46 auto DescribeRecord = [&](unsigned RecordID, StringRef Name) {
47 Data.clear();
48 Data.push_back(RecordID);
49 llvm::append_range(Data, Name);
50 Writer.EmitRecord(bitc::BLOCKINFO_CODE_SETRECORDNAME, Data);
51 };
52 DescribeBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID, "Metadata");
53 DescribeRecord(PGOCtxProfileRecords::Version, "Version");
54 DescribeBlock(PGOCtxProfileBlockIDs::ContextsSectionBlockID, "Contexts");
55 DescribeBlock(PGOCtxProfileBlockIDs::ContextRootBlockID, "Root");
56 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");
57 DescribeRecord(PGOCtxProfileRecords::TotalRootEntryCount,
58 "TotalRootEntryCount");
59 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");
60 DescribeBlock(PGOCtxProfileBlockIDs::UnhandledBlockID, "Unhandled");
61 DescribeBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID, "Context");
62 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");
63 DescribeRecord(PGOCtxProfileRecords::CallsiteIndex, "CalleeIndex");
64 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");
65 DescribeBlock(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID,
66 "FlatProfiles");
67 DescribeBlock(PGOCtxProfileBlockIDs::FlatProfileBlockID, "Flat");
68 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");
69 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");
70 }
71 Writer.ExitBlock();
72 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID, CodeLen);
73 const auto Version = VersionOverride.value_or(CurrentVersion);
74 Writer.EmitRecord(PGOCtxProfileRecords::Version,
75 SmallVector<unsigned, 1>({Version}));
76 }
77
writeCounters(ArrayRef<uint64_t> Counters)78 void PGOCtxProfileWriter::writeCounters(ArrayRef<uint64_t> Counters) {
79 Writer.EmitCode(bitc::UNABBREV_RECORD);
80 Writer.EmitVBR(PGOCtxProfileRecords::Counters, VBREncodingBits);
81 Writer.EmitVBR(Counters.size(), VBREncodingBits);
82 for (uint64_t C : Counters)
83 Writer.EmitVBR64(C, VBREncodingBits);
84 }
85
writeGuid(ctx_profile::GUID Guid)86 void PGOCtxProfileWriter::writeGuid(ctx_profile::GUID Guid) {
87 Writer.EmitRecord(PGOCtxProfileRecords::Guid, SmallVector<uint64_t, 1>{Guid});
88 }
89
writeCallsiteIndex(uint32_t CallsiteIndex)90 void PGOCtxProfileWriter::writeCallsiteIndex(uint32_t CallsiteIndex) {
91 Writer.EmitRecord(PGOCtxProfileRecords::CallsiteIndex,
92 SmallVector<uint64_t, 1>{CallsiteIndex});
93 }
94
writeRootEntryCount(uint64_t TotalRootEntryCount)95 void PGOCtxProfileWriter::writeRootEntryCount(uint64_t TotalRootEntryCount) {
96 Writer.EmitRecord(PGOCtxProfileRecords::TotalRootEntryCount,
97 SmallVector<uint64_t, 1>{TotalRootEntryCount});
98 }
99
100 // recursively write all the subcontexts. We do need to traverse depth first to
101 // model the context->subcontext implicitly, and since this captures call
102 // stacks, we don't really need to be worried about stack overflow and we can
103 // keep the implementation simple.
writeNode(uint32_t CallsiteIndex,const ContextNode & Node)104 void PGOCtxProfileWriter::writeNode(uint32_t CallsiteIndex,
105 const ContextNode &Node) {
106 // A node with no counters is an error. We don't expect this to happen from
107 // the runtime, rather, this is interesting for testing the reader.
108 if (!IncludeEmpty && (Node.counters_size() > 0 && Node.entrycount() == 0))
109 return;
110 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextNodeBlockID, CodeLen);
111 writeGuid(Node.guid());
112 writeCallsiteIndex(CallsiteIndex);
113 writeCounters({Node.counters(), Node.counters_size()});
114 writeSubcontexts(Node);
115 Writer.ExitBlock();
116 }
117
writeSubcontexts(const ContextNode & Node)118 void PGOCtxProfileWriter::writeSubcontexts(const ContextNode &Node) {
119 for (uint32_t I = 0U; I < Node.callsites_size(); ++I)
120 for (const auto *Subcontext = Node.subContexts()[I]; Subcontext;
121 Subcontext = Subcontext->next())
122 writeNode(I, *Subcontext);
123 }
124
startContextSection()125 void PGOCtxProfileWriter::startContextSection() {
126 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextsSectionBlockID, CodeLen);
127 }
128
startFlatSection()129 void PGOCtxProfileWriter::startFlatSection() {
130 Writer.EnterSubblock(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID,
131 CodeLen);
132 }
133
endContextSection()134 void PGOCtxProfileWriter::endContextSection() { Writer.ExitBlock(); }
endFlatSection()135 void PGOCtxProfileWriter::endFlatSection() { Writer.ExitBlock(); }
136
writeContextual(const ContextNode & RootNode,const ContextNode * Unhandled,uint64_t TotalRootEntryCount)137 void PGOCtxProfileWriter::writeContextual(const ContextNode &RootNode,
138 const ContextNode *Unhandled,
139 uint64_t TotalRootEntryCount) {
140 if (!IncludeEmpty && (!TotalRootEntryCount || (RootNode.counters_size() > 0 &&
141 RootNode.entrycount() == 0)))
142 return;
143 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextRootBlockID, CodeLen);
144 writeGuid(RootNode.guid());
145 writeRootEntryCount(TotalRootEntryCount);
146 writeCounters({RootNode.counters(), RootNode.counters_size()});
147
148 Writer.EnterSubblock(PGOCtxProfileBlockIDs::UnhandledBlockID, CodeLen);
149 for (const auto *P = Unhandled; P; P = P->next())
150 writeFlat(P->guid(), P->counters(), P->counters_size());
151 Writer.ExitBlock();
152
153 writeSubcontexts(RootNode);
154 Writer.ExitBlock();
155 }
156
writeFlat(ctx_profile::GUID Guid,const uint64_t * Buffer,size_t Size)157 void PGOCtxProfileWriter::writeFlat(ctx_profile::GUID Guid,
158 const uint64_t *Buffer, size_t Size) {
159 Writer.EnterSubblock(PGOCtxProfileBlockIDs::FlatProfileBlockID, CodeLen);
160 writeGuid(Guid);
161 writeCounters({Buffer, Size});
162 Writer.ExitBlock();
163 }
164
165 namespace {
166
167 /// Representation of the context node suitable for yaml serialization /
168 /// deserialization.
169 using SerializableFlatProfileRepresentation =
170 std::pair<ctx_profile::GUID, std::vector<uint64_t>>;
171
172 struct SerializableCtxRepresentation {
173 ctx_profile::GUID Guid = 0;
174 std::vector<uint64_t> Counters;
175 std::vector<std::vector<SerializableCtxRepresentation>> Callsites;
176 };
177
178 struct SerializableRootRepresentation : public SerializableCtxRepresentation {
179 uint64_t TotalRootEntryCount = 0;
180 std::vector<SerializableFlatProfileRepresentation> Unhandled;
181 };
182
183 struct SerializableProfileRepresentation {
184 std::vector<SerializableRootRepresentation> Contexts;
185 std::vector<SerializableFlatProfileRepresentation> FlatProfiles;
186 };
187
188 ctx_profile::ContextNode *
189 createNode(std::vector<std::unique_ptr<char[]>> &Nodes,
190 const std::vector<SerializableCtxRepresentation> &DCList);
191
192 // Convert a DeserializableCtx into a ContextNode, potentially linking it to
193 // its sibling (e.g. callee at same callsite) "Next".
194 ctx_profile::ContextNode *
createNode(std::vector<std::unique_ptr<char[]>> & Nodes,const SerializableCtxRepresentation & DC,ctx_profile::ContextNode * Next=nullptr)195 createNode(std::vector<std::unique_ptr<char[]>> &Nodes,
196 const SerializableCtxRepresentation &DC,
197 ctx_profile::ContextNode *Next = nullptr) {
198 auto AllocSize = ctx_profile::ContextNode::getAllocSize(DC.Counters.size(),
199 DC.Callsites.size());
200 auto *Mem = Nodes.emplace_back(std::make_unique<char[]>(AllocSize)).get();
201 std::memset(Mem, 0, AllocSize);
202 auto *Ret = new (Mem) ctx_profile::ContextNode(DC.Guid, DC.Counters.size(),
203 DC.Callsites.size(), Next);
204 std::memcpy(Ret->counters(), DC.Counters.data(),
205 sizeof(uint64_t) * DC.Counters.size());
206 for (const auto &[I, DCList] : llvm::enumerate(DC.Callsites))
207 Ret->subContexts()[I] = createNode(Nodes, DCList);
208 return Ret;
209 }
210
211 // Convert a list of SerializableCtxRepresentation into a linked list of
212 // ContextNodes.
213 ctx_profile::ContextNode *
createNode(std::vector<std::unique_ptr<char[]>> & Nodes,const std::vector<SerializableCtxRepresentation> & DCList)214 createNode(std::vector<std::unique_ptr<char[]>> &Nodes,
215 const std::vector<SerializableCtxRepresentation> &DCList) {
216 ctx_profile::ContextNode *List = nullptr;
217 for (const auto &DC : DCList)
218 List = createNode(Nodes, DC, List);
219 return List;
220 }
221 } // namespace
222
223 LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableCtxRepresentation)
224 LLVM_YAML_IS_SEQUENCE_VECTOR(std::vector<SerializableCtxRepresentation>)
225 LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableRootRepresentation)
226 LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableFlatProfileRepresentation)
227 template <> struct yaml::MappingTraits<SerializableCtxRepresentation> {
mappingyaml::MappingTraits228 static void mapping(yaml::IO &IO, SerializableCtxRepresentation &SCR) {
229 IO.mapRequired("Guid", SCR.Guid);
230 IO.mapRequired("Counters", SCR.Counters);
231 IO.mapOptional("Callsites", SCR.Callsites);
232 }
233 };
234
235 template <> struct yaml::MappingTraits<SerializableRootRepresentation> {
mappingyaml::MappingTraits236 static void mapping(yaml::IO &IO, SerializableRootRepresentation &R) {
237 yaml::MappingTraits<SerializableCtxRepresentation>::mapping(IO, R);
238 IO.mapRequired("TotalRootEntryCount", R.TotalRootEntryCount);
239 IO.mapOptional("Unhandled", R.Unhandled);
240 }
241 };
242
243 template <> struct yaml::MappingTraits<SerializableProfileRepresentation> {
mappingyaml::MappingTraits244 static void mapping(yaml::IO &IO, SerializableProfileRepresentation &SPR) {
245 IO.mapOptional("Contexts", SPR.Contexts);
246 IO.mapOptional("FlatProfiles", SPR.FlatProfiles);
247 }
248 };
249
250 template <> struct yaml::MappingTraits<SerializableFlatProfileRepresentation> {
mappingyaml::MappingTraits251 static void mapping(yaml::IO &IO,
252 SerializableFlatProfileRepresentation &SFPR) {
253 IO.mapRequired("Guid", SFPR.first);
254 IO.mapRequired("Counters", SFPR.second);
255 }
256 };
257
createCtxProfFromYAML(StringRef Profile,raw_ostream & Out)258 Error llvm::createCtxProfFromYAML(StringRef Profile, raw_ostream &Out) {
259 yaml::Input In(Profile);
260 SerializableProfileRepresentation SPR;
261 In >> SPR;
262 if (In.error())
263 return createStringError(In.error(), "incorrect yaml content");
264 std::vector<std::unique_ptr<char[]>> Nodes;
265 std::error_code EC;
266 if (EC)
267 return createStringError(EC, "failed to open output");
268 PGOCtxProfileWriter Writer(Out);
269
270 if (!SPR.Contexts.empty()) {
271 Writer.startContextSection();
272 for (const auto &DC : SPR.Contexts) {
273 auto *TopList = createNode(Nodes, DC);
274 if (!TopList)
275 return createStringError(
276 "Unexpected error converting internal structure to ctx profile");
277
278 ctx_profile::ContextNode *FirstUnhandled = nullptr;
279 for (const auto &U : DC.Unhandled) {
280 SerializableCtxRepresentation Unhandled;
281 Unhandled.Guid = U.first;
282 Unhandled.Counters = U.second;
283 FirstUnhandled = createNode(Nodes, Unhandled, FirstUnhandled);
284 }
285 Writer.writeContextual(*TopList, FirstUnhandled, DC.TotalRootEntryCount);
286 }
287 Writer.endContextSection();
288 }
289 if (!SPR.FlatProfiles.empty()) {
290 Writer.startFlatSection();
291 for (const auto &[Guid, Counters] : SPR.FlatProfiles)
292 Writer.writeFlat(Guid, Counters.data(), Counters.size());
293 Writer.endFlatSection();
294 }
295 if (EC)
296 return createStringError(EC, "failed to write output");
297 return Error::success();
298 }
299