xref: /freebsd/contrib/llvm-project/compiler-rt/include/profile/InstrProfData.inc (revision ec0ea6efa1ad229d75c394c1a9b9cac33af2b1d3)
1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8/*
9 * This is the main file that defines all the data structure, signature,
10 * constant literals that are shared across profiling runtime library,
11 * compiler (instrumentation), and host tools (reader/writer). The entities
12 * defined in this file affect the profile runtime ABI, the raw profile format,
13 * or both.
14 *
15 * The file has two identical copies. The primary copy lives in LLVM and
16 * the other one  sits in compiler-rt/lib/profile directory. To make changes
17 * in this file, first modify the primary copy and copy it over to compiler-rt.
18 * Testing of any change in this file can start only after the two copies are
19 * synced up.
20 *
21 * The first part of the file includes macros that defines types, names, and
22 * initializers for the member fields of the core data structures. The field
23 * declarations for one structure is enabled by defining the field activation
24 * macro associated with that structure. Only one field activation record
25 * can be defined at one time and the rest definitions will be filtered out by
26 * the preprocessor.
27 *
28 * Examples of how the template is used to instantiate structure definition:
29 * 1. To declare a structure:
30 *
31 * struct ProfData {
32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33 *    Type Name;
34 * #include "llvm/ProfileData/InstrProfData.inc"
35 * };
36 *
37 * 2. To construct LLVM type arrays for the struct type:
38 *
39 * Type *DataTypes[] = {
40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41 *   LLVMType,
42 * #include "llvm/ProfileData/InstrProfData.inc"
43 * };
44 *
45 * 4. To construct constant array for the initializers:
46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47 *   Initializer,
48 * Constant *ConstantVals[] = {
49 * #include "llvm/ProfileData/InstrProfData.inc"
50 * };
51 *
52 *
53 * The second part of the file includes definitions all other entities that
54 * are related to runtime ABI and format. When no field activation macro is
55 * defined, this file can be included to introduce the definitions.
56 *
57\*===----------------------------------------------------------------------===*/
58
59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60 * the compiler runtime. */
61#ifndef INSTR_PROF_VISIBILITY
62#define INSTR_PROF_VISIBILITY
63#endif
64
65/* INSTR_PROF_DATA start. */
66/* Definition of member fields of the per-function control structure. */
67#ifndef INSTR_PROF_DATA
68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
69#else
70#define INSTR_PROF_DATA_DEFINED
71#endif
72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
73                ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
74                IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
76                ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
77                Inc->getHash()->getZExtValue()))
78INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
79                ConstantExpr::getBitCast(CounterPtr, \
80                llvm::Type::getInt64PtrTy(Ctx)))
81/* This is used to map function pointers for the indirect call targets to
82 * function name hashes during the conversion from raw to merged profile
83 * data.
84 */
85INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
86                FunctionAddr)
87INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
88                ValuesPtrExpr)
89INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
90                ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
91INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
92                ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
93#undef INSTR_PROF_DATA
94/* INSTR_PROF_DATA end. */
95
96
97/* This is an internal data structure used by value profiler. It
98 * is defined here to allow serialization code sharing by LLVM
99 * to be used in unit test.
100 *
101 * typedef struct ValueProfNode {
102 *   // InstrProfValueData VData;
103 *   uint64_t Value;
104 *   uint64_t Count;
105 *   struct ValueProfNode *Next;
106 * } ValueProfNode;
107 */
108/* INSTR_PROF_VALUE_NODE start. */
109#ifndef INSTR_PROF_VALUE_NODE
110#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
111#else
112#define INSTR_PROF_DATA_DEFINED
113#endif
114INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
115                      ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
116INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
117                      ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
118INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \
119                      ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
120#undef INSTR_PROF_VALUE_NODE
121/* INSTR_PROF_VALUE_NODE end. */
122
123/* INSTR_PROF_RAW_HEADER  start */
124/* Definition of member fields of the raw profile header data structure. */
125#ifndef INSTR_PROF_RAW_HEADER
126#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
127#else
128#define INSTR_PROF_DATA_DEFINED
129#endif
130INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
131INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
132INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
133INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
134INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
135INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
136INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
137INSTR_PROF_RAW_HEADER(uint64_t, NamesSize,  NamesSize)
138INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
139INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
140INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
141#undef INSTR_PROF_RAW_HEADER
142/* INSTR_PROF_RAW_HEADER  end */
143
144/* VALUE_PROF_FUNC_PARAM start */
145/* Definition of parameter types of the runtime API used to do value profiling
146 * for a given value site.
147 */
148#ifndef VALUE_PROF_FUNC_PARAM
149#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
150#define INSTR_PROF_COMMA
151#else
152#define INSTR_PROF_DATA_DEFINED
153#define INSTR_PROF_COMMA ,
154#endif
155VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
156                      INSTR_PROF_COMMA
157VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
158VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
159#undef VALUE_PROF_FUNC_PARAM
160#undef INSTR_PROF_COMMA
161/* VALUE_PROF_FUNC_PARAM end */
162
163/* VALUE_PROF_KIND start */
164#ifndef VALUE_PROF_KIND
165#define VALUE_PROF_KIND(Enumerator, Value, Descr)
166#else
167#define INSTR_PROF_DATA_DEFINED
168#endif
169/* For indirect function call value profiling, the addresses of the target
170 * functions are profiled by the instrumented code. The target addresses are
171 * written in the raw profile data and converted to target function name's MD5
172 * hash by the profile reader during deserialization.  Typically, this happens
173 * when the raw profile data is read during profile merging.
174 *
175 * For this remapping the ProfData is used.  ProfData contains both the function
176 * name hash and the function address.
177 */
178VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
179/* For memory intrinsic functions size profiling. */
180VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
181/* These two kinds must be the last to be
182 * declared. This is to make sure the string
183 * array created with the template can be
184 * indexed with the kind value.
185 */
186VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
187VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
188
189#undef VALUE_PROF_KIND
190/* VALUE_PROF_KIND end */
191
192#undef COVMAP_V2_OR_V3
193#ifdef COVMAP_V2
194#define COVMAP_V2_OR_V3
195#endif
196#ifdef COVMAP_V3
197#define COVMAP_V2_OR_V3
198#endif
199
200/* COVMAP_FUNC_RECORD start */
201/* Definition of member fields of the function record structure in coverage
202 * map.
203 */
204#ifndef COVMAP_FUNC_RECORD
205#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
206#else
207#define INSTR_PROF_DATA_DEFINED
208#endif
209#ifdef COVMAP_V1
210COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
211                   NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
212                   llvm::Type::getInt8PtrTy(Ctx)))
213COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
214                   llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
215                   NameValue.size()))
216#endif
217#ifdef COVMAP_V2_OR_V3
218COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
219                   llvm::ConstantInt::get( \
220                     llvm::Type::getInt64Ty(Ctx), NameHash))
221#endif
222COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
223                   llvm::ConstantInt::get( \
224                     llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
225COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
226                   llvm::ConstantInt::get( \
227                     llvm::Type::getInt64Ty(Ctx), FuncHash))
228#ifdef COVMAP_V3
229COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
230                   llvm::ConstantInt::get( \
231                     llvm::Type::getInt64Ty(Ctx), FilenamesRef))
232COVMAP_FUNC_RECORD(const char, \
233                   llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
234                                        CoverageMapping.size()), \
235                   CoverageMapping,
236                   llvm::ConstantDataArray::getRaw( \
237                     CoverageMapping, CoverageMapping.size(), \
238                     llvm::Type::getInt8Ty(Ctx)))
239#endif
240#undef COVMAP_FUNC_RECORD
241/* COVMAP_FUNC_RECORD end.  */
242
243/* COVMAP_HEADER start */
244/* Definition of member fields of coverage map header.
245 */
246#ifndef COVMAP_HEADER
247#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
248#else
249#define INSTR_PROF_DATA_DEFINED
250#endif
251COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
252              llvm::ConstantInt::get(Int32Ty, NRecords))
253COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
254              llvm::ConstantInt::get(Int32Ty, FilenamesSize))
255COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
256              llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
257COVMAP_HEADER(uint32_t, Int32Ty, Version, \
258              llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
259#undef COVMAP_HEADER
260/* COVMAP_HEADER end.  */
261
262
263#ifdef INSTR_PROF_SECT_ENTRY
264#define INSTR_PROF_DATA_DEFINED
265INSTR_PROF_SECT_ENTRY(IPSK_data, \
266                      INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
267                      INSTR_PROF_DATA_COFF, "__DATA,")
268INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
269                      INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
270                      INSTR_PROF_CNTS_COFF, "__DATA,")
271INSTR_PROF_SECT_ENTRY(IPSK_name, \
272                      INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
273                      INSTR_PROF_NAME_COFF, "__DATA,")
274INSTR_PROF_SECT_ENTRY(IPSK_vals, \
275                      INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
276                      INSTR_PROF_VALS_COFF, "__DATA,")
277INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
278                      INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
279                      INSTR_PROF_VNODES_COFF, "__DATA,")
280INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
281                      INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
282                      INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
283INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
284                      INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
285                      INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
286INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
287                      INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
288                      INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
289
290#undef INSTR_PROF_SECT_ENTRY
291#endif
292
293
294#ifdef INSTR_PROF_VALUE_PROF_DATA
295#define INSTR_PROF_DATA_DEFINED
296
297#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
298/*!
299 * This is the header of the data structure that defines the on-disk
300 * layout of the value profile data of a particular kind for one function.
301 */
302typedef struct ValueProfRecord {
303  /* The kind of the value profile record. */
304  uint32_t Kind;
305  /*
306   * The number of value profile sites. It is guaranteed to be non-zero;
307   * otherwise the record for this kind won't be emitted.
308   */
309  uint32_t NumValueSites;
310  /*
311   * The first element of the array that stores the number of profiled
312   * values for each value site. The size of the array is NumValueSites.
313   * Since NumValueSites is greater than zero, there is at least one
314   * element in the array.
315   */
316  uint8_t SiteCountArray[1];
317
318  /*
319   * The fake declaration is for documentation purpose only.
320   * Align the start of next field to be on 8 byte boundaries.
321  uint8_t Padding[X];
322   */
323
324  /* The array of value profile data. The size of the array is the sum
325   * of all elements in SiteCountArray[].
326  InstrProfValueData ValueData[];
327   */
328
329#ifdef __cplusplus
330  /*!
331   * Return the number of value sites.
332   */
333  uint32_t getNumValueSites() const { return NumValueSites; }
334  /*!
335   * Read data from this record and save it to Record.
336   */
337  void deserializeTo(InstrProfRecord &Record,
338                     InstrProfSymtab *SymTab);
339  /*
340   * In-place byte swap:
341   * Do byte swap for this instance. \c Old is the original order before
342   * the swap, and \c New is the New byte order.
343   */
344  void swapBytes(support::endianness Old, support::endianness New);
345#endif
346} ValueProfRecord;
347
348/*!
349 * Per-function header/control data structure for value profiling
350 * data in indexed format.
351 */
352typedef struct ValueProfData {
353  /*
354   * Total size in bytes including this field. It must be a multiple
355   * of sizeof(uint64_t).
356   */
357  uint32_t TotalSize;
358  /*
359   *The number of value profile kinds that has value profile data.
360   * In this implementation, a value profile kind is considered to
361   * have profile data if the number of value profile sites for the
362   * kind is not zero. More aggressively, the implementation can
363   * choose to check the actual data value: if none of the value sites
364   * has any profiled values, the kind can be skipped.
365   */
366  uint32_t NumValueKinds;
367
368  /*
369   * Following are a sequence of variable length records. The prefix/header
370   * of each record is defined by ValueProfRecord type. The number of
371   * records is NumValueKinds.
372   * ValueProfRecord Record_1;
373   * ValueProfRecord Record_N;
374   */
375
376#if __cplusplus
377  /*!
378   * Return the total size in bytes of the on-disk value profile data
379   * given the data stored in Record.
380   */
381  static uint32_t getSize(const InstrProfRecord &Record);
382  /*!
383   * Return a pointer to \c ValueProfData instance ready to be streamed.
384   */
385  static std::unique_ptr<ValueProfData>
386  serializeFrom(const InstrProfRecord &Record);
387  /*!
388   * Check the integrity of the record.
389   */
390  Error checkIntegrity();
391  /*!
392   * Return a pointer to \c ValueProfileData instance ready to be read.
393   * All data in the instance are properly byte swapped. The input
394   * data is assumed to be in little endian order.
395   */
396  static Expected<std::unique_ptr<ValueProfData>>
397  getValueProfData(const unsigned char *SrcBuffer,
398                   const unsigned char *const SrcBufferEnd,
399                   support::endianness SrcDataEndianness);
400  /*!
401   * Swap byte order from \c Endianness order to host byte order.
402   */
403  void swapBytesToHost(support::endianness Endianness);
404  /*!
405   * Swap byte order from host byte order to \c Endianness order.
406   */
407  void swapBytesFromHost(support::endianness Endianness);
408  /*!
409   * Return the total size of \c ValueProfileData.
410   */
411  uint32_t getSize() const { return TotalSize; }
412  /*!
413   * Read data from this data and save it to \c Record.
414   */
415  void deserializeTo(InstrProfRecord &Record,
416                     InstrProfSymtab *SymTab);
417  void operator delete(void *ptr) { ::operator delete(ptr); }
418#endif
419} ValueProfData;
420
421/*
422 * The closure is designed to abstact away two types of value profile data:
423 * - InstrProfRecord which is the primary data structure used to
424 *   represent profile data in host tools (reader, writer, and profile-use)
425 * - value profile runtime data structure suitable to be used by C
426 *   runtime library.
427 *
428 * Both sources of data need to serialize to disk/memory-buffer in common
429 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
430 * writer to share the same format and code with indexed profile writer.
431 *
432 * For documentation of the member methods below, refer to corresponding methods
433 * in class InstrProfRecord.
434 */
435typedef struct ValueProfRecordClosure {
436  const void *Record;
437  uint32_t (*GetNumValueKinds)(const void *Record);
438  uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
439  uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
440  uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
441
442  /*
443   * After extracting the value profile data from the value profile record,
444   * this method is used to map the in-memory value to on-disk value. If
445   * the method is null, value will be written out untranslated.
446   */
447  uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
448  void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
449                          uint32_t S);
450  ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
451} ValueProfRecordClosure;
452
453INSTR_PROF_VISIBILITY ValueProfRecord *
454getFirstValueProfRecord(ValueProfData *VPD);
455INSTR_PROF_VISIBILITY ValueProfRecord *
456getValueProfRecordNext(ValueProfRecord *VPR);
457INSTR_PROF_VISIBILITY InstrProfValueData *
458getValueProfRecordValueData(ValueProfRecord *VPR);
459INSTR_PROF_VISIBILITY uint32_t
460getValueProfRecordHeaderSize(uint32_t NumValueSites);
461
462#undef INSTR_PROF_VALUE_PROF_DATA
463#endif  /* INSTR_PROF_VALUE_PROF_DATA */
464
465
466#ifdef INSTR_PROF_COMMON_API_IMPL
467#define INSTR_PROF_DATA_DEFINED
468#ifdef __cplusplus
469#define INSTR_PROF_INLINE inline
470#define INSTR_PROF_NULLPTR nullptr
471#else
472#define INSTR_PROF_INLINE
473#define INSTR_PROF_NULLPTR NULL
474#endif
475
476#ifndef offsetof
477#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
478#endif
479
480/*!
481 * Return the \c ValueProfRecord header size including the
482 * padding bytes.
483 */
484INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
485uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
486  uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
487                  sizeof(uint8_t) * NumValueSites;
488  /* Round the size to multiple of 8 bytes. */
489  Size = (Size + 7) & ~7;
490  return Size;
491}
492
493/*!
494 * Return the total size of the value profile record including the
495 * header and the value data.
496 */
497INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
498uint32_t getValueProfRecordSize(uint32_t NumValueSites,
499                                uint32_t NumValueData) {
500  return getValueProfRecordHeaderSize(NumValueSites) +
501         sizeof(InstrProfValueData) * NumValueData;
502}
503
504/*!
505 * Return the pointer to the start of value data array.
506 */
507INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
508InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
509  return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
510                                                   This->NumValueSites));
511}
512
513/*!
514 * Return the total number of value data for \c This record.
515 */
516INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
517uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
518  uint32_t NumValueData = 0;
519  uint32_t I;
520  for (I = 0; I < This->NumValueSites; I++)
521    NumValueData += This->SiteCountArray[I];
522  return NumValueData;
523}
524
525/*!
526 * Use this method to advance to the next \c This \c ValueProfRecord.
527 */
528INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
529ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
530  uint32_t NumValueData = getValueProfRecordNumValueData(This);
531  return (ValueProfRecord *)((char *)This +
532                             getValueProfRecordSize(This->NumValueSites,
533                                                    NumValueData));
534}
535
536/*!
537 * Return the first \c ValueProfRecord instance.
538 */
539INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
540ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
541  return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
542}
543
544/* Closure based interfaces.  */
545
546/*!
547 * Return the total size in bytes of the on-disk value profile data
548 * given the data stored in Record.
549 */
550INSTR_PROF_VISIBILITY uint32_t
551getValueProfDataSize(ValueProfRecordClosure *Closure) {
552  uint32_t Kind;
553  uint32_t TotalSize = sizeof(ValueProfData);
554  const void *Record = Closure->Record;
555
556  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
557    uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
558    if (!NumValueSites)
559      continue;
560    TotalSize += getValueProfRecordSize(NumValueSites,
561                                        Closure->GetNumValueData(Record, Kind));
562  }
563  return TotalSize;
564}
565
566/*!
567 * Extract value profile data of a function for the profile kind \c ValueKind
568 * from the \c Closure and serialize the data into \c This record instance.
569 */
570INSTR_PROF_VISIBILITY void
571serializeValueProfRecordFrom(ValueProfRecord *This,
572                             ValueProfRecordClosure *Closure,
573                             uint32_t ValueKind, uint32_t NumValueSites) {
574  uint32_t S;
575  const void *Record = Closure->Record;
576  This->Kind = ValueKind;
577  This->NumValueSites = NumValueSites;
578  InstrProfValueData *DstVD = getValueProfRecordValueData(This);
579
580  for (S = 0; S < NumValueSites; S++) {
581    uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
582    This->SiteCountArray[S] = ND;
583    Closure->GetValueForSite(Record, DstVD, ValueKind, S);
584    DstVD += ND;
585  }
586}
587
588/*!
589 * Extract value profile data of a function  from the \c Closure
590 * and serialize the data into \c DstData if it is not NULL or heap
591 * memory allocated by the \c Closure's allocator method. If \c
592 * DstData is not null, the caller is expected to set the TotalSize
593 * in DstData.
594 */
595INSTR_PROF_VISIBILITY ValueProfData *
596serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
597                           ValueProfData *DstData) {
598  uint32_t Kind;
599  uint32_t TotalSize =
600      DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
601
602  ValueProfData *VPD =
603      DstData ? DstData : Closure->AllocValueProfData(TotalSize);
604
605  VPD->TotalSize = TotalSize;
606  VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
607  ValueProfRecord *VR = getFirstValueProfRecord(VPD);
608  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
609    uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
610    if (!NumValueSites)
611      continue;
612    serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
613    VR = getValueProfRecordNext(VR);
614  }
615  return VPD;
616}
617
618#undef INSTR_PROF_COMMON_API_IMPL
619#endif /* INSTR_PROF_COMMON_API_IMPL */
620
621/*============================================================================*/
622
623#ifndef INSTR_PROF_DATA_DEFINED
624
625#ifndef INSTR_PROF_DATA_INC
626#define INSTR_PROF_DATA_INC
627
628/* Helper macros.  */
629#define INSTR_PROF_SIMPLE_QUOTE(x) #x
630#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
631#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
632#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
633
634/* Magic number to detect file format and endianness.
635 * Use 255 at one end, since no UTF-8 file can use that character.  Avoid 0,
636 * so that utilities, like strings, don't grab it as a string.  129 is also
637 * invalid UTF-8, and high enough to be interesting.
638 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
639 * for 32-bit platforms.
640 */
641#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
642       (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 |  \
643        (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
644#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
645       (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 |  \
646        (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
647
648/* Raw profile format version (start from 1). */
649#define INSTR_PROF_RAW_VERSION 7
650/* Indexed profile format version (start from 1). */
651#define INSTR_PROF_INDEX_VERSION 7
652/* Coverage mapping format version (start from 0). */
653#define INSTR_PROF_COVMAP_VERSION 5
654
655/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
656 * version for other variants of profile. We set the lowest bit of the upper 8
657 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
658 * generated profile, and 0 if this is a Clang FE generated profile.
659 * 1 in bit 57 indicates there are context-sensitive records in the profile.
660 */
661#define VARIANT_MASKS_ALL 0xff00000000000000ULL
662#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
663#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
664#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
665#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
666#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
667#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
668#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
669
670/* The variable that holds the name of the profile data
671 * specified via command line. */
672#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
673
674/* section name strings common to all targets other
675   than WIN32 */
676#define INSTR_PROF_DATA_COMMON __llvm_prf_data
677#define INSTR_PROF_NAME_COMMON __llvm_prf_names
678#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
679#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
680#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
681#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
682#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
683#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
684/* Windows section names. Because these section names contain dollar characters,
685 * they must be quoted.
686 */
687#define INSTR_PROF_DATA_COFF ".lprfd$M"
688#define INSTR_PROF_NAME_COFF ".lprfn$M"
689#define INSTR_PROF_CNTS_COFF ".lprfc$M"
690#define INSTR_PROF_VALS_COFF ".lprfv$M"
691#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
692#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
693#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
694#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
695
696#ifdef _WIN32
697/* Runtime section names and name strings.  */
698#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
699#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
700#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
701/* Array of pointers. Each pointer points to a list
702 * of value nodes associated with one value site.
703 */
704#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
705/* Value profile nodes section. */
706#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
707#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
708#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
709#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
710#else
711/* Runtime section names and name strings.  */
712#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
713#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
714#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
715/* Array of pointers. Each pointer points to a list
716 * of value nodes associated with one value site.
717 */
718#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
719/* Value profile nodes section. */
720#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
721#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
722#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
723/* Order file instrumentation. */
724#define INSTR_PROF_ORDERFILE_SECT_NAME                                         \
725  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
726#endif
727
728#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
729#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR                                   \
730  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
731#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
732#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR                               \
733  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
734
735/* Macros to define start/stop section symbol for a given
736 * section on Linux. For instance
737 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
738 * expand to __start___llvm_prof_data
739 */
740#define INSTR_PROF_SECT_START(Sect) \
741        INSTR_PROF_CONCAT(__start_,Sect)
742#define INSTR_PROF_SECT_STOP(Sect) \
743        INSTR_PROF_CONCAT(__stop_,Sect)
744
745/* Value Profiling API linkage name.  */
746#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
747#define INSTR_PROF_VALUE_PROF_FUNC_STR \
748        INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
749#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
750#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR                                   \
751  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
752
753/* InstrProfile per-function control data alignment.  */
754#define INSTR_PROF_DATA_ALIGNMENT 8
755
756/* The data structure that represents a tracked value by the
757 * value profiler.
758 */
759typedef struct InstrProfValueData {
760  /* Profiled value. */
761  uint64_t Value;
762  /* Number of times the value appears in the training run. */
763  uint64_t Count;
764} InstrProfValueData;
765
766#endif /* INSTR_PROF_DATA_INC */
767
768#ifndef INSTR_ORDER_FILE_INC
769/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
770#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
771#define INSTR_ORDER_FILE_BUFFER_BITS 17
772#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
773#endif /* INSTR_ORDER_FILE_INC */
774#else
775#undef INSTR_PROF_DATA_DEFINED
776#endif
777
778#undef COVMAP_V2_OR_V3
779
780#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
781
782#ifdef __cplusplus
783#define INSTR_PROF_INLINE inline
784#else
785#define INSTR_PROF_INLINE
786#endif
787
788/* The value range buckets (22 buckets) for the memop size value profiling looks
789 * like:
790 *
791 *   [0, 0]
792 *   [1, 1]
793 *   [2, 2]
794 *   [3, 3]
795 *   [4, 4]
796 *   [5, 5]
797 *   [6, 6]
798 *   [7, 7]
799 *   [8, 8]
800 *   [9, 15]
801 *   [16, 16]
802 *   [17, 31]
803 *   [32, 32]
804 *   [33, 63]
805 *   [64, 64]
806 *   [65, 127]
807 *   [128, 128]
808 *   [129, 255]
809 *   [256, 256]
810 *   [257, 511]
811 *   [512, 512]
812 *   [513, UINT64_MAX]
813 *
814 * Each range has a 'representative value' which is the lower end value of the
815 * range and used to store in the runtime profile data records and the VP
816 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
817 */
818#define INSTR_PROF_NUM_BUCKETS 22
819
820/*
821 * Clz and Popcount. This code was copied from
822 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
823 * llvm/include/llvm/Support/MathExtras.h.
824 */
825#if defined(_MSC_VER) && !defined(__clang__)
826
827#include <intrin.h>
828INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
829int InstProfClzll(unsigned long long X) {
830  unsigned long LeadZeroIdx = 0;
831#if !defined(_M_ARM64) && !defined(_M_X64)
832  // Scan the high 32 bits.
833  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
834    return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
835                                                      // from the MSB.
836  // Scan the low 32 bits.
837  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
838    return (int)(63 - LeadZeroIdx);
839#else
840  if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
841#endif
842  return 64;
843}
844INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
845int InstProfPopcountll(unsigned long long X) {
846  // This code originates from https://reviews.llvm.org/rG30626254510f.
847  unsigned long long v = X;
848  v = v - ((v >> 1) & 0x5555555555555555ULL);
849  v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
850  v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
851  return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
852}
853
854#else
855
856INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
857int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
858INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
859int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
860
861#endif  /* defined(_MSC_VER) && !defined(__clang__) */
862
863/* Map an (observed) memop size value to the representative value of its range.
864 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
865INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
866InstrProfGetRangeRepValue(uint64_t Value) {
867  if (Value <= 8)
868    // The first ranges are individually tracked. Use the value as is.
869    return Value;
870  else if (Value >= 513)
871    // The last range is mapped to its lowest value.
872    return 513;
873  else if (InstProfPopcountll(Value) == 1)
874    // If it's a power of two, use it as is.
875    return Value;
876  else
877    // Otherwise, take to the previous power of two + 1.
878    return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
879}
880
881/* Return true if the range that an (observed) memop size value belongs to has
882 * only a single value in the range.  For example, 0 -> true, 8 -> true, 10 ->
883 * false, 64 -> true, 100 -> false, 513 -> false. */
884INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
885InstrProfIsSingleValRange(uint64_t Value) {
886  if (Value <= 8)
887    // The first ranges are individually tracked.
888    return 1;
889  else if (InstProfPopcountll(Value) == 1)
890    // If it's a power of two, there's only one value.
891    return 1;
892  else
893    // Otherwise, there's more than one value in the range.
894    return 0;
895}
896
897#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
898