1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2|* 3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4|* See https://llvm.org/LICENSE.txt for license information. 5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6|* 7\*===----------------------------------------------------------------------===*/ 8/* 9 * This is the main file that defines all the data structure, signature, 10 * constant literals that are shared across profiling runtime library, 11 * compiler (instrumentation), and host tools (reader/writer). The entities 12 * defined in this file affect the profile runtime ABI, the raw profile format, 13 * or both. 14 * 15 * The file has two identical copies. The primary copy lives in LLVM and 16 * the other one sits in compiler-rt/lib/profile directory. To make changes 17 * in this file, first modify the primary copy and copy it over to compiler-rt. 18 * Testing of any change in this file can start only after the two copies are 19 * synced up. 20 * 21 * The first part of the file includes macros that defines types, names, and 22 * initializers for the member fields of the core data structures. The field 23 * declarations for one structure is enabled by defining the field activation 24 * macro associated with that structure. Only one field activation record 25 * can be defined at one time and the rest definitions will be filtered out by 26 * the preprocessor. 27 * 28 * Examples of how the template is used to instantiate structure definition: 29 * 1. To declare a structure: 30 * 31 * struct ProfData { 32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 33 * Type Name; 34 * #include "llvm/ProfileData/InstrProfData.inc" 35 * }; 36 * 37 * 2. To construct LLVM type arrays for the struct type: 38 * 39 * Type *DataTypes[] = { 40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 41 * LLVMType, 42 * #include "llvm/ProfileData/InstrProfData.inc" 43 * }; 44 * 45 * 4. To construct constant array for the initializers: 46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 47 * Initializer, 48 * Constant *ConstantVals[] = { 49 * #include "llvm/ProfileData/InstrProfData.inc" 50 * }; 51 * 52 * 53 * The second part of the file includes definitions all other entities that 54 * are related to runtime ABI and format. When no field activation macro is 55 * defined, this file can be included to introduce the definitions. 56 * 57\*===----------------------------------------------------------------------===*/ 58 59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in 60 * the compiler runtime. */ 61#ifndef INSTR_PROF_VISIBILITY 62#define INSTR_PROF_VISIBILITY 63#endif 64 65/* INSTR_PROF_DATA start. */ 66/* Definition of member fields of the per-function control structure. */ 67#ifndef INSTR_PROF_DATA 68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 69#else 70#define INSTR_PROF_DATA_DEFINED 71#endif 72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) 75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 77 Inc->getHash()->getZExtValue())) 78INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) 79INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) 80/* This is used to map function pointers for the indirect call targets to 81 * function name hashes during the conversion from raw to merged profile 82 * data. 83 */ 84INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \ 85 FunctionAddr) 86INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \ 87 ValuesPtrExpr) 88INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 89 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 90INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 91 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ 92INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ 93 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) 94#undef INSTR_PROF_DATA 95/* INSTR_PROF_DATA end. */ 96 97 98/* This is an internal data structure used by value profiler. It 99 * is defined here to allow serialization code sharing by LLVM 100 * to be used in unit test. 101 * 102 * typedef struct ValueProfNode { 103 * // InstrProfValueData VData; 104 * uint64_t Value; 105 * uint64_t Count; 106 * struct ValueProfNode *Next; 107 * } ValueProfNode; 108 */ 109/* INSTR_PROF_VALUE_NODE start. */ 110#ifndef INSTR_PROF_VALUE_NODE 111#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) 112#else 113#define INSTR_PROF_DATA_DEFINED 114#endif 115INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ 116 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 117INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ 118 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 119INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \ 120 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) 121#undef INSTR_PROF_VALUE_NODE 122/* INSTR_PROF_VALUE_NODE end. */ 123 124/* INSTR_PROF_RAW_HEADER start */ 125/* Definition of member fields of the raw profile header data structure. */ 126#ifndef INSTR_PROF_RAW_HEADER 127#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 128#else 129#define INSTR_PROF_DATA_DEFINED 130#endif 131INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 132INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 133INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) 134INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) 135INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) 136INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) 137INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) 138INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) 139INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) 140INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 141INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, 142 (uintptr_t)CountersBegin - (uintptr_t)DataBegin) 143INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, 144 (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) 145INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 146INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 147#undef INSTR_PROF_RAW_HEADER 148/* INSTR_PROF_RAW_HEADER end */ 149 150/* VALUE_PROF_FUNC_PARAM start */ 151/* Definition of parameter types of the runtime API used to do value profiling 152 * for a given value site. 153 */ 154#ifndef VALUE_PROF_FUNC_PARAM 155#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 156#define INSTR_PROF_COMMA 157#else 158#define INSTR_PROF_DATA_DEFINED 159#define INSTR_PROF_COMMA , 160#endif 161VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 162 INSTR_PROF_COMMA 163VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA 164VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 165#undef VALUE_PROF_FUNC_PARAM 166#undef INSTR_PROF_COMMA 167/* VALUE_PROF_FUNC_PARAM end */ 168 169/* VALUE_PROF_KIND start */ 170#ifndef VALUE_PROF_KIND 171#define VALUE_PROF_KIND(Enumerator, Value, Descr) 172#else 173#define INSTR_PROF_DATA_DEFINED 174#endif 175/* For indirect function call value profiling, the addresses of the target 176 * functions are profiled by the instrumented code. The target addresses are 177 * written in the raw profile data and converted to target function name's MD5 178 * hash by the profile reader during deserialization. Typically, this happens 179 * when the raw profile data is read during profile merging. 180 * 181 * For this remapping the ProfData is used. ProfData contains both the function 182 * name hash and the function address. 183 */ 184VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") 185/* For memory intrinsic functions size profiling. */ 186VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") 187/* These two kinds must be the last to be 188 * declared. This is to make sure the string 189 * array created with the template can be 190 * indexed with the kind value. 191 */ 192VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") 193VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") 194 195#undef VALUE_PROF_KIND 196/* VALUE_PROF_KIND end */ 197 198#undef COVMAP_V2_OR_V3 199#ifdef COVMAP_V2 200#define COVMAP_V2_OR_V3 201#endif 202#ifdef COVMAP_V3 203#define COVMAP_V2_OR_V3 204#endif 205 206/* COVMAP_FUNC_RECORD start */ 207/* Definition of member fields of the function record structure in coverage 208 * map. 209 */ 210#ifndef COVMAP_FUNC_RECORD 211#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 212#else 213#define INSTR_PROF_DATA_DEFINED 214#endif 215#ifdef COVMAP_V1 216COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ 217 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 218 llvm::PointerType::getUnqual(Ctx))) 219COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 220 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 221 NameValue.size())) 222#endif 223#ifdef COVMAP_V2_OR_V3 224COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 225 llvm::ConstantInt::get( \ 226 llvm::Type::getInt64Ty(Ctx), NameHash)) 227#endif 228COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 229 llvm::ConstantInt::get( \ 230 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) 231COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 232 llvm::ConstantInt::get( \ 233 llvm::Type::getInt64Ty(Ctx), FuncHash)) 234#ifdef COVMAP_V3 235COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ 236 llvm::ConstantInt::get( \ 237 llvm::Type::getInt64Ty(Ctx), FilenamesRef)) 238COVMAP_FUNC_RECORD(const char, \ 239 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ 240 CoverageMapping.size()), \ 241 CoverageMapping, 242 llvm::ConstantDataArray::getRaw( \ 243 CoverageMapping, CoverageMapping.size(), \ 244 llvm::Type::getInt8Ty(Ctx))) 245#endif 246#undef COVMAP_FUNC_RECORD 247/* COVMAP_FUNC_RECORD end. */ 248 249/* COVMAP_HEADER start */ 250/* Definition of member fields of coverage map header. 251 */ 252#ifndef COVMAP_HEADER 253#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 254#else 255#define INSTR_PROF_DATA_DEFINED 256#endif 257COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 258 llvm::ConstantInt::get(Int32Ty, NRecords)) 259COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 260 llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 261COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 262 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 263COVMAP_HEADER(uint32_t, Int32Ty, Version, \ 264 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 265#undef COVMAP_HEADER 266/* COVMAP_HEADER end. */ 267 268 269#ifdef INSTR_PROF_SECT_ENTRY 270#define INSTR_PROF_DATA_DEFINED 271INSTR_PROF_SECT_ENTRY(IPSK_data, \ 272 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ 273 INSTR_PROF_DATA_COFF, "__DATA,") 274INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ 275 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ 276 INSTR_PROF_CNTS_COFF, "__DATA,") 277INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ 278 INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ 279 INSTR_PROF_BITS_COFF, "__DATA,") 280INSTR_PROF_SECT_ENTRY(IPSK_name, \ 281 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ 282 INSTR_PROF_NAME_COFF, "__DATA,") 283INSTR_PROF_SECT_ENTRY(IPSK_vals, \ 284 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ 285 INSTR_PROF_VALS_COFF, "__DATA,") 286INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ 287 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ 288 INSTR_PROF_VNODES_COFF, "__DATA,") 289INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ 290 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ 291 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") 292INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ 293 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ 294 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,") 295INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ 296 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ 297 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") 298INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ 299 INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ 300 INSTR_PROF_COVDATA_COFF, "__LLVM_COV,") 301INSTR_PROF_SECT_ENTRY(IPSK_covname, \ 302 INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ 303 INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") 304 305#undef INSTR_PROF_SECT_ENTRY 306#endif 307 308 309#ifdef INSTR_PROF_VALUE_PROF_DATA 310#define INSTR_PROF_DATA_DEFINED 311 312#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 313/*! 314 * This is the header of the data structure that defines the on-disk 315 * layout of the value profile data of a particular kind for one function. 316 */ 317typedef struct ValueProfRecord { 318 /* The kind of the value profile record. */ 319 uint32_t Kind; 320 /* 321 * The number of value profile sites. It is guaranteed to be non-zero; 322 * otherwise the record for this kind won't be emitted. 323 */ 324 uint32_t NumValueSites; 325 /* 326 * The first element of the array that stores the number of profiled 327 * values for each value site. The size of the array is NumValueSites. 328 * Since NumValueSites is greater than zero, there is at least one 329 * element in the array. 330 */ 331 uint8_t SiteCountArray[1]; 332 333 /* 334 * The fake declaration is for documentation purpose only. 335 * Align the start of next field to be on 8 byte boundaries. 336 uint8_t Padding[X]; 337 */ 338 339 /* The array of value profile data. The size of the array is the sum 340 * of all elements in SiteCountArray[]. 341 InstrProfValueData ValueData[]; 342 */ 343 344#ifdef __cplusplus 345 /*! 346 * Return the number of value sites. 347 */ 348 uint32_t getNumValueSites() const { return NumValueSites; } 349 /*! 350 * Read data from this record and save it to Record. 351 */ 352 void deserializeTo(InstrProfRecord &Record, 353 InstrProfSymtab *SymTab); 354 /* 355 * In-place byte swap: 356 * Do byte swap for this instance. \c Old is the original order before 357 * the swap, and \c New is the New byte order. 358 */ 359 void swapBytes(llvm::endianness Old, llvm::endianness New); 360#endif 361} ValueProfRecord; 362 363/*! 364 * Per-function header/control data structure for value profiling 365 * data in indexed format. 366 */ 367typedef struct ValueProfData { 368 /* 369 * Total size in bytes including this field. It must be a multiple 370 * of sizeof(uint64_t). 371 */ 372 uint32_t TotalSize; 373 /* 374 *The number of value profile kinds that has value profile data. 375 * In this implementation, a value profile kind is considered to 376 * have profile data if the number of value profile sites for the 377 * kind is not zero. More aggressively, the implementation can 378 * choose to check the actual data value: if none of the value sites 379 * has any profiled values, the kind can be skipped. 380 */ 381 uint32_t NumValueKinds; 382 383 /* 384 * Following are a sequence of variable length records. The prefix/header 385 * of each record is defined by ValueProfRecord type. The number of 386 * records is NumValueKinds. 387 * ValueProfRecord Record_1; 388 * ValueProfRecord Record_N; 389 */ 390 391#if __cplusplus 392 /*! 393 * Return the total size in bytes of the on-disk value profile data 394 * given the data stored in Record. 395 */ 396 static uint32_t getSize(const InstrProfRecord &Record); 397 /*! 398 * Return a pointer to \c ValueProfData instance ready to be streamed. 399 */ 400 static std::unique_ptr<ValueProfData> 401 serializeFrom(const InstrProfRecord &Record); 402 /*! 403 * Check the integrity of the record. 404 */ 405 Error checkIntegrity(); 406 /*! 407 * Return a pointer to \c ValueProfileData instance ready to be read. 408 * All data in the instance are properly byte swapped. The input 409 * data is assumed to be in little endian order. 410 */ 411 static Expected<std::unique_ptr<ValueProfData>> 412 getValueProfData(const unsigned char *SrcBuffer, 413 const unsigned char *const SrcBufferEnd, 414 llvm::endianness SrcDataEndianness); 415 /*! 416 * Swap byte order from \c Endianness order to host byte order. 417 */ 418 void swapBytesToHost(llvm::endianness Endianness); 419 /*! 420 * Swap byte order from host byte order to \c Endianness order. 421 */ 422 void swapBytesFromHost(llvm::endianness Endianness); 423 /*! 424 * Return the total size of \c ValueProfileData. 425 */ 426 uint32_t getSize() const { return TotalSize; } 427 /*! 428 * Read data from this data and save it to \c Record. 429 */ 430 void deserializeTo(InstrProfRecord &Record, 431 InstrProfSymtab *SymTab); 432 void operator delete(void *ptr) { ::operator delete(ptr); } 433#endif 434} ValueProfData; 435 436/* 437 * The closure is designed to abstact away two types of value profile data: 438 * - InstrProfRecord which is the primary data structure used to 439 * represent profile data in host tools (reader, writer, and profile-use) 440 * - value profile runtime data structure suitable to be used by C 441 * runtime library. 442 * 443 * Both sources of data need to serialize to disk/memory-buffer in common 444 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 445 * writer to share the same format and code with indexed profile writer. 446 * 447 * For documentation of the member methods below, refer to corresponding methods 448 * in class InstrProfRecord. 449 */ 450typedef struct ValueProfRecordClosure { 451 const void *Record; 452 uint32_t (*GetNumValueKinds)(const void *Record); 453 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 454 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 455 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 456 457 /* 458 * After extracting the value profile data from the value profile record, 459 * this method is used to map the in-memory value to on-disk value. If 460 * the method is null, value will be written out untranslated. 461 */ 462 uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 463 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 464 uint32_t S); 465 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 466} ValueProfRecordClosure; 467 468INSTR_PROF_VISIBILITY ValueProfRecord * 469getFirstValueProfRecord(ValueProfData *VPD); 470INSTR_PROF_VISIBILITY ValueProfRecord * 471getValueProfRecordNext(ValueProfRecord *VPR); 472INSTR_PROF_VISIBILITY InstrProfValueData * 473getValueProfRecordValueData(ValueProfRecord *VPR); 474INSTR_PROF_VISIBILITY uint32_t 475getValueProfRecordHeaderSize(uint32_t NumValueSites); 476 477#undef INSTR_PROF_VALUE_PROF_DATA 478#endif /* INSTR_PROF_VALUE_PROF_DATA */ 479 480 481#ifdef INSTR_PROF_COMMON_API_IMPL 482#define INSTR_PROF_DATA_DEFINED 483#ifdef __cplusplus 484#define INSTR_PROF_INLINE inline 485#define INSTR_PROF_NULLPTR nullptr 486#else 487#define INSTR_PROF_INLINE 488#define INSTR_PROF_NULLPTR NULL 489#endif 490 491#ifndef offsetof 492#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 493#endif 494 495/*! 496 * Return the \c ValueProfRecord header size including the 497 * padding bytes. 498 */ 499INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 500uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 501 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 502 sizeof(uint8_t) * NumValueSites; 503 /* Round the size to multiple of 8 bytes. */ 504 Size = (Size + 7) & ~7; 505 return Size; 506} 507 508/*! 509 * Return the total size of the value profile record including the 510 * header and the value data. 511 */ 512INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 513uint32_t getValueProfRecordSize(uint32_t NumValueSites, 514 uint32_t NumValueData) { 515 return getValueProfRecordHeaderSize(NumValueSites) + 516 sizeof(InstrProfValueData) * NumValueData; 517} 518 519/*! 520 * Return the pointer to the start of value data array. 521 */ 522INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 523InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 524 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 525 This->NumValueSites)); 526} 527 528/*! 529 * Return the total number of value data for \c This record. 530 */ 531INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 532uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 533 uint32_t NumValueData = 0; 534 uint32_t I; 535 for (I = 0; I < This->NumValueSites; I++) 536 NumValueData += This->SiteCountArray[I]; 537 return NumValueData; 538} 539 540/*! 541 * Use this method to advance to the next \c This \c ValueProfRecord. 542 */ 543INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 544ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 545 uint32_t NumValueData = getValueProfRecordNumValueData(This); 546 return (ValueProfRecord *)((char *)This + 547 getValueProfRecordSize(This->NumValueSites, 548 NumValueData)); 549} 550 551/*! 552 * Return the first \c ValueProfRecord instance. 553 */ 554INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 555ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 556 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 557} 558 559/* Closure based interfaces. */ 560 561/*! 562 * Return the total size in bytes of the on-disk value profile data 563 * given the data stored in Record. 564 */ 565INSTR_PROF_VISIBILITY uint32_t 566getValueProfDataSize(ValueProfRecordClosure *Closure) { 567 uint32_t Kind; 568 uint32_t TotalSize = sizeof(ValueProfData); 569 const void *Record = Closure->Record; 570 571 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 572 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 573 if (!NumValueSites) 574 continue; 575 TotalSize += getValueProfRecordSize(NumValueSites, 576 Closure->GetNumValueData(Record, Kind)); 577 } 578 return TotalSize; 579} 580 581/*! 582 * Extract value profile data of a function for the profile kind \c ValueKind 583 * from the \c Closure and serialize the data into \c This record instance. 584 */ 585INSTR_PROF_VISIBILITY void 586serializeValueProfRecordFrom(ValueProfRecord *This, 587 ValueProfRecordClosure *Closure, 588 uint32_t ValueKind, uint32_t NumValueSites) { 589 uint32_t S; 590 const void *Record = Closure->Record; 591 This->Kind = ValueKind; 592 This->NumValueSites = NumValueSites; 593 InstrProfValueData *DstVD = getValueProfRecordValueData(This); 594 595 for (S = 0; S < NumValueSites; S++) { 596 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 597 This->SiteCountArray[S] = ND; 598 Closure->GetValueForSite(Record, DstVD, ValueKind, S); 599 DstVD += ND; 600 } 601} 602 603/*! 604 * Extract value profile data of a function from the \c Closure 605 * and serialize the data into \c DstData if it is not NULL or heap 606 * memory allocated by the \c Closure's allocator method. If \c 607 * DstData is not null, the caller is expected to set the TotalSize 608 * in DstData. 609 */ 610INSTR_PROF_VISIBILITY ValueProfData * 611serializeValueProfDataFrom(ValueProfRecordClosure *Closure, 612 ValueProfData *DstData) { 613 uint32_t Kind; 614 uint32_t TotalSize = 615 DstData ? DstData->TotalSize : getValueProfDataSize(Closure); 616 617 ValueProfData *VPD = 618 DstData ? DstData : Closure->AllocValueProfData(TotalSize); 619 620 VPD->TotalSize = TotalSize; 621 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 622 ValueProfRecord *VR = getFirstValueProfRecord(VPD); 623 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 624 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 625 if (!NumValueSites) 626 continue; 627 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 628 VR = getValueProfRecordNext(VR); 629 } 630 return VPD; 631} 632 633#undef INSTR_PROF_COMMON_API_IMPL 634#endif /* INSTR_PROF_COMMON_API_IMPL */ 635 636/*============================================================================*/ 637 638#ifndef INSTR_PROF_DATA_DEFINED 639 640#ifndef INSTR_PROF_DATA_INC 641#define INSTR_PROF_DATA_INC 642 643/* Helper macros. */ 644#define INSTR_PROF_SIMPLE_QUOTE(x) #x 645#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 646#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 647#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 648 649/* Magic number to detect file format and endianness. 650 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 651 * so that utilities, like strings, don't grab it as a string. 129 is also 652 * invalid UTF-8, and high enough to be interesting. 653 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 654 * for 32-bit platforms. 655 */ 656#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 657 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 658 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 659#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 660 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 661 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 662 663/* Raw profile format version (start from 1). */ 664#define INSTR_PROF_RAW_VERSION 9 665/* Indexed profile format version (start from 1). */ 666#define INSTR_PROF_INDEX_VERSION 11 667/* Coverage mapping format version (start from 0). */ 668#define INSTR_PROF_COVMAP_VERSION 6 669 670/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the 671 * version for other variants of profile. We set the 8th most significant bit 672 * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation 673 * generated profile, and 0 if this is a Clang FE generated profile. 674 * 1 in bit 57 indicates there are context-sensitive records in the profile. 675 * The 59th bit indicates whether to use debug info to correlate profiles. 676 * The 60th bit indicates single byte coverage instrumentation. 677 * The 61st bit indicates function entry instrumentation only. 678 * The 62nd bit indicates whether memory profile information is present. 679 * The 63rd bit indicates if this is a temporal profile. 680 */ 681#define VARIANT_MASKS_ALL 0xffffffff00000000ULL 682#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 683#define VARIANT_MASK_IR_PROF (0x1ULL << 56) 684#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) 685#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) 686#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59) 687#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60) 688#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61) 689#define VARIANT_MASK_MEMPROF (0x1ULL << 62) 690#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63) 691#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version 692#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime 693#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias 694#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp 695 696/* The variable that holds the name of the profile data 697 * specified via command line. */ 698#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename 699 700/* section name strings common to all targets other 701 than WIN32 */ 702#define INSTR_PROF_DATA_COMMON __llvm_prf_data 703#define INSTR_PROF_NAME_COMMON __llvm_prf_names 704#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts 705#define INSTR_PROF_BITS_COMMON __llvm_prf_bits 706#define INSTR_PROF_VALS_COMMON __llvm_prf_vals 707#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds 708#define INSTR_PROF_COVMAP_COMMON __llvm_covmap 709#define INSTR_PROF_COVFUN_COMMON __llvm_covfun 710#define INSTR_PROF_COVDATA_COMMON __llvm_covdata 711#define INSTR_PROF_COVNAME_COMMON __llvm_covnames 712#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile 713/* Windows section names. Because these section names contain dollar characters, 714 * they must be quoted. 715 */ 716#define INSTR_PROF_DATA_COFF ".lprfd$M" 717#define INSTR_PROF_NAME_COFF ".lprfn$M" 718#define INSTR_PROF_CNTS_COFF ".lprfc$M" 719#define INSTR_PROF_BITS_COFF ".lprfb$M" 720#define INSTR_PROF_VALS_COFF ".lprfv$M" 721#define INSTR_PROF_VNODES_COFF ".lprfnd$M" 722#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" 723#define INSTR_PROF_COVFUN_COFF ".lcovfun$M" 724/* Since cov data and cov names sections are not allocated, we don't need to 725 * access them at runtime. 726 */ 727#define INSTR_PROF_COVDATA_COFF ".lcovd" 728#define INSTR_PROF_COVNAME_COFF ".lcovn" 729#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" 730 731#ifdef _WIN32 732/* Runtime section names and name strings. */ 733#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF 734#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF 735#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF 736#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF 737/* Array of pointers. Each pointer points to a list 738 * of value nodes associated with one value site. 739 */ 740#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF 741/* Value profile nodes section. */ 742#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF 743#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF 744#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF 745#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF 746#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF 747#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF 748#else 749/* Runtime section names and name strings. */ 750#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) 751#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) 752#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) 753#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) 754/* Array of pointers. Each pointer points to a list 755 * of value nodes associated with one value site. 756 */ 757#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) 758/* Value profile nodes section. */ 759#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) 760#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) 761#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) 762#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) 763#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) 764/* Order file instrumentation. */ 765#define INSTR_PROF_ORDERFILE_SECT_NAME \ 766 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) 767#endif 768 769#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer 770#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ 771 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) 772#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx 773#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ 774 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) 775 776/* Macros to define start/stop section symbol for a given 777 * section on Linux. For instance 778 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 779 * expand to __start___llvm_prof_data 780 */ 781#define INSTR_PROF_SECT_START(Sect) \ 782 INSTR_PROF_CONCAT(__start_,Sect) 783#define INSTR_PROF_SECT_STOP(Sect) \ 784 INSTR_PROF_CONCAT(__stop_,Sect) 785 786/* Value Profiling API linkage name. */ 787#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 788#define INSTR_PROF_VALUE_PROF_FUNC_STR \ 789 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 790#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop 791#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ 792 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) 793 794/* InstrProfile per-function control data alignment. */ 795#define INSTR_PROF_DATA_ALIGNMENT 8 796 797/* The data structure that represents a tracked value by the 798 * value profiler. 799 */ 800typedef struct InstrProfValueData { 801 /* Profiled value. */ 802 uint64_t Value; 803 /* Number of times the value appears in the training run. */ 804 uint64_t Count; 805} InstrProfValueData; 806 807#endif /* INSTR_PROF_DATA_INC */ 808 809#ifndef INSTR_ORDER_FILE_INC 810/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ 811#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 812#define INSTR_ORDER_FILE_BUFFER_BITS 17 813#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff 814#endif /* INSTR_ORDER_FILE_INC */ 815#else 816#undef INSTR_PROF_DATA_DEFINED 817#endif 818 819#undef COVMAP_V2_OR_V3 820 821#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API 822 823#ifdef __cplusplus 824#define INSTR_PROF_INLINE inline 825#else 826#define INSTR_PROF_INLINE 827#endif 828 829/* The value range buckets (22 buckets) for the memop size value profiling looks 830 * like: 831 * 832 * [0, 0] 833 * [1, 1] 834 * [2, 2] 835 * [3, 3] 836 * [4, 4] 837 * [5, 5] 838 * [6, 6] 839 * [7, 7] 840 * [8, 8] 841 * [9, 15] 842 * [16, 16] 843 * [17, 31] 844 * [32, 32] 845 * [33, 63] 846 * [64, 64] 847 * [65, 127] 848 * [128, 128] 849 * [129, 255] 850 * [256, 256] 851 * [257, 511] 852 * [512, 512] 853 * [513, UINT64_MAX] 854 * 855 * Each range has a 'representative value' which is the lower end value of the 856 * range and used to store in the runtime profile data records and the VP 857 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. 858 */ 859#define INSTR_PROF_NUM_BUCKETS 22 860 861/* 862 * Clz and Popcount. This code was copied from 863 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and 864 * llvm/include/llvm/Support/MathExtras.h. 865 */ 866#if defined(_MSC_VER) && !defined(__clang__) 867 868#include <intrin.h> 869INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 870int InstProfClzll(unsigned long long X) { 871 unsigned long LeadZeroIdx = 0; 872#if !defined(_M_ARM64) && !defined(_M_X64) 873 // Scan the high 32 bits. 874 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) 875 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset 876 // from the MSB. 877 // Scan the low 32 bits. 878 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) 879 return (int)(63 - LeadZeroIdx); 880#else 881 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; 882#endif 883 return 64; 884} 885INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 886int InstProfPopcountll(unsigned long long X) { 887 // This code originates from https://reviews.llvm.org/rG30626254510f. 888 unsigned long long v = X; 889 v = v - ((v >> 1) & 0x5555555555555555ULL); 890 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); 891 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; 892 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); 893} 894 895#else 896 897INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 898int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } 899INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 900int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } 901 902#endif /* defined(_MSC_VER) && !defined(__clang__) */ 903 904/* Map an (observed) memop size value to the representative value of its range. 905 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ 906INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t 907InstrProfGetRangeRepValue(uint64_t Value) { 908 if (Value <= 8) 909 // The first ranges are individually tracked. Use the value as is. 910 return Value; 911 else if (Value >= 513) 912 // The last range is mapped to its lowest value. 913 return 513; 914 else if (InstProfPopcountll(Value) == 1) 915 // If it's a power of two, use it as is. 916 return Value; 917 else 918 // Otherwise, take to the previous power of two + 1. 919 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1; 920} 921 922/* Return true if the range that an (observed) memop size value belongs to has 923 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> 924 * false, 64 -> true, 100 -> false, 513 -> false. */ 925INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned 926InstrProfIsSingleValRange(uint64_t Value) { 927 if (Value <= 8) 928 // The first ranges are individually tracked. 929 return 1; 930 else if (InstProfPopcountll(Value) == 1) 931 // If it's a power of two, there's only one value. 932 return 1; 933 else 934 // Otherwise, there's more than one value in the range. 935 return 0; 936} 937 938#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ 939