1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2|* 3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4|* See https://llvm.org/LICENSE.txt for license information. 5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6|* 7\*===----------------------------------------------------------------------===*/ 8/* 9 * This is the main file that defines all the data structure, signature, 10 * constant literals that are shared across profiling runtime library, 11 * compiler (instrumentation), and host tools (reader/writer). The entities 12 * defined in this file affect the profile runtime ABI, the raw profile format, 13 * or both. 14 * 15 * The file has two identical copies. The primary copy lives in LLVM and 16 * the other one sits in compiler-rt/lib/profile directory. To make changes 17 * in this file, first modify the primary copy and copy it over to compiler-rt. 18 * Testing of any change in this file can start only after the two copies are 19 * synced up. 20 * 21 * The first part of the file includes macros that defines types, names, and 22 * initializers for the member fields of the core data structures. The field 23 * declarations for one structure is enabled by defining the field activation 24 * macro associated with that structure. Only one field activation record 25 * can be defined at one time and the rest definitions will be filtered out by 26 * the preprocessor. 27 * 28 * Examples of how the template is used to instantiate structure definition: 29 * 1. To declare a structure: 30 * 31 * struct ProfData { 32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 33 * Type Name; 34 * #include "llvm/ProfileData/InstrProfData.inc" 35 * }; 36 * 37 * 2. To construct LLVM type arrays for the struct type: 38 * 39 * Type *DataTypes[] = { 40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 41 * LLVMType, 42 * #include "llvm/ProfileData/InstrProfData.inc" 43 * }; 44 * 45 * 4. To construct constant array for the initializers: 46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 47 * Initializer, 48 * Constant *ConstantVals[] = { 49 * #include "llvm/ProfileData/InstrProfData.inc" 50 * }; 51 * 52 * 53 * The second part of the file includes definitions all other entities that 54 * are related to runtime ABI and format. When no field activation macro is 55 * defined, this file can be included to introduce the definitions. 56 * 57\*===----------------------------------------------------------------------===*/ 58 59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in 60 * the compiler runtime. */ 61#ifndef INSTR_PROF_VISIBILITY 62#define INSTR_PROF_VISIBILITY 63#endif 64 65/* INSTR_PROF_DATA start. */ 66/* Definition of member fields of the per-function control structure. */ 67#ifndef INSTR_PROF_DATA 68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 69#else 70#define INSTR_PROF_DATA_DEFINED 71#endif 72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) 75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 77 Inc->getHash()->getZExtValue())) 78INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) 79/* This is used to map function pointers for the indirect call targets to 80 * function name hashes during the conversion from raw to merged profile 81 * data. 82 */ 83INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ 84 FunctionAddr) 85INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ 86 ValuesPtrExpr) 87INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 88 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 89INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 90 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) 91#undef INSTR_PROF_DATA 92/* INSTR_PROF_DATA end. */ 93 94 95/* This is an internal data structure used by value profiler. It 96 * is defined here to allow serialization code sharing by LLVM 97 * to be used in unit test. 98 * 99 * typedef struct ValueProfNode { 100 * // InstrProfValueData VData; 101 * uint64_t Value; 102 * uint64_t Count; 103 * struct ValueProfNode *Next; 104 * } ValueProfNode; 105 */ 106/* INSTR_PROF_VALUE_NODE start. */ 107#ifndef INSTR_PROF_VALUE_NODE 108#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) 109#else 110#define INSTR_PROF_DATA_DEFINED 111#endif 112INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ 113 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 114INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ 115 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 116INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ 117 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) 118#undef INSTR_PROF_VALUE_NODE 119/* INSTR_PROF_VALUE_NODE end. */ 120 121/* INSTR_PROF_RAW_HEADER start */ 122/* Definition of member fields of the raw profile header data structure. */ 123#ifndef INSTR_PROF_RAW_HEADER 124#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 125#else 126#define INSTR_PROF_DATA_DEFINED 127#endif 128INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 129INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 130INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) 131/* FIXME: A more accurate name is NumData */ 132INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) 133INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) 134/* FIXME: A more accurate name is NumCounters */ 135INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) 136INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) 137INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 138INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, 139 (uintptr_t)CountersBegin - (uintptr_t)DataBegin) 140INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 141INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 142#undef INSTR_PROF_RAW_HEADER 143/* INSTR_PROF_RAW_HEADER end */ 144 145/* VALUE_PROF_FUNC_PARAM start */ 146/* Definition of parameter types of the runtime API used to do value profiling 147 * for a given value site. 148 */ 149#ifndef VALUE_PROF_FUNC_PARAM 150#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 151#define INSTR_PROF_COMMA 152#else 153#define INSTR_PROF_DATA_DEFINED 154#define INSTR_PROF_COMMA , 155#endif 156VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 157 INSTR_PROF_COMMA 158VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA 159VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 160#undef VALUE_PROF_FUNC_PARAM 161#undef INSTR_PROF_COMMA 162/* VALUE_PROF_FUNC_PARAM end */ 163 164/* VALUE_PROF_KIND start */ 165#ifndef VALUE_PROF_KIND 166#define VALUE_PROF_KIND(Enumerator, Value, Descr) 167#else 168#define INSTR_PROF_DATA_DEFINED 169#endif 170/* For indirect function call value profiling, the addresses of the target 171 * functions are profiled by the instrumented code. The target addresses are 172 * written in the raw profile data and converted to target function name's MD5 173 * hash by the profile reader during deserialization. Typically, this happens 174 * when the raw profile data is read during profile merging. 175 * 176 * For this remapping the ProfData is used. ProfData contains both the function 177 * name hash and the function address. 178 */ 179VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") 180/* For memory intrinsic functions size profiling. */ 181VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") 182/* These two kinds must be the last to be 183 * declared. This is to make sure the string 184 * array created with the template can be 185 * indexed with the kind value. 186 */ 187VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") 188VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") 189 190#undef VALUE_PROF_KIND 191/* VALUE_PROF_KIND end */ 192 193#undef COVMAP_V2_OR_V3 194#ifdef COVMAP_V2 195#define COVMAP_V2_OR_V3 196#endif 197#ifdef COVMAP_V3 198#define COVMAP_V2_OR_V3 199#endif 200 201/* COVMAP_FUNC_RECORD start */ 202/* Definition of member fields of the function record structure in coverage 203 * map. 204 */ 205#ifndef COVMAP_FUNC_RECORD 206#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 207#else 208#define INSTR_PROF_DATA_DEFINED 209#endif 210#ifdef COVMAP_V1 211COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ 212 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 213 llvm::Type::getInt8PtrTy(Ctx))) 214COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 215 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 216 NameValue.size())) 217#endif 218#ifdef COVMAP_V2_OR_V3 219COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 220 llvm::ConstantInt::get( \ 221 llvm::Type::getInt64Ty(Ctx), NameHash)) 222#endif 223COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 224 llvm::ConstantInt::get( \ 225 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) 226COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 227 llvm::ConstantInt::get( \ 228 llvm::Type::getInt64Ty(Ctx), FuncHash)) 229#ifdef COVMAP_V3 230COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ 231 llvm::ConstantInt::get( \ 232 llvm::Type::getInt64Ty(Ctx), FilenamesRef)) 233COVMAP_FUNC_RECORD(const char, \ 234 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ 235 CoverageMapping.size()), \ 236 CoverageMapping, 237 llvm::ConstantDataArray::getRaw( \ 238 CoverageMapping, CoverageMapping.size(), \ 239 llvm::Type::getInt8Ty(Ctx))) 240#endif 241#undef COVMAP_FUNC_RECORD 242/* COVMAP_FUNC_RECORD end. */ 243 244/* COVMAP_HEADER start */ 245/* Definition of member fields of coverage map header. 246 */ 247#ifndef COVMAP_HEADER 248#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 249#else 250#define INSTR_PROF_DATA_DEFINED 251#endif 252COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 253 llvm::ConstantInt::get(Int32Ty, NRecords)) 254COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 255 llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 256COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 257 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 258COVMAP_HEADER(uint32_t, Int32Ty, Version, \ 259 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 260#undef COVMAP_HEADER 261/* COVMAP_HEADER end. */ 262 263 264#ifdef INSTR_PROF_SECT_ENTRY 265#define INSTR_PROF_DATA_DEFINED 266INSTR_PROF_SECT_ENTRY(IPSK_data, \ 267 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ 268 INSTR_PROF_DATA_COFF, "__DATA,") 269INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ 270 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ 271 INSTR_PROF_CNTS_COFF, "__DATA,") 272INSTR_PROF_SECT_ENTRY(IPSK_name, \ 273 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ 274 INSTR_PROF_NAME_COFF, "__DATA,") 275INSTR_PROF_SECT_ENTRY(IPSK_vals, \ 276 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ 277 INSTR_PROF_VALS_COFF, "__DATA,") 278INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ 279 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ 280 INSTR_PROF_VNODES_COFF, "__DATA,") 281INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ 282 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ 283 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") 284INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ 285 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ 286 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,") 287INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ 288 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ 289 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") 290 291#undef INSTR_PROF_SECT_ENTRY 292#endif 293 294 295#ifdef INSTR_PROF_VALUE_PROF_DATA 296#define INSTR_PROF_DATA_DEFINED 297 298#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 299/*! 300 * This is the header of the data structure that defines the on-disk 301 * layout of the value profile data of a particular kind for one function. 302 */ 303typedef struct ValueProfRecord { 304 /* The kind of the value profile record. */ 305 uint32_t Kind; 306 /* 307 * The number of value profile sites. It is guaranteed to be non-zero; 308 * otherwise the record for this kind won't be emitted. 309 */ 310 uint32_t NumValueSites; 311 /* 312 * The first element of the array that stores the number of profiled 313 * values for each value site. The size of the array is NumValueSites. 314 * Since NumValueSites is greater than zero, there is at least one 315 * element in the array. 316 */ 317 uint8_t SiteCountArray[1]; 318 319 /* 320 * The fake declaration is for documentation purpose only. 321 * Align the start of next field to be on 8 byte boundaries. 322 uint8_t Padding[X]; 323 */ 324 325 /* The array of value profile data. The size of the array is the sum 326 * of all elements in SiteCountArray[]. 327 InstrProfValueData ValueData[]; 328 */ 329 330#ifdef __cplusplus 331 /*! 332 * Return the number of value sites. 333 */ 334 uint32_t getNumValueSites() const { return NumValueSites; } 335 /*! 336 * Read data from this record and save it to Record. 337 */ 338 void deserializeTo(InstrProfRecord &Record, 339 InstrProfSymtab *SymTab); 340 /* 341 * In-place byte swap: 342 * Do byte swap for this instance. \c Old is the original order before 343 * the swap, and \c New is the New byte order. 344 */ 345 void swapBytes(support::endianness Old, support::endianness New); 346#endif 347} ValueProfRecord; 348 349/*! 350 * Per-function header/control data structure for value profiling 351 * data in indexed format. 352 */ 353typedef struct ValueProfData { 354 /* 355 * Total size in bytes including this field. It must be a multiple 356 * of sizeof(uint64_t). 357 */ 358 uint32_t TotalSize; 359 /* 360 *The number of value profile kinds that has value profile data. 361 * In this implementation, a value profile kind is considered to 362 * have profile data if the number of value profile sites for the 363 * kind is not zero. More aggressively, the implementation can 364 * choose to check the actual data value: if none of the value sites 365 * has any profiled values, the kind can be skipped. 366 */ 367 uint32_t NumValueKinds; 368 369 /* 370 * Following are a sequence of variable length records. The prefix/header 371 * of each record is defined by ValueProfRecord type. The number of 372 * records is NumValueKinds. 373 * ValueProfRecord Record_1; 374 * ValueProfRecord Record_N; 375 */ 376 377#if __cplusplus 378 /*! 379 * Return the total size in bytes of the on-disk value profile data 380 * given the data stored in Record. 381 */ 382 static uint32_t getSize(const InstrProfRecord &Record); 383 /*! 384 * Return a pointer to \c ValueProfData instance ready to be streamed. 385 */ 386 static std::unique_ptr<ValueProfData> 387 serializeFrom(const InstrProfRecord &Record); 388 /*! 389 * Check the integrity of the record. 390 */ 391 Error checkIntegrity(); 392 /*! 393 * Return a pointer to \c ValueProfileData instance ready to be read. 394 * All data in the instance are properly byte swapped. The input 395 * data is assumed to be in little endian order. 396 */ 397 static Expected<std::unique_ptr<ValueProfData>> 398 getValueProfData(const unsigned char *SrcBuffer, 399 const unsigned char *const SrcBufferEnd, 400 support::endianness SrcDataEndianness); 401 /*! 402 * Swap byte order from \c Endianness order to host byte order. 403 */ 404 void swapBytesToHost(support::endianness Endianness); 405 /*! 406 * Swap byte order from host byte order to \c Endianness order. 407 */ 408 void swapBytesFromHost(support::endianness Endianness); 409 /*! 410 * Return the total size of \c ValueProfileData. 411 */ 412 uint32_t getSize() const { return TotalSize; } 413 /*! 414 * Read data from this data and save it to \c Record. 415 */ 416 void deserializeTo(InstrProfRecord &Record, 417 InstrProfSymtab *SymTab); 418 void operator delete(void *ptr) { ::operator delete(ptr); } 419#endif 420} ValueProfData; 421 422/* 423 * The closure is designed to abstact away two types of value profile data: 424 * - InstrProfRecord which is the primary data structure used to 425 * represent profile data in host tools (reader, writer, and profile-use) 426 * - value profile runtime data structure suitable to be used by C 427 * runtime library. 428 * 429 * Both sources of data need to serialize to disk/memory-buffer in common 430 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 431 * writer to share the same format and code with indexed profile writer. 432 * 433 * For documentation of the member methods below, refer to corresponding methods 434 * in class InstrProfRecord. 435 */ 436typedef struct ValueProfRecordClosure { 437 const void *Record; 438 uint32_t (*GetNumValueKinds)(const void *Record); 439 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 440 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 441 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 442 443 /* 444 * After extracting the value profile data from the value profile record, 445 * this method is used to map the in-memory value to on-disk value. If 446 * the method is null, value will be written out untranslated. 447 */ 448 uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 449 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 450 uint32_t S); 451 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 452} ValueProfRecordClosure; 453 454INSTR_PROF_VISIBILITY ValueProfRecord * 455getFirstValueProfRecord(ValueProfData *VPD); 456INSTR_PROF_VISIBILITY ValueProfRecord * 457getValueProfRecordNext(ValueProfRecord *VPR); 458INSTR_PROF_VISIBILITY InstrProfValueData * 459getValueProfRecordValueData(ValueProfRecord *VPR); 460INSTR_PROF_VISIBILITY uint32_t 461getValueProfRecordHeaderSize(uint32_t NumValueSites); 462 463#undef INSTR_PROF_VALUE_PROF_DATA 464#endif /* INSTR_PROF_VALUE_PROF_DATA */ 465 466 467#ifdef INSTR_PROF_COMMON_API_IMPL 468#define INSTR_PROF_DATA_DEFINED 469#ifdef __cplusplus 470#define INSTR_PROF_INLINE inline 471#define INSTR_PROF_NULLPTR nullptr 472#else 473#define INSTR_PROF_INLINE 474#define INSTR_PROF_NULLPTR NULL 475#endif 476 477#ifndef offsetof 478#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 479#endif 480 481/*! 482 * Return the \c ValueProfRecord header size including the 483 * padding bytes. 484 */ 485INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 486uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 487 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 488 sizeof(uint8_t) * NumValueSites; 489 /* Round the size to multiple of 8 bytes. */ 490 Size = (Size + 7) & ~7; 491 return Size; 492} 493 494/*! 495 * Return the total size of the value profile record including the 496 * header and the value data. 497 */ 498INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 499uint32_t getValueProfRecordSize(uint32_t NumValueSites, 500 uint32_t NumValueData) { 501 return getValueProfRecordHeaderSize(NumValueSites) + 502 sizeof(InstrProfValueData) * NumValueData; 503} 504 505/*! 506 * Return the pointer to the start of value data array. 507 */ 508INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 509InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 510 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 511 This->NumValueSites)); 512} 513 514/*! 515 * Return the total number of value data for \c This record. 516 */ 517INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 518uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 519 uint32_t NumValueData = 0; 520 uint32_t I; 521 for (I = 0; I < This->NumValueSites; I++) 522 NumValueData += This->SiteCountArray[I]; 523 return NumValueData; 524} 525 526/*! 527 * Use this method to advance to the next \c This \c ValueProfRecord. 528 */ 529INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 530ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 531 uint32_t NumValueData = getValueProfRecordNumValueData(This); 532 return (ValueProfRecord *)((char *)This + 533 getValueProfRecordSize(This->NumValueSites, 534 NumValueData)); 535} 536 537/*! 538 * Return the first \c ValueProfRecord instance. 539 */ 540INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 541ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 542 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 543} 544 545/* Closure based interfaces. */ 546 547/*! 548 * Return the total size in bytes of the on-disk value profile data 549 * given the data stored in Record. 550 */ 551INSTR_PROF_VISIBILITY uint32_t 552getValueProfDataSize(ValueProfRecordClosure *Closure) { 553 uint32_t Kind; 554 uint32_t TotalSize = sizeof(ValueProfData); 555 const void *Record = Closure->Record; 556 557 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 558 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 559 if (!NumValueSites) 560 continue; 561 TotalSize += getValueProfRecordSize(NumValueSites, 562 Closure->GetNumValueData(Record, Kind)); 563 } 564 return TotalSize; 565} 566 567/*! 568 * Extract value profile data of a function for the profile kind \c ValueKind 569 * from the \c Closure and serialize the data into \c This record instance. 570 */ 571INSTR_PROF_VISIBILITY void 572serializeValueProfRecordFrom(ValueProfRecord *This, 573 ValueProfRecordClosure *Closure, 574 uint32_t ValueKind, uint32_t NumValueSites) { 575 uint32_t S; 576 const void *Record = Closure->Record; 577 This->Kind = ValueKind; 578 This->NumValueSites = NumValueSites; 579 InstrProfValueData *DstVD = getValueProfRecordValueData(This); 580 581 for (S = 0; S < NumValueSites; S++) { 582 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 583 This->SiteCountArray[S] = ND; 584 Closure->GetValueForSite(Record, DstVD, ValueKind, S); 585 DstVD += ND; 586 } 587} 588 589/*! 590 * Extract value profile data of a function from the \c Closure 591 * and serialize the data into \c DstData if it is not NULL or heap 592 * memory allocated by the \c Closure's allocator method. If \c 593 * DstData is not null, the caller is expected to set the TotalSize 594 * in DstData. 595 */ 596INSTR_PROF_VISIBILITY ValueProfData * 597serializeValueProfDataFrom(ValueProfRecordClosure *Closure, 598 ValueProfData *DstData) { 599 uint32_t Kind; 600 uint32_t TotalSize = 601 DstData ? DstData->TotalSize : getValueProfDataSize(Closure); 602 603 ValueProfData *VPD = 604 DstData ? DstData : Closure->AllocValueProfData(TotalSize); 605 606 VPD->TotalSize = TotalSize; 607 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 608 ValueProfRecord *VR = getFirstValueProfRecord(VPD); 609 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 610 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 611 if (!NumValueSites) 612 continue; 613 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 614 VR = getValueProfRecordNext(VR); 615 } 616 return VPD; 617} 618 619#undef INSTR_PROF_COMMON_API_IMPL 620#endif /* INSTR_PROF_COMMON_API_IMPL */ 621 622/*============================================================================*/ 623 624#ifndef INSTR_PROF_DATA_DEFINED 625 626#ifndef INSTR_PROF_DATA_INC 627#define INSTR_PROF_DATA_INC 628 629/* Helper macros. */ 630#define INSTR_PROF_SIMPLE_QUOTE(x) #x 631#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 632#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 633#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 634 635/* Magic number to detect file format and endianness. 636 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 637 * so that utilities, like strings, don't grab it as a string. 129 is also 638 * invalid UTF-8, and high enough to be interesting. 639 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 640 * for 32-bit platforms. 641 */ 642#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 643 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 644 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 645#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 646 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 647 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 648 649/* FIXME: Please remedy the fixme in the header before bumping the version. */ 650/* Raw profile format version (start from 1). */ 651#define INSTR_PROF_RAW_VERSION 8 652/* Indexed profile format version (start from 1). */ 653#define INSTR_PROF_INDEX_VERSION 7 654/* Coverage mapping format version (start from 0). */ 655#define INSTR_PROF_COVMAP_VERSION 5 656 657/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the 658 * version for other variants of profile. We set the lowest bit of the upper 8 659 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation 660 * generated profile, and 0 if this is a Clang FE generated profile. 661 * 1 in bit 57 indicates there are context-sensitive records in the profile. 662 * The 59th bit indicates whether to use debug info to correlate profiles. 663 * The 60th bit indicates single byte coverage instrumentation. 664 * The 61st bit indicates function entry instrumentation only. 665 */ 666#define VARIANT_MASKS_ALL 0xff00000000000000ULL 667#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 668#define VARIANT_MASK_IR_PROF (0x1ULL << 56) 669#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) 670#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) 671#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59) 672#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60) 673#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61) 674#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version 675#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime 676#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias 677 678/* The variable that holds the name of the profile data 679 * specified via command line. */ 680#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename 681 682/* section name strings common to all targets other 683 than WIN32 */ 684#define INSTR_PROF_DATA_COMMON __llvm_prf_data 685#define INSTR_PROF_NAME_COMMON __llvm_prf_names 686#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts 687#define INSTR_PROF_VALS_COMMON __llvm_prf_vals 688#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds 689#define INSTR_PROF_COVMAP_COMMON __llvm_covmap 690#define INSTR_PROF_COVFUN_COMMON __llvm_covfun 691#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile 692/* Windows section names. Because these section names contain dollar characters, 693 * they must be quoted. 694 */ 695#define INSTR_PROF_DATA_COFF ".lprfd$M" 696#define INSTR_PROF_NAME_COFF ".lprfn$M" 697#define INSTR_PROF_CNTS_COFF ".lprfc$M" 698#define INSTR_PROF_VALS_COFF ".lprfv$M" 699#define INSTR_PROF_VNODES_COFF ".lprfnd$M" 700#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" 701#define INSTR_PROF_COVFUN_COFF ".lcovfun$M" 702#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" 703 704#ifdef _WIN32 705/* Runtime section names and name strings. */ 706#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF 707#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF 708#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF 709/* Array of pointers. Each pointer points to a list 710 * of value nodes associated with one value site. 711 */ 712#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF 713/* Value profile nodes section. */ 714#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF 715#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF 716#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF 717#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF 718#else 719/* Runtime section names and name strings. */ 720#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) 721#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) 722#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) 723/* Array of pointers. Each pointer points to a list 724 * of value nodes associated with one value site. 725 */ 726#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) 727/* Value profile nodes section. */ 728#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) 729#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) 730#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) 731/* Order file instrumentation. */ 732#define INSTR_PROF_ORDERFILE_SECT_NAME \ 733 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) 734#endif 735 736#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer 737#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ 738 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) 739#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx 740#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ 741 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) 742 743/* Macros to define start/stop section symbol for a given 744 * section on Linux. For instance 745 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 746 * expand to __start___llvm_prof_data 747 */ 748#define INSTR_PROF_SECT_START(Sect) \ 749 INSTR_PROF_CONCAT(__start_,Sect) 750#define INSTR_PROF_SECT_STOP(Sect) \ 751 INSTR_PROF_CONCAT(__stop_,Sect) 752 753/* Value Profiling API linkage name. */ 754#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 755#define INSTR_PROF_VALUE_PROF_FUNC_STR \ 756 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 757#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop 758#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ 759 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) 760 761/* InstrProfile per-function control data alignment. */ 762#define INSTR_PROF_DATA_ALIGNMENT 8 763 764/* The data structure that represents a tracked value by the 765 * value profiler. 766 */ 767typedef struct InstrProfValueData { 768 /* Profiled value. */ 769 uint64_t Value; 770 /* Number of times the value appears in the training run. */ 771 uint64_t Count; 772} InstrProfValueData; 773 774#endif /* INSTR_PROF_DATA_INC */ 775 776#ifndef INSTR_ORDER_FILE_INC 777/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ 778#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 779#define INSTR_ORDER_FILE_BUFFER_BITS 17 780#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff 781#endif /* INSTR_ORDER_FILE_INC */ 782#else 783#undef INSTR_PROF_DATA_DEFINED 784#endif 785 786#undef COVMAP_V2_OR_V3 787 788#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API 789 790#ifdef __cplusplus 791#define INSTR_PROF_INLINE inline 792#else 793#define INSTR_PROF_INLINE 794#endif 795 796/* The value range buckets (22 buckets) for the memop size value profiling looks 797 * like: 798 * 799 * [0, 0] 800 * [1, 1] 801 * [2, 2] 802 * [3, 3] 803 * [4, 4] 804 * [5, 5] 805 * [6, 6] 806 * [7, 7] 807 * [8, 8] 808 * [9, 15] 809 * [16, 16] 810 * [17, 31] 811 * [32, 32] 812 * [33, 63] 813 * [64, 64] 814 * [65, 127] 815 * [128, 128] 816 * [129, 255] 817 * [256, 256] 818 * [257, 511] 819 * [512, 512] 820 * [513, UINT64_MAX] 821 * 822 * Each range has a 'representative value' which is the lower end value of the 823 * range and used to store in the runtime profile data records and the VP 824 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. 825 */ 826#define INSTR_PROF_NUM_BUCKETS 22 827 828/* 829 * Clz and Popcount. This code was copied from 830 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and 831 * llvm/include/llvm/Support/MathExtras.h. 832 */ 833#if defined(_MSC_VER) && !defined(__clang__) 834 835#include <intrin.h> 836INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 837int InstProfClzll(unsigned long long X) { 838 unsigned long LeadZeroIdx = 0; 839#if !defined(_M_ARM64) && !defined(_M_X64) 840 // Scan the high 32 bits. 841 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) 842 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset 843 // from the MSB. 844 // Scan the low 32 bits. 845 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) 846 return (int)(63 - LeadZeroIdx); 847#else 848 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; 849#endif 850 return 64; 851} 852INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 853int InstProfPopcountll(unsigned long long X) { 854 // This code originates from https://reviews.llvm.org/rG30626254510f. 855 unsigned long long v = X; 856 v = v - ((v >> 1) & 0x5555555555555555ULL); 857 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); 858 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; 859 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); 860} 861 862#else 863 864INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 865int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } 866INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 867int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } 868 869#endif /* defined(_MSC_VER) && !defined(__clang__) */ 870 871/* Map an (observed) memop size value to the representative value of its range. 872 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ 873INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t 874InstrProfGetRangeRepValue(uint64_t Value) { 875 if (Value <= 8) 876 // The first ranges are individually tracked. Use the value as is. 877 return Value; 878 else if (Value >= 513) 879 // The last range is mapped to its lowest value. 880 return 513; 881 else if (InstProfPopcountll(Value) == 1) 882 // If it's a power of two, use it as is. 883 return Value; 884 else 885 // Otherwise, take to the previous power of two + 1. 886 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1; 887} 888 889/* Return true if the range that an (observed) memop size value belongs to has 890 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> 891 * false, 64 -> true, 100 -> false, 513 -> false. */ 892INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned 893InstrProfIsSingleValRange(uint64_t Value) { 894 if (Value <= 8) 895 // The first ranges are individually tracked. 896 return 1; 897 else if (InstProfPopcountll(Value) == 1) 898 // If it's a power of two, there's only one value. 899 return 1; 900 else 901 // Otherwise, there's more than one value in the range. 902 return 0; 903} 904 905#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ 906