1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2|* 3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4|* See https://llvm.org/LICENSE.txt for license information. 5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6|* 7\*===----------------------------------------------------------------------===*/ 8/* 9 * This is the main file that defines all the data structure, signature, 10 * constant literals that are shared across profiling runtime library, 11 * compiler (instrumentation), and host tools (reader/writer). The entities 12 * defined in this file affect the profile runtime ABI, the raw profile format, 13 * or both. 14 * 15 * The file has two identical copies. The primary copy lives in LLVM and 16 * the other one sits in compiler-rt/lib/profile directory. To make changes 17 * in this file, first modify the primary copy and copy it over to compiler-rt. 18 * Testing of any change in this file can start only after the two copies are 19 * synced up. 20 * 21 * The first part of the file includes macros that defines types, names, and 22 * initializers for the member fields of the core data structures. The field 23 * declarations for one structure is enabled by defining the field activation 24 * macro associated with that structure. Only one field activation record 25 * can be defined at one time and the rest definitions will be filtered out by 26 * the preprocessor. 27 * 28 * Examples of how the template is used to instantiate structure definition: 29 * 1. To declare a structure: 30 * 31 * struct ProfData { 32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 33 * Type Name; 34 * #include "llvm/ProfileData/InstrProfData.inc" 35 * }; 36 * 37 * 2. To construct LLVM type arrays for the struct type: 38 * 39 * Type *DataTypes[] = { 40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 41 * LLVMType, 42 * #include "llvm/ProfileData/InstrProfData.inc" 43 * }; 44 * 45 * 4. To construct constant array for the initializers: 46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 47 * Initializer, 48 * Constant *ConstantVals[] = { 49 * #include "llvm/ProfileData/InstrProfData.inc" 50 * }; 51 * 52 * 53 * The second part of the file includes definitions all other entities that 54 * are related to runtime ABI and format. When no field activation macro is 55 * defined, this file can be included to introduce the definitions. 56 * 57\*===----------------------------------------------------------------------===*/ 58 59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in 60 * the compiler runtime. */ 61#ifndef INSTR_PROF_VISIBILITY 62#define INSTR_PROF_VISIBILITY 63#endif 64 65/* INSTR_PROF_DATA start. */ 66/* Definition of member fields of the per-function control structure. */ 67#ifndef INSTR_PROF_DATA 68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 69#else 70#define INSTR_PROF_DATA_DEFINED 71#endif 72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) 75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 77 Inc->getHash()->getZExtValue())) 78INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) 79/* This is used to map function pointers for the indirect call targets to 80 * function name hashes during the conversion from raw to merged profile 81 * data. 82 */ 83INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ 84 FunctionAddr) 85INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ 86 ValuesPtrExpr) 87INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 88 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 89INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 90 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) 91#undef INSTR_PROF_DATA 92/* INSTR_PROF_DATA end. */ 93 94 95/* This is an internal data structure used by value profiler. It 96 * is defined here to allow serialization code sharing by LLVM 97 * to be used in unit test. 98 * 99 * typedef struct ValueProfNode { 100 * // InstrProfValueData VData; 101 * uint64_t Value; 102 * uint64_t Count; 103 * struct ValueProfNode *Next; 104 * } ValueProfNode; 105 */ 106/* INSTR_PROF_VALUE_NODE start. */ 107#ifndef INSTR_PROF_VALUE_NODE 108#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) 109#else 110#define INSTR_PROF_DATA_DEFINED 111#endif 112INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ 113 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 114INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ 115 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 116INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ 117 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) 118#undef INSTR_PROF_VALUE_NODE 119/* INSTR_PROF_VALUE_NODE end. */ 120 121/* INSTR_PROF_RAW_HEADER start */ 122/* Definition of member fields of the raw profile header data structure. */ 123#ifndef INSTR_PROF_RAW_HEADER 124#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 125#else 126#define INSTR_PROF_DATA_DEFINED 127#endif 128INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 129INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 130INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) 131INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) 132INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) 133INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) 134INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) 135INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 136INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, 137 (uintptr_t)CountersBegin - (uintptr_t)DataBegin) 138INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 139INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 140#undef INSTR_PROF_RAW_HEADER 141/* INSTR_PROF_RAW_HEADER end */ 142 143/* VALUE_PROF_FUNC_PARAM start */ 144/* Definition of parameter types of the runtime API used to do value profiling 145 * for a given value site. 146 */ 147#ifndef VALUE_PROF_FUNC_PARAM 148#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 149#define INSTR_PROF_COMMA 150#else 151#define INSTR_PROF_DATA_DEFINED 152#define INSTR_PROF_COMMA , 153#endif 154VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 155 INSTR_PROF_COMMA 156VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA 157VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 158#undef VALUE_PROF_FUNC_PARAM 159#undef INSTR_PROF_COMMA 160/* VALUE_PROF_FUNC_PARAM end */ 161 162/* VALUE_PROF_KIND start */ 163#ifndef VALUE_PROF_KIND 164#define VALUE_PROF_KIND(Enumerator, Value, Descr) 165#else 166#define INSTR_PROF_DATA_DEFINED 167#endif 168/* For indirect function call value profiling, the addresses of the target 169 * functions are profiled by the instrumented code. The target addresses are 170 * written in the raw profile data and converted to target function name's MD5 171 * hash by the profile reader during deserialization. Typically, this happens 172 * when the raw profile data is read during profile merging. 173 * 174 * For this remapping the ProfData is used. ProfData contains both the function 175 * name hash and the function address. 176 */ 177VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") 178/* For memory intrinsic functions size profiling. */ 179VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") 180/* These two kinds must be the last to be 181 * declared. This is to make sure the string 182 * array created with the template can be 183 * indexed with the kind value. 184 */ 185VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") 186VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") 187 188#undef VALUE_PROF_KIND 189/* VALUE_PROF_KIND end */ 190 191#undef COVMAP_V2_OR_V3 192#ifdef COVMAP_V2 193#define COVMAP_V2_OR_V3 194#endif 195#ifdef COVMAP_V3 196#define COVMAP_V2_OR_V3 197#endif 198 199/* COVMAP_FUNC_RECORD start */ 200/* Definition of member fields of the function record structure in coverage 201 * map. 202 */ 203#ifndef COVMAP_FUNC_RECORD 204#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 205#else 206#define INSTR_PROF_DATA_DEFINED 207#endif 208#ifdef COVMAP_V1 209COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ 210 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 211 llvm::Type::getInt8PtrTy(Ctx))) 212COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 213 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 214 NameValue.size())) 215#endif 216#ifdef COVMAP_V2_OR_V3 217COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 218 llvm::ConstantInt::get( \ 219 llvm::Type::getInt64Ty(Ctx), NameHash)) 220#endif 221COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 222 llvm::ConstantInt::get( \ 223 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) 224COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 225 llvm::ConstantInt::get( \ 226 llvm::Type::getInt64Ty(Ctx), FuncHash)) 227#ifdef COVMAP_V3 228COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ 229 llvm::ConstantInt::get( \ 230 llvm::Type::getInt64Ty(Ctx), FilenamesRef)) 231COVMAP_FUNC_RECORD(const char, \ 232 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ 233 CoverageMapping.size()), \ 234 CoverageMapping, 235 llvm::ConstantDataArray::getRaw( \ 236 CoverageMapping, CoverageMapping.size(), \ 237 llvm::Type::getInt8Ty(Ctx))) 238#endif 239#undef COVMAP_FUNC_RECORD 240/* COVMAP_FUNC_RECORD end. */ 241 242/* COVMAP_HEADER start */ 243/* Definition of member fields of coverage map header. 244 */ 245#ifndef COVMAP_HEADER 246#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 247#else 248#define INSTR_PROF_DATA_DEFINED 249#endif 250COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 251 llvm::ConstantInt::get(Int32Ty, NRecords)) 252COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 253 llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 254COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 255 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 256COVMAP_HEADER(uint32_t, Int32Ty, Version, \ 257 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 258#undef COVMAP_HEADER 259/* COVMAP_HEADER end. */ 260 261 262#ifdef INSTR_PROF_SECT_ENTRY 263#define INSTR_PROF_DATA_DEFINED 264INSTR_PROF_SECT_ENTRY(IPSK_data, \ 265 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ 266 INSTR_PROF_DATA_COFF, "__DATA,") 267INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ 268 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ 269 INSTR_PROF_CNTS_COFF, "__DATA,") 270INSTR_PROF_SECT_ENTRY(IPSK_name, \ 271 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ 272 INSTR_PROF_NAME_COFF, "__DATA,") 273INSTR_PROF_SECT_ENTRY(IPSK_vals, \ 274 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ 275 INSTR_PROF_VALS_COFF, "__DATA,") 276INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ 277 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ 278 INSTR_PROF_VNODES_COFF, "__DATA,") 279INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ 280 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ 281 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") 282INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ 283 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ 284 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,") 285INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ 286 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ 287 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") 288 289#undef INSTR_PROF_SECT_ENTRY 290#endif 291 292 293#ifdef INSTR_PROF_VALUE_PROF_DATA 294#define INSTR_PROF_DATA_DEFINED 295 296#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 297/*! 298 * This is the header of the data structure that defines the on-disk 299 * layout of the value profile data of a particular kind for one function. 300 */ 301typedef struct ValueProfRecord { 302 /* The kind of the value profile record. */ 303 uint32_t Kind; 304 /* 305 * The number of value profile sites. It is guaranteed to be non-zero; 306 * otherwise the record for this kind won't be emitted. 307 */ 308 uint32_t NumValueSites; 309 /* 310 * The first element of the array that stores the number of profiled 311 * values for each value site. The size of the array is NumValueSites. 312 * Since NumValueSites is greater than zero, there is at least one 313 * element in the array. 314 */ 315 uint8_t SiteCountArray[1]; 316 317 /* 318 * The fake declaration is for documentation purpose only. 319 * Align the start of next field to be on 8 byte boundaries. 320 uint8_t Padding[X]; 321 */ 322 323 /* The array of value profile data. The size of the array is the sum 324 * of all elements in SiteCountArray[]. 325 InstrProfValueData ValueData[]; 326 */ 327 328#ifdef __cplusplus 329 /*! 330 * Return the number of value sites. 331 */ 332 uint32_t getNumValueSites() const { return NumValueSites; } 333 /*! 334 * Read data from this record and save it to Record. 335 */ 336 void deserializeTo(InstrProfRecord &Record, 337 InstrProfSymtab *SymTab); 338 /* 339 * In-place byte swap: 340 * Do byte swap for this instance. \c Old is the original order before 341 * the swap, and \c New is the New byte order. 342 */ 343 void swapBytes(support::endianness Old, support::endianness New); 344#endif 345} ValueProfRecord; 346 347/*! 348 * Per-function header/control data structure for value profiling 349 * data in indexed format. 350 */ 351typedef struct ValueProfData { 352 /* 353 * Total size in bytes including this field. It must be a multiple 354 * of sizeof(uint64_t). 355 */ 356 uint32_t TotalSize; 357 /* 358 *The number of value profile kinds that has value profile data. 359 * In this implementation, a value profile kind is considered to 360 * have profile data if the number of value profile sites for the 361 * kind is not zero. More aggressively, the implementation can 362 * choose to check the actual data value: if none of the value sites 363 * has any profiled values, the kind can be skipped. 364 */ 365 uint32_t NumValueKinds; 366 367 /* 368 * Following are a sequence of variable length records. The prefix/header 369 * of each record is defined by ValueProfRecord type. The number of 370 * records is NumValueKinds. 371 * ValueProfRecord Record_1; 372 * ValueProfRecord Record_N; 373 */ 374 375#if __cplusplus 376 /*! 377 * Return the total size in bytes of the on-disk value profile data 378 * given the data stored in Record. 379 */ 380 static uint32_t getSize(const InstrProfRecord &Record); 381 /*! 382 * Return a pointer to \c ValueProfData instance ready to be streamed. 383 */ 384 static std::unique_ptr<ValueProfData> 385 serializeFrom(const InstrProfRecord &Record); 386 /*! 387 * Check the integrity of the record. 388 */ 389 Error checkIntegrity(); 390 /*! 391 * Return a pointer to \c ValueProfileData instance ready to be read. 392 * All data in the instance are properly byte swapped. The input 393 * data is assumed to be in little endian order. 394 */ 395 static Expected<std::unique_ptr<ValueProfData>> 396 getValueProfData(const unsigned char *SrcBuffer, 397 const unsigned char *const SrcBufferEnd, 398 support::endianness SrcDataEndianness); 399 /*! 400 * Swap byte order from \c Endianness order to host byte order. 401 */ 402 void swapBytesToHost(support::endianness Endianness); 403 /*! 404 * Swap byte order from host byte order to \c Endianness order. 405 */ 406 void swapBytesFromHost(support::endianness Endianness); 407 /*! 408 * Return the total size of \c ValueProfileData. 409 */ 410 uint32_t getSize() const { return TotalSize; } 411 /*! 412 * Read data from this data and save it to \c Record. 413 */ 414 void deserializeTo(InstrProfRecord &Record, 415 InstrProfSymtab *SymTab); 416 void operator delete(void *ptr) { ::operator delete(ptr); } 417#endif 418} ValueProfData; 419 420/* 421 * The closure is designed to abstact away two types of value profile data: 422 * - InstrProfRecord which is the primary data structure used to 423 * represent profile data in host tools (reader, writer, and profile-use) 424 * - value profile runtime data structure suitable to be used by C 425 * runtime library. 426 * 427 * Both sources of data need to serialize to disk/memory-buffer in common 428 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 429 * writer to share the same format and code with indexed profile writer. 430 * 431 * For documentation of the member methods below, refer to corresponding methods 432 * in class InstrProfRecord. 433 */ 434typedef struct ValueProfRecordClosure { 435 const void *Record; 436 uint32_t (*GetNumValueKinds)(const void *Record); 437 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 438 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 439 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 440 441 /* 442 * After extracting the value profile data from the value profile record, 443 * this method is used to map the in-memory value to on-disk value. If 444 * the method is null, value will be written out untranslated. 445 */ 446 uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 447 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 448 uint32_t S); 449 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 450} ValueProfRecordClosure; 451 452INSTR_PROF_VISIBILITY ValueProfRecord * 453getFirstValueProfRecord(ValueProfData *VPD); 454INSTR_PROF_VISIBILITY ValueProfRecord * 455getValueProfRecordNext(ValueProfRecord *VPR); 456INSTR_PROF_VISIBILITY InstrProfValueData * 457getValueProfRecordValueData(ValueProfRecord *VPR); 458INSTR_PROF_VISIBILITY uint32_t 459getValueProfRecordHeaderSize(uint32_t NumValueSites); 460 461#undef INSTR_PROF_VALUE_PROF_DATA 462#endif /* INSTR_PROF_VALUE_PROF_DATA */ 463 464 465#ifdef INSTR_PROF_COMMON_API_IMPL 466#define INSTR_PROF_DATA_DEFINED 467#ifdef __cplusplus 468#define INSTR_PROF_INLINE inline 469#define INSTR_PROF_NULLPTR nullptr 470#else 471#define INSTR_PROF_INLINE 472#define INSTR_PROF_NULLPTR NULL 473#endif 474 475#ifndef offsetof 476#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 477#endif 478 479/*! 480 * Return the \c ValueProfRecord header size including the 481 * padding bytes. 482 */ 483INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 484uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 485 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 486 sizeof(uint8_t) * NumValueSites; 487 /* Round the size to multiple of 8 bytes. */ 488 Size = (Size + 7) & ~7; 489 return Size; 490} 491 492/*! 493 * Return the total size of the value profile record including the 494 * header and the value data. 495 */ 496INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 497uint32_t getValueProfRecordSize(uint32_t NumValueSites, 498 uint32_t NumValueData) { 499 return getValueProfRecordHeaderSize(NumValueSites) + 500 sizeof(InstrProfValueData) * NumValueData; 501} 502 503/*! 504 * Return the pointer to the start of value data array. 505 */ 506INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 507InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 508 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 509 This->NumValueSites)); 510} 511 512/*! 513 * Return the total number of value data for \c This record. 514 */ 515INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 516uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 517 uint32_t NumValueData = 0; 518 uint32_t I; 519 for (I = 0; I < This->NumValueSites; I++) 520 NumValueData += This->SiteCountArray[I]; 521 return NumValueData; 522} 523 524/*! 525 * Use this method to advance to the next \c This \c ValueProfRecord. 526 */ 527INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 528ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 529 uint32_t NumValueData = getValueProfRecordNumValueData(This); 530 return (ValueProfRecord *)((char *)This + 531 getValueProfRecordSize(This->NumValueSites, 532 NumValueData)); 533} 534 535/*! 536 * Return the first \c ValueProfRecord instance. 537 */ 538INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 539ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 540 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 541} 542 543/* Closure based interfaces. */ 544 545/*! 546 * Return the total size in bytes of the on-disk value profile data 547 * given the data stored in Record. 548 */ 549INSTR_PROF_VISIBILITY uint32_t 550getValueProfDataSize(ValueProfRecordClosure *Closure) { 551 uint32_t Kind; 552 uint32_t TotalSize = sizeof(ValueProfData); 553 const void *Record = Closure->Record; 554 555 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 556 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 557 if (!NumValueSites) 558 continue; 559 TotalSize += getValueProfRecordSize(NumValueSites, 560 Closure->GetNumValueData(Record, Kind)); 561 } 562 return TotalSize; 563} 564 565/*! 566 * Extract value profile data of a function for the profile kind \c ValueKind 567 * from the \c Closure and serialize the data into \c This record instance. 568 */ 569INSTR_PROF_VISIBILITY void 570serializeValueProfRecordFrom(ValueProfRecord *This, 571 ValueProfRecordClosure *Closure, 572 uint32_t ValueKind, uint32_t NumValueSites) { 573 uint32_t S; 574 const void *Record = Closure->Record; 575 This->Kind = ValueKind; 576 This->NumValueSites = NumValueSites; 577 InstrProfValueData *DstVD = getValueProfRecordValueData(This); 578 579 for (S = 0; S < NumValueSites; S++) { 580 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 581 This->SiteCountArray[S] = ND; 582 Closure->GetValueForSite(Record, DstVD, ValueKind, S); 583 DstVD += ND; 584 } 585} 586 587/*! 588 * Extract value profile data of a function from the \c Closure 589 * and serialize the data into \c DstData if it is not NULL or heap 590 * memory allocated by the \c Closure's allocator method. If \c 591 * DstData is not null, the caller is expected to set the TotalSize 592 * in DstData. 593 */ 594INSTR_PROF_VISIBILITY ValueProfData * 595serializeValueProfDataFrom(ValueProfRecordClosure *Closure, 596 ValueProfData *DstData) { 597 uint32_t Kind; 598 uint32_t TotalSize = 599 DstData ? DstData->TotalSize : getValueProfDataSize(Closure); 600 601 ValueProfData *VPD = 602 DstData ? DstData : Closure->AllocValueProfData(TotalSize); 603 604 VPD->TotalSize = TotalSize; 605 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 606 ValueProfRecord *VR = getFirstValueProfRecord(VPD); 607 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 608 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 609 if (!NumValueSites) 610 continue; 611 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 612 VR = getValueProfRecordNext(VR); 613 } 614 return VPD; 615} 616 617#undef INSTR_PROF_COMMON_API_IMPL 618#endif /* INSTR_PROF_COMMON_API_IMPL */ 619 620/*============================================================================*/ 621 622#ifndef INSTR_PROF_DATA_DEFINED 623 624#ifndef INSTR_PROF_DATA_INC 625#define INSTR_PROF_DATA_INC 626 627/* Helper macros. */ 628#define INSTR_PROF_SIMPLE_QUOTE(x) #x 629#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 630#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 631#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 632 633/* Magic number to detect file format and endianness. 634 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 635 * so that utilities, like strings, don't grab it as a string. 129 is also 636 * invalid UTF-8, and high enough to be interesting. 637 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 638 * for 32-bit platforms. 639 */ 640#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 641 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 642 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 643#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 644 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 645 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 646 647/* Raw profile format version (start from 1). */ 648#define INSTR_PROF_RAW_VERSION 8 649/* Indexed profile format version (start from 1). */ 650#define INSTR_PROF_INDEX_VERSION 7 651/* Coverage mapping format version (start from 0). */ 652#define INSTR_PROF_COVMAP_VERSION 5 653 654/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the 655 * version for other variants of profile. We set the lowest bit of the upper 8 656 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton 657 * generated profile, and 0 if this is a Clang FE generated profile. 658 * 1 in bit 57 indicates there are context-sensitive records in the profile. 659 */ 660#define VARIANT_MASKS_ALL 0xff00000000000000ULL 661#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 662#define VARIANT_MASK_IR_PROF (0x1ULL << 56) 663#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) 664#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) 665#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version 666#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime 667#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias 668 669/* The variable that holds the name of the profile data 670 * specified via command line. */ 671#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename 672 673/* section name strings common to all targets other 674 than WIN32 */ 675#define INSTR_PROF_DATA_COMMON __llvm_prf_data 676#define INSTR_PROF_NAME_COMMON __llvm_prf_names 677#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts 678#define INSTR_PROF_VALS_COMMON __llvm_prf_vals 679#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds 680#define INSTR_PROF_COVMAP_COMMON __llvm_covmap 681#define INSTR_PROF_COVFUN_COMMON __llvm_covfun 682#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile 683/* Windows section names. Because these section names contain dollar characters, 684 * they must be quoted. 685 */ 686#define INSTR_PROF_DATA_COFF ".lprfd$M" 687#define INSTR_PROF_NAME_COFF ".lprfn$M" 688#define INSTR_PROF_CNTS_COFF ".lprfc$M" 689#define INSTR_PROF_VALS_COFF ".lprfv$M" 690#define INSTR_PROF_VNODES_COFF ".lprfnd$M" 691#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" 692#define INSTR_PROF_COVFUN_COFF ".lcovfun$M" 693#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" 694 695#ifdef _WIN32 696/* Runtime section names and name strings. */ 697#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF 698#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF 699#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF 700/* Array of pointers. Each pointer points to a list 701 * of value nodes associated with one value site. 702 */ 703#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF 704/* Value profile nodes section. */ 705#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF 706#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF 707#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF 708#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF 709#else 710/* Runtime section names and name strings. */ 711#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) 712#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) 713#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) 714/* Array of pointers. Each pointer points to a list 715 * of value nodes associated with one value site. 716 */ 717#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) 718/* Value profile nodes section. */ 719#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) 720#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) 721#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) 722/* Order file instrumentation. */ 723#define INSTR_PROF_ORDERFILE_SECT_NAME \ 724 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) 725#endif 726 727#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer 728#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ 729 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) 730#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx 731#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ 732 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) 733 734/* Macros to define start/stop section symbol for a given 735 * section on Linux. For instance 736 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 737 * expand to __start___llvm_prof_data 738 */ 739#define INSTR_PROF_SECT_START(Sect) \ 740 INSTR_PROF_CONCAT(__start_,Sect) 741#define INSTR_PROF_SECT_STOP(Sect) \ 742 INSTR_PROF_CONCAT(__stop_,Sect) 743 744/* Value Profiling API linkage name. */ 745#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 746#define INSTR_PROF_VALUE_PROF_FUNC_STR \ 747 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 748#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop 749#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ 750 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) 751 752/* InstrProfile per-function control data alignment. */ 753#define INSTR_PROF_DATA_ALIGNMENT 8 754 755/* The data structure that represents a tracked value by the 756 * value profiler. 757 */ 758typedef struct InstrProfValueData { 759 /* Profiled value. */ 760 uint64_t Value; 761 /* Number of times the value appears in the training run. */ 762 uint64_t Count; 763} InstrProfValueData; 764 765#endif /* INSTR_PROF_DATA_INC */ 766 767#ifndef INSTR_ORDER_FILE_INC 768/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ 769#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 770#define INSTR_ORDER_FILE_BUFFER_BITS 17 771#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff 772#endif /* INSTR_ORDER_FILE_INC */ 773#else 774#undef INSTR_PROF_DATA_DEFINED 775#endif 776 777#undef COVMAP_V2_OR_V3 778 779#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API 780 781#ifdef __cplusplus 782#define INSTR_PROF_INLINE inline 783#else 784#define INSTR_PROF_INLINE 785#endif 786 787/* The value range buckets (22 buckets) for the memop size value profiling looks 788 * like: 789 * 790 * [0, 0] 791 * [1, 1] 792 * [2, 2] 793 * [3, 3] 794 * [4, 4] 795 * [5, 5] 796 * [6, 6] 797 * [7, 7] 798 * [8, 8] 799 * [9, 15] 800 * [16, 16] 801 * [17, 31] 802 * [32, 32] 803 * [33, 63] 804 * [64, 64] 805 * [65, 127] 806 * [128, 128] 807 * [129, 255] 808 * [256, 256] 809 * [257, 511] 810 * [512, 512] 811 * [513, UINT64_MAX] 812 * 813 * Each range has a 'representative value' which is the lower end value of the 814 * range and used to store in the runtime profile data records and the VP 815 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. 816 */ 817#define INSTR_PROF_NUM_BUCKETS 22 818 819/* 820 * Clz and Popcount. This code was copied from 821 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and 822 * llvm/include/llvm/Support/MathExtras.h. 823 */ 824#if defined(_MSC_VER) && !defined(__clang__) 825 826#include <intrin.h> 827INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 828int InstProfClzll(unsigned long long X) { 829 unsigned long LeadZeroIdx = 0; 830#if !defined(_M_ARM64) && !defined(_M_X64) 831 // Scan the high 32 bits. 832 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) 833 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset 834 // from the MSB. 835 // Scan the low 32 bits. 836 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) 837 return (int)(63 - LeadZeroIdx); 838#else 839 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; 840#endif 841 return 64; 842} 843INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 844int InstProfPopcountll(unsigned long long X) { 845 // This code originates from https://reviews.llvm.org/rG30626254510f. 846 unsigned long long v = X; 847 v = v - ((v >> 1) & 0x5555555555555555ULL); 848 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); 849 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; 850 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); 851} 852 853#else 854 855INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 856int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } 857INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 858int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } 859 860#endif /* defined(_MSC_VER) && !defined(__clang__) */ 861 862/* Map an (observed) memop size value to the representative value of its range. 863 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ 864INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t 865InstrProfGetRangeRepValue(uint64_t Value) { 866 if (Value <= 8) 867 // The first ranges are individually tracked. Use the value as is. 868 return Value; 869 else if (Value >= 513) 870 // The last range is mapped to its lowest value. 871 return 513; 872 else if (InstProfPopcountll(Value) == 1) 873 // If it's a power of two, use it as is. 874 return Value; 875 else 876 // Otherwise, take to the previous power of two + 1. 877 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1; 878} 879 880/* Return true if the range that an (observed) memop size value belongs to has 881 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> 882 * false, 64 -> true, 100 -> false, 513 -> false. */ 883INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned 884InstrProfIsSingleValRange(uint64_t Value) { 885 if (Value <= 8) 886 // The first ranges are individually tracked. 887 return 1; 888 else if (InstProfPopcountll(Value) == 1) 889 // If it's a power of two, there's only one value. 890 return 1; 891 else 892 // Otherwise, there's more than one value in the range. 893 return 0; 894} 895 896#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ 897