1 /*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\ 2 |* 3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 |* See https://llvm.org/LICENSE.txt for license information. 5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 |* 7 |*===----------------------------------------------------------------------===* 8 |* This file defines the API needed for in-process merging of profile data 9 |* stored in memory buffer. 10 \*===---------------------------------------------------------------------===*/ 11 12 #include "InstrProfiling.h" 13 #include "InstrProfilingInternal.h" 14 #include "InstrProfilingUtil.h" 15 16 #define INSTR_PROF_VALUE_PROF_DATA 17 #include "profile/InstrProfData.inc" 18 19 COMPILER_RT_VISIBILITY 20 void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *); 21 22 COMPILER_RT_VISIBILITY 23 uint64_t lprofGetLoadModuleSignature(void) { 24 /* A very fast way to compute a module signature. */ 25 uint64_t Version = __llvm_profile_get_version(); 26 uint64_t NumCounters = __llvm_profile_get_num_counters( 27 __llvm_profile_begin_counters(), __llvm_profile_end_counters()); 28 uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(), 29 __llvm_profile_end_data()); 30 uint64_t NamesSize = 31 (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()); 32 uint64_t NumVnodes = 33 (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes()); 34 const __llvm_profile_data *FirstD = __llvm_profile_begin_data(); 35 36 return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) + 37 (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version + 38 __llvm_profile_get_magic(); 39 } 40 41 #ifdef __GNUC__ 42 #pragma GCC diagnostic push 43 #pragma GCC diagnostic ignored "-Wcast-qual" 44 #elif defined(__clang__) 45 #pragma clang diagnostic push 46 #pragma clang diagnostic ignored "-Wcast-qual" 47 #endif 48 49 /* Returns 1 if profile is not structurally compatible. */ 50 COMPILER_RT_VISIBILITY 51 int __llvm_profile_check_compatibility(const char *ProfileData, 52 uint64_t ProfileSize) { 53 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; 54 __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; 55 SrcDataStart = 56 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + 57 Header->BinaryIdsSize); 58 SrcDataEnd = SrcDataStart + Header->NumData; 59 60 if (ProfileSize < sizeof(__llvm_profile_header)) 61 return 1; 62 63 /* Check the header first. */ 64 if (Header->Magic != __llvm_profile_get_magic() || 65 Header->Version != __llvm_profile_get_version() || 66 Header->NumData != 67 __llvm_profile_get_num_data(__llvm_profile_begin_data(), 68 __llvm_profile_end_data()) || 69 Header->NumCounters != 70 __llvm_profile_get_num_counters(__llvm_profile_begin_counters(), 71 __llvm_profile_end_counters()) || 72 Header->NumBitmapBytes != 73 __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(), 74 __llvm_profile_end_bitmap()) || 75 Header->NamesSize != 76 __llvm_profile_get_name_size(__llvm_profile_begin_names(), 77 __llvm_profile_end_names()) || 78 Header->ValueKindLast != IPVK_Last) 79 return 1; 80 81 if (ProfileSize < 82 sizeof(__llvm_profile_header) + Header->BinaryIdsSize + 83 Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize + 84 Header->NumCounters * __llvm_profile_counter_entry_size() + 85 Header->NumBitmapBytes) 86 return 1; 87 88 for (SrcData = SrcDataStart, 89 DstData = (__llvm_profile_data *)__llvm_profile_begin_data(); 90 SrcData < SrcDataEnd; ++SrcData, ++DstData) { 91 if (SrcData->NameRef != DstData->NameRef || 92 SrcData->FuncHash != DstData->FuncHash || 93 SrcData->NumCounters != DstData->NumCounters || 94 SrcData->NumBitmapBytes != DstData->NumBitmapBytes) 95 return 1; 96 } 97 98 /* Matched! */ 99 return 0; 100 } 101 102 static uintptr_t signextIfWin64(void *V) { 103 #ifdef _WIN64 104 return (uintptr_t)(int32_t)(uintptr_t)V; 105 #else 106 return (uintptr_t)V; 107 #endif 108 } 109 110 // Skip names section, vtable profile data section and vtable names section 111 // for runtime profile merge. To merge runtime addresses from multiple 112 // profiles collected from the same instrumented binary, the binary should be 113 // loaded at fixed base address (e.g., build with -no-pie, or run with ASLR 114 // disabled). In this set-up these three sections remain unchanged. 115 static uint64_t 116 getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) { 117 const uint64_t VTableSectionSize = 118 Header->NumVTables * sizeof(VTableProfData); 119 const uint64_t PaddingBytesAfterVTableSection = 120 __llvm_profile_get_num_padding_bytes(VTableSectionSize); 121 const uint64_t VNamesSize = Header->VNamesSize; 122 const uint64_t PaddingBytesAfterVNamesSize = 123 __llvm_profile_get_num_padding_bytes(VNamesSize); 124 return Header->NamesSize + 125 __llvm_profile_get_num_padding_bytes(Header->NamesSize) + 126 VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize + 127 PaddingBytesAfterVNamesSize; 128 } 129 130 COMPILER_RT_VISIBILITY 131 int __llvm_profile_merge_from_buffer(const char *ProfileData, 132 uint64_t ProfileSize) { 133 if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) { 134 PROF_ERR("%s\n", 135 "Temporal profiles do not support profile merging at runtime. " 136 "Instead, merge raw profiles using the llvm-profdata tool."); 137 return 1; 138 } 139 140 __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; 141 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; 142 char *SrcCountersStart, *DstCounter; 143 const char *SrcCountersEnd, *SrcCounter; 144 const char *SrcBitmapStart; 145 const char *SrcNameStart; 146 const char *SrcValueProfDataStart, *SrcValueProfData; 147 uintptr_t CountersDelta = Header->CountersDelta; 148 uintptr_t BitmapDelta = Header->BitmapDelta; 149 150 SrcDataStart = 151 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + 152 Header->BinaryIdsSize); 153 SrcDataEnd = SrcDataStart + Header->NumData; 154 SrcCountersStart = (char *)SrcDataEnd; 155 SrcCountersEnd = SrcCountersStart + 156 Header->NumCounters * __llvm_profile_counter_entry_size(); 157 SrcBitmapStart = SrcCountersEnd; 158 SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; 159 SrcValueProfDataStart = 160 SrcNameStart + getDistanceFromCounterToValueProf(Header); 161 if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) 162 return 1; 163 164 // Merge counters by iterating the entire counter section when data section is 165 // empty due to correlation. 166 if (Header->NumData == 0) { 167 for (SrcCounter = SrcCountersStart, 168 DstCounter = __llvm_profile_begin_counters(); 169 SrcCounter < SrcCountersEnd;) { 170 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) { 171 *DstCounter &= *SrcCounter; 172 } else { 173 *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter; 174 } 175 SrcCounter += __llvm_profile_counter_entry_size(); 176 DstCounter += __llvm_profile_counter_entry_size(); 177 } 178 return 0; 179 } 180 181 for (SrcData = SrcDataStart, 182 DstData = (__llvm_profile_data *)__llvm_profile_begin_data(), 183 SrcValueProfData = SrcValueProfDataStart; 184 SrcData < SrcDataEnd; ++SrcData, ++DstData) { 185 // For the in-memory destination, CounterPtr is the distance from the start 186 // address of the data to the start address of the counter. On WIN64, 187 // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign 188 // extend CounterPtr to get the original value. 189 char *DstCounters = 190 (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr)); 191 char *DstBitmap = 192 (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr)); 193 unsigned NVK = 0; 194 195 // SrcData is a serialized representation of the memory image. We need to 196 // compute the in-buffer counter offset from the in-memory address distance. 197 // The initial CountersDelta is the in-memory address difference 198 // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr - 199 // CountersDelta computes the offset into the in-buffer counter section. 200 // 201 // On WIN64, CountersDelta is truncated as well, so no need for signext. 202 char *SrcCounters = 203 SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta); 204 // CountersDelta needs to be decreased as we advance to the next data 205 // record. 206 CountersDelta -= sizeof(*SrcData); 207 unsigned NC = SrcData->NumCounters; 208 if (NC == 0) 209 return 1; 210 if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart || 211 (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart) 212 return 1; 213 for (unsigned I = 0; I < NC; I++) { 214 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) { 215 // A value of zero signifies the function is covered. 216 DstCounters[I] &= SrcCounters[I]; 217 } else { 218 ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I]; 219 } 220 } 221 222 const char *SrcBitmap = 223 SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta); 224 // BitmapDelta also needs to be decreased as we advance to the next data 225 // record. 226 BitmapDelta -= sizeof(*SrcData); 227 unsigned NB = SrcData->NumBitmapBytes; 228 // NumBitmapBytes may legitimately be 0. Just keep going. 229 if (NB != 0) { 230 if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart) 231 return 1; 232 // Merge Src and Dst Bitmap bytes by simply ORing them together. 233 for (unsigned I = 0; I < NB; I++) 234 DstBitmap[I] |= SrcBitmap[I]; 235 } 236 237 /* Now merge value profile data. */ 238 if (!VPMergeHook) 239 continue; 240 241 for (unsigned I = 0; I <= IPVK_Last; I++) 242 NVK += (SrcData->NumValueSites[I] != 0); 243 244 if (!NVK) 245 continue; 246 247 if (SrcValueProfData >= ProfileData + ProfileSize) 248 return 1; 249 VPMergeHook((ValueProfData *)SrcValueProfData, DstData); 250 SrcValueProfData = 251 SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize; 252 } 253 254 return 0; 255 } 256 257 #ifdef __GNUC__ 258 #pragma GCC diagnostic pop 259 #elif defined(__clang__) 260 #pragma clang diagnostic pop 261 #endif 262