1 /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ 2 |* 3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 |* See https://llvm.org/LICENSE.txt for license information. 5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 |* 7 \*===----------------------------------------------------------------------===*/ 8 9 // Note: This is linked into the Darwin kernel, and must remain compatible 10 // with freestanding compilation. See `darwin_add_builtin_libraries`. 11 12 #ifdef _MSC_VER 13 /* For _alloca */ 14 #include <malloc.h> 15 #endif 16 #include <string.h> 17 18 #include "InstrProfiling.h" 19 #include "InstrProfilingInternal.h" 20 #include "InstrProfilingPort.h" 21 22 #define INSTR_PROF_VALUE_PROF_DATA 23 #include "profile/InstrProfData.inc" 24 25 COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL; 26 static ProfBufferIO TheBufferIO; 27 #define VP_BUFFER_SIZE 8 * 1024 28 static uint8_t BufferIOBuffer[VP_BUFFER_SIZE]; 29 static InstrProfValueData VPDataArray[16]; 30 static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray); 31 32 COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0; 33 COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0; 34 35 /* The buffer writer is responsible in keeping writer state 36 * across the call. 37 */ 38 COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This, 39 ProfDataIOVec *IOVecs, 40 uint32_t NumIOVecs) { 41 uint32_t I; 42 char **Buffer = (char **)&This->WriterCtx; 43 for (I = 0; I < NumIOVecs; I++) { 44 size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; 45 if (IOVecs[I].Data) 46 memcpy(*Buffer, IOVecs[I].Data, Length); 47 else if (IOVecs[I].UseZeroPadding) { 48 /* Allocating the buffer should zero fill. */ 49 } 50 *Buffer += Length; 51 } 52 return 0; 53 } 54 55 static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter, 56 uint8_t *Buffer, uint32_t BufferSz) { 57 BufferIO->FileWriter = FileWriter; 58 BufferIO->OwnFileWriter = 0; 59 BufferIO->BufferStart = Buffer; 60 BufferIO->BufferSz = BufferSz; 61 BufferIO->CurOffset = 0; 62 } 63 64 COMPILER_RT_VISIBILITY ProfBufferIO * 65 lprofCreateBufferIO(ProfDataWriter *FileWriter) { 66 uint8_t *Buffer = DynamicBufferIOBuffer; 67 uint32_t BufferSize = VPBufferSize; 68 if (!Buffer) { 69 Buffer = &BufferIOBuffer[0]; 70 BufferSize = sizeof(BufferIOBuffer); 71 } 72 llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize); 73 return &TheBufferIO; 74 } 75 76 COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) { 77 if (BufferIO->OwnFileWriter) 78 FreeHook(BufferIO->FileWriter); 79 if (DynamicBufferIOBuffer) { 80 FreeHook(DynamicBufferIOBuffer); 81 DynamicBufferIOBuffer = 0; 82 VPBufferSize = 0; 83 } 84 } 85 86 COMPILER_RT_VISIBILITY int 87 lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) { 88 /* Buffer is not large enough, it is time to flush. */ 89 if (Size + BufferIO->CurOffset > BufferIO->BufferSz) { 90 if (lprofBufferIOFlush(BufferIO) != 0) 91 return -1; 92 } 93 /* Special case, bypass the buffer completely. */ 94 ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}}; 95 if (Size > BufferIO->BufferSz) { 96 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) 97 return -1; 98 } else { 99 /* Write the data to buffer */ 100 uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset; 101 ProfDataWriter BufferWriter; 102 initBufferWriter(&BufferWriter, (char *)Buffer); 103 lprofBufferWriter(&BufferWriter, IO, 1); 104 BufferIO->CurOffset = 105 (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart; 106 } 107 return 0; 108 } 109 110 COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) { 111 if (BufferIO->CurOffset) { 112 ProfDataIOVec IO[] = { 113 {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}}; 114 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) 115 return -1; 116 BufferIO->CurOffset = 0; 117 } 118 return 0; 119 } 120 121 /* Write out value profile data for function specified with \c Data. 122 * The implementation does not use the method \c serializeValueProfData 123 * which depends on dynamic memory allocation. In this implementation, 124 * value profile data is written out to \c BufferIO piecemeal. 125 */ 126 static int writeOneValueProfData(ProfBufferIO *BufferIO, 127 VPDataReaderType *VPDataReader, 128 const __llvm_profile_data *Data) { 129 unsigned I, NumValueKinds = 0; 130 ValueProfData VPHeader; 131 uint8_t *SiteCountArray[IPVK_Last + 1]; 132 133 for (I = 0; I <= IPVK_Last; I++) { 134 if (!Data->NumValueSites[I]) 135 SiteCountArray[I] = 0; 136 else { 137 uint32_t Sz = 138 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - 139 offsetof(ValueProfRecord, SiteCountArray); 140 /* Only use alloca for this small byte array to avoid excessive 141 * stack growth. */ 142 SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz); 143 memset(SiteCountArray[I], 0, Sz); 144 } 145 } 146 147 /* If NumValueKinds returned is 0, there is nothing to write, report 148 success and return. This should match the raw profile reader's behavior. */ 149 if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray))) 150 return 0; 151 152 /* First write the header structure. */ 153 VPHeader.TotalSize = VPDataReader->GetValueProfDataSize(); 154 VPHeader.NumValueKinds = NumValueKinds; 155 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader, 156 sizeof(ValueProfData))) 157 return -1; 158 159 /* Make sure nothing else needs to be written before value profile 160 * records. */ 161 if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) != 162 (void *)(&VPHeader + 1)) 163 return -1; 164 165 /* Write out the value profile record for each value kind 166 * one by one. */ 167 for (I = 0; I <= IPVK_Last; I++) { 168 uint32_t J; 169 ValueProfRecord RecordHeader; 170 /* The size of the value prof record header without counting the 171 * site count array .*/ 172 uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray); 173 uint32_t SiteCountArraySize; 174 175 if (!Data->NumValueSites[I]) 176 continue; 177 178 /* Write out the record header. */ 179 RecordHeader.Kind = I; 180 RecordHeader.NumValueSites = Data->NumValueSites[I]; 181 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader, 182 RecordHeaderSize)) 183 return -1; 184 185 /* Write out the site value count array including padding space. */ 186 SiteCountArraySize = 187 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - 188 RecordHeaderSize; 189 if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize)) 190 return -1; 191 192 /* Write out the value profile data for each value site. */ 193 for (J = 0; J < Data->NumValueSites[I]; J++) { 194 uint32_t NRead, NRemain; 195 ValueProfNode *NextStartNode = 0; 196 NRemain = VPDataReader->GetNumValueDataForSite(I, J); 197 if (!NRemain) 198 continue; 199 /* Read and write out value data in small chunks till it is done. */ 200 do { 201 NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain); 202 NextStartNode = 203 VPDataReader->GetValueData(I, /* ValueKind */ 204 J, /* Site */ 205 &VPDataArray[0], NextStartNode, NRead); 206 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0], 207 NRead * sizeof(InstrProfValueData))) 208 return -1; 209 NRemain -= NRead; 210 } while (NRemain != 0); 211 } 212 } 213 /* All done report success. */ 214 return 0; 215 } 216 217 static int writeValueProfData(ProfDataWriter *Writer, 218 VPDataReaderType *VPDataReader, 219 const __llvm_profile_data *DataBegin, 220 const __llvm_profile_data *DataEnd) { 221 ProfBufferIO *BufferIO; 222 const __llvm_profile_data *DI = 0; 223 224 if (!VPDataReader) 225 return 0; 226 227 BufferIO = lprofCreateBufferIO(Writer); 228 229 for (DI = DataBegin; DI < DataEnd; DI++) { 230 if (writeOneValueProfData(BufferIO, VPDataReader, DI)) 231 return -1; 232 } 233 234 if (lprofBufferIOFlush(BufferIO) != 0) 235 return -1; 236 lprofDeleteBufferIO(BufferIO); 237 238 return 0; 239 } 240 241 COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, 242 VPDataReaderType *VPDataReader, 243 int SkipNameDataWrite) { 244 /* Match logic in __llvm_profile_write_buffer(). */ 245 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); 246 const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); 247 const char *CountersBegin = __llvm_profile_begin_counters(); 248 const char *CountersEnd = __llvm_profile_end_counters(); 249 const char *BitmapBegin = __llvm_profile_begin_bitmap(); 250 const char *BitmapEnd = __llvm_profile_end_bitmap(); 251 const char *NamesBegin = __llvm_profile_begin_names(); 252 const char *NamesEnd = __llvm_profile_end_names(); 253 const VTableProfData *VTableBegin = __llvm_profile_begin_vtables(); 254 const VTableProfData *VTableEnd = __llvm_profile_end_vtables(); 255 const char *VNamesBegin = __llvm_profile_begin_vtabnames(); 256 const char *VNamesEnd = __llvm_profile_end_vtabnames(); 257 uint64_t Version = __llvm_profile_get_version(); 258 return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, 259 CountersEnd, BitmapBegin, BitmapEnd, VPDataReader, 260 NamesBegin, NamesEnd, VTableBegin, VTableEnd, 261 VNamesBegin, VNamesEnd, SkipNameDataWrite, Version); 262 } 263 264 COMPILER_RT_VISIBILITY int lprofWriteDataImpl( 265 ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, 266 const __llvm_profile_data *DataEnd, const char *CountersBegin, 267 const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, 268 VPDataReaderType *VPDataReader, const char *NamesBegin, 269 const char *NamesEnd, const VTableProfData *VTableBegin, 270 const VTableProfData *VTableEnd, const char *VNamesBegin, 271 const char *VNamesEnd, int SkipNameDataWrite, uint64_t Version) { 272 /* Calculate size of sections. */ 273 const uint64_t DataSectionSize = 274 __llvm_profile_get_data_size(DataBegin, DataEnd); 275 const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd); 276 const uint64_t CountersSectionSize = 277 __llvm_profile_get_counters_size(CountersBegin, CountersEnd); 278 const uint64_t NumCounters = 279 __llvm_profile_get_num_counters(CountersBegin, CountersEnd); 280 const uint64_t NumBitmapBytes = 281 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); 282 const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd); 283 const uint64_t NumVTables = 284 __llvm_profile_get_num_vtable(VTableBegin, VTableEnd); 285 const uint64_t VTableSectionSize = 286 __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd); 287 const uint64_t VNamesSize = 288 __llvm_profile_get_name_size(VNamesBegin, VNamesEnd); 289 290 /* Create the header. */ 291 __llvm_profile_header Header; 292 293 /* Determine how much padding is needed before/after the counters and after 294 * the names. */ 295 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, 296 PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames, 297 PaddingBytesAfterVTable, PaddingBytesAfterVNames; 298 if (__llvm_profile_get_padding_sizes_for_counters( 299 DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, 300 VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters, 301 &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, 302 &PaddingBytesAfterNames, &PaddingBytesAfterVTable, 303 &PaddingBytesAfterVNames) == -1) 304 return -1; 305 306 { 307 /* Initialize header structure. */ 308 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; 309 #include "profile/InstrProfData.inc" 310 } 311 Header.Version = Version; 312 313 /* On WIN64, label differences are truncated 32-bit values. Truncate 314 * CountersDelta to match. */ 315 #ifdef _WIN64 316 Header.CountersDelta = (uint32_t)Header.CountersDelta; 317 Header.BitmapDelta = (uint32_t)Header.BitmapDelta; 318 #endif 319 320 /* The data and names sections are omitted in lightweight mode. */ 321 if (NumData == 0 && NamesSize == 0) { 322 Header.CountersDelta = 0; 323 Header.NamesDelta = 0; 324 } 325 326 /* Write the profile header. */ 327 ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}}; 328 if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) 329 return -1; 330 331 /* Write the binary id lengths and data. */ 332 if (__llvm_write_binary_ids(Writer) == -1) 333 return -1; 334 335 /* Write the profile data. */ 336 ProfDataIOVec IOVecData[] = { 337 {DataBegin, sizeof(uint8_t), DataSectionSize, 0}, 338 {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, 339 {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, 340 {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, 341 {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, 342 {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, 343 {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, 344 {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}, 345 {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0}, 346 {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1}, 347 {SkipNameDataWrite ? NULL : VNamesBegin, sizeof(uint8_t), VNamesSize, 0}, 348 {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}}; 349 if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData))) 350 return -1; 351 352 /* Value profiling is not yet supported in continuous mode and profile 353 * correlation mode. */ 354 if (__llvm_profile_is_continuous_mode_enabled() || 355 (NumData == 0 && NamesSize == 0)) 356 return 0; 357 358 return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); 359 } 360 361 /* 362 * Write binary id length and then its data, because binary id does not 363 * have a fixed length. 364 */ 365 COMPILER_RT_VISIBILITY 366 int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen, 367 const uint8_t *BinaryIdData, 368 uint64_t BinaryIdPadding) { 369 ProfDataIOVec BinaryIdIOVec[] = { 370 {&BinaryIdLen, sizeof(uint64_t), 1, 0}, 371 {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0}, 372 {NULL, sizeof(uint8_t), BinaryIdPadding, 1}, 373 }; 374 if (Writer->Write(Writer, BinaryIdIOVec, 375 sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec))) 376 return -1; 377 378 /* Successfully wrote binary id, report success. */ 379 return 0; 380 } 381