1 /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ 2 |* 3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 |* See https://llvm.org/LICENSE.txt for license information. 5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 |* 7 \*===----------------------------------------------------------------------===*/ 8 9 // Note: This is linked into the Darwin kernel, and must remain compatible 10 // with freestanding compilation. See `darwin_add_builtin_libraries`. 11 12 #ifdef _MSC_VER 13 /* For _alloca */ 14 #include <malloc.h> 15 #endif 16 #include <string.h> 17 18 #include "InstrProfiling.h" 19 #include "InstrProfilingInternal.h" 20 #include "InstrProfilingPort.h" 21 22 #define INSTR_PROF_VALUE_PROF_DATA 23 #include "profile/InstrProfData.inc" 24 25 COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL; 26 static ProfBufferIO TheBufferIO; 27 #define VP_BUFFER_SIZE 8 * 1024 28 static uint8_t BufferIOBuffer[VP_BUFFER_SIZE]; 29 static InstrProfValueData VPDataArray[16]; 30 static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray); 31 32 COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0; 33 COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0; 34 35 /* The buffer writer is responsible in keeping writer state 36 * across the call. 37 */ 38 COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This, 39 ProfDataIOVec *IOVecs, 40 uint32_t NumIOVecs) { 41 uint32_t I; 42 char **Buffer = (char **)&This->WriterCtx; 43 for (I = 0; I < NumIOVecs; I++) { 44 size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; 45 if (IOVecs[I].Data) 46 memcpy(*Buffer, IOVecs[I].Data, Length); 47 else if (IOVecs[I].UseZeroPadding) { 48 /* Allocating the buffer should zero fill. */ 49 } 50 *Buffer += Length; 51 } 52 return 0; 53 } 54 55 static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter, 56 uint8_t *Buffer, uint32_t BufferSz) { 57 BufferIO->FileWriter = FileWriter; 58 BufferIO->OwnFileWriter = 0; 59 BufferIO->BufferStart = Buffer; 60 BufferIO->BufferSz = BufferSz; 61 BufferIO->CurOffset = 0; 62 } 63 64 COMPILER_RT_VISIBILITY ProfBufferIO * 65 lprofCreateBufferIO(ProfDataWriter *FileWriter) { 66 uint8_t *Buffer = DynamicBufferIOBuffer; 67 uint32_t BufferSize = VPBufferSize; 68 if (!Buffer) { 69 Buffer = &BufferIOBuffer[0]; 70 BufferSize = sizeof(BufferIOBuffer); 71 } 72 llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize); 73 return &TheBufferIO; 74 } 75 76 COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) { 77 if (BufferIO->OwnFileWriter) 78 FreeHook(BufferIO->FileWriter); 79 if (DynamicBufferIOBuffer) { 80 FreeHook(DynamicBufferIOBuffer); 81 DynamicBufferIOBuffer = 0; 82 VPBufferSize = 0; 83 } 84 } 85 86 COMPILER_RT_VISIBILITY int 87 lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) { 88 /* Buffer is not large enough, it is time to flush. */ 89 if (Size + BufferIO->CurOffset > BufferIO->BufferSz) { 90 if (lprofBufferIOFlush(BufferIO) != 0) 91 return -1; 92 } 93 /* Special case, bypass the buffer completely. */ 94 ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}}; 95 if (Size > BufferIO->BufferSz) { 96 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) 97 return -1; 98 } else { 99 /* Write the data to buffer */ 100 uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset; 101 ProfDataWriter BufferWriter; 102 initBufferWriter(&BufferWriter, (char *)Buffer); 103 lprofBufferWriter(&BufferWriter, IO, 1); 104 BufferIO->CurOffset = 105 (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart; 106 } 107 return 0; 108 } 109 110 COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) { 111 if (BufferIO->CurOffset) { 112 ProfDataIOVec IO[] = { 113 {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}}; 114 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) 115 return -1; 116 BufferIO->CurOffset = 0; 117 } 118 return 0; 119 } 120 121 /* Write out value profile data for function specified with \c Data. 122 * The implementation does not use the method \c serializeValueProfData 123 * which depends on dynamic memory allocation. In this implementation, 124 * value profile data is written out to \c BufferIO piecemeal. 125 */ 126 static int writeOneValueProfData(ProfBufferIO *BufferIO, 127 VPDataReaderType *VPDataReader, 128 const __llvm_profile_data *Data) { 129 unsigned I, NumValueKinds = 0; 130 ValueProfData VPHeader; 131 uint8_t *SiteCountArray[IPVK_Last + 1]; 132 133 for (I = 0; I <= IPVK_Last; I++) { 134 if (!Data->NumValueSites[I]) 135 SiteCountArray[I] = 0; 136 else { 137 uint32_t Sz = 138 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - 139 offsetof(ValueProfRecord, SiteCountArray); 140 /* Only use alloca for this small byte array to avoid excessive 141 * stack growth. */ 142 SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz); 143 memset(SiteCountArray[I], 0, Sz); 144 } 145 } 146 147 /* If NumValueKinds returned is 0, there is nothing to write, report 148 success and return. This should match the raw profile reader's behavior. */ 149 if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray))) 150 return 0; 151 152 /* First write the header structure. */ 153 VPHeader.TotalSize = VPDataReader->GetValueProfDataSize(); 154 VPHeader.NumValueKinds = NumValueKinds; 155 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader, 156 sizeof(ValueProfData))) 157 return -1; 158 159 /* Make sure nothing else needs to be written before value profile 160 * records. */ 161 if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) != 162 (void *)(&VPHeader + 1)) 163 return -1; 164 165 /* Write out the value profile record for each value kind 166 * one by one. */ 167 for (I = 0; I <= IPVK_Last; I++) { 168 uint32_t J; 169 ValueProfRecord RecordHeader; 170 /* The size of the value prof record header without counting the 171 * site count array .*/ 172 uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray); 173 uint32_t SiteCountArraySize; 174 175 if (!Data->NumValueSites[I]) 176 continue; 177 178 /* Write out the record header. */ 179 RecordHeader.Kind = I; 180 RecordHeader.NumValueSites = Data->NumValueSites[I]; 181 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader, 182 RecordHeaderSize)) 183 return -1; 184 185 /* Write out the site value count array including padding space. */ 186 SiteCountArraySize = 187 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - 188 RecordHeaderSize; 189 if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize)) 190 return -1; 191 192 /* Write out the value profile data for each value site. */ 193 for (J = 0; J < Data->NumValueSites[I]; J++) { 194 uint32_t NRead, NRemain; 195 ValueProfNode *NextStartNode = 0; 196 NRemain = VPDataReader->GetNumValueDataForSite(I, J); 197 if (!NRemain) 198 continue; 199 /* Read and write out value data in small chunks till it is done. */ 200 do { 201 NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain); 202 NextStartNode = 203 VPDataReader->GetValueData(I, /* ValueKind */ 204 J, /* Site */ 205 &VPDataArray[0], NextStartNode, NRead); 206 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0], 207 NRead * sizeof(InstrProfValueData))) 208 return -1; 209 NRemain -= NRead; 210 } while (NRemain != 0); 211 } 212 } 213 /* All done report success. */ 214 return 0; 215 } 216 217 static int writeValueProfData(ProfDataWriter *Writer, 218 VPDataReaderType *VPDataReader, 219 const __llvm_profile_data *DataBegin, 220 const __llvm_profile_data *DataEnd) { 221 ProfBufferIO *BufferIO; 222 const __llvm_profile_data *DI = 0; 223 224 if (!VPDataReader) 225 return 0; 226 227 BufferIO = lprofCreateBufferIO(Writer); 228 229 for (DI = DataBegin; DI < DataEnd; DI++) { 230 if (writeOneValueProfData(BufferIO, VPDataReader, DI)) 231 return -1; 232 } 233 234 if (lprofBufferIOFlush(BufferIO) != 0) 235 return -1; 236 lprofDeleteBufferIO(BufferIO); 237 238 return 0; 239 } 240 241 COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, 242 VPDataReaderType *VPDataReader, 243 int SkipNameDataWrite) { 244 /* Match logic in __llvm_profile_write_buffer(). */ 245 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); 246 const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); 247 const char *CountersBegin = __llvm_profile_begin_counters(); 248 const char *CountersEnd = __llvm_profile_end_counters(); 249 const char *BitmapBegin = __llvm_profile_begin_bitmap(); 250 const char *BitmapEnd = __llvm_profile_end_bitmap(); 251 const char *NamesBegin = __llvm_profile_begin_names(); 252 const char *NamesEnd = __llvm_profile_end_names(); 253 return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, 254 CountersEnd, BitmapBegin, BitmapEnd, VPDataReader, 255 NamesBegin, NamesEnd, SkipNameDataWrite); 256 } 257 258 COMPILER_RT_VISIBILITY int 259 lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, 260 const __llvm_profile_data *DataEnd, 261 const char *CountersBegin, const char *CountersEnd, 262 const char *BitmapBegin, const char *BitmapEnd, 263 VPDataReaderType *VPDataReader, const char *NamesBegin, 264 const char *NamesEnd, int SkipNameDataWrite) { 265 /* Calculate size of sections. */ 266 const uint64_t DataSectionSize = 267 __llvm_profile_get_data_size(DataBegin, DataEnd); 268 const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd); 269 const uint64_t CountersSectionSize = 270 __llvm_profile_get_counters_size(CountersBegin, CountersEnd); 271 const uint64_t NumCounters = 272 __llvm_profile_get_num_counters(CountersBegin, CountersEnd); 273 const uint64_t NumBitmapBytes = 274 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); 275 const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd); 276 277 /* Create the header. */ 278 __llvm_profile_header Header; 279 280 /* Determine how much padding is needed before/after the counters and after 281 * the names. */ 282 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, 283 PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; 284 __llvm_profile_get_padding_sizes_for_counters( 285 DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, 286 &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, 287 &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); 288 289 { 290 /* Initialize header structure. */ 291 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; 292 #include "profile/InstrProfData.inc" 293 } 294 295 /* On WIN64, label differences are truncated 32-bit values. Truncate 296 * CountersDelta to match. */ 297 #ifdef _WIN64 298 Header.CountersDelta = (uint32_t)Header.CountersDelta; 299 Header.BitmapDelta = (uint32_t)Header.BitmapDelta; 300 #endif 301 302 /* The data and names sections are omitted in lightweight mode. */ 303 if (NumData == 0 && NamesSize == 0) { 304 Header.CountersDelta = 0; 305 Header.NamesDelta = 0; 306 } 307 308 /* Write the profile header. */ 309 ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}}; 310 if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) 311 return -1; 312 313 /* Write the binary id lengths and data. */ 314 if (__llvm_write_binary_ids(Writer) == -1) 315 return -1; 316 317 /* Write the profile data. */ 318 ProfDataIOVec IOVecData[] = { 319 {DataBegin, sizeof(uint8_t), DataSectionSize, 0}, 320 {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, 321 {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, 322 {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, 323 {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, 324 {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, 325 {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, 326 {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; 327 if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData))) 328 return -1; 329 330 /* Value profiling is not yet supported in continuous mode and profile 331 * correlation mode. */ 332 if (__llvm_profile_is_continuous_mode_enabled() || 333 (NumData == 0 && NamesSize == 0)) 334 return 0; 335 336 return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); 337 } 338 339 /* 340 * Write binary id length and then its data, because binary id does not 341 * have a fixed length. 342 */ 343 COMPILER_RT_VISIBILITY 344 int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen, 345 const uint8_t *BinaryIdData, 346 uint64_t BinaryIdPadding) { 347 ProfDataIOVec BinaryIdIOVec[] = { 348 {&BinaryIdLen, sizeof(uint64_t), 1, 0}, 349 {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0}, 350 {NULL, sizeof(uint8_t), BinaryIdPadding, 1}, 351 }; 352 if (Writer->Write(Writer, BinaryIdIOVec, 353 sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec))) 354 return -1; 355 356 /* Successfully wrote binary id, report success. */ 357 return 0; 358 } 359