1 /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ 2 |* 3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 |* See https://llvm.org/LICENSE.txt for license information. 5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 |* 7 \*===----------------------------------------------------------------------===*/ 8 9 // Note: This is linked into the Darwin kernel, and must remain compatible 10 // with freestanding compilation. See `darwin_add_builtin_libraries`. 11 12 #ifdef _MSC_VER 13 /* For _alloca */ 14 #include <malloc.h> 15 #endif 16 #include <string.h> 17 18 #include "InstrProfiling.h" 19 #include "InstrProfilingInternal.h" 20 #include "InstrProfilingPort.h" 21 22 #define INSTR_PROF_VALUE_PROF_DATA 23 #include "profile/InstrProfData.inc" 24 25 COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL; 26 static ProfBufferIO TheBufferIO; 27 #define VP_BUFFER_SIZE 8 * 1024 28 static uint8_t BufferIOBuffer[VP_BUFFER_SIZE]; 29 static InstrProfValueData VPDataArray[16]; 30 static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray); 31 32 COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0; 33 COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0; 34 35 /* The buffer writer is responsible in keeping writer state 36 * across the call. 37 */ 38 COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This, 39 ProfDataIOVec *IOVecs, 40 uint32_t NumIOVecs) { 41 uint32_t I; 42 char **Buffer = (char **)&This->WriterCtx; 43 for (I = 0; I < NumIOVecs; I++) { 44 size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; 45 if (IOVecs[I].Data) 46 memcpy(*Buffer, IOVecs[I].Data, Length); 47 else if (IOVecs[I].UseZeroPadding) { 48 /* Allocating the buffer should zero fill. */ 49 } 50 *Buffer += Length; 51 } 52 return 0; 53 } 54 55 static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter, 56 uint8_t *Buffer, uint32_t BufferSz) { 57 BufferIO->FileWriter = FileWriter; 58 BufferIO->OwnFileWriter = 0; 59 BufferIO->BufferStart = Buffer; 60 BufferIO->BufferSz = BufferSz; 61 BufferIO->CurOffset = 0; 62 } 63 64 COMPILER_RT_VISIBILITY ProfBufferIO * 65 lprofCreateBufferIO(ProfDataWriter *FileWriter) { 66 uint8_t *Buffer = DynamicBufferIOBuffer; 67 uint32_t BufferSize = VPBufferSize; 68 if (!Buffer) { 69 Buffer = &BufferIOBuffer[0]; 70 BufferSize = sizeof(BufferIOBuffer); 71 } 72 llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize); 73 return &TheBufferIO; 74 } 75 76 COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) { 77 if (BufferIO->OwnFileWriter) 78 FreeHook(BufferIO->FileWriter); 79 if (DynamicBufferIOBuffer) { 80 FreeHook(DynamicBufferIOBuffer); 81 DynamicBufferIOBuffer = 0; 82 VPBufferSize = 0; 83 } 84 } 85 86 COMPILER_RT_VISIBILITY int 87 lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) { 88 /* Buffer is not large enough, it is time to flush. */ 89 if (Size + BufferIO->CurOffset > BufferIO->BufferSz) { 90 if (lprofBufferIOFlush(BufferIO) != 0) 91 return -1; 92 } 93 /* Special case, bypass the buffer completely. */ 94 ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}}; 95 if (Size > BufferIO->BufferSz) { 96 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) 97 return -1; 98 } else { 99 /* Write the data to buffer */ 100 uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset; 101 ProfDataWriter BufferWriter; 102 initBufferWriter(&BufferWriter, (char *)Buffer); 103 lprofBufferWriter(&BufferWriter, IO, 1); 104 BufferIO->CurOffset = 105 (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart; 106 } 107 return 0; 108 } 109 110 COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) { 111 if (BufferIO->CurOffset) { 112 ProfDataIOVec IO[] = { 113 {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}}; 114 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) 115 return -1; 116 BufferIO->CurOffset = 0; 117 } 118 return 0; 119 } 120 121 /* Write out value profile data for function specified with \c Data. 122 * The implementation does not use the method \c serializeValueProfData 123 * which depends on dynamic memory allocation. In this implementation, 124 * value profile data is written out to \c BufferIO piecemeal. 125 */ 126 static int writeOneValueProfData(ProfBufferIO *BufferIO, 127 VPDataReaderType *VPDataReader, 128 const __llvm_profile_data *Data) { 129 unsigned I, NumValueKinds = 0; 130 ValueProfData VPHeader; 131 uint8_t *SiteCountArray[IPVK_Last + 1]; 132 133 for (I = 0; I <= IPVK_Last; I++) { 134 if (!Data->NumValueSites[I]) 135 SiteCountArray[I] = 0; 136 else { 137 uint32_t Sz = 138 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - 139 offsetof(ValueProfRecord, SiteCountArray); 140 /* Only use alloca for this small byte array to avoid excessive 141 * stack growth. */ 142 SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz); 143 memset(SiteCountArray[I], 0, Sz); 144 } 145 } 146 147 /* If NumValueKinds returned is 0, there is nothing to write, report 148 success and return. This should match the raw profile reader's behavior. */ 149 if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray))) 150 return 0; 151 152 /* First write the header structure. */ 153 VPHeader.TotalSize = VPDataReader->GetValueProfDataSize(); 154 VPHeader.NumValueKinds = NumValueKinds; 155 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader, 156 sizeof(ValueProfData))) 157 return -1; 158 159 /* Make sure nothing else needs to be written before value profile 160 * records. */ 161 if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) != 162 (void *)(&VPHeader + 1)) 163 return -1; 164 165 /* Write out the value profile record for each value kind 166 * one by one. */ 167 for (I = 0; I <= IPVK_Last; I++) { 168 uint32_t J; 169 ValueProfRecord RecordHeader; 170 /* The size of the value prof record header without counting the 171 * site count array .*/ 172 uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray); 173 uint32_t SiteCountArraySize; 174 175 if (!Data->NumValueSites[I]) 176 continue; 177 178 /* Write out the record header. */ 179 RecordHeader.Kind = I; 180 RecordHeader.NumValueSites = Data->NumValueSites[I]; 181 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader, 182 RecordHeaderSize)) 183 return -1; 184 185 /* Write out the site value count array including padding space. */ 186 SiteCountArraySize = 187 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - 188 RecordHeaderSize; 189 if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize)) 190 return -1; 191 192 /* Write out the value profile data for each value site. */ 193 for (J = 0; J < Data->NumValueSites[I]; J++) { 194 uint32_t NRead, NRemain; 195 ValueProfNode *NextStartNode = 0; 196 NRemain = VPDataReader->GetNumValueDataForSite(I, J); 197 if (!NRemain) 198 continue; 199 /* Read and write out value data in small chunks till it is done. */ 200 do { 201 NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain); 202 NextStartNode = 203 VPDataReader->GetValueData(I, /* ValueKind */ 204 J, /* Site */ 205 &VPDataArray[0], NextStartNode, NRead); 206 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0], 207 NRead * sizeof(InstrProfValueData))) 208 return -1; 209 NRemain -= NRead; 210 } while (NRemain != 0); 211 } 212 } 213 /* All done report success. */ 214 return 0; 215 } 216 217 static int writeValueProfData(ProfDataWriter *Writer, 218 VPDataReaderType *VPDataReader, 219 const __llvm_profile_data *DataBegin, 220 const __llvm_profile_data *DataEnd) { 221 ProfBufferIO *BufferIO; 222 const __llvm_profile_data *DI = 0; 223 224 if (!VPDataReader) 225 return 0; 226 227 BufferIO = lprofCreateBufferIO(Writer); 228 229 for (DI = DataBegin; DI < DataEnd; DI++) { 230 if (writeOneValueProfData(BufferIO, VPDataReader, DI)) 231 return -1; 232 } 233 234 if (lprofBufferIOFlush(BufferIO) != 0) 235 return -1; 236 lprofDeleteBufferIO(BufferIO); 237 238 return 0; 239 } 240 241 COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, 242 VPDataReaderType *VPDataReader, 243 int SkipNameDataWrite) { 244 /* Match logic in __llvm_profile_write_buffer(). */ 245 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); 246 const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); 247 const char *CountersBegin = __llvm_profile_begin_counters(); 248 const char *CountersEnd = __llvm_profile_end_counters(); 249 const char *NamesBegin = __llvm_profile_begin_names(); 250 const char *NamesEnd = __llvm_profile_end_names(); 251 return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, 252 CountersEnd, VPDataReader, NamesBegin, NamesEnd, 253 SkipNameDataWrite); 254 } 255 256 COMPILER_RT_VISIBILITY int 257 lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, 258 const __llvm_profile_data *DataEnd, 259 const char *CountersBegin, const char *CountersEnd, 260 VPDataReaderType *VPDataReader, const char *NamesBegin, 261 const char *NamesEnd, int SkipNameDataWrite) { 262 int DebugInfoCorrelate = 263 (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL; 264 265 /* Calculate size of sections. */ 266 const uint64_t DataSize = 267 DebugInfoCorrelate ? 0 : __llvm_profile_get_data_size(DataBegin, DataEnd); 268 const uint64_t NumData = 269 DebugInfoCorrelate ? 0 : __llvm_profile_get_num_data(DataBegin, DataEnd); 270 const uint64_t CountersSize = 271 __llvm_profile_get_counters_size(CountersBegin, CountersEnd); 272 const uint64_t NumCounters = 273 __llvm_profile_get_num_counters(CountersBegin, CountersEnd); 274 const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin; 275 276 /* Create the header. */ 277 __llvm_profile_header Header; 278 279 if (!NumData && (!DebugInfoCorrelate || !NumCounters)) 280 return 0; 281 282 /* Determine how much padding is needed before/after the counters and after 283 * the names. */ 284 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, 285 PaddingBytesAfterNames; 286 __llvm_profile_get_padding_sizes_for_counters( 287 DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, 288 &PaddingBytesAfterCounters, &PaddingBytesAfterNames); 289 290 { 291 // TODO: Unfortunately the header's fields are named DataSize and 292 // CountersSize when they should be named NumData and NumCounters, 293 // respectively. 294 const uint64_t CountersSize = NumCounters; 295 const uint64_t DataSize = NumData; 296 /* Initialize header structure. */ 297 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; 298 #include "profile/InstrProfData.inc" 299 } 300 301 /* On WIN64, label differences are truncated 32-bit values. Truncate 302 * CountersDelta to match. */ 303 #ifdef _WIN64 304 Header.CountersDelta = (uint32_t)Header.CountersDelta; 305 #endif 306 307 /* The data and names sections are omitted in lightweight mode. */ 308 if (DebugInfoCorrelate) { 309 Header.CountersDelta = 0; 310 Header.NamesDelta = 0; 311 } 312 313 /* Write the profile header. */ 314 ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}}; 315 if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) 316 return -1; 317 318 /* Write the binary id lengths and data. */ 319 if (__llvm_write_binary_ids(Writer) == -1) 320 return -1; 321 322 /* Write the profile data. */ 323 ProfDataIOVec IOVecData[] = { 324 {DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSize, 0}, 325 {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, 326 {CountersBegin, sizeof(uint8_t), CountersSize, 0}, 327 {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, 328 {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin, 329 sizeof(uint8_t), NamesSize, 0}, 330 {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; 331 if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData))) 332 return -1; 333 334 /* Value profiling is not yet supported in continuous mode. */ 335 if (__llvm_profile_is_continuous_mode_enabled()) 336 return 0; 337 338 return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); 339 } 340