1 /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ 2 |* 3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 |* See https://llvm.org/LICENSE.txt for license information. 5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 |* 7 \*===----------------------------------------------------------------------===*/ 8 9 #ifdef _MSC_VER 10 /* For _alloca */ 11 #include <malloc.h> 12 #endif 13 #include <string.h> 14 15 #include "InstrProfiling.h" 16 #include "InstrProfilingInternal.h" 17 #include "InstrProfilingPort.h" 18 19 #define INSTR_PROF_VALUE_PROF_DATA 20 #include "profile/InstrProfData.inc" 21 22 COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL; 23 static ProfBufferIO TheBufferIO; 24 #define VP_BUFFER_SIZE 8 * 1024 25 static uint8_t BufferIOBuffer[VP_BUFFER_SIZE]; 26 static InstrProfValueData VPDataArray[16]; 27 static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray); 28 29 COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0; 30 COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0; 31 32 /* The buffer writer is reponsponsible in keeping writer state 33 * across the call. 34 */ 35 COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This, 36 ProfDataIOVec *IOVecs, 37 uint32_t NumIOVecs) { 38 uint32_t I; 39 char **Buffer = (char **)&This->WriterCtx; 40 for (I = 0; I < NumIOVecs; I++) { 41 size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; 42 if (IOVecs[I].Data) 43 memcpy(*Buffer, IOVecs[I].Data, Length); 44 else if (IOVecs[I].UseZeroPadding) { 45 /* Allocating the buffer should zero fill. */ 46 } 47 *Buffer += Length; 48 } 49 return 0; 50 } 51 52 static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter, 53 uint8_t *Buffer, uint32_t BufferSz) { 54 BufferIO->FileWriter = FileWriter; 55 BufferIO->OwnFileWriter = 0; 56 BufferIO->BufferStart = Buffer; 57 BufferIO->BufferSz = BufferSz; 58 BufferIO->CurOffset = 0; 59 } 60 61 COMPILER_RT_VISIBILITY ProfBufferIO * 62 lprofCreateBufferIO(ProfDataWriter *FileWriter) { 63 uint8_t *Buffer = DynamicBufferIOBuffer; 64 uint32_t BufferSize = VPBufferSize; 65 if (!Buffer) { 66 Buffer = &BufferIOBuffer[0]; 67 BufferSize = sizeof(BufferIOBuffer); 68 } 69 llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize); 70 return &TheBufferIO; 71 } 72 73 COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) { 74 if (BufferIO->OwnFileWriter) 75 FreeHook(BufferIO->FileWriter); 76 if (DynamicBufferIOBuffer) { 77 FreeHook(DynamicBufferIOBuffer); 78 DynamicBufferIOBuffer = 0; 79 VPBufferSize = 0; 80 } 81 } 82 83 COMPILER_RT_VISIBILITY int 84 lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) { 85 /* Buffer is not large enough, it is time to flush. */ 86 if (Size + BufferIO->CurOffset > BufferIO->BufferSz) { 87 if (lprofBufferIOFlush(BufferIO) != 0) 88 return -1; 89 } 90 /* Special case, bypass the buffer completely. */ 91 ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}}; 92 if (Size > BufferIO->BufferSz) { 93 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) 94 return -1; 95 } else { 96 /* Write the data to buffer */ 97 uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset; 98 ProfDataWriter BufferWriter; 99 initBufferWriter(&BufferWriter, (char *)Buffer); 100 lprofBufferWriter(&BufferWriter, IO, 1); 101 BufferIO->CurOffset = 102 (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart; 103 } 104 return 0; 105 } 106 107 COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) { 108 if (BufferIO->CurOffset) { 109 ProfDataIOVec IO[] = { 110 {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}}; 111 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) 112 return -1; 113 BufferIO->CurOffset = 0; 114 } 115 return 0; 116 } 117 118 /* Write out value profile data for function specified with \c Data. 119 * The implementation does not use the method \c serializeValueProfData 120 * which depends on dynamic memory allocation. In this implementation, 121 * value profile data is written out to \c BufferIO piecemeal. 122 */ 123 static int writeOneValueProfData(ProfBufferIO *BufferIO, 124 VPDataReaderType *VPDataReader, 125 const __llvm_profile_data *Data) { 126 unsigned I, NumValueKinds = 0; 127 ValueProfData VPHeader; 128 uint8_t *SiteCountArray[IPVK_Last + 1]; 129 130 for (I = 0; I <= IPVK_Last; I++) { 131 if (!Data->NumValueSites[I]) 132 SiteCountArray[I] = 0; 133 else { 134 uint32_t Sz = 135 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - 136 offsetof(ValueProfRecord, SiteCountArray); 137 /* Only use alloca for this small byte array to avoid excessive 138 * stack growth. */ 139 SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz); 140 memset(SiteCountArray[I], 0, Sz); 141 } 142 } 143 144 /* If NumValueKinds returned is 0, there is nothing to write, report 145 success and return. This should match the raw profile reader's behavior. */ 146 if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray))) 147 return 0; 148 149 /* First write the header structure. */ 150 VPHeader.TotalSize = VPDataReader->GetValueProfDataSize(); 151 VPHeader.NumValueKinds = NumValueKinds; 152 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader, 153 sizeof(ValueProfData))) 154 return -1; 155 156 /* Make sure nothing else needs to be written before value profile 157 * records. */ 158 if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) != 159 (void *)(&VPHeader + 1)) 160 return -1; 161 162 /* Write out the value profile record for each value kind 163 * one by one. */ 164 for (I = 0; I <= IPVK_Last; I++) { 165 uint32_t J; 166 ValueProfRecord RecordHeader; 167 /* The size of the value prof record header without counting the 168 * site count array .*/ 169 uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray); 170 uint32_t SiteCountArraySize; 171 172 if (!Data->NumValueSites[I]) 173 continue; 174 175 /* Write out the record header. */ 176 RecordHeader.Kind = I; 177 RecordHeader.NumValueSites = Data->NumValueSites[I]; 178 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader, 179 RecordHeaderSize)) 180 return -1; 181 182 /* Write out the site value count array including padding space. */ 183 SiteCountArraySize = 184 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - 185 RecordHeaderSize; 186 if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize)) 187 return -1; 188 189 /* Write out the value profile data for each value site. */ 190 for (J = 0; J < Data->NumValueSites[I]; J++) { 191 uint32_t NRead, NRemain; 192 ValueProfNode *NextStartNode = 0; 193 NRemain = VPDataReader->GetNumValueDataForSite(I, J); 194 if (!NRemain) 195 continue; 196 /* Read and write out value data in small chunks till it is done. */ 197 do { 198 NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain); 199 NextStartNode = 200 VPDataReader->GetValueData(I, /* ValueKind */ 201 J, /* Site */ 202 &VPDataArray[0], NextStartNode, NRead); 203 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0], 204 NRead * sizeof(InstrProfValueData))) 205 return -1; 206 NRemain -= NRead; 207 } while (NRemain != 0); 208 } 209 } 210 /* All done report success. */ 211 return 0; 212 } 213 214 static int writeValueProfData(ProfDataWriter *Writer, 215 VPDataReaderType *VPDataReader, 216 const __llvm_profile_data *DataBegin, 217 const __llvm_profile_data *DataEnd) { 218 ProfBufferIO *BufferIO; 219 const __llvm_profile_data *DI = 0; 220 221 if (!VPDataReader) 222 return 0; 223 224 BufferIO = lprofCreateBufferIO(Writer); 225 226 for (DI = DataBegin; DI < DataEnd; DI++) { 227 if (writeOneValueProfData(BufferIO, VPDataReader, DI)) 228 return -1; 229 } 230 231 if (lprofBufferIOFlush(BufferIO) != 0) 232 return -1; 233 lprofDeleteBufferIO(BufferIO); 234 235 return 0; 236 } 237 238 COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, 239 VPDataReaderType *VPDataReader, 240 int SkipNameDataWrite) { 241 /* Match logic in __llvm_profile_write_buffer(). */ 242 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); 243 const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); 244 const uint64_t *CountersBegin = __llvm_profile_begin_counters(); 245 const uint64_t *CountersEnd = __llvm_profile_end_counters(); 246 const char *NamesBegin = __llvm_profile_begin_names(); 247 const char *NamesEnd = __llvm_profile_end_names(); 248 return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, 249 CountersEnd, VPDataReader, NamesBegin, NamesEnd, 250 SkipNameDataWrite); 251 } 252 253 COMPILER_RT_VISIBILITY int 254 lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, 255 const __llvm_profile_data *DataEnd, 256 const uint64_t *CountersBegin, const uint64_t *CountersEnd, 257 VPDataReaderType *VPDataReader, const char *NamesBegin, 258 const char *NamesEnd, int SkipNameDataWrite) { 259 260 /* Calculate size of sections. */ 261 const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); 262 const uint64_t CountersSize = CountersEnd - CountersBegin; 263 const uint64_t NamesSize = NamesEnd - NamesBegin; 264 265 /* Create the header. */ 266 __llvm_profile_header Header; 267 268 if (!DataSize) 269 return 0; 270 271 /* Determine how much padding is needed before/after the counters and after 272 * the names. */ 273 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, 274 PaddingBytesAfterNames; 275 __llvm_profile_get_padding_sizes_for_counters( 276 DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, 277 &PaddingBytesAfterCounters, &PaddingBytesAfterNames); 278 279 /* Initialize header structure. */ 280 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; 281 #include "profile/InstrProfData.inc" 282 283 /* Write the data. */ 284 ProfDataIOVec IOVec[] = { 285 {&Header, sizeof(__llvm_profile_header), 1, 0}, 286 {DataBegin, sizeof(__llvm_profile_data), DataSize, 0}, 287 {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, 288 {CountersBegin, sizeof(uint64_t), CountersSize, 0}, 289 {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, 290 {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, 291 {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; 292 if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) 293 return -1; 294 295 /* Value profiling is not yet supported in continuous mode. */ 296 if (__llvm_profile_is_continuous_mode_enabled()) 297 return 0; 298 299 return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); 300 } 301