1 /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ 2 |* 3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 |* See https://llvm.org/LICENSE.txt for license information. 5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 |* 7 \*===----------------------------------------------------------------------===*/ 8 9 #ifdef _MSC_VER 10 /* For _alloca */ 11 #include <malloc.h> 12 #endif 13 #include <string.h> 14 15 #include "InstrProfiling.h" 16 #include "InstrProfilingInternal.h" 17 18 #define INSTR_PROF_VALUE_PROF_DATA 19 #include "InstrProfData.inc" 20 21 COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL; 22 static ProfBufferIO TheBufferIO; 23 #define VP_BUFFER_SIZE 8 * 1024 24 static uint8_t BufferIOBuffer[VP_BUFFER_SIZE]; 25 static InstrProfValueData VPDataArray[16]; 26 static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray); 27 28 COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0; 29 COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0; 30 31 /* The buffer writer is reponsponsible in keeping writer state 32 * across the call. 33 */ 34 COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This, 35 ProfDataIOVec *IOVecs, 36 uint32_t NumIOVecs) { 37 uint32_t I; 38 char **Buffer = (char **)&This->WriterCtx; 39 for (I = 0; I < NumIOVecs; I++) { 40 size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; 41 if (IOVecs[I].Data) 42 memcpy(*Buffer, IOVecs[I].Data, Length); 43 *Buffer += Length; 44 } 45 return 0; 46 } 47 48 static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter, 49 uint8_t *Buffer, uint32_t BufferSz) { 50 BufferIO->FileWriter = FileWriter; 51 BufferIO->OwnFileWriter = 0; 52 BufferIO->BufferStart = Buffer; 53 BufferIO->BufferSz = BufferSz; 54 BufferIO->CurOffset = 0; 55 } 56 57 COMPILER_RT_VISIBILITY ProfBufferIO * 58 lprofCreateBufferIO(ProfDataWriter *FileWriter) { 59 uint8_t *Buffer = DynamicBufferIOBuffer; 60 uint32_t BufferSize = VPBufferSize; 61 if (!Buffer) { 62 Buffer = &BufferIOBuffer[0]; 63 BufferSize = sizeof(BufferIOBuffer); 64 } 65 llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize); 66 return &TheBufferIO; 67 } 68 69 COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) { 70 if (BufferIO->OwnFileWriter) 71 FreeHook(BufferIO->FileWriter); 72 if (DynamicBufferIOBuffer) { 73 FreeHook(DynamicBufferIOBuffer); 74 DynamicBufferIOBuffer = 0; 75 VPBufferSize = 0; 76 } 77 } 78 79 COMPILER_RT_VISIBILITY int 80 lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) { 81 /* Buffer is not large enough, it is time to flush. */ 82 if (Size + BufferIO->CurOffset > BufferIO->BufferSz) { 83 if (lprofBufferIOFlush(BufferIO) != 0) 84 return -1; 85 } 86 /* Special case, bypass the buffer completely. */ 87 ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size}}; 88 if (Size > BufferIO->BufferSz) { 89 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) 90 return -1; 91 } else { 92 /* Write the data to buffer */ 93 uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset; 94 ProfDataWriter BufferWriter; 95 initBufferWriter(&BufferWriter, (char *)Buffer); 96 lprofBufferWriter(&BufferWriter, IO, 1); 97 BufferIO->CurOffset = 98 (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart; 99 } 100 return 0; 101 } 102 103 COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) { 104 if (BufferIO->CurOffset) { 105 ProfDataIOVec IO[] = { 106 {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset}}; 107 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) 108 return -1; 109 BufferIO->CurOffset = 0; 110 } 111 return 0; 112 } 113 114 /* Write out value profile data for function specified with \c Data. 115 * The implementation does not use the method \c serializeValueProfData 116 * which depends on dynamic memory allocation. In this implementation, 117 * value profile data is written out to \c BufferIO piecemeal. 118 */ 119 static int writeOneValueProfData(ProfBufferIO *BufferIO, 120 VPDataReaderType *VPDataReader, 121 const __llvm_profile_data *Data) { 122 unsigned I, NumValueKinds = 0; 123 ValueProfData VPHeader; 124 uint8_t *SiteCountArray[IPVK_Last + 1]; 125 126 for (I = 0; I <= IPVK_Last; I++) { 127 if (!Data->NumValueSites[I]) 128 SiteCountArray[I] = 0; 129 else { 130 uint32_t Sz = 131 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - 132 offsetof(ValueProfRecord, SiteCountArray); 133 /* Only use alloca for this small byte array to avoid excessive 134 * stack growth. */ 135 SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz); 136 memset(SiteCountArray[I], 0, Sz); 137 } 138 } 139 140 /* If NumValueKinds returned is 0, there is nothing to write, report 141 success and return. This should match the raw profile reader's behavior. */ 142 if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray))) 143 return 0; 144 145 /* First write the header structure. */ 146 VPHeader.TotalSize = VPDataReader->GetValueProfDataSize(); 147 VPHeader.NumValueKinds = NumValueKinds; 148 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader, 149 sizeof(ValueProfData))) 150 return -1; 151 152 /* Make sure nothing else needs to be written before value profile 153 * records. */ 154 if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) != 155 (void *)(&VPHeader + 1)) 156 return -1; 157 158 /* Write out the value profile record for each value kind 159 * one by one. */ 160 for (I = 0; I <= IPVK_Last; I++) { 161 uint32_t J; 162 ValueProfRecord RecordHeader; 163 /* The size of the value prof record header without counting the 164 * site count array .*/ 165 uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray); 166 uint32_t SiteCountArraySize; 167 168 if (!Data->NumValueSites[I]) 169 continue; 170 171 /* Write out the record header. */ 172 RecordHeader.Kind = I; 173 RecordHeader.NumValueSites = Data->NumValueSites[I]; 174 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader, 175 RecordHeaderSize)) 176 return -1; 177 178 /* Write out the site value count array including padding space. */ 179 SiteCountArraySize = 180 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - 181 RecordHeaderSize; 182 if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize)) 183 return -1; 184 185 /* Write out the value profile data for each value site. */ 186 for (J = 0; J < Data->NumValueSites[I]; J++) { 187 uint32_t NRead, NRemain; 188 ValueProfNode *NextStartNode = 0; 189 NRemain = VPDataReader->GetNumValueDataForSite(I, J); 190 if (!NRemain) 191 continue; 192 /* Read and write out value data in small chunks till it is done. */ 193 do { 194 NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain); 195 NextStartNode = 196 VPDataReader->GetValueData(I, /* ValueKind */ 197 J, /* Site */ 198 &VPDataArray[0], NextStartNode, NRead); 199 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0], 200 NRead * sizeof(InstrProfValueData))) 201 return -1; 202 NRemain -= NRead; 203 } while (NRemain != 0); 204 } 205 } 206 /* All done report success. */ 207 return 0; 208 } 209 210 static int writeValueProfData(ProfDataWriter *Writer, 211 VPDataReaderType *VPDataReader, 212 const __llvm_profile_data *DataBegin, 213 const __llvm_profile_data *DataEnd) { 214 ProfBufferIO *BufferIO; 215 const __llvm_profile_data *DI = 0; 216 217 if (!VPDataReader) 218 return 0; 219 220 BufferIO = lprofCreateBufferIO(Writer); 221 222 for (DI = DataBegin; DI < DataEnd; DI++) { 223 if (writeOneValueProfData(BufferIO, VPDataReader, DI)) 224 return -1; 225 } 226 227 if (lprofBufferIOFlush(BufferIO) != 0) 228 return -1; 229 lprofDeleteBufferIO(BufferIO); 230 231 return 0; 232 } 233 234 COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, 235 VPDataReaderType *VPDataReader, 236 int SkipNameDataWrite) { 237 /* Match logic in __llvm_profile_write_buffer(). */ 238 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); 239 const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); 240 const uint64_t *CountersBegin = __llvm_profile_begin_counters(); 241 const uint64_t *CountersEnd = __llvm_profile_end_counters(); 242 const char *NamesBegin = __llvm_profile_begin_names(); 243 const char *NamesEnd = __llvm_profile_end_names(); 244 return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, 245 CountersEnd, VPDataReader, NamesBegin, NamesEnd, 246 SkipNameDataWrite); 247 } 248 249 COMPILER_RT_VISIBILITY int 250 lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, 251 const __llvm_profile_data *DataEnd, 252 const uint64_t *CountersBegin, const uint64_t *CountersEnd, 253 VPDataReaderType *VPDataReader, const char *NamesBegin, 254 const char *NamesEnd, int SkipNameDataWrite) { 255 256 /* Calculate size of sections. */ 257 const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); 258 const uint64_t CountersSize = CountersEnd - CountersBegin; 259 const uint64_t NamesSize = NamesEnd - NamesBegin; 260 const uint64_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize); 261 262 /* Enough zeroes for padding. */ 263 const char Zeroes[sizeof(uint64_t)] = {0}; 264 265 /* Create the header. */ 266 __llvm_profile_header Header; 267 268 if (!DataSize) 269 return 0; 270 271 /* Initialize header structure. */ 272 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; 273 #include "InstrProfData.inc" 274 275 /* Write the data. */ 276 ProfDataIOVec IOVec[] = { 277 {&Header, sizeof(__llvm_profile_header), 1}, 278 {DataBegin, sizeof(__llvm_profile_data), DataSize}, 279 {CountersBegin, sizeof(uint64_t), CountersSize}, 280 {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize}, 281 {Zeroes, sizeof(uint8_t), Padding}}; 282 if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) 283 return -1; 284 285 return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); 286 } 287