1 /*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\ 2 |* 3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 |* See https://llvm.org/LICENSE.txt for license information. 5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 |* 7 \*===----------------------------------------------------------------------===*/ 8 9 // Note: This is linked into the Darwin kernel, and must remain compatible 10 // with freestanding compilation. See `darwin_add_builtin_libraries`. 11 12 #include "InstrProfiling.h" 13 #include "InstrProfilingInternal.h" 14 #include "InstrProfilingPort.h" 15 16 /* When continuous mode is enabled (%c), this parameter is set to 1. 17 * 18 * This parameter is defined here in InstrProfilingBuffer.o, instead of in 19 * InstrProfilingFile.o, to sequester all libc-dependent code in 20 * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this 21 * layering is violated. */ 22 static int ContinuouslySyncProfile = 0; 23 24 /* The system page size. Only valid when non-zero. If 0, the page size is 25 * unavailable. */ 26 static unsigned PageSize = 0; 27 28 COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) { 29 return ContinuouslySyncProfile && PageSize; 30 } 31 32 COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) { 33 ContinuouslySyncProfile = 1; 34 } 35 36 COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) { 37 ContinuouslySyncProfile = 0; 38 } 39 40 COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) { 41 PageSize = PS; 42 } 43 44 COMPILER_RT_VISIBILITY 45 uint64_t __llvm_profile_get_size_for_buffer(void) { 46 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); 47 const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); 48 const char *CountersBegin = __llvm_profile_begin_counters(); 49 const char *CountersEnd = __llvm_profile_end_counters(); 50 const char *BitmapBegin = __llvm_profile_begin_bitmap(); 51 const char *BitmapEnd = __llvm_profile_end_bitmap(); 52 const char *NamesBegin = __llvm_profile_begin_names(); 53 const char *NamesEnd = __llvm_profile_end_names(); 54 const VTableProfData *VTableBegin = __llvm_profile_begin_vtables(); 55 const VTableProfData *VTableEnd = __llvm_profile_end_vtables(); 56 const char *VNamesBegin = __llvm_profile_begin_vtabnames(); 57 const char *VNamesEnd = __llvm_profile_end_vtabnames(); 58 59 return __llvm_profile_get_size_for_buffer_internal( 60 DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, 61 NamesBegin, NamesEnd, VTableBegin, VTableEnd, VNamesBegin, VNamesEnd); 62 } 63 64 // NOTE: Caller should guarantee that `Begin` and `End` specifies a half-open 65 // interval [Begin, End). Namely, `End` is one-byte past the end of the array. 66 COMPILER_RT_VISIBILITY 67 uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, 68 const __llvm_profile_data *End) { 69 intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; 70 return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) / 71 sizeof(__llvm_profile_data); 72 } 73 74 COMPILER_RT_VISIBILITY 75 uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, 76 const __llvm_profile_data *End) { 77 return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data); 78 } 79 80 // Counts the number of `VTableProfData` elements within the range of [Begin, 81 // End). Caller should guarantee that End points to one byte past the inclusive 82 // range. 83 // FIXME: Add a compiler-rt test to make sure the number of vtables in the 84 // raw profile is the same as the number of vtable elements in the instrumented 85 // binary. 86 COMPILER_RT_VISIBILITY 87 uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, 88 const VTableProfData *End) { 89 // Convert pointers to intptr_t to use integer arithmetic. 90 intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin; 91 return (EndI - BeginI) / sizeof(VTableProfData); 92 } 93 94 COMPILER_RT_VISIBILITY 95 uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin, 96 const VTableProfData *End) { 97 return (intptr_t)(End) - (intptr_t)(Begin); 98 } 99 100 COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) { 101 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) 102 return sizeof(uint8_t); 103 return sizeof(uint64_t); 104 } 105 106 COMPILER_RT_VISIBILITY 107 uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) { 108 intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; 109 return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) / 110 __llvm_profile_counter_entry_size(); 111 } 112 113 COMPILER_RT_VISIBILITY 114 uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) { 115 return __llvm_profile_get_num_counters(Begin, End) * 116 __llvm_profile_counter_entry_size(); 117 } 118 119 COMPILER_RT_VISIBILITY 120 uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, 121 const char *End) { 122 return (End - Begin); 123 } 124 125 COMPILER_RT_VISIBILITY 126 uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) { 127 return End - Begin; 128 } 129 130 /// Calculate the number of padding bytes needed to add to \p Offset in order 131 /// for (\p Offset + Padding) to be page-aligned. 132 static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) { 133 uint64_t OffsetModPage = Offset % PageSize; 134 if (OffsetModPage > 0) 135 return PageSize - OffsetModPage; 136 return 0; 137 } 138 139 static int needsCounterPadding(void) { 140 #if defined(__APPLE__) 141 return __llvm_profile_is_continuous_mode_enabled(); 142 #else 143 return 0; 144 #endif 145 } 146 147 COMPILER_RT_VISIBILITY 148 int __llvm_profile_get_padding_sizes_for_counters( 149 uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, 150 uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize, 151 uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, 152 uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames, 153 uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVName) { 154 // Counter padding is needed only if continuous mode is enabled. 155 if (!needsCounterPadding()) { 156 *PaddingBytesBeforeCounters = 0; 157 *PaddingBytesAfterCounters = 158 __llvm_profile_get_num_padding_bytes(CountersSize); 159 *PaddingBytesAfterBitmapBytes = 160 __llvm_profile_get_num_padding_bytes(NumBitmapBytes); 161 *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); 162 if (PaddingBytesAfterVTable != NULL) 163 *PaddingBytesAfterVTable = 164 __llvm_profile_get_num_padding_bytes(VTableSize); 165 if (PaddingBytesAfterVName != NULL) 166 *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize); 167 return 0; 168 } 169 170 // Value profiling not supported in continuous mode at profile-write time. 171 // Return -1 to alert the incompatibility. 172 if (VTableSize != 0 || VNameSize != 0) 173 return -1; 174 175 // In continuous mode, the file offsets for headers and for the start of 176 // counter sections need to be page-aligned. 177 *PaddingBytesBeforeCounters = 178 calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize); 179 *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize); 180 *PaddingBytesAfterBitmapBytes = 181 calculateBytesNeededToPageAlign(NumBitmapBytes); 182 *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize); 183 // Set these two variables to zero to avoid uninitialized variables 184 // even if VTableSize and VNameSize are known to be zero. 185 if (PaddingBytesAfterVTable != NULL) 186 *PaddingBytesAfterVTable = 0; 187 if (PaddingBytesAfterVName != NULL) 188 *PaddingBytesAfterVName = 0; 189 return 0; 190 } 191 192 COMPILER_RT_VISIBILITY 193 uint64_t __llvm_profile_get_size_for_buffer_internal( 194 const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, 195 const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, 196 const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd, 197 const VTableProfData *VTableBegin, const VTableProfData *VTableEnd, 198 const char *VNamesBegin, const char *VNamesEnd) { 199 /* Match logic in __llvm_profile_write_buffer(). */ 200 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); 201 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); 202 uint64_t CountersSize = 203 __llvm_profile_get_counters_size(CountersBegin, CountersEnd); 204 const uint64_t NumBitmapBytes = 205 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); 206 const uint64_t VTableSize = 207 __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd); 208 const uint64_t VNameSize = 209 __llvm_profile_get_name_size(VNamesBegin, VNamesEnd); 210 211 /* Determine how much padding is needed before/after the counters and after 212 * the names. */ 213 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, 214 PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes, 215 PaddingBytesAfterVTable, PaddingBytesAfterVNames; 216 __llvm_profile_get_padding_sizes_for_counters( 217 DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */, 218 0 /* VNameSize */, &PaddingBytesBeforeCounters, 219 &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, 220 &PaddingBytesAfterNames, &PaddingBytesAfterVTable, 221 &PaddingBytesAfterVNames); 222 223 return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + 224 DataSize + PaddingBytesBeforeCounters + CountersSize + 225 PaddingBytesAfterCounters + NumBitmapBytes + 226 PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames + 227 VTableSize + PaddingBytesAfterVTable + VNameSize + 228 PaddingBytesAfterVNames; 229 } 230 231 COMPILER_RT_VISIBILITY 232 void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer) { 233 BufferWriter->Write = lprofBufferWriter; 234 BufferWriter->WriterCtx = Buffer; 235 } 236 237 COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) { 238 ProfDataWriter BufferWriter; 239 initBufferWriter(&BufferWriter, Buffer); 240 return lprofWriteData(&BufferWriter, 0, 0); 241 } 242 243 COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( 244 char *Buffer, const __llvm_profile_data *DataBegin, 245 const __llvm_profile_data *DataEnd, const char *CountersBegin, 246 const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, 247 const char *NamesBegin, const char *NamesEnd) { 248 ProfDataWriter BufferWriter; 249 initBufferWriter(&BufferWriter, Buffer); 250 // Set virtual table arguments to NULL since they are not supported yet. 251 return lprofWriteDataImpl( 252 &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd, 253 BitmapBegin, BitmapEnd, /*VPDataReader=*/0, NamesBegin, NamesEnd, 254 /*VTableBegin=*/NULL, /*VTableEnd=*/NULL, /*VNamesBegin=*/NULL, 255 /*VNamesEnd=*/NULL, /*SkipNameDataWrite=*/0); 256 } 257