1 /*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8
9 #if !defined(__Fuchsia__)
10
11 #include <assert.h>
12 #include <errno.h>
13 #include <stdio.h>
14 #include <stdlib.h>
15 #include <string.h>
16 #ifdef _MSC_VER
17 /* For _alloca. */
18 #include <malloc.h>
19 #endif
20 #if defined(_WIN32)
21 #include "WindowsMMap.h"
22 /* For _chsize_s */
23 #include <io.h>
24 #include <process.h>
25 #else
26 #include <sys/file.h>
27 #include <sys/mman.h>
28 #include <unistd.h>
29 #if defined(__linux__)
30 #include <sys/types.h>
31 #endif
32 #endif
33
34 #include "InstrProfiling.h"
35 #include "InstrProfilingInternal.h"
36 #include "InstrProfilingPort.h"
37 #include "InstrProfilingUtil.h"
38
39 /* From where is profile name specified.
40 * The order the enumerators define their
41 * precedence. Re-order them may lead to
42 * runtime behavior change. */
43 typedef enum ProfileNameSpecifier {
44 PNS_unknown = 0,
45 PNS_default,
46 PNS_command_line,
47 PNS_environment,
48 PNS_runtime_api
49 } ProfileNameSpecifier;
50
getPNSStr(ProfileNameSpecifier PNS)51 static const char *getPNSStr(ProfileNameSpecifier PNS) {
52 switch (PNS) {
53 case PNS_default:
54 return "default setting";
55 case PNS_command_line:
56 return "command line";
57 case PNS_environment:
58 return "environment variable";
59 case PNS_runtime_api:
60 return "runtime API";
61 default:
62 return "Unknown";
63 }
64 }
65
66 #define MAX_PID_SIZE 16
67 /* Data structure holding the result of parsed filename pattern. */
68 typedef struct lprofFilename {
69 /* File name string possibly with %p or %h specifiers. */
70 const char *FilenamePat;
71 /* A flag indicating if FilenamePat's memory is allocated
72 * by runtime. */
73 unsigned OwnsFilenamePat;
74 const char *ProfilePathPrefix;
75 char PidChars[MAX_PID_SIZE];
76 char *TmpDir;
77 char Hostname[COMPILER_RT_MAX_HOSTLEN];
78 unsigned NumPids;
79 unsigned NumHosts;
80 unsigned NumBinaryIds;
81 /* When in-process merging is enabled, this parameter specifies
82 * the total number of profile data files shared by all the processes
83 * spawned from the same binary. By default the value is 1. If merging
84 * is not enabled, its value should be 0. This parameter is specified
85 * by the %[0-9]m specifier. For instance %2m enables merging using
86 * 2 profile data files. %1m is equivalent to %m. Also %m specifier
87 * can only appear once at the end of the name pattern. */
88 unsigned MergePoolSize;
89 ProfileNameSpecifier PNS;
90 } lprofFilename;
91
92 static lprofFilename lprofCurFilename = {0, 0, 0, {0}, NULL, {0},
93 0, 0, 0, 0, PNS_unknown};
94
95 static int ProfileMergeRequested = 0;
96 static int getProfileFileSizeForMerging(FILE *ProfileFile,
97 uint64_t *ProfileFileSize);
98
99 #if defined(__APPLE__)
100 static const int ContinuousModeSupported = 1;
101 static const int UseBiasVar = 0;
102 static const char *FileOpenMode = "a+b";
103 static void *BiasAddr = NULL;
104 static void *BiasDefaultAddr = NULL;
105 static void *BitmapBiasAddr = NULL;
106 static void *BitmapBiasDefaultAddr = NULL;
mmapForContinuousMode(uint64_t CurrentFileOffset,FILE * File)107 static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
108 /* Get the sizes of various profile data sections. Taken from
109 * __llvm_profile_get_size_for_buffer(). */
110 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
111 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
112 const char *CountersBegin = __llvm_profile_begin_counters();
113 const char *CountersEnd = __llvm_profile_end_counters();
114 const char *BitmapBegin = __llvm_profile_begin_bitmap();
115 const char *BitmapEnd = __llvm_profile_end_bitmap();
116 const char *NamesBegin = __llvm_profile_begin_names();
117 const char *NamesEnd = __llvm_profile_end_names();
118 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
119 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
120 uint64_t CountersSize =
121 __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
122 uint64_t NumBitmapBytes =
123 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
124
125 /* Check that the counter, bitmap, and data sections in this image are
126 * page-aligned. */
127 unsigned PageSize = getpagesize();
128 if ((intptr_t)CountersBegin % PageSize != 0) {
129 PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n",
130 CountersBegin, PageSize);
131 return 1;
132 }
133 if ((intptr_t)BitmapBegin % PageSize != 0) {
134 PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n",
135 BitmapBegin, PageSize);
136 return 1;
137 }
138 if ((intptr_t)DataBegin % PageSize != 0) {
139 PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n",
140 DataBegin, PageSize);
141 return 1;
142 }
143
144 int Fileno = fileno(File);
145 /* Determine how much padding is needed before/after the counters and
146 * after the names. */
147 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
148 PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes,
149 PaddingBytesAfterVTable, PaddingBytesAfterVNames;
150 __llvm_profile_get_padding_sizes_for_counters(
151 DataSize, CountersSize, NumBitmapBytes, NamesSize, /*VTableSize=*/0,
152 /*VNameSize=*/0, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
153 &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames,
154 &PaddingBytesAfterVTable, &PaddingBytesAfterVNames);
155
156 uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters;
157 uint64_t FileOffsetToCounters = CurrentFileOffset +
158 sizeof(__llvm_profile_header) + DataSize +
159 PaddingBytesBeforeCounters;
160 void *CounterMmap = mmap((void *)CountersBegin, PageAlignedCountersLength,
161 PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED,
162 Fileno, FileOffsetToCounters);
163 if (CounterMmap != CountersBegin) {
164 PROF_ERR(
165 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
166 " - CountersBegin: %p\n"
167 " - PageAlignedCountersLength: %" PRIu64 "\n"
168 " - Fileno: %d\n"
169 " - FileOffsetToCounters: %" PRIu64 "\n",
170 strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
171 FileOffsetToCounters);
172 return 1;
173 }
174
175 /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap()
176 * will fail with EINVAL. */
177 if (NumBitmapBytes == 0)
178 return 0;
179
180 uint64_t PageAlignedBitmapLength =
181 NumBitmapBytes + PaddingBytesAfterBitmapBytes;
182 uint64_t FileOffsetToBitmap =
183 FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters;
184 void *BitmapMmap =
185 mmap((void *)BitmapBegin, PageAlignedBitmapLength, PROT_READ | PROT_WRITE,
186 MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToBitmap);
187 if (BitmapMmap != BitmapBegin) {
188 PROF_ERR(
189 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
190 " - BitmapBegin: %p\n"
191 " - PageAlignedBitmapLength: %" PRIu64 "\n"
192 " - Fileno: %d\n"
193 " - FileOffsetToBitmap: %" PRIu64 "\n",
194 strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno,
195 FileOffsetToBitmap);
196 return 1;
197 }
198 return 0;
199 }
200 #elif defined(__ELF__) || defined(_WIN32) || defined(_AIX)
201
202 #define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \
203 INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default)
204 COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0;
205 #define INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR \
206 INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_BITMAP_BIAS_VAR, _default)
207 COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR = 0;
208
209 /* This variable is a weak external reference which could be used to detect
210 * whether or not the compiler defined this symbol. */
211 #if defined(_MSC_VER)
212 COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
213 COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
214 #if defined(_M_IX86) || defined(__i386__)
215 #define WIN_SYM_PREFIX "_"
216 #else
217 #define WIN_SYM_PREFIX
218 #endif
219 #pragma comment( \
220 linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
221 INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
222 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
223 #pragma comment( \
224 linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
225 INSTR_PROF_PROFILE_BITMAP_BIAS_VAR) "=" WIN_SYM_PREFIX \
226 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))
227 #else
228 COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
229 __attribute__((weak, alias(INSTR_PROF_QUOTE(
230 INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))));
231 COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR
232 __attribute__((weak, alias(INSTR_PROF_QUOTE(
233 INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR))));
234 #endif
235 static const int ContinuousModeSupported = 1;
236 static const int UseBiasVar = 1;
237 /* TODO: If there are two DSOs, the second DSO initialization will truncate the
238 * first profile file. */
239 static const char *FileOpenMode = "w+b";
240 /* This symbol is defined by the compiler when runtime counter relocation is
241 * used and runtime provides a weak alias so we can check if it's defined. */
242 static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
243 static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR;
244 static void *BitmapBiasAddr = &INSTR_PROF_PROFILE_BITMAP_BIAS_VAR;
245 static void *BitmapBiasDefaultAddr =
246 &INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR;
mmapForContinuousMode(uint64_t CurrentFileOffset,FILE * File)247 static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
248 /* Get the sizes of various profile data sections. Taken from
249 * __llvm_profile_get_size_for_buffer(). */
250 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
251 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
252 const char *CountersBegin = __llvm_profile_begin_counters();
253 const char *CountersEnd = __llvm_profile_end_counters();
254 const char *BitmapBegin = __llvm_profile_begin_bitmap();
255 const char *BitmapEnd = __llvm_profile_end_bitmap();
256 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
257 uint64_t CountersSize =
258 __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
259 uint64_t NumBitmapBytes =
260 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
261 /* Get the file size. */
262 uint64_t FileSize = 0;
263 if (getProfileFileSizeForMerging(File, &FileSize))
264 return 1;
265
266 int Fileno = fileno(File);
267 uint64_t PaddingBytesAfterCounters =
268 __llvm_profile_get_num_padding_bytes(CountersSize);
269 uint64_t FileOffsetToCounters =
270 sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize;
271
272 /* Map the profile. */
273 char *Profile = (char *)mmap(NULL, FileSize, PROT_READ | PROT_WRITE,
274 MAP_SHARED, Fileno, 0);
275 if (Profile == MAP_FAILED) {
276 PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
277 return 1;
278 }
279 /* Update the profile fields based on the current mapping. */
280 INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
281 (intptr_t)Profile - (uintptr_t)CountersBegin + FileOffsetToCounters;
282
283 /* Return the memory allocated for counters to OS. */
284 lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd);
285
286 /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap()
287 * will fail with EINVAL. */
288 if (NumBitmapBytes == 0)
289 return 0;
290
291 /* Update profbm_bias. */
292 uint64_t FileOffsetToBitmap =
293 FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters;
294 /* Update the profile fields based on the current mapping. */
295 INSTR_PROF_PROFILE_BITMAP_BIAS_VAR =
296 (uintptr_t)Profile - (uintptr_t)BitmapBegin + FileOffsetToBitmap;
297
298 /* Return the memory allocated for counters to OS. */
299 lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd);
300 return 0;
301 }
302 #else
303 static const int ContinuousModeSupported = 0;
304 static const int UseBiasVar = 0;
305 static const char *FileOpenMode = "a+b";
306 static void *BiasAddr = NULL;
307 static void *BiasDefaultAddr = NULL;
308 static void *BitmapBiasAddr = NULL;
309 static void *BitmapBiasDefaultAddr = NULL;
mmapForContinuousMode(uint64_t CurrentFileOffset,FILE * File)310 static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
311 return 0;
312 }
313 #endif
314
isProfileMergeRequested(void)315 static int isProfileMergeRequested(void) { return ProfileMergeRequested; }
setProfileMergeRequested(int EnableMerge)316 static void setProfileMergeRequested(int EnableMerge) {
317 ProfileMergeRequested = EnableMerge;
318 }
319
320 static FILE *ProfileFile = NULL;
getProfileFile(void)321 static FILE *getProfileFile(void) { return ProfileFile; }
setProfileFile(FILE * File)322 static void setProfileFile(FILE *File) { ProfileFile = File; }
323
324 static int getCurFilenameLength(void);
325 static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf);
doMerging(void)326 static unsigned doMerging(void) {
327 return lprofCurFilename.MergePoolSize || isProfileMergeRequested();
328 }
329
330 /* Return 1 if there is an error, otherwise return 0. */
fileWriter(ProfDataWriter * This,ProfDataIOVec * IOVecs,uint32_t NumIOVecs)331 static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs,
332 uint32_t NumIOVecs) {
333 uint32_t I;
334 FILE *File = (FILE *)This->WriterCtx;
335 char Zeroes[sizeof(uint64_t)] = {0};
336 for (I = 0; I < NumIOVecs; I++) {
337 if (IOVecs[I].Data) {
338 if (fwrite(IOVecs[I].Data, IOVecs[I].ElmSize, IOVecs[I].NumElm, File) !=
339 IOVecs[I].NumElm)
340 return 1;
341 } else if (IOVecs[I].UseZeroPadding) {
342 size_t BytesToWrite = IOVecs[I].ElmSize * IOVecs[I].NumElm;
343 while (BytesToWrite > 0) {
344 size_t PartialWriteLen =
345 (sizeof(uint64_t) > BytesToWrite) ? BytesToWrite : sizeof(uint64_t);
346 if (fwrite(Zeroes, sizeof(uint8_t), PartialWriteLen, File) !=
347 PartialWriteLen) {
348 return 1;
349 }
350 BytesToWrite -= PartialWriteLen;
351 }
352 } else {
353 if (fseek(File, IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1)
354 return 1;
355 }
356 }
357 return 0;
358 }
359
initFileWriter(ProfDataWriter * This,FILE * File)360 static void initFileWriter(ProfDataWriter *This, FILE *File) {
361 This->Write = fileWriter;
362 This->WriterCtx = File;
363 }
364
365 COMPILER_RT_VISIBILITY ProfBufferIO *
lprofCreateBufferIOInternal(void * File,uint32_t BufferSz)366 lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) {
367 FreeHook = &free;
368 DynamicBufferIOBuffer = (uint8_t *)calloc(1, BufferSz);
369 VPBufferSize = BufferSz;
370 ProfDataWriter *fileWriter =
371 (ProfDataWriter *)calloc(1, sizeof(ProfDataWriter));
372 initFileWriter(fileWriter, File);
373 ProfBufferIO *IO = lprofCreateBufferIO(fileWriter);
374 IO->OwnFileWriter = 1;
375 return IO;
376 }
377
setupIOBuffer(void)378 static void setupIOBuffer(void) {
379 const char *BufferSzStr = 0;
380 BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE");
381 if (BufferSzStr && BufferSzStr[0]) {
382 VPBufferSize = atoi(BufferSzStr);
383 DynamicBufferIOBuffer = (uint8_t *)calloc(VPBufferSize, 1);
384 }
385 }
386
387 /* Get the size of the profile file. If there are any errors, print the
388 * message under the assumption that the profile is being read for merging
389 * purposes, and return -1. Otherwise return the file size in the inout param
390 * \p ProfileFileSize. */
getProfileFileSizeForMerging(FILE * ProfileFile,uint64_t * ProfileFileSize)391 static int getProfileFileSizeForMerging(FILE *ProfileFile,
392 uint64_t *ProfileFileSize) {
393 if (fseek(ProfileFile, 0L, SEEK_END) == -1) {
394 PROF_ERR("Unable to merge profile data, unable to get size: %s\n",
395 strerror(errno));
396 return -1;
397 }
398 *ProfileFileSize = ftell(ProfileFile);
399
400 /* Restore file offset. */
401 if (fseek(ProfileFile, 0L, SEEK_SET) == -1) {
402 PROF_ERR("Unable to merge profile data, unable to rewind: %s\n",
403 strerror(errno));
404 return -1;
405 }
406
407 if (*ProfileFileSize > 0 &&
408 *ProfileFileSize < sizeof(__llvm_profile_header)) {
409 PROF_WARN("Unable to merge profile data: %s\n",
410 "source profile file is too small.");
411 return -1;
412 }
413 return 0;
414 }
415
416 /* mmap() \p ProfileFile for profile merging purposes, assuming that an
417 * exclusive lock is held on the file and that \p ProfileFileSize is the
418 * length of the file. Return the mmap'd buffer in the inout variable
419 * \p ProfileBuffer. Returns -1 on failure. On success, the caller is
420 * responsible for unmapping the mmap'd buffer in \p ProfileBuffer. */
mmapProfileForMerging(FILE * ProfileFile,uint64_t ProfileFileSize,ManagedMemory * ProfileBuffer)421 static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize,
422 ManagedMemory *ProfileBuffer) {
423 lprofGetFileContentBuffer(ProfileFile, ProfileFileSize, ProfileBuffer);
424
425 if (ProfileBuffer->Status == MS_INVALID) {
426 PROF_ERR("Unable to merge profile data: %s\n", "reading file failed");
427 return -1;
428 }
429
430 if (__llvm_profile_check_compatibility(ProfileBuffer->Addr,
431 ProfileFileSize)) {
432 (void)lprofReleaseBuffer(ProfileBuffer, ProfileFileSize);
433 PROF_WARN("Unable to merge profile data: %s\n",
434 "source profile file is not compatible.");
435 return -1;
436 }
437 return 0;
438 }
439
440 /* Read profile data in \c ProfileFile and merge with in-memory
441 profile counters. Returns -1 if there is fatal error, otherwise
442 0 is returned. Returning 0 does not mean merge is actually
443 performed. If merge is actually done, *MergeDone is set to 1.
444 */
doProfileMerging(FILE * ProfileFile,int * MergeDone)445 static int doProfileMerging(FILE *ProfileFile, int *MergeDone) {
446 uint64_t ProfileFileSize;
447 ManagedMemory ProfileBuffer;
448
449 /* Get the size of the profile on disk. */
450 if (getProfileFileSizeForMerging(ProfileFile, &ProfileFileSize) == -1)
451 return -1;
452
453 /* Nothing to merge. */
454 if (!ProfileFileSize)
455 return 0;
456
457 /* mmap() the profile and check that it is compatible with the data in
458 * the current image. */
459 if (mmapProfileForMerging(ProfileFile, ProfileFileSize, &ProfileBuffer) == -1)
460 return -1;
461
462 /* Now start merging */
463 if (__llvm_profile_merge_from_buffer(ProfileBuffer.Addr, ProfileFileSize)) {
464 PROF_ERR("%s\n", "Invalid profile data to merge");
465 (void)lprofReleaseBuffer(&ProfileBuffer, ProfileFileSize);
466 return -1;
467 }
468
469 // Truncate the file in case merging of value profile did not happen to
470 // prevent from leaving garbage data at the end of the profile file.
471 (void)COMPILER_RT_FTRUNCATE(ProfileFile,
472 __llvm_profile_get_size_for_buffer());
473
474 (void)lprofReleaseBuffer(&ProfileBuffer, ProfileFileSize);
475 *MergeDone = 1;
476
477 return 0;
478 }
479
480 /* Create the directory holding the file, if needed. */
createProfileDir(const char * Filename)481 static void createProfileDir(const char *Filename) {
482 size_t Length = strlen(Filename);
483 if (lprofFindFirstDirSeparator(Filename)) {
484 char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1);
485 strncpy(Copy, Filename, Length + 1);
486 __llvm_profile_recursive_mkdir(Copy);
487 }
488 }
489
490 /* Open the profile data for merging. It opens the file in r+b mode with
491 * file locking. If the file has content which is compatible with the
492 * current process, it also reads in the profile data in the file and merge
493 * it with in-memory counters. After the profile data is merged in memory,
494 * the original profile data is truncated and gets ready for the profile
495 * dumper. With profile merging enabled, each executable as well as any of
496 * its instrumented shared libraries dump profile data into their own data file.
497 */
openFileForMerging(const char * ProfileFileName,int * MergeDone)498 static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) {
499 FILE *ProfileFile = getProfileFile();
500 int rc;
501 // initializeProfileForContinuousMode will lock the profile, but if
502 // ProfileFile is set by user via __llvm_profile_set_file_object, it's assumed
503 // unlocked at this point.
504 if (ProfileFile && !__llvm_profile_is_continuous_mode_enabled()) {
505 lprofLockFileHandle(ProfileFile);
506 }
507 if (!ProfileFile) {
508 createProfileDir(ProfileFileName);
509 ProfileFile = lprofOpenFileEx(ProfileFileName);
510 }
511 if (!ProfileFile)
512 return NULL;
513
514 rc = doProfileMerging(ProfileFile, MergeDone);
515 if (rc || (!*MergeDone && COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) ||
516 fseek(ProfileFile, 0L, SEEK_SET) == -1) {
517 PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName,
518 strerror(errno));
519 fclose(ProfileFile);
520 return NULL;
521 }
522 return ProfileFile;
523 }
524
getFileObject(const char * OutputName)525 static FILE *getFileObject(const char *OutputName) {
526 FILE *File;
527 File = getProfileFile();
528 if (File != NULL) {
529 return File;
530 }
531
532 return fopen(OutputName, "ab");
533 }
534
closeFileObject(FILE * OutputFile)535 static void closeFileObject(FILE *OutputFile) {
536 if (OutputFile == getProfileFile()) {
537 fflush(OutputFile);
538 if (doMerging() && !__llvm_profile_is_continuous_mode_enabled()) {
539 lprofUnlockFileHandle(OutputFile);
540 }
541 } else {
542 fclose(OutputFile);
543 }
544 }
545
546 /* Write profile data to file \c OutputName. */
writeFile(const char * OutputName)547 static int writeFile(const char *OutputName) {
548 int RetVal;
549 FILE *OutputFile;
550
551 int MergeDone = 0;
552 VPMergeHook = &lprofMergeValueProfData;
553 if (doMerging())
554 OutputFile = openFileForMerging(OutputName, &MergeDone);
555 else
556 OutputFile = getFileObject(OutputName);
557
558 if (!OutputFile)
559 return -1;
560
561 FreeHook = &free;
562 setupIOBuffer();
563 ProfDataWriter fileWriter;
564 initFileWriter(&fileWriter, OutputFile);
565 RetVal = lprofWriteData(&fileWriter, lprofGetVPDataReader(), MergeDone);
566
567 closeFileObject(OutputFile);
568 return RetVal;
569 }
570
571 #define LPROF_INIT_ONCE_ENV "__LLVM_PROFILE_RT_INIT_ONCE"
572
truncateCurrentFile(void)573 static void truncateCurrentFile(void) {
574 const char *Filename;
575 char *FilenameBuf;
576 FILE *File;
577 int Length;
578
579 Length = getCurFilenameLength();
580 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
581 Filename = getCurFilename(FilenameBuf, 0);
582 if (!Filename)
583 return;
584
585 /* Only create the profile directory and truncate an existing profile once.
586 * In continuous mode, this is necessary, as the profile is written-to by the
587 * runtime initializer. */
588 int initialized = getenv(LPROF_INIT_ONCE_ENV) != NULL;
589 if (initialized)
590 return;
591 #if defined(_WIN32)
592 _putenv(LPROF_INIT_ONCE_ENV "=" LPROF_INIT_ONCE_ENV);
593 #else
594 setenv(LPROF_INIT_ONCE_ENV, LPROF_INIT_ONCE_ENV, 1);
595 #endif
596
597 /* Create the profile dir (even if online merging is enabled), so that
598 * the profile file can be set up if continuous mode is enabled. */
599 createProfileDir(Filename);
600
601 /* By pass file truncation to allow online raw profile merging. */
602 if (lprofCurFilename.MergePoolSize)
603 return;
604
605 /* Truncate the file. Later we'll reopen and append. */
606 File = fopen(Filename, "w");
607 if (!File)
608 return;
609 fclose(File);
610 }
611
612 /* Write a partial profile to \p Filename, which is required to be backed by
613 * the open file object \p File. */
writeProfileWithFileObject(const char * Filename,FILE * File)614 static int writeProfileWithFileObject(const char *Filename, FILE *File) {
615 setProfileFile(File);
616 int rc = writeFile(Filename);
617 if (rc)
618 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
619 setProfileFile(NULL);
620 return rc;
621 }
622
initializeProfileForContinuousMode(void)623 static void initializeProfileForContinuousMode(void) {
624 if (!__llvm_profile_is_continuous_mode_enabled())
625 return;
626 if (!ContinuousModeSupported) {
627 PROF_ERR("%s\n", "continuous mode is unsupported on this platform");
628 return;
629 }
630 if (UseBiasVar && BiasAddr == BiasDefaultAddr &&
631 BitmapBiasAddr == BitmapBiasDefaultAddr) {
632 PROF_ERR("%s\n", "Neither __llvm_profile_counter_bias nor "
633 "__llvm_profile_bitmap_bias is defined");
634 return;
635 }
636
637 /* Get the sizes of counter section. */
638 uint64_t CountersSize = __llvm_profile_get_counters_size(
639 __llvm_profile_begin_counters(), __llvm_profile_end_counters());
640
641 int Length = getCurFilenameLength();
642 char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
643 const char *Filename = getCurFilename(FilenameBuf, 0);
644 if (!Filename)
645 return;
646
647 FILE *File = NULL;
648 uint64_t CurrentFileOffset = 0;
649 if (doMerging()) {
650 /* We are merging profiles. Map the counter section as shared memory into
651 * the profile, i.e. into each participating process. An increment in one
652 * process should be visible to every other process with the same counter
653 * section mapped. */
654 File = lprofOpenFileEx(Filename);
655 if (!File)
656 return;
657
658 uint64_t ProfileFileSize = 0;
659 if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
660 lprofUnlockFileHandle(File);
661 fclose(File);
662 return;
663 }
664 if (ProfileFileSize == 0) {
665 /* Grow the profile so that mmap() can succeed. Leak the file handle, as
666 * the file should stay open. */
667 if (writeProfileWithFileObject(Filename, File) != 0) {
668 lprofUnlockFileHandle(File);
669 fclose(File);
670 return;
671 }
672 } else {
673 /* The merged profile has a non-zero length. Check that it is compatible
674 * with the data in this process. */
675 ManagedMemory ProfileBuffer;
676 if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) {
677 lprofUnlockFileHandle(File);
678 fclose(File);
679 return;
680 }
681 (void)lprofReleaseBuffer(&ProfileBuffer, ProfileFileSize);
682 }
683 } else {
684 File = fopen(Filename, FileOpenMode);
685 if (!File)
686 return;
687 /* Check that the offset within the file is page-aligned. */
688 CurrentFileOffset = ftell(File);
689 unsigned PageSize = getpagesize();
690 if (CurrentFileOffset % PageSize != 0) {
691 PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not"
692 "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n",
693 (uint64_t)CurrentFileOffset, PageSize);
694 fclose(File);
695 return;
696 }
697 if (writeProfileWithFileObject(Filename, File) != 0) {
698 fclose(File);
699 return;
700 }
701 }
702
703 /* mmap() the profile counters so long as there is at least one counter.
704 * If there aren't any counters, mmap() would fail with EINVAL. */
705 if (CountersSize > 0)
706 mmapForContinuousMode(CurrentFileOffset, File);
707
708 if (doMerging()) {
709 lprofUnlockFileHandle(File);
710 }
711 if (File != NULL) {
712 fclose(File);
713 }
714 }
715
716 static const char *DefaultProfileName = "default.profraw";
resetFilenameToDefault(void)717 static void resetFilenameToDefault(void) {
718 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
719 #ifdef __GNUC__
720 #pragma GCC diagnostic push
721 #pragma GCC diagnostic ignored "-Wcast-qual"
722 #elif defined(__clang__)
723 #pragma clang diagnostic push
724 #pragma clang diagnostic ignored "-Wcast-qual"
725 #endif
726 free((void *)lprofCurFilename.FilenamePat);
727 #ifdef __GNUC__
728 #pragma GCC diagnostic pop
729 #elif defined(__clang__)
730 #pragma clang diagnostic pop
731 #endif
732 }
733 memset(&lprofCurFilename, 0, sizeof(lprofCurFilename));
734 lprofCurFilename.FilenamePat = DefaultProfileName;
735 lprofCurFilename.PNS = PNS_default;
736 }
737
getMergePoolSize(const char * FilenamePat,int * I)738 static unsigned getMergePoolSize(const char *FilenamePat, int *I) {
739 unsigned J = 0, Num = 0;
740 for (;; ++J) {
741 char C = FilenamePat[*I + J];
742 if (C == 'm') {
743 *I += J;
744 return Num ? Num : 1;
745 }
746 if (C < '0' || C > '9')
747 break;
748 Num = Num * 10 + C - '0';
749
750 /* If FilenamePat[*I+J] is between '0' and '9', the next byte is guaranteed
751 * to be in-bound as the string is null terminated. */
752 }
753 return 0;
754 }
755
756 /* Assert that Idx does index past a string null terminator. Return the
757 * result of the check. */
checkBounds(int Idx,int Strlen)758 static int checkBounds(int Idx, int Strlen) {
759 assert(Idx <= Strlen && "Indexing past string null terminator");
760 return Idx <= Strlen;
761 }
762
763 /* Parses the pattern string \p FilenamePat and stores the result to
764 * lprofcurFilename structure. */
parseFilenamePattern(const char * FilenamePat,unsigned CopyFilenamePat)765 static int parseFilenamePattern(const char *FilenamePat,
766 unsigned CopyFilenamePat) {
767 int NumPids = 0, NumHosts = 0, NumBinaryIds = 0, I;
768 char *PidChars = &lprofCurFilename.PidChars[0];
769 char *Hostname = &lprofCurFilename.Hostname[0];
770 int MergingEnabled = 0;
771 int FilenamePatLen = strlen(FilenamePat);
772
773 #ifdef __GNUC__
774 #pragma GCC diagnostic push
775 #pragma GCC diagnostic ignored "-Wcast-qual"
776 #elif defined(__clang__)
777 #pragma clang diagnostic push
778 #pragma clang diagnostic ignored "-Wcast-qual"
779 #endif
780 /* Clean up cached prefix and filename. */
781 if (lprofCurFilename.ProfilePathPrefix)
782 free((void *)lprofCurFilename.ProfilePathPrefix);
783
784 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
785 free((void *)lprofCurFilename.FilenamePat);
786 }
787 #ifdef __GNUC__
788 #pragma GCC diagnostic pop
789 #elif defined(__clang__)
790 #pragma clang diagnostic pop
791 #endif
792
793 memset(&lprofCurFilename, 0, sizeof(lprofCurFilename));
794
795 if (!CopyFilenamePat)
796 lprofCurFilename.FilenamePat = FilenamePat;
797 else {
798 lprofCurFilename.FilenamePat = strdup(FilenamePat);
799 lprofCurFilename.OwnsFilenamePat = 1;
800 }
801 /* Check the filename for "%p", which indicates a pid-substitution. */
802 for (I = 0; checkBounds(I, FilenamePatLen) && FilenamePat[I]; ++I) {
803 if (FilenamePat[I] == '%') {
804 ++I; /* Advance to the next character. */
805 if (!checkBounds(I, FilenamePatLen))
806 break;
807 if (FilenamePat[I] == 'p') {
808 if (!NumPids++) {
809 if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) {
810 PROF_WARN("Unable to get pid for filename pattern %s. Using the "
811 "default name.",
812 FilenamePat);
813 return -1;
814 }
815 }
816 } else if (FilenamePat[I] == 'h') {
817 if (!NumHosts++)
818 if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) {
819 PROF_WARN("Unable to get hostname for filename pattern %s. Using "
820 "the default name.",
821 FilenamePat);
822 return -1;
823 }
824 } else if (FilenamePat[I] == 't') {
825 lprofCurFilename.TmpDir = getenv("TMPDIR");
826 if (!lprofCurFilename.TmpDir) {
827 PROF_WARN("Unable to get the TMPDIR environment variable, referenced "
828 "in %s. Using the default path.",
829 FilenamePat);
830 return -1;
831 }
832 } else if (FilenamePat[I] == 'b') {
833 if (!NumBinaryIds++) {
834 /* Check if binary ID does not exist or if its size is 0. */
835 if (__llvm_write_binary_ids(NULL) <= 0) {
836 PROF_WARN("Unable to get binary ID for filename pattern %s. Using "
837 "the default name.",
838 FilenamePat);
839 return -1;
840 }
841 }
842 } else if (FilenamePat[I] == 'c') {
843 if (__llvm_profile_is_continuous_mode_enabled()) {
844 PROF_WARN("%%c specifier can only be specified once in %s.\n",
845 FilenamePat);
846 __llvm_profile_disable_continuous_mode();
847 return -1;
848 }
849 #if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) || defined(_AIX)
850 __llvm_profile_set_page_size(getpagesize());
851 __llvm_profile_enable_continuous_mode();
852 #else
853 PROF_WARN("%s",
854 "Continuous mode is currently only supported for Mach-O,"
855 " ELF and COFF formats.");
856 return -1;
857 #endif
858 } else {
859 unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I);
860 if (!MergePoolSize)
861 continue;
862 if (MergingEnabled) {
863 PROF_WARN("%%m specifier can only be specified once in %s.\n",
864 FilenamePat);
865 return -1;
866 }
867 MergingEnabled = 1;
868 lprofCurFilename.MergePoolSize = MergePoolSize;
869 }
870 }
871 }
872
873 lprofCurFilename.NumPids = NumPids;
874 lprofCurFilename.NumHosts = NumHosts;
875 lprofCurFilename.NumBinaryIds = NumBinaryIds;
876 return 0;
877 }
878
parseAndSetFilename(const char * FilenamePat,ProfileNameSpecifier PNS,unsigned CopyFilenamePat)879 static void parseAndSetFilename(const char *FilenamePat,
880 ProfileNameSpecifier PNS,
881 unsigned CopyFilenamePat) {
882
883 const char *OldFilenamePat = lprofCurFilename.FilenamePat;
884 ProfileNameSpecifier OldPNS = lprofCurFilename.PNS;
885
886 /* The old profile name specifier takes precedence over the old one. */
887 if (PNS < OldPNS)
888 return;
889
890 if (!FilenamePat)
891 FilenamePat = DefaultProfileName;
892
893 if (OldFilenamePat && !strcmp(OldFilenamePat, FilenamePat)) {
894 lprofCurFilename.PNS = PNS;
895 return;
896 }
897
898 /* When PNS >= OldPNS, the last one wins. */
899 if (!FilenamePat || parseFilenamePattern(FilenamePat, CopyFilenamePat))
900 resetFilenameToDefault();
901 lprofCurFilename.PNS = PNS;
902
903 if (!OldFilenamePat) {
904 if (getenv("LLVM_PROFILE_VERBOSE"))
905 PROF_NOTE("Set profile file path to \"%s\" via %s.\n",
906 lprofCurFilename.FilenamePat, getPNSStr(PNS));
907 } else {
908 if (getenv("LLVM_PROFILE_VERBOSE"))
909 PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n",
910 OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat,
911 getPNSStr(PNS));
912 }
913
914 truncateCurrentFile();
915 if (__llvm_profile_is_continuous_mode_enabled())
916 initializeProfileForContinuousMode();
917 }
918
919 /* Return buffer length that is required to store the current profile
920 * filename with PID and hostname substitutions. */
921 /* The length to hold uint64_t followed by 3 digits pool id including '_' */
922 #define SIGLEN 24
923 /* The length to hold 160-bit hash in hexadecimal form */
924 #define BINARY_ID_LEN 40
getCurFilenameLength(void)925 static int getCurFilenameLength(void) {
926 int Len;
927 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
928 return 0;
929
930 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
931 lprofCurFilename.NumBinaryIds || lprofCurFilename.TmpDir ||
932 lprofCurFilename.MergePoolSize))
933 return strlen(lprofCurFilename.FilenamePat);
934
935 Len = strlen(lprofCurFilename.FilenamePat) +
936 lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) +
937 lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2) +
938 lprofCurFilename.NumBinaryIds * BINARY_ID_LEN +
939 (lprofCurFilename.TmpDir ? (strlen(lprofCurFilename.TmpDir) - 1) : 0);
940 if (lprofCurFilename.MergePoolSize)
941 Len += SIGLEN;
942 return Len;
943 }
944
945 typedef struct lprofBinaryIdsBuffer {
946 char String[BINARY_ID_LEN + 1];
947 int Length;
948 } lprofBinaryIdsBuffer;
949
950 /* Reads binary ID length and then its data, writes it into lprofBinaryIdsBuffer
951 * in hexadecimal form. */
binaryIdsStringWriter(ProfDataWriter * This,ProfDataIOVec * IOVecs,uint32_t NumIOVecs)952 static uint32_t binaryIdsStringWriter(ProfDataWriter *This,
953 ProfDataIOVec *IOVecs,
954 uint32_t NumIOVecs) {
955 if (NumIOVecs < 2 || IOVecs[0].ElmSize != sizeof(uint64_t))
956 return -1;
957 uint64_t BinaryIdLen = *(const uint64_t *)IOVecs[0].Data;
958 if (IOVecs[1].ElmSize != sizeof(uint8_t) || IOVecs[1].NumElm != BinaryIdLen)
959 return -1;
960 const uint8_t *BinaryIdData = (const uint8_t *)IOVecs[1].Data;
961 lprofBinaryIdsBuffer *Data = (lprofBinaryIdsBuffer *)This->WriterCtx;
962 for (uint64_t I = 0; I < BinaryIdLen; I++) {
963 Data->Length +=
964 snprintf(Data->String + Data->Length, BINARY_ID_LEN + 1 - Data->Length,
965 "%02hhx", BinaryIdData[I]);
966 }
967 return 0;
968 }
969
970 /* Return the pointer to the current profile file name (after substituting
971 * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer
972 * to store the resulting filename. If no substitution is needed, the
973 * current filename pattern string is directly returned, unless ForceUseBuf
974 * is enabled. */
getCurFilename(char * FilenameBuf,int ForceUseBuf)975 static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) {
976 int I, J, PidLength, HostNameLength, TmpDirLength, FilenamePatLength;
977 const char *FilenamePat = lprofCurFilename.FilenamePat;
978
979 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
980 return 0;
981
982 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
983 lprofCurFilename.NumBinaryIds || lprofCurFilename.TmpDir ||
984 lprofCurFilename.MergePoolSize ||
985 __llvm_profile_is_continuous_mode_enabled())) {
986 if (!ForceUseBuf)
987 return lprofCurFilename.FilenamePat;
988
989 FilenamePatLength = strlen(lprofCurFilename.FilenamePat);
990 memcpy(FilenameBuf, lprofCurFilename.FilenamePat, FilenamePatLength);
991 FilenameBuf[FilenamePatLength] = '\0';
992 return FilenameBuf;
993 }
994
995 PidLength = strlen(lprofCurFilename.PidChars);
996 HostNameLength = strlen(lprofCurFilename.Hostname);
997 TmpDirLength = lprofCurFilename.TmpDir ? strlen(lprofCurFilename.TmpDir) : 0;
998 /* Construct the new filename. */
999 for (I = 0, J = 0; FilenamePat[I]; ++I)
1000 if (FilenamePat[I] == '%') {
1001 if (FilenamePat[++I] == 'p') {
1002 memcpy(FilenameBuf + J, lprofCurFilename.PidChars, PidLength);
1003 J += PidLength;
1004 } else if (FilenamePat[I] == 'h') {
1005 memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength);
1006 J += HostNameLength;
1007 } else if (FilenamePat[I] == 't') {
1008 memcpy(FilenameBuf + J, lprofCurFilename.TmpDir, TmpDirLength);
1009 FilenameBuf[J + TmpDirLength] = DIR_SEPARATOR;
1010 J += TmpDirLength + 1;
1011 } else if (FilenamePat[I] == 'b') {
1012 lprofBinaryIdsBuffer Data = {{0}, 0};
1013 ProfDataWriter Writer = {binaryIdsStringWriter, &Data};
1014 __llvm_write_binary_ids(&Writer);
1015 memcpy(FilenameBuf + J, Data.String, Data.Length);
1016 J += Data.Length;
1017 } else {
1018 if (!getMergePoolSize(FilenamePat, &I))
1019 continue;
1020 char LoadModuleSignature[SIGLEN + 1];
1021 int S;
1022 int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize;
1023 S = snprintf(LoadModuleSignature, SIGLEN + 1, "%" PRIu64 "_%d",
1024 lprofGetLoadModuleSignature(), ProfilePoolId);
1025 if (S == -1 || S > SIGLEN)
1026 S = SIGLEN;
1027 memcpy(FilenameBuf + J, LoadModuleSignature, S);
1028 J += S;
1029 }
1030 /* Drop any unknown substitutions. */
1031 } else
1032 FilenameBuf[J++] = FilenamePat[I];
1033 FilenameBuf[J] = 0;
1034
1035 return FilenameBuf;
1036 }
1037
1038 /* Returns the pointer to the environment variable
1039 * string. Returns null if the env var is not set. */
getFilenamePatFromEnv(void)1040 static const char *getFilenamePatFromEnv(void) {
1041 const char *Filename = getenv("LLVM_PROFILE_FILE");
1042 if (!Filename || !Filename[0])
1043 return 0;
1044 return Filename;
1045 }
1046
1047 COMPILER_RT_VISIBILITY
__llvm_profile_get_path_prefix(void)1048 const char *__llvm_profile_get_path_prefix(void) {
1049 int Length;
1050 char *FilenameBuf, *Prefix;
1051 const char *Filename, *PrefixEnd;
1052
1053 if (lprofCurFilename.ProfilePathPrefix)
1054 return lprofCurFilename.ProfilePathPrefix;
1055
1056 Length = getCurFilenameLength();
1057 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1058 Filename = getCurFilename(FilenameBuf, 0);
1059 if (!Filename)
1060 return "\0";
1061
1062 PrefixEnd = lprofFindLastDirSeparator(Filename);
1063 if (!PrefixEnd)
1064 return "\0";
1065
1066 Length = PrefixEnd - Filename + 1;
1067 Prefix = (char *)malloc(Length + 1);
1068 if (!Prefix) {
1069 PROF_ERR("Failed to %s\n", "allocate memory.");
1070 return "\0";
1071 }
1072 memcpy(Prefix, Filename, Length);
1073 Prefix[Length] = '\0';
1074 lprofCurFilename.ProfilePathPrefix = Prefix;
1075 return Prefix;
1076 }
1077
1078 COMPILER_RT_VISIBILITY
__llvm_profile_get_filename(void)1079 const char *__llvm_profile_get_filename(void) {
1080 int Length;
1081 char *FilenameBuf;
1082 const char *Filename;
1083
1084 Length = getCurFilenameLength();
1085 FilenameBuf = (char *)malloc(Length + 1);
1086 if (!FilenameBuf) {
1087 PROF_ERR("Failed to %s\n", "allocate memory.");
1088 return "\0";
1089 }
1090 Filename = getCurFilename(FilenameBuf, 1);
1091 if (!Filename) {
1092 free(FilenameBuf);
1093 return "\0";
1094 }
1095
1096 return FilenameBuf;
1097 }
1098
1099 /* This API initializes the file handling, both user specified
1100 * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE
1101 * environment variable can override this default value.
1102 */
1103 COMPILER_RT_VISIBILITY
__llvm_profile_initialize_file(void)1104 void __llvm_profile_initialize_file(void) {
1105 const char *EnvFilenamePat;
1106 const char *SelectedPat = NULL;
1107 ProfileNameSpecifier PNS = PNS_unknown;
1108 int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
1109
1110 EnvFilenamePat = getFilenamePatFromEnv();
1111 if (EnvFilenamePat) {
1112 /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid
1113 at the moment when __llvm_profile_write_file() gets executed. */
1114 parseAndSetFilename(EnvFilenamePat, PNS_environment, 1);
1115 return;
1116 } else if (hasCommandLineOverrider) {
1117 SelectedPat = INSTR_PROF_PROFILE_NAME_VAR;
1118 PNS = PNS_command_line;
1119 } else {
1120 SelectedPat = NULL;
1121 PNS = PNS_default;
1122 }
1123
1124 parseAndSetFilename(SelectedPat, PNS, 0);
1125 }
1126
1127 /* This method is invoked by the runtime initialization hook
1128 * InstrProfilingRuntime.o if it is linked in.
1129 */
1130 COMPILER_RT_VISIBILITY
__llvm_profile_initialize(void)1131 void __llvm_profile_initialize(void) {
1132 __llvm_profile_initialize_file();
1133 if (!__llvm_profile_is_continuous_mode_enabled())
1134 __llvm_profile_register_write_file_atexit();
1135 }
1136
1137 /* This API is directly called by the user application code. It has the
1138 * highest precedence compared with LLVM_PROFILE_FILE environment variable
1139 * and command line option -fprofile-instr-generate=<profile_name>.
1140 */
1141 COMPILER_RT_VISIBILITY
__llvm_profile_set_filename(const char * FilenamePat)1142 void __llvm_profile_set_filename(const char *FilenamePat) {
1143 if (__llvm_profile_is_continuous_mode_enabled())
1144 return;
1145 parseAndSetFilename(FilenamePat, PNS_runtime_api, 1);
1146 }
1147
1148 /* The public API for writing profile data into the file with name
1149 * set by previous calls to __llvm_profile_set_filename or
1150 * __llvm_profile_override_default_filename or
1151 * __llvm_profile_initialize_file. */
1152 COMPILER_RT_VISIBILITY
__llvm_profile_write_file(void)1153 int __llvm_profile_write_file(void) {
1154 int rc, Length;
1155 const char *Filename;
1156 char *FilenameBuf;
1157
1158 // Temporarily suspend getting SIGKILL when the parent exits.
1159 int PDeathSig = lprofSuspendSigKill();
1160
1161 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
1162 PROF_NOTE("Profile data not written to file: %s.\n", "already written");
1163 if (PDeathSig == 1)
1164 lprofRestoreSigKill();
1165 return 0;
1166 }
1167
1168 Length = getCurFilenameLength();
1169 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1170 Filename = getCurFilename(FilenameBuf, 0);
1171
1172 /* Check the filename. */
1173 if (!Filename) {
1174 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1175 if (PDeathSig == 1)
1176 lprofRestoreSigKill();
1177 return -1;
1178 }
1179
1180 /* Check if there is llvm/runtime version mismatch. */
1181 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1182 PROF_ERR("Runtime and instrumentation version mismatch : "
1183 "expected %d, but get %d\n",
1184 INSTR_PROF_RAW_VERSION,
1185 (int)GET_VERSION(__llvm_profile_get_version()));
1186 if (PDeathSig == 1)
1187 lprofRestoreSigKill();
1188 return -1;
1189 }
1190
1191 /* Write profile data to the file. */
1192 rc = writeFile(Filename);
1193 if (rc)
1194 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
1195
1196 // Restore SIGKILL.
1197 if (PDeathSig == 1)
1198 lprofRestoreSigKill();
1199
1200 return rc;
1201 }
1202
1203 COMPILER_RT_VISIBILITY
__llvm_profile_dump(void)1204 int __llvm_profile_dump(void) {
1205 if (!doMerging())
1206 PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering "
1207 " of previously dumped profile data : %s. Either use %%m "
1208 "in profile name or change profile name before dumping.\n",
1209 "online profile merging is not on");
1210 int rc = __llvm_profile_write_file();
1211 lprofSetProfileDumped(1);
1212 return rc;
1213 }
1214
writeFileWithoutReturn(void)1215 static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); }
1216
1217 COMPILER_RT_VISIBILITY
__llvm_profile_register_write_file_atexit(void)1218 int __llvm_profile_register_write_file_atexit(void) {
1219 static int HasBeenRegistered = 0;
1220
1221 if (HasBeenRegistered)
1222 return 0;
1223
1224 lprofSetupValueProfiler();
1225
1226 HasBeenRegistered = 1;
1227 return lprofAtExit(writeFileWithoutReturn);
1228 }
1229
__llvm_profile_set_file_object(FILE * File,int EnableMerge)1230 COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File,
1231 int EnableMerge) {
1232 if (__llvm_profile_is_continuous_mode_enabled()) {
1233 if (!EnableMerge) {
1234 PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported in "
1235 "continuous sync mode when merging is disabled\n",
1236 fileno(File));
1237 return 1;
1238 }
1239 if (lprofLockFileHandle(File) != 0) {
1240 PROF_WARN("Data may be corrupted during profile merging : %s\n",
1241 "Fail to obtain file lock due to system limit.");
1242 }
1243 uint64_t ProfileFileSize = 0;
1244 if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
1245 lprofUnlockFileHandle(File);
1246 return 1;
1247 }
1248 if (ProfileFileSize == 0) {
1249 FreeHook = &free;
1250 setupIOBuffer();
1251 ProfDataWriter fileWriter;
1252 initFileWriter(&fileWriter, File);
1253 if (lprofWriteData(&fileWriter, 0, 0)) {
1254 lprofUnlockFileHandle(File);
1255 PROF_ERR("Failed to write file \"%d\": %s\n", fileno(File),
1256 strerror(errno));
1257 return 1;
1258 }
1259 fflush(File);
1260 } else {
1261 /* The merged profile has a non-zero length. Check that it is compatible
1262 * with the data in this process. */
1263 ManagedMemory ProfileBuffer;
1264 if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) {
1265 lprofUnlockFileHandle(File);
1266 return 1;
1267 }
1268 (void)lprofReleaseBuffer(&ProfileBuffer, ProfileFileSize);
1269 }
1270 mmapForContinuousMode(0, File);
1271 lprofUnlockFileHandle(File);
1272 } else {
1273 setProfileFile(File);
1274 setProfileMergeRequested(EnableMerge);
1275 }
1276 return 0;
1277 }
1278
1279 #ifndef __APPLE__
__llvm_write_custom_profile(const char * Target,const __llvm_profile_data * DataBegin,const __llvm_profile_data * DataEnd,const char * CountersBegin,const char * CountersEnd,const char * NamesBegin,const char * NamesEnd,const uint64_t * VersionOverride)1280 int __llvm_write_custom_profile(const char *Target,
1281 const __llvm_profile_data *DataBegin,
1282 const __llvm_profile_data *DataEnd,
1283 const char *CountersBegin,
1284 const char *CountersEnd, const char *NamesBegin,
1285 const char *NamesEnd,
1286 const uint64_t *VersionOverride) {
1287 int ReturnValue = 0, FilenameLength, TargetLength;
1288 char *FilenameBuf, *TargetFilename;
1289 const char *Filename;
1290
1291 /* Save old profile data */
1292 FILE *oldFile = getProfileFile();
1293
1294 // Temporarily suspend getting SIGKILL when the parent exits.
1295 int PDeathSig = lprofSuspendSigKill();
1296
1297 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
1298 PROF_NOTE("Profile data not written to file: %s.\n", "already written");
1299 if (PDeathSig == 1)
1300 lprofRestoreSigKill();
1301 return 0;
1302 }
1303
1304 /* Check if there is llvm/runtime version mismatch. */
1305 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1306 PROF_ERR("Runtime and instrumentation version mismatch : "
1307 "expected %d, but get %d\n",
1308 INSTR_PROF_RAW_VERSION,
1309 (int)GET_VERSION(__llvm_profile_get_version()));
1310 if (PDeathSig == 1)
1311 lprofRestoreSigKill();
1312 return -1;
1313 }
1314
1315 /* Get current filename */
1316 FilenameLength = getCurFilenameLength();
1317 FilenameBuf = (char *)COMPILER_RT_ALLOCA(FilenameLength + 1);
1318 Filename = getCurFilename(FilenameBuf, 0);
1319
1320 /* Check the filename. */
1321 if (!Filename) {
1322 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1323 if (PDeathSig == 1)
1324 lprofRestoreSigKill();
1325 return -1;
1326 }
1327
1328 /* Allocate new space for our target-specific PGO filename */
1329 TargetLength = strlen(Target);
1330 TargetFilename =
1331 (char *)COMPILER_RT_ALLOCA(FilenameLength + TargetLength + 2);
1332
1333 /* Find file basename and path sizes */
1334 int32_t DirEnd = FilenameLength - 1;
1335 while (DirEnd >= 0 && !IS_DIR_SEPARATOR(Filename[DirEnd])) {
1336 DirEnd--;
1337 }
1338 uint32_t DirSize = DirEnd + 1, BaseSize = FilenameLength - DirSize;
1339
1340 /* Prepend "TARGET." to current filename */
1341 if (DirSize > 0) {
1342 memcpy(TargetFilename, Filename, DirSize);
1343 }
1344 memcpy(TargetFilename + DirSize, Target, TargetLength);
1345 TargetFilename[TargetLength + DirSize] = '.';
1346 memcpy(TargetFilename + DirSize + 1 + TargetLength, Filename + DirSize,
1347 BaseSize);
1348 TargetFilename[FilenameLength + 1 + TargetLength] = 0;
1349
1350 /* Open and truncate target-specific PGO file */
1351 FILE *OutputFile = fopen(TargetFilename, "w");
1352 setProfileFile(OutputFile);
1353
1354 if (!OutputFile) {
1355 PROF_ERR("Failed to open file : %s\n", TargetFilename);
1356 if (PDeathSig == 1)
1357 lprofRestoreSigKill();
1358 return -1;
1359 }
1360
1361 FreeHook = &free;
1362 setupIOBuffer();
1363
1364 /* Write custom data */
1365 ProfDataWriter fileWriter;
1366 initFileWriter(&fileWriter, OutputFile);
1367
1368 uint64_t Version = __llvm_profile_get_version();
1369 if (VersionOverride)
1370 Version = *VersionOverride;
1371
1372 /* Write custom data to the file */
1373 ReturnValue =
1374 lprofWriteDataImpl(&fileWriter, DataBegin, DataEnd, CountersBegin,
1375 CountersEnd, NULL, NULL, lprofGetVPDataReader(), NULL,
1376 NULL, NULL, NULL, NamesBegin, NamesEnd, 0, Version);
1377 closeFileObject(OutputFile);
1378
1379 // Restore SIGKILL.
1380 if (PDeathSig == 1)
1381 lprofRestoreSigKill();
1382
1383 /* Restore old profiling file */
1384 setProfileFile(oldFile);
1385
1386 return ReturnValue;
1387 }
1388 #endif
1389
1390 #endif
1391