1 /*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\ 2 |* 3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 |* See https://llvm.org/LICENSE.txt for license information. 5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 |* 7 \*===----------------------------------------------------------------------===*/ 8 9 #if !defined(__Fuchsia__) 10 11 #include <errno.h> 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <string.h> 15 #ifdef _MSC_VER 16 /* For _alloca. */ 17 #include <malloc.h> 18 #endif 19 #if defined(_WIN32) 20 #include "WindowsMMap.h" 21 /* For _chsize_s */ 22 #include <io.h> 23 #include <process.h> 24 #else 25 #include <sys/file.h> 26 #include <sys/mman.h> 27 #include <unistd.h> 28 #if defined(__linux__) 29 #include <sys/types.h> 30 #endif 31 #endif 32 33 #include "InstrProfiling.h" 34 #include "InstrProfilingInternal.h" 35 #include "InstrProfilingPort.h" 36 #include "InstrProfilingUtil.h" 37 38 /* From where is profile name specified. 39 * The order the enumerators define their 40 * precedence. Re-order them may lead to 41 * runtime behavior change. */ 42 typedef enum ProfileNameSpecifier { 43 PNS_unknown = 0, 44 PNS_default, 45 PNS_command_line, 46 PNS_environment, 47 PNS_runtime_api 48 } ProfileNameSpecifier; 49 50 static const char *getPNSStr(ProfileNameSpecifier PNS) { 51 switch (PNS) { 52 case PNS_default: 53 return "default setting"; 54 case PNS_command_line: 55 return "command line"; 56 case PNS_environment: 57 return "environment variable"; 58 case PNS_runtime_api: 59 return "runtime API"; 60 default: 61 return "Unknown"; 62 } 63 } 64 65 #define MAX_PID_SIZE 16 66 /* Data structure holding the result of parsed filename pattern. */ 67 typedef struct lprofFilename { 68 /* File name string possibly with %p or %h specifiers. */ 69 const char *FilenamePat; 70 /* A flag indicating if FilenamePat's memory is allocated 71 * by runtime. */ 72 unsigned OwnsFilenamePat; 73 const char *ProfilePathPrefix; 74 char PidChars[MAX_PID_SIZE]; 75 char Hostname[COMPILER_RT_MAX_HOSTLEN]; 76 unsigned NumPids; 77 unsigned NumHosts; 78 /* When in-process merging is enabled, this parameter specifies 79 * the total number of profile data files shared by all the processes 80 * spawned from the same binary. By default the value is 1. If merging 81 * is not enabled, its value should be 0. This parameter is specified 82 * by the %[0-9]m specifier. For instance %2m enables merging using 83 * 2 profile data files. %1m is equivalent to %m. Also %m specifier 84 * can only appear once at the end of the name pattern. */ 85 unsigned MergePoolSize; 86 ProfileNameSpecifier PNS; 87 } lprofFilename; 88 89 static lprofFilename lprofCurFilename = {0, 0, 0, {0}, {0}, 90 0, 0, 0, PNS_unknown}; 91 92 static int ProfileMergeRequested = 0; 93 static int isProfileMergeRequested() { return ProfileMergeRequested; } 94 static void setProfileMergeRequested(int EnableMerge) { 95 ProfileMergeRequested = EnableMerge; 96 } 97 98 static FILE *ProfileFile = NULL; 99 static FILE *getProfileFile() { return ProfileFile; } 100 static void setProfileFile(FILE *File) { ProfileFile = File; } 101 102 COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File, 103 int EnableMerge) { 104 if (__llvm_profile_is_continuous_mode_enabled()) { 105 PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported, because " 106 "continuous sync mode (%%c) is enabled", 107 fileno(File)); 108 return; 109 } 110 setProfileFile(File); 111 setProfileMergeRequested(EnableMerge); 112 } 113 114 static int getCurFilenameLength(); 115 static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf); 116 static unsigned doMerging() { 117 return lprofCurFilename.MergePoolSize || isProfileMergeRequested(); 118 } 119 120 /* Return 1 if there is an error, otherwise return 0. */ 121 static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, 122 uint32_t NumIOVecs) { 123 uint32_t I; 124 FILE *File = (FILE *)This->WriterCtx; 125 char Zeroes[sizeof(uint64_t)] = {0}; 126 for (I = 0; I < NumIOVecs; I++) { 127 if (IOVecs[I].Data) { 128 if (fwrite(IOVecs[I].Data, IOVecs[I].ElmSize, IOVecs[I].NumElm, File) != 129 IOVecs[I].NumElm) 130 return 1; 131 } else if (IOVecs[I].UseZeroPadding) { 132 size_t BytesToWrite = IOVecs[I].ElmSize * IOVecs[I].NumElm; 133 while (BytesToWrite > 0) { 134 size_t PartialWriteLen = 135 (sizeof(uint64_t) > BytesToWrite) ? BytesToWrite : sizeof(uint64_t); 136 if (fwrite(Zeroes, sizeof(uint8_t), PartialWriteLen, File) != 137 PartialWriteLen) { 138 return 1; 139 } 140 BytesToWrite -= PartialWriteLen; 141 } 142 } else { 143 if (fseek(File, IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1) 144 return 1; 145 } 146 } 147 return 0; 148 } 149 150 /* TODO: make buffer size controllable by an internal option, and compiler can pass the size 151 to runtime via a variable. */ 152 static uint32_t orderFileWriter(FILE *File, const uint32_t *DataStart) { 153 if (fwrite(DataStart, sizeof(uint32_t), INSTR_ORDER_FILE_BUFFER_SIZE, File) != 154 INSTR_ORDER_FILE_BUFFER_SIZE) 155 return 1; 156 return 0; 157 } 158 159 static void initFileWriter(ProfDataWriter *This, FILE *File) { 160 This->Write = fileWriter; 161 This->WriterCtx = File; 162 } 163 164 COMPILER_RT_VISIBILITY ProfBufferIO * 165 lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) { 166 FreeHook = &free; 167 DynamicBufferIOBuffer = (uint8_t *)calloc(BufferSz, 1); 168 VPBufferSize = BufferSz; 169 ProfDataWriter *fileWriter = 170 (ProfDataWriter *)calloc(sizeof(ProfDataWriter), 1); 171 initFileWriter(fileWriter, File); 172 ProfBufferIO *IO = lprofCreateBufferIO(fileWriter); 173 IO->OwnFileWriter = 1; 174 return IO; 175 } 176 177 static void setupIOBuffer() { 178 const char *BufferSzStr = 0; 179 BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE"); 180 if (BufferSzStr && BufferSzStr[0]) { 181 VPBufferSize = atoi(BufferSzStr); 182 DynamicBufferIOBuffer = (uint8_t *)calloc(VPBufferSize, 1); 183 } 184 } 185 186 /* Get the size of the profile file. If there are any errors, print the 187 * message under the assumption that the profile is being read for merging 188 * purposes, and return -1. Otherwise return the file size in the inout param 189 * \p ProfileFileSize. */ 190 static int getProfileFileSizeForMerging(FILE *ProfileFile, 191 uint64_t *ProfileFileSize) { 192 if (fseek(ProfileFile, 0L, SEEK_END) == -1) { 193 PROF_ERR("Unable to merge profile data, unable to get size: %s\n", 194 strerror(errno)); 195 return -1; 196 } 197 *ProfileFileSize = ftell(ProfileFile); 198 199 /* Restore file offset. */ 200 if (fseek(ProfileFile, 0L, SEEK_SET) == -1) { 201 PROF_ERR("Unable to merge profile data, unable to rewind: %s\n", 202 strerror(errno)); 203 return -1; 204 } 205 206 if (*ProfileFileSize > 0 && 207 *ProfileFileSize < sizeof(__llvm_profile_header)) { 208 PROF_WARN("Unable to merge profile data: %s\n", 209 "source profile file is too small."); 210 return -1; 211 } 212 return 0; 213 } 214 215 /* mmap() \p ProfileFile for profile merging purposes, assuming that an 216 * exclusive lock is held on the file and that \p ProfileFileSize is the 217 * length of the file. Return the mmap'd buffer in the inout variable 218 * \p ProfileBuffer. Returns -1 on failure. On success, the caller is 219 * responsible for unmapping the mmap'd buffer in \p ProfileBuffer. */ 220 static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize, 221 char **ProfileBuffer) { 222 *ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE, 223 fileno(ProfileFile), 0); 224 if (*ProfileBuffer == MAP_FAILED) { 225 PROF_ERR("Unable to merge profile data, mmap failed: %s\n", 226 strerror(errno)); 227 return -1; 228 } 229 230 if (__llvm_profile_check_compatibility(*ProfileBuffer, ProfileFileSize)) { 231 (void)munmap(*ProfileBuffer, ProfileFileSize); 232 PROF_WARN("Unable to merge profile data: %s\n", 233 "source profile file is not compatible."); 234 return -1; 235 } 236 return 0; 237 } 238 239 /* Read profile data in \c ProfileFile and merge with in-memory 240 profile counters. Returns -1 if there is fatal error, otheriwse 241 0 is returned. Returning 0 does not mean merge is actually 242 performed. If merge is actually done, *MergeDone is set to 1. 243 */ 244 static int doProfileMerging(FILE *ProfileFile, int *MergeDone) { 245 uint64_t ProfileFileSize; 246 char *ProfileBuffer; 247 248 /* Get the size of the profile on disk. */ 249 if (getProfileFileSizeForMerging(ProfileFile, &ProfileFileSize) == -1) 250 return -1; 251 252 /* Nothing to merge. */ 253 if (!ProfileFileSize) 254 return 0; 255 256 /* mmap() the profile and check that it is compatible with the data in 257 * the current image. */ 258 if (mmapProfileForMerging(ProfileFile, ProfileFileSize, &ProfileBuffer) == -1) 259 return -1; 260 261 /* Now start merging */ 262 __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize); 263 264 // Truncate the file in case merging of value profile did not happend to 265 // prevent from leaving garbage data at the end of the profile file. 266 COMPILER_RT_FTRUNCATE(ProfileFile, __llvm_profile_get_size_for_buffer()); 267 268 (void)munmap(ProfileBuffer, ProfileFileSize); 269 *MergeDone = 1; 270 271 return 0; 272 } 273 274 /* Create the directory holding the file, if needed. */ 275 static void createProfileDir(const char *Filename) { 276 size_t Length = strlen(Filename); 277 if (lprofFindFirstDirSeparator(Filename)) { 278 char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1); 279 strncpy(Copy, Filename, Length + 1); 280 __llvm_profile_recursive_mkdir(Copy); 281 } 282 } 283 284 /* Open the profile data for merging. It opens the file in r+b mode with 285 * file locking. If the file has content which is compatible with the 286 * current process, it also reads in the profile data in the file and merge 287 * it with in-memory counters. After the profile data is merged in memory, 288 * the original profile data is truncated and gets ready for the profile 289 * dumper. With profile merging enabled, each executable as well as any of 290 * its instrumented shared libraries dump profile data into their own data file. 291 */ 292 static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) { 293 FILE *ProfileFile = NULL; 294 int rc; 295 296 ProfileFile = getProfileFile(); 297 if (ProfileFile) { 298 lprofLockFileHandle(ProfileFile); 299 } else { 300 createProfileDir(ProfileFileName); 301 ProfileFile = lprofOpenFileEx(ProfileFileName); 302 } 303 if (!ProfileFile) 304 return NULL; 305 306 rc = doProfileMerging(ProfileFile, MergeDone); 307 if (rc || (!*MergeDone && COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) || 308 fseek(ProfileFile, 0L, SEEK_SET) == -1) { 309 PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName, 310 strerror(errno)); 311 fclose(ProfileFile); 312 return NULL; 313 } 314 return ProfileFile; 315 } 316 317 static FILE *getFileObject(const char *OutputName) { 318 FILE *File; 319 File = getProfileFile(); 320 if (File != NULL) { 321 return File; 322 } 323 324 return fopen(OutputName, "ab"); 325 } 326 327 /* Write profile data to file \c OutputName. */ 328 static int writeFile(const char *OutputName) { 329 int RetVal; 330 FILE *OutputFile; 331 332 int MergeDone = 0; 333 VPMergeHook = &lprofMergeValueProfData; 334 if (doMerging()) 335 OutputFile = openFileForMerging(OutputName, &MergeDone); 336 else 337 OutputFile = getFileObject(OutputName); 338 339 if (!OutputFile) 340 return -1; 341 342 FreeHook = &free; 343 setupIOBuffer(); 344 ProfDataWriter fileWriter; 345 initFileWriter(&fileWriter, OutputFile); 346 RetVal = lprofWriteData(&fileWriter, lprofGetVPDataReader(), MergeDone); 347 348 if (OutputFile == getProfileFile()) { 349 fflush(OutputFile); 350 if (doMerging()) { 351 lprofUnlockFileHandle(OutputFile); 352 } 353 } else { 354 fclose(OutputFile); 355 } 356 357 return RetVal; 358 } 359 360 /* Write order data to file \c OutputName. */ 361 static int writeOrderFile(const char *OutputName) { 362 int RetVal; 363 FILE *OutputFile; 364 365 OutputFile = fopen(OutputName, "w"); 366 367 if (!OutputFile) { 368 PROF_WARN("can't open file with mode ab: %s\n", OutputName); 369 return -1; 370 } 371 372 FreeHook = &free; 373 setupIOBuffer(); 374 const uint32_t *DataBegin = __llvm_profile_begin_orderfile(); 375 RetVal = orderFileWriter(OutputFile, DataBegin); 376 377 fclose(OutputFile); 378 return RetVal; 379 } 380 381 #define LPROF_INIT_ONCE_ENV "__LLVM_PROFILE_RT_INIT_ONCE" 382 383 static void truncateCurrentFile(void) { 384 const char *Filename; 385 char *FilenameBuf; 386 FILE *File; 387 int Length; 388 389 Length = getCurFilenameLength(); 390 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); 391 Filename = getCurFilename(FilenameBuf, 0); 392 if (!Filename) 393 return; 394 395 /* Only create the profile directory and truncate an existing profile once. 396 * In continuous mode, this is necessary, as the profile is written-to by the 397 * runtime initializer. */ 398 int initialized = getenv(LPROF_INIT_ONCE_ENV) != NULL; 399 if (initialized) 400 return; 401 #if defined(_WIN32) 402 _putenv(LPROF_INIT_ONCE_ENV "=" LPROF_INIT_ONCE_ENV); 403 #else 404 setenv(LPROF_INIT_ONCE_ENV, LPROF_INIT_ONCE_ENV, 1); 405 #endif 406 407 /* Create the profile dir (even if online merging is enabled), so that 408 * the profile file can be set up if continuous mode is enabled. */ 409 createProfileDir(Filename); 410 411 /* By pass file truncation to allow online raw profile merging. */ 412 if (lprofCurFilename.MergePoolSize) 413 return; 414 415 /* Truncate the file. Later we'll reopen and append. */ 416 File = fopen(Filename, "w"); 417 if (!File) 418 return; 419 fclose(File); 420 } 421 422 #ifndef _MSC_VER 423 static void assertIsZero(int *i) { 424 if (*i) 425 PROF_WARN("Expected flag to be 0, but got: %d\n", *i); 426 } 427 #endif 428 429 #if !defined(__Fuchsia__) && !defined(_WIN32) 430 /* Write a partial profile to \p Filename, which is required to be backed by 431 * the open file object \p File. */ 432 static int writeProfileWithFileObject(const char *Filename, FILE *File) { 433 setProfileFile(File); 434 int rc = writeFile(Filename); 435 if (rc) 436 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); 437 setProfileFile(NULL); 438 return rc; 439 } 440 441 /* Unlock the profile \p File and clear the unlock flag. */ 442 static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) { 443 if (!*ProfileRequiresUnlock) { 444 PROF_WARN("%s", "Expected to require profile unlock\n"); 445 } 446 lprofUnlockFileHandle(File); 447 *ProfileRequiresUnlock = 0; 448 } 449 #endif // !defined(__Fuchsia__) && !defined(_WIN32) 450 451 static int writeMMappedFile(FILE *OutputFile, char **Profile) { 452 if (!OutputFile) 453 return -1; 454 455 /* Write the data into a file. */ 456 setupIOBuffer(); 457 ProfDataWriter fileWriter; 458 initFileWriter(&fileWriter, OutputFile); 459 if (lprofWriteData(&fileWriter, NULL, 0)) { 460 PROF_ERR("Failed to write profile: %s\n", strerror(errno)); 461 return -1; 462 } 463 fflush(OutputFile); 464 465 /* Get the file size. */ 466 uint64_t FileSize = ftell(OutputFile); 467 468 /* Map the profile. */ 469 *Profile = (char *)mmap( 470 NULL, FileSize, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(OutputFile), 0); 471 if (*Profile == MAP_FAILED) { 472 PROF_ERR("Unable to mmap profile: %s\n", strerror(errno)); 473 return -1; 474 } 475 476 return 0; 477 } 478 479 static void relocateCounters(void) { 480 if (!__llvm_profile_is_continuous_mode_enabled() || 481 !lprofRuntimeCounterRelocation()) 482 return; 483 484 /* Get the sizes of various profile data sections. Taken from 485 * __llvm_profile_get_size_for_buffer(). */ 486 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); 487 const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); 488 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); 489 const uint64_t CountersOffset = sizeof(__llvm_profile_header) + 490 (DataSize * sizeof(__llvm_profile_data)); 491 492 int Length = getCurFilenameLength(); 493 char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); 494 const char *Filename = getCurFilename(FilenameBuf, 0); 495 if (!Filename) 496 return; 497 498 FILE *File = NULL; 499 char *Profile = NULL; 500 501 if (!doMerging()) { 502 File = fopen(Filename, "w+b"); 503 if (!File) 504 return; 505 506 if (writeMMappedFile(File, &Profile) == -1) { 507 fclose(File); 508 return; 509 } 510 } else { 511 File = lprofOpenFileEx(Filename); 512 if (!File) 513 return; 514 515 uint64_t ProfileFileSize = 0; 516 if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) { 517 lprofUnlockFileHandle(File); 518 fclose(File); 519 return; 520 } 521 522 if (!ProfileFileSize) { 523 if (writeMMappedFile(File, &Profile) == -1) { 524 fclose(File); 525 return; 526 } 527 } else { 528 /* The merged profile has a non-zero length. Check that it is compatible 529 * with the data in this process. */ 530 if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) { 531 fclose(File); 532 return; 533 } 534 } 535 536 lprofUnlockFileHandle(File); 537 } 538 539 /* Update the profile fields based on the current mapping. */ 540 __llvm_profile_counter_bias = (intptr_t)Profile - 541 (uintptr_t)__llvm_profile_begin_counters() + CountersOffset; 542 } 543 544 static void initializeProfileForContinuousMode(void) { 545 if (!__llvm_profile_is_continuous_mode_enabled()) 546 return; 547 548 #if defined(__Fuchsia__) || defined(_WIN32) 549 PROF_ERR("%s\n", "Continuous mode not yet supported on Fuchsia or Windows."); 550 #else // defined(__Fuchsia__) || defined(_WIN32) 551 /* Get the sizes of various profile data sections. Taken from 552 * __llvm_profile_get_size_for_buffer(). */ 553 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); 554 const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); 555 const uint64_t *CountersBegin = __llvm_profile_begin_counters(); 556 const uint64_t *CountersEnd = __llvm_profile_end_counters(); 557 const char *NamesBegin = __llvm_profile_begin_names(); 558 const char *NamesEnd = __llvm_profile_end_names(); 559 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); 560 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); 561 uint64_t CountersSize = CountersEnd - CountersBegin; 562 563 /* Check that the counter and data sections in this image are page-aligned. */ 564 unsigned PageSize = getpagesize(); 565 if ((intptr_t)CountersBegin % PageSize != 0) { 566 PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n", 567 CountersBegin, PageSize); 568 return; 569 } 570 if ((intptr_t)DataBegin % PageSize != 0) { 571 PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", 572 DataBegin, PageSize); 573 return; 574 } 575 576 int Length = getCurFilenameLength(); 577 char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); 578 const char *Filename = getCurFilename(FilenameBuf, 0); 579 if (!Filename) 580 return; 581 582 FILE *File = NULL; 583 off_t CurrentFileOffset = 0; 584 off_t OffsetModPage = 0; 585 586 /* Whether an exclusive lock on the profile must be dropped after init. 587 * Use a cleanup to warn if the unlock does not occur. */ 588 COMPILER_RT_CLEANUP(assertIsZero) int ProfileRequiresUnlock = 0; 589 590 if (!doMerging()) { 591 /* We are not merging profiles, so open the raw profile in append mode. */ 592 File = fopen(Filename, "a+b"); 593 if (!File) 594 return; 595 596 /* Check that the offset within the file is page-aligned. */ 597 CurrentFileOffset = ftello(File); 598 OffsetModPage = CurrentFileOffset % PageSize; 599 if (OffsetModPage != 0) { 600 PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not" 601 "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n", 602 (uint64_t)CurrentFileOffset, PageSize); 603 return; 604 } 605 606 /* Grow the profile so that mmap() can succeed. Leak the file handle, as 607 * the file should stay open. */ 608 if (writeProfileWithFileObject(Filename, File) != 0) 609 return; 610 } else { 611 /* We are merging profiles. Map the counter section as shared memory into 612 * the profile, i.e. into each participating process. An increment in one 613 * process should be visible to every other process with the same counter 614 * section mapped. */ 615 File = lprofOpenFileEx(Filename); 616 if (!File) 617 return; 618 619 ProfileRequiresUnlock = 1; 620 621 uint64_t ProfileFileSize; 622 if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) 623 return unlockProfile(&ProfileRequiresUnlock, File); 624 625 if (ProfileFileSize == 0) { 626 /* Grow the profile so that mmap() can succeed. Leak the file handle, as 627 * the file should stay open. */ 628 if (writeProfileWithFileObject(Filename, File) != 0) 629 return unlockProfile(&ProfileRequiresUnlock, File); 630 } else { 631 /* The merged profile has a non-zero length. Check that it is compatible 632 * with the data in this process. */ 633 char *ProfileBuffer; 634 if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1 || 635 munmap(ProfileBuffer, ProfileFileSize) == -1) 636 return unlockProfile(&ProfileRequiresUnlock, File); 637 } 638 } 639 640 int Fileno = fileno(File); 641 642 /* Determine how much padding is needed before/after the counters and after 643 * the names. */ 644 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, 645 PaddingBytesAfterNames; 646 __llvm_profile_get_padding_sizes_for_counters( 647 DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, 648 &PaddingBytesAfterCounters, &PaddingBytesAfterNames); 649 650 uint64_t PageAlignedCountersLength = 651 (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters; 652 uint64_t FileOffsetToCounters = 653 CurrentFileOffset + sizeof(__llvm_profile_header) + 654 (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters; 655 656 uint64_t *CounterMmap = (uint64_t *)mmap( 657 (void *)CountersBegin, PageAlignedCountersLength, PROT_READ | PROT_WRITE, 658 MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters); 659 if (CounterMmap != CountersBegin) { 660 PROF_ERR( 661 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" 662 " - CountersBegin: %p\n" 663 " - PageAlignedCountersLength: %" PRIu64 "\n" 664 " - Fileno: %d\n" 665 " - FileOffsetToCounters: %" PRIu64 "\n", 666 strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno, 667 FileOffsetToCounters); 668 } 669 670 unlockProfile(&ProfileRequiresUnlock, File); 671 #endif // defined(__Fuchsia__) || defined(_WIN32) 672 } 673 674 static const char *DefaultProfileName = "default.profraw"; 675 static void resetFilenameToDefault(void) { 676 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { 677 free((void *)lprofCurFilename.FilenamePat); 678 } 679 memset(&lprofCurFilename, 0, sizeof(lprofCurFilename)); 680 lprofCurFilename.FilenamePat = DefaultProfileName; 681 lprofCurFilename.PNS = PNS_default; 682 } 683 684 static unsigned getMergePoolSize(const char *FilenamePat, int *I) { 685 unsigned J = 0, Num = 0; 686 for (;; ++J) { 687 char C = FilenamePat[*I + J]; 688 if (C == 'm') { 689 *I += J; 690 return Num ? Num : 1; 691 } 692 if (C < '0' || C > '9') 693 break; 694 Num = Num * 10 + C - '0'; 695 696 /* If FilenamePat[*I+J] is between '0' and '9', the next byte is guaranteed 697 * to be in-bound as the string is null terminated. */ 698 } 699 return 0; 700 } 701 702 /* Parses the pattern string \p FilenamePat and stores the result to 703 * lprofcurFilename structure. */ 704 static int parseFilenamePattern(const char *FilenamePat, 705 unsigned CopyFilenamePat) { 706 int NumPids = 0, NumHosts = 0, I; 707 char *PidChars = &lprofCurFilename.PidChars[0]; 708 char *Hostname = &lprofCurFilename.Hostname[0]; 709 int MergingEnabled = 0; 710 711 /* Clean up cached prefix and filename. */ 712 if (lprofCurFilename.ProfilePathPrefix) 713 free((void *)lprofCurFilename.ProfilePathPrefix); 714 715 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { 716 free((void *)lprofCurFilename.FilenamePat); 717 } 718 719 memset(&lprofCurFilename, 0, sizeof(lprofCurFilename)); 720 721 if (!CopyFilenamePat) 722 lprofCurFilename.FilenamePat = FilenamePat; 723 else { 724 lprofCurFilename.FilenamePat = strdup(FilenamePat); 725 lprofCurFilename.OwnsFilenamePat = 1; 726 } 727 /* Check the filename for "%p", which indicates a pid-substitution. */ 728 for (I = 0; FilenamePat[I]; ++I) 729 if (FilenamePat[I] == '%') { 730 if (FilenamePat[++I] == 'p') { 731 if (!NumPids++) { 732 if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) { 733 PROF_WARN("Unable to get pid for filename pattern %s. Using the " 734 "default name.", 735 FilenamePat); 736 return -1; 737 } 738 } 739 } else if (FilenamePat[I] == 'h') { 740 if (!NumHosts++) 741 if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) { 742 PROF_WARN("Unable to get hostname for filename pattern %s. Using " 743 "the default name.", 744 FilenamePat); 745 return -1; 746 } 747 } else if (FilenamePat[I] == 'c') { 748 if (__llvm_profile_is_continuous_mode_enabled()) { 749 PROF_WARN("%%c specifier can only be specified once in %s.\n", 750 FilenamePat); 751 return -1; 752 } 753 754 __llvm_profile_enable_continuous_mode(); 755 I++; /* advance to 'c' */ 756 } else { 757 unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I); 758 if (!MergePoolSize) 759 continue; 760 if (MergingEnabled) { 761 PROF_WARN("%%m specifier can only be specified once in %s.\n", 762 FilenamePat); 763 return -1; 764 } 765 MergingEnabled = 1; 766 lprofCurFilename.MergePoolSize = MergePoolSize; 767 } 768 } 769 770 lprofCurFilename.NumPids = NumPids; 771 lprofCurFilename.NumHosts = NumHosts; 772 return 0; 773 } 774 775 static void parseAndSetFilename(const char *FilenamePat, 776 ProfileNameSpecifier PNS, 777 unsigned CopyFilenamePat) { 778 779 const char *OldFilenamePat = lprofCurFilename.FilenamePat; 780 ProfileNameSpecifier OldPNS = lprofCurFilename.PNS; 781 782 /* The old profile name specifier takes precedence over the old one. */ 783 if (PNS < OldPNS) 784 return; 785 786 if (!FilenamePat) 787 FilenamePat = DefaultProfileName; 788 789 if (OldFilenamePat && !strcmp(OldFilenamePat, FilenamePat)) { 790 lprofCurFilename.PNS = PNS; 791 return; 792 } 793 794 /* When PNS >= OldPNS, the last one wins. */ 795 if (!FilenamePat || parseFilenamePattern(FilenamePat, CopyFilenamePat)) 796 resetFilenameToDefault(); 797 lprofCurFilename.PNS = PNS; 798 799 if (!OldFilenamePat) { 800 if (getenv("LLVM_PROFILE_VERBOSE")) 801 PROF_NOTE("Set profile file path to \"%s\" via %s.\n", 802 lprofCurFilename.FilenamePat, getPNSStr(PNS)); 803 } else { 804 if (getenv("LLVM_PROFILE_VERBOSE")) 805 PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n", 806 OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat, 807 getPNSStr(PNS)); 808 } 809 810 truncateCurrentFile(); 811 if (__llvm_profile_is_continuous_mode_enabled()) { 812 if (lprofRuntimeCounterRelocation()) 813 relocateCounters(); 814 else 815 initializeProfileForContinuousMode(); 816 } 817 } 818 819 /* Return buffer length that is required to store the current profile 820 * filename with PID and hostname substitutions. */ 821 /* The length to hold uint64_t followed by 3 digits pool id including '_' */ 822 #define SIGLEN 24 823 static int getCurFilenameLength() { 824 int Len; 825 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0]) 826 return 0; 827 828 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || 829 lprofCurFilename.MergePoolSize)) 830 return strlen(lprofCurFilename.FilenamePat); 831 832 Len = strlen(lprofCurFilename.FilenamePat) + 833 lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) + 834 lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2); 835 if (lprofCurFilename.MergePoolSize) 836 Len += SIGLEN; 837 return Len; 838 } 839 840 /* Return the pointer to the current profile file name (after substituting 841 * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer 842 * to store the resulting filename. If no substitution is needed, the 843 * current filename pattern string is directly returned, unless ForceUseBuf 844 * is enabled. */ 845 static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) { 846 int I, J, PidLength, HostNameLength, FilenamePatLength; 847 const char *FilenamePat = lprofCurFilename.FilenamePat; 848 849 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0]) 850 return 0; 851 852 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || 853 lprofCurFilename.MergePoolSize || 854 __llvm_profile_is_continuous_mode_enabled())) { 855 if (!ForceUseBuf) 856 return lprofCurFilename.FilenamePat; 857 858 FilenamePatLength = strlen(lprofCurFilename.FilenamePat); 859 memcpy(FilenameBuf, lprofCurFilename.FilenamePat, FilenamePatLength); 860 FilenameBuf[FilenamePatLength] = '\0'; 861 return FilenameBuf; 862 } 863 864 PidLength = strlen(lprofCurFilename.PidChars); 865 HostNameLength = strlen(lprofCurFilename.Hostname); 866 /* Construct the new filename. */ 867 for (I = 0, J = 0; FilenamePat[I]; ++I) 868 if (FilenamePat[I] == '%') { 869 if (FilenamePat[++I] == 'p') { 870 memcpy(FilenameBuf + J, lprofCurFilename.PidChars, PidLength); 871 J += PidLength; 872 } else if (FilenamePat[I] == 'h') { 873 memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength); 874 J += HostNameLength; 875 } else { 876 if (!getMergePoolSize(FilenamePat, &I)) 877 continue; 878 char LoadModuleSignature[SIGLEN + 1]; 879 int S; 880 int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize; 881 S = snprintf(LoadModuleSignature, SIGLEN + 1, "%" PRIu64 "_%d", 882 lprofGetLoadModuleSignature(), ProfilePoolId); 883 if (S == -1 || S > SIGLEN) 884 S = SIGLEN; 885 memcpy(FilenameBuf + J, LoadModuleSignature, S); 886 J += S; 887 } 888 /* Drop any unknown substitutions. */ 889 } else 890 FilenameBuf[J++] = FilenamePat[I]; 891 FilenameBuf[J] = 0; 892 893 return FilenameBuf; 894 } 895 896 /* Returns the pointer to the environment variable 897 * string. Returns null if the env var is not set. */ 898 static const char *getFilenamePatFromEnv(void) { 899 const char *Filename = getenv("LLVM_PROFILE_FILE"); 900 if (!Filename || !Filename[0]) 901 return 0; 902 return Filename; 903 } 904 905 COMPILER_RT_VISIBILITY 906 const char *__llvm_profile_get_path_prefix(void) { 907 int Length; 908 char *FilenameBuf, *Prefix; 909 const char *Filename, *PrefixEnd; 910 911 if (lprofCurFilename.ProfilePathPrefix) 912 return lprofCurFilename.ProfilePathPrefix; 913 914 Length = getCurFilenameLength(); 915 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); 916 Filename = getCurFilename(FilenameBuf, 0); 917 if (!Filename) 918 return "\0"; 919 920 PrefixEnd = lprofFindLastDirSeparator(Filename); 921 if (!PrefixEnd) 922 return "\0"; 923 924 Length = PrefixEnd - Filename + 1; 925 Prefix = (char *)malloc(Length + 1); 926 if (!Prefix) { 927 PROF_ERR("Failed to %s\n", "allocate memory."); 928 return "\0"; 929 } 930 memcpy(Prefix, Filename, Length); 931 Prefix[Length] = '\0'; 932 lprofCurFilename.ProfilePathPrefix = Prefix; 933 return Prefix; 934 } 935 936 COMPILER_RT_VISIBILITY 937 const char *__llvm_profile_get_filename(void) { 938 int Length; 939 char *FilenameBuf; 940 const char *Filename; 941 942 Length = getCurFilenameLength(); 943 FilenameBuf = (char *)malloc(Length + 1); 944 if (!FilenameBuf) { 945 PROF_ERR("Failed to %s\n", "allocate memory."); 946 return "\0"; 947 } 948 Filename = getCurFilename(FilenameBuf, 1); 949 if (!Filename) 950 return "\0"; 951 952 return FilenameBuf; 953 } 954 955 /* This API initializes the file handling, both user specified 956 * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE 957 * environment variable can override this default value. 958 */ 959 COMPILER_RT_VISIBILITY 960 void __llvm_profile_initialize_file(void) { 961 const char *EnvFilenamePat; 962 const char *SelectedPat = NULL; 963 ProfileNameSpecifier PNS = PNS_unknown; 964 int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0); 965 966 if (__llvm_profile_counter_bias != -1) 967 lprofSetRuntimeCounterRelocation(1); 968 969 EnvFilenamePat = getFilenamePatFromEnv(); 970 if (EnvFilenamePat) { 971 /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid 972 at the moment when __llvm_profile_write_file() gets executed. */ 973 parseAndSetFilename(EnvFilenamePat, PNS_environment, 1); 974 return; 975 } else if (hasCommandLineOverrider) { 976 SelectedPat = INSTR_PROF_PROFILE_NAME_VAR; 977 PNS = PNS_command_line; 978 } else { 979 SelectedPat = NULL; 980 PNS = PNS_default; 981 } 982 983 parseAndSetFilename(SelectedPat, PNS, 0); 984 } 985 986 /* This method is invoked by the runtime initialization hook 987 * InstrProfilingRuntime.o if it is linked in. 988 */ 989 COMPILER_RT_VISIBILITY 990 void __llvm_profile_initialize(void) { 991 __llvm_profile_initialize_file(); 992 if (!__llvm_profile_is_continuous_mode_enabled()) 993 __llvm_profile_register_write_file_atexit(); 994 } 995 996 /* This API is directly called by the user application code. It has the 997 * highest precedence compared with LLVM_PROFILE_FILE environment variable 998 * and command line option -fprofile-instr-generate=<profile_name>. 999 */ 1000 COMPILER_RT_VISIBILITY 1001 void __llvm_profile_set_filename(const char *FilenamePat) { 1002 if (__llvm_profile_is_continuous_mode_enabled()) 1003 return; 1004 parseAndSetFilename(FilenamePat, PNS_runtime_api, 1); 1005 } 1006 1007 /* The public API for writing profile data into the file with name 1008 * set by previous calls to __llvm_profile_set_filename or 1009 * __llvm_profile_override_default_filename or 1010 * __llvm_profile_initialize_file. */ 1011 COMPILER_RT_VISIBILITY 1012 int __llvm_profile_write_file(void) { 1013 int rc, Length; 1014 const char *Filename; 1015 char *FilenameBuf; 1016 int PDeathSig = 0; 1017 1018 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) { 1019 PROF_NOTE("Profile data not written to file: %s.\n", "already written"); 1020 return 0; 1021 } 1022 1023 Length = getCurFilenameLength(); 1024 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); 1025 Filename = getCurFilename(FilenameBuf, 0); 1026 1027 /* Check the filename. */ 1028 if (!Filename) { 1029 PROF_ERR("Failed to write file : %s\n", "Filename not set"); 1030 return -1; 1031 } 1032 1033 /* Check if there is llvm/runtime version mismatch. */ 1034 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { 1035 PROF_ERR("Runtime and instrumentation version mismatch : " 1036 "expected %d, but get %d\n", 1037 INSTR_PROF_RAW_VERSION, 1038 (int)GET_VERSION(__llvm_profile_get_version())); 1039 return -1; 1040 } 1041 1042 // Temporarily suspend getting SIGKILL when the parent exits. 1043 PDeathSig = lprofSuspendSigKill(); 1044 1045 /* Write profile data to the file. */ 1046 rc = writeFile(Filename); 1047 if (rc) 1048 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); 1049 1050 // Restore SIGKILL. 1051 if (PDeathSig == 1) 1052 lprofRestoreSigKill(); 1053 1054 return rc; 1055 } 1056 1057 COMPILER_RT_VISIBILITY 1058 int __llvm_profile_dump(void) { 1059 if (!doMerging()) 1060 PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering " 1061 " of previously dumped profile data : %s. Either use %%m " 1062 "in profile name or change profile name before dumping.\n", 1063 "online profile merging is not on"); 1064 int rc = __llvm_profile_write_file(); 1065 lprofSetProfileDumped(1); 1066 return rc; 1067 } 1068 1069 /* Order file data will be saved in a file with suffx .order. */ 1070 static const char *OrderFileSuffix = ".order"; 1071 1072 COMPILER_RT_VISIBILITY 1073 int __llvm_orderfile_write_file(void) { 1074 int rc, Length, LengthBeforeAppend, SuffixLength; 1075 const char *Filename; 1076 char *FilenameBuf; 1077 int PDeathSig = 0; 1078 1079 SuffixLength = strlen(OrderFileSuffix); 1080 Length = getCurFilenameLength() + SuffixLength; 1081 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); 1082 Filename = getCurFilename(FilenameBuf, 1); 1083 1084 /* Check the filename. */ 1085 if (!Filename) { 1086 PROF_ERR("Failed to write file : %s\n", "Filename not set"); 1087 return -1; 1088 } 1089 1090 /* Append order file suffix */ 1091 LengthBeforeAppend = strlen(Filename); 1092 memcpy(FilenameBuf + LengthBeforeAppend, OrderFileSuffix, SuffixLength); 1093 FilenameBuf[LengthBeforeAppend + SuffixLength] = '\0'; 1094 1095 /* Check if there is llvm/runtime version mismatch. */ 1096 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { 1097 PROF_ERR("Runtime and instrumentation version mismatch : " 1098 "expected %d, but get %d\n", 1099 INSTR_PROF_RAW_VERSION, 1100 (int)GET_VERSION(__llvm_profile_get_version())); 1101 return -1; 1102 } 1103 1104 // Temporarily suspend getting SIGKILL when the parent exits. 1105 PDeathSig = lprofSuspendSigKill(); 1106 1107 /* Write order data to the file. */ 1108 rc = writeOrderFile(Filename); 1109 if (rc) 1110 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); 1111 1112 // Restore SIGKILL. 1113 if (PDeathSig == 1) 1114 lprofRestoreSigKill(); 1115 1116 return rc; 1117 } 1118 1119 COMPILER_RT_VISIBILITY 1120 int __llvm_orderfile_dump(void) { 1121 int rc = __llvm_orderfile_write_file(); 1122 return rc; 1123 } 1124 1125 static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); } 1126 1127 COMPILER_RT_VISIBILITY 1128 int __llvm_profile_register_write_file_atexit(void) { 1129 static int HasBeenRegistered = 0; 1130 1131 if (HasBeenRegistered) 1132 return 0; 1133 1134 lprofSetupValueProfiler(); 1135 1136 HasBeenRegistered = 1; 1137 return atexit(writeFileWithoutReturn); 1138 } 1139 1140 #endif 1141