1 /*===- InstrProfilingUtil.c - Support library for PGO instrumentation -----===*\ 2 |* 3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 |* See https://llvm.org/LICENSE.txt for license information. 5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 |* 7 \*===----------------------------------------------------------------------===*/ 8 9 #ifdef _WIN32 10 #include <direct.h> 11 #include <process.h> 12 #include <windows.h> 13 #include "WindowsMMap.h" 14 #else 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <sys/file.h> 18 #include <sys/mman.h> 19 #include <sys/stat.h> 20 #include <sys/types.h> 21 #include <unistd.h> 22 #endif 23 24 #ifdef _AIX 25 #include <sys/statfs.h> 26 // <sys/vmount.h> depends on `uint` to be a typedef from <sys/types.h> to 27 // `uint_t`; however, <sys/types.h> does not always declare `uint`. We provide 28 // the typedef prior to including <sys/vmount.h> to work around this issue. 29 typedef uint_t uint; 30 #include <sys/vmount.h> 31 #endif 32 33 #ifdef COMPILER_RT_HAS_UNAME 34 #include <sys/utsname.h> 35 #endif 36 37 #include <stdlib.h> 38 #include <string.h> 39 40 #if defined(__linux__) 41 #include <signal.h> 42 #include <sys/prctl.h> 43 #endif 44 45 #if defined(__Fuchsia__) 46 #include <zircon/process.h> 47 #include <zircon/syscalls.h> 48 #endif 49 50 #if defined(__FreeBSD__) 51 #include <signal.h> 52 #include <sys/procctl.h> 53 #endif 54 55 #include "InstrProfiling.h" 56 #include "InstrProfilingUtil.h" 57 58 COMPILER_RT_VISIBILITY unsigned lprofDirMode = 0755; 59 60 COMPILER_RT_VISIBILITY 61 void __llvm_profile_recursive_mkdir(char *path) { 62 int i; 63 int start = 1; 64 65 #if defined(__ANDROID__) && defined(__ANDROID_API__) && \ 66 defined(__ANDROID_API_FUTURE__) && \ 67 __ANDROID_API__ == __ANDROID_API_FUTURE__ 68 // Avoid spammy selinux denial messages in Android by not attempting to 69 // create directories in GCOV_PREFIX. These denials occur when creating (or 70 // even attempting to stat()) top-level directories like "/data". 71 // 72 // Do so by ignoring ${GCOV_PREFIX} when invoking mkdir(). 73 const char *gcov_prefix = getenv("GCOV_PREFIX"); 74 if (gcov_prefix != NULL) { 75 const int gcov_prefix_len = strlen(gcov_prefix); 76 if (strncmp(path, gcov_prefix, gcov_prefix_len) == 0) 77 start = gcov_prefix_len; 78 } 79 #endif 80 81 for (i = start; path[i] != '\0'; ++i) { 82 char save = path[i]; 83 if (!IS_DIR_SEPARATOR(path[i])) 84 continue; 85 path[i] = '\0'; 86 #ifdef _WIN32 87 _mkdir(path); 88 #else 89 /* Some of these will fail, ignore it. */ 90 mkdir(path, __llvm_profile_get_dir_mode()); 91 #endif 92 path[i] = save; 93 } 94 } 95 96 COMPILER_RT_VISIBILITY 97 void __llvm_profile_set_dir_mode(unsigned Mode) { lprofDirMode = Mode; } 98 99 COMPILER_RT_VISIBILITY 100 unsigned __llvm_profile_get_dir_mode(void) { return lprofDirMode; } 101 102 #if COMPILER_RT_HAS_ATOMICS != 1 103 COMPILER_RT_VISIBILITY 104 uint32_t lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV) { 105 void *R = *Ptr; 106 if (R == OldV) { 107 *Ptr = NewV; 108 return 1; 109 } 110 return 0; 111 } 112 COMPILER_RT_VISIBILITY 113 void *lprofPtrFetchAdd(void **Mem, long ByteIncr) { 114 void *Old = *Mem; 115 *((char **)Mem) += ByteIncr; 116 return Old; 117 } 118 119 #endif 120 121 #ifdef _WIN32 122 COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { 123 WCHAR Buffer[COMPILER_RT_MAX_HOSTLEN]; 124 DWORD BufferSize = sizeof(Buffer); 125 BOOL Result = 126 GetComputerNameExW(ComputerNameDnsFullyQualified, Buffer, &BufferSize); 127 if (!Result) 128 return -1; 129 if (WideCharToMultiByte(CP_UTF8, 0, Buffer, -1, Name, Len, NULL, NULL) == 0) 130 return -1; 131 return 0; 132 } 133 #elif defined(COMPILER_RT_HAS_UNAME) 134 COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { 135 struct utsname N; 136 int R = uname(&N); 137 if (R >= 0) { 138 strncpy(Name, N.nodename, Len); 139 return 0; 140 } 141 return R; 142 } 143 #endif 144 145 COMPILER_RT_VISIBILITY int lprofLockFd(int fd) { 146 #ifdef COMPILER_RT_HAS_FCNTL_LCK 147 struct flock s_flock; 148 149 s_flock.l_whence = SEEK_SET; 150 s_flock.l_start = 0; 151 s_flock.l_len = 0; /* Until EOF. */ 152 s_flock.l_pid = getpid(); 153 s_flock.l_type = F_WRLCK; 154 155 while (fcntl(fd, F_SETLKW, &s_flock) == -1) { 156 if (errno != EINTR) { 157 if (errno == ENOLCK) { 158 return -1; 159 } 160 break; 161 } 162 } 163 return 0; 164 #elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32) 165 // Windows doesn't have flock but WindowsMMap.h provides a shim 166 flock(fd, LOCK_EX); 167 return 0; 168 #else 169 return 0; 170 #endif 171 } 172 173 COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) { 174 #ifdef COMPILER_RT_HAS_FCNTL_LCK 175 struct flock s_flock; 176 177 s_flock.l_whence = SEEK_SET; 178 s_flock.l_start = 0; 179 s_flock.l_len = 0; /* Until EOF. */ 180 s_flock.l_pid = getpid(); 181 s_flock.l_type = F_UNLCK; 182 183 while (fcntl(fd, F_SETLKW, &s_flock) == -1) { 184 if (errno != EINTR) { 185 if (errno == ENOLCK) { 186 return -1; 187 } 188 break; 189 } 190 } 191 return 0; 192 #elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32) 193 // Windows doesn't have flock but WindowsMMap.h provides a shim 194 flock(fd, LOCK_UN); 195 return 0; 196 #else 197 return 0; 198 #endif 199 } 200 201 COMPILER_RT_VISIBILITY int lprofLockFileHandle(FILE *F) { 202 int fd; 203 #if defined(_WIN32) 204 fd = _fileno(F); 205 #else 206 fd = fileno(F); 207 #endif 208 return lprofLockFd(fd); 209 } 210 211 COMPILER_RT_VISIBILITY int lprofUnlockFileHandle(FILE *F) { 212 int fd; 213 #if defined(_WIN32) 214 fd = _fileno(F); 215 #else 216 fd = fileno(F); 217 #endif 218 return lprofUnlockFd(fd); 219 } 220 221 COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) { 222 FILE *f; 223 int fd; 224 #ifdef COMPILER_RT_HAS_FCNTL_LCK 225 fd = open(ProfileName, O_RDWR | O_CREAT, 0666); 226 if (fd < 0) 227 return NULL; 228 229 if (lprofLockFd(fd) != 0) 230 PROF_WARN("Data may be corrupted during profile merging : %s\n", 231 "Fail to obtain file lock due to system limit."); 232 233 f = fdopen(fd, "r+b"); 234 #elif defined(_WIN32) 235 // FIXME: Use the wide variants to handle Unicode filenames. 236 HANDLE h = CreateFileA(ProfileName, GENERIC_READ | GENERIC_WRITE, 237 FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_ALWAYS, 238 FILE_ATTRIBUTE_NORMAL, 0); 239 if (h == INVALID_HANDLE_VALUE) 240 return NULL; 241 242 fd = _open_osfhandle((intptr_t)h, 0); 243 if (fd == -1) { 244 CloseHandle(h); 245 return NULL; 246 } 247 248 if (lprofLockFd(fd) != 0) 249 PROF_WARN("Data may be corrupted during profile merging : %s\n", 250 "Fail to obtain file lock due to system limit."); 251 252 f = _fdopen(fd, "r+b"); 253 if (f == 0) { 254 CloseHandle(h); 255 return NULL; 256 } 257 #else 258 /* Worst case no locking applied. */ 259 PROF_WARN("Concurrent file access is not supported : %s\n", 260 "lack file locking"); 261 fd = open(ProfileName, O_RDWR | O_CREAT, 0666); 262 if (fd < 0) 263 return NULL; 264 f = fdopen(fd, "r+b"); 265 #endif 266 267 return f; 268 } 269 270 #if defined(_AIX) 271 // Return 1 (true) if the file descriptor Fd represents a file that is on a 272 // local filesystem, otherwise return 0. 273 static int isLocalFilesystem(int Fd) { 274 struct statfs Vfs; 275 if (fstatfs(Fd, &Vfs) != 0) { 276 PROF_ERR("%s: fstatfs(%d) failed: %s\n", __func__, Fd, strerror(errno)); 277 return 0; 278 } 279 280 int Ret; 281 size_t BufSize = 2048u; 282 char *Buf; 283 int Tries = 3; 284 while (Tries--) { 285 Buf = malloc(BufSize); 286 // mntctl returns -1 if `Buf` is `NULL`. 287 Ret = mntctl(MCTL_QUERY, BufSize, Buf); 288 if (Ret != 0) 289 break; 290 BufSize = *(unsigned int *)Buf; 291 free(Buf); 292 } 293 294 if (Ret != -1) { 295 // Look for the correct vmount entry. 296 char *CurObjPtr = Buf; 297 while (Ret--) { 298 struct vmount *Vp = (struct vmount *)CurObjPtr; 299 _Static_assert(sizeof(Vfs.f_fsid) == sizeof(Vp->vmt_fsid), 300 "fsid length mismatch"); 301 if (memcmp(&Vfs.f_fsid, &Vp->vmt_fsid, sizeof Vfs.f_fsid) == 0) { 302 int Answer = (Vp->vmt_flags & MNT_REMOTE) == 0; 303 free(Buf); 304 return Answer; 305 } 306 CurObjPtr += Vp->vmt_length; 307 } 308 } 309 310 free(Buf); 311 // There was an error in mntctl or vmount entry not found; "remote" is the 312 // conservative answer. 313 return 0; 314 } 315 #endif 316 317 static int isMmapSafe(int Fd) { 318 if (getenv("LLVM_PROFILE_NO_MMAP")) // For testing purposes. 319 return 0; 320 #ifdef _AIX 321 return isLocalFilesystem(Fd); 322 #else 323 return 1; 324 #endif 325 } 326 327 COMPILER_RT_VISIBILITY void lprofGetFileContentBuffer(FILE *F, uint64_t Length, 328 ManagedMemory *Buf) { 329 Buf->Status = MS_INVALID; 330 if (isMmapSafe(fileno(F))) { 331 Buf->Addr = 332 mmap(NULL, Length, PROT_READ, MAP_SHARED | MAP_FILE, fileno(F), 0); 333 if (Buf->Addr == MAP_FAILED) 334 PROF_ERR("%s: mmap failed: %s\n", __func__, strerror(errno)) 335 else 336 Buf->Status = MS_MMAP; 337 return; 338 } 339 340 if (getenv("LLVM_PROFILE_VERBOSE")) 341 PROF_NOTE("%s\n", "could not use mmap; using fread instead"); 342 343 void *Buffer = malloc(Length); 344 if (!Buffer) { 345 PROF_ERR("%s: malloc failed: %s\n", __func__, strerror(errno)); 346 return; 347 } 348 if (ftell(F) != 0) { 349 PROF_ERR("%s: expecting ftell to return zero\n", __func__); 350 free(Buffer); 351 return; 352 } 353 354 // Read the entire file into memory. 355 size_t BytesRead = fread(Buffer, 1, Length, F); 356 if (BytesRead != (size_t)Length) { 357 PROF_ERR("%s: fread failed%s\n", __func__, 358 feof(F) ? ": end of file reached" : ""); 359 free(Buffer); 360 return; 361 } 362 363 // Reading was successful, record the result in the Buf parameter. 364 Buf->Addr = Buffer; 365 Buf->Status = MS_MALLOC; 366 } 367 368 COMPILER_RT_VISIBILITY 369 void lprofReleaseBuffer(ManagedMemory *Buf, size_t Length) { 370 switch (Buf->Status) { 371 case MS_MALLOC: 372 free(Buf->Addr); 373 break; 374 case MS_MMAP: 375 (void)munmap(Buf->Addr, Length); 376 break; 377 default: 378 PROF_ERR("%s: Buffer has invalid state: %d\n", __func__, Buf->Status); 379 break; 380 } 381 Buf->Addr = NULL; 382 Buf->Status = MS_INVALID; 383 } 384 385 COMPILER_RT_VISIBILITY const char *lprofGetPathPrefix(int *PrefixStrip, 386 size_t *PrefixLen) { 387 const char *Prefix = getenv("GCOV_PREFIX"); 388 const char *PrefixStripStr = getenv("GCOV_PREFIX_STRIP"); 389 390 *PrefixLen = 0; 391 *PrefixStrip = 0; 392 if (Prefix == NULL || Prefix[0] == '\0') 393 return NULL; 394 395 if (PrefixStripStr) { 396 *PrefixStrip = atoi(PrefixStripStr); 397 398 /* Negative GCOV_PREFIX_STRIP values are ignored */ 399 if (*PrefixStrip < 0) 400 *PrefixStrip = 0; 401 } else { 402 *PrefixStrip = 0; 403 } 404 *PrefixLen = strlen(Prefix); 405 406 return Prefix; 407 } 408 409 COMPILER_RT_VISIBILITY void 410 lprofApplyPathPrefix(char *Dest, const char *PathStr, const char *Prefix, 411 size_t PrefixLen, int PrefixStrip) { 412 413 const char *Ptr; 414 int Level; 415 const char *StrippedPathStr = PathStr; 416 417 for (Level = 0, Ptr = PathStr + 1; Level < PrefixStrip; ++Ptr) { 418 if (*Ptr == '\0') 419 break; 420 421 if (!IS_DIR_SEPARATOR(*Ptr)) 422 continue; 423 424 StrippedPathStr = Ptr; 425 ++Level; 426 } 427 428 memcpy(Dest, Prefix, PrefixLen); 429 430 if (!IS_DIR_SEPARATOR(Prefix[PrefixLen - 1])) 431 Dest[PrefixLen++] = DIR_SEPARATOR; 432 433 memcpy(Dest + PrefixLen, StrippedPathStr, strlen(StrippedPathStr) + 1); 434 } 435 436 COMPILER_RT_VISIBILITY const char * 437 lprofFindFirstDirSeparator(const char *Path) { 438 const char *Sep = strchr(Path, DIR_SEPARATOR); 439 #if defined(DIR_SEPARATOR_2) 440 const char *Sep2 = strchr(Path, DIR_SEPARATOR_2); 441 if (Sep2 && (!Sep || Sep2 < Sep)) 442 Sep = Sep2; 443 #endif 444 return Sep; 445 } 446 447 COMPILER_RT_VISIBILITY const char *lprofFindLastDirSeparator(const char *Path) { 448 const char *Sep = strrchr(Path, DIR_SEPARATOR); 449 #if defined(DIR_SEPARATOR_2) 450 const char *Sep2 = strrchr(Path, DIR_SEPARATOR_2); 451 if (Sep2 && (!Sep || Sep2 > Sep)) 452 Sep = Sep2; 453 #endif 454 return Sep; 455 } 456 457 COMPILER_RT_VISIBILITY int lprofSuspendSigKill(void) { 458 #if defined(__linux__) 459 int PDeachSig = 0; 460 /* Temporarily suspend getting SIGKILL upon exit of the parent process. */ 461 if (prctl(PR_GET_PDEATHSIG, &PDeachSig) == 0 && PDeachSig == SIGKILL) 462 prctl(PR_SET_PDEATHSIG, 0); 463 return (PDeachSig == SIGKILL); 464 #elif defined(__FreeBSD__) 465 int PDeachSig = 0, PDisableSig = 0; 466 if (procctl(P_PID, 0, PROC_PDEATHSIG_STATUS, &PDeachSig) == 0 && 467 PDeachSig == SIGKILL) 468 procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PDisableSig); 469 return (PDeachSig == SIGKILL); 470 #else 471 return 0; 472 #endif 473 } 474 475 COMPILER_RT_VISIBILITY void lprofRestoreSigKill(void) { 476 #if defined(__linux__) 477 prctl(PR_SET_PDEATHSIG, SIGKILL); 478 #elif defined(__FreeBSD__) 479 int PEnableSig = SIGKILL; 480 procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PEnableSig); 481 #endif 482 } 483 484 COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin, 485 uintptr_t End) { 486 #if defined(__ve__) || defined(__wasi__) 487 // VE and WASI doesn't support madvise. 488 return 0; 489 #else 490 size_t PageSize = getpagesize(); 491 uintptr_t BeginAligned = lprofRoundUpTo((uintptr_t)Begin, PageSize); 492 uintptr_t EndAligned = lprofRoundDownTo((uintptr_t)End, PageSize); 493 if (BeginAligned < EndAligned) { 494 #if defined(__Fuchsia__) 495 return _zx_vmar_op_range(_zx_vmar_root_self(), ZX_VMAR_OP_DECOMMIT, 496 (zx_vaddr_t)BeginAligned, 497 EndAligned - BeginAligned, NULL, 0); 498 #else 499 return madvise((void *)BeginAligned, EndAligned - BeginAligned, 500 MADV_DONTNEED); 501 #endif 502 } 503 return 0; 504 #endif 505 } 506 507 #ifdef _AIX 508 typedef struct fn_node { 509 AtExit_Fn_ptr func; 510 struct fn_node *next; 511 } fn_node; 512 typedef struct { 513 fn_node *top; 514 } fn_stack; 515 516 static void fn_stack_push(fn_stack *, AtExit_Fn_ptr); 517 static AtExit_Fn_ptr fn_stack_pop(fn_stack *); 518 /* return 1 if stack is empty, 0 otherwise */ 519 static int fn_stack_is_empty(fn_stack *); 520 521 static fn_stack AtExit_stack = {0}; 522 #define ATEXIT_STACK (&AtExit_stack) 523 524 /* On AIX, atexit() functions registered by a shared library do not get called 525 * when the library is dlclose'd, causing a crash when they are eventually 526 * called at main program exit. However, a destructor does get called. So we 527 * collect all atexit functions registered by profile-rt and at program 528 * termination time (normal exit, shared library unload, or dlclose) we walk 529 * the list and execute any function that is still sitting in the atexit system 530 * queue. 531 */ 532 __attribute__((__destructor__)) static void cleanup() { 533 while (!fn_stack_is_empty(ATEXIT_STACK)) { 534 AtExit_Fn_ptr func = fn_stack_pop(ATEXIT_STACK); 535 if (func && unatexit(func) == 0) 536 func(); 537 } 538 } 539 540 static void fn_stack_push(fn_stack *st, AtExit_Fn_ptr func) { 541 fn_node *old_top, *n = (fn_node *)malloc(sizeof(fn_node)); 542 n->func = func; 543 544 while (1) { 545 old_top = st->top; 546 n->next = old_top; 547 if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, n)) 548 return; 549 } 550 } 551 static AtExit_Fn_ptr fn_stack_pop(fn_stack *st) { 552 fn_node *old_top, *new_top; 553 while (1) { 554 old_top = st->top; 555 if (old_top == 0) 556 return 0; 557 new_top = old_top->next; 558 if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, new_top)) { 559 AtExit_Fn_ptr func = old_top->func; 560 free(old_top); 561 return func; 562 } 563 } 564 } 565 566 static int fn_stack_is_empty(fn_stack *st) { return st->top == 0; } 567 #endif 568 569 COMPILER_RT_VISIBILITY int lprofAtExit(AtExit_Fn_ptr func) { 570 #ifdef _AIX 571 fn_stack_push(ATEXIT_STACK, func); 572 #endif 573 return atexit(func); 574 } 575