1 /*===- InstrProfilingUtil.c - Support library for PGO instrumentation -----===*\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8
9 #ifdef _WIN32
10 #include <direct.h>
11 #include <process.h>
12 #include <windows.h>
13 #include "WindowsMMap.h"
14 #else
15 #include <errno.h>
16 #include <fcntl.h>
17 #include <sys/file.h>
18 #include <sys/mman.h>
19 #include <sys/stat.h>
20 #include <sys/types.h>
21 #include <unistd.h>
22 #endif
23
24 #ifdef _AIX
25 #include <sys/statfs.h>
26 // <sys/vmount.h> depends on `uint` to be a typedef from <sys/types.h> to
27 // `uint_t`; however, <sys/types.h> does not always declare `uint`. We provide
28 // the typedef prior to including <sys/vmount.h> to work around this issue.
29 typedef uint_t uint;
30 #include <sys/vmount.h>
31 #endif
32
33 #ifdef COMPILER_RT_HAS_UNAME
34 #include <sys/utsname.h>
35 #endif
36
37 #include <stdlib.h>
38 #include <string.h>
39
40 #if defined(__linux__)
41 #include <signal.h>
42 #include <sys/prctl.h>
43 #endif
44
45 #if defined(__Fuchsia__)
46 #include <zircon/process.h>
47 #include <zircon/syscalls.h>
48 #endif
49
50 #if defined(__FreeBSD__)
51 #include <signal.h>
52 #include <sys/procctl.h>
53 #endif
54
55 #include "InstrProfiling.h"
56 #include "InstrProfilingUtil.h"
57
58 COMPILER_RT_VISIBILITY unsigned lprofDirMode = 0755;
59
60 COMPILER_RT_VISIBILITY
__llvm_profile_recursive_mkdir(char * path)61 void __llvm_profile_recursive_mkdir(char *path) {
62 int i;
63 int start = 1;
64
65 #if defined(__ANDROID__) && defined(__ANDROID_API__) && \
66 defined(__ANDROID_API_FUTURE__) && \
67 __ANDROID_API__ == __ANDROID_API_FUTURE__
68 // Avoid spammy selinux denial messages in Android by not attempting to
69 // create directories in GCOV_PREFIX. These denials occur when creating (or
70 // even attempting to stat()) top-level directories like "/data".
71 //
72 // Do so by ignoring ${GCOV_PREFIX} when invoking mkdir().
73 const char *gcov_prefix = getenv("GCOV_PREFIX");
74 if (gcov_prefix != NULL) {
75 const int gcov_prefix_len = strlen(gcov_prefix);
76 if (strncmp(path, gcov_prefix, gcov_prefix_len) == 0)
77 start = gcov_prefix_len;
78 }
79 #endif
80
81 for (i = start; path[i] != '\0'; ++i) {
82 char save = path[i];
83 if (!IS_DIR_SEPARATOR(path[i]))
84 continue;
85 path[i] = '\0';
86 #ifdef _WIN32
87 _mkdir(path);
88 #else
89 /* Some of these will fail, ignore it. */
90 mkdir(path, __llvm_profile_get_dir_mode());
91 #endif
92 path[i] = save;
93 }
94 }
95
96 COMPILER_RT_VISIBILITY
__llvm_profile_set_dir_mode(unsigned Mode)97 void __llvm_profile_set_dir_mode(unsigned Mode) { lprofDirMode = Mode; }
98
99 COMPILER_RT_VISIBILITY
__llvm_profile_get_dir_mode(void)100 unsigned __llvm_profile_get_dir_mode(void) { return lprofDirMode; }
101
102 #if COMPILER_RT_HAS_ATOMICS != 1
103 COMPILER_RT_VISIBILITY
lprofBoolCmpXchg(void ** Ptr,void * OldV,void * NewV)104 uint32_t lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
105 void *R = *Ptr;
106 if (R == OldV) {
107 *Ptr = NewV;
108 return 1;
109 }
110 return 0;
111 }
112 COMPILER_RT_VISIBILITY
lprofPtrFetchAdd(void ** Mem,long ByteIncr)113 void *lprofPtrFetchAdd(void **Mem, long ByteIncr) {
114 void *Old = *Mem;
115 *((char **)Mem) += ByteIncr;
116 return Old;
117 }
118
119 #endif
120
121 #ifdef _WIN32
lprofGetHostName(char * Name,int Len)122 COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) {
123 WCHAR Buffer[COMPILER_RT_MAX_HOSTLEN];
124 DWORD BufferSize = sizeof(Buffer);
125 BOOL Result =
126 GetComputerNameExW(ComputerNameDnsFullyQualified, Buffer, &BufferSize);
127 if (!Result)
128 return -1;
129 if (WideCharToMultiByte(CP_UTF8, 0, Buffer, -1, Name, Len, NULL, NULL) == 0)
130 return -1;
131 return 0;
132 }
133 #elif defined(COMPILER_RT_HAS_UNAME)
lprofGetHostName(char * Name,int Len)134 COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) {
135 struct utsname N;
136 int R = uname(&N);
137 if (R >= 0) {
138 strncpy(Name, N.nodename, Len);
139 return 0;
140 }
141 return R;
142 }
143 #endif
144
lprofLockFd(int fd)145 COMPILER_RT_VISIBILITY int lprofLockFd(int fd) {
146 #ifdef COMPILER_RT_HAS_FCNTL_LCK
147 struct flock s_flock;
148
149 s_flock.l_whence = SEEK_SET;
150 s_flock.l_start = 0;
151 s_flock.l_len = 0; /* Until EOF. */
152 s_flock.l_pid = getpid();
153 s_flock.l_type = F_WRLCK;
154
155 while (fcntl(fd, F_SETLKW, &s_flock) == -1) {
156 if (errno != EINTR) {
157 if (errno == ENOLCK) {
158 return -1;
159 }
160 break;
161 }
162 }
163 return 0;
164 #elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32)
165 // Windows doesn't have flock but WindowsMMap.h provides a shim
166 flock(fd, LOCK_EX);
167 return 0;
168 #else
169 return 0;
170 #endif
171 }
172
lprofUnlockFd(int fd)173 COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) {
174 #ifdef COMPILER_RT_HAS_FCNTL_LCK
175 struct flock s_flock;
176
177 s_flock.l_whence = SEEK_SET;
178 s_flock.l_start = 0;
179 s_flock.l_len = 0; /* Until EOF. */
180 s_flock.l_pid = getpid();
181 s_flock.l_type = F_UNLCK;
182
183 while (fcntl(fd, F_SETLKW, &s_flock) == -1) {
184 if (errno != EINTR) {
185 if (errno == ENOLCK) {
186 return -1;
187 }
188 break;
189 }
190 }
191 return 0;
192 #elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32)
193 // Windows doesn't have flock but WindowsMMap.h provides a shim
194 flock(fd, LOCK_UN);
195 return 0;
196 #else
197 return 0;
198 #endif
199 }
200
lprofLockFileHandle(FILE * F)201 COMPILER_RT_VISIBILITY int lprofLockFileHandle(FILE *F) {
202 int fd;
203 #if defined(_WIN32)
204 fd = _fileno(F);
205 #else
206 fd = fileno(F);
207 #endif
208 return lprofLockFd(fd);
209 }
210
lprofUnlockFileHandle(FILE * F)211 COMPILER_RT_VISIBILITY int lprofUnlockFileHandle(FILE *F) {
212 int fd;
213 #if defined(_WIN32)
214 fd = _fileno(F);
215 #else
216 fd = fileno(F);
217 #endif
218 return lprofUnlockFd(fd);
219 }
220
lprofOpenFileEx(const char * ProfileName)221 COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) {
222 FILE *f;
223 int fd;
224 #ifdef COMPILER_RT_HAS_FCNTL_LCK
225 fd = open(ProfileName, O_RDWR | O_CREAT, 0666);
226 if (fd < 0)
227 return NULL;
228
229 if (lprofLockFd(fd) != 0)
230 PROF_WARN("Data may be corrupted during profile merging : %s\n",
231 "Fail to obtain file lock due to system limit.");
232
233 f = fdopen(fd, "r+b");
234 #elif defined(_WIN32)
235 // FIXME: Use the wide variants to handle Unicode filenames.
236 HANDLE h = CreateFileA(ProfileName, GENERIC_READ | GENERIC_WRITE,
237 FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_ALWAYS,
238 FILE_ATTRIBUTE_NORMAL, 0);
239 if (h == INVALID_HANDLE_VALUE)
240 return NULL;
241
242 fd = _open_osfhandle((intptr_t)h, 0);
243 if (fd == -1) {
244 CloseHandle(h);
245 return NULL;
246 }
247
248 if (lprofLockFd(fd) != 0)
249 PROF_WARN("Data may be corrupted during profile merging : %s\n",
250 "Fail to obtain file lock due to system limit.");
251
252 f = _fdopen(fd, "r+b");
253 if (f == 0) {
254 CloseHandle(h);
255 return NULL;
256 }
257 #else
258 /* Worst case no locking applied. */
259 PROF_WARN("Concurrent file access is not supported : %s\n",
260 "lack file locking");
261 fd = open(ProfileName, O_RDWR | O_CREAT, 0666);
262 if (fd < 0)
263 return NULL;
264 f = fdopen(fd, "r+b");
265 #endif
266
267 return f;
268 }
269
270 #if defined(_AIX)
271 // Return 1 (true) if the file descriptor Fd represents a file that is on a
272 // local filesystem, otherwise return 0.
isLocalFilesystem(int Fd)273 static int isLocalFilesystem(int Fd) {
274 struct statfs Vfs;
275 if (fstatfs(Fd, &Vfs) != 0) {
276 PROF_ERR("%s: fstatfs(%d) failed: %s\n", __func__, Fd, strerror(errno));
277 return 0;
278 }
279
280 int Ret;
281 size_t BufSize = 2048u;
282 char *Buf;
283 int Tries = 3;
284 while (Tries--) {
285 Buf = malloc(BufSize);
286 // mntctl returns -1 if `Buf` is `NULL`.
287 Ret = mntctl(MCTL_QUERY, BufSize, Buf);
288 if (Ret != 0)
289 break;
290 BufSize = *(unsigned int *)Buf;
291 free(Buf);
292 }
293
294 if (Ret != -1) {
295 // Look for the correct vmount entry.
296 char *CurObjPtr = Buf;
297 while (Ret--) {
298 struct vmount *Vp = (struct vmount *)CurObjPtr;
299 _Static_assert(sizeof(Vfs.f_fsid) == sizeof(Vp->vmt_fsid),
300 "fsid length mismatch");
301 if (memcmp(&Vfs.f_fsid, &Vp->vmt_fsid, sizeof Vfs.f_fsid) == 0) {
302 int Answer = (Vp->vmt_flags & MNT_REMOTE) == 0;
303 free(Buf);
304 return Answer;
305 }
306 CurObjPtr += Vp->vmt_length;
307 }
308 }
309
310 free(Buf);
311 // There was an error in mntctl or vmount entry not found; "remote" is the
312 // conservative answer.
313 return 0;
314 }
315 #endif
316
isMmapSafe(int Fd)317 static int isMmapSafe(int Fd) {
318 if (getenv("LLVM_PROFILE_NO_MMAP")) // For testing purposes.
319 return 0;
320 #ifdef _AIX
321 return isLocalFilesystem(Fd);
322 #else
323 return 1;
324 #endif
325 }
326
lprofGetFileContentBuffer(FILE * F,uint64_t Length,ManagedMemory * Buf)327 COMPILER_RT_VISIBILITY void lprofGetFileContentBuffer(FILE *F, uint64_t Length,
328 ManagedMemory *Buf) {
329 Buf->Status = MS_INVALID;
330 if (isMmapSafe(fileno(F))) {
331 Buf->Addr =
332 mmap(NULL, Length, PROT_READ, MAP_SHARED | MAP_FILE, fileno(F), 0);
333 if (Buf->Addr == MAP_FAILED)
334 PROF_ERR("%s: mmap failed: %s\n", __func__, strerror(errno))
335 else
336 Buf->Status = MS_MMAP;
337 return;
338 }
339
340 if (getenv("LLVM_PROFILE_VERBOSE"))
341 PROF_NOTE("%s\n", "could not use mmap; using fread instead");
342
343 void *Buffer = malloc(Length);
344 if (!Buffer) {
345 PROF_ERR("%s: malloc failed: %s\n", __func__, strerror(errno));
346 return;
347 }
348 if (ftell(F) != 0) {
349 PROF_ERR("%s: expecting ftell to return zero\n", __func__);
350 free(Buffer);
351 return;
352 }
353
354 // Read the entire file into memory.
355 size_t BytesRead = fread(Buffer, 1, Length, F);
356 if (BytesRead != (size_t)Length) {
357 PROF_ERR("%s: fread failed%s\n", __func__,
358 feof(F) ? ": end of file reached" : "");
359 free(Buffer);
360 return;
361 }
362
363 // Reading was successful, record the result in the Buf parameter.
364 Buf->Addr = Buffer;
365 Buf->Status = MS_MALLOC;
366 }
367
368 COMPILER_RT_VISIBILITY
lprofReleaseBuffer(ManagedMemory * Buf,size_t Length)369 void lprofReleaseBuffer(ManagedMemory *Buf, size_t Length) {
370 switch (Buf->Status) {
371 case MS_MALLOC:
372 free(Buf->Addr);
373 break;
374 case MS_MMAP:
375 (void)munmap(Buf->Addr, Length);
376 break;
377 default:
378 PROF_ERR("%s: Buffer has invalid state: %d\n", __func__, Buf->Status);
379 break;
380 }
381 Buf->Addr = NULL;
382 Buf->Status = MS_INVALID;
383 }
384
lprofGetPathPrefix(int * PrefixStrip,size_t * PrefixLen)385 COMPILER_RT_VISIBILITY const char *lprofGetPathPrefix(int *PrefixStrip,
386 size_t *PrefixLen) {
387 const char *Prefix = getenv("GCOV_PREFIX");
388 const char *PrefixStripStr = getenv("GCOV_PREFIX_STRIP");
389
390 *PrefixLen = 0;
391 *PrefixStrip = 0;
392 if (Prefix == NULL || Prefix[0] == '\0')
393 return NULL;
394
395 if (PrefixStripStr) {
396 *PrefixStrip = atoi(PrefixStripStr);
397
398 /* Negative GCOV_PREFIX_STRIP values are ignored */
399 if (*PrefixStrip < 0)
400 *PrefixStrip = 0;
401 } else {
402 *PrefixStrip = 0;
403 }
404 *PrefixLen = strlen(Prefix);
405
406 return Prefix;
407 }
408
409 COMPILER_RT_VISIBILITY void
lprofApplyPathPrefix(char * Dest,const char * PathStr,const char * Prefix,size_t PrefixLen,int PrefixStrip)410 lprofApplyPathPrefix(char *Dest, const char *PathStr, const char *Prefix,
411 size_t PrefixLen, int PrefixStrip) {
412
413 const char *Ptr;
414 int Level;
415 const char *StrippedPathStr = PathStr;
416
417 for (Level = 0, Ptr = PathStr + 1; Level < PrefixStrip; ++Ptr) {
418 if (*Ptr == '\0')
419 break;
420
421 if (!IS_DIR_SEPARATOR(*Ptr))
422 continue;
423
424 StrippedPathStr = Ptr;
425 ++Level;
426 }
427
428 memcpy(Dest, Prefix, PrefixLen);
429
430 if (!IS_DIR_SEPARATOR(Prefix[PrefixLen - 1]))
431 Dest[PrefixLen++] = DIR_SEPARATOR;
432
433 memcpy(Dest + PrefixLen, StrippedPathStr, strlen(StrippedPathStr) + 1);
434 }
435
436 COMPILER_RT_VISIBILITY const char *
lprofFindFirstDirSeparator(const char * Path)437 lprofFindFirstDirSeparator(const char *Path) {
438 const char *Sep = strchr(Path, DIR_SEPARATOR);
439 #if defined(DIR_SEPARATOR_2)
440 const char *Sep2 = strchr(Path, DIR_SEPARATOR_2);
441 if (Sep2 && (!Sep || Sep2 < Sep))
442 Sep = Sep2;
443 #endif
444 return Sep;
445 }
446
lprofFindLastDirSeparator(const char * Path)447 COMPILER_RT_VISIBILITY const char *lprofFindLastDirSeparator(const char *Path) {
448 const char *Sep = strrchr(Path, DIR_SEPARATOR);
449 #if defined(DIR_SEPARATOR_2)
450 const char *Sep2 = strrchr(Path, DIR_SEPARATOR_2);
451 if (Sep2 && (!Sep || Sep2 > Sep))
452 Sep = Sep2;
453 #endif
454 return Sep;
455 }
456
lprofSuspendSigKill(void)457 COMPILER_RT_VISIBILITY int lprofSuspendSigKill(void) {
458 #if defined(__linux__)
459 int PDeachSig = 0;
460 /* Temporarily suspend getting SIGKILL upon exit of the parent process. */
461 if (prctl(PR_GET_PDEATHSIG, &PDeachSig) == 0 && PDeachSig == SIGKILL)
462 prctl(PR_SET_PDEATHSIG, 0);
463 return (PDeachSig == SIGKILL);
464 #elif defined(__FreeBSD__)
465 int PDeachSig = 0, PDisableSig = 0;
466 if (procctl(P_PID, 0, PROC_PDEATHSIG_STATUS, &PDeachSig) == 0 &&
467 PDeachSig == SIGKILL)
468 procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PDisableSig);
469 return (PDeachSig == SIGKILL);
470 #else
471 return 0;
472 #endif
473 }
474
lprofRestoreSigKill(void)475 COMPILER_RT_VISIBILITY void lprofRestoreSigKill(void) {
476 #if defined(__linux__)
477 prctl(PR_SET_PDEATHSIG, SIGKILL);
478 #elif defined(__FreeBSD__)
479 int PEnableSig = SIGKILL;
480 procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PEnableSig);
481 #endif
482 }
483
lprofReleaseMemoryPagesToOS(uintptr_t Begin,uintptr_t End)484 COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin,
485 uintptr_t End) {
486 #if defined(__ve__) || defined(__wasi__)
487 // VE and WASI doesn't support madvise.
488 return 0;
489 #else
490 size_t PageSize = getpagesize();
491 uintptr_t BeginAligned = lprofRoundUpTo((uintptr_t)Begin, PageSize);
492 uintptr_t EndAligned = lprofRoundDownTo((uintptr_t)End, PageSize);
493 if (BeginAligned < EndAligned) {
494 #if defined(__Fuchsia__)
495 return _zx_vmar_op_range(_zx_vmar_root_self(), ZX_VMAR_OP_DECOMMIT,
496 (zx_vaddr_t)BeginAligned,
497 EndAligned - BeginAligned, NULL, 0);
498 #else
499 return madvise((void *)BeginAligned, EndAligned - BeginAligned,
500 MADV_DONTNEED);
501 #endif
502 }
503 return 0;
504 #endif
505 }
506
507 #ifdef _AIX
508 typedef struct fn_node {
509 AtExit_Fn_ptr func;
510 struct fn_node *next;
511 } fn_node;
512 typedef struct {
513 fn_node *top;
514 } fn_stack;
515
516 static void fn_stack_push(fn_stack *, AtExit_Fn_ptr);
517 static AtExit_Fn_ptr fn_stack_pop(fn_stack *);
518 /* return 1 if stack is empty, 0 otherwise */
519 static int fn_stack_is_empty(fn_stack *);
520
521 static fn_stack AtExit_stack = {0};
522 #define ATEXIT_STACK (&AtExit_stack)
523
524 /* On AIX, atexit() functions registered by a shared library do not get called
525 * when the library is dlclose'd, causing a crash when they are eventually
526 * called at main program exit. However, a destructor does get called. So we
527 * collect all atexit functions registered by profile-rt and at program
528 * termination time (normal exit, shared library unload, or dlclose) we walk
529 * the list and execute any function that is still sitting in the atexit system
530 * queue.
531 */
cleanup()532 __attribute__((__destructor__)) static void cleanup() {
533 while (!fn_stack_is_empty(ATEXIT_STACK)) {
534 AtExit_Fn_ptr func = fn_stack_pop(ATEXIT_STACK);
535 if (func && unatexit(func) == 0)
536 func();
537 }
538 }
539
fn_stack_push(fn_stack * st,AtExit_Fn_ptr func)540 static void fn_stack_push(fn_stack *st, AtExit_Fn_ptr func) {
541 fn_node *old_top, *n = (fn_node *)malloc(sizeof(fn_node));
542 n->func = func;
543
544 while (1) {
545 old_top = st->top;
546 n->next = old_top;
547 if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, n))
548 return;
549 }
550 }
fn_stack_pop(fn_stack * st)551 static AtExit_Fn_ptr fn_stack_pop(fn_stack *st) {
552 fn_node *old_top, *new_top;
553 while (1) {
554 old_top = st->top;
555 if (old_top == 0)
556 return 0;
557 new_top = old_top->next;
558 if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, new_top)) {
559 AtExit_Fn_ptr func = old_top->func;
560 free(old_top);
561 return func;
562 }
563 }
564 }
565
fn_stack_is_empty(fn_stack * st)566 static int fn_stack_is_empty(fn_stack *st) { return st->top == 0; }
567 #endif
568
lprofAtExit(AtExit_Fn_ptr func)569 COMPILER_RT_VISIBILITY int lprofAtExit(AtExit_Fn_ptr func) {
570 #ifdef _AIX
571 fn_stack_push(ATEXIT_STACK, func);
572 #endif
573 return atexit(func);
574 }
575