xref: /freebsd/contrib/llvm-project/compiler-rt/lib/xray/xray_interface.cpp (revision c66ec88fed842fbaad62c30d510644ceb7bd2d71)
1 //===-- xray_interface.cpp --------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of XRay, a dynamic runtime instrumentation system.
10 //
11 // Implementation of the API functions.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "xray_interface_internal.h"
16 
17 #include <cstdint>
18 #include <cstdio>
19 #include <errno.h>
20 #include <limits>
21 #include <string.h>
22 #include <sys/mman.h>
23 
24 #if SANITIZER_FUCHSIA
25 #include <zircon/process.h>
26 #include <zircon/sanitizer.h>
27 #include <zircon/status.h>
28 #include <zircon/syscalls.h>
29 #endif
30 
31 #include "sanitizer_common/sanitizer_addrhashmap.h"
32 #include "sanitizer_common/sanitizer_common.h"
33 
34 #include "xray_defs.h"
35 #include "xray_flags.h"
36 
37 extern __sanitizer::SpinMutex XRayInstrMapMutex;
38 extern __sanitizer::atomic_uint8_t XRayInitialized;
39 extern __xray::XRaySledMap XRayInstrMap;
40 
41 namespace __xray {
42 
43 #if defined(__x86_64__)
44 static const int16_t cSledLength = 12;
45 #elif defined(__aarch64__)
46 static const int16_t cSledLength = 32;
47 #elif defined(__arm__)
48 static const int16_t cSledLength = 28;
49 #elif SANITIZER_MIPS32
50 static const int16_t cSledLength = 48;
51 #elif SANITIZER_MIPS64
52 static const int16_t cSledLength = 64;
53 #elif defined(__powerpc64__)
54 static const int16_t cSledLength = 8;
55 #else
56 #error "Unsupported CPU Architecture"
57 #endif /* CPU architecture */
58 
59 // This is the function to call when we encounter the entry or exit sleds.
60 atomic_uintptr_t XRayPatchedFunction{0};
61 
62 // This is the function to call from the arg1-enabled sleds/trampolines.
63 atomic_uintptr_t XRayArgLogger{0};
64 
65 // This is the function to call when we encounter a custom event log call.
66 atomic_uintptr_t XRayPatchedCustomEvent{0};
67 
68 // This is the function to call when we encounter a typed event log call.
69 atomic_uintptr_t XRayPatchedTypedEvent{0};
70 
71 // This is the global status to determine whether we are currently
72 // patching/unpatching.
73 atomic_uint8_t XRayPatching{0};
74 
75 struct TypeDescription {
76   uint32_t type_id;
77   std::size_t description_string_length;
78 };
79 
80 using TypeDescriptorMapType = AddrHashMap<TypeDescription, 11>;
81 // An address map from immutable descriptors to type ids.
82 TypeDescriptorMapType TypeDescriptorAddressMap{};
83 
84 atomic_uint32_t TypeEventDescriptorCounter{0};
85 
86 // MProtectHelper is an RAII wrapper for calls to mprotect(...) that will
87 // undo any successful mprotect(...) changes. This is used to make a page
88 // writeable and executable, and upon destruction if it was successful in
89 // doing so returns the page into a read-only and executable page.
90 //
91 // This is only used specifically for runtime-patching of the XRay
92 // instrumentation points. This assumes that the executable pages are
93 // originally read-and-execute only.
94 class MProtectHelper {
95   void *PageAlignedAddr;
96   std::size_t MProtectLen;
97   bool MustCleanup;
98 
99 public:
100   explicit MProtectHelper(void *PageAlignedAddr,
101                           std::size_t MProtectLen,
102                           std::size_t PageSize) XRAY_NEVER_INSTRUMENT
103       : PageAlignedAddr(PageAlignedAddr),
104         MProtectLen(MProtectLen),
105         MustCleanup(false) {
106 #if SANITIZER_FUCHSIA
107     MProtectLen = RoundUpTo(MProtectLen, PageSize);
108 #endif
109   }
110 
111   int MakeWriteable() XRAY_NEVER_INSTRUMENT {
112 #if SANITIZER_FUCHSIA
113     auto R = __sanitizer_change_code_protection(
114         reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true);
115     if (R != ZX_OK) {
116       Report("XRay: cannot change code protection: %s\n",
117              _zx_status_get_string(R));
118       return -1;
119     }
120     MustCleanup = true;
121     return 0;
122 #else
123     auto R = mprotect(PageAlignedAddr, MProtectLen,
124                       PROT_READ | PROT_WRITE | PROT_EXEC);
125     if (R != -1)
126       MustCleanup = true;
127     return R;
128 #endif
129   }
130 
131   ~MProtectHelper() XRAY_NEVER_INSTRUMENT {
132     if (MustCleanup) {
133 #if SANITIZER_FUCHSIA
134       auto R = __sanitizer_change_code_protection(
135           reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false);
136       if (R != ZX_OK) {
137         Report("XRay: cannot change code protection: %s\n",
138                _zx_status_get_string(R));
139       }
140 #else
141       mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC);
142 #endif
143     }
144   }
145 };
146 
147 namespace {
148 
149 bool patchSled(const XRaySledEntry &Sled, bool Enable,
150                int32_t FuncId) XRAY_NEVER_INSTRUMENT {
151   bool Success = false;
152   switch (Sled.Kind) {
153   case XRayEntryType::ENTRY:
154     Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry);
155     break;
156   case XRayEntryType::EXIT:
157     Success = patchFunctionExit(Enable, FuncId, Sled);
158     break;
159   case XRayEntryType::TAIL:
160     Success = patchFunctionTailExit(Enable, FuncId, Sled);
161     break;
162   case XRayEntryType::LOG_ARGS_ENTRY:
163     Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry);
164     break;
165   case XRayEntryType::CUSTOM_EVENT:
166     Success = patchCustomEvent(Enable, FuncId, Sled);
167     break;
168   case XRayEntryType::TYPED_EVENT:
169     Success = patchTypedEvent(Enable, FuncId, Sled);
170     break;
171   default:
172     Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind));
173     return false;
174   }
175   return Success;
176 }
177 
178 const XRayFunctionSledIndex
179 findFunctionSleds(int32_t FuncId,
180                   const XRaySledMap &InstrMap) XRAY_NEVER_INSTRUMENT {
181   int32_t CurFn = 0;
182   uint64_t LastFnAddr = 0;
183   XRayFunctionSledIndex Index = {nullptr, nullptr};
184 
185   for (std::size_t I = 0; I < InstrMap.Entries && CurFn <= FuncId; I++) {
186     const auto &Sled = InstrMap.Sleds[I];
187     const auto Function = Sled.function();
188     if (Function != LastFnAddr) {
189       CurFn++;
190       LastFnAddr = Function;
191     }
192 
193     if (CurFn == FuncId) {
194       if (Index.Begin == nullptr)
195         Index.Begin = &Sled;
196       Index.End = &Sled;
197     }
198   }
199 
200   Index.End += 1;
201 
202   return Index;
203 }
204 
205 XRayPatchingStatus patchFunction(int32_t FuncId,
206                                  bool Enable) XRAY_NEVER_INSTRUMENT {
207   if (!atomic_load(&XRayInitialized,
208                                 memory_order_acquire))
209     return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
210 
211   uint8_t NotPatching = false;
212   if (!atomic_compare_exchange_strong(
213           &XRayPatching, &NotPatching, true, memory_order_acq_rel))
214     return XRayPatchingStatus::ONGOING; // Already patching.
215 
216   // Next, we look for the function index.
217   XRaySledMap InstrMap;
218   {
219     SpinMutexLock Guard(&XRayInstrMapMutex);
220     InstrMap = XRayInstrMap;
221   }
222 
223   // If we don't have an index, we can't patch individual functions.
224   if (InstrMap.Functions == 0)
225     return XRayPatchingStatus::NOT_INITIALIZED;
226 
227   // FuncId must be a positive number, less than the number of functions
228   // instrumented.
229   if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
230     Report("Invalid function id provided: %d\n", FuncId);
231     return XRayPatchingStatus::FAILED;
232   }
233 
234   // Now we patch ths sleds for this specific function.
235   auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
236                                        : findFunctionSleds(FuncId, InstrMap);
237   auto *f = SledRange.Begin;
238   auto *e = SledRange.End;
239   bool SucceedOnce = false;
240   while (f != e)
241     SucceedOnce |= patchSled(*f++, Enable, FuncId);
242 
243   atomic_store(&XRayPatching, false,
244                             memory_order_release);
245 
246   if (!SucceedOnce) {
247     Report("Failed patching any sled for function '%d'.", FuncId);
248     return XRayPatchingStatus::FAILED;
249   }
250 
251   return XRayPatchingStatus::SUCCESS;
252 }
253 
254 // controlPatching implements the common internals of the patching/unpatching
255 // implementation. |Enable| defines whether we're enabling or disabling the
256 // runtime XRay instrumentation.
257 XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
258   if (!atomic_load(&XRayInitialized,
259                                 memory_order_acquire))
260     return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
261 
262   uint8_t NotPatching = false;
263   if (!atomic_compare_exchange_strong(
264           &XRayPatching, &NotPatching, true, memory_order_acq_rel))
265     return XRayPatchingStatus::ONGOING; // Already patching.
266 
267   uint8_t PatchingSuccess = false;
268   auto XRayPatchingStatusResetter =
269       at_scope_exit([&PatchingSuccess] {
270         if (!PatchingSuccess)
271           atomic_store(&XRayPatching, false,
272                                     memory_order_release);
273       });
274 
275   XRaySledMap InstrMap;
276   {
277     SpinMutexLock Guard(&XRayInstrMapMutex);
278     InstrMap = XRayInstrMap;
279   }
280   if (InstrMap.Entries == 0)
281     return XRayPatchingStatus::NOT_INITIALIZED;
282 
283   uint32_t FuncId = 1;
284   uint64_t CurFun = 0;
285 
286   // First we want to find the bounds for which we have instrumentation points,
287   // and try to get as few calls to mprotect(...) as possible. We're assuming
288   // that all the sleds for the instrumentation map are contiguous as a single
289   // set of pages. When we do support dynamic shared object instrumentation,
290   // we'll need to do this for each set of page load offsets per DSO loaded. For
291   // now we're assuming we can mprotect the whole section of text between the
292   // minimum sled address and the maximum sled address (+ the largest sled
293   // size).
294   auto *MinSled = &InstrMap.Sleds[0];
295   auto *MaxSled = &InstrMap.Sleds[InstrMap.Entries - 1];
296   for (std::size_t I = 0; I < InstrMap.Entries; I++) {
297     const auto &Sled = InstrMap.Sleds[I];
298     if (Sled.address() < MinSled->address())
299       MinSled = &Sled;
300     if (Sled.address() > MaxSled->address())
301       MaxSled = &Sled;
302   }
303 
304   const size_t PageSize = flags()->xray_page_size_override > 0
305                               ? flags()->xray_page_size_override
306                               : GetPageSizeCached();
307   if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
308     Report("System page size is not a power of two: %lld\n", PageSize);
309     return XRayPatchingStatus::FAILED;
310   }
311 
312   void *PageAlignedAddr =
313       reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
314   size_t MProtectLen =
315       (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
316       cSledLength;
317   MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
318   if (Protector.MakeWriteable() == -1) {
319     Report("Failed mprotect: %d\n", errno);
320     return XRayPatchingStatus::FAILED;
321   }
322 
323   for (std::size_t I = 0; I < InstrMap.Entries; ++I) {
324     auto &Sled = InstrMap.Sleds[I];
325     auto F = Sled.function();
326     if (CurFun == 0)
327       CurFun = F;
328     if (F != CurFun) {
329       ++FuncId;
330       CurFun = F;
331     }
332     patchSled(Sled, Enable, FuncId);
333   }
334   atomic_store(&XRayPatching, false,
335                             memory_order_release);
336   PatchingSuccess = true;
337   return XRayPatchingStatus::SUCCESS;
338 }
339 
340 XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
341                                             bool Enable) XRAY_NEVER_INSTRUMENT {
342   XRaySledMap InstrMap;
343   {
344     SpinMutexLock Guard(&XRayInstrMapMutex);
345     InstrMap = XRayInstrMap;
346   }
347 
348   // FuncId must be a positive number, less than the number of functions
349   // instrumented.
350   if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
351     Report("Invalid function id provided: %d\n", FuncId);
352     return XRayPatchingStatus::FAILED;
353   }
354 
355   const size_t PageSize = flags()->xray_page_size_override > 0
356                               ? flags()->xray_page_size_override
357                               : GetPageSizeCached();
358   if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
359     Report("Provided page size is not a power of two: %lld\n", PageSize);
360     return XRayPatchingStatus::FAILED;
361   }
362 
363   // Here we compute the minumum sled and maximum sled associated with a
364   // particular function ID.
365   auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
366                                        : findFunctionSleds(FuncId, InstrMap);
367   auto *f = SledRange.Begin;
368   auto *e = SledRange.End;
369   auto *MinSled = f;
370   auto *MaxSled = (SledRange.End - 1);
371   while (f != e) {
372     if (f->address() < MinSled->address())
373       MinSled = f;
374     if (f->address() > MaxSled->address())
375       MaxSled = f;
376     ++f;
377   }
378 
379   void *PageAlignedAddr =
380       reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
381   size_t MProtectLen =
382       (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
383       cSledLength;
384   MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
385   if (Protector.MakeWriteable() == -1) {
386     Report("Failed mprotect: %d\n", errno);
387     return XRayPatchingStatus::FAILED;
388   }
389   return patchFunction(FuncId, Enable);
390 }
391 
392 } // namespace
393 
394 } // namespace __xray
395 
396 using namespace __xray;
397 
398 // The following functions are declared `extern "C" {...}` in the header, hence
399 // they're defined in the global namespace.
400 
401 int __xray_set_handler(void (*entry)(int32_t,
402                                      XRayEntryType)) XRAY_NEVER_INSTRUMENT {
403   if (atomic_load(&XRayInitialized,
404                                memory_order_acquire)) {
405 
406     atomic_store(&__xray::XRayPatchedFunction,
407                               reinterpret_cast<uintptr_t>(entry),
408                               memory_order_release);
409     return 1;
410   }
411   return 0;
412 }
413 
414 int __xray_set_customevent_handler(void (*entry)(void *, size_t))
415     XRAY_NEVER_INSTRUMENT {
416   if (atomic_load(&XRayInitialized,
417                                memory_order_acquire)) {
418     atomic_store(&__xray::XRayPatchedCustomEvent,
419                               reinterpret_cast<uintptr_t>(entry),
420                               memory_order_release);
421     return 1;
422   }
423   return 0;
424 }
425 
426 int __xray_set_typedevent_handler(void (*entry)(
427     uint16_t, const void *, size_t)) XRAY_NEVER_INSTRUMENT {
428   if (atomic_load(&XRayInitialized,
429                                memory_order_acquire)) {
430     atomic_store(&__xray::XRayPatchedTypedEvent,
431                               reinterpret_cast<uintptr_t>(entry),
432                               memory_order_release);
433     return 1;
434   }
435   return 0;
436 }
437 
438 int __xray_remove_handler() XRAY_NEVER_INSTRUMENT {
439   return __xray_set_handler(nullptr);
440 }
441 
442 int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT {
443   return __xray_set_customevent_handler(nullptr);
444 }
445 
446 int __xray_remove_typedevent_handler() XRAY_NEVER_INSTRUMENT {
447   return __xray_set_typedevent_handler(nullptr);
448 }
449 
450 uint16_t __xray_register_event_type(
451     const char *const event_type) XRAY_NEVER_INSTRUMENT {
452   TypeDescriptorMapType::Handle h(&TypeDescriptorAddressMap, (uptr)event_type);
453   if (h.created()) {
454     h->type_id = atomic_fetch_add(
455         &TypeEventDescriptorCounter, 1, memory_order_acq_rel);
456     h->description_string_length = strnlen(event_type, 1024);
457   }
458   return h->type_id;
459 }
460 
461 XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
462   return controlPatching(true);
463 }
464 
465 XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
466   return controlPatching(false);
467 }
468 
469 XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
470   return mprotectAndPatchFunction(FuncId, true);
471 }
472 
473 XRayPatchingStatus
474 __xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
475   return mprotectAndPatchFunction(FuncId, false);
476 }
477 
478 int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
479   if (!atomic_load(&XRayInitialized,
480                                 memory_order_acquire))
481     return 0;
482 
483   // A relaxed write might not be visible even if the current thread gets
484   // scheduled on a different CPU/NUMA node.  We need to wait for everyone to
485   // have this handler installed for consistency of collected data across CPUs.
486   atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry),
487                             memory_order_release);
488   return 1;
489 }
490 
491 int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); }
492 
493 uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
494   XRaySledMap InstrMap;
495   {
496     SpinMutexLock Guard(&XRayInstrMapMutex);
497     InstrMap = XRayInstrMap;
498   }
499 
500   if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions)
501     return 0;
502   const XRaySledEntry *Sled = InstrMap.SledsIndex
503                                   ? InstrMap.SledsIndex[FuncId - 1].Begin
504                                   : findFunctionSleds(FuncId, InstrMap).Begin;
505   return Sled->function()
506 // On PPC, function entries are always aligned to 16 bytes. The beginning of a
507 // sled might be a local entry, which is always +8 based on the global entry.
508 // Always return the global entry.
509 #ifdef __PPC__
510          & ~0xf
511 #endif
512       ;
513 }
514 
515 size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT {
516   SpinMutexLock Guard(&XRayInstrMapMutex);
517   return XRayInstrMap.Functions;
518 }
519