1 //===-- xray_interface.cpp --------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of XRay, a dynamic runtime instrumentation system.
10 //
11 // Implementation of the API functions.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "xray_interface_internal.h"
16
17 #include <cinttypes>
18 #include <cstdio>
19 #include <errno.h>
20 #include <limits>
21 #include <string.h>
22 #include <sys/mman.h>
23
24 #if SANITIZER_FUCHSIA
25 #include <zircon/process.h>
26 #include <zircon/sanitizer.h>
27 #include <zircon/status.h>
28 #include <zircon/syscalls.h>
29 #endif
30
31 #include "sanitizer_common/sanitizer_addrhashmap.h"
32 #include "sanitizer_common/sanitizer_common.h"
33
34 #include "xray_defs.h"
35 #include "xray_flags.h"
36
37 extern __sanitizer::SpinMutex XRayInstrMapMutex;
38 extern __sanitizer::atomic_uint8_t XRayInitialized;
39 extern __xray::XRaySledMap XRayInstrMap;
40
41 namespace __xray {
42
43 #if defined(__x86_64__)
44 static const int16_t cSledLength = 12;
45 #elif defined(__aarch64__)
46 static const int16_t cSledLength = 32;
47 #elif defined(__arm__)
48 static const int16_t cSledLength = 28;
49 #elif SANITIZER_LOONGARCH64
50 static const int16_t cSledLength = 48;
51 #elif SANITIZER_MIPS32
52 static const int16_t cSledLength = 48;
53 #elif SANITIZER_MIPS64
54 static const int16_t cSledLength = 64;
55 #elif defined(__powerpc64__)
56 static const int16_t cSledLength = 8;
57 #elif defined(__hexagon__)
58 static const int16_t cSledLength = 20;
59 #else
60 #error "Unsupported CPU Architecture"
61 #endif /* CPU architecture */
62
63 // This is the function to call when we encounter the entry or exit sleds.
64 atomic_uintptr_t XRayPatchedFunction{0};
65
66 // This is the function to call from the arg1-enabled sleds/trampolines.
67 atomic_uintptr_t XRayArgLogger{0};
68
69 // This is the function to call when we encounter a custom event log call.
70 atomic_uintptr_t XRayPatchedCustomEvent{0};
71
72 // This is the function to call when we encounter a typed event log call.
73 atomic_uintptr_t XRayPatchedTypedEvent{0};
74
75 // This is the global status to determine whether we are currently
76 // patching/unpatching.
77 atomic_uint8_t XRayPatching{0};
78
79 struct TypeDescription {
80 uint32_t type_id;
81 std::size_t description_string_length;
82 };
83
84 using TypeDescriptorMapType = AddrHashMap<TypeDescription, 11>;
85 // An address map from immutable descriptors to type ids.
86 TypeDescriptorMapType TypeDescriptorAddressMap{};
87
88 atomic_uint32_t TypeEventDescriptorCounter{0};
89
90 // MProtectHelper is an RAII wrapper for calls to mprotect(...) that will
91 // undo any successful mprotect(...) changes. This is used to make a page
92 // writeable and executable, and upon destruction if it was successful in
93 // doing so returns the page into a read-only and executable page.
94 //
95 // This is only used specifically for runtime-patching of the XRay
96 // instrumentation points. This assumes that the executable pages are
97 // originally read-and-execute only.
98 class MProtectHelper {
99 void *PageAlignedAddr;
100 std::size_t MProtectLen;
101 bool MustCleanup;
102
103 public:
MProtectHelper(void * PageAlignedAddr,std::size_t MProtectLen,std::size_t PageSize)104 explicit MProtectHelper(void *PageAlignedAddr,
105 std::size_t MProtectLen,
106 std::size_t PageSize) XRAY_NEVER_INSTRUMENT
107 : PageAlignedAddr(PageAlignedAddr),
108 MProtectLen(MProtectLen),
109 MustCleanup(false) {
110 #if SANITIZER_FUCHSIA
111 MProtectLen = RoundUpTo(MProtectLen, PageSize);
112 #endif
113 }
114
MakeWriteable()115 int MakeWriteable() XRAY_NEVER_INSTRUMENT {
116 #if SANITIZER_FUCHSIA
117 auto R = __sanitizer_change_code_protection(
118 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true);
119 if (R != ZX_OK) {
120 Report("XRay: cannot change code protection: %s\n",
121 _zx_status_get_string(R));
122 return -1;
123 }
124 MustCleanup = true;
125 return 0;
126 #else
127 auto R = mprotect(PageAlignedAddr, MProtectLen,
128 PROT_READ | PROT_WRITE | PROT_EXEC);
129 if (R != -1)
130 MustCleanup = true;
131 return R;
132 #endif
133 }
134
~MProtectHelper()135 ~MProtectHelper() XRAY_NEVER_INSTRUMENT {
136 if (MustCleanup) {
137 #if SANITIZER_FUCHSIA
138 auto R = __sanitizer_change_code_protection(
139 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false);
140 if (R != ZX_OK) {
141 Report("XRay: cannot change code protection: %s\n",
142 _zx_status_get_string(R));
143 }
144 #else
145 mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC);
146 #endif
147 }
148 }
149 };
150
151 namespace {
152
patchSled(const XRaySledEntry & Sled,bool Enable,int32_t FuncId)153 bool patchSled(const XRaySledEntry &Sled, bool Enable,
154 int32_t FuncId) XRAY_NEVER_INSTRUMENT {
155 bool Success = false;
156 switch (Sled.Kind) {
157 case XRayEntryType::ENTRY:
158 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry);
159 break;
160 case XRayEntryType::EXIT:
161 Success = patchFunctionExit(Enable, FuncId, Sled);
162 break;
163 case XRayEntryType::TAIL:
164 Success = patchFunctionTailExit(Enable, FuncId, Sled);
165 break;
166 case XRayEntryType::LOG_ARGS_ENTRY:
167 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry);
168 break;
169 case XRayEntryType::CUSTOM_EVENT:
170 Success = patchCustomEvent(Enable, FuncId, Sled);
171 break;
172 case XRayEntryType::TYPED_EVENT:
173 Success = patchTypedEvent(Enable, FuncId, Sled);
174 break;
175 default:
176 Report("Unsupported sled kind '%" PRIu64 "' @%04x\n", Sled.Address,
177 int(Sled.Kind));
178 return false;
179 }
180 return Success;
181 }
182
183 const XRayFunctionSledIndex
findFunctionSleds(int32_t FuncId,const XRaySledMap & InstrMap)184 findFunctionSleds(int32_t FuncId,
185 const XRaySledMap &InstrMap) XRAY_NEVER_INSTRUMENT {
186 int32_t CurFn = 0;
187 uint64_t LastFnAddr = 0;
188 XRayFunctionSledIndex Index = {nullptr, 0};
189
190 for (std::size_t I = 0; I < InstrMap.Entries && CurFn <= FuncId; I++) {
191 const auto &Sled = InstrMap.Sleds[I];
192 const auto Function = Sled.function();
193 if (Function != LastFnAddr) {
194 CurFn++;
195 LastFnAddr = Function;
196 }
197
198 if (CurFn == FuncId) {
199 if (Index.Begin == nullptr)
200 Index.Begin = &Sled;
201 Index.Size = &Sled - Index.Begin + 1;
202 }
203 }
204
205 return Index;
206 }
207
patchFunction(int32_t FuncId,bool Enable)208 XRayPatchingStatus patchFunction(int32_t FuncId,
209 bool Enable) XRAY_NEVER_INSTRUMENT {
210 if (!atomic_load(&XRayInitialized,
211 memory_order_acquire))
212 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
213
214 uint8_t NotPatching = false;
215 if (!atomic_compare_exchange_strong(
216 &XRayPatching, &NotPatching, true, memory_order_acq_rel))
217 return XRayPatchingStatus::ONGOING; // Already patching.
218
219 // Next, we look for the function index.
220 XRaySledMap InstrMap;
221 {
222 SpinMutexLock Guard(&XRayInstrMapMutex);
223 InstrMap = XRayInstrMap;
224 }
225
226 // If we don't have an index, we can't patch individual functions.
227 if (InstrMap.Functions == 0)
228 return XRayPatchingStatus::NOT_INITIALIZED;
229
230 // FuncId must be a positive number, less than the number of functions
231 // instrumented.
232 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
233 Report("Invalid function id provided: %d\n", FuncId);
234 return XRayPatchingStatus::FAILED;
235 }
236
237 // Now we patch ths sleds for this specific function.
238 XRayFunctionSledIndex SledRange;
239 if (InstrMap.SledsIndex) {
240 SledRange = {InstrMap.SledsIndex[FuncId - 1].fromPCRelative(),
241 InstrMap.SledsIndex[FuncId - 1].Size};
242 } else {
243 SledRange = findFunctionSleds(FuncId, InstrMap);
244 }
245 auto *f = SledRange.Begin;
246 bool SucceedOnce = false;
247 for (size_t i = 0; i != SledRange.Size; ++i)
248 SucceedOnce |= patchSled(f[i], Enable, FuncId);
249
250 atomic_store(&XRayPatching, false,
251 memory_order_release);
252
253 if (!SucceedOnce) {
254 Report("Failed patching any sled for function '%d'.", FuncId);
255 return XRayPatchingStatus::FAILED;
256 }
257
258 return XRayPatchingStatus::SUCCESS;
259 }
260
261 // controlPatching implements the common internals of the patching/unpatching
262 // implementation. |Enable| defines whether we're enabling or disabling the
263 // runtime XRay instrumentation.
controlPatching(bool Enable)264 XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
265 if (!atomic_load(&XRayInitialized,
266 memory_order_acquire))
267 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
268
269 uint8_t NotPatching = false;
270 if (!atomic_compare_exchange_strong(
271 &XRayPatching, &NotPatching, true, memory_order_acq_rel))
272 return XRayPatchingStatus::ONGOING; // Already patching.
273
274 uint8_t PatchingSuccess = false;
275 auto XRayPatchingStatusResetter =
276 at_scope_exit([&PatchingSuccess] {
277 if (!PatchingSuccess)
278 atomic_store(&XRayPatching, false,
279 memory_order_release);
280 });
281
282 XRaySledMap InstrMap;
283 {
284 SpinMutexLock Guard(&XRayInstrMapMutex);
285 InstrMap = XRayInstrMap;
286 }
287 if (InstrMap.Entries == 0)
288 return XRayPatchingStatus::NOT_INITIALIZED;
289
290 uint32_t FuncId = 1;
291 uint64_t CurFun = 0;
292
293 // First we want to find the bounds for which we have instrumentation points,
294 // and try to get as few calls to mprotect(...) as possible. We're assuming
295 // that all the sleds for the instrumentation map are contiguous as a single
296 // set of pages. When we do support dynamic shared object instrumentation,
297 // we'll need to do this for each set of page load offsets per DSO loaded. For
298 // now we're assuming we can mprotect the whole section of text between the
299 // minimum sled address and the maximum sled address (+ the largest sled
300 // size).
301 auto *MinSled = &InstrMap.Sleds[0];
302 auto *MaxSled = &InstrMap.Sleds[InstrMap.Entries - 1];
303 for (std::size_t I = 0; I < InstrMap.Entries; I++) {
304 const auto &Sled = InstrMap.Sleds[I];
305 if (Sled.address() < MinSled->address())
306 MinSled = &Sled;
307 if (Sled.address() > MaxSled->address())
308 MaxSled = &Sled;
309 }
310
311 const size_t PageSize = flags()->xray_page_size_override > 0
312 ? flags()->xray_page_size_override
313 : GetPageSizeCached();
314 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
315 Report("System page size is not a power of two: %zu\n", PageSize);
316 return XRayPatchingStatus::FAILED;
317 }
318
319 void *PageAlignedAddr =
320 reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
321 size_t MProtectLen =
322 (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
323 cSledLength;
324 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
325 if (Protector.MakeWriteable() == -1) {
326 Report("Failed mprotect: %d\n", errno);
327 return XRayPatchingStatus::FAILED;
328 }
329
330 for (std::size_t I = 0; I < InstrMap.Entries; ++I) {
331 auto &Sled = InstrMap.Sleds[I];
332 auto F = Sled.function();
333 if (CurFun == 0)
334 CurFun = F;
335 if (F != CurFun) {
336 ++FuncId;
337 CurFun = F;
338 }
339 patchSled(Sled, Enable, FuncId);
340 }
341 atomic_store(&XRayPatching, false,
342 memory_order_release);
343 PatchingSuccess = true;
344 return XRayPatchingStatus::SUCCESS;
345 }
346
mprotectAndPatchFunction(int32_t FuncId,bool Enable)347 XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
348 bool Enable) XRAY_NEVER_INSTRUMENT {
349 XRaySledMap InstrMap;
350 {
351 SpinMutexLock Guard(&XRayInstrMapMutex);
352 InstrMap = XRayInstrMap;
353 }
354
355 // FuncId must be a positive number, less than the number of functions
356 // instrumented.
357 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
358 Report("Invalid function id provided: %d\n", FuncId);
359 return XRayPatchingStatus::FAILED;
360 }
361
362 const size_t PageSize = flags()->xray_page_size_override > 0
363 ? flags()->xray_page_size_override
364 : GetPageSizeCached();
365 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
366 Report("Provided page size is not a power of two: %zu\n", PageSize);
367 return XRayPatchingStatus::FAILED;
368 }
369
370 // Here we compute the minimum sled and maximum sled associated with a
371 // particular function ID.
372 XRayFunctionSledIndex SledRange;
373 if (InstrMap.SledsIndex) {
374 SledRange = {InstrMap.SledsIndex[FuncId - 1].fromPCRelative(),
375 InstrMap.SledsIndex[FuncId - 1].Size};
376 } else {
377 SledRange = findFunctionSleds(FuncId, InstrMap);
378 }
379 auto *f = SledRange.Begin;
380 auto *e = SledRange.Begin + SledRange.Size;
381 auto *MinSled = f;
382 auto *MaxSled = e - 1;
383 while (f != e) {
384 if (f->address() < MinSled->address())
385 MinSled = f;
386 if (f->address() > MaxSled->address())
387 MaxSled = f;
388 ++f;
389 }
390
391 void *PageAlignedAddr =
392 reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
393 size_t MProtectLen =
394 (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
395 cSledLength;
396 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
397 if (Protector.MakeWriteable() == -1) {
398 Report("Failed mprotect: %d\n", errno);
399 return XRayPatchingStatus::FAILED;
400 }
401 return patchFunction(FuncId, Enable);
402 }
403
404 } // namespace
405
406 } // namespace __xray
407
408 using namespace __xray;
409
410 // The following functions are declared `extern "C" {...}` in the header, hence
411 // they're defined in the global namespace.
412
__xray_set_handler(void (* entry)(int32_t,XRayEntryType))413 int __xray_set_handler(void (*entry)(int32_t,
414 XRayEntryType)) XRAY_NEVER_INSTRUMENT {
415 if (atomic_load(&XRayInitialized,
416 memory_order_acquire)) {
417
418 atomic_store(&__xray::XRayPatchedFunction,
419 reinterpret_cast<uintptr_t>(entry),
420 memory_order_release);
421 return 1;
422 }
423 return 0;
424 }
425
__xray_set_customevent_handler(void (* entry)(void *,size_t))426 int __xray_set_customevent_handler(void (*entry)(void *, size_t))
427 XRAY_NEVER_INSTRUMENT {
428 if (atomic_load(&XRayInitialized,
429 memory_order_acquire)) {
430 atomic_store(&__xray::XRayPatchedCustomEvent,
431 reinterpret_cast<uintptr_t>(entry),
432 memory_order_release);
433 return 1;
434 }
435 return 0;
436 }
437
__xray_set_typedevent_handler(void (* entry)(size_t,const void *,size_t))438 int __xray_set_typedevent_handler(void (*entry)(size_t, const void *,
439 size_t)) XRAY_NEVER_INSTRUMENT {
440 if (atomic_load(&XRayInitialized,
441 memory_order_acquire)) {
442 atomic_store(&__xray::XRayPatchedTypedEvent,
443 reinterpret_cast<uintptr_t>(entry),
444 memory_order_release);
445 return 1;
446 }
447 return 0;
448 }
449
__xray_remove_handler()450 int __xray_remove_handler() XRAY_NEVER_INSTRUMENT {
451 return __xray_set_handler(nullptr);
452 }
453
__xray_remove_customevent_handler()454 int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT {
455 return __xray_set_customevent_handler(nullptr);
456 }
457
__xray_remove_typedevent_handler()458 int __xray_remove_typedevent_handler() XRAY_NEVER_INSTRUMENT {
459 return __xray_set_typedevent_handler(nullptr);
460 }
461
__xray_register_event_type(const char * const event_type)462 uint16_t __xray_register_event_type(
463 const char *const event_type) XRAY_NEVER_INSTRUMENT {
464 TypeDescriptorMapType::Handle h(&TypeDescriptorAddressMap, (uptr)event_type);
465 if (h.created()) {
466 h->type_id = atomic_fetch_add(
467 &TypeEventDescriptorCounter, 1, memory_order_acq_rel);
468 h->description_string_length = strnlen(event_type, 1024);
469 }
470 return h->type_id;
471 }
472
__xray_patch()473 XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
474 return controlPatching(true);
475 }
476
__xray_unpatch()477 XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
478 return controlPatching(false);
479 }
480
__xray_patch_function(int32_t FuncId)481 XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
482 return mprotectAndPatchFunction(FuncId, true);
483 }
484
485 XRayPatchingStatus
__xray_unpatch_function(int32_t FuncId)486 __xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
487 return mprotectAndPatchFunction(FuncId, false);
488 }
489
__xray_set_handler_arg1(void (* entry)(int32_t,XRayEntryType,uint64_t))490 int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
491 if (!atomic_load(&XRayInitialized,
492 memory_order_acquire))
493 return 0;
494
495 // A relaxed write might not be visible even if the current thread gets
496 // scheduled on a different CPU/NUMA node. We need to wait for everyone to
497 // have this handler installed for consistency of collected data across CPUs.
498 atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry),
499 memory_order_release);
500 return 1;
501 }
502
__xray_remove_handler_arg1()503 int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); }
504
__xray_function_address(int32_t FuncId)505 uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
506 XRaySledMap InstrMap;
507 {
508 SpinMutexLock Guard(&XRayInstrMapMutex);
509 InstrMap = XRayInstrMap;
510 }
511
512 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions)
513 return 0;
514 const XRaySledEntry *Sled =
515 InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1].fromPCRelative()
516 : findFunctionSleds(FuncId, InstrMap).Begin;
517 return Sled->function()
518 // On PPC, function entries are always aligned to 16 bytes. The beginning of a
519 // sled might be a local entry, which is always +8 based on the global entry.
520 // Always return the global entry.
521 #ifdef __PPC__
522 & ~0xf
523 #endif
524 ;
525 }
526
__xray_max_function_id()527 size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT {
528 SpinMutexLock Guard(&XRayInstrMapMutex);
529 return XRayInstrMap.Functions;
530 }
531