1 #include "cpuid.h" 2 #include "sanitizer_common/sanitizer_common.h" 3 #if !SANITIZER_FUCHSIA 4 #include "sanitizer_common/sanitizer_posix.h" 5 #endif 6 #include "xray_defs.h" 7 #include "xray_interface_internal.h" 8 9 #if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE 10 #include <sys/types.h> 11 #include <sys/sysctl.h> 12 #elif SANITIZER_FUCHSIA 13 #include <zircon/syscalls.h> 14 #endif 15 16 #include <atomic> 17 #include <cstdint> 18 #include <errno.h> 19 #include <fcntl.h> 20 #include <iterator> 21 #include <limits> 22 #include <tuple> 23 #include <unistd.h> 24 25 namespace __xray { 26 27 #if SANITIZER_LINUX 28 static std::pair<ssize_t, bool> 29 retryingReadSome(int Fd, char *Begin, char *End) XRAY_NEVER_INSTRUMENT { 30 auto BytesToRead = std::distance(Begin, End); 31 ssize_t BytesRead; 32 ssize_t TotalBytesRead = 0; 33 while (BytesToRead && (BytesRead = read(Fd, Begin, BytesToRead))) { 34 if (BytesRead == -1) { 35 if (errno == EINTR) 36 continue; 37 Report("Read error; errno = %d\n", errno); 38 return std::make_pair(TotalBytesRead, false); 39 } 40 41 TotalBytesRead += BytesRead; 42 BytesToRead -= BytesRead; 43 Begin += BytesRead; 44 } 45 return std::make_pair(TotalBytesRead, true); 46 } 47 48 static bool readValueFromFile(const char *Filename, 49 long long *Value) XRAY_NEVER_INSTRUMENT { 50 int Fd = open(Filename, O_RDONLY | O_CLOEXEC); 51 if (Fd == -1) 52 return false; 53 static constexpr size_t BufSize = 256; 54 char Line[BufSize] = {}; 55 ssize_t BytesRead; 56 bool Success; 57 std::tie(BytesRead, Success) = retryingReadSome(Fd, Line, Line + BufSize); 58 close(Fd); 59 if (!Success) 60 return false; 61 const char *End = nullptr; 62 long long Tmp = internal_simple_strtoll(Line, &End, 10); 63 bool Result = false; 64 if (Line[0] != '\0' && (*End == '\n' || *End == '\0')) { 65 *Value = Tmp; 66 Result = true; 67 } 68 return Result; 69 } 70 71 uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { 72 long long TSCFrequency = -1; 73 if (readValueFromFile("/sys/devices/system/cpu/cpu0/tsc_freq_khz", 74 &TSCFrequency)) { 75 TSCFrequency *= 1000; 76 } else if (readValueFromFile( 77 "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq", 78 &TSCFrequency)) { 79 TSCFrequency *= 1000; 80 } else { 81 Report("Unable to determine CPU frequency for TSC accounting.\n"); 82 } 83 return TSCFrequency == -1 ? 0 : static_cast<uint64_t>(TSCFrequency); 84 } 85 #elif SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE 86 uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { 87 long long TSCFrequency = -1; 88 size_t tscfreqsz = sizeof(TSCFrequency); 89 #if SANITIZER_APPLE 90 if (internal_sysctlbyname("machdep.tsc.frequency", &TSCFrequency, 91 &tscfreqsz, NULL, 0) != -1) { 92 93 #else 94 if (internal_sysctlbyname("machdep.tsc_freq", &TSCFrequency, &tscfreqsz, 95 NULL, 0) != -1) { 96 #endif 97 return static_cast<uint64_t>(TSCFrequency); 98 } else { 99 Report("Unable to determine CPU frequency for TSC accounting.\n"); 100 } 101 102 return 0; 103 } 104 #elif !SANITIZER_FUCHSIA 105 uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT { 106 /* Not supported */ 107 return 0; 108 } 109 #endif 110 111 static constexpr uint8_t CallOpCode = 0xe8; 112 static constexpr uint16_t MovR10Seq = 0xba41; 113 static constexpr uint16_t Jmp9Seq = 0x09eb; 114 static constexpr uint16_t Jmp20Seq = 0x14eb; 115 static constexpr uint16_t Jmp15Seq = 0x0feb; 116 static constexpr uint8_t JmpOpCode = 0xe9; 117 static constexpr uint8_t RetOpCode = 0xc3; 118 static constexpr uint16_t NopwSeq = 0x9066; 119 120 static constexpr int64_t MinOffset{std::numeric_limits<int32_t>::min()}; 121 static constexpr int64_t MaxOffset{std::numeric_limits<int32_t>::max()}; 122 123 bool patchFunctionEntry(const bool Enable, const uint32_t FuncId, 124 const XRaySledEntry &Sled, 125 void (*Trampoline)()) XRAY_NEVER_INSTRUMENT { 126 // Here we do the dance of replacing the following sled: 127 // 128 // xray_sled_n: 129 // jmp +9 130 // <9 byte nop> 131 // 132 // With the following: 133 // 134 // mov r10d, <function id> 135 // call <relative 32bit offset to entry trampoline> 136 // 137 // We need to do this in the following order: 138 // 139 // 1. Put the function id first, 2 bytes from the start of the sled (just 140 // after the 2-byte jmp instruction). 141 // 2. Put the call opcode 6 bytes from the start of the sled. 142 // 3. Put the relative offset 7 bytes from the start of the sled. 143 // 4. Do an atomic write over the jmp instruction for the "mov r10d" 144 // opcode and first operand. 145 // 146 // Prerequisite is to compute the relative offset to the trampoline's address. 147 const uint64_t Address = Sled.address(); 148 int64_t TrampolineOffset = reinterpret_cast<int64_t>(Trampoline) - 149 (static_cast<int64_t>(Address) + 11); 150 if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) { 151 Report("XRay Entry trampoline (%p) too far from sled (%p)\n", 152 reinterpret_cast<void *>(Trampoline), 153 reinterpret_cast<void *>(Address)); 154 return false; 155 } 156 if (Enable) { 157 *reinterpret_cast<uint32_t *>(Address + 2) = FuncId; 158 *reinterpret_cast<uint8_t *>(Address + 6) = CallOpCode; 159 *reinterpret_cast<uint32_t *>(Address + 7) = TrampolineOffset; 160 std::atomic_store_explicit( 161 reinterpret_cast<std::atomic<uint16_t> *>(Address), MovR10Seq, 162 std::memory_order_release); 163 } else { 164 std::atomic_store_explicit( 165 reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp9Seq, 166 std::memory_order_release); 167 // FIXME: Write out the nops still? 168 } 169 return true; 170 } 171 172 bool patchFunctionExit(const bool Enable, const uint32_t FuncId, 173 const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { 174 // Here we do the dance of replacing the following sled: 175 // 176 // xray_sled_n: 177 // ret 178 // <10 byte nop> 179 // 180 // With the following: 181 // 182 // mov r10d, <function id> 183 // jmp <relative 32bit offset to exit trampoline> 184 // 185 // 1. Put the function id first, 2 bytes from the start of the sled (just 186 // after the 1-byte ret instruction). 187 // 2. Put the jmp opcode 6 bytes from the start of the sled. 188 // 3. Put the relative offset 7 bytes from the start of the sled. 189 // 4. Do an atomic write over the jmp instruction for the "mov r10d" 190 // opcode and first operand. 191 // 192 // Prerequisite is to compute the relative offset fo the 193 // __xray_FunctionExit function's address. 194 const uint64_t Address = Sled.address(); 195 int64_t TrampolineOffset = reinterpret_cast<int64_t>(__xray_FunctionExit) - 196 (static_cast<int64_t>(Address) + 11); 197 if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) { 198 Report("XRay Exit trampoline (%p) too far from sled (%p)\n", 199 reinterpret_cast<void *>(__xray_FunctionExit), 200 reinterpret_cast<void *>(Address)); 201 return false; 202 } 203 if (Enable) { 204 *reinterpret_cast<uint32_t *>(Address + 2) = FuncId; 205 *reinterpret_cast<uint8_t *>(Address + 6) = JmpOpCode; 206 *reinterpret_cast<uint32_t *>(Address + 7) = TrampolineOffset; 207 std::atomic_store_explicit( 208 reinterpret_cast<std::atomic<uint16_t> *>(Address), MovR10Seq, 209 std::memory_order_release); 210 } else { 211 std::atomic_store_explicit( 212 reinterpret_cast<std::atomic<uint8_t> *>(Address), RetOpCode, 213 std::memory_order_release); 214 // FIXME: Write out the nops still? 215 } 216 return true; 217 } 218 219 bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId, 220 const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { 221 // Here we do the dance of replacing the tail call sled with a similar 222 // sequence as the entry sled, but calls the tail exit sled instead. 223 const uint64_t Address = Sled.address(); 224 int64_t TrampolineOffset = 225 reinterpret_cast<int64_t>(__xray_FunctionTailExit) - 226 (static_cast<int64_t>(Address) + 11); 227 if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) { 228 Report("XRay Tail Exit trampoline (%p) too far from sled (%p)\n", 229 reinterpret_cast<void *>(__xray_FunctionTailExit), 230 reinterpret_cast<void *>(Address)); 231 return false; 232 } 233 if (Enable) { 234 *reinterpret_cast<uint32_t *>(Address + 2) = FuncId; 235 *reinterpret_cast<uint8_t *>(Address + 6) = CallOpCode; 236 *reinterpret_cast<uint32_t *>(Address + 7) = TrampolineOffset; 237 std::atomic_store_explicit( 238 reinterpret_cast<std::atomic<uint16_t> *>(Address), MovR10Seq, 239 std::memory_order_release); 240 } else { 241 std::atomic_store_explicit( 242 reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp9Seq, 243 std::memory_order_release); 244 // FIXME: Write out the nops still? 245 } 246 return true; 247 } 248 249 bool patchCustomEvent(const bool Enable, const uint32_t FuncId, 250 const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { 251 // Here we do the dance of replacing the following sled: 252 // 253 // xray_sled_n: 254 // jmp +15 // 2 bytes 255 // ... 256 // 257 // With the following: 258 // 259 // nopw // 2 bytes* 260 // ... 261 // 262 // 263 // The "unpatch" should just turn the 'nopw' back to a 'jmp +15'. 264 const uint64_t Address = Sled.address(); 265 if (Enable) { 266 std::atomic_store_explicit( 267 reinterpret_cast<std::atomic<uint16_t> *>(Address), NopwSeq, 268 std::memory_order_release); 269 } else { 270 std::atomic_store_explicit( 271 reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp15Seq, 272 std::memory_order_release); 273 } 274 return false; 275 } 276 277 bool patchTypedEvent(const bool Enable, const uint32_t FuncId, 278 const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { 279 // Here we do the dance of replacing the following sled: 280 // 281 // xray_sled_n: 282 // jmp +20 // 2 byte instruction 283 // ... 284 // 285 // With the following: 286 // 287 // nopw // 2 bytes 288 // ... 289 // 290 // 291 // The "unpatch" should just turn the 'nopw' back to a 'jmp +20'. 292 // The 20 byte sled stashes three argument registers, calls the trampoline, 293 // unstashes the registers and returns. If the arguments are already in 294 // the correct registers, the stashing and unstashing become equivalently 295 // sized nops. 296 const uint64_t Address = Sled.address(); 297 if (Enable) { 298 std::atomic_store_explicit( 299 reinterpret_cast<std::atomic<uint16_t> *>(Address), NopwSeq, 300 std::memory_order_release); 301 } else { 302 std::atomic_store_explicit( 303 reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp20Seq, 304 std::memory_order_release); 305 } 306 return false; 307 } 308 309 #if !SANITIZER_FUCHSIA 310 // We determine whether the CPU we're running on has the correct features we 311 // need. In x86_64 this will be rdtscp support. 312 bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT { 313 unsigned int EAX, EBX, ECX, EDX; 314 315 // We check whether rdtscp support is enabled. According to the x86_64 manual, 316 // level should be set at 0x80000001, and we should have a look at bit 27 in 317 // EDX. That's 0x8000000 (or 1u << 27). 318 __asm__ __volatile__("cpuid" : "=a"(EAX), "=b"(EBX), "=c"(ECX), "=d"(EDX) 319 : "0"(0x80000001)); 320 if (!(EDX & (1u << 27))) { 321 Report("Missing rdtscp support.\n"); 322 return false; 323 } 324 // Also check whether we can determine the CPU frequency, since if we cannot, 325 // we should use the emulated TSC instead. 326 if (!getTSCFrequency()) { 327 Report("Unable to determine CPU frequency.\n"); 328 return false; 329 } 330 return true; 331 } 332 #endif 333 334 } // namespace __xray 335