1 //===-- hwasan_interceptors.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of HWAddressSanitizer. 10 // 11 // Interceptors for standard library functions. 12 // 13 // FIXME: move as many interceptors as possible into 14 // sanitizer_common/sanitizer_common_interceptors.h 15 //===----------------------------------------------------------------------===// 16 17 #include "interception/interception.h" 18 #include "hwasan.h" 19 #include "hwasan_allocator.h" 20 #include "hwasan_mapping.h" 21 #include "hwasan_thread.h" 22 #include "hwasan_poisoning.h" 23 #include "hwasan_report.h" 24 #include "sanitizer_common/sanitizer_platform_limits_posix.h" 25 #include "sanitizer_common/sanitizer_allocator.h" 26 #include "sanitizer_common/sanitizer_allocator_interface.h" 27 #include "sanitizer_common/sanitizer_allocator_internal.h" 28 #include "sanitizer_common/sanitizer_atomic.h" 29 #include "sanitizer_common/sanitizer_common.h" 30 #include "sanitizer_common/sanitizer_errno.h" 31 #include "sanitizer_common/sanitizer_stackdepot.h" 32 #include "sanitizer_common/sanitizer_libc.h" 33 #include "sanitizer_common/sanitizer_linux.h" 34 #include "sanitizer_common/sanitizer_tls_get_addr.h" 35 36 #include <stdarg.h> 37 // ACHTUNG! No other system header includes in this file. 38 // Ideally, we should get rid of stdarg.h as well. 39 40 using namespace __hwasan; 41 42 using __sanitizer::memory_order; 43 using __sanitizer::atomic_load; 44 using __sanitizer::atomic_store; 45 using __sanitizer::atomic_uintptr_t; 46 47 static uptr allocated_for_dlsym; 48 static const uptr kDlsymAllocPoolSize = 1024; 49 static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; 50 51 static bool IsInDlsymAllocPool(const void *ptr) { 52 uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; 53 return off < sizeof(alloc_memory_for_dlsym); 54 } 55 56 static void *AllocateFromLocalPool(uptr size_in_bytes) { 57 uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; 58 void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; 59 allocated_for_dlsym += size_in_words; 60 CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); 61 return mem; 62 } 63 64 #define ENSURE_HWASAN_INITED() do { \ 65 CHECK(!hwasan_init_is_running); \ 66 if (!hwasan_inited) { \ 67 __hwasan_init(); \ 68 } \ 69 } while (0) 70 71 72 int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) { 73 GET_MALLOC_STACK_TRACE; 74 CHECK_NE(memptr, 0); 75 int res = hwasan_posix_memalign(memptr, alignment, size, &stack); 76 return res; 77 } 78 79 void * __sanitizer_memalign(uptr alignment, uptr size) { 80 GET_MALLOC_STACK_TRACE; 81 return hwasan_memalign(alignment, size, &stack); 82 } 83 84 void * __sanitizer_aligned_alloc(uptr alignment, uptr size) { 85 GET_MALLOC_STACK_TRACE; 86 return hwasan_aligned_alloc(alignment, size, &stack); 87 } 88 89 void * __sanitizer___libc_memalign(uptr alignment, uptr size) { 90 GET_MALLOC_STACK_TRACE; 91 void *ptr = hwasan_memalign(alignment, size, &stack); 92 if (ptr) 93 DTLS_on_libc_memalign(ptr, size); 94 return ptr; 95 } 96 97 void * __sanitizer_valloc(uptr size) { 98 GET_MALLOC_STACK_TRACE; 99 return hwasan_valloc(size, &stack); 100 } 101 102 void * __sanitizer_pvalloc(uptr size) { 103 GET_MALLOC_STACK_TRACE; 104 return hwasan_pvalloc(size, &stack); 105 } 106 107 void __sanitizer_free(void *ptr) { 108 GET_MALLOC_STACK_TRACE; 109 if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; 110 hwasan_free(ptr, &stack); 111 } 112 113 void __sanitizer_cfree(void *ptr) { 114 GET_MALLOC_STACK_TRACE; 115 if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; 116 hwasan_free(ptr, &stack); 117 } 118 119 uptr __sanitizer_malloc_usable_size(const void *ptr) { 120 return __sanitizer_get_allocated_size(ptr); 121 } 122 123 struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() { 124 __sanitizer_struct_mallinfo sret; 125 internal_memset(&sret, 0, sizeof(sret)); 126 return sret; 127 } 128 129 int __sanitizer_mallopt(int cmd, int value) { 130 return 0; 131 } 132 133 void __sanitizer_malloc_stats(void) { 134 // FIXME: implement, but don't call REAL(malloc_stats)! 135 } 136 137 void * __sanitizer_calloc(uptr nmemb, uptr size) { 138 GET_MALLOC_STACK_TRACE; 139 if (UNLIKELY(!hwasan_inited)) 140 // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. 141 return AllocateFromLocalPool(nmemb * size); 142 return hwasan_calloc(nmemb, size, &stack); 143 } 144 145 void * __sanitizer_realloc(void *ptr, uptr size) { 146 GET_MALLOC_STACK_TRACE; 147 if (UNLIKELY(IsInDlsymAllocPool(ptr))) { 148 uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; 149 uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); 150 void *new_ptr; 151 if (UNLIKELY(!hwasan_inited)) { 152 new_ptr = AllocateFromLocalPool(copy_size); 153 } else { 154 copy_size = size; 155 new_ptr = hwasan_malloc(copy_size, &stack); 156 } 157 internal_memcpy(new_ptr, ptr, copy_size); 158 return new_ptr; 159 } 160 return hwasan_realloc(ptr, size, &stack); 161 } 162 163 void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) { 164 GET_MALLOC_STACK_TRACE; 165 return hwasan_reallocarray(ptr, nmemb, size, &stack); 166 } 167 168 void * __sanitizer_malloc(uptr size) { 169 GET_MALLOC_STACK_TRACE; 170 if (UNLIKELY(!hwasan_init_is_running)) 171 ENSURE_HWASAN_INITED(); 172 if (UNLIKELY(!hwasan_inited)) 173 // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. 174 return AllocateFromLocalPool(size); 175 return hwasan_malloc(size, &stack); 176 } 177 178 #if HWASAN_WITH_INTERCEPTORS 179 #define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \ 180 extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \ 181 ALIAS("__sanitizer_" #FN); \ 182 extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \ 183 ARGS) ALIAS("__sanitizer_" #FN) 184 185 INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment, 186 SIZE_T size); 187 INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size); 188 INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size); 189 INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size); 190 INTERCEPTOR_ALIAS(void, free, void *ptr); 191 INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); 192 INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); 193 INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); 194 INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size); 195 INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); 196 197 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD 198 INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size); 199 INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size); 200 INTERCEPTOR_ALIAS(void, cfree, void *ptr); 201 INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo); 202 INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value); 203 INTERCEPTOR_ALIAS(void, malloc_stats, void); 204 #endif 205 206 struct ThreadStartArg { 207 thread_callback_t callback; 208 void *param; 209 }; 210 211 static void *HwasanThreadStartFunc(void *arg) { 212 __hwasan_thread_enter(); 213 ThreadStartArg A = *reinterpret_cast<ThreadStartArg*>(arg); 214 UnmapOrDie(arg, GetPageSizeCached()); 215 return A.callback(A.param); 216 } 217 218 INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), 219 void * param) { 220 ScopedTaggingDisabler disabler; 221 ThreadStartArg *A = reinterpret_cast<ThreadStartArg *> (MmapOrDie( 222 GetPageSizeCached(), "pthread_create")); 223 *A = {callback, param}; 224 int res = REAL(pthread_create)(UntagPtr(th), UntagPtr(attr), 225 &HwasanThreadStartFunc, A); 226 return res; 227 } 228 229 DEFINE_REAL(int, vfork) 230 DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork) 231 #endif // HWASAN_WITH_INTERCEPTORS 232 233 #if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__) 234 // Get and/or change the set of blocked signals. 235 extern "C" int sigprocmask(int __how, const __hw_sigset_t *__restrict __set, 236 __hw_sigset_t *__restrict __oset); 237 #define SIG_BLOCK 0 238 #define SIG_SETMASK 2 239 extern "C" int __sigjmp_save(__hw_sigjmp_buf env, int savemask) { 240 env[0].__mask_was_saved = 241 (savemask && sigprocmask(SIG_BLOCK, (__hw_sigset_t *)0, 242 &env[0].__saved_mask) == 0); 243 return 0; 244 } 245 246 static void __attribute__((always_inline)) 247 InternalLongjmp(__hw_register_buf env, int retval) { 248 // Clear all memory tags on the stack between here and where we're going. 249 unsigned long long stack_pointer = env[13]; 250 // The stack pointer should never be tagged, so we don't need to clear the 251 // tag for this function call. 252 __hwasan_handle_longjmp((void *)stack_pointer); 253 254 // Run code for handling a longjmp. 255 // Need to use a register that isn't going to be loaded from the environment 256 // buffer -- hence why we need to specify the register to use. 257 // Must implement this ourselves, since we don't know the order of registers 258 // in different libc implementations and many implementations mangle the 259 // stack pointer so we can't use it without knowing the demangling scheme. 260 register long int retval_tmp asm("x1") = retval; 261 register void *env_address asm("x0") = &env[0]; 262 asm volatile("ldp x19, x20, [%0, #0<<3];" 263 "ldp x21, x22, [%0, #2<<3];" 264 "ldp x23, x24, [%0, #4<<3];" 265 "ldp x25, x26, [%0, #6<<3];" 266 "ldp x27, x28, [%0, #8<<3];" 267 "ldp x29, x30, [%0, #10<<3];" 268 "ldp d8, d9, [%0, #14<<3];" 269 "ldp d10, d11, [%0, #16<<3];" 270 "ldp d12, d13, [%0, #18<<3];" 271 "ldp d14, d15, [%0, #20<<3];" 272 "ldr x5, [%0, #13<<3];" 273 "mov sp, x5;" 274 // Return the value requested to return through arguments. 275 // This should be in x1 given what we requested above. 276 "cmp %1, #0;" 277 "mov x0, #1;" 278 "csel x0, %1, x0, ne;" 279 "br x30;" 280 : "+r"(env_address) 281 : "r"(retval_tmp)); 282 } 283 284 INTERCEPTOR(void, siglongjmp, __hw_sigjmp_buf env, int val) { 285 if (env[0].__mask_was_saved) 286 // Restore the saved signal mask. 287 (void)sigprocmask(SIG_SETMASK, &env[0].__saved_mask, 288 (__hw_sigset_t *)0); 289 InternalLongjmp(env[0].__jmpbuf, val); 290 } 291 292 // Required since glibc libpthread calls __libc_longjmp on pthread_exit, and 293 // _setjmp on start_thread. Hence we have to intercept the longjmp on 294 // pthread_exit so the __hw_jmp_buf order matches. 295 INTERCEPTOR(void, __libc_longjmp, __hw_jmp_buf env, int val) { 296 InternalLongjmp(env[0].__jmpbuf, val); 297 } 298 299 INTERCEPTOR(void, longjmp, __hw_jmp_buf env, int val) { 300 InternalLongjmp(env[0].__jmpbuf, val); 301 } 302 #undef SIG_BLOCK 303 #undef SIG_SETMASK 304 305 #endif // HWASAN_WITH_INTERCEPTORS && __aarch64__ 306 307 static void BeforeFork() { 308 StackDepotLockAll(); 309 } 310 311 static void AfterFork() { 312 StackDepotUnlockAll(); 313 } 314 315 INTERCEPTOR(int, fork, void) { 316 ENSURE_HWASAN_INITED(); 317 BeforeFork(); 318 int pid = REAL(fork)(); 319 AfterFork(); 320 return pid; 321 } 322 323 namespace __hwasan { 324 325 int OnExit() { 326 // FIXME: ask frontend whether we need to return failure. 327 return 0; 328 } 329 330 } // namespace __hwasan 331 332 namespace __hwasan { 333 334 void InitializeInterceptors() { 335 static int inited = 0; 336 CHECK_EQ(inited, 0); 337 338 INTERCEPT_FUNCTION(fork); 339 340 #if HWASAN_WITH_INTERCEPTORS 341 #if defined(__linux__) 342 INTERCEPT_FUNCTION(vfork); 343 #endif // __linux__ 344 INTERCEPT_FUNCTION(pthread_create); 345 #endif 346 347 inited = 1; 348 } 349 } // namespace __hwasan 350