1 #ifndef JEMALLOC_INTERNAL_UTIL_H 2 #define JEMALLOC_INTERNAL_UTIL_H 3 4 #define UTIL_INLINE static inline 5 6 /* Junk fill patterns. */ 7 #ifndef JEMALLOC_ALLOC_JUNK 8 # define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) 9 #endif 10 #ifndef JEMALLOC_FREE_JUNK 11 # define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) 12 #endif 13 14 /* 15 * Wrap a cpp argument that contains commas such that it isn't broken up into 16 * multiple arguments. 17 */ 18 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ 19 20 /* cpp macro definition stringification. */ 21 #define STRINGIFY_HELPER(x) #x 22 #define STRINGIFY(x) STRINGIFY_HELPER(x) 23 24 /* 25 * Silence compiler warnings due to uninitialized values. This is used 26 * wherever the compiler fails to recognize that the variable is never used 27 * uninitialized. 28 */ 29 #define JEMALLOC_CC_SILENCE_INIT(v) = v 30 31 #ifdef __GNUC__ 32 # define likely(x) __builtin_expect(!!(x), 1) 33 # define unlikely(x) __builtin_expect(!!(x), 0) 34 #else 35 # define likely(x) !!(x) 36 # define unlikely(x) !!(x) 37 #endif 38 39 #if !defined(JEMALLOC_INTERNAL_UNREACHABLE) 40 # error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure 41 #endif 42 43 #define unreachable() JEMALLOC_INTERNAL_UNREACHABLE() 44 45 /* Set error code. */ 46 UTIL_INLINE void 47 set_errno(int errnum) { 48 #ifdef _WIN32 49 SetLastError(errnum); 50 #else 51 errno = errnum; 52 #endif 53 } 54 55 /* Get last error code. */ 56 UTIL_INLINE int 57 get_errno(void) { 58 #ifdef _WIN32 59 return GetLastError(); 60 #else 61 return errno; 62 #endif 63 } 64 65 JEMALLOC_ALWAYS_INLINE void 66 util_assume(bool b) { 67 if (!b) { 68 unreachable(); 69 } 70 } 71 72 /* ptr should be valid. */ 73 JEMALLOC_ALWAYS_INLINE void 74 util_prefetch_read(void *ptr) { 75 /* 76 * This should arguably be a config check; but any version of GCC so old 77 * that it doesn't support __builtin_prefetch is also too old to build 78 * jemalloc. 79 */ 80 #ifdef __GNUC__ 81 if (config_debug) { 82 /* Enforce the "valid ptr" requirement. */ 83 *(volatile char *)ptr; 84 } 85 __builtin_prefetch(ptr, /* read or write */ 0, /* locality hint */ 3); 86 #else 87 *(volatile char *)ptr; 88 #endif 89 } 90 91 JEMALLOC_ALWAYS_INLINE void 92 util_prefetch_write(void *ptr) { 93 #ifdef __GNUC__ 94 if (config_debug) { 95 *(volatile char *)ptr; 96 } 97 /* 98 * The only difference from the read variant is that this has a 1 as the 99 * second argument (the write hint). 100 */ 101 __builtin_prefetch(ptr, 1, 3); 102 #else 103 *(volatile char *)ptr; 104 #endif 105 } 106 107 JEMALLOC_ALWAYS_INLINE void 108 util_prefetch_read_range(void *ptr, size_t sz) { 109 for (size_t i = 0; i < sz; i += CACHELINE) { 110 util_prefetch_read((void *)((uintptr_t)ptr + i)); 111 } 112 } 113 114 JEMALLOC_ALWAYS_INLINE void 115 util_prefetch_write_range(void *ptr, size_t sz) { 116 for (size_t i = 0; i < sz; i += CACHELINE) { 117 util_prefetch_write((void *)((uintptr_t)ptr + i)); 118 } 119 } 120 121 #undef UTIL_INLINE 122 123 #endif /* JEMALLOC_INTERNAL_UTIL_H */ 124