1 #ifndef JEMALLOC_INTERNAL_UTIL_H
2 #define JEMALLOC_INTERNAL_UTIL_H
3
4 #define UTIL_INLINE static inline
5
6 /* Junk fill patterns. */
7 #ifndef JEMALLOC_ALLOC_JUNK
8 # define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5)
9 #endif
10 #ifndef JEMALLOC_FREE_JUNK
11 # define JEMALLOC_FREE_JUNK ((uint8_t)0x5a)
12 #endif
13
14 /*
15 * Wrap a cpp argument that contains commas such that it isn't broken up into
16 * multiple arguments.
17 */
18 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
19
20 /* cpp macro definition stringification. */
21 #define STRINGIFY_HELPER(x) #x
22 #define STRINGIFY(x) STRINGIFY_HELPER(x)
23
24 /*
25 * Silence compiler warnings due to uninitialized values. This is used
26 * wherever the compiler fails to recognize that the variable is never used
27 * uninitialized.
28 */
29 #define JEMALLOC_CC_SILENCE_INIT(v) = v
30
31 #ifdef __GNUC__
32 # define likely(x) __builtin_expect(!!(x), 1)
33 # define unlikely(x) __builtin_expect(!!(x), 0)
34 #else
35 # define likely(x) !!(x)
36 # define unlikely(x) !!(x)
37 #endif
38
39 #if !defined(JEMALLOC_INTERNAL_UNREACHABLE)
40 # error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure
41 #endif
42
43 #ifdef unreachable
44 # undef unreachable
45 #endif
46 #define unreachable() JEMALLOC_INTERNAL_UNREACHABLE()
47
48 /* Set error code. */
49 UTIL_INLINE void
set_errno(int errnum)50 set_errno(int errnum) {
51 #ifdef _WIN32
52 SetLastError(errnum);
53 #else
54 errno = errnum;
55 #endif
56 }
57
58 /* Get last error code. */
59 UTIL_INLINE int
get_errno(void)60 get_errno(void) {
61 #ifdef _WIN32
62 return GetLastError();
63 #else
64 return errno;
65 #endif
66 }
67
68 JEMALLOC_ALWAYS_INLINE void
util_assume(bool b)69 util_assume(bool b) {
70 if (!b) {
71 unreachable();
72 }
73 }
74
75 /* ptr should be valid. */
76 JEMALLOC_ALWAYS_INLINE void
util_prefetch_read(void * ptr)77 util_prefetch_read(void *ptr) {
78 /*
79 * This should arguably be a config check; but any version of GCC so old
80 * that it doesn't support __builtin_prefetch is also too old to build
81 * jemalloc.
82 */
83 #ifdef __GNUC__
84 if (config_debug) {
85 /* Enforce the "valid ptr" requirement. */
86 *(volatile char *)ptr;
87 }
88 __builtin_prefetch(ptr, /* read or write */ 0, /* locality hint */ 3);
89 #else
90 *(volatile char *)ptr;
91 #endif
92 }
93
94 JEMALLOC_ALWAYS_INLINE void
util_prefetch_write(void * ptr)95 util_prefetch_write(void *ptr) {
96 #ifdef __GNUC__
97 if (config_debug) {
98 *(volatile char *)ptr;
99 }
100 /*
101 * The only difference from the read variant is that this has a 1 as the
102 * second argument (the write hint).
103 */
104 __builtin_prefetch(ptr, 1, 3);
105 #else
106 *(volatile char *)ptr;
107 #endif
108 }
109
110 JEMALLOC_ALWAYS_INLINE void
util_prefetch_read_range(void * ptr,size_t sz)111 util_prefetch_read_range(void *ptr, size_t sz) {
112 for (size_t i = 0; i < sz; i += CACHELINE) {
113 util_prefetch_read((void *)((uintptr_t)ptr + i));
114 }
115 }
116
117 JEMALLOC_ALWAYS_INLINE void
util_prefetch_write_range(void * ptr,size_t sz)118 util_prefetch_write_range(void *ptr, size_t sz) {
119 for (size_t i = 0; i < sz; i += CACHELINE) {
120 util_prefetch_write((void *)((uintptr_t)ptr + i));
121 }
122 }
123
124 #undef UTIL_INLINE
125
126 #endif /* JEMALLOC_INTERNAL_UTIL_H */
127