1 #include <mutex>
2 #include <new>
3
4 #define JEMALLOC_CPP_CPP_
5 #ifdef __cplusplus
6 extern "C" {
7 #endif
8
9 #include "jemalloc/internal/jemalloc_preamble.h"
10 #include "jemalloc/internal/jemalloc_internal_includes.h"
11
12 #ifdef __cplusplus
13 }
14 #endif
15
16 // All operators in this file are exported.
17
18 // Possibly alias hidden versions of malloc and sdallocx to avoid an extra plt
19 // thunk?
20 //
21 // extern __typeof (sdallocx) sdallocx_int
22 // __attribute ((alias ("sdallocx"),
23 // visibility ("hidden")));
24 //
25 // ... but it needs to work with jemalloc namespaces.
26
27 void *operator new(std::size_t size);
28 void *operator new[](std::size_t size);
29 void *operator new(std::size_t size, const std::nothrow_t &) noexcept;
30 void *operator new[](std::size_t size, const std::nothrow_t &) noexcept;
31 void operator delete(void *ptr) noexcept;
32 void operator delete[](void *ptr) noexcept;
33 void operator delete(void *ptr, const std::nothrow_t &) noexcept;
34 void operator delete[](void *ptr, const std::nothrow_t &) noexcept;
35
36 #if __cpp_sized_deallocation >= 201309
37 /* C++14's sized-delete operators. */
38 void operator delete(void *ptr, std::size_t size) noexcept;
39 void operator delete[](void *ptr, std::size_t size) noexcept;
40 #endif
41
42 #if __cpp_aligned_new >= 201606
43 /* C++17's over-aligned operators. */
44 void *operator new(std::size_t size, std::align_val_t);
45 void *operator new(std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept;
46 void *operator new[](std::size_t size, std::align_val_t);
47 void *operator new[](std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept;
48 void operator delete(void* ptr, std::align_val_t) noexcept;
49 void operator delete(void* ptr, std::align_val_t, const std::nothrow_t &) noexcept;
50 void operator delete(void* ptr, std::size_t size, std::align_val_t al) noexcept;
51 void operator delete[](void* ptr, std::align_val_t) noexcept;
52 void operator delete[](void* ptr, std::align_val_t, const std::nothrow_t &) noexcept;
53 void operator delete[](void* ptr, std::size_t size, std::align_val_t al) noexcept;
54 #endif
55
56 JEMALLOC_NOINLINE
57 static void *
handleOOM(std::size_t size,bool nothrow)58 handleOOM(std::size_t size, bool nothrow) {
59 if (opt_experimental_infallible_new) {
60 safety_check_fail("<jemalloc>: Allocation failed and "
61 "opt.experimental_infallible_new is true. Aborting.\n");
62 return nullptr;
63 }
64
65 void *ptr = nullptr;
66
67 while (ptr == nullptr) {
68 std::new_handler handler;
69 // GCC-4.8 and clang 4.0 do not have std::get_new_handler.
70 {
71 static std::mutex mtx;
72 std::lock_guard<std::mutex> lock(mtx);
73
74 handler = std::set_new_handler(nullptr);
75 std::set_new_handler(handler);
76 }
77 if (handler == nullptr)
78 break;
79
80 try {
81 handler();
82 } catch (const std::bad_alloc &) {
83 break;
84 }
85
86 ptr = je_malloc(size);
87 }
88
89 if (ptr == nullptr && !nothrow)
90 std::__throw_bad_alloc();
91 return ptr;
92 }
93
94 template <bool IsNoExcept>
95 JEMALLOC_NOINLINE
96 static void *
fallback_impl(std::size_t size)97 fallback_impl(std::size_t size) noexcept(IsNoExcept) {
98 void *ptr = malloc_default(size);
99 if (likely(ptr != nullptr)) {
100 return ptr;
101 }
102 return handleOOM(size, IsNoExcept);
103 }
104
105 template <bool IsNoExcept>
106 JEMALLOC_ALWAYS_INLINE
107 void *
newImpl(std::size_t size)108 newImpl(std::size_t size) noexcept(IsNoExcept) {
109 return imalloc_fastpath(size, &fallback_impl<IsNoExcept>);
110 }
111
112 void *
operator new(std::size_t size)113 operator new(std::size_t size) {
114 return newImpl<false>(size);
115 }
116
117 void *
operator new[](std::size_t size)118 operator new[](std::size_t size) {
119 return newImpl<false>(size);
120 }
121
122 void *
operator new(std::size_t size,const std::nothrow_t &)123 operator new(std::size_t size, const std::nothrow_t &) noexcept {
124 return newImpl<true>(size);
125 }
126
127 void *
operator new[](std::size_t size,const std::nothrow_t &)128 operator new[](std::size_t size, const std::nothrow_t &) noexcept {
129 return newImpl<true>(size);
130 }
131
132 #if __cpp_aligned_new >= 201606
133
134 template <bool IsNoExcept>
135 JEMALLOC_ALWAYS_INLINE
136 void *
alignedNewImpl(std::size_t size,std::align_val_t alignment)137 alignedNewImpl(std::size_t size, std::align_val_t alignment) noexcept(IsNoExcept) {
138 void *ptr = je_aligned_alloc(static_cast<std::size_t>(alignment), size);
139 if (likely(ptr != nullptr)) {
140 return ptr;
141 }
142
143 return handleOOM(size, IsNoExcept);
144 }
145
146 void *
operator new(std::size_t size,std::align_val_t alignment)147 operator new(std::size_t size, std::align_val_t alignment) {
148 return alignedNewImpl<false>(size, alignment);
149 }
150
151 void *
operator new[](std::size_t size,std::align_val_t alignment)152 operator new[](std::size_t size, std::align_val_t alignment) {
153 return alignedNewImpl<false>(size, alignment);
154 }
155
156 void *
operator new(std::size_t size,std::align_val_t alignment,const std::nothrow_t &)157 operator new(std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept {
158 return alignedNewImpl<true>(size, alignment);
159 }
160
161 void *
operator new[](std::size_t size,std::align_val_t alignment,const std::nothrow_t &)162 operator new[](std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept {
163 return alignedNewImpl<true>(size, alignment);
164 }
165
166 #endif // __cpp_aligned_new
167
168 void
operator delete(void * ptr)169 operator delete(void *ptr) noexcept {
170 je_free(ptr);
171 }
172
173 void
operator delete[](void * ptr)174 operator delete[](void *ptr) noexcept {
175 je_free(ptr);
176 }
177
178 void
operator delete(void * ptr,const std::nothrow_t &)179 operator delete(void *ptr, const std::nothrow_t &) noexcept {
180 je_free(ptr);
181 }
182
operator delete[](void * ptr,const std::nothrow_t &)183 void operator delete[](void *ptr, const std::nothrow_t &) noexcept {
184 je_free(ptr);
185 }
186
187 #if __cpp_sized_deallocation >= 201309
188
189 JEMALLOC_ALWAYS_INLINE
190 void
sizedDeleteImpl(void * ptr,std::size_t size)191 sizedDeleteImpl(void* ptr, std::size_t size) noexcept {
192 if (unlikely(ptr == nullptr)) {
193 return;
194 }
195 je_sdallocx_noflags(ptr, size);
196 }
197
198 void
operator delete(void * ptr,std::size_t size)199 operator delete(void *ptr, std::size_t size) noexcept {
200 sizedDeleteImpl(ptr, size);
201 }
202
203 void
operator delete[](void * ptr,std::size_t size)204 operator delete[](void *ptr, std::size_t size) noexcept {
205 sizedDeleteImpl(ptr, size);
206 }
207
208 #endif // __cpp_sized_deallocation
209
210 #if __cpp_aligned_new >= 201606
211
212 JEMALLOC_ALWAYS_INLINE
213 void
alignedSizedDeleteImpl(void * ptr,std::size_t size,std::align_val_t alignment)214 alignedSizedDeleteImpl(void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
215 if (config_debug) {
216 assert(((size_t)alignment & ((size_t)alignment - 1)) == 0);
217 }
218 if (unlikely(ptr == nullptr)) {
219 return;
220 }
221 je_sdallocx(ptr, size, MALLOCX_ALIGN(alignment));
222 }
223
224 void
operator delete(void * ptr,std::align_val_t)225 operator delete(void* ptr, std::align_val_t) noexcept {
226 je_free(ptr);
227 }
228
229 void
operator delete[](void * ptr,std::align_val_t)230 operator delete[](void* ptr, std::align_val_t) noexcept {
231 je_free(ptr);
232 }
233
234 void
operator delete(void * ptr,std::align_val_t,const std::nothrow_t &)235 operator delete(void* ptr, std::align_val_t, const std::nothrow_t&) noexcept {
236 je_free(ptr);
237 }
238
239 void
operator delete[](void * ptr,std::align_val_t,const std::nothrow_t &)240 operator delete[](void* ptr, std::align_val_t, const std::nothrow_t&) noexcept {
241 je_free(ptr);
242 }
243
244 void
operator delete(void * ptr,std::size_t size,std::align_val_t alignment)245 operator delete(void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
246 alignedSizedDeleteImpl(ptr, size, alignment);
247 }
248
249 void
operator delete[](void * ptr,std::size_t size,std::align_val_t alignment)250 operator delete[](void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
251 alignedSizedDeleteImpl(ptr, size, alignment);
252 }
253
254 #endif // __cpp_aligned_new
255