xref: /freebsd/contrib/jemalloc/include/jemalloc/internal/atomic.h (revision 38f0b757fd84d17d0fc24739a7cda160c4516d81)
1 /******************************************************************************/
2 #ifdef JEMALLOC_H_TYPES
3 
4 #endif /* JEMALLOC_H_TYPES */
5 /******************************************************************************/
6 #ifdef JEMALLOC_H_STRUCTS
7 
8 #endif /* JEMALLOC_H_STRUCTS */
9 /******************************************************************************/
10 #ifdef JEMALLOC_H_EXTERNS
11 
12 #define	atomic_read_uint64(p)	atomic_add_uint64(p, 0)
13 #define	atomic_read_uint32(p)	atomic_add_uint32(p, 0)
14 #define	atomic_read_z(p)	atomic_add_z(p, 0)
15 #define	atomic_read_u(p)	atomic_add_u(p, 0)
16 
17 #endif /* JEMALLOC_H_EXTERNS */
18 /******************************************************************************/
19 #ifdef JEMALLOC_H_INLINES
20 
21 #ifndef JEMALLOC_ENABLE_INLINE
22 uint64_t	atomic_add_uint64(uint64_t *p, uint64_t x);
23 uint64_t	atomic_sub_uint64(uint64_t *p, uint64_t x);
24 uint32_t	atomic_add_uint32(uint32_t *p, uint32_t x);
25 uint32_t	atomic_sub_uint32(uint32_t *p, uint32_t x);
26 size_t	atomic_add_z(size_t *p, size_t x);
27 size_t	atomic_sub_z(size_t *p, size_t x);
28 unsigned	atomic_add_u(unsigned *p, unsigned x);
29 unsigned	atomic_sub_u(unsigned *p, unsigned x);
30 #endif
31 
32 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
33 /******************************************************************************/
34 /* 64-bit operations. */
35 #if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
36 #  ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
37 JEMALLOC_INLINE uint64_t
38 atomic_add_uint64(uint64_t *p, uint64_t x)
39 {
40 
41 	return (__sync_add_and_fetch(p, x));
42 }
43 
44 JEMALLOC_INLINE uint64_t
45 atomic_sub_uint64(uint64_t *p, uint64_t x)
46 {
47 
48 	return (__sync_sub_and_fetch(p, x));
49 }
50 #elif (defined(_MSC_VER))
51 JEMALLOC_INLINE uint64_t
52 atomic_add_uint64(uint64_t *p, uint64_t x)
53 {
54 
55 	return (InterlockedExchangeAdd64(p, x));
56 }
57 
58 JEMALLOC_INLINE uint64_t
59 atomic_sub_uint64(uint64_t *p, uint64_t x)
60 {
61 
62 	return (InterlockedExchangeAdd64(p, -((int64_t)x)));
63 }
64 #elif (defined(JEMALLOC_OSATOMIC))
65 JEMALLOC_INLINE uint64_t
66 atomic_add_uint64(uint64_t *p, uint64_t x)
67 {
68 
69 	return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
70 }
71 
72 JEMALLOC_INLINE uint64_t
73 atomic_sub_uint64(uint64_t *p, uint64_t x)
74 {
75 
76 	return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
77 }
78 #  elif (defined(__amd64__) || defined(__x86_64__))
79 JEMALLOC_INLINE uint64_t
80 atomic_add_uint64(uint64_t *p, uint64_t x)
81 {
82 
83 	asm volatile (
84 	    "lock; xaddq %0, %1;"
85 	    : "+r" (x), "=m" (*p) /* Outputs. */
86 	    : "m" (*p) /* Inputs. */
87 	    );
88 
89 	return (x);
90 }
91 
92 JEMALLOC_INLINE uint64_t
93 atomic_sub_uint64(uint64_t *p, uint64_t x)
94 {
95 
96 	x = (uint64_t)(-(int64_t)x);
97 	asm volatile (
98 	    "lock; xaddq %0, %1;"
99 	    : "+r" (x), "=m" (*p) /* Outputs. */
100 	    : "m" (*p) /* Inputs. */
101 	    );
102 
103 	return (x);
104 }
105 #  elif (defined(JEMALLOC_ATOMIC9))
106 JEMALLOC_INLINE uint64_t
107 atomic_add_uint64(uint64_t *p, uint64_t x)
108 {
109 
110 	/*
111 	 * atomic_fetchadd_64() doesn't exist, but we only ever use this
112 	 * function on LP64 systems, so atomic_fetchadd_long() will do.
113 	 */
114 	assert(sizeof(uint64_t) == sizeof(unsigned long));
115 
116 	return (atomic_fetchadd_long(p, (unsigned long)x) + x);
117 }
118 
119 JEMALLOC_INLINE uint64_t
120 atomic_sub_uint64(uint64_t *p, uint64_t x)
121 {
122 
123 	assert(sizeof(uint64_t) == sizeof(unsigned long));
124 
125 	return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);
126 }
127 #  elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
128 JEMALLOC_INLINE uint64_t
129 atomic_add_uint64(uint64_t *p, uint64_t x)
130 {
131 
132 	return (__sync_add_and_fetch(p, x));
133 }
134 
135 JEMALLOC_INLINE uint64_t
136 atomic_sub_uint64(uint64_t *p, uint64_t x)
137 {
138 
139 	return (__sync_sub_and_fetch(p, x));
140 }
141 #  else
142 #    error "Missing implementation for 64-bit atomic operations"
143 #  endif
144 #endif
145 
146 /******************************************************************************/
147 /* 32-bit operations. */
148 #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4
149 JEMALLOC_INLINE uint32_t
150 atomic_add_uint32(uint32_t *p, uint32_t x)
151 {
152 
153 	return (__sync_add_and_fetch(p, x));
154 }
155 
156 JEMALLOC_INLINE uint32_t
157 atomic_sub_uint32(uint32_t *p, uint32_t x)
158 {
159 
160 	return (__sync_sub_and_fetch(p, x));
161 }
162 #elif (defined(_MSC_VER))
163 JEMALLOC_INLINE uint32_t
164 atomic_add_uint32(uint32_t *p, uint32_t x)
165 {
166 
167 	return (InterlockedExchangeAdd(p, x));
168 }
169 
170 JEMALLOC_INLINE uint32_t
171 atomic_sub_uint32(uint32_t *p, uint32_t x)
172 {
173 
174 	return (InterlockedExchangeAdd(p, -((int32_t)x)));
175 }
176 #elif (defined(JEMALLOC_OSATOMIC))
177 JEMALLOC_INLINE uint32_t
178 atomic_add_uint32(uint32_t *p, uint32_t x)
179 {
180 
181 	return (OSAtomicAdd32((int32_t)x, (int32_t *)p));
182 }
183 
184 JEMALLOC_INLINE uint32_t
185 atomic_sub_uint32(uint32_t *p, uint32_t x)
186 {
187 
188 	return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
189 }
190 #elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
191 JEMALLOC_INLINE uint32_t
192 atomic_add_uint32(uint32_t *p, uint32_t x)
193 {
194 
195 	asm volatile (
196 	    "lock; xaddl %0, %1;"
197 	    : "+r" (x), "=m" (*p) /* Outputs. */
198 	    : "m" (*p) /* Inputs. */
199 	    );
200 
201 	return (x);
202 }
203 
204 JEMALLOC_INLINE uint32_t
205 atomic_sub_uint32(uint32_t *p, uint32_t x)
206 {
207 
208 	x = (uint32_t)(-(int32_t)x);
209 	asm volatile (
210 	    "lock; xaddl %0, %1;"
211 	    : "+r" (x), "=m" (*p) /* Outputs. */
212 	    : "m" (*p) /* Inputs. */
213 	    );
214 
215 	return (x);
216 }
217 #elif (defined(JEMALLOC_ATOMIC9))
218 JEMALLOC_INLINE uint32_t
219 atomic_add_uint32(uint32_t *p, uint32_t x)
220 {
221 
222 	return (atomic_fetchadd_32(p, x) + x);
223 }
224 
225 JEMALLOC_INLINE uint32_t
226 atomic_sub_uint32(uint32_t *p, uint32_t x)
227 {
228 
229 	return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
230 }
231 #elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
232 JEMALLOC_INLINE uint32_t
233 atomic_add_uint32(uint32_t *p, uint32_t x)
234 {
235 
236 	return (__sync_add_and_fetch(p, x));
237 }
238 
239 JEMALLOC_INLINE uint32_t
240 atomic_sub_uint32(uint32_t *p, uint32_t x)
241 {
242 
243 	return (__sync_sub_and_fetch(p, x));
244 }
245 #else
246 #  error "Missing implementation for 32-bit atomic operations"
247 #endif
248 
249 /******************************************************************************/
250 /* size_t operations. */
251 JEMALLOC_INLINE size_t
252 atomic_add_z(size_t *p, size_t x)
253 {
254 
255 #if (LG_SIZEOF_PTR == 3)
256 	return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
257 #elif (LG_SIZEOF_PTR == 2)
258 	return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
259 #endif
260 }
261 
262 JEMALLOC_INLINE size_t
263 atomic_sub_z(size_t *p, size_t x)
264 {
265 
266 #if (LG_SIZEOF_PTR == 3)
267 	return ((size_t)atomic_add_uint64((uint64_t *)p,
268 	    (uint64_t)-((int64_t)x)));
269 #elif (LG_SIZEOF_PTR == 2)
270 	return ((size_t)atomic_add_uint32((uint32_t *)p,
271 	    (uint32_t)-((int32_t)x)));
272 #endif
273 }
274 
275 /******************************************************************************/
276 /* unsigned operations. */
277 JEMALLOC_INLINE unsigned
278 atomic_add_u(unsigned *p, unsigned x)
279 {
280 
281 #if (LG_SIZEOF_INT == 3)
282 	return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
283 #elif (LG_SIZEOF_INT == 2)
284 	return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
285 #endif
286 }
287 
288 JEMALLOC_INLINE unsigned
289 atomic_sub_u(unsigned *p, unsigned x)
290 {
291 
292 #if (LG_SIZEOF_INT == 3)
293 	return ((unsigned)atomic_add_uint64((uint64_t *)p,
294 	    (uint64_t)-((int64_t)x)));
295 #elif (LG_SIZEOF_INT == 2)
296 	return ((unsigned)atomic_add_uint32((uint32_t *)p,
297 	    (uint32_t)-((int32_t)x)));
298 #endif
299 }
300 /******************************************************************************/
301 #endif
302 
303 #endif /* JEMALLOC_H_INLINES */
304 /******************************************************************************/
305