1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 #endif /* JEMALLOC_H_TYPES */ 5 /******************************************************************************/ 6 #ifdef JEMALLOC_H_STRUCTS 7 8 #endif /* JEMALLOC_H_STRUCTS */ 9 /******************************************************************************/ 10 #ifdef JEMALLOC_H_EXTERNS 11 12 #define atomic_read_uint64(p) atomic_add_uint64(p, 0) 13 #define atomic_read_uint32(p) atomic_add_uint32(p, 0) 14 #define atomic_read_z(p) atomic_add_z(p, 0) 15 #define atomic_read_u(p) atomic_add_u(p, 0) 16 17 #endif /* JEMALLOC_H_EXTERNS */ 18 /******************************************************************************/ 19 #ifdef JEMALLOC_H_INLINES 20 21 #ifndef JEMALLOC_ENABLE_INLINE 22 uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); 23 uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); 24 uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); 25 uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); 26 size_t atomic_add_z(size_t *p, size_t x); 27 size_t atomic_sub_z(size_t *p, size_t x); 28 unsigned atomic_add_u(unsigned *p, unsigned x); 29 unsigned atomic_sub_u(unsigned *p, unsigned x); 30 #endif 31 32 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) 33 /******************************************************************************/ 34 /* 64-bit operations. */ 35 #if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) 36 # ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 37 JEMALLOC_INLINE uint64_t 38 atomic_add_uint64(uint64_t *p, uint64_t x) 39 { 40 41 return (__sync_add_and_fetch(p, x)); 42 } 43 44 JEMALLOC_INLINE uint64_t 45 atomic_sub_uint64(uint64_t *p, uint64_t x) 46 { 47 48 return (__sync_sub_and_fetch(p, x)); 49 } 50 #elif (defined(_MSC_VER)) 51 JEMALLOC_INLINE uint64_t 52 atomic_add_uint64(uint64_t *p, uint64_t x) 53 { 54 55 return (InterlockedExchangeAdd64(p, x)); 56 } 57 58 JEMALLOC_INLINE uint64_t 59 atomic_sub_uint64(uint64_t *p, uint64_t x) 60 { 61 62 return (InterlockedExchangeAdd64(p, -((int64_t)x))); 63 } 64 #elif (defined(JEMALLOC_OSATOMIC)) 65 JEMALLOC_INLINE uint64_t 66 atomic_add_uint64(uint64_t *p, uint64_t x) 67 { 68 69 return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); 70 } 71 72 JEMALLOC_INLINE uint64_t 73 atomic_sub_uint64(uint64_t *p, uint64_t x) 74 { 75 76 return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); 77 } 78 # elif (defined(__amd64__) || defined(__x86_64__)) 79 JEMALLOC_INLINE uint64_t 80 atomic_add_uint64(uint64_t *p, uint64_t x) 81 { 82 83 asm volatile ( 84 "lock; xaddq %0, %1;" 85 : "+r" (x), "=m" (*p) /* Outputs. */ 86 : "m" (*p) /* Inputs. */ 87 ); 88 89 return (x); 90 } 91 92 JEMALLOC_INLINE uint64_t 93 atomic_sub_uint64(uint64_t *p, uint64_t x) 94 { 95 96 x = (uint64_t)(-(int64_t)x); 97 asm volatile ( 98 "lock; xaddq %0, %1;" 99 : "+r" (x), "=m" (*p) /* Outputs. */ 100 : "m" (*p) /* Inputs. */ 101 ); 102 103 return (x); 104 } 105 # elif (defined(JEMALLOC_ATOMIC9)) 106 JEMALLOC_INLINE uint64_t 107 atomic_add_uint64(uint64_t *p, uint64_t x) 108 { 109 110 /* 111 * atomic_fetchadd_64() doesn't exist, but we only ever use this 112 * function on LP64 systems, so atomic_fetchadd_long() will do. 113 */ 114 assert(sizeof(uint64_t) == sizeof(unsigned long)); 115 116 return (atomic_fetchadd_long(p, (unsigned long)x) + x); 117 } 118 119 JEMALLOC_INLINE uint64_t 120 atomic_sub_uint64(uint64_t *p, uint64_t x) 121 { 122 123 assert(sizeof(uint64_t) == sizeof(unsigned long)); 124 125 return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x); 126 } 127 # elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8)) 128 JEMALLOC_INLINE uint64_t 129 atomic_add_uint64(uint64_t *p, uint64_t x) 130 { 131 132 return (__sync_add_and_fetch(p, x)); 133 } 134 135 JEMALLOC_INLINE uint64_t 136 atomic_sub_uint64(uint64_t *p, uint64_t x) 137 { 138 139 return (__sync_sub_and_fetch(p, x)); 140 } 141 # else 142 # error "Missing implementation for 64-bit atomic operations" 143 # endif 144 #endif 145 146 /******************************************************************************/ 147 /* 32-bit operations. */ 148 #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 149 JEMALLOC_INLINE uint32_t 150 atomic_add_uint32(uint32_t *p, uint32_t x) 151 { 152 153 return (__sync_add_and_fetch(p, x)); 154 } 155 156 JEMALLOC_INLINE uint32_t 157 atomic_sub_uint32(uint32_t *p, uint32_t x) 158 { 159 160 return (__sync_sub_and_fetch(p, x)); 161 } 162 #elif (defined(_MSC_VER)) 163 JEMALLOC_INLINE uint32_t 164 atomic_add_uint32(uint32_t *p, uint32_t x) 165 { 166 167 return (InterlockedExchangeAdd(p, x)); 168 } 169 170 JEMALLOC_INLINE uint32_t 171 atomic_sub_uint32(uint32_t *p, uint32_t x) 172 { 173 174 return (InterlockedExchangeAdd(p, -((int32_t)x))); 175 } 176 #elif (defined(JEMALLOC_OSATOMIC)) 177 JEMALLOC_INLINE uint32_t 178 atomic_add_uint32(uint32_t *p, uint32_t x) 179 { 180 181 return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); 182 } 183 184 JEMALLOC_INLINE uint32_t 185 atomic_sub_uint32(uint32_t *p, uint32_t x) 186 { 187 188 return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); 189 } 190 #elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 191 JEMALLOC_INLINE uint32_t 192 atomic_add_uint32(uint32_t *p, uint32_t x) 193 { 194 195 asm volatile ( 196 "lock; xaddl %0, %1;" 197 : "+r" (x), "=m" (*p) /* Outputs. */ 198 : "m" (*p) /* Inputs. */ 199 ); 200 201 return (x); 202 } 203 204 JEMALLOC_INLINE uint32_t 205 atomic_sub_uint32(uint32_t *p, uint32_t x) 206 { 207 208 x = (uint32_t)(-(int32_t)x); 209 asm volatile ( 210 "lock; xaddl %0, %1;" 211 : "+r" (x), "=m" (*p) /* Outputs. */ 212 : "m" (*p) /* Inputs. */ 213 ); 214 215 return (x); 216 } 217 #elif (defined(JEMALLOC_ATOMIC9)) 218 JEMALLOC_INLINE uint32_t 219 atomic_add_uint32(uint32_t *p, uint32_t x) 220 { 221 222 return (atomic_fetchadd_32(p, x) + x); 223 } 224 225 JEMALLOC_INLINE uint32_t 226 atomic_sub_uint32(uint32_t *p, uint32_t x) 227 { 228 229 return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x); 230 } 231 #elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4)) 232 JEMALLOC_INLINE uint32_t 233 atomic_add_uint32(uint32_t *p, uint32_t x) 234 { 235 236 return (__sync_add_and_fetch(p, x)); 237 } 238 239 JEMALLOC_INLINE uint32_t 240 atomic_sub_uint32(uint32_t *p, uint32_t x) 241 { 242 243 return (__sync_sub_and_fetch(p, x)); 244 } 245 #else 246 # error "Missing implementation for 32-bit atomic operations" 247 #endif 248 249 /******************************************************************************/ 250 /* size_t operations. */ 251 JEMALLOC_INLINE size_t 252 atomic_add_z(size_t *p, size_t x) 253 { 254 255 #if (LG_SIZEOF_PTR == 3) 256 return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); 257 #elif (LG_SIZEOF_PTR == 2) 258 return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); 259 #endif 260 } 261 262 JEMALLOC_INLINE size_t 263 atomic_sub_z(size_t *p, size_t x) 264 { 265 266 #if (LG_SIZEOF_PTR == 3) 267 return ((size_t)atomic_add_uint64((uint64_t *)p, 268 (uint64_t)-((int64_t)x))); 269 #elif (LG_SIZEOF_PTR == 2) 270 return ((size_t)atomic_add_uint32((uint32_t *)p, 271 (uint32_t)-((int32_t)x))); 272 #endif 273 } 274 275 /******************************************************************************/ 276 /* unsigned operations. */ 277 JEMALLOC_INLINE unsigned 278 atomic_add_u(unsigned *p, unsigned x) 279 { 280 281 #if (LG_SIZEOF_INT == 3) 282 return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); 283 #elif (LG_SIZEOF_INT == 2) 284 return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); 285 #endif 286 } 287 288 JEMALLOC_INLINE unsigned 289 atomic_sub_u(unsigned *p, unsigned x) 290 { 291 292 #if (LG_SIZEOF_INT == 3) 293 return ((unsigned)atomic_add_uint64((uint64_t *)p, 294 (uint64_t)-((int64_t)x))); 295 #elif (LG_SIZEOF_INT == 2) 296 return ((unsigned)atomic_add_uint32((uint32_t *)p, 297 (uint32_t)-((int32_t)x))); 298 #endif 299 } 300 /******************************************************************************/ 301 #endif 302 303 #endif /* JEMALLOC_H_INLINES */ 304 /******************************************************************************/ 305