xref: /freebsd/contrib/llvm-project/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp (revision 86dc8398c9ca2283c5d6984992b7a585257b5adb)
1 //===-- tsan_interface_atomic.cpp -----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 // ThreadSanitizer atomic operations are based on C++11/C1x standards.
14 // For background see C++11 standard.  A slightly older, publicly
15 // available draft of the standard (not entirely up-to-date, but close enough
16 // for casual browsing) is available here:
17 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
18 // The following page contains more background information:
19 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
20 
21 #include "sanitizer_common/sanitizer_placement_new.h"
22 #include "sanitizer_common/sanitizer_stacktrace.h"
23 #include "sanitizer_common/sanitizer_mutex.h"
24 #include "tsan_flags.h"
25 #include "tsan_interface.h"
26 #include "tsan_rtl.h"
27 
28 using namespace __tsan;
29 
30 #if !SANITIZER_GO && __TSAN_HAS_INT128
31 // Protects emulation of 128-bit atomic operations.
32 static StaticSpinMutex mutex128;
33 #endif
34 
35 static bool IsLoadOrder(morder mo) {
36   return mo == mo_relaxed || mo == mo_consume
37       || mo == mo_acquire || mo == mo_seq_cst;
38 }
39 
40 static bool IsStoreOrder(morder mo) {
41   return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
42 }
43 
44 static bool IsReleaseOrder(morder mo) {
45   return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
46 }
47 
48 static bool IsAcquireOrder(morder mo) {
49   return mo == mo_consume || mo == mo_acquire
50       || mo == mo_acq_rel || mo == mo_seq_cst;
51 }
52 
53 static bool IsAcqRelOrder(morder mo) {
54   return mo == mo_acq_rel || mo == mo_seq_cst;
55 }
56 
57 template<typename T> T func_xchg(volatile T *v, T op) {
58   T res = __sync_lock_test_and_set(v, op);
59   // __sync_lock_test_and_set does not contain full barrier.
60   __sync_synchronize();
61   return res;
62 }
63 
64 template<typename T> T func_add(volatile T *v, T op) {
65   return __sync_fetch_and_add(v, op);
66 }
67 
68 template<typename T> T func_sub(volatile T *v, T op) {
69   return __sync_fetch_and_sub(v, op);
70 }
71 
72 template<typename T> T func_and(volatile T *v, T op) {
73   return __sync_fetch_and_and(v, op);
74 }
75 
76 template<typename T> T func_or(volatile T *v, T op) {
77   return __sync_fetch_and_or(v, op);
78 }
79 
80 template<typename T> T func_xor(volatile T *v, T op) {
81   return __sync_fetch_and_xor(v, op);
82 }
83 
84 template<typename T> T func_nand(volatile T *v, T op) {
85   // clang does not support __sync_fetch_and_nand.
86   T cmp = *v;
87   for (;;) {
88     T newv = ~(cmp & op);
89     T cur = __sync_val_compare_and_swap(v, cmp, newv);
90     if (cmp == cur)
91       return cmp;
92     cmp = cur;
93   }
94 }
95 
96 template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
97   return __sync_val_compare_and_swap(v, cmp, xch);
98 }
99 
100 // clang does not support 128-bit atomic ops.
101 // Atomic ops are executed under tsan internal mutex,
102 // here we assume that the atomic variables are not accessed
103 // from non-instrumented code.
104 #if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \
105     && __TSAN_HAS_INT128
106 a128 func_xchg(volatile a128 *v, a128 op) {
107   SpinMutexLock lock(&mutex128);
108   a128 cmp = *v;
109   *v = op;
110   return cmp;
111 }
112 
113 a128 func_add(volatile a128 *v, a128 op) {
114   SpinMutexLock lock(&mutex128);
115   a128 cmp = *v;
116   *v = cmp + op;
117   return cmp;
118 }
119 
120 a128 func_sub(volatile a128 *v, a128 op) {
121   SpinMutexLock lock(&mutex128);
122   a128 cmp = *v;
123   *v = cmp - op;
124   return cmp;
125 }
126 
127 a128 func_and(volatile a128 *v, a128 op) {
128   SpinMutexLock lock(&mutex128);
129   a128 cmp = *v;
130   *v = cmp & op;
131   return cmp;
132 }
133 
134 a128 func_or(volatile a128 *v, a128 op) {
135   SpinMutexLock lock(&mutex128);
136   a128 cmp = *v;
137   *v = cmp | op;
138   return cmp;
139 }
140 
141 a128 func_xor(volatile a128 *v, a128 op) {
142   SpinMutexLock lock(&mutex128);
143   a128 cmp = *v;
144   *v = cmp ^ op;
145   return cmp;
146 }
147 
148 a128 func_nand(volatile a128 *v, a128 op) {
149   SpinMutexLock lock(&mutex128);
150   a128 cmp = *v;
151   *v = ~(cmp & op);
152   return cmp;
153 }
154 
155 a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
156   SpinMutexLock lock(&mutex128);
157   a128 cur = *v;
158   if (cur == cmp)
159     *v = xch;
160   return cur;
161 }
162 #endif
163 
164 template<typename T>
165 static int SizeLog() {
166   if (sizeof(T) <= 1)
167     return kSizeLog1;
168   else if (sizeof(T) <= 2)
169     return kSizeLog2;
170   else if (sizeof(T) <= 4)
171     return kSizeLog4;
172   else
173     return kSizeLog8;
174   // For 16-byte atomics we also use 8-byte memory access,
175   // this leads to false negatives only in very obscure cases.
176 }
177 
178 #if !SANITIZER_GO
179 static atomic_uint8_t *to_atomic(const volatile a8 *a) {
180   return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a));
181 }
182 
183 static atomic_uint16_t *to_atomic(const volatile a16 *a) {
184   return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a));
185 }
186 #endif
187 
188 static atomic_uint32_t *to_atomic(const volatile a32 *a) {
189   return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a));
190 }
191 
192 static atomic_uint64_t *to_atomic(const volatile a64 *a) {
193   return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a));
194 }
195 
196 static memory_order to_mo(morder mo) {
197   switch (mo) {
198   case mo_relaxed: return memory_order_relaxed;
199   case mo_consume: return memory_order_consume;
200   case mo_acquire: return memory_order_acquire;
201   case mo_release: return memory_order_release;
202   case mo_acq_rel: return memory_order_acq_rel;
203   case mo_seq_cst: return memory_order_seq_cst;
204   }
205   CHECK(0);
206   return memory_order_seq_cst;
207 }
208 
209 template<typename T>
210 static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
211   return atomic_load(to_atomic(a), to_mo(mo));
212 }
213 
214 #if __TSAN_HAS_INT128 && !SANITIZER_GO
215 static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
216   SpinMutexLock lock(&mutex128);
217   return *a;
218 }
219 #endif
220 
221 template <typename T>
222 static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
223                     morder mo) NO_THREAD_SAFETY_ANALYSIS {
224   CHECK(IsLoadOrder(mo));
225   // This fast-path is critical for performance.
226   // Assume the access is atomic.
227   if (!IsAcquireOrder(mo)) {
228     MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
229     return NoTsanAtomicLoad(a, mo);
230   }
231   // Don't create sync object if it does not exist yet. For example, an atomic
232   // pointer is initialized to nullptr and then periodically acquire-loaded.
233   T v = NoTsanAtomicLoad(a, mo);
234   SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false);
235   if (s) {
236     AcquireImpl(thr, pc, &s->clock);
237     // Re-read under sync mutex because we need a consistent snapshot
238     // of the value and the clock we acquire.
239     v = NoTsanAtomicLoad(a, mo);
240     s->mtx.ReadUnlock();
241   }
242   MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
243   return v;
244 }
245 
246 template<typename T>
247 static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
248   atomic_store(to_atomic(a), v, to_mo(mo));
249 }
250 
251 #if __TSAN_HAS_INT128 && !SANITIZER_GO
252 static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
253   SpinMutexLock lock(&mutex128);
254   *a = v;
255 }
256 #endif
257 
258 template <typename T>
259 static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
260                         morder mo) NO_THREAD_SAFETY_ANALYSIS {
261   CHECK(IsStoreOrder(mo));
262   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
263   // This fast-path is critical for performance.
264   // Assume the access is atomic.
265   // Strictly saying even relaxed store cuts off release sequence,
266   // so must reset the clock.
267   if (!IsReleaseOrder(mo)) {
268     NoTsanAtomicStore(a, v, mo);
269     return;
270   }
271   __sync_synchronize();
272   SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
273   thr->fast_state.IncrementEpoch();
274   // Can't increment epoch w/o writing to the trace as well.
275   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
276   ReleaseStoreImpl(thr, pc, &s->clock);
277   NoTsanAtomicStore(a, v, mo);
278   s->mtx.Unlock();
279 }
280 
281 template <typename T, T (*F)(volatile T *v, T op)>
282 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v,
283                    morder mo) NO_THREAD_SAFETY_ANALYSIS {
284   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
285   SyncVar *s = 0;
286   if (mo != mo_relaxed) {
287     s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
288     thr->fast_state.IncrementEpoch();
289     // Can't increment epoch w/o writing to the trace as well.
290     TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
291     if (IsAcqRelOrder(mo))
292       AcquireReleaseImpl(thr, pc, &s->clock);
293     else if (IsReleaseOrder(mo))
294       ReleaseImpl(thr, pc, &s->clock);
295     else if (IsAcquireOrder(mo))
296       AcquireImpl(thr, pc, &s->clock);
297   }
298   v = F(a, v);
299   if (s)
300     s->mtx.Unlock();
301   return v;
302 }
303 
304 template<typename T>
305 static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {
306   return func_xchg(a, v);
307 }
308 
309 template<typename T>
310 static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {
311   return func_add(a, v);
312 }
313 
314 template<typename T>
315 static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {
316   return func_sub(a, v);
317 }
318 
319 template<typename T>
320 static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {
321   return func_and(a, v);
322 }
323 
324 template<typename T>
325 static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {
326   return func_or(a, v);
327 }
328 
329 template<typename T>
330 static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {
331   return func_xor(a, v);
332 }
333 
334 template<typename T>
335 static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {
336   return func_nand(a, v);
337 }
338 
339 template<typename T>
340 static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,
341     morder mo) {
342   return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
343 }
344 
345 template<typename T>
346 static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,
347     morder mo) {
348   return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
349 }
350 
351 template<typename T>
352 static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,
353     morder mo) {
354   return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
355 }
356 
357 template<typename T>
358 static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,
359     morder mo) {
360   return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
361 }
362 
363 template<typename T>
364 static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,
365     morder mo) {
366   return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
367 }
368 
369 template<typename T>
370 static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,
371     morder mo) {
372   return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
373 }
374 
375 template<typename T>
376 static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,
377     morder mo) {
378   return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
379 }
380 
381 template<typename T>
382 static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) {
383   return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
384 }
385 
386 #if __TSAN_HAS_INT128
387 static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v,
388     morder mo, morder fmo) {
389   a128 old = *c;
390   a128 cur = func_cas(a, old, v);
391   if (cur == old)
392     return true;
393   *c = cur;
394   return false;
395 }
396 #endif
397 
398 template<typename T>
399 static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
400   NoTsanAtomicCAS(a, &c, v, mo, fmo);
401   return c;
402 }
403 
404 template <typename T>
405 static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, morder mo,
406                       morder fmo) NO_THREAD_SAFETY_ANALYSIS {
407   // 31.7.2.18: "The failure argument shall not be memory_order_release
408   // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
409   // (mo_relaxed) when those are used.
410   CHECK(IsLoadOrder(fmo));
411 
412   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
413   SyncVar *s = 0;
414   bool write_lock = IsReleaseOrder(mo);
415 
416   if (mo != mo_relaxed || fmo != mo_relaxed)
417     s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);
418 
419   T cc = *c;
420   T pr = func_cas(a, cc, v);
421   bool success = pr == cc;
422   if (!success) {
423     *c = pr;
424     mo = fmo;
425   }
426 
427   if (s) {
428     thr->fast_state.IncrementEpoch();
429     // Can't increment epoch w/o writing to the trace as well.
430     TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
431 
432     if (success && IsAcqRelOrder(mo))
433       AcquireReleaseImpl(thr, pc, &s->clock);
434     else if (success && IsReleaseOrder(mo))
435       ReleaseImpl(thr, pc, &s->clock);
436     else if (IsAcquireOrder(mo))
437       AcquireImpl(thr, pc, &s->clock);
438 
439     if (write_lock)
440       s->mtx.Unlock();
441     else
442       s->mtx.ReadUnlock();
443   }
444 
445   return success;
446 }
447 
448 template<typename T>
449 static T AtomicCAS(ThreadState *thr, uptr pc,
450     volatile T *a, T c, T v, morder mo, morder fmo) {
451   AtomicCAS(thr, pc, a, &c, v, mo, fmo);
452   return c;
453 }
454 
455 #if !SANITIZER_GO
456 static void NoTsanAtomicFence(morder mo) {
457   __sync_synchronize();
458 }
459 
460 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
461   // FIXME(dvyukov): not implemented.
462   __sync_synchronize();
463 }
464 #endif
465 
466 // Interface functions follow.
467 #if !SANITIZER_GO
468 
469 // C/C++
470 
471 static morder convert_morder(morder mo) {
472   if (flags()->force_seq_cst_atomics)
473     return (morder)mo_seq_cst;
474 
475   // Filter out additional memory order flags:
476   // MEMMODEL_SYNC        = 1 << 15
477   // __ATOMIC_HLE_ACQUIRE = 1 << 16
478   // __ATOMIC_HLE_RELEASE = 1 << 17
479   //
480   // HLE is an optimization, and we pretend that elision always fails.
481   // MEMMODEL_SYNC is used when lowering __sync_ atomics,
482   // since we use __sync_ atomics for actual atomic operations,
483   // we can safely ignore it as well. It also subtly affects semantics,
484   // but we don't model the difference.
485   return (morder)(mo & 0x7fff);
486 }
487 
488 #define SCOPED_ATOMIC(func, ...) \
489     ThreadState *const thr = cur_thread(); \
490     if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \
491       ProcessPendingSignals(thr); \
492       return NoTsanAtomic##func(__VA_ARGS__); \
493     } \
494     const uptr callpc = (uptr)__builtin_return_address(0); \
495     uptr pc = StackTrace::GetCurrentPc(); \
496     mo = convert_morder(mo); \
497     ScopedAtomic sa(thr, callpc, a, mo, __func__); \
498     return Atomic##func(thr, pc, __VA_ARGS__); \
499 /**/
500 
501 class ScopedAtomic {
502  public:
503   ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a,
504                morder mo, const char *func)
505       : thr_(thr) {
506     FuncEntry(thr_, pc);
507     DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo);
508   }
509   ~ScopedAtomic() {
510     ProcessPendingSignals(thr_);
511     FuncExit(thr_);
512   }
513  private:
514   ThreadState *thr_;
515 };
516 
517 extern "C" {
518 SANITIZER_INTERFACE_ATTRIBUTE
519 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
520   SCOPED_ATOMIC(Load, a, mo);
521 }
522 
523 SANITIZER_INTERFACE_ATTRIBUTE
524 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
525   SCOPED_ATOMIC(Load, a, mo);
526 }
527 
528 SANITIZER_INTERFACE_ATTRIBUTE
529 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
530   SCOPED_ATOMIC(Load, a, mo);
531 }
532 
533 SANITIZER_INTERFACE_ATTRIBUTE
534 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
535   SCOPED_ATOMIC(Load, a, mo);
536 }
537 
538 #if __TSAN_HAS_INT128
539 SANITIZER_INTERFACE_ATTRIBUTE
540 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
541   SCOPED_ATOMIC(Load, a, mo);
542 }
543 #endif
544 
545 SANITIZER_INTERFACE_ATTRIBUTE
546 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
547   SCOPED_ATOMIC(Store, a, v, mo);
548 }
549 
550 SANITIZER_INTERFACE_ATTRIBUTE
551 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
552   SCOPED_ATOMIC(Store, a, v, mo);
553 }
554 
555 SANITIZER_INTERFACE_ATTRIBUTE
556 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
557   SCOPED_ATOMIC(Store, a, v, mo);
558 }
559 
560 SANITIZER_INTERFACE_ATTRIBUTE
561 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
562   SCOPED_ATOMIC(Store, a, v, mo);
563 }
564 
565 #if __TSAN_HAS_INT128
566 SANITIZER_INTERFACE_ATTRIBUTE
567 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
568   SCOPED_ATOMIC(Store, a, v, mo);
569 }
570 #endif
571 
572 SANITIZER_INTERFACE_ATTRIBUTE
573 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
574   SCOPED_ATOMIC(Exchange, a, v, mo);
575 }
576 
577 SANITIZER_INTERFACE_ATTRIBUTE
578 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
579   SCOPED_ATOMIC(Exchange, a, v, mo);
580 }
581 
582 SANITIZER_INTERFACE_ATTRIBUTE
583 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
584   SCOPED_ATOMIC(Exchange, a, v, mo);
585 }
586 
587 SANITIZER_INTERFACE_ATTRIBUTE
588 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
589   SCOPED_ATOMIC(Exchange, a, v, mo);
590 }
591 
592 #if __TSAN_HAS_INT128
593 SANITIZER_INTERFACE_ATTRIBUTE
594 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
595   SCOPED_ATOMIC(Exchange, a, v, mo);
596 }
597 #endif
598 
599 SANITIZER_INTERFACE_ATTRIBUTE
600 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
601   SCOPED_ATOMIC(FetchAdd, a, v, mo);
602 }
603 
604 SANITIZER_INTERFACE_ATTRIBUTE
605 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
606   SCOPED_ATOMIC(FetchAdd, a, v, mo);
607 }
608 
609 SANITIZER_INTERFACE_ATTRIBUTE
610 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
611   SCOPED_ATOMIC(FetchAdd, a, v, mo);
612 }
613 
614 SANITIZER_INTERFACE_ATTRIBUTE
615 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
616   SCOPED_ATOMIC(FetchAdd, a, v, mo);
617 }
618 
619 #if __TSAN_HAS_INT128
620 SANITIZER_INTERFACE_ATTRIBUTE
621 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
622   SCOPED_ATOMIC(FetchAdd, a, v, mo);
623 }
624 #endif
625 
626 SANITIZER_INTERFACE_ATTRIBUTE
627 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
628   SCOPED_ATOMIC(FetchSub, a, v, mo);
629 }
630 
631 SANITIZER_INTERFACE_ATTRIBUTE
632 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
633   SCOPED_ATOMIC(FetchSub, a, v, mo);
634 }
635 
636 SANITIZER_INTERFACE_ATTRIBUTE
637 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
638   SCOPED_ATOMIC(FetchSub, a, v, mo);
639 }
640 
641 SANITIZER_INTERFACE_ATTRIBUTE
642 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
643   SCOPED_ATOMIC(FetchSub, a, v, mo);
644 }
645 
646 #if __TSAN_HAS_INT128
647 SANITIZER_INTERFACE_ATTRIBUTE
648 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
649   SCOPED_ATOMIC(FetchSub, a, v, mo);
650 }
651 #endif
652 
653 SANITIZER_INTERFACE_ATTRIBUTE
654 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
655   SCOPED_ATOMIC(FetchAnd, a, v, mo);
656 }
657 
658 SANITIZER_INTERFACE_ATTRIBUTE
659 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
660   SCOPED_ATOMIC(FetchAnd, a, v, mo);
661 }
662 
663 SANITIZER_INTERFACE_ATTRIBUTE
664 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
665   SCOPED_ATOMIC(FetchAnd, a, v, mo);
666 }
667 
668 SANITIZER_INTERFACE_ATTRIBUTE
669 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
670   SCOPED_ATOMIC(FetchAnd, a, v, mo);
671 }
672 
673 #if __TSAN_HAS_INT128
674 SANITIZER_INTERFACE_ATTRIBUTE
675 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
676   SCOPED_ATOMIC(FetchAnd, a, v, mo);
677 }
678 #endif
679 
680 SANITIZER_INTERFACE_ATTRIBUTE
681 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
682   SCOPED_ATOMIC(FetchOr, a, v, mo);
683 }
684 
685 SANITIZER_INTERFACE_ATTRIBUTE
686 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
687   SCOPED_ATOMIC(FetchOr, a, v, mo);
688 }
689 
690 SANITIZER_INTERFACE_ATTRIBUTE
691 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
692   SCOPED_ATOMIC(FetchOr, a, v, mo);
693 }
694 
695 SANITIZER_INTERFACE_ATTRIBUTE
696 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
697   SCOPED_ATOMIC(FetchOr, a, v, mo);
698 }
699 
700 #if __TSAN_HAS_INT128
701 SANITIZER_INTERFACE_ATTRIBUTE
702 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
703   SCOPED_ATOMIC(FetchOr, a, v, mo);
704 }
705 #endif
706 
707 SANITIZER_INTERFACE_ATTRIBUTE
708 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
709   SCOPED_ATOMIC(FetchXor, a, v, mo);
710 }
711 
712 SANITIZER_INTERFACE_ATTRIBUTE
713 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
714   SCOPED_ATOMIC(FetchXor, a, v, mo);
715 }
716 
717 SANITIZER_INTERFACE_ATTRIBUTE
718 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
719   SCOPED_ATOMIC(FetchXor, a, v, mo);
720 }
721 
722 SANITIZER_INTERFACE_ATTRIBUTE
723 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
724   SCOPED_ATOMIC(FetchXor, a, v, mo);
725 }
726 
727 #if __TSAN_HAS_INT128
728 SANITIZER_INTERFACE_ATTRIBUTE
729 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
730   SCOPED_ATOMIC(FetchXor, a, v, mo);
731 }
732 #endif
733 
734 SANITIZER_INTERFACE_ATTRIBUTE
735 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
736   SCOPED_ATOMIC(FetchNand, a, v, mo);
737 }
738 
739 SANITIZER_INTERFACE_ATTRIBUTE
740 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
741   SCOPED_ATOMIC(FetchNand, a, v, mo);
742 }
743 
744 SANITIZER_INTERFACE_ATTRIBUTE
745 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
746   SCOPED_ATOMIC(FetchNand, a, v, mo);
747 }
748 
749 SANITIZER_INTERFACE_ATTRIBUTE
750 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
751   SCOPED_ATOMIC(FetchNand, a, v, mo);
752 }
753 
754 #if __TSAN_HAS_INT128
755 SANITIZER_INTERFACE_ATTRIBUTE
756 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
757   SCOPED_ATOMIC(FetchNand, a, v, mo);
758 }
759 #endif
760 
761 SANITIZER_INTERFACE_ATTRIBUTE
762 int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
763     morder mo, morder fmo) {
764   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
765 }
766 
767 SANITIZER_INTERFACE_ATTRIBUTE
768 int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
769     morder mo, morder fmo) {
770   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
771 }
772 
773 SANITIZER_INTERFACE_ATTRIBUTE
774 int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
775     morder mo, morder fmo) {
776   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
777 }
778 
779 SANITIZER_INTERFACE_ATTRIBUTE
780 int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
781     morder mo, morder fmo) {
782   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
783 }
784 
785 #if __TSAN_HAS_INT128
786 SANITIZER_INTERFACE_ATTRIBUTE
787 int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
788     morder mo, morder fmo) {
789   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
790 }
791 #endif
792 
793 SANITIZER_INTERFACE_ATTRIBUTE
794 int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
795     morder mo, morder fmo) {
796   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
797 }
798 
799 SANITIZER_INTERFACE_ATTRIBUTE
800 int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
801     morder mo, morder fmo) {
802   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
803 }
804 
805 SANITIZER_INTERFACE_ATTRIBUTE
806 int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
807     morder mo, morder fmo) {
808   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
809 }
810 
811 SANITIZER_INTERFACE_ATTRIBUTE
812 int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
813     morder mo, morder fmo) {
814   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
815 }
816 
817 #if __TSAN_HAS_INT128
818 SANITIZER_INTERFACE_ATTRIBUTE
819 int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
820     morder mo, morder fmo) {
821   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
822 }
823 #endif
824 
825 SANITIZER_INTERFACE_ATTRIBUTE
826 a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
827     morder mo, morder fmo) {
828   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
829 }
830 
831 SANITIZER_INTERFACE_ATTRIBUTE
832 a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
833     morder mo, morder fmo) {
834   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
835 }
836 
837 SANITIZER_INTERFACE_ATTRIBUTE
838 a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
839     morder mo, morder fmo) {
840   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
841 }
842 
843 SANITIZER_INTERFACE_ATTRIBUTE
844 a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
845     morder mo, morder fmo) {
846   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
847 }
848 
849 #if __TSAN_HAS_INT128
850 SANITIZER_INTERFACE_ATTRIBUTE
851 a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
852     morder mo, morder fmo) {
853   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
854 }
855 #endif
856 
857 SANITIZER_INTERFACE_ATTRIBUTE
858 void __tsan_atomic_thread_fence(morder mo) {
859   char* a = 0;
860   SCOPED_ATOMIC(Fence, mo);
861 }
862 
863 SANITIZER_INTERFACE_ATTRIBUTE
864 void __tsan_atomic_signal_fence(morder mo) {
865 }
866 }  // extern "C"
867 
868 #else  // #if !SANITIZER_GO
869 
870 // Go
871 
872 #define ATOMIC(func, ...) \
873     if (thr->ignore_sync) { \
874       NoTsanAtomic##func(__VA_ARGS__); \
875     } else { \
876       FuncEntry(thr, cpc); \
877       Atomic##func(thr, pc, __VA_ARGS__); \
878       FuncExit(thr); \
879     } \
880 /**/
881 
882 #define ATOMIC_RET(func, ret, ...) \
883     if (thr->ignore_sync) { \
884       (ret) = NoTsanAtomic##func(__VA_ARGS__); \
885     } else { \
886       FuncEntry(thr, cpc); \
887       (ret) = Atomic##func(thr, pc, __VA_ARGS__); \
888       FuncExit(thr); \
889     } \
890 /**/
891 
892 extern "C" {
893 SANITIZER_INTERFACE_ATTRIBUTE
894 void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
895   ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire);
896 }
897 
898 SANITIZER_INTERFACE_ATTRIBUTE
899 void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
900   ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire);
901 }
902 
903 SANITIZER_INTERFACE_ATTRIBUTE
904 void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
905   ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release);
906 }
907 
908 SANITIZER_INTERFACE_ATTRIBUTE
909 void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
910   ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release);
911 }
912 
913 SANITIZER_INTERFACE_ATTRIBUTE
914 void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
915   ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
916 }
917 
918 SANITIZER_INTERFACE_ATTRIBUTE
919 void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
920   ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
921 }
922 
923 SANITIZER_INTERFACE_ATTRIBUTE
924 void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
925   ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
926 }
927 
928 SANITIZER_INTERFACE_ATTRIBUTE
929 void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
930   ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
931 }
932 
933 SANITIZER_INTERFACE_ATTRIBUTE
934 void __tsan_go_atomic32_compare_exchange(
935     ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
936   a32 cur = 0;
937   a32 cmp = *(a32*)(a+8);
938   ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire);
939   *(bool*)(a+16) = (cur == cmp);
940 }
941 
942 SANITIZER_INTERFACE_ATTRIBUTE
943 void __tsan_go_atomic64_compare_exchange(
944     ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
945   a64 cur = 0;
946   a64 cmp = *(a64*)(a+8);
947   ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire);
948   *(bool*)(a+24) = (cur == cmp);
949 }
950 }  // extern "C"
951 #endif  // #if !SANITIZER_GO
952