xref: /freebsd/contrib/llvm-project/libcxx/include/atomic (revision 5ffd83dbcc34f10e07f6d3e968ae6365869615f4)
10b57cec5SDimitry Andric// -*- C++ -*-
20b57cec5SDimitry Andric//===--------------------------- atomic -----------------------------------===//
30b57cec5SDimitry Andric//
40b57cec5SDimitry Andric// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
50b57cec5SDimitry Andric// See https://llvm.org/LICENSE.txt for license information.
60b57cec5SDimitry Andric// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
70b57cec5SDimitry Andric//
80b57cec5SDimitry Andric//===----------------------------------------------------------------------===//
90b57cec5SDimitry Andric
100b57cec5SDimitry Andric#ifndef _LIBCPP_ATOMIC
110b57cec5SDimitry Andric#define _LIBCPP_ATOMIC
120b57cec5SDimitry Andric
130b57cec5SDimitry Andric/*
140b57cec5SDimitry Andric    atomic synopsis
150b57cec5SDimitry Andric
160b57cec5SDimitry Andricnamespace std
170b57cec5SDimitry Andric{
180b57cec5SDimitry Andric
190b57cec5SDimitry Andric// feature test macro
200b57cec5SDimitry Andric
210b57cec5SDimitry Andric#define __cpp_lib_atomic_is_always_lock_free // as specified by SG10
220b57cec5SDimitry Andric
230b57cec5SDimitry Andric // order and consistency
240b57cec5SDimitry Andric
250b57cec5SDimitry Andric enum memory_order: unspecified // enum class in C++20
260b57cec5SDimitry Andric {
270b57cec5SDimitry Andric    relaxed,
280b57cec5SDimitry Andric    consume, // load-consume
290b57cec5SDimitry Andric    acquire, // load-acquire
300b57cec5SDimitry Andric    release, // store-release
310b57cec5SDimitry Andric    acq_rel, // store-release load-acquire
320b57cec5SDimitry Andric    seq_cst // store-release load-acquire
330b57cec5SDimitry Andric };
340b57cec5SDimitry Andric
350b57cec5SDimitry Andric inline constexpr auto memory_order_relaxed = memory_order::relaxed;
360b57cec5SDimitry Andric inline constexpr auto memory_order_consume = memory_order::consume;
370b57cec5SDimitry Andric inline constexpr auto memory_order_acquire = memory_order::acquire;
380b57cec5SDimitry Andric inline constexpr auto memory_order_release = memory_order::release;
390b57cec5SDimitry Andric inline constexpr auto memory_order_acq_rel = memory_order::acq_rel;
400b57cec5SDimitry Andric inline constexpr auto memory_order_seq_cst = memory_order::seq_cst;
410b57cec5SDimitry Andric
420b57cec5SDimitry Andrictemplate <class T> T kill_dependency(T y) noexcept;
430b57cec5SDimitry Andric
440b57cec5SDimitry Andric// lock-free property
450b57cec5SDimitry Andric
460b57cec5SDimitry Andric#define ATOMIC_BOOL_LOCK_FREE unspecified
470b57cec5SDimitry Andric#define ATOMIC_CHAR_LOCK_FREE unspecified
480b57cec5SDimitry Andric#define ATOMIC_CHAR16_T_LOCK_FREE unspecified
490b57cec5SDimitry Andric#define ATOMIC_CHAR32_T_LOCK_FREE unspecified
500b57cec5SDimitry Andric#define ATOMIC_WCHAR_T_LOCK_FREE unspecified
510b57cec5SDimitry Andric#define ATOMIC_SHORT_LOCK_FREE unspecified
520b57cec5SDimitry Andric#define ATOMIC_INT_LOCK_FREE unspecified
530b57cec5SDimitry Andric#define ATOMIC_LONG_LOCK_FREE unspecified
540b57cec5SDimitry Andric#define ATOMIC_LLONG_LOCK_FREE unspecified
550b57cec5SDimitry Andric#define ATOMIC_POINTER_LOCK_FREE unspecified
560b57cec5SDimitry Andric
570b57cec5SDimitry Andrictemplate <class T>
580b57cec5SDimitry Andricstruct atomic
590b57cec5SDimitry Andric{
60*5ffd83dbSDimitry Andric    using value_type = T;
61*5ffd83dbSDimitry Andric
620b57cec5SDimitry Andric    static constexpr bool is_always_lock_free;
630b57cec5SDimitry Andric    bool is_lock_free() const volatile noexcept;
640b57cec5SDimitry Andric    bool is_lock_free() const noexcept;
65*5ffd83dbSDimitry Andric
66*5ffd83dbSDimitry Andric    atomic() noexcept = default;
67*5ffd83dbSDimitry Andric    constexpr atomic(T desr) noexcept;
68*5ffd83dbSDimitry Andric    atomic(const atomic&) = delete;
69*5ffd83dbSDimitry Andric    atomic& operator=(const atomic&) = delete;
70*5ffd83dbSDimitry Andric    atomic& operator=(const atomic&) volatile = delete;
71*5ffd83dbSDimitry Andric
720b57cec5SDimitry Andric    T load(memory_order m = memory_order_seq_cst) const volatile noexcept;
730b57cec5SDimitry Andric    T load(memory_order m = memory_order_seq_cst) const noexcept;
740b57cec5SDimitry Andric    operator T() const volatile noexcept;
750b57cec5SDimitry Andric    operator T() const noexcept;
76*5ffd83dbSDimitry Andric    void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;
77*5ffd83dbSDimitry Andric    void store(T desr, memory_order m = memory_order_seq_cst) noexcept;
78*5ffd83dbSDimitry Andric    T operator=(T) volatile noexcept;
79*5ffd83dbSDimitry Andric    T operator=(T) noexcept;
80*5ffd83dbSDimitry Andric
810b57cec5SDimitry Andric    T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;
820b57cec5SDimitry Andric    T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept;
830b57cec5SDimitry Andric    bool compare_exchange_weak(T& expc, T desr,
840b57cec5SDimitry Andric                               memory_order s, memory_order f) volatile noexcept;
850b57cec5SDimitry Andric    bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept;
860b57cec5SDimitry Andric    bool compare_exchange_strong(T& expc, T desr,
870b57cec5SDimitry Andric                                 memory_order s, memory_order f) volatile noexcept;
880b57cec5SDimitry Andric    bool compare_exchange_strong(T& expc, T desr,
890b57cec5SDimitry Andric                                 memory_order s, memory_order f) noexcept;
900b57cec5SDimitry Andric    bool compare_exchange_weak(T& expc, T desr,
910b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) volatile noexcept;
920b57cec5SDimitry Andric    bool compare_exchange_weak(T& expc, T desr,
930b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) noexcept;
940b57cec5SDimitry Andric    bool compare_exchange_strong(T& expc, T desr,
950b57cec5SDimitry Andric                                memory_order m = memory_order_seq_cst) volatile noexcept;
960b57cec5SDimitry Andric    bool compare_exchange_strong(T& expc, T desr,
970b57cec5SDimitry Andric                                 memory_order m = memory_order_seq_cst) noexcept;
980b57cec5SDimitry Andric
99*5ffd83dbSDimitry Andric    void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept;
100*5ffd83dbSDimitry Andric    void wait(T, memory_order = memory_order::seq_cst) const noexcept;
101*5ffd83dbSDimitry Andric    void notify_one() volatile noexcept;
102*5ffd83dbSDimitry Andric    void notify_one() noexcept;
103*5ffd83dbSDimitry Andric    void notify_all() volatile noexcept;
104*5ffd83dbSDimitry Andric    void notify_all() noexcept;
1050b57cec5SDimitry Andric};
1060b57cec5SDimitry Andric
1070b57cec5SDimitry Andrictemplate <>
1080b57cec5SDimitry Andricstruct atomic<integral>
1090b57cec5SDimitry Andric{
110*5ffd83dbSDimitry Andric    using value_type = integral;
111*5ffd83dbSDimitry Andric
1120b57cec5SDimitry Andric    static constexpr bool is_always_lock_free;
1130b57cec5SDimitry Andric    bool is_lock_free() const volatile noexcept;
1140b57cec5SDimitry Andric    bool is_lock_free() const noexcept;
115*5ffd83dbSDimitry Andric
116*5ffd83dbSDimitry Andric    atomic() noexcept = default;
117*5ffd83dbSDimitry Andric    constexpr atomic(integral desr) noexcept;
118*5ffd83dbSDimitry Andric    atomic(const atomic&) = delete;
119*5ffd83dbSDimitry Andric    atomic& operator=(const atomic&) = delete;
120*5ffd83dbSDimitry Andric    atomic& operator=(const atomic&) volatile = delete;
121*5ffd83dbSDimitry Andric
1220b57cec5SDimitry Andric    integral load(memory_order m = memory_order_seq_cst) const volatile noexcept;
1230b57cec5SDimitry Andric    integral load(memory_order m = memory_order_seq_cst) const noexcept;
1240b57cec5SDimitry Andric    operator integral() const volatile noexcept;
1250b57cec5SDimitry Andric    operator integral() const noexcept;
126*5ffd83dbSDimitry Andric    void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept;
127*5ffd83dbSDimitry Andric    void store(integral desr, memory_order m = memory_order_seq_cst) noexcept;
128*5ffd83dbSDimitry Andric    integral operator=(integral desr) volatile noexcept;
129*5ffd83dbSDimitry Andric    integral operator=(integral desr) noexcept;
130*5ffd83dbSDimitry Andric
1310b57cec5SDimitry Andric    integral exchange(integral desr,
1320b57cec5SDimitry Andric                      memory_order m = memory_order_seq_cst) volatile noexcept;
1330b57cec5SDimitry Andric    integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept;
1340b57cec5SDimitry Andric    bool compare_exchange_weak(integral& expc, integral desr,
1350b57cec5SDimitry Andric                               memory_order s, memory_order f) volatile noexcept;
1360b57cec5SDimitry Andric    bool compare_exchange_weak(integral& expc, integral desr,
1370b57cec5SDimitry Andric                               memory_order s, memory_order f) noexcept;
1380b57cec5SDimitry Andric    bool compare_exchange_strong(integral& expc, integral desr,
1390b57cec5SDimitry Andric                                 memory_order s, memory_order f) volatile noexcept;
1400b57cec5SDimitry Andric    bool compare_exchange_strong(integral& expc, integral desr,
1410b57cec5SDimitry Andric                                 memory_order s, memory_order f) noexcept;
1420b57cec5SDimitry Andric    bool compare_exchange_weak(integral& expc, integral desr,
1430b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) volatile noexcept;
1440b57cec5SDimitry Andric    bool compare_exchange_weak(integral& expc, integral desr,
1450b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) noexcept;
1460b57cec5SDimitry Andric    bool compare_exchange_strong(integral& expc, integral desr,
1470b57cec5SDimitry Andric                                memory_order m = memory_order_seq_cst) volatile noexcept;
1480b57cec5SDimitry Andric    bool compare_exchange_strong(integral& expc, integral desr,
1490b57cec5SDimitry Andric                                 memory_order m = memory_order_seq_cst) noexcept;
1500b57cec5SDimitry Andric
151*5ffd83dbSDimitry Andric    integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
1520b57cec5SDimitry Andric    integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept;
153*5ffd83dbSDimitry Andric    integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
1540b57cec5SDimitry Andric    integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept;
155*5ffd83dbSDimitry Andric    integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
1560b57cec5SDimitry Andric    integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept;
157*5ffd83dbSDimitry Andric    integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
1580b57cec5SDimitry Andric    integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept;
159*5ffd83dbSDimitry Andric    integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
1600b57cec5SDimitry Andric    integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept;
1610b57cec5SDimitry Andric
1620b57cec5SDimitry Andric    integral operator++(int) volatile noexcept;
1630b57cec5SDimitry Andric    integral operator++(int) noexcept;
1640b57cec5SDimitry Andric    integral operator--(int) volatile noexcept;
1650b57cec5SDimitry Andric    integral operator--(int) noexcept;
1660b57cec5SDimitry Andric    integral operator++() volatile noexcept;
1670b57cec5SDimitry Andric    integral operator++() noexcept;
1680b57cec5SDimitry Andric    integral operator--() volatile noexcept;
1690b57cec5SDimitry Andric    integral operator--() noexcept;
1700b57cec5SDimitry Andric    integral operator+=(integral op) volatile noexcept;
1710b57cec5SDimitry Andric    integral operator+=(integral op) noexcept;
1720b57cec5SDimitry Andric    integral operator-=(integral op) volatile noexcept;
1730b57cec5SDimitry Andric    integral operator-=(integral op) noexcept;
1740b57cec5SDimitry Andric    integral operator&=(integral op) volatile noexcept;
1750b57cec5SDimitry Andric    integral operator&=(integral op) noexcept;
1760b57cec5SDimitry Andric    integral operator|=(integral op) volatile noexcept;
1770b57cec5SDimitry Andric    integral operator|=(integral op) noexcept;
1780b57cec5SDimitry Andric    integral operator^=(integral op) volatile noexcept;
1790b57cec5SDimitry Andric    integral operator^=(integral op) noexcept;
180*5ffd83dbSDimitry Andric
181*5ffd83dbSDimitry Andric    void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept;
182*5ffd83dbSDimitry Andric    void wait(integral, memory_order = memory_order::seq_cst) const noexcept;
183*5ffd83dbSDimitry Andric    void notify_one() volatile noexcept;
184*5ffd83dbSDimitry Andric    void notify_one() noexcept;
185*5ffd83dbSDimitry Andric    void notify_all() volatile noexcept;
186*5ffd83dbSDimitry Andric    void notify_all() noexcept;
1870b57cec5SDimitry Andric};
1880b57cec5SDimitry Andric
1890b57cec5SDimitry Andrictemplate <class T>
1900b57cec5SDimitry Andricstruct atomic<T*>
1910b57cec5SDimitry Andric{
192*5ffd83dbSDimitry Andric    using value_type = T*;
193*5ffd83dbSDimitry Andric
1940b57cec5SDimitry Andric    static constexpr bool is_always_lock_free;
1950b57cec5SDimitry Andric    bool is_lock_free() const volatile noexcept;
1960b57cec5SDimitry Andric    bool is_lock_free() const noexcept;
197*5ffd83dbSDimitry Andric
198*5ffd83dbSDimitry Andric    atomic() noexcept = default;
199*5ffd83dbSDimitry Andric    constexpr atomic(T* desr) noexcept;
200*5ffd83dbSDimitry Andric    atomic(const atomic&) = delete;
201*5ffd83dbSDimitry Andric    atomic& operator=(const atomic&) = delete;
202*5ffd83dbSDimitry Andric    atomic& operator=(const atomic&) volatile = delete;
203*5ffd83dbSDimitry Andric
2040b57cec5SDimitry Andric    T* load(memory_order m = memory_order_seq_cst) const volatile noexcept;
2050b57cec5SDimitry Andric    T* load(memory_order m = memory_order_seq_cst) const noexcept;
2060b57cec5SDimitry Andric    operator T*() const volatile noexcept;
2070b57cec5SDimitry Andric    operator T*() const noexcept;
208*5ffd83dbSDimitry Andric    void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;
209*5ffd83dbSDimitry Andric    void store(T* desr, memory_order m = memory_order_seq_cst) noexcept;
210*5ffd83dbSDimitry Andric    T* operator=(T*) volatile noexcept;
211*5ffd83dbSDimitry Andric    T* operator=(T*) noexcept;
212*5ffd83dbSDimitry Andric
2130b57cec5SDimitry Andric    T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;
2140b57cec5SDimitry Andric    T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept;
2150b57cec5SDimitry Andric    bool compare_exchange_weak(T*& expc, T* desr,
2160b57cec5SDimitry Andric                               memory_order s, memory_order f) volatile noexcept;
2170b57cec5SDimitry Andric    bool compare_exchange_weak(T*& expc, T* desr,
2180b57cec5SDimitry Andric                               memory_order s, memory_order f) noexcept;
2190b57cec5SDimitry Andric    bool compare_exchange_strong(T*& expc, T* desr,
2200b57cec5SDimitry Andric                                 memory_order s, memory_order f) volatile noexcept;
2210b57cec5SDimitry Andric    bool compare_exchange_strong(T*& expc, T* desr,
2220b57cec5SDimitry Andric                                 memory_order s, memory_order f) noexcept;
2230b57cec5SDimitry Andric    bool compare_exchange_weak(T*& expc, T* desr,
2240b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) volatile noexcept;
2250b57cec5SDimitry Andric    bool compare_exchange_weak(T*& expc, T* desr,
2260b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) noexcept;
2270b57cec5SDimitry Andric    bool compare_exchange_strong(T*& expc, T* desr,
2280b57cec5SDimitry Andric                                memory_order m = memory_order_seq_cst) volatile noexcept;
2290b57cec5SDimitry Andric    bool compare_exchange_strong(T*& expc, T* desr,
2300b57cec5SDimitry Andric                                 memory_order m = memory_order_seq_cst) noexcept;
2310b57cec5SDimitry Andric    T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;
2320b57cec5SDimitry Andric    T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;
2330b57cec5SDimitry Andric    T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;
2340b57cec5SDimitry Andric    T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;
2350b57cec5SDimitry Andric
2360b57cec5SDimitry Andric    T* operator++(int) volatile noexcept;
2370b57cec5SDimitry Andric    T* operator++(int) noexcept;
2380b57cec5SDimitry Andric    T* operator--(int) volatile noexcept;
2390b57cec5SDimitry Andric    T* operator--(int) noexcept;
2400b57cec5SDimitry Andric    T* operator++() volatile noexcept;
2410b57cec5SDimitry Andric    T* operator++() noexcept;
2420b57cec5SDimitry Andric    T* operator--() volatile noexcept;
2430b57cec5SDimitry Andric    T* operator--() noexcept;
2440b57cec5SDimitry Andric    T* operator+=(ptrdiff_t op) volatile noexcept;
2450b57cec5SDimitry Andric    T* operator+=(ptrdiff_t op) noexcept;
2460b57cec5SDimitry Andric    T* operator-=(ptrdiff_t op) volatile noexcept;
2470b57cec5SDimitry Andric    T* operator-=(ptrdiff_t op) noexcept;
248*5ffd83dbSDimitry Andric
249*5ffd83dbSDimitry Andric    void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept;
250*5ffd83dbSDimitry Andric    void wait(T*, memory_order = memory_order::seq_cst) const noexcept;
251*5ffd83dbSDimitry Andric    void notify_one() volatile noexcept;
252*5ffd83dbSDimitry Andric    void notify_one() noexcept;
253*5ffd83dbSDimitry Andric    void notify_all() volatile noexcept;
254*5ffd83dbSDimitry Andric    void notify_all() noexcept;
2550b57cec5SDimitry Andric};
2560b57cec5SDimitry Andric
2570b57cec5SDimitry Andric
2580b57cec5SDimitry Andrictemplate <class T>
259*5ffd83dbSDimitry Andric  bool atomic_is_lock_free(const volatile atomic<T>* obj) noexcept;
2600b57cec5SDimitry Andric
2610b57cec5SDimitry Andrictemplate <class T>
262*5ffd83dbSDimitry Andric  bool atomic_is_lock_free(const atomic<T>* obj) noexcept;
2630b57cec5SDimitry Andric
2640b57cec5SDimitry Andrictemplate <class T>
265*5ffd83dbSDimitry Andric  void atomic_store(volatile atomic<T>* obj, T desr) noexcept;
2660b57cec5SDimitry Andric
2670b57cec5SDimitry Andrictemplate <class T>
268*5ffd83dbSDimitry Andric  void atomic_store(atomic<T>* obj, T desr) noexcept;
2690b57cec5SDimitry Andric
2700b57cec5SDimitry Andrictemplate <class T>
271*5ffd83dbSDimitry Andric  void atomic_store_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept;
2720b57cec5SDimitry Andric
2730b57cec5SDimitry Andrictemplate <class T>
274*5ffd83dbSDimitry Andric  void atomic_store_explicit(atomic<T>* obj, T desr, memory_order m) noexcept;
2750b57cec5SDimitry Andric
2760b57cec5SDimitry Andrictemplate <class T>
277*5ffd83dbSDimitry Andric  T atomic_load(const volatile atomic<T>* obj) noexcept;
2780b57cec5SDimitry Andric
2790b57cec5SDimitry Andrictemplate <class T>
280*5ffd83dbSDimitry Andric  T atomic_load(const atomic<T>* obj) noexcept;
2810b57cec5SDimitry Andric
2820b57cec5SDimitry Andrictemplate <class T>
283*5ffd83dbSDimitry Andric  T atomic_load_explicit(const volatile atomic<T>* obj, memory_order m) noexcept;
2840b57cec5SDimitry Andric
2850b57cec5SDimitry Andrictemplate <class T>
286*5ffd83dbSDimitry Andric  T atomic_load_explicit(const atomic<T>* obj, memory_order m) noexcept;
2870b57cec5SDimitry Andric
2880b57cec5SDimitry Andrictemplate <class T>
289*5ffd83dbSDimitry Andric  T atomic_exchange(volatile atomic<T>* obj, T desr) noexcept;
2900b57cec5SDimitry Andric
2910b57cec5SDimitry Andrictemplate <class T>
292*5ffd83dbSDimitry Andric  T atomic_exchange(atomic<T>* obj, T desr) noexcept;
2930b57cec5SDimitry Andric
2940b57cec5SDimitry Andrictemplate <class T>
295*5ffd83dbSDimitry Andric  T atomic_exchange_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept;
2960b57cec5SDimitry Andric
2970b57cec5SDimitry Andrictemplate <class T>
298*5ffd83dbSDimitry Andric  T atomic_exchange_explicit(atomic<T>* obj, T desr, memory_order m) noexcept;
2990b57cec5SDimitry Andric
3000b57cec5SDimitry Andrictemplate <class T>
301*5ffd83dbSDimitry Andric  bool atomic_compare_exchange_weak(volatile atomic<T>* obj, T* expc, T desr) noexcept;
3020b57cec5SDimitry Andric
3030b57cec5SDimitry Andrictemplate <class T>
304*5ffd83dbSDimitry Andric  bool atomic_compare_exchange_weak(atomic<T>* obj, T* expc, T desr) noexcept;
3050b57cec5SDimitry Andric
3060b57cec5SDimitry Andrictemplate <class T>
307*5ffd83dbSDimitry Andric  bool atomic_compare_exchange_strong(volatile atomic<T>* obj, T* expc, T desr) noexcept;
3080b57cec5SDimitry Andric
3090b57cec5SDimitry Andrictemplate <class T>
310*5ffd83dbSDimitry Andric  bool atomic_compare_exchange_strong(atomic<T>* obj, T* expc, T desr) noexcept;
3110b57cec5SDimitry Andric
3120b57cec5SDimitry Andrictemplate <class T>
313*5ffd83dbSDimitry Andric  bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* obj, T* expc,
3140b57cec5SDimitry Andric                                             T desr,
3150b57cec5SDimitry Andric                                             memory_order s, memory_order f) noexcept;
3160b57cec5SDimitry Andric
3170b57cec5SDimitry Andrictemplate <class T>
318*5ffd83dbSDimitry Andric  bool atomic_compare_exchange_weak_explicit(atomic<T>* obj, T* expc, T desr,
3190b57cec5SDimitry Andric                                             memory_order s, memory_order f) noexcept;
3200b57cec5SDimitry Andric
3210b57cec5SDimitry Andrictemplate <class T>
322*5ffd83dbSDimitry Andric  bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* obj,
3230b57cec5SDimitry Andric                                               T* expc, T desr,
3240b57cec5SDimitry Andric                                               memory_order s, memory_order f) noexcept;
3250b57cec5SDimitry Andric
3260b57cec5SDimitry Andrictemplate <class T>
327*5ffd83dbSDimitry Andric  bool atomic_compare_exchange_strong_explicit(atomic<T>* obj, T* expc,
3280b57cec5SDimitry Andric                                               T desr,
3290b57cec5SDimitry Andric                                               memory_order s, memory_order f) noexcept;
3300b57cec5SDimitry Andric
331*5ffd83dbSDimitry Andrictemplate <class T>
332*5ffd83dbSDimitry Andric  void atomic_wait(const volatile atomic<T>* obj, T old) noexcept;
333*5ffd83dbSDimitry Andric
334*5ffd83dbSDimitry Andrictemplate <class T>
335*5ffd83dbSDimitry Andric  void atomic_wait(const atomic<T>* obj, T old) noexcept;
336*5ffd83dbSDimitry Andric
337*5ffd83dbSDimitry Andrictemplate <class T>
338*5ffd83dbSDimitry Andric  void atomic_wait_explicit(const volatile atomic<T>* obj, T old, memory_order m) noexcept;
339*5ffd83dbSDimitry Andric
340*5ffd83dbSDimitry Andrictemplate <class T>
341*5ffd83dbSDimitry Andric  void atomic_wait_explicit(const atomic<T>* obj, T old, memory_order m) noexcept;
342*5ffd83dbSDimitry Andric
343*5ffd83dbSDimitry Andrictemplate <class T>
344*5ffd83dbSDimitry Andric  void atomic_one(volatile atomic<T>* obj) noexcept;
345*5ffd83dbSDimitry Andric
346*5ffd83dbSDimitry Andrictemplate <class T>
347*5ffd83dbSDimitry Andric  void atomic_one(atomic<T>* obj) noexcept;
348*5ffd83dbSDimitry Andric
349*5ffd83dbSDimitry Andrictemplate <class T>
350*5ffd83dbSDimitry Andric  void atomic_all(volatile atomic<T>* obj) noexcept;
351*5ffd83dbSDimitry Andric
352*5ffd83dbSDimitry Andrictemplate <class T>
353*5ffd83dbSDimitry Andric  void atomic_all(atomic<T>* obj) noexcept;
3540b57cec5SDimitry Andric
3550b57cec5SDimitry Andrictemplate <class Integral>
356*5ffd83dbSDimitry Andric  Integral atomic_fetch_add(volatile atomic<Integral>* obj, Integral op) noexcept;
3570b57cec5SDimitry Andric
3580b57cec5SDimitry Andrictemplate <class Integral>
359*5ffd83dbSDimitry Andric  Integral atomic_fetch_add(atomic<Integral>* obj, Integral op) noexcept;
360*5ffd83dbSDimitry Andric
361*5ffd83dbSDimitry Andrictemplate <class Integral>
362*5ffd83dbSDimitry Andric  Integral atomic_fetch_add_explicit(volatile atomic<Integral>* obj, Integral op,
3630b57cec5SDimitry Andric                              memory_order m) noexcept;
3640b57cec5SDimitry Andrictemplate <class Integral>
365*5ffd83dbSDimitry Andric  Integral atomic_fetch_add_explicit(atomic<Integral>* obj, Integral op,
3660b57cec5SDimitry Andric                              memory_order m) noexcept;
3670b57cec5SDimitry Andrictemplate <class Integral>
368*5ffd83dbSDimitry Andric  Integral atomic_fetch_sub(volatile atomic<Integral>* obj, Integral op) noexcept;
3690b57cec5SDimitry Andric
3700b57cec5SDimitry Andrictemplate <class Integral>
371*5ffd83dbSDimitry Andric  Integral atomic_fetch_sub(atomic<Integral>* obj, Integral op) noexcept;
3720b57cec5SDimitry Andric
3730b57cec5SDimitry Andrictemplate <class Integral>
374*5ffd83dbSDimitry Andric  Integral atomic_fetch_sub_explicit(volatile atomic<Integral>* obj, Integral op,
3750b57cec5SDimitry Andric                                     memory_order m) noexcept;
3760b57cec5SDimitry Andric
3770b57cec5SDimitry Andrictemplate <class Integral>
378*5ffd83dbSDimitry Andric  Integral atomic_fetch_sub_explicit(atomic<Integral>* obj, Integral op,
379*5ffd83dbSDimitry Andric                                     memory_order m) noexcept;
3800b57cec5SDimitry Andric
3810b57cec5SDimitry Andrictemplate <class Integral>
382*5ffd83dbSDimitry Andric  Integral atomic_fetch_and(volatile atomic<Integral>* obj, Integral op) noexcept;
3830b57cec5SDimitry Andric
3840b57cec5SDimitry Andrictemplate <class Integral>
385*5ffd83dbSDimitry Andric  Integral atomic_fetch_and(atomic<Integral>* obj, Integral op) noexcept;
3860b57cec5SDimitry Andric
3870b57cec5SDimitry Andrictemplate <class Integral>
388*5ffd83dbSDimitry Andric  Integral atomic_fetch_and_explicit(volatile atomic<Integral>* obj, Integral op,
3890b57cec5SDimitry Andric                                     memory_order m) noexcept;
3900b57cec5SDimitry Andric
3910b57cec5SDimitry Andrictemplate <class Integral>
392*5ffd83dbSDimitry Andric  Integral atomic_fetch_and_explicit(atomic<Integral>* obj, Integral op,
393*5ffd83dbSDimitry Andric                                     memory_order m) noexcept;
3940b57cec5SDimitry Andric
3950b57cec5SDimitry Andrictemplate <class Integral>
396*5ffd83dbSDimitry Andric  Integral atomic_fetch_or(volatile atomic<Integral>* obj, Integral op) noexcept;
397*5ffd83dbSDimitry Andric
3980b57cec5SDimitry Andrictemplate <class Integral>
399*5ffd83dbSDimitry Andric  Integral atomic_fetch_or(atomic<Integral>* obj, Integral op) noexcept;
400*5ffd83dbSDimitry Andric
401*5ffd83dbSDimitry Andrictemplate <class Integral>
402*5ffd83dbSDimitry Andric  Integral atomic_fetch_or_explicit(volatile atomic<Integral>* obj, Integral op,
403*5ffd83dbSDimitry Andric                             memory_order m) noexcept;
404*5ffd83dbSDimitry Andric
405*5ffd83dbSDimitry Andrictemplate <class Integral>
406*5ffd83dbSDimitry Andric  Integral atomic_fetch_or_explicit(atomic<Integral>* obj, Integral op,
407*5ffd83dbSDimitry Andric                             memory_order m) noexcept;
408*5ffd83dbSDimitry Andric
409*5ffd83dbSDimitry Andrictemplate <class Integral>
410*5ffd83dbSDimitry Andric  Integral atomic_fetch_xor(volatile atomic<Integral>* obj, Integral op) noexcept;
411*5ffd83dbSDimitry Andric
412*5ffd83dbSDimitry Andrictemplate <class Integral>
413*5ffd83dbSDimitry Andric  Integral atomic_fetch_xor(atomic<Integral>* obj, Integral op) noexcept;
414*5ffd83dbSDimitry Andric
415*5ffd83dbSDimitry Andrictemplate <class Integral>
416*5ffd83dbSDimitry Andric  Integral atomic_fetch_xor_explicit(volatile atomic<Integral>* obj, Integral op,
417*5ffd83dbSDimitry Andric                                     memory_order m) noexcept;
418*5ffd83dbSDimitry Andric
419*5ffd83dbSDimitry Andrictemplate <class Integral>
420*5ffd83dbSDimitry Andric  Integral atomic_fetch_xor_explicit(atomic<Integral>* obj, Integral op,
4210b57cec5SDimitry Andric                                     memory_order m) noexcept;
4220b57cec5SDimitry Andric
4230b57cec5SDimitry Andrictemplate <class T>
424*5ffd83dbSDimitry Andric  T* atomic_fetch_add(volatile atomic<T*>* obj, ptrdiff_t op) noexcept;
4250b57cec5SDimitry Andric
4260b57cec5SDimitry Andrictemplate <class T>
427*5ffd83dbSDimitry Andric  T* atomic_fetch_add(atomic<T*>* obj, ptrdiff_t op) noexcept;
4280b57cec5SDimitry Andric
4290b57cec5SDimitry Andrictemplate <class T>
430*5ffd83dbSDimitry Andric  T* atomic_fetch_add_explicit(volatile atomic<T*>* obj, ptrdiff_t op,
4310b57cec5SDimitry Andric                               memory_order m) noexcept;
4320b57cec5SDimitry Andric
4330b57cec5SDimitry Andrictemplate <class T>
434*5ffd83dbSDimitry Andric  T* atomic_fetch_add_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept;
4350b57cec5SDimitry Andric
4360b57cec5SDimitry Andrictemplate <class T>
437*5ffd83dbSDimitry Andric  T* atomic_fetch_sub(volatile atomic<T*>* obj, ptrdiff_t op) noexcept;
4380b57cec5SDimitry Andric
4390b57cec5SDimitry Andrictemplate <class T>
440*5ffd83dbSDimitry Andric  T* atomic_fetch_sub(atomic<T*>* obj, ptrdiff_t op) noexcept;
441*5ffd83dbSDimitry Andric
442*5ffd83dbSDimitry Andrictemplate <class T>
443*5ffd83dbSDimitry Andric  T* atomic_fetch_sub_explicit(volatile atomic<T*>* obj, ptrdiff_t op,
4440b57cec5SDimitry Andric                               memory_order m) noexcept;
445*5ffd83dbSDimitry Andric
4460b57cec5SDimitry Andrictemplate <class T>
447*5ffd83dbSDimitry Andric  T* atomic_fetch_sub_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept;
4480b57cec5SDimitry Andric
4490b57cec5SDimitry Andric// Atomics for standard typedef types
4500b57cec5SDimitry Andric
4510b57cec5SDimitry Andrictypedef atomic<bool>               atomic_bool;
4520b57cec5SDimitry Andrictypedef atomic<char>               atomic_char;
4530b57cec5SDimitry Andrictypedef atomic<signed char>        atomic_schar;
4540b57cec5SDimitry Andrictypedef atomic<unsigned char>      atomic_uchar;
4550b57cec5SDimitry Andrictypedef atomic<short>              atomic_short;
4560b57cec5SDimitry Andrictypedef atomic<unsigned short>     atomic_ushort;
4570b57cec5SDimitry Andrictypedef atomic<int>                atomic_int;
4580b57cec5SDimitry Andrictypedef atomic<unsigned int>       atomic_uint;
4590b57cec5SDimitry Andrictypedef atomic<long>               atomic_long;
4600b57cec5SDimitry Andrictypedef atomic<unsigned long>      atomic_ulong;
4610b57cec5SDimitry Andrictypedef atomic<long long>          atomic_llong;
4620b57cec5SDimitry Andrictypedef atomic<unsigned long long> atomic_ullong;
4630b57cec5SDimitry Andrictypedef atomic<char16_t>           atomic_char16_t;
4640b57cec5SDimitry Andrictypedef atomic<char32_t>           atomic_char32_t;
4650b57cec5SDimitry Andrictypedef atomic<wchar_t>            atomic_wchar_t;
4660b57cec5SDimitry Andric
4670b57cec5SDimitry Andrictypedef atomic<int_least8_t>   atomic_int_least8_t;
4680b57cec5SDimitry Andrictypedef atomic<uint_least8_t>  atomic_uint_least8_t;
4690b57cec5SDimitry Andrictypedef atomic<int_least16_t>  atomic_int_least16_t;
4700b57cec5SDimitry Andrictypedef atomic<uint_least16_t> atomic_uint_least16_t;
4710b57cec5SDimitry Andrictypedef atomic<int_least32_t>  atomic_int_least32_t;
4720b57cec5SDimitry Andrictypedef atomic<uint_least32_t> atomic_uint_least32_t;
4730b57cec5SDimitry Andrictypedef atomic<int_least64_t>  atomic_int_least64_t;
4740b57cec5SDimitry Andrictypedef atomic<uint_least64_t> atomic_uint_least64_t;
4750b57cec5SDimitry Andric
4760b57cec5SDimitry Andrictypedef atomic<int_fast8_t>   atomic_int_fast8_t;
4770b57cec5SDimitry Andrictypedef atomic<uint_fast8_t>  atomic_uint_fast8_t;
4780b57cec5SDimitry Andrictypedef atomic<int_fast16_t>  atomic_int_fast16_t;
4790b57cec5SDimitry Andrictypedef atomic<uint_fast16_t> atomic_uint_fast16_t;
4800b57cec5SDimitry Andrictypedef atomic<int_fast32_t>  atomic_int_fast32_t;
4810b57cec5SDimitry Andrictypedef atomic<uint_fast32_t> atomic_uint_fast32_t;
4820b57cec5SDimitry Andrictypedef atomic<int_fast64_t>  atomic_int_fast64_t;
4830b57cec5SDimitry Andrictypedef atomic<uint_fast64_t> atomic_uint_fast64_t;
4840b57cec5SDimitry Andric
4850b57cec5SDimitry Andrictypedef atomic<int8_t>   atomic_int8_t;
4860b57cec5SDimitry Andrictypedef atomic<uint8_t>  atomic_uint8_t;
4870b57cec5SDimitry Andrictypedef atomic<int16_t>  atomic_int16_t;
4880b57cec5SDimitry Andrictypedef atomic<uint16_t> atomic_uint16_t;
4890b57cec5SDimitry Andrictypedef atomic<int32_t>  atomic_int32_t;
4900b57cec5SDimitry Andrictypedef atomic<uint32_t> atomic_uint32_t;
4910b57cec5SDimitry Andrictypedef atomic<int64_t>  atomic_int64_t;
4920b57cec5SDimitry Andrictypedef atomic<uint64_t> atomic_uint64_t;
4930b57cec5SDimitry Andric
4940b57cec5SDimitry Andrictypedef atomic<intptr_t>  atomic_intptr_t;
4950b57cec5SDimitry Andrictypedef atomic<uintptr_t> atomic_uintptr_t;
4960b57cec5SDimitry Andrictypedef atomic<size_t>    atomic_size_t;
4970b57cec5SDimitry Andrictypedef atomic<ptrdiff_t> atomic_ptrdiff_t;
4980b57cec5SDimitry Andrictypedef atomic<intmax_t>  atomic_intmax_t;
4990b57cec5SDimitry Andrictypedef atomic<uintmax_t> atomic_uintmax_t;
5000b57cec5SDimitry Andric
501*5ffd83dbSDimitry Andric// flag type and operations
502*5ffd83dbSDimitry Andric
503*5ffd83dbSDimitry Andrictypedef struct atomic_flag
504*5ffd83dbSDimitry Andric{
505*5ffd83dbSDimitry Andric    atomic_flag() noexcept = default;
506*5ffd83dbSDimitry Andric    atomic_flag(const atomic_flag&) = delete;
507*5ffd83dbSDimitry Andric    atomic_flag& operator=(const atomic_flag&) = delete;
508*5ffd83dbSDimitry Andric    atomic_flag& operator=(const atomic_flag&) volatile = delete;
509*5ffd83dbSDimitry Andric
510*5ffd83dbSDimitry Andric    bool test(memory_order m = memory_order_seq_cst) volatile noexcept;
511*5ffd83dbSDimitry Andric    bool test(memory_order m = memory_order_seq_cst) noexcept;
512*5ffd83dbSDimitry Andric    bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept;
513*5ffd83dbSDimitry Andric    bool test_and_set(memory_order m = memory_order_seq_cst) noexcept;
514*5ffd83dbSDimitry Andric    void clear(memory_order m = memory_order_seq_cst) volatile noexcept;
515*5ffd83dbSDimitry Andric    void clear(memory_order m = memory_order_seq_cst) noexcept;
516*5ffd83dbSDimitry Andric
517*5ffd83dbSDimitry Andric    void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept;
518*5ffd83dbSDimitry Andric    void wait(bool, memory_order = memory_order::seq_cst) const noexcept;
519*5ffd83dbSDimitry Andric    void notify_one() volatile noexcept;
520*5ffd83dbSDimitry Andric    void notify_one() noexcept;
521*5ffd83dbSDimitry Andric    void notify_all() volatile noexcept;
522*5ffd83dbSDimitry Andric    void notify_all() noexcept;
523*5ffd83dbSDimitry Andric} atomic_flag;
524*5ffd83dbSDimitry Andric
525*5ffd83dbSDimitry Andricbool atomic_flag_test(volatile atomic_flag* obj) noexcept;
526*5ffd83dbSDimitry Andricbool atomic_flag_test(atomic_flag* obj) noexcept;
527*5ffd83dbSDimitry Andricbool atomic_flag_test_explicit(volatile atomic_flag* obj,
528*5ffd83dbSDimitry Andric                               memory_order m) noexcept;
529*5ffd83dbSDimitry Andricbool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept;
530*5ffd83dbSDimitry Andricbool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept;
531*5ffd83dbSDimitry Andricbool atomic_flag_test_and_set(atomic_flag* obj) noexcept;
532*5ffd83dbSDimitry Andricbool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj,
533*5ffd83dbSDimitry Andric                                       memory_order m) noexcept;
534*5ffd83dbSDimitry Andricbool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept;
535*5ffd83dbSDimitry Andricvoid atomic_flag_clear(volatile atomic_flag* obj) noexcept;
536*5ffd83dbSDimitry Andricvoid atomic_flag_clear(atomic_flag* obj) noexcept;
537*5ffd83dbSDimitry Andricvoid atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept;
538*5ffd83dbSDimitry Andricvoid atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept;
539*5ffd83dbSDimitry Andric
540*5ffd83dbSDimitry Andricvoid atomic_wait(const volatile atomic_flag* obj, T old) noexcept;
541*5ffd83dbSDimitry Andricvoid atomic_wait(const atomic_flag* obj, T old) noexcept;
542*5ffd83dbSDimitry Andricvoid atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept;
543*5ffd83dbSDimitry Andricvoid atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept;
544*5ffd83dbSDimitry Andricvoid atomic_one(volatile atomic_flag* obj) noexcept;
545*5ffd83dbSDimitry Andricvoid atomic_one(atomic_flag* obj) noexcept;
546*5ffd83dbSDimitry Andricvoid atomic_all(volatile atomic_flag* obj) noexcept;
547*5ffd83dbSDimitry Andricvoid atomic_all(atomic_flag* obj) noexcept;
548*5ffd83dbSDimitry Andric
5490b57cec5SDimitry Andric// fences
5500b57cec5SDimitry Andric
5510b57cec5SDimitry Andricvoid atomic_thread_fence(memory_order m) noexcept;
5520b57cec5SDimitry Andricvoid atomic_signal_fence(memory_order m) noexcept;
5530b57cec5SDimitry Andric
554*5ffd83dbSDimitry Andric// deprecated
555*5ffd83dbSDimitry Andric
556*5ffd83dbSDimitry Andrictemplate <class T>
557*5ffd83dbSDimitry Andric  void atomic_init(volatile atomic<T>* obj, typename atomic<T>::value_type desr) noexcept;
558*5ffd83dbSDimitry Andric
559*5ffd83dbSDimitry Andrictemplate <class T>
560*5ffd83dbSDimitry Andric  void atomic_init(atomic<T>* obj, typename atomic<T>::value_type desr) noexcept;
561*5ffd83dbSDimitry Andric
562*5ffd83dbSDimitry Andric#define ATOMIC_VAR_INIT(value) see below
563*5ffd83dbSDimitry Andric
564*5ffd83dbSDimitry Andric#define ATOMIC_FLAG_INIT see below
565*5ffd83dbSDimitry Andric
5660b57cec5SDimitry Andric}  // std
5670b57cec5SDimitry Andric
5680b57cec5SDimitry Andric*/
5690b57cec5SDimitry Andric
5700b57cec5SDimitry Andric#include <__config>
571*5ffd83dbSDimitry Andric#include <__threading_support>
5720b57cec5SDimitry Andric#include <cstddef>
5730b57cec5SDimitry Andric#include <cstdint>
574*5ffd83dbSDimitry Andric#include <cstring>
5750b57cec5SDimitry Andric#include <type_traits>
5760b57cec5SDimitry Andric#include <version>
5770b57cec5SDimitry Andric
5780b57cec5SDimitry Andric#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
5790b57cec5SDimitry Andric#pragma GCC system_header
5800b57cec5SDimitry Andric#endif
5810b57cec5SDimitry Andric
5820b57cec5SDimitry Andric#ifdef _LIBCPP_HAS_NO_THREADS
5830b57cec5SDimitry Andric# error <atomic> is not supported on this single threaded system
5840b57cec5SDimitry Andric#endif
5850b57cec5SDimitry Andric#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER
5860b57cec5SDimitry Andric# error <atomic> is not implemented
5870b57cec5SDimitry Andric#endif
5880b57cec5SDimitry Andric#ifdef kill_dependency
5890b57cec5SDimitry Andric# error C++ standard library is incompatible with <stdatomic.h>
5900b57cec5SDimitry Andric#endif
5910b57cec5SDimitry Andric
5920b57cec5SDimitry Andric#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \
5930b57cec5SDimitry Andric  _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \
5940b57cec5SDimitry Andric                           __m == memory_order_acquire || \
5950b57cec5SDimitry Andric                           __m == memory_order_acq_rel,   \
5960b57cec5SDimitry Andric                        "memory order argument to atomic operation is invalid")
5970b57cec5SDimitry Andric
5980b57cec5SDimitry Andric#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \
5990b57cec5SDimitry Andric  _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \
6000b57cec5SDimitry Andric                           __m == memory_order_acq_rel,   \
6010b57cec5SDimitry Andric                        "memory order argument to atomic operation is invalid")
6020b57cec5SDimitry Andric
6030b57cec5SDimitry Andric#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \
6040b57cec5SDimitry Andric  _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \
6050b57cec5SDimitry Andric                           __f == memory_order_acq_rel,   \
6060b57cec5SDimitry Andric                        "memory order argument to atomic operation is invalid")
6070b57cec5SDimitry Andric
6080b57cec5SDimitry Andric_LIBCPP_BEGIN_NAMESPACE_STD
6090b57cec5SDimitry Andric
6100b57cec5SDimitry Andric// Figure out what the underlying type for `memory_order` would be if it were
6110b57cec5SDimitry Andric// declared as an unscoped enum (accounting for -fshort-enums). Use this result
6120b57cec5SDimitry Andric// to pin the underlying type in C++20.
6130b57cec5SDimitry Andricenum __legacy_memory_order {
6140b57cec5SDimitry Andric    __mo_relaxed,
6150b57cec5SDimitry Andric    __mo_consume,
6160b57cec5SDimitry Andric    __mo_acquire,
6170b57cec5SDimitry Andric    __mo_release,
6180b57cec5SDimitry Andric    __mo_acq_rel,
6190b57cec5SDimitry Andric    __mo_seq_cst
6200b57cec5SDimitry Andric};
6210b57cec5SDimitry Andric
6220b57cec5SDimitry Andrictypedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t;
6230b57cec5SDimitry Andric
6240b57cec5SDimitry Andric#if _LIBCPP_STD_VER > 17
6250b57cec5SDimitry Andric
6260b57cec5SDimitry Andricenum class memory_order : __memory_order_underlying_t {
6270b57cec5SDimitry Andric  relaxed = __mo_relaxed,
6280b57cec5SDimitry Andric  consume = __mo_consume,
6290b57cec5SDimitry Andric  acquire = __mo_acquire,
6300b57cec5SDimitry Andric  release = __mo_release,
6310b57cec5SDimitry Andric  acq_rel = __mo_acq_rel,
6320b57cec5SDimitry Andric  seq_cst = __mo_seq_cst
6330b57cec5SDimitry Andric};
6340b57cec5SDimitry Andric
6350b57cec5SDimitry Andricinline constexpr auto memory_order_relaxed = memory_order::relaxed;
6360b57cec5SDimitry Andricinline constexpr auto memory_order_consume = memory_order::consume;
6370b57cec5SDimitry Andricinline constexpr auto memory_order_acquire = memory_order::acquire;
6380b57cec5SDimitry Andricinline constexpr auto memory_order_release = memory_order::release;
6390b57cec5SDimitry Andricinline constexpr auto memory_order_acq_rel = memory_order::acq_rel;
6400b57cec5SDimitry Andricinline constexpr auto memory_order_seq_cst = memory_order::seq_cst;
6410b57cec5SDimitry Andric
6420b57cec5SDimitry Andric#else
6430b57cec5SDimitry Andric
6440b57cec5SDimitry Andrictypedef enum memory_order {
6450b57cec5SDimitry Andric  memory_order_relaxed = __mo_relaxed,
6460b57cec5SDimitry Andric  memory_order_consume = __mo_consume,
6470b57cec5SDimitry Andric  memory_order_acquire = __mo_acquire,
6480b57cec5SDimitry Andric  memory_order_release = __mo_release,
6490b57cec5SDimitry Andric  memory_order_acq_rel = __mo_acq_rel,
6500b57cec5SDimitry Andric  memory_order_seq_cst = __mo_seq_cst,
6510b57cec5SDimitry Andric} memory_order;
6520b57cec5SDimitry Andric
6530b57cec5SDimitry Andric#endif // _LIBCPP_STD_VER > 17
6540b57cec5SDimitry Andric
655*5ffd83dbSDimitry Andrictemplate <typename _Tp> _LIBCPP_INLINE_VISIBILITY
656*5ffd83dbSDimitry Andricbool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) {
657*5ffd83dbSDimitry Andric    return memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0;
658*5ffd83dbSDimitry Andric}
659*5ffd83dbSDimitry Andric
6600b57cec5SDimitry Andricstatic_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value),
6610b57cec5SDimitry Andric  "unexpected underlying type for std::memory_order");
6620b57cec5SDimitry Andric
6630b57cec5SDimitry Andric#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \
6640b57cec5SDimitry Andric	defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS)
6650b57cec5SDimitry Andric
6660b57cec5SDimitry Andric// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because
6670b57cec5SDimitry Andric// the default operator= in an object is not volatile, a byte-by-byte copy
6680b57cec5SDimitry Andric// is required.
6690b57cec5SDimitry Andrictemplate <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY
6700b57cec5SDimitry Andrictypename enable_if<is_assignable<_Tp&, _Tv>::value>::type
6710b57cec5SDimitry Andric__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) {
6720b57cec5SDimitry Andric  __a_value = __val;
6730b57cec5SDimitry Andric}
6740b57cec5SDimitry Andrictemplate <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY
6750b57cec5SDimitry Andrictypename enable_if<is_assignable<_Tp&, _Tv>::value>::type
6760b57cec5SDimitry Andric__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) {
6770b57cec5SDimitry Andric  volatile char* __to = reinterpret_cast<volatile char*>(&__a_value);
6780b57cec5SDimitry Andric  volatile char* __end = __to + sizeof(_Tp);
6790b57cec5SDimitry Andric  volatile const char* __from = reinterpret_cast<volatile const char*>(&__val);
6800b57cec5SDimitry Andric  while (__to != __end)
6810b57cec5SDimitry Andric    *__to++ = *__from++;
6820b57cec5SDimitry Andric}
6830b57cec5SDimitry Andric
6840b57cec5SDimitry Andric#endif
6850b57cec5SDimitry Andric
6860b57cec5SDimitry Andric#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP)
6870b57cec5SDimitry Andric
6880b57cec5SDimitry Andrictemplate <typename _Tp>
6890b57cec5SDimitry Andricstruct __cxx_atomic_base_impl {
6900b57cec5SDimitry Andric
6910b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY
6920b57cec5SDimitry Andric#ifndef _LIBCPP_CXX03_LANG
6930b57cec5SDimitry Andric    __cxx_atomic_base_impl() _NOEXCEPT = default;
6940b57cec5SDimitry Andric#else
6950b57cec5SDimitry Andric    __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {}
6960b57cec5SDimitry Andric#endif // _LIBCPP_CXX03_LANG
6970b57cec5SDimitry Andric  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT
6980b57cec5SDimitry Andric    : __a_value(value) {}
6990b57cec5SDimitry Andric  _Tp __a_value;
7000b57cec5SDimitry Andric};
7010b57cec5SDimitry Andric
7020b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) {
7030b57cec5SDimitry Andric  // Avoid switch statement to make this a constexpr.
7040b57cec5SDimitry Andric  return __order == memory_order_relaxed ? __ATOMIC_RELAXED:
7050b57cec5SDimitry Andric         (__order == memory_order_acquire ? __ATOMIC_ACQUIRE:
7060b57cec5SDimitry Andric          (__order == memory_order_release ? __ATOMIC_RELEASE:
7070b57cec5SDimitry Andric           (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST:
7080b57cec5SDimitry Andric            (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL:
7090b57cec5SDimitry Andric              __ATOMIC_CONSUME))));
7100b57cec5SDimitry Andric}
7110b57cec5SDimitry Andric
7120b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) {
7130b57cec5SDimitry Andric  // Avoid switch statement to make this a constexpr.
7140b57cec5SDimitry Andric  return __order == memory_order_relaxed ? __ATOMIC_RELAXED:
7150b57cec5SDimitry Andric         (__order == memory_order_acquire ? __ATOMIC_ACQUIRE:
7160b57cec5SDimitry Andric          (__order == memory_order_release ? __ATOMIC_RELAXED:
7170b57cec5SDimitry Andric           (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST:
7180b57cec5SDimitry Andric            (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE:
7190b57cec5SDimitry Andric              __ATOMIC_CONSUME))));
7200b57cec5SDimitry Andric}
7210b57cec5SDimitry Andric
7220b57cec5SDimitry Andrictemplate <typename _Tp>
7230b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7240b57cec5SDimitry Andricvoid __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a,  _Tp __val) {
7250b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, __val);
7260b57cec5SDimitry Andric}
7270b57cec5SDimitry Andric
7280b57cec5SDimitry Andrictemplate <typename _Tp>
7290b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7300b57cec5SDimitry Andricvoid __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a,  _Tp __val) {
7310b57cec5SDimitry Andric  __a->__a_value = __val;
7320b57cec5SDimitry Andric}
7330b57cec5SDimitry Andric
7340b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline
7350b57cec5SDimitry Andricvoid __cxx_atomic_thread_fence(memory_order __order) {
7360b57cec5SDimitry Andric  __atomic_thread_fence(__to_gcc_order(__order));
7370b57cec5SDimitry Andric}
7380b57cec5SDimitry Andric
7390b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline
7400b57cec5SDimitry Andricvoid __cxx_atomic_signal_fence(memory_order __order) {
7410b57cec5SDimitry Andric  __atomic_signal_fence(__to_gcc_order(__order));
7420b57cec5SDimitry Andric}
7430b57cec5SDimitry Andric
7440b57cec5SDimitry Andrictemplate <typename _Tp>
7450b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7460b57cec5SDimitry Andricvoid __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a,  _Tp __val,
7470b57cec5SDimitry Andric                        memory_order __order) {
7480b57cec5SDimitry Andric  __atomic_store(&__a->__a_value, &__val,
7490b57cec5SDimitry Andric                 __to_gcc_order(__order));
7500b57cec5SDimitry Andric}
7510b57cec5SDimitry Andric
7520b57cec5SDimitry Andrictemplate <typename _Tp>
7530b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7540b57cec5SDimitry Andricvoid __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a,  _Tp __val,
7550b57cec5SDimitry Andric                        memory_order __order) {
7560b57cec5SDimitry Andric  __atomic_store(&__a->__a_value, &__val,
7570b57cec5SDimitry Andric                 __to_gcc_order(__order));
7580b57cec5SDimitry Andric}
7590b57cec5SDimitry Andric
7600b57cec5SDimitry Andrictemplate <typename _Tp>
7610b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7620b57cec5SDimitry Andric_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a,
7630b57cec5SDimitry Andric                      memory_order __order) {
7640b57cec5SDimitry Andric  _Tp __ret;
7650b57cec5SDimitry Andric  __atomic_load(&__a->__a_value, &__ret,
7660b57cec5SDimitry Andric                __to_gcc_order(__order));
7670b57cec5SDimitry Andric  return __ret;
7680b57cec5SDimitry Andric}
7690b57cec5SDimitry Andric
7700b57cec5SDimitry Andrictemplate <typename _Tp>
7710b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7720b57cec5SDimitry Andric_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
7730b57cec5SDimitry Andric  _Tp __ret;
7740b57cec5SDimitry Andric  __atomic_load(&__a->__a_value, &__ret,
7750b57cec5SDimitry Andric                __to_gcc_order(__order));
7760b57cec5SDimitry Andric  return __ret;
7770b57cec5SDimitry Andric}
7780b57cec5SDimitry Andric
7790b57cec5SDimitry Andrictemplate <typename _Tp>
7800b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7810b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a,
7820b57cec5SDimitry Andric                          _Tp __value, memory_order __order) {
7830b57cec5SDimitry Andric  _Tp __ret;
7840b57cec5SDimitry Andric  __atomic_exchange(&__a->__a_value, &__value, &__ret,
7850b57cec5SDimitry Andric                    __to_gcc_order(__order));
7860b57cec5SDimitry Andric  return __ret;
7870b57cec5SDimitry Andric}
7880b57cec5SDimitry Andric
7890b57cec5SDimitry Andrictemplate <typename _Tp>
7900b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7910b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value,
7920b57cec5SDimitry Andric                          memory_order __order) {
7930b57cec5SDimitry Andric  _Tp __ret;
7940b57cec5SDimitry Andric  __atomic_exchange(&__a->__a_value, &__value, &__ret,
7950b57cec5SDimitry Andric                    __to_gcc_order(__order));
7960b57cec5SDimitry Andric  return __ret;
7970b57cec5SDimitry Andric}
7980b57cec5SDimitry Andric
7990b57cec5SDimitry Andrictemplate <typename _Tp>
8000b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8010b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(
8020b57cec5SDimitry Andric    volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value,
8030b57cec5SDimitry Andric    memory_order __success, memory_order __failure) {
8040b57cec5SDimitry Andric  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
8050b57cec5SDimitry Andric                                   false,
8060b57cec5SDimitry Andric                                   __to_gcc_order(__success),
8070b57cec5SDimitry Andric                                   __to_gcc_failure_order(__failure));
8080b57cec5SDimitry Andric}
8090b57cec5SDimitry Andric
8100b57cec5SDimitry Andrictemplate <typename _Tp>
8110b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8120b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(
8130b57cec5SDimitry Andric    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success,
8140b57cec5SDimitry Andric    memory_order __failure) {
8150b57cec5SDimitry Andric  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
8160b57cec5SDimitry Andric                                   false,
8170b57cec5SDimitry Andric                                   __to_gcc_order(__success),
8180b57cec5SDimitry Andric                                   __to_gcc_failure_order(__failure));
8190b57cec5SDimitry Andric}
8200b57cec5SDimitry Andric
8210b57cec5SDimitry Andrictemplate <typename _Tp>
8220b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8230b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(
8240b57cec5SDimitry Andric    volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value,
8250b57cec5SDimitry Andric    memory_order __success, memory_order __failure) {
8260b57cec5SDimitry Andric  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
8270b57cec5SDimitry Andric                                   true,
8280b57cec5SDimitry Andric                                   __to_gcc_order(__success),
8290b57cec5SDimitry Andric                                   __to_gcc_failure_order(__failure));
8300b57cec5SDimitry Andric}
8310b57cec5SDimitry Andric
8320b57cec5SDimitry Andrictemplate <typename _Tp>
8330b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8340b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(
8350b57cec5SDimitry Andric    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success,
8360b57cec5SDimitry Andric    memory_order __failure) {
8370b57cec5SDimitry Andric  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
8380b57cec5SDimitry Andric                                   true,
8390b57cec5SDimitry Andric                                   __to_gcc_order(__success),
8400b57cec5SDimitry Andric                                   __to_gcc_failure_order(__failure));
8410b57cec5SDimitry Andric}
8420b57cec5SDimitry Andric
8430b57cec5SDimitry Andrictemplate <typename _Tp>
8440b57cec5SDimitry Andricstruct __skip_amt { enum {value = 1}; };
8450b57cec5SDimitry Andric
8460b57cec5SDimitry Andrictemplate <typename _Tp>
8470b57cec5SDimitry Andricstruct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; };
8480b57cec5SDimitry Andric
8490b57cec5SDimitry Andric// FIXME: Haven't figured out what the spec says about using arrays with
8500b57cec5SDimitry Andric// atomic_fetch_add. Force a failure rather than creating bad behavior.
8510b57cec5SDimitry Andrictemplate <typename _Tp>
8520b57cec5SDimitry Andricstruct __skip_amt<_Tp[]> { };
8530b57cec5SDimitry Andrictemplate <typename _Tp, int n>
8540b57cec5SDimitry Andricstruct __skip_amt<_Tp[n]> { };
8550b57cec5SDimitry Andric
8560b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
8570b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8580b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a,
8590b57cec5SDimitry Andric                           _Td __delta, memory_order __order) {
8600b57cec5SDimitry Andric  return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
8610b57cec5SDimitry Andric                            __to_gcc_order(__order));
8620b57cec5SDimitry Andric}
8630b57cec5SDimitry Andric
8640b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
8650b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8660b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta,
8670b57cec5SDimitry Andric                           memory_order __order) {
8680b57cec5SDimitry Andric  return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
8690b57cec5SDimitry Andric                            __to_gcc_order(__order));
8700b57cec5SDimitry Andric}
8710b57cec5SDimitry Andric
8720b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
8730b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8740b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a,
8750b57cec5SDimitry Andric                           _Td __delta, memory_order __order) {
8760b57cec5SDimitry Andric  return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
8770b57cec5SDimitry Andric                            __to_gcc_order(__order));
8780b57cec5SDimitry Andric}
8790b57cec5SDimitry Andric
8800b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
8810b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8820b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta,
8830b57cec5SDimitry Andric                           memory_order __order) {
8840b57cec5SDimitry Andric  return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
8850b57cec5SDimitry Andric                            __to_gcc_order(__order));
8860b57cec5SDimitry Andric}
8870b57cec5SDimitry Andric
8880b57cec5SDimitry Andrictemplate <typename _Tp>
8890b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8900b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a,
8910b57cec5SDimitry Andric                           _Tp __pattern, memory_order __order) {
8920b57cec5SDimitry Andric  return __atomic_fetch_and(&__a->__a_value, __pattern,
8930b57cec5SDimitry Andric                            __to_gcc_order(__order));
8940b57cec5SDimitry Andric}
8950b57cec5SDimitry Andric
8960b57cec5SDimitry Andrictemplate <typename _Tp>
8970b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8980b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a,
8990b57cec5SDimitry Andric                           _Tp __pattern, memory_order __order) {
9000b57cec5SDimitry Andric  return __atomic_fetch_and(&__a->__a_value, __pattern,
9010b57cec5SDimitry Andric                            __to_gcc_order(__order));
9020b57cec5SDimitry Andric}
9030b57cec5SDimitry Andric
9040b57cec5SDimitry Andrictemplate <typename _Tp>
9050b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9060b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a,
9070b57cec5SDimitry Andric                          _Tp __pattern, memory_order __order) {
9080b57cec5SDimitry Andric  return __atomic_fetch_or(&__a->__a_value, __pattern,
9090b57cec5SDimitry Andric                           __to_gcc_order(__order));
9100b57cec5SDimitry Andric}
9110b57cec5SDimitry Andric
9120b57cec5SDimitry Andrictemplate <typename _Tp>
9130b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9140b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern,
9150b57cec5SDimitry Andric                          memory_order __order) {
9160b57cec5SDimitry Andric  return __atomic_fetch_or(&__a->__a_value, __pattern,
9170b57cec5SDimitry Andric                           __to_gcc_order(__order));
9180b57cec5SDimitry Andric}
9190b57cec5SDimitry Andric
9200b57cec5SDimitry Andrictemplate <typename _Tp>
9210b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9220b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a,
9230b57cec5SDimitry Andric                           _Tp __pattern, memory_order __order) {
9240b57cec5SDimitry Andric  return __atomic_fetch_xor(&__a->__a_value, __pattern,
9250b57cec5SDimitry Andric                            __to_gcc_order(__order));
9260b57cec5SDimitry Andric}
9270b57cec5SDimitry Andric
9280b57cec5SDimitry Andrictemplate <typename _Tp>
9290b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9300b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern,
9310b57cec5SDimitry Andric                           memory_order __order) {
9320b57cec5SDimitry Andric  return __atomic_fetch_xor(&__a->__a_value, __pattern,
9330b57cec5SDimitry Andric                            __to_gcc_order(__order));
9340b57cec5SDimitry Andric}
9350b57cec5SDimitry Andric
9360b57cec5SDimitry Andric#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0)
9370b57cec5SDimitry Andric
9380b57cec5SDimitry Andric#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP)
9390b57cec5SDimitry Andric
9400b57cec5SDimitry Andrictemplate <typename _Tp>
9410b57cec5SDimitry Andricstruct __cxx_atomic_base_impl {
9420b57cec5SDimitry Andric
9430b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY
9440b57cec5SDimitry Andric#ifndef _LIBCPP_CXX03_LANG
9450b57cec5SDimitry Andric    __cxx_atomic_base_impl() _NOEXCEPT = default;
9460b57cec5SDimitry Andric#else
9470b57cec5SDimitry Andric    __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {}
9480b57cec5SDimitry Andric#endif // _LIBCPP_CXX03_LANG
9490b57cec5SDimitry Andric  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT
9500b57cec5SDimitry Andric    : __a_value(value) {}
951e40139ffSDimitry Andric  _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value;
9520b57cec5SDimitry Andric};
9530b57cec5SDimitry Andric
9540b57cec5SDimitry Andric#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
9550b57cec5SDimitry Andric
9560b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline
9570b57cec5SDimitry Andricvoid __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
9580b57cec5SDimitry Andric    __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
9590b57cec5SDimitry Andric}
9600b57cec5SDimitry Andric
9610b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline
9620b57cec5SDimitry Andricvoid __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
9630b57cec5SDimitry Andric    __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
9640b57cec5SDimitry Andric}
9650b57cec5SDimitry Andric
9660b57cec5SDimitry Andrictemplate<class _Tp>
9670b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9680b57cec5SDimitry Andricvoid __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
9690b57cec5SDimitry Andric    __c11_atomic_init(&__a->__a_value, __val);
9700b57cec5SDimitry Andric}
9710b57cec5SDimitry Andrictemplate<class _Tp>
9720b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9730b57cec5SDimitry Andricvoid __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT {
9740b57cec5SDimitry Andric    __c11_atomic_init(&__a->__a_value, __val);
9750b57cec5SDimitry Andric}
9760b57cec5SDimitry Andric
9770b57cec5SDimitry Andrictemplate<class _Tp>
9780b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9790b57cec5SDimitry Andricvoid __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
9800b57cec5SDimitry Andric    __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order));
9810b57cec5SDimitry Andric}
9820b57cec5SDimitry Andrictemplate<class _Tp>
9830b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9840b57cec5SDimitry Andricvoid __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT {
9850b57cec5SDimitry Andric    __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order));
9860b57cec5SDimitry Andric}
9870b57cec5SDimitry Andric
9880b57cec5SDimitry Andrictemplate<class _Tp>
9890b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9900b57cec5SDimitry Andric_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
9910b57cec5SDimitry Andric    using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*;
9920b57cec5SDimitry Andric    return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order));
9930b57cec5SDimitry Andric}
9940b57cec5SDimitry Andrictemplate<class _Tp>
9950b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9960b57cec5SDimitry Andric_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
9970b57cec5SDimitry Andric    using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*;
9980b57cec5SDimitry Andric    return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order));
9990b57cec5SDimitry Andric}
10000b57cec5SDimitry Andric
10010b57cec5SDimitry Andrictemplate<class _Tp>
10020b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10030b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
10040b57cec5SDimitry Andric    return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order));
10050b57cec5SDimitry Andric}
10060b57cec5SDimitry Andrictemplate<class _Tp>
10070b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10080b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT {
10090b57cec5SDimitry Andric    return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order));
10100b57cec5SDimitry Andric}
10110b57cec5SDimitry Andric
10120b57cec5SDimitry Andrictemplate<class _Tp>
10130b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10140b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
10150b57cec5SDimitry Andric    return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure));
10160b57cec5SDimitry Andric}
10170b57cec5SDimitry Andrictemplate<class _Tp>
10180b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10190b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
10200b57cec5SDimitry Andric    return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure));
10210b57cec5SDimitry Andric}
10220b57cec5SDimitry Andric
10230b57cec5SDimitry Andrictemplate<class _Tp>
10240b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10250b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
10260b57cec5SDimitry Andric    return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure));
10270b57cec5SDimitry Andric}
10280b57cec5SDimitry Andrictemplate<class _Tp>
10290b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10300b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
10310b57cec5SDimitry Andric    return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value,  static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure));
10320b57cec5SDimitry Andric}
10330b57cec5SDimitry Andric
10340b57cec5SDimitry Andrictemplate<class _Tp>
10350b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10360b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
10370b57cec5SDimitry Andric    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10380b57cec5SDimitry Andric}
10390b57cec5SDimitry Andrictemplate<class _Tp>
10400b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10410b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT {
10420b57cec5SDimitry Andric    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10430b57cec5SDimitry Andric}
10440b57cec5SDimitry Andric
10450b57cec5SDimitry Andrictemplate<class _Tp>
10460b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10470b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
10480b57cec5SDimitry Andric    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10490b57cec5SDimitry Andric}
10500b57cec5SDimitry Andrictemplate<class _Tp>
10510b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10520b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
10530b57cec5SDimitry Andric    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10540b57cec5SDimitry Andric}
10550b57cec5SDimitry Andric
10560b57cec5SDimitry Andrictemplate<class _Tp>
10570b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10580b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
10590b57cec5SDimitry Andric    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10600b57cec5SDimitry Andric}
10610b57cec5SDimitry Andrictemplate<class _Tp>
10620b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10630b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT {
10640b57cec5SDimitry Andric    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10650b57cec5SDimitry Andric}
10660b57cec5SDimitry Andrictemplate<class _Tp>
10670b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10680b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
10690b57cec5SDimitry Andric    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10700b57cec5SDimitry Andric}
10710b57cec5SDimitry Andrictemplate<class _Tp>
10720b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10730b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
10740b57cec5SDimitry Andric    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10750b57cec5SDimitry Andric}
10760b57cec5SDimitry Andric
10770b57cec5SDimitry Andrictemplate<class _Tp>
10780b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10790b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
10800b57cec5SDimitry Andric    return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
10810b57cec5SDimitry Andric}
10820b57cec5SDimitry Andrictemplate<class _Tp>
10830b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10840b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
10850b57cec5SDimitry Andric    return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
10860b57cec5SDimitry Andric}
10870b57cec5SDimitry Andric
10880b57cec5SDimitry Andrictemplate<class _Tp>
10890b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10900b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
10910b57cec5SDimitry Andric    return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
10920b57cec5SDimitry Andric}
10930b57cec5SDimitry Andrictemplate<class _Tp>
10940b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10950b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
10960b57cec5SDimitry Andric    return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
10970b57cec5SDimitry Andric}
10980b57cec5SDimitry Andric
10990b57cec5SDimitry Andrictemplate<class _Tp>
11000b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11010b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
11020b57cec5SDimitry Andric    return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
11030b57cec5SDimitry Andric}
11040b57cec5SDimitry Andrictemplate<class _Tp>
11050b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11060b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
11070b57cec5SDimitry Andric    return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
11080b57cec5SDimitry Andric}
11090b57cec5SDimitry Andric
11100b57cec5SDimitry Andric#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP
11110b57cec5SDimitry Andric
11120b57cec5SDimitry Andrictemplate <class _Tp>
11130b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11140b57cec5SDimitry Andric_Tp kill_dependency(_Tp __y) _NOEXCEPT
11150b57cec5SDimitry Andric{
11160b57cec5SDimitry Andric    return __y;
11170b57cec5SDimitry Andric}
11180b57cec5SDimitry Andric
11190b57cec5SDimitry Andric#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE)
11200b57cec5SDimitry Andric# define ATOMIC_BOOL_LOCK_FREE      __CLANG_ATOMIC_BOOL_LOCK_FREE
11210b57cec5SDimitry Andric# define ATOMIC_CHAR_LOCK_FREE      __CLANG_ATOMIC_CHAR_LOCK_FREE
11220b57cec5SDimitry Andric# define ATOMIC_CHAR16_T_LOCK_FREE  __CLANG_ATOMIC_CHAR16_T_LOCK_FREE
11230b57cec5SDimitry Andric# define ATOMIC_CHAR32_T_LOCK_FREE  __CLANG_ATOMIC_CHAR32_T_LOCK_FREE
11240b57cec5SDimitry Andric# define ATOMIC_WCHAR_T_LOCK_FREE   __CLANG_ATOMIC_WCHAR_T_LOCK_FREE
11250b57cec5SDimitry Andric# define ATOMIC_SHORT_LOCK_FREE     __CLANG_ATOMIC_SHORT_LOCK_FREE
11260b57cec5SDimitry Andric# define ATOMIC_INT_LOCK_FREE       __CLANG_ATOMIC_INT_LOCK_FREE
11270b57cec5SDimitry Andric# define ATOMIC_LONG_LOCK_FREE      __CLANG_ATOMIC_LONG_LOCK_FREE
11280b57cec5SDimitry Andric# define ATOMIC_LLONG_LOCK_FREE     __CLANG_ATOMIC_LLONG_LOCK_FREE
11290b57cec5SDimitry Andric# define ATOMIC_POINTER_LOCK_FREE   __CLANG_ATOMIC_POINTER_LOCK_FREE
11300b57cec5SDimitry Andric#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE)
11310b57cec5SDimitry Andric# define ATOMIC_BOOL_LOCK_FREE      __GCC_ATOMIC_BOOL_LOCK_FREE
11320b57cec5SDimitry Andric# define ATOMIC_CHAR_LOCK_FREE      __GCC_ATOMIC_CHAR_LOCK_FREE
11330b57cec5SDimitry Andric# define ATOMIC_CHAR16_T_LOCK_FREE  __GCC_ATOMIC_CHAR16_T_LOCK_FREE
11340b57cec5SDimitry Andric# define ATOMIC_CHAR32_T_LOCK_FREE  __GCC_ATOMIC_CHAR32_T_LOCK_FREE
11350b57cec5SDimitry Andric# define ATOMIC_WCHAR_T_LOCK_FREE   __GCC_ATOMIC_WCHAR_T_LOCK_FREE
11360b57cec5SDimitry Andric# define ATOMIC_SHORT_LOCK_FREE     __GCC_ATOMIC_SHORT_LOCK_FREE
11370b57cec5SDimitry Andric# define ATOMIC_INT_LOCK_FREE       __GCC_ATOMIC_INT_LOCK_FREE
11380b57cec5SDimitry Andric# define ATOMIC_LONG_LOCK_FREE      __GCC_ATOMIC_LONG_LOCK_FREE
11390b57cec5SDimitry Andric# define ATOMIC_LLONG_LOCK_FREE     __GCC_ATOMIC_LLONG_LOCK_FREE
11400b57cec5SDimitry Andric# define ATOMIC_POINTER_LOCK_FREE   __GCC_ATOMIC_POINTER_LOCK_FREE
11410b57cec5SDimitry Andric#endif
11420b57cec5SDimitry Andric
11430b57cec5SDimitry Andric#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS
11440b57cec5SDimitry Andric
11450b57cec5SDimitry Andrictemplate<typename _Tp>
11460b57cec5SDimitry Andricstruct __cxx_atomic_lock_impl {
11470b57cec5SDimitry Andric
11480b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY
11490b57cec5SDimitry Andric  __cxx_atomic_lock_impl() _NOEXCEPT
11500b57cec5SDimitry Andric    : __a_value(), __a_lock(0) {}
11510b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit
11520b57cec5SDimitry Andric  __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT
11530b57cec5SDimitry Andric    : __a_value(value), __a_lock(0) {}
11540b57cec5SDimitry Andric
11550b57cec5SDimitry Andric  _Tp __a_value;
11560b57cec5SDimitry Andric  mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock;
11570b57cec5SDimitry Andric
11580b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY void __lock() const volatile {
11590b57cec5SDimitry Andric    while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire))
11600b57cec5SDimitry Andric        /*spin*/;
11610b57cec5SDimitry Andric  }
11620b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY void __lock() const {
11630b57cec5SDimitry Andric    while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire))
11640b57cec5SDimitry Andric        /*spin*/;
11650b57cec5SDimitry Andric  }
11660b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile {
11670b57cec5SDimitry Andric    __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release);
11680b57cec5SDimitry Andric  }
11690b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY void __unlock() const {
11700b57cec5SDimitry Andric    __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release);
11710b57cec5SDimitry Andric  }
11720b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile {
11730b57cec5SDimitry Andric    __lock();
11740b57cec5SDimitry Andric    _Tp __old;
11750b57cec5SDimitry Andric    __cxx_atomic_assign_volatile(__old, __a_value);
11760b57cec5SDimitry Andric    __unlock();
11770b57cec5SDimitry Andric    return __old;
11780b57cec5SDimitry Andric  }
11790b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY _Tp __read() const {
11800b57cec5SDimitry Andric    __lock();
11810b57cec5SDimitry Andric    _Tp __old = __a_value;
11820b57cec5SDimitry Andric    __unlock();
11830b57cec5SDimitry Andric    return __old;
11840b57cec5SDimitry Andric  }
11850b57cec5SDimitry Andric};
11860b57cec5SDimitry Andric
11870b57cec5SDimitry Andrictemplate <typename _Tp>
11880b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11890b57cec5SDimitry Andricvoid __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val) {
11900b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, __val);
11910b57cec5SDimitry Andric}
11920b57cec5SDimitry Andrictemplate <typename _Tp>
11930b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11940b57cec5SDimitry Andricvoid __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val) {
11950b57cec5SDimitry Andric  __a->__a_value = __val;
11960b57cec5SDimitry Andric}
11970b57cec5SDimitry Andric
11980b57cec5SDimitry Andrictemplate <typename _Tp>
11990b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12000b57cec5SDimitry Andricvoid __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val, memory_order) {
12010b57cec5SDimitry Andric  __a->__lock();
12020b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, __val);
12030b57cec5SDimitry Andric  __a->__unlock();
12040b57cec5SDimitry Andric}
12050b57cec5SDimitry Andrictemplate <typename _Tp>
12060b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12070b57cec5SDimitry Andricvoid __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val, memory_order) {
12080b57cec5SDimitry Andric  __a->__lock();
12090b57cec5SDimitry Andric  __a->__a_value = __val;
12100b57cec5SDimitry Andric  __a->__unlock();
12110b57cec5SDimitry Andric}
12120b57cec5SDimitry Andric
12130b57cec5SDimitry Andrictemplate <typename _Tp>
12140b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12150b57cec5SDimitry Andric_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
12160b57cec5SDimitry Andric  return __a->__read();
12170b57cec5SDimitry Andric}
12180b57cec5SDimitry Andrictemplate <typename _Tp>
12190b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12200b57cec5SDimitry Andric_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
12210b57cec5SDimitry Andric  return __a->__read();
12220b57cec5SDimitry Andric}
12230b57cec5SDimitry Andric
12240b57cec5SDimitry Andrictemplate <typename _Tp>
12250b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12260b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
12270b57cec5SDimitry Andric  __a->__lock();
12280b57cec5SDimitry Andric  _Tp __old;
12290b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
12300b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, __value);
12310b57cec5SDimitry Andric  __a->__unlock();
12320b57cec5SDimitry Andric  return __old;
12330b57cec5SDimitry Andric}
12340b57cec5SDimitry Andrictemplate <typename _Tp>
12350b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12360b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
12370b57cec5SDimitry Andric  __a->__lock();
12380b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
12390b57cec5SDimitry Andric  __a->__a_value = __value;
12400b57cec5SDimitry Andric  __a->__unlock();
12410b57cec5SDimitry Andric  return __old;
12420b57cec5SDimitry Andric}
12430b57cec5SDimitry Andric
12440b57cec5SDimitry Andrictemplate <typename _Tp>
12450b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12460b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a,
12470b57cec5SDimitry Andric                                          _Tp* __expected, _Tp __value, memory_order, memory_order) {
12480b57cec5SDimitry Andric  __a->__lock();
1249*5ffd83dbSDimitry Andric  _Tp __temp;
1250*5ffd83dbSDimitry Andric  __cxx_atomic_assign_volatile(__temp, __a->__a_value);
1251*5ffd83dbSDimitry Andric  bool __ret = __temp == *__expected;
12520b57cec5SDimitry Andric  if(__ret)
12530b57cec5SDimitry Andric    __cxx_atomic_assign_volatile(__a->__a_value, __value);
12540b57cec5SDimitry Andric  else
12550b57cec5SDimitry Andric    __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
12560b57cec5SDimitry Andric  __a->__unlock();
12570b57cec5SDimitry Andric  return __ret;
12580b57cec5SDimitry Andric}
12590b57cec5SDimitry Andrictemplate <typename _Tp>
12600b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12610b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a,
12620b57cec5SDimitry Andric                                          _Tp* __expected, _Tp __value, memory_order, memory_order) {
12630b57cec5SDimitry Andric  __a->__lock();
12640b57cec5SDimitry Andric  bool __ret = __a->__a_value == *__expected;
12650b57cec5SDimitry Andric  if(__ret)
12660b57cec5SDimitry Andric    __a->__a_value = __value;
12670b57cec5SDimitry Andric  else
12680b57cec5SDimitry Andric    *__expected = __a->__a_value;
12690b57cec5SDimitry Andric  __a->__unlock();
12700b57cec5SDimitry Andric  return __ret;
12710b57cec5SDimitry Andric}
12720b57cec5SDimitry Andric
12730b57cec5SDimitry Andrictemplate <typename _Tp>
12740b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12750b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a,
12760b57cec5SDimitry Andric                                        _Tp* __expected, _Tp __value, memory_order, memory_order) {
12770b57cec5SDimitry Andric  __a->__lock();
1278*5ffd83dbSDimitry Andric  _Tp __temp;
1279*5ffd83dbSDimitry Andric  __cxx_atomic_assign_volatile(__temp, __a->__a_value);
1280*5ffd83dbSDimitry Andric  bool __ret = __temp == *__expected;
12810b57cec5SDimitry Andric  if(__ret)
12820b57cec5SDimitry Andric    __cxx_atomic_assign_volatile(__a->__a_value, __value);
12830b57cec5SDimitry Andric  else
12840b57cec5SDimitry Andric    __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
12850b57cec5SDimitry Andric  __a->__unlock();
12860b57cec5SDimitry Andric  return __ret;
12870b57cec5SDimitry Andric}
12880b57cec5SDimitry Andrictemplate <typename _Tp>
12890b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12900b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a,
12910b57cec5SDimitry Andric                                        _Tp* __expected, _Tp __value, memory_order, memory_order) {
12920b57cec5SDimitry Andric  __a->__lock();
12930b57cec5SDimitry Andric  bool __ret = __a->__a_value == *__expected;
12940b57cec5SDimitry Andric  if(__ret)
12950b57cec5SDimitry Andric    __a->__a_value = __value;
12960b57cec5SDimitry Andric  else
12970b57cec5SDimitry Andric    *__expected = __a->__a_value;
12980b57cec5SDimitry Andric  __a->__unlock();
12990b57cec5SDimitry Andric  return __ret;
13000b57cec5SDimitry Andric}
13010b57cec5SDimitry Andric
13020b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
13030b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13040b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a,
13050b57cec5SDimitry Andric                           _Td __delta, memory_order) {
13060b57cec5SDimitry Andric  __a->__lock();
13070b57cec5SDimitry Andric  _Tp __old;
13080b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
13090b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta));
13100b57cec5SDimitry Andric  __a->__unlock();
13110b57cec5SDimitry Andric  return __old;
13120b57cec5SDimitry Andric}
13130b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
13140b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13150b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a,
13160b57cec5SDimitry Andric                           _Td __delta, memory_order) {
13170b57cec5SDimitry Andric  __a->__lock();
13180b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
13190b57cec5SDimitry Andric  __a->__a_value += __delta;
13200b57cec5SDimitry Andric  __a->__unlock();
13210b57cec5SDimitry Andric  return __old;
13220b57cec5SDimitry Andric}
13230b57cec5SDimitry Andric
13240b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
13250b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13260b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a,
13270b57cec5SDimitry Andric                           ptrdiff_t __delta, memory_order) {
13280b57cec5SDimitry Andric  __a->__lock();
13290b57cec5SDimitry Andric  _Tp* __old;
13300b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
13310b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta);
13320b57cec5SDimitry Andric  __a->__unlock();
13330b57cec5SDimitry Andric  return __old;
13340b57cec5SDimitry Andric}
13350b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
13360b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13370b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a,
13380b57cec5SDimitry Andric                           ptrdiff_t __delta, memory_order) {
13390b57cec5SDimitry Andric  __a->__lock();
13400b57cec5SDimitry Andric  _Tp* __old = __a->__a_value;
13410b57cec5SDimitry Andric  __a->__a_value += __delta;
13420b57cec5SDimitry Andric  __a->__unlock();
13430b57cec5SDimitry Andric  return __old;
13440b57cec5SDimitry Andric}
13450b57cec5SDimitry Andric
13460b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
13470b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13480b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a,
13490b57cec5SDimitry Andric                           _Td __delta, memory_order) {
13500b57cec5SDimitry Andric  __a->__lock();
13510b57cec5SDimitry Andric  _Tp __old;
13520b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
13530b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta));
13540b57cec5SDimitry Andric  __a->__unlock();
13550b57cec5SDimitry Andric  return __old;
13560b57cec5SDimitry Andric}
13570b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
13580b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13590b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a,
13600b57cec5SDimitry Andric                           _Td __delta, memory_order) {
13610b57cec5SDimitry Andric  __a->__lock();
13620b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
13630b57cec5SDimitry Andric  __a->__a_value -= __delta;
13640b57cec5SDimitry Andric  __a->__unlock();
13650b57cec5SDimitry Andric  return __old;
13660b57cec5SDimitry Andric}
13670b57cec5SDimitry Andric
13680b57cec5SDimitry Andrictemplate <typename _Tp>
13690b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13700b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a,
13710b57cec5SDimitry Andric                           _Tp __pattern, memory_order) {
13720b57cec5SDimitry Andric  __a->__lock();
13730b57cec5SDimitry Andric  _Tp __old;
13740b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
13750b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern));
13760b57cec5SDimitry Andric  __a->__unlock();
13770b57cec5SDimitry Andric  return __old;
13780b57cec5SDimitry Andric}
13790b57cec5SDimitry Andrictemplate <typename _Tp>
13800b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13810b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a,
13820b57cec5SDimitry Andric                           _Tp __pattern, memory_order) {
13830b57cec5SDimitry Andric  __a->__lock();
13840b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
13850b57cec5SDimitry Andric  __a->__a_value &= __pattern;
13860b57cec5SDimitry Andric  __a->__unlock();
13870b57cec5SDimitry Andric  return __old;
13880b57cec5SDimitry Andric}
13890b57cec5SDimitry Andric
13900b57cec5SDimitry Andrictemplate <typename _Tp>
13910b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13920b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a,
13930b57cec5SDimitry Andric                          _Tp __pattern, memory_order) {
13940b57cec5SDimitry Andric  __a->__lock();
13950b57cec5SDimitry Andric  _Tp __old;
13960b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
13970b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern));
13980b57cec5SDimitry Andric  __a->__unlock();
13990b57cec5SDimitry Andric  return __old;
14000b57cec5SDimitry Andric}
14010b57cec5SDimitry Andrictemplate <typename _Tp>
14020b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
14030b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a,
14040b57cec5SDimitry Andric                          _Tp __pattern, memory_order) {
14050b57cec5SDimitry Andric  __a->__lock();
14060b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
14070b57cec5SDimitry Andric  __a->__a_value |= __pattern;
14080b57cec5SDimitry Andric  __a->__unlock();
14090b57cec5SDimitry Andric  return __old;
14100b57cec5SDimitry Andric}
14110b57cec5SDimitry Andric
14120b57cec5SDimitry Andrictemplate <typename _Tp>
14130b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
14140b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a,
14150b57cec5SDimitry Andric                           _Tp __pattern, memory_order) {
14160b57cec5SDimitry Andric  __a->__lock();
14170b57cec5SDimitry Andric  _Tp __old;
14180b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
14190b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern));
14200b57cec5SDimitry Andric  __a->__unlock();
14210b57cec5SDimitry Andric  return __old;
14220b57cec5SDimitry Andric}
14230b57cec5SDimitry Andrictemplate <typename _Tp>
14240b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
14250b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a,
14260b57cec5SDimitry Andric                           _Tp __pattern, memory_order) {
14270b57cec5SDimitry Andric  __a->__lock();
14280b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
14290b57cec5SDimitry Andric  __a->__a_value ^= __pattern;
14300b57cec5SDimitry Andric  __a->__unlock();
14310b57cec5SDimitry Andric  return __old;
14320b57cec5SDimitry Andric}
14330b57cec5SDimitry Andric
14340b57cec5SDimitry Andric#ifdef __cpp_lib_atomic_is_always_lock_free
14350b57cec5SDimitry Andric
14360b57cec5SDimitry Andrictemplate<typename _Tp> struct __cxx_is_always_lock_free {
14370b57cec5SDimitry Andric    enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; };
14380b57cec5SDimitry Andric
14390b57cec5SDimitry Andric#else
14400b57cec5SDimitry Andric
14410b57cec5SDimitry Andrictemplate<typename _Tp> struct __cxx_is_always_lock_free { enum { __value = false }; };
14420b57cec5SDimitry Andric// Implementations must match the C ATOMIC_*_LOCK_FREE macro values.
14430b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<bool> { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; };
14440b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; };
14450b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<signed char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; };
14460b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<unsigned char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; };
14470b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<char16_t> { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; };
14480b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<char32_t> { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; };
14490b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<wchar_t> { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; };
14500b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; };
14510b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<unsigned short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; };
14520b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; };
14530b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<unsigned int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; };
14540b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; };
14550b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<unsigned long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; };
14560b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; };
14570b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<unsigned long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; };
14580b57cec5SDimitry Andrictemplate<typename _Tp> struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; };
14590b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<std::nullptr_t> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; };
14600b57cec5SDimitry Andric
14610b57cec5SDimitry Andric#endif //__cpp_lib_atomic_is_always_lock_free
14620b57cec5SDimitry Andric
14630b57cec5SDimitry Andrictemplate <typename _Tp,
14640b57cec5SDimitry Andric          typename _Base = typename conditional<__cxx_is_always_lock_free<_Tp>::__value,
14650b57cec5SDimitry Andric                                                __cxx_atomic_base_impl<_Tp>,
14660b57cec5SDimitry Andric                                                __cxx_atomic_lock_impl<_Tp> >::type>
14670b57cec5SDimitry Andric#else
14680b57cec5SDimitry Andrictemplate <typename _Tp,
14690b57cec5SDimitry Andric          typename _Base = __cxx_atomic_base_impl<_Tp> >
14700b57cec5SDimitry Andric#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS
14710b57cec5SDimitry Andricstruct __cxx_atomic_impl : public _Base {
14720b57cec5SDimitry Andric
14730b57cec5SDimitry Andric#if _GNUC_VER >= 501
14740b57cec5SDimitry Andric    static_assert(is_trivially_copyable<_Tp>::value,
14750b57cec5SDimitry Andric      "std::atomic<Tp> requires that 'Tp' be a trivially copyable type");
14760b57cec5SDimitry Andric#endif
14770b57cec5SDimitry Andric
14780b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT
14790b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT
14800b57cec5SDimitry Andric    : _Base(value) {}
14810b57cec5SDimitry Andric};
14820b57cec5SDimitry Andric
1483*5ffd83dbSDimitry Andric#ifdef __linux__
1484*5ffd83dbSDimitry Andric    using __cxx_contention_t = int32_t;
1485*5ffd83dbSDimitry Andric#else
1486*5ffd83dbSDimitry Andric    using __cxx_contention_t = int64_t;
1487*5ffd83dbSDimitry Andric#endif //__linux__
1488*5ffd83dbSDimitry Andric
1489*5ffd83dbSDimitry Andric#if _LIBCPP_STD_VER >= 11
1490*5ffd83dbSDimitry Andric
1491*5ffd83dbSDimitry Andricusing __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>;
1492*5ffd83dbSDimitry Andric
1493*5ffd83dbSDimitry Andric#ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT
1494*5ffd83dbSDimitry Andric
1495*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*);
1496*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*);
1497*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*);
1498*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t);
1499*5ffd83dbSDimitry Andric
1500*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*);
1501*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*);
1502*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*);
1503*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t);
1504*5ffd83dbSDimitry Andric
1505*5ffd83dbSDimitry Andrictemplate <class _Atp, class _Fn>
1506*5ffd83dbSDimitry Andricstruct __libcpp_atomic_wait_backoff_impl {
1507*5ffd83dbSDimitry Andric    _Atp* __a;
1508*5ffd83dbSDimitry Andric    _Fn __test_fn;
1509*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC
1510*5ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const
1511*5ffd83dbSDimitry Andric    {
1512*5ffd83dbSDimitry Andric        if(__elapsed > chrono::microseconds(64))
1513*5ffd83dbSDimitry Andric        {
1514*5ffd83dbSDimitry Andric            auto const __monitor = __libcpp_atomic_monitor(__a);
1515*5ffd83dbSDimitry Andric            if(__test_fn())
1516*5ffd83dbSDimitry Andric                return true;
1517*5ffd83dbSDimitry Andric            __libcpp_atomic_wait(__a, __monitor);
1518*5ffd83dbSDimitry Andric        }
1519*5ffd83dbSDimitry Andric        else if(__elapsed > chrono::microseconds(4))
1520*5ffd83dbSDimitry Andric            __libcpp_thread_yield();
1521*5ffd83dbSDimitry Andric        else
1522*5ffd83dbSDimitry Andric            ; // poll
1523*5ffd83dbSDimitry Andric        return false;
1524*5ffd83dbSDimitry Andric    }
1525*5ffd83dbSDimitry Andric};
1526*5ffd83dbSDimitry Andric
1527*5ffd83dbSDimitry Andrictemplate <class _Atp, class _Fn>
1528*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC
1529*5ffd83dbSDimitry Andric_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn)
1530*5ffd83dbSDimitry Andric{
1531*5ffd83dbSDimitry Andric    __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn};
1532*5ffd83dbSDimitry Andric    return __libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn);
1533*5ffd83dbSDimitry Andric}
1534*5ffd83dbSDimitry Andric
1535*5ffd83dbSDimitry Andric#else // _LIBCPP_HAS_NO_PLATFORM_WAIT
1536*5ffd83dbSDimitry Andric
1537*5ffd83dbSDimitry Andrictemplate <class _Tp>
1538*5ffd83dbSDimitry Andric_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { }
1539*5ffd83dbSDimitry Andrictemplate <class _Tp>
1540*5ffd83dbSDimitry Andric_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { }
1541*5ffd83dbSDimitry Andrictemplate <class _Atp, class _Fn>
1542*5ffd83dbSDimitry Andric_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn)
1543*5ffd83dbSDimitry Andric{
1544*5ffd83dbSDimitry Andric    return __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy());
1545*5ffd83dbSDimitry Andric}
1546*5ffd83dbSDimitry Andric
1547*5ffd83dbSDimitry Andric#endif // _LIBCPP_HAS_NO_PLATFORM_WAIT
1548*5ffd83dbSDimitry Andric
1549*5ffd83dbSDimitry Andrictemplate <class _Atp, class _Tp>
1550*5ffd83dbSDimitry Andricstruct __cxx_atomic_wait_test_fn_impl {
1551*5ffd83dbSDimitry Andric    _Atp* __a;
1552*5ffd83dbSDimitry Andric    _Tp __val;
1553*5ffd83dbSDimitry Andric    memory_order __order;
1554*5ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY bool operator()() const
1555*5ffd83dbSDimitry Andric    {
1556*5ffd83dbSDimitry Andric        return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val);
1557*5ffd83dbSDimitry Andric    }
1558*5ffd83dbSDimitry Andric};
1559*5ffd83dbSDimitry Andric
1560*5ffd83dbSDimitry Andrictemplate <class _Atp, class _Tp>
1561*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC
1562*5ffd83dbSDimitry Andric_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order)
1563*5ffd83dbSDimitry Andric{
1564*5ffd83dbSDimitry Andric    __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order};
1565*5ffd83dbSDimitry Andric    return __cxx_atomic_wait(__a, __test_fn);
1566*5ffd83dbSDimitry Andric}
1567*5ffd83dbSDimitry Andric
1568*5ffd83dbSDimitry Andric#endif //_LIBCPP_STD_VER >= 11
1569*5ffd83dbSDimitry Andric
15700b57cec5SDimitry Andric// general atomic<T>
15710b57cec5SDimitry Andric
15720b57cec5SDimitry Andrictemplate <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>
15730b57cec5SDimitry Andricstruct __atomic_base  // false
15740b57cec5SDimitry Andric{
15750b57cec5SDimitry Andric    mutable __cxx_atomic_impl<_Tp> __a_;
15760b57cec5SDimitry Andric
15770b57cec5SDimitry Andric#if defined(__cpp_lib_atomic_is_always_lock_free)
15780b57cec5SDimitry Andric  static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0);
15790b57cec5SDimitry Andric#endif
15800b57cec5SDimitry Andric
15810b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15820b57cec5SDimitry Andric    bool is_lock_free() const volatile _NOEXCEPT
15830b57cec5SDimitry Andric        {return __cxx_atomic_is_lock_free(sizeof(_Tp));}
15840b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15850b57cec5SDimitry Andric    bool is_lock_free() const _NOEXCEPT
15860b57cec5SDimitry Andric        {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();}
15870b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15880b57cec5SDimitry Andric    void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
15890b57cec5SDimitry Andric      _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
15900b57cec5SDimitry Andric        {__cxx_atomic_store(&__a_, __d, __m);}
15910b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15920b57cec5SDimitry Andric    void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
15930b57cec5SDimitry Andric      _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
15940b57cec5SDimitry Andric        {__cxx_atomic_store(&__a_, __d, __m);}
15950b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15960b57cec5SDimitry Andric    _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
15970b57cec5SDimitry Andric      _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
15980b57cec5SDimitry Andric        {return __cxx_atomic_load(&__a_, __m);}
15990b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16000b57cec5SDimitry Andric    _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
16010b57cec5SDimitry Andric      _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
16020b57cec5SDimitry Andric        {return __cxx_atomic_load(&__a_, __m);}
16030b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16040b57cec5SDimitry Andric    operator _Tp() const volatile _NOEXCEPT {return load();}
16050b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16060b57cec5SDimitry Andric    operator _Tp() const _NOEXCEPT          {return load();}
16070b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16080b57cec5SDimitry Andric    _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
16090b57cec5SDimitry Andric        {return __cxx_atomic_exchange(&__a_, __d, __m);}
16100b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16110b57cec5SDimitry Andric    _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
16120b57cec5SDimitry Andric        {return __cxx_atomic_exchange(&__a_, __d, __m);}
16130b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16140b57cec5SDimitry Andric    bool compare_exchange_weak(_Tp& __e, _Tp __d,
16150b57cec5SDimitry Andric                               memory_order __s, memory_order __f) volatile _NOEXCEPT
16160b57cec5SDimitry Andric      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
16170b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);}
16180b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16190b57cec5SDimitry Andric    bool compare_exchange_weak(_Tp& __e, _Tp __d,
16200b57cec5SDimitry Andric                               memory_order __s, memory_order __f) _NOEXCEPT
16210b57cec5SDimitry Andric      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
16220b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);}
16230b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16240b57cec5SDimitry Andric    bool compare_exchange_strong(_Tp& __e, _Tp __d,
16250b57cec5SDimitry Andric                                 memory_order __s, memory_order __f) volatile _NOEXCEPT
16260b57cec5SDimitry Andric      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
16270b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);}
16280b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16290b57cec5SDimitry Andric    bool compare_exchange_strong(_Tp& __e, _Tp __d,
16300b57cec5SDimitry Andric                                 memory_order __s, memory_order __f) _NOEXCEPT
16310b57cec5SDimitry Andric      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
16320b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);}
16330b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16340b57cec5SDimitry Andric    bool compare_exchange_weak(_Tp& __e, _Tp __d,
16350b57cec5SDimitry Andric                              memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
16360b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);}
16370b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16380b57cec5SDimitry Andric    bool compare_exchange_weak(_Tp& __e, _Tp __d,
16390b57cec5SDimitry Andric                               memory_order __m = memory_order_seq_cst) _NOEXCEPT
16400b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);}
16410b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16420b57cec5SDimitry Andric    bool compare_exchange_strong(_Tp& __e, _Tp __d,
16430b57cec5SDimitry Andric                              memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
16440b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);}
16450b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16460b57cec5SDimitry Andric    bool compare_exchange_strong(_Tp& __e, _Tp __d,
16470b57cec5SDimitry Andric                                 memory_order __m = memory_order_seq_cst) _NOEXCEPT
16480b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);}
16490b57cec5SDimitry Andric
1650*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
1651*5ffd83dbSDimitry Andric        {__cxx_atomic_wait(&__a_, __v, __m);}
1652*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT
1653*5ffd83dbSDimitry Andric        {__cxx_atomic_wait(&__a_, __v, __m);}
1654*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT
1655*5ffd83dbSDimitry Andric        {__cxx_atomic_notify_one(&__a_);}
1656*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT
1657*5ffd83dbSDimitry Andric        {__cxx_atomic_notify_one(&__a_);}
1658*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT
1659*5ffd83dbSDimitry Andric        {__cxx_atomic_notify_all(&__a_);}
1660*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT
1661*5ffd83dbSDimitry Andric        {__cxx_atomic_notify_all(&__a_);}
1662*5ffd83dbSDimitry Andric
16630b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16640b57cec5SDimitry Andric    __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT
16650b57cec5SDimitry Andric
16660b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
16670b57cec5SDimitry Andric    __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {}
16680b57cec5SDimitry Andric
16690b57cec5SDimitry Andric#ifndef _LIBCPP_CXX03_LANG
16700b57cec5SDimitry Andric    __atomic_base(const __atomic_base&) = delete;
16710b57cec5SDimitry Andric    __atomic_base& operator=(const __atomic_base&) = delete;
16720b57cec5SDimitry Andric    __atomic_base& operator=(const __atomic_base&) volatile = delete;
16730b57cec5SDimitry Andric#else
16740b57cec5SDimitry Andricprivate:
1675*5ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16760b57cec5SDimitry Andric    __atomic_base(const __atomic_base&);
1677*5ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16780b57cec5SDimitry Andric    __atomic_base& operator=(const __atomic_base&);
1679*5ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16800b57cec5SDimitry Andric    __atomic_base& operator=(const __atomic_base&) volatile;
16810b57cec5SDimitry Andric#endif
16820b57cec5SDimitry Andric};
16830b57cec5SDimitry Andric
16840b57cec5SDimitry Andric#if defined(__cpp_lib_atomic_is_always_lock_free)
16850b57cec5SDimitry Andrictemplate <class _Tp, bool __b>
16860b57cec5SDimitry Andric_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free;
16870b57cec5SDimitry Andric#endif
16880b57cec5SDimitry Andric
16890b57cec5SDimitry Andric// atomic<Integral>
16900b57cec5SDimitry Andric
16910b57cec5SDimitry Andrictemplate <class _Tp>
16920b57cec5SDimitry Andricstruct __atomic_base<_Tp, true>
16930b57cec5SDimitry Andric    : public __atomic_base<_Tp, false>
16940b57cec5SDimitry Andric{
16950b57cec5SDimitry Andric    typedef __atomic_base<_Tp, false> __base;
16960b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16970b57cec5SDimitry Andric    __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT
16980b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16990b57cec5SDimitry Andric    _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}
17000b57cec5SDimitry Andric
17010b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17020b57cec5SDimitry Andric    _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
17030b57cec5SDimitry Andric        {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);}
17040b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17050b57cec5SDimitry Andric    _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
17060b57cec5SDimitry Andric        {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);}
17070b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17080b57cec5SDimitry Andric    _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
17090b57cec5SDimitry Andric        {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);}
17100b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17110b57cec5SDimitry Andric    _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
17120b57cec5SDimitry Andric        {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);}
17130b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17140b57cec5SDimitry Andric    _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
17150b57cec5SDimitry Andric        {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);}
17160b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17170b57cec5SDimitry Andric    _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
17180b57cec5SDimitry Andric        {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);}
17190b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17200b57cec5SDimitry Andric    _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
17210b57cec5SDimitry Andric        {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);}
17220b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17230b57cec5SDimitry Andric    _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
17240b57cec5SDimitry Andric        {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);}
17250b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17260b57cec5SDimitry Andric    _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
17270b57cec5SDimitry Andric        {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);}
17280b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17290b57cec5SDimitry Andric    _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
17300b57cec5SDimitry Andric        {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);}
17310b57cec5SDimitry Andric
17320b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17330b57cec5SDimitry Andric    _Tp operator++(int) volatile _NOEXCEPT      {return fetch_add(_Tp(1));}
17340b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17350b57cec5SDimitry Andric    _Tp operator++(int) _NOEXCEPT               {return fetch_add(_Tp(1));}
17360b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17370b57cec5SDimitry Andric    _Tp operator--(int) volatile _NOEXCEPT      {return fetch_sub(_Tp(1));}
17380b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17390b57cec5SDimitry Andric    _Tp operator--(int) _NOEXCEPT               {return fetch_sub(_Tp(1));}
17400b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17410b57cec5SDimitry Andric    _Tp operator++() volatile _NOEXCEPT         {return fetch_add(_Tp(1)) + _Tp(1);}
17420b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17430b57cec5SDimitry Andric    _Tp operator++() _NOEXCEPT                  {return fetch_add(_Tp(1)) + _Tp(1);}
17440b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17450b57cec5SDimitry Andric    _Tp operator--() volatile _NOEXCEPT         {return fetch_sub(_Tp(1)) - _Tp(1);}
17460b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17470b57cec5SDimitry Andric    _Tp operator--() _NOEXCEPT                  {return fetch_sub(_Tp(1)) - _Tp(1);}
17480b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17490b57cec5SDimitry Andric    _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;}
17500b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17510b57cec5SDimitry Andric    _Tp operator+=(_Tp __op) _NOEXCEPT          {return fetch_add(__op) + __op;}
17520b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17530b57cec5SDimitry Andric    _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;}
17540b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17550b57cec5SDimitry Andric    _Tp operator-=(_Tp __op) _NOEXCEPT          {return fetch_sub(__op) - __op;}
17560b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17570b57cec5SDimitry Andric    _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;}
17580b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17590b57cec5SDimitry Andric    _Tp operator&=(_Tp __op) _NOEXCEPT          {return fetch_and(__op) & __op;}
17600b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17610b57cec5SDimitry Andric    _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;}
17620b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17630b57cec5SDimitry Andric    _Tp operator|=(_Tp __op) _NOEXCEPT          {return fetch_or(__op) | __op;}
17640b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17650b57cec5SDimitry Andric    _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;}
17660b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17670b57cec5SDimitry Andric    _Tp operator^=(_Tp __op) _NOEXCEPT          {return fetch_xor(__op) ^ __op;}
17680b57cec5SDimitry Andric};
17690b57cec5SDimitry Andric
17700b57cec5SDimitry Andric// atomic<T>
17710b57cec5SDimitry Andric
17720b57cec5SDimitry Andrictemplate <class _Tp>
17730b57cec5SDimitry Andricstruct atomic
17740b57cec5SDimitry Andric    : public __atomic_base<_Tp>
17750b57cec5SDimitry Andric{
17760b57cec5SDimitry Andric    typedef __atomic_base<_Tp> __base;
1777*5ffd83dbSDimitry Andric    typedef _Tp value_type;
17780b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17790b57cec5SDimitry Andric    atomic() _NOEXCEPT _LIBCPP_DEFAULT
17800b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17810b57cec5SDimitry Andric    _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {}
17820b57cec5SDimitry Andric
17830b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17840b57cec5SDimitry Andric    _Tp operator=(_Tp __d) volatile _NOEXCEPT
17850b57cec5SDimitry Andric        {__base::store(__d); return __d;}
17860b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17870b57cec5SDimitry Andric    _Tp operator=(_Tp __d) _NOEXCEPT
17880b57cec5SDimitry Andric        {__base::store(__d); return __d;}
17890b57cec5SDimitry Andric};
17900b57cec5SDimitry Andric
17910b57cec5SDimitry Andric// atomic<T*>
17920b57cec5SDimitry Andric
17930b57cec5SDimitry Andrictemplate <class _Tp>
17940b57cec5SDimitry Andricstruct atomic<_Tp*>
17950b57cec5SDimitry Andric    : public __atomic_base<_Tp*>
17960b57cec5SDimitry Andric{
17970b57cec5SDimitry Andric    typedef __atomic_base<_Tp*> __base;
1798*5ffd83dbSDimitry Andric    typedef _Tp* value_type;
17990b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18000b57cec5SDimitry Andric    atomic() _NOEXCEPT _LIBCPP_DEFAULT
18010b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18020b57cec5SDimitry Andric    _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}
18030b57cec5SDimitry Andric
18040b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18050b57cec5SDimitry Andric    _Tp* operator=(_Tp* __d) volatile _NOEXCEPT
18060b57cec5SDimitry Andric        {__base::store(__d); return __d;}
18070b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18080b57cec5SDimitry Andric    _Tp* operator=(_Tp* __d) _NOEXCEPT
18090b57cec5SDimitry Andric        {__base::store(__d); return __d;}
18100b57cec5SDimitry Andric
18110b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18120b57cec5SDimitry Andric    _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst)
18130b57cec5SDimitry Andric                                                                        volatile _NOEXCEPT
18140b57cec5SDimitry Andric        {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);}
18150b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18160b57cec5SDimitry Andric    _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
18170b57cec5SDimitry Andric        {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);}
18180b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18190b57cec5SDimitry Andric    _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst)
18200b57cec5SDimitry Andric                                                                        volatile _NOEXCEPT
18210b57cec5SDimitry Andric        {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);}
18220b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18230b57cec5SDimitry Andric    _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
18240b57cec5SDimitry Andric        {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);}
18250b57cec5SDimitry Andric
18260b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18270b57cec5SDimitry Andric    _Tp* operator++(int) volatile _NOEXCEPT            {return fetch_add(1);}
18280b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18290b57cec5SDimitry Andric    _Tp* operator++(int) _NOEXCEPT                     {return fetch_add(1);}
18300b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18310b57cec5SDimitry Andric    _Tp* operator--(int) volatile _NOEXCEPT            {return fetch_sub(1);}
18320b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18330b57cec5SDimitry Andric    _Tp* operator--(int) _NOEXCEPT                     {return fetch_sub(1);}
18340b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18350b57cec5SDimitry Andric    _Tp* operator++() volatile _NOEXCEPT               {return fetch_add(1) + 1;}
18360b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18370b57cec5SDimitry Andric    _Tp* operator++() _NOEXCEPT                        {return fetch_add(1) + 1;}
18380b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18390b57cec5SDimitry Andric    _Tp* operator--() volatile _NOEXCEPT               {return fetch_sub(1) - 1;}
18400b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18410b57cec5SDimitry Andric    _Tp* operator--() _NOEXCEPT                        {return fetch_sub(1) - 1;}
18420b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18430b57cec5SDimitry Andric    _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;}
18440b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18450b57cec5SDimitry Andric    _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT          {return fetch_add(__op) + __op;}
18460b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18470b57cec5SDimitry Andric    _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;}
18480b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18490b57cec5SDimitry Andric    _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT          {return fetch_sub(__op) - __op;}
18500b57cec5SDimitry Andric};
18510b57cec5SDimitry Andric
18520b57cec5SDimitry Andric// atomic_is_lock_free
18530b57cec5SDimitry Andric
18540b57cec5SDimitry Andrictemplate <class _Tp>
18550b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
18560b57cec5SDimitry Andricbool
18570b57cec5SDimitry Andricatomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT
18580b57cec5SDimitry Andric{
18590b57cec5SDimitry Andric    return __o->is_lock_free();
18600b57cec5SDimitry Andric}
18610b57cec5SDimitry Andric
18620b57cec5SDimitry Andrictemplate <class _Tp>
18630b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
18640b57cec5SDimitry Andricbool
18650b57cec5SDimitry Andricatomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT
18660b57cec5SDimitry Andric{
18670b57cec5SDimitry Andric    return __o->is_lock_free();
18680b57cec5SDimitry Andric}
18690b57cec5SDimitry Andric
18700b57cec5SDimitry Andric// atomic_init
18710b57cec5SDimitry Andric
18720b57cec5SDimitry Andrictemplate <class _Tp>
18730b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
18740b57cec5SDimitry Andricvoid
18750b57cec5SDimitry Andricatomic_init(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
18760b57cec5SDimitry Andric{
18770b57cec5SDimitry Andric    __cxx_atomic_init(&__o->__a_, __d);
18780b57cec5SDimitry Andric}
18790b57cec5SDimitry Andric
18800b57cec5SDimitry Andrictemplate <class _Tp>
18810b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
18820b57cec5SDimitry Andricvoid
18830b57cec5SDimitry Andricatomic_init(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
18840b57cec5SDimitry Andric{
18850b57cec5SDimitry Andric    __cxx_atomic_init(&__o->__a_, __d);
18860b57cec5SDimitry Andric}
18870b57cec5SDimitry Andric
18880b57cec5SDimitry Andric// atomic_store
18890b57cec5SDimitry Andric
18900b57cec5SDimitry Andrictemplate <class _Tp>
18910b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
18920b57cec5SDimitry Andricvoid
18930b57cec5SDimitry Andricatomic_store(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
18940b57cec5SDimitry Andric{
18950b57cec5SDimitry Andric    __o->store(__d);
18960b57cec5SDimitry Andric}
18970b57cec5SDimitry Andric
18980b57cec5SDimitry Andrictemplate <class _Tp>
18990b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19000b57cec5SDimitry Andricvoid
19010b57cec5SDimitry Andricatomic_store(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
19020b57cec5SDimitry Andric{
19030b57cec5SDimitry Andric    __o->store(__d);
19040b57cec5SDimitry Andric}
19050b57cec5SDimitry Andric
19060b57cec5SDimitry Andric// atomic_store_explicit
19070b57cec5SDimitry Andric
19080b57cec5SDimitry Andrictemplate <class _Tp>
19090b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19100b57cec5SDimitry Andricvoid
19110b57cec5SDimitry Andricatomic_store_explicit(volatile atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT
19120b57cec5SDimitry Andric  _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
19130b57cec5SDimitry Andric{
19140b57cec5SDimitry Andric    __o->store(__d, __m);
19150b57cec5SDimitry Andric}
19160b57cec5SDimitry Andric
19170b57cec5SDimitry Andrictemplate <class _Tp>
19180b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19190b57cec5SDimitry Andricvoid
19200b57cec5SDimitry Andricatomic_store_explicit(atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT
19210b57cec5SDimitry Andric  _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
19220b57cec5SDimitry Andric{
19230b57cec5SDimitry Andric    __o->store(__d, __m);
19240b57cec5SDimitry Andric}
19250b57cec5SDimitry Andric
19260b57cec5SDimitry Andric// atomic_load
19270b57cec5SDimitry Andric
19280b57cec5SDimitry Andrictemplate <class _Tp>
19290b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19300b57cec5SDimitry Andric_Tp
19310b57cec5SDimitry Andricatomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT
19320b57cec5SDimitry Andric{
19330b57cec5SDimitry Andric    return __o->load();
19340b57cec5SDimitry Andric}
19350b57cec5SDimitry Andric
19360b57cec5SDimitry Andrictemplate <class _Tp>
19370b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19380b57cec5SDimitry Andric_Tp
19390b57cec5SDimitry Andricatomic_load(const atomic<_Tp>* __o) _NOEXCEPT
19400b57cec5SDimitry Andric{
19410b57cec5SDimitry Andric    return __o->load();
19420b57cec5SDimitry Andric}
19430b57cec5SDimitry Andric
19440b57cec5SDimitry Andric// atomic_load_explicit
19450b57cec5SDimitry Andric
19460b57cec5SDimitry Andrictemplate <class _Tp>
19470b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19480b57cec5SDimitry Andric_Tp
19490b57cec5SDimitry Andricatomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
19500b57cec5SDimitry Andric  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
19510b57cec5SDimitry Andric{
19520b57cec5SDimitry Andric    return __o->load(__m);
19530b57cec5SDimitry Andric}
19540b57cec5SDimitry Andric
19550b57cec5SDimitry Andrictemplate <class _Tp>
19560b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19570b57cec5SDimitry Andric_Tp
19580b57cec5SDimitry Andricatomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
19590b57cec5SDimitry Andric  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
19600b57cec5SDimitry Andric{
19610b57cec5SDimitry Andric    return __o->load(__m);
19620b57cec5SDimitry Andric}
19630b57cec5SDimitry Andric
19640b57cec5SDimitry Andric// atomic_exchange
19650b57cec5SDimitry Andric
19660b57cec5SDimitry Andrictemplate <class _Tp>
19670b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19680b57cec5SDimitry Andric_Tp
19690b57cec5SDimitry Andricatomic_exchange(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
19700b57cec5SDimitry Andric{
19710b57cec5SDimitry Andric    return __o->exchange(__d);
19720b57cec5SDimitry Andric}
19730b57cec5SDimitry Andric
19740b57cec5SDimitry Andrictemplate <class _Tp>
19750b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19760b57cec5SDimitry Andric_Tp
19770b57cec5SDimitry Andricatomic_exchange(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT
19780b57cec5SDimitry Andric{
19790b57cec5SDimitry Andric    return __o->exchange(__d);
19800b57cec5SDimitry Andric}
19810b57cec5SDimitry Andric
19820b57cec5SDimitry Andric// atomic_exchange_explicit
19830b57cec5SDimitry Andric
19840b57cec5SDimitry Andrictemplate <class _Tp>
19850b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19860b57cec5SDimitry Andric_Tp
19870b57cec5SDimitry Andricatomic_exchange_explicit(volatile atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT
19880b57cec5SDimitry Andric{
19890b57cec5SDimitry Andric    return __o->exchange(__d, __m);
19900b57cec5SDimitry Andric}
19910b57cec5SDimitry Andric
19920b57cec5SDimitry Andrictemplate <class _Tp>
19930b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19940b57cec5SDimitry Andric_Tp
19950b57cec5SDimitry Andricatomic_exchange_explicit(atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT
19960b57cec5SDimitry Andric{
19970b57cec5SDimitry Andric    return __o->exchange(__d, __m);
19980b57cec5SDimitry Andric}
19990b57cec5SDimitry Andric
20000b57cec5SDimitry Andric// atomic_compare_exchange_weak
20010b57cec5SDimitry Andric
20020b57cec5SDimitry Andrictemplate <class _Tp>
20030b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20040b57cec5SDimitry Andricbool
20050b57cec5SDimitry Andricatomic_compare_exchange_weak(volatile atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT
20060b57cec5SDimitry Andric{
20070b57cec5SDimitry Andric    return __o->compare_exchange_weak(*__e, __d);
20080b57cec5SDimitry Andric}
20090b57cec5SDimitry Andric
20100b57cec5SDimitry Andrictemplate <class _Tp>
20110b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20120b57cec5SDimitry Andricbool
20130b57cec5SDimitry Andricatomic_compare_exchange_weak(atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT
20140b57cec5SDimitry Andric{
20150b57cec5SDimitry Andric    return __o->compare_exchange_weak(*__e, __d);
20160b57cec5SDimitry Andric}
20170b57cec5SDimitry Andric
20180b57cec5SDimitry Andric// atomic_compare_exchange_strong
20190b57cec5SDimitry Andric
20200b57cec5SDimitry Andrictemplate <class _Tp>
20210b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20220b57cec5SDimitry Andricbool
20230b57cec5SDimitry Andricatomic_compare_exchange_strong(volatile atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT
20240b57cec5SDimitry Andric{
20250b57cec5SDimitry Andric    return __o->compare_exchange_strong(*__e, __d);
20260b57cec5SDimitry Andric}
20270b57cec5SDimitry Andric
20280b57cec5SDimitry Andrictemplate <class _Tp>
20290b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20300b57cec5SDimitry Andricbool
20310b57cec5SDimitry Andricatomic_compare_exchange_strong(atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT
20320b57cec5SDimitry Andric{
20330b57cec5SDimitry Andric    return __o->compare_exchange_strong(*__e, __d);
20340b57cec5SDimitry Andric}
20350b57cec5SDimitry Andric
20360b57cec5SDimitry Andric// atomic_compare_exchange_weak_explicit
20370b57cec5SDimitry Andric
20380b57cec5SDimitry Andrictemplate <class _Tp>
20390b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20400b57cec5SDimitry Andricbool
20410b57cec5SDimitry Andricatomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, _Tp* __e,
20420b57cec5SDimitry Andric                                      _Tp __d,
20430b57cec5SDimitry Andric                                      memory_order __s, memory_order __f) _NOEXCEPT
20440b57cec5SDimitry Andric  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
20450b57cec5SDimitry Andric{
20460b57cec5SDimitry Andric    return __o->compare_exchange_weak(*__e, __d, __s, __f);
20470b57cec5SDimitry Andric}
20480b57cec5SDimitry Andric
20490b57cec5SDimitry Andrictemplate <class _Tp>
20500b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20510b57cec5SDimitry Andricbool
20520b57cec5SDimitry Andricatomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, _Tp* __e, _Tp __d,
20530b57cec5SDimitry Andric                                      memory_order __s, memory_order __f) _NOEXCEPT
20540b57cec5SDimitry Andric  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
20550b57cec5SDimitry Andric{
20560b57cec5SDimitry Andric    return __o->compare_exchange_weak(*__e, __d, __s, __f);
20570b57cec5SDimitry Andric}
20580b57cec5SDimitry Andric
20590b57cec5SDimitry Andric// atomic_compare_exchange_strong_explicit
20600b57cec5SDimitry Andric
20610b57cec5SDimitry Andrictemplate <class _Tp>
20620b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20630b57cec5SDimitry Andricbool
20640b57cec5SDimitry Andricatomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o,
20650b57cec5SDimitry Andric                                        _Tp* __e, _Tp __d,
20660b57cec5SDimitry Andric                                        memory_order __s, memory_order __f) _NOEXCEPT
20670b57cec5SDimitry Andric  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
20680b57cec5SDimitry Andric{
20690b57cec5SDimitry Andric    return __o->compare_exchange_strong(*__e, __d, __s, __f);
20700b57cec5SDimitry Andric}
20710b57cec5SDimitry Andric
20720b57cec5SDimitry Andrictemplate <class _Tp>
20730b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20740b57cec5SDimitry Andricbool
20750b57cec5SDimitry Andricatomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, _Tp* __e,
20760b57cec5SDimitry Andric                                        _Tp __d,
20770b57cec5SDimitry Andric                                        memory_order __s, memory_order __f) _NOEXCEPT
20780b57cec5SDimitry Andric  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
20790b57cec5SDimitry Andric{
20800b57cec5SDimitry Andric    return __o->compare_exchange_strong(*__e, __d, __s, __f);
20810b57cec5SDimitry Andric}
20820b57cec5SDimitry Andric
2083*5ffd83dbSDimitry Andric// atomic_wait
2084*5ffd83dbSDimitry Andric
2085*5ffd83dbSDimitry Andrictemplate <class _Tp>
2086*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2087*5ffd83dbSDimitry Andricvoid atomic_wait(const volatile atomic<_Tp>* __o,
2088*5ffd83dbSDimitry Andric                 typename atomic<_Tp>::value_type __v) _NOEXCEPT
2089*5ffd83dbSDimitry Andric{
2090*5ffd83dbSDimitry Andric    return __o->wait(__v);
2091*5ffd83dbSDimitry Andric}
2092*5ffd83dbSDimitry Andric
2093*5ffd83dbSDimitry Andrictemplate <class _Tp>
2094*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2095*5ffd83dbSDimitry Andricvoid atomic_wait(const atomic<_Tp>* __o,
2096*5ffd83dbSDimitry Andric                 typename atomic<_Tp>::value_type __v) _NOEXCEPT
2097*5ffd83dbSDimitry Andric{
2098*5ffd83dbSDimitry Andric    return __o->wait(__v);
2099*5ffd83dbSDimitry Andric}
2100*5ffd83dbSDimitry Andric
2101*5ffd83dbSDimitry Andric// atomic_wait_explicit
2102*5ffd83dbSDimitry Andric
2103*5ffd83dbSDimitry Andrictemplate <class _Tp>
2104*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2105*5ffd83dbSDimitry Andricvoid atomic_wait_explicit(const volatile atomic<_Tp>* __o,
2106*5ffd83dbSDimitry Andric                          typename atomic<_Tp>::value_type __v,
2107*5ffd83dbSDimitry Andric                          memory_order __m) _NOEXCEPT
2108*5ffd83dbSDimitry Andric  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
2109*5ffd83dbSDimitry Andric{
2110*5ffd83dbSDimitry Andric    return __o->wait(__v, __m);
2111*5ffd83dbSDimitry Andric}
2112*5ffd83dbSDimitry Andric
2113*5ffd83dbSDimitry Andrictemplate <class _Tp>
2114*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2115*5ffd83dbSDimitry Andricvoid atomic_wait_explicit(const atomic<_Tp>* __o,
2116*5ffd83dbSDimitry Andric                          typename atomic<_Tp>::value_type __v,
2117*5ffd83dbSDimitry Andric                          memory_order __m) _NOEXCEPT
2118*5ffd83dbSDimitry Andric  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
2119*5ffd83dbSDimitry Andric{
2120*5ffd83dbSDimitry Andric    return __o->wait(__v, __m);
2121*5ffd83dbSDimitry Andric}
2122*5ffd83dbSDimitry Andric
2123*5ffd83dbSDimitry Andric// atomic_notify_one
2124*5ffd83dbSDimitry Andric
2125*5ffd83dbSDimitry Andrictemplate <class _Tp>
2126*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2127*5ffd83dbSDimitry Andricvoid atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT
2128*5ffd83dbSDimitry Andric{
2129*5ffd83dbSDimitry Andric    __o->notify_one();
2130*5ffd83dbSDimitry Andric}
2131*5ffd83dbSDimitry Andrictemplate <class _Tp>
2132*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2133*5ffd83dbSDimitry Andricvoid atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT
2134*5ffd83dbSDimitry Andric{
2135*5ffd83dbSDimitry Andric    __o->notify_one();
2136*5ffd83dbSDimitry Andric}
2137*5ffd83dbSDimitry Andric
2138*5ffd83dbSDimitry Andric// atomic_notify_one
2139*5ffd83dbSDimitry Andric
2140*5ffd83dbSDimitry Andrictemplate <class _Tp>
2141*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2142*5ffd83dbSDimitry Andricvoid atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT
2143*5ffd83dbSDimitry Andric{
2144*5ffd83dbSDimitry Andric    __o->notify_all();
2145*5ffd83dbSDimitry Andric}
2146*5ffd83dbSDimitry Andrictemplate <class _Tp>
2147*5ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2148*5ffd83dbSDimitry Andricvoid atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT
2149*5ffd83dbSDimitry Andric{
2150*5ffd83dbSDimitry Andric    __o->notify_all();
2151*5ffd83dbSDimitry Andric}
2152*5ffd83dbSDimitry Andric
21530b57cec5SDimitry Andric// atomic_fetch_add
21540b57cec5SDimitry Andric
21550b57cec5SDimitry Andrictemplate <class _Tp>
21560b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
21570b57cec5SDimitry Andrictypename enable_if
21580b57cec5SDimitry Andric<
21590b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
21600b57cec5SDimitry Andric    _Tp
21610b57cec5SDimitry Andric>::type
21620b57cec5SDimitry Andricatomic_fetch_add(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
21630b57cec5SDimitry Andric{
21640b57cec5SDimitry Andric    return __o->fetch_add(__op);
21650b57cec5SDimitry Andric}
21660b57cec5SDimitry Andric
21670b57cec5SDimitry Andrictemplate <class _Tp>
21680b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
21690b57cec5SDimitry Andrictypename enable_if
21700b57cec5SDimitry Andric<
21710b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
21720b57cec5SDimitry Andric    _Tp
21730b57cec5SDimitry Andric>::type
21740b57cec5SDimitry Andricatomic_fetch_add(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
21750b57cec5SDimitry Andric{
21760b57cec5SDimitry Andric    return __o->fetch_add(__op);
21770b57cec5SDimitry Andric}
21780b57cec5SDimitry Andric
21790b57cec5SDimitry Andrictemplate <class _Tp>
21800b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
21810b57cec5SDimitry Andric_Tp*
21820b57cec5SDimitry Andricatomic_fetch_add(volatile atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT
21830b57cec5SDimitry Andric{
21840b57cec5SDimitry Andric    return __o->fetch_add(__op);
21850b57cec5SDimitry Andric}
21860b57cec5SDimitry Andric
21870b57cec5SDimitry Andrictemplate <class _Tp>
21880b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
21890b57cec5SDimitry Andric_Tp*
21900b57cec5SDimitry Andricatomic_fetch_add(atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT
21910b57cec5SDimitry Andric{
21920b57cec5SDimitry Andric    return __o->fetch_add(__op);
21930b57cec5SDimitry Andric}
21940b57cec5SDimitry Andric
21950b57cec5SDimitry Andric// atomic_fetch_add_explicit
21960b57cec5SDimitry Andric
21970b57cec5SDimitry Andrictemplate <class _Tp>
21980b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
21990b57cec5SDimitry Andrictypename enable_if
22000b57cec5SDimitry Andric<
22010b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22020b57cec5SDimitry Andric    _Tp
22030b57cec5SDimitry Andric>::type
22040b57cec5SDimitry Andricatomic_fetch_add_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
22050b57cec5SDimitry Andric{
22060b57cec5SDimitry Andric    return __o->fetch_add(__op, __m);
22070b57cec5SDimitry Andric}
22080b57cec5SDimitry Andric
22090b57cec5SDimitry Andrictemplate <class _Tp>
22100b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22110b57cec5SDimitry Andrictypename enable_if
22120b57cec5SDimitry Andric<
22130b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22140b57cec5SDimitry Andric    _Tp
22150b57cec5SDimitry Andric>::type
22160b57cec5SDimitry Andricatomic_fetch_add_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
22170b57cec5SDimitry Andric{
22180b57cec5SDimitry Andric    return __o->fetch_add(__op, __m);
22190b57cec5SDimitry Andric}
22200b57cec5SDimitry Andric
22210b57cec5SDimitry Andrictemplate <class _Tp>
22220b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22230b57cec5SDimitry Andric_Tp*
22240b57cec5SDimitry Andricatomic_fetch_add_explicit(volatile atomic<_Tp*>* __o, ptrdiff_t __op,
22250b57cec5SDimitry Andric                          memory_order __m) _NOEXCEPT
22260b57cec5SDimitry Andric{
22270b57cec5SDimitry Andric    return __o->fetch_add(__op, __m);
22280b57cec5SDimitry Andric}
22290b57cec5SDimitry Andric
22300b57cec5SDimitry Andrictemplate <class _Tp>
22310b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22320b57cec5SDimitry Andric_Tp*
22330b57cec5SDimitry Andricatomic_fetch_add_explicit(atomic<_Tp*>* __o, ptrdiff_t __op, memory_order __m) _NOEXCEPT
22340b57cec5SDimitry Andric{
22350b57cec5SDimitry Andric    return __o->fetch_add(__op, __m);
22360b57cec5SDimitry Andric}
22370b57cec5SDimitry Andric
22380b57cec5SDimitry Andric// atomic_fetch_sub
22390b57cec5SDimitry Andric
22400b57cec5SDimitry Andrictemplate <class _Tp>
22410b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22420b57cec5SDimitry Andrictypename enable_if
22430b57cec5SDimitry Andric<
22440b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22450b57cec5SDimitry Andric    _Tp
22460b57cec5SDimitry Andric>::type
22470b57cec5SDimitry Andricatomic_fetch_sub(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
22480b57cec5SDimitry Andric{
22490b57cec5SDimitry Andric    return __o->fetch_sub(__op);
22500b57cec5SDimitry Andric}
22510b57cec5SDimitry Andric
22520b57cec5SDimitry Andrictemplate <class _Tp>
22530b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22540b57cec5SDimitry Andrictypename enable_if
22550b57cec5SDimitry Andric<
22560b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22570b57cec5SDimitry Andric    _Tp
22580b57cec5SDimitry Andric>::type
22590b57cec5SDimitry Andricatomic_fetch_sub(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
22600b57cec5SDimitry Andric{
22610b57cec5SDimitry Andric    return __o->fetch_sub(__op);
22620b57cec5SDimitry Andric}
22630b57cec5SDimitry Andric
22640b57cec5SDimitry Andrictemplate <class _Tp>
22650b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22660b57cec5SDimitry Andric_Tp*
22670b57cec5SDimitry Andricatomic_fetch_sub(volatile atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT
22680b57cec5SDimitry Andric{
22690b57cec5SDimitry Andric    return __o->fetch_sub(__op);
22700b57cec5SDimitry Andric}
22710b57cec5SDimitry Andric
22720b57cec5SDimitry Andrictemplate <class _Tp>
22730b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22740b57cec5SDimitry Andric_Tp*
22750b57cec5SDimitry Andricatomic_fetch_sub(atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT
22760b57cec5SDimitry Andric{
22770b57cec5SDimitry Andric    return __o->fetch_sub(__op);
22780b57cec5SDimitry Andric}
22790b57cec5SDimitry Andric
22800b57cec5SDimitry Andric// atomic_fetch_sub_explicit
22810b57cec5SDimitry Andric
22820b57cec5SDimitry Andrictemplate <class _Tp>
22830b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22840b57cec5SDimitry Andrictypename enable_if
22850b57cec5SDimitry Andric<
22860b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22870b57cec5SDimitry Andric    _Tp
22880b57cec5SDimitry Andric>::type
22890b57cec5SDimitry Andricatomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
22900b57cec5SDimitry Andric{
22910b57cec5SDimitry Andric    return __o->fetch_sub(__op, __m);
22920b57cec5SDimitry Andric}
22930b57cec5SDimitry Andric
22940b57cec5SDimitry Andrictemplate <class _Tp>
22950b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22960b57cec5SDimitry Andrictypename enable_if
22970b57cec5SDimitry Andric<
22980b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22990b57cec5SDimitry Andric    _Tp
23000b57cec5SDimitry Andric>::type
23010b57cec5SDimitry Andricatomic_fetch_sub_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
23020b57cec5SDimitry Andric{
23030b57cec5SDimitry Andric    return __o->fetch_sub(__op, __m);
23040b57cec5SDimitry Andric}
23050b57cec5SDimitry Andric
23060b57cec5SDimitry Andrictemplate <class _Tp>
23070b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23080b57cec5SDimitry Andric_Tp*
23090b57cec5SDimitry Andricatomic_fetch_sub_explicit(volatile atomic<_Tp*>* __o, ptrdiff_t __op,
23100b57cec5SDimitry Andric                          memory_order __m) _NOEXCEPT
23110b57cec5SDimitry Andric{
23120b57cec5SDimitry Andric    return __o->fetch_sub(__op, __m);
23130b57cec5SDimitry Andric}
23140b57cec5SDimitry Andric
23150b57cec5SDimitry Andrictemplate <class _Tp>
23160b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23170b57cec5SDimitry Andric_Tp*
23180b57cec5SDimitry Andricatomic_fetch_sub_explicit(atomic<_Tp*>* __o, ptrdiff_t __op, memory_order __m) _NOEXCEPT
23190b57cec5SDimitry Andric{
23200b57cec5SDimitry Andric    return __o->fetch_sub(__op, __m);
23210b57cec5SDimitry Andric}
23220b57cec5SDimitry Andric
23230b57cec5SDimitry Andric// atomic_fetch_and
23240b57cec5SDimitry Andric
23250b57cec5SDimitry Andrictemplate <class _Tp>
23260b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23270b57cec5SDimitry Andrictypename enable_if
23280b57cec5SDimitry Andric<
23290b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23300b57cec5SDimitry Andric    _Tp
23310b57cec5SDimitry Andric>::type
23320b57cec5SDimitry Andricatomic_fetch_and(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
23330b57cec5SDimitry Andric{
23340b57cec5SDimitry Andric    return __o->fetch_and(__op);
23350b57cec5SDimitry Andric}
23360b57cec5SDimitry Andric
23370b57cec5SDimitry Andrictemplate <class _Tp>
23380b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23390b57cec5SDimitry Andrictypename enable_if
23400b57cec5SDimitry Andric<
23410b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23420b57cec5SDimitry Andric    _Tp
23430b57cec5SDimitry Andric>::type
23440b57cec5SDimitry Andricatomic_fetch_and(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
23450b57cec5SDimitry Andric{
23460b57cec5SDimitry Andric    return __o->fetch_and(__op);
23470b57cec5SDimitry Andric}
23480b57cec5SDimitry Andric
23490b57cec5SDimitry Andric// atomic_fetch_and_explicit
23500b57cec5SDimitry Andric
23510b57cec5SDimitry Andrictemplate <class _Tp>
23520b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23530b57cec5SDimitry Andrictypename enable_if
23540b57cec5SDimitry Andric<
23550b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23560b57cec5SDimitry Andric    _Tp
23570b57cec5SDimitry Andric>::type
23580b57cec5SDimitry Andricatomic_fetch_and_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
23590b57cec5SDimitry Andric{
23600b57cec5SDimitry Andric    return __o->fetch_and(__op, __m);
23610b57cec5SDimitry Andric}
23620b57cec5SDimitry Andric
23630b57cec5SDimitry Andrictemplate <class _Tp>
23640b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23650b57cec5SDimitry Andrictypename enable_if
23660b57cec5SDimitry Andric<
23670b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23680b57cec5SDimitry Andric    _Tp
23690b57cec5SDimitry Andric>::type
23700b57cec5SDimitry Andricatomic_fetch_and_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
23710b57cec5SDimitry Andric{
23720b57cec5SDimitry Andric    return __o->fetch_and(__op, __m);
23730b57cec5SDimitry Andric}
23740b57cec5SDimitry Andric
23750b57cec5SDimitry Andric// atomic_fetch_or
23760b57cec5SDimitry Andric
23770b57cec5SDimitry Andrictemplate <class _Tp>
23780b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23790b57cec5SDimitry Andrictypename enable_if
23800b57cec5SDimitry Andric<
23810b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23820b57cec5SDimitry Andric    _Tp
23830b57cec5SDimitry Andric>::type
23840b57cec5SDimitry Andricatomic_fetch_or(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
23850b57cec5SDimitry Andric{
23860b57cec5SDimitry Andric    return __o->fetch_or(__op);
23870b57cec5SDimitry Andric}
23880b57cec5SDimitry Andric
23890b57cec5SDimitry Andrictemplate <class _Tp>
23900b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23910b57cec5SDimitry Andrictypename enable_if
23920b57cec5SDimitry Andric<
23930b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23940b57cec5SDimitry Andric    _Tp
23950b57cec5SDimitry Andric>::type
23960b57cec5SDimitry Andricatomic_fetch_or(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
23970b57cec5SDimitry Andric{
23980b57cec5SDimitry Andric    return __o->fetch_or(__op);
23990b57cec5SDimitry Andric}
24000b57cec5SDimitry Andric
24010b57cec5SDimitry Andric// atomic_fetch_or_explicit
24020b57cec5SDimitry Andric
24030b57cec5SDimitry Andrictemplate <class _Tp>
24040b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
24050b57cec5SDimitry Andrictypename enable_if
24060b57cec5SDimitry Andric<
24070b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
24080b57cec5SDimitry Andric    _Tp
24090b57cec5SDimitry Andric>::type
24100b57cec5SDimitry Andricatomic_fetch_or_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
24110b57cec5SDimitry Andric{
24120b57cec5SDimitry Andric    return __o->fetch_or(__op, __m);
24130b57cec5SDimitry Andric}
24140b57cec5SDimitry Andric
24150b57cec5SDimitry Andrictemplate <class _Tp>
24160b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
24170b57cec5SDimitry Andrictypename enable_if
24180b57cec5SDimitry Andric<
24190b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
24200b57cec5SDimitry Andric    _Tp
24210b57cec5SDimitry Andric>::type
24220b57cec5SDimitry Andricatomic_fetch_or_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
24230b57cec5SDimitry Andric{
24240b57cec5SDimitry Andric    return __o->fetch_or(__op, __m);
24250b57cec5SDimitry Andric}
24260b57cec5SDimitry Andric
24270b57cec5SDimitry Andric// atomic_fetch_xor
24280b57cec5SDimitry Andric
24290b57cec5SDimitry Andrictemplate <class _Tp>
24300b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
24310b57cec5SDimitry Andrictypename enable_if
24320b57cec5SDimitry Andric<
24330b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
24340b57cec5SDimitry Andric    _Tp
24350b57cec5SDimitry Andric>::type
24360b57cec5SDimitry Andricatomic_fetch_xor(volatile atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
24370b57cec5SDimitry Andric{
24380b57cec5SDimitry Andric    return __o->fetch_xor(__op);
24390b57cec5SDimitry Andric}
24400b57cec5SDimitry Andric
24410b57cec5SDimitry Andrictemplate <class _Tp>
24420b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
24430b57cec5SDimitry Andrictypename enable_if
24440b57cec5SDimitry Andric<
24450b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
24460b57cec5SDimitry Andric    _Tp
24470b57cec5SDimitry Andric>::type
24480b57cec5SDimitry Andricatomic_fetch_xor(atomic<_Tp>* __o, _Tp __op) _NOEXCEPT
24490b57cec5SDimitry Andric{
24500b57cec5SDimitry Andric    return __o->fetch_xor(__op);
24510b57cec5SDimitry Andric}
24520b57cec5SDimitry Andric
24530b57cec5SDimitry Andric// atomic_fetch_xor_explicit
24540b57cec5SDimitry Andric
24550b57cec5SDimitry Andrictemplate <class _Tp>
24560b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
24570b57cec5SDimitry Andrictypename enable_if
24580b57cec5SDimitry Andric<
24590b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
24600b57cec5SDimitry Andric    _Tp
24610b57cec5SDimitry Andric>::type
24620b57cec5SDimitry Andricatomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
24630b57cec5SDimitry Andric{
24640b57cec5SDimitry Andric    return __o->fetch_xor(__op, __m);
24650b57cec5SDimitry Andric}
24660b57cec5SDimitry Andric
24670b57cec5SDimitry Andrictemplate <class _Tp>
24680b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
24690b57cec5SDimitry Andrictypename enable_if
24700b57cec5SDimitry Andric<
24710b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
24720b57cec5SDimitry Andric    _Tp
24730b57cec5SDimitry Andric>::type
24740b57cec5SDimitry Andricatomic_fetch_xor_explicit(atomic<_Tp>* __o, _Tp __op, memory_order __m) _NOEXCEPT
24750b57cec5SDimitry Andric{
24760b57cec5SDimitry Andric    return __o->fetch_xor(__op, __m);
24770b57cec5SDimitry Andric}
24780b57cec5SDimitry Andric
24790b57cec5SDimitry Andric// flag type and operations
24800b57cec5SDimitry Andric
24810b57cec5SDimitry Andrictypedef struct atomic_flag
24820b57cec5SDimitry Andric{
24830b57cec5SDimitry Andric    __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_;
24840b57cec5SDimitry Andric
24850b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
2486*5ffd83dbSDimitry Andric    bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
2487*5ffd83dbSDimitry Andric        {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);}
2488*5ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
2489*5ffd83dbSDimitry Andric    bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
2490*5ffd83dbSDimitry Andric        {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);}
2491*5ffd83dbSDimitry Andric
2492*5ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
24930b57cec5SDimitry Andric    bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
24940b57cec5SDimitry Andric        {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);}
24950b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
24960b57cec5SDimitry Andric    bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT
24970b57cec5SDimitry Andric        {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);}
24980b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
24990b57cec5SDimitry Andric    void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
25000b57cec5SDimitry Andric        {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);}
25010b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
25020b57cec5SDimitry Andric    void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT
25030b57cec5SDimitry Andric        {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);}
25040b57cec5SDimitry Andric
2505*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2506*5ffd83dbSDimitry Andric    void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
2507*5ffd83dbSDimitry Andric        {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);}
2508*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2509*5ffd83dbSDimitry Andric    void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT
2510*5ffd83dbSDimitry Andric        {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);}
2511*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2512*5ffd83dbSDimitry Andric    void notify_one() volatile _NOEXCEPT
2513*5ffd83dbSDimitry Andric        {__cxx_atomic_notify_one(&__a_);}
2514*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2515*5ffd83dbSDimitry Andric    void notify_one() _NOEXCEPT
2516*5ffd83dbSDimitry Andric        {__cxx_atomic_notify_one(&__a_);}
2517*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2518*5ffd83dbSDimitry Andric    void notify_all() volatile _NOEXCEPT
2519*5ffd83dbSDimitry Andric        {__cxx_atomic_notify_all(&__a_);}
2520*5ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
2521*5ffd83dbSDimitry Andric    void notify_all() _NOEXCEPT
2522*5ffd83dbSDimitry Andric        {__cxx_atomic_notify_all(&__a_);}
2523*5ffd83dbSDimitry Andric
25240b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
25250b57cec5SDimitry Andric    atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT
25260b57cec5SDimitry Andric
25270b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
25280b57cec5SDimitry Andric    atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION
25290b57cec5SDimitry Andric
25300b57cec5SDimitry Andric#ifndef _LIBCPP_CXX03_LANG
25310b57cec5SDimitry Andric    atomic_flag(const atomic_flag&) = delete;
25320b57cec5SDimitry Andric    atomic_flag& operator=(const atomic_flag&) = delete;
25330b57cec5SDimitry Andric    atomic_flag& operator=(const atomic_flag&) volatile = delete;
25340b57cec5SDimitry Andric#else
25350b57cec5SDimitry Andricprivate:
2536*5ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
25370b57cec5SDimitry Andric    atomic_flag(const atomic_flag&);
2538*5ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
25390b57cec5SDimitry Andric    atomic_flag& operator=(const atomic_flag&);
2540*5ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
25410b57cec5SDimitry Andric    atomic_flag& operator=(const atomic_flag&) volatile;
25420b57cec5SDimitry Andric#endif
25430b57cec5SDimitry Andric} atomic_flag;
25440b57cec5SDimitry Andric
2545*5ffd83dbSDimitry Andric
2546*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
2547*5ffd83dbSDimitry Andricbool
2548*5ffd83dbSDimitry Andricatomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT
2549*5ffd83dbSDimitry Andric{
2550*5ffd83dbSDimitry Andric    return __o->test();
2551*5ffd83dbSDimitry Andric}
2552*5ffd83dbSDimitry Andric
2553*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
2554*5ffd83dbSDimitry Andricbool
2555*5ffd83dbSDimitry Andricatomic_flag_test(const atomic_flag* __o) _NOEXCEPT
2556*5ffd83dbSDimitry Andric{
2557*5ffd83dbSDimitry Andric    return __o->test();
2558*5ffd83dbSDimitry Andric}
2559*5ffd83dbSDimitry Andric
2560*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
2561*5ffd83dbSDimitry Andricbool
2562*5ffd83dbSDimitry Andricatomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
2563*5ffd83dbSDimitry Andric{
2564*5ffd83dbSDimitry Andric    return __o->test(__m);
2565*5ffd83dbSDimitry Andric}
2566*5ffd83dbSDimitry Andric
2567*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
2568*5ffd83dbSDimitry Andricbool
2569*5ffd83dbSDimitry Andricatomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT
2570*5ffd83dbSDimitry Andric{
2571*5ffd83dbSDimitry Andric    return __o->test(__m);
2572*5ffd83dbSDimitry Andric}
2573*5ffd83dbSDimitry Andric
25740b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
25750b57cec5SDimitry Andricbool
25760b57cec5SDimitry Andricatomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT
25770b57cec5SDimitry Andric{
25780b57cec5SDimitry Andric    return __o->test_and_set();
25790b57cec5SDimitry Andric}
25800b57cec5SDimitry Andric
25810b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
25820b57cec5SDimitry Andricbool
25830b57cec5SDimitry Andricatomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT
25840b57cec5SDimitry Andric{
25850b57cec5SDimitry Andric    return __o->test_and_set();
25860b57cec5SDimitry Andric}
25870b57cec5SDimitry Andric
25880b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
25890b57cec5SDimitry Andricbool
25900b57cec5SDimitry Andricatomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
25910b57cec5SDimitry Andric{
25920b57cec5SDimitry Andric    return __o->test_and_set(__m);
25930b57cec5SDimitry Andric}
25940b57cec5SDimitry Andric
25950b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
25960b57cec5SDimitry Andricbool
25970b57cec5SDimitry Andricatomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT
25980b57cec5SDimitry Andric{
25990b57cec5SDimitry Andric    return __o->test_and_set(__m);
26000b57cec5SDimitry Andric}
26010b57cec5SDimitry Andric
26020b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
26030b57cec5SDimitry Andricvoid
26040b57cec5SDimitry Andricatomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT
26050b57cec5SDimitry Andric{
26060b57cec5SDimitry Andric    __o->clear();
26070b57cec5SDimitry Andric}
26080b57cec5SDimitry Andric
26090b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
26100b57cec5SDimitry Andricvoid
26110b57cec5SDimitry Andricatomic_flag_clear(atomic_flag* __o) _NOEXCEPT
26120b57cec5SDimitry Andric{
26130b57cec5SDimitry Andric    __o->clear();
26140b57cec5SDimitry Andric}
26150b57cec5SDimitry Andric
26160b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
26170b57cec5SDimitry Andricvoid
26180b57cec5SDimitry Andricatomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
26190b57cec5SDimitry Andric{
26200b57cec5SDimitry Andric    __o->clear(__m);
26210b57cec5SDimitry Andric}
26220b57cec5SDimitry Andric
26230b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
26240b57cec5SDimitry Andricvoid
26250b57cec5SDimitry Andricatomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT
26260b57cec5SDimitry Andric{
26270b57cec5SDimitry Andric    __o->clear(__m);
26280b57cec5SDimitry Andric}
26290b57cec5SDimitry Andric
2630*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2631*5ffd83dbSDimitry Andricvoid
2632*5ffd83dbSDimitry Andricatomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT
2633*5ffd83dbSDimitry Andric{
2634*5ffd83dbSDimitry Andric    __o->wait(__v);
2635*5ffd83dbSDimitry Andric}
2636*5ffd83dbSDimitry Andric
2637*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2638*5ffd83dbSDimitry Andricvoid
2639*5ffd83dbSDimitry Andricatomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT
2640*5ffd83dbSDimitry Andric{
2641*5ffd83dbSDimitry Andric    __o->wait(__v);
2642*5ffd83dbSDimitry Andric}
2643*5ffd83dbSDimitry Andric
2644*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2645*5ffd83dbSDimitry Andricvoid
2646*5ffd83dbSDimitry Andricatomic_flag_wait_explicit(const volatile atomic_flag* __o,
2647*5ffd83dbSDimitry Andric                          bool __v, memory_order __m) _NOEXCEPT
2648*5ffd83dbSDimitry Andric{
2649*5ffd83dbSDimitry Andric    __o->wait(__v, __m);
2650*5ffd83dbSDimitry Andric}
2651*5ffd83dbSDimitry Andric
2652*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2653*5ffd83dbSDimitry Andricvoid
2654*5ffd83dbSDimitry Andricatomic_flag_wait_explicit(const atomic_flag* __o,
2655*5ffd83dbSDimitry Andric                          bool __v, memory_order __m) _NOEXCEPT
2656*5ffd83dbSDimitry Andric{
2657*5ffd83dbSDimitry Andric    __o->wait(__v, __m);
2658*5ffd83dbSDimitry Andric}
2659*5ffd83dbSDimitry Andric
2660*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2661*5ffd83dbSDimitry Andricvoid
2662*5ffd83dbSDimitry Andricatomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT
2663*5ffd83dbSDimitry Andric{
2664*5ffd83dbSDimitry Andric    __o->notify_one();
2665*5ffd83dbSDimitry Andric}
2666*5ffd83dbSDimitry Andric
2667*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2668*5ffd83dbSDimitry Andricvoid
2669*5ffd83dbSDimitry Andricatomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT
2670*5ffd83dbSDimitry Andric{
2671*5ffd83dbSDimitry Andric    __o->notify_one();
2672*5ffd83dbSDimitry Andric}
2673*5ffd83dbSDimitry Andric
2674*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2675*5ffd83dbSDimitry Andricvoid
2676*5ffd83dbSDimitry Andricatomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT
2677*5ffd83dbSDimitry Andric{
2678*5ffd83dbSDimitry Andric    __o->notify_all();
2679*5ffd83dbSDimitry Andric}
2680*5ffd83dbSDimitry Andric
2681*5ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
2682*5ffd83dbSDimitry Andricvoid
2683*5ffd83dbSDimitry Andricatomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT
2684*5ffd83dbSDimitry Andric{
2685*5ffd83dbSDimitry Andric    __o->notify_all();
2686*5ffd83dbSDimitry Andric}
2687*5ffd83dbSDimitry Andric
26880b57cec5SDimitry Andric// fences
26890b57cec5SDimitry Andric
26900b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
26910b57cec5SDimitry Andricvoid
26920b57cec5SDimitry Andricatomic_thread_fence(memory_order __m) _NOEXCEPT
26930b57cec5SDimitry Andric{
26940b57cec5SDimitry Andric    __cxx_atomic_thread_fence(__m);
26950b57cec5SDimitry Andric}
26960b57cec5SDimitry Andric
26970b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
26980b57cec5SDimitry Andricvoid
26990b57cec5SDimitry Andricatomic_signal_fence(memory_order __m) _NOEXCEPT
27000b57cec5SDimitry Andric{
27010b57cec5SDimitry Andric    __cxx_atomic_signal_fence(__m);
27020b57cec5SDimitry Andric}
27030b57cec5SDimitry Andric
27040b57cec5SDimitry Andric// Atomics for standard typedef types
27050b57cec5SDimitry Andric
27060b57cec5SDimitry Andrictypedef atomic<bool>               atomic_bool;
27070b57cec5SDimitry Andrictypedef atomic<char>               atomic_char;
27080b57cec5SDimitry Andrictypedef atomic<signed char>        atomic_schar;
27090b57cec5SDimitry Andrictypedef atomic<unsigned char>      atomic_uchar;
27100b57cec5SDimitry Andrictypedef atomic<short>              atomic_short;
27110b57cec5SDimitry Andrictypedef atomic<unsigned short>     atomic_ushort;
27120b57cec5SDimitry Andrictypedef atomic<int>                atomic_int;
27130b57cec5SDimitry Andrictypedef atomic<unsigned int>       atomic_uint;
27140b57cec5SDimitry Andrictypedef atomic<long>               atomic_long;
27150b57cec5SDimitry Andrictypedef atomic<unsigned long>      atomic_ulong;
27160b57cec5SDimitry Andrictypedef atomic<long long>          atomic_llong;
27170b57cec5SDimitry Andrictypedef atomic<unsigned long long> atomic_ullong;
27180b57cec5SDimitry Andrictypedef atomic<char16_t>           atomic_char16_t;
27190b57cec5SDimitry Andrictypedef atomic<char32_t>           atomic_char32_t;
27200b57cec5SDimitry Andrictypedef atomic<wchar_t>            atomic_wchar_t;
27210b57cec5SDimitry Andric
27220b57cec5SDimitry Andrictypedef atomic<int_least8_t>   atomic_int_least8_t;
27230b57cec5SDimitry Andrictypedef atomic<uint_least8_t>  atomic_uint_least8_t;
27240b57cec5SDimitry Andrictypedef atomic<int_least16_t>  atomic_int_least16_t;
27250b57cec5SDimitry Andrictypedef atomic<uint_least16_t> atomic_uint_least16_t;
27260b57cec5SDimitry Andrictypedef atomic<int_least32_t>  atomic_int_least32_t;
27270b57cec5SDimitry Andrictypedef atomic<uint_least32_t> atomic_uint_least32_t;
27280b57cec5SDimitry Andrictypedef atomic<int_least64_t>  atomic_int_least64_t;
27290b57cec5SDimitry Andrictypedef atomic<uint_least64_t> atomic_uint_least64_t;
27300b57cec5SDimitry Andric
27310b57cec5SDimitry Andrictypedef atomic<int_fast8_t>   atomic_int_fast8_t;
27320b57cec5SDimitry Andrictypedef atomic<uint_fast8_t>  atomic_uint_fast8_t;
27330b57cec5SDimitry Andrictypedef atomic<int_fast16_t>  atomic_int_fast16_t;
27340b57cec5SDimitry Andrictypedef atomic<uint_fast16_t> atomic_uint_fast16_t;
27350b57cec5SDimitry Andrictypedef atomic<int_fast32_t>  atomic_int_fast32_t;
27360b57cec5SDimitry Andrictypedef atomic<uint_fast32_t> atomic_uint_fast32_t;
27370b57cec5SDimitry Andrictypedef atomic<int_fast64_t>  atomic_int_fast64_t;
27380b57cec5SDimitry Andrictypedef atomic<uint_fast64_t> atomic_uint_fast64_t;
27390b57cec5SDimitry Andric
27400b57cec5SDimitry Andrictypedef atomic< int8_t>  atomic_int8_t;
27410b57cec5SDimitry Andrictypedef atomic<uint8_t>  atomic_uint8_t;
27420b57cec5SDimitry Andrictypedef atomic< int16_t> atomic_int16_t;
27430b57cec5SDimitry Andrictypedef atomic<uint16_t> atomic_uint16_t;
27440b57cec5SDimitry Andrictypedef atomic< int32_t> atomic_int32_t;
27450b57cec5SDimitry Andrictypedef atomic<uint32_t> atomic_uint32_t;
27460b57cec5SDimitry Andrictypedef atomic< int64_t> atomic_int64_t;
27470b57cec5SDimitry Andrictypedef atomic<uint64_t> atomic_uint64_t;
27480b57cec5SDimitry Andric
27490b57cec5SDimitry Andrictypedef atomic<intptr_t>  atomic_intptr_t;
27500b57cec5SDimitry Andrictypedef atomic<uintptr_t> atomic_uintptr_t;
27510b57cec5SDimitry Andrictypedef atomic<size_t>    atomic_size_t;
27520b57cec5SDimitry Andrictypedef atomic<ptrdiff_t> atomic_ptrdiff_t;
27530b57cec5SDimitry Andrictypedef atomic<intmax_t>  atomic_intmax_t;
27540b57cec5SDimitry Andrictypedef atomic<uintmax_t> atomic_uintmax_t;
27550b57cec5SDimitry Andric
2756*5ffd83dbSDimitry Andric// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type
2757*5ffd83dbSDimitry Andric
2758*5ffd83dbSDimitry Andric#ifdef __cpp_lib_atomic_is_always_lock_free
2759*5ffd83dbSDimitry Andric# define _LIBCPP_CONTENTION_LOCK_FREE __atomic_always_lock_free(sizeof(__cxx_contention_t), 0)
2760*5ffd83dbSDimitry Andric#else
2761*5ffd83dbSDimitry Andric# define _LIBCPP_CONTENTION_LOCK_FREE false
2762*5ffd83dbSDimitry Andric#endif
2763*5ffd83dbSDimitry Andric
2764*5ffd83dbSDimitry Andric#if ATOMIC_LLONG_LOCK_FREE == 2
2765*5ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>::type          __libcpp_signed_lock_free;
2766*5ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long>::type __libcpp_unsigned_lock_free;
2767*5ffd83dbSDimitry Andric#elif ATOMIC_INT_LOCK_FREE == 2
2768*5ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>::type                __libcpp_signed_lock_free;
2769*5ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>::type       __libcpp_unsigned_lock_free;
2770*5ffd83dbSDimitry Andric#elif ATOMIC_SHORT_LOCK_FREE == 2
2771*5ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>::type              __libcpp_signed_lock_free;
2772*5ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>::type     __libcpp_unsigned_lock_free;
2773*5ffd83dbSDimitry Andric#elif ATOMIC_CHAR_LOCK_FREE == 2
2774*5ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>::type               __libcpp_signed_lock_free;
2775*5ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>::type      __libcpp_unsigned_lock_free;
2776*5ffd83dbSDimitry Andric#else
2777*5ffd83dbSDimitry Andric    // No signed/unsigned lock-free types
2778*5ffd83dbSDimitry Andric#endif
2779*5ffd83dbSDimitry Andric
2780*5ffd83dbSDimitry Andrictypedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free;
2781*5ffd83dbSDimitry Andrictypedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free;
2782*5ffd83dbSDimitry Andric
27830b57cec5SDimitry Andric#define ATOMIC_FLAG_INIT {false}
27840b57cec5SDimitry Andric#define ATOMIC_VAR_INIT(__v) {__v}
27850b57cec5SDimitry Andric
27860b57cec5SDimitry Andric_LIBCPP_END_NAMESPACE_STD
27870b57cec5SDimitry Andric
27880b57cec5SDimitry Andric#endif  // _LIBCPP_ATOMIC
2789