xref: /freebsd/contrib/llvm-project/libcxx/include/atomic (revision 349cc55c9796c4596a5b9904cd3281af295f878f)
10b57cec5SDimitry Andric// -*- C++ -*-
2*349cc55cSDimitry Andric//===----------------------------------------------------------------------===//
30b57cec5SDimitry Andric//
40b57cec5SDimitry Andric// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
50b57cec5SDimitry Andric// See https://llvm.org/LICENSE.txt for license information.
60b57cec5SDimitry Andric// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
70b57cec5SDimitry Andric//
80b57cec5SDimitry Andric//===----------------------------------------------------------------------===//
90b57cec5SDimitry Andric
100b57cec5SDimitry Andric#ifndef _LIBCPP_ATOMIC
110b57cec5SDimitry Andric#define _LIBCPP_ATOMIC
120b57cec5SDimitry Andric
130b57cec5SDimitry Andric/*
140b57cec5SDimitry Andric    atomic synopsis
150b57cec5SDimitry Andric
160b57cec5SDimitry Andricnamespace std
170b57cec5SDimitry Andric{
180b57cec5SDimitry Andric
19e8d8bef9SDimitry Andric// feature test macro [version.syn]
200b57cec5SDimitry Andric
21e8d8bef9SDimitry Andric#define __cpp_lib_atomic_is_always_lock_free
22e8d8bef9SDimitry Andric#define __cpp_lib_atomic_flag_test
23e8d8bef9SDimitry Andric#define __cpp_lib_atomic_lock_free_type_aliases
24e8d8bef9SDimitry Andric#define __cpp_lib_atomic_wait
250b57cec5SDimitry Andric
260b57cec5SDimitry Andric // order and consistency
270b57cec5SDimitry Andric
280b57cec5SDimitry Andric enum memory_order: unspecified // enum class in C++20
290b57cec5SDimitry Andric {
300b57cec5SDimitry Andric    relaxed,
310b57cec5SDimitry Andric    consume, // load-consume
320b57cec5SDimitry Andric    acquire, // load-acquire
330b57cec5SDimitry Andric    release, // store-release
340b57cec5SDimitry Andric    acq_rel, // store-release load-acquire
350b57cec5SDimitry Andric    seq_cst // store-release load-acquire
360b57cec5SDimitry Andric };
370b57cec5SDimitry Andric
380b57cec5SDimitry Andric inline constexpr auto memory_order_relaxed = memory_order::relaxed;
390b57cec5SDimitry Andric inline constexpr auto memory_order_consume = memory_order::consume;
400b57cec5SDimitry Andric inline constexpr auto memory_order_acquire = memory_order::acquire;
410b57cec5SDimitry Andric inline constexpr auto memory_order_release = memory_order::release;
420b57cec5SDimitry Andric inline constexpr auto memory_order_acq_rel = memory_order::acq_rel;
430b57cec5SDimitry Andric inline constexpr auto memory_order_seq_cst = memory_order::seq_cst;
440b57cec5SDimitry Andric
450b57cec5SDimitry Andrictemplate <class T> T kill_dependency(T y) noexcept;
460b57cec5SDimitry Andric
470b57cec5SDimitry Andric// lock-free property
480b57cec5SDimitry Andric
490b57cec5SDimitry Andric#define ATOMIC_BOOL_LOCK_FREE unspecified
500b57cec5SDimitry Andric#define ATOMIC_CHAR_LOCK_FREE unspecified
51e8d8bef9SDimitry Andric#define ATOMIC_CHAR8_T_LOCK_FREE unspecified // C++20
520b57cec5SDimitry Andric#define ATOMIC_CHAR16_T_LOCK_FREE unspecified
530b57cec5SDimitry Andric#define ATOMIC_CHAR32_T_LOCK_FREE unspecified
540b57cec5SDimitry Andric#define ATOMIC_WCHAR_T_LOCK_FREE unspecified
550b57cec5SDimitry Andric#define ATOMIC_SHORT_LOCK_FREE unspecified
560b57cec5SDimitry Andric#define ATOMIC_INT_LOCK_FREE unspecified
570b57cec5SDimitry Andric#define ATOMIC_LONG_LOCK_FREE unspecified
580b57cec5SDimitry Andric#define ATOMIC_LLONG_LOCK_FREE unspecified
590b57cec5SDimitry Andric#define ATOMIC_POINTER_LOCK_FREE unspecified
600b57cec5SDimitry Andric
610b57cec5SDimitry Andrictemplate <class T>
620b57cec5SDimitry Andricstruct atomic
630b57cec5SDimitry Andric{
645ffd83dbSDimitry Andric    using value_type = T;
655ffd83dbSDimitry Andric
660b57cec5SDimitry Andric    static constexpr bool is_always_lock_free;
670b57cec5SDimitry Andric    bool is_lock_free() const volatile noexcept;
680b57cec5SDimitry Andric    bool is_lock_free() const noexcept;
695ffd83dbSDimitry Andric
70fe6060f1SDimitry Andric    atomic() noexcept = default; // until C++20
71fe6060f1SDimitry Andric    constexpr atomic() noexcept(is_nothrow_default_constructible_v<T>); // since C++20
725ffd83dbSDimitry Andric    constexpr atomic(T desr) noexcept;
735ffd83dbSDimitry Andric    atomic(const atomic&) = delete;
745ffd83dbSDimitry Andric    atomic& operator=(const atomic&) = delete;
755ffd83dbSDimitry Andric    atomic& operator=(const atomic&) volatile = delete;
765ffd83dbSDimitry Andric
770b57cec5SDimitry Andric    T load(memory_order m = memory_order_seq_cst) const volatile noexcept;
780b57cec5SDimitry Andric    T load(memory_order m = memory_order_seq_cst) const noexcept;
790b57cec5SDimitry Andric    operator T() const volatile noexcept;
800b57cec5SDimitry Andric    operator T() const noexcept;
815ffd83dbSDimitry Andric    void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;
825ffd83dbSDimitry Andric    void store(T desr, memory_order m = memory_order_seq_cst) noexcept;
835ffd83dbSDimitry Andric    T operator=(T) volatile noexcept;
845ffd83dbSDimitry Andric    T operator=(T) noexcept;
855ffd83dbSDimitry Andric
860b57cec5SDimitry Andric    T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;
870b57cec5SDimitry Andric    T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept;
880b57cec5SDimitry Andric    bool compare_exchange_weak(T& expc, T desr,
890b57cec5SDimitry Andric                               memory_order s, memory_order f) volatile noexcept;
900b57cec5SDimitry Andric    bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept;
910b57cec5SDimitry Andric    bool compare_exchange_strong(T& expc, T desr,
920b57cec5SDimitry Andric                                 memory_order s, memory_order f) volatile noexcept;
930b57cec5SDimitry Andric    bool compare_exchange_strong(T& expc, T desr,
940b57cec5SDimitry Andric                                 memory_order s, memory_order f) noexcept;
950b57cec5SDimitry Andric    bool compare_exchange_weak(T& expc, T desr,
960b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) volatile noexcept;
970b57cec5SDimitry Andric    bool compare_exchange_weak(T& expc, T desr,
980b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) noexcept;
990b57cec5SDimitry Andric    bool compare_exchange_strong(T& expc, T desr,
1000b57cec5SDimitry Andric                                memory_order m = memory_order_seq_cst) volatile noexcept;
1010b57cec5SDimitry Andric    bool compare_exchange_strong(T& expc, T desr,
1020b57cec5SDimitry Andric                                 memory_order m = memory_order_seq_cst) noexcept;
1030b57cec5SDimitry Andric
1045ffd83dbSDimitry Andric    void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept;
1055ffd83dbSDimitry Andric    void wait(T, memory_order = memory_order::seq_cst) const noexcept;
1065ffd83dbSDimitry Andric    void notify_one() volatile noexcept;
1075ffd83dbSDimitry Andric    void notify_one() noexcept;
1085ffd83dbSDimitry Andric    void notify_all() volatile noexcept;
1095ffd83dbSDimitry Andric    void notify_all() noexcept;
1100b57cec5SDimitry Andric};
1110b57cec5SDimitry Andric
1120b57cec5SDimitry Andrictemplate <>
1130b57cec5SDimitry Andricstruct atomic<integral>
1140b57cec5SDimitry Andric{
1155ffd83dbSDimitry Andric    using value_type = integral;
116e8d8bef9SDimitry Andric    using difference_type = value_type;
1175ffd83dbSDimitry Andric
1180b57cec5SDimitry Andric    static constexpr bool is_always_lock_free;
1190b57cec5SDimitry Andric    bool is_lock_free() const volatile noexcept;
1200b57cec5SDimitry Andric    bool is_lock_free() const noexcept;
1215ffd83dbSDimitry Andric
1225ffd83dbSDimitry Andric    atomic() noexcept = default;
1235ffd83dbSDimitry Andric    constexpr atomic(integral desr) noexcept;
1245ffd83dbSDimitry Andric    atomic(const atomic&) = delete;
1255ffd83dbSDimitry Andric    atomic& operator=(const atomic&) = delete;
1265ffd83dbSDimitry Andric    atomic& operator=(const atomic&) volatile = delete;
1275ffd83dbSDimitry Andric
1280b57cec5SDimitry Andric    integral load(memory_order m = memory_order_seq_cst) const volatile noexcept;
1290b57cec5SDimitry Andric    integral load(memory_order m = memory_order_seq_cst) const noexcept;
1300b57cec5SDimitry Andric    operator integral() const volatile noexcept;
1310b57cec5SDimitry Andric    operator integral() const noexcept;
1325ffd83dbSDimitry Andric    void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept;
1335ffd83dbSDimitry Andric    void store(integral desr, memory_order m = memory_order_seq_cst) noexcept;
1345ffd83dbSDimitry Andric    integral operator=(integral desr) volatile noexcept;
1355ffd83dbSDimitry Andric    integral operator=(integral desr) noexcept;
1365ffd83dbSDimitry Andric
1370b57cec5SDimitry Andric    integral exchange(integral desr,
1380b57cec5SDimitry Andric                      memory_order m = memory_order_seq_cst) volatile noexcept;
1390b57cec5SDimitry Andric    integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept;
1400b57cec5SDimitry Andric    bool compare_exchange_weak(integral& expc, integral desr,
1410b57cec5SDimitry Andric                               memory_order s, memory_order f) volatile noexcept;
1420b57cec5SDimitry Andric    bool compare_exchange_weak(integral& expc, integral desr,
1430b57cec5SDimitry Andric                               memory_order s, memory_order f) noexcept;
1440b57cec5SDimitry Andric    bool compare_exchange_strong(integral& expc, integral desr,
1450b57cec5SDimitry Andric                                 memory_order s, memory_order f) volatile noexcept;
1460b57cec5SDimitry Andric    bool compare_exchange_strong(integral& expc, integral desr,
1470b57cec5SDimitry Andric                                 memory_order s, memory_order f) noexcept;
1480b57cec5SDimitry Andric    bool compare_exchange_weak(integral& expc, integral desr,
1490b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) volatile noexcept;
1500b57cec5SDimitry Andric    bool compare_exchange_weak(integral& expc, integral desr,
1510b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) noexcept;
1520b57cec5SDimitry Andric    bool compare_exchange_strong(integral& expc, integral desr,
1530b57cec5SDimitry Andric                                memory_order m = memory_order_seq_cst) volatile noexcept;
1540b57cec5SDimitry Andric    bool compare_exchange_strong(integral& expc, integral desr,
1550b57cec5SDimitry Andric                                 memory_order m = memory_order_seq_cst) noexcept;
1560b57cec5SDimitry Andric
1575ffd83dbSDimitry Andric    integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
1580b57cec5SDimitry Andric    integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept;
1595ffd83dbSDimitry Andric    integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
1600b57cec5SDimitry Andric    integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept;
1615ffd83dbSDimitry Andric    integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
1620b57cec5SDimitry Andric    integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept;
1635ffd83dbSDimitry Andric    integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
1640b57cec5SDimitry Andric    integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept;
1655ffd83dbSDimitry Andric    integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;
1660b57cec5SDimitry Andric    integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept;
1670b57cec5SDimitry Andric
1680b57cec5SDimitry Andric    integral operator++(int) volatile noexcept;
1690b57cec5SDimitry Andric    integral operator++(int) noexcept;
1700b57cec5SDimitry Andric    integral operator--(int) volatile noexcept;
1710b57cec5SDimitry Andric    integral operator--(int) noexcept;
1720b57cec5SDimitry Andric    integral operator++() volatile noexcept;
1730b57cec5SDimitry Andric    integral operator++() noexcept;
1740b57cec5SDimitry Andric    integral operator--() volatile noexcept;
1750b57cec5SDimitry Andric    integral operator--() noexcept;
1760b57cec5SDimitry Andric    integral operator+=(integral op) volatile noexcept;
1770b57cec5SDimitry Andric    integral operator+=(integral op) noexcept;
1780b57cec5SDimitry Andric    integral operator-=(integral op) volatile noexcept;
1790b57cec5SDimitry Andric    integral operator-=(integral op) noexcept;
1800b57cec5SDimitry Andric    integral operator&=(integral op) volatile noexcept;
1810b57cec5SDimitry Andric    integral operator&=(integral op) noexcept;
1820b57cec5SDimitry Andric    integral operator|=(integral op) volatile noexcept;
1830b57cec5SDimitry Andric    integral operator|=(integral op) noexcept;
1840b57cec5SDimitry Andric    integral operator^=(integral op) volatile noexcept;
1850b57cec5SDimitry Andric    integral operator^=(integral op) noexcept;
1865ffd83dbSDimitry Andric
1875ffd83dbSDimitry Andric    void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept;
1885ffd83dbSDimitry Andric    void wait(integral, memory_order = memory_order::seq_cst) const noexcept;
1895ffd83dbSDimitry Andric    void notify_one() volatile noexcept;
1905ffd83dbSDimitry Andric    void notify_one() noexcept;
1915ffd83dbSDimitry Andric    void notify_all() volatile noexcept;
1925ffd83dbSDimitry Andric    void notify_all() noexcept;
1930b57cec5SDimitry Andric};
1940b57cec5SDimitry Andric
1950b57cec5SDimitry Andrictemplate <class T>
1960b57cec5SDimitry Andricstruct atomic<T*>
1970b57cec5SDimitry Andric{
1985ffd83dbSDimitry Andric    using value_type = T*;
199e8d8bef9SDimitry Andric    using difference_type = ptrdiff_t;
2005ffd83dbSDimitry Andric
2010b57cec5SDimitry Andric    static constexpr bool is_always_lock_free;
2020b57cec5SDimitry Andric    bool is_lock_free() const volatile noexcept;
2030b57cec5SDimitry Andric    bool is_lock_free() const noexcept;
2045ffd83dbSDimitry Andric
205fe6060f1SDimitry Andric    atomic() noexcept = default; // until C++20
206fe6060f1SDimitry Andric    constexpr atomic() noexcept; // since C++20
2075ffd83dbSDimitry Andric    constexpr atomic(T* desr) noexcept;
2085ffd83dbSDimitry Andric    atomic(const atomic&) = delete;
2095ffd83dbSDimitry Andric    atomic& operator=(const atomic&) = delete;
2105ffd83dbSDimitry Andric    atomic& operator=(const atomic&) volatile = delete;
2115ffd83dbSDimitry Andric
2120b57cec5SDimitry Andric    T* load(memory_order m = memory_order_seq_cst) const volatile noexcept;
2130b57cec5SDimitry Andric    T* load(memory_order m = memory_order_seq_cst) const noexcept;
2140b57cec5SDimitry Andric    operator T*() const volatile noexcept;
2150b57cec5SDimitry Andric    operator T*() const noexcept;
2165ffd83dbSDimitry Andric    void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;
2175ffd83dbSDimitry Andric    void store(T* desr, memory_order m = memory_order_seq_cst) noexcept;
2185ffd83dbSDimitry Andric    T* operator=(T*) volatile noexcept;
2195ffd83dbSDimitry Andric    T* operator=(T*) noexcept;
2205ffd83dbSDimitry Andric
2210b57cec5SDimitry Andric    T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;
2220b57cec5SDimitry Andric    T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept;
2230b57cec5SDimitry Andric    bool compare_exchange_weak(T*& expc, T* desr,
2240b57cec5SDimitry Andric                               memory_order s, memory_order f) volatile noexcept;
2250b57cec5SDimitry Andric    bool compare_exchange_weak(T*& expc, T* desr,
2260b57cec5SDimitry Andric                               memory_order s, memory_order f) noexcept;
2270b57cec5SDimitry Andric    bool compare_exchange_strong(T*& expc, T* desr,
2280b57cec5SDimitry Andric                                 memory_order s, memory_order f) volatile noexcept;
2290b57cec5SDimitry Andric    bool compare_exchange_strong(T*& expc, T* desr,
2300b57cec5SDimitry Andric                                 memory_order s, memory_order f) noexcept;
2310b57cec5SDimitry Andric    bool compare_exchange_weak(T*& expc, T* desr,
2320b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) volatile noexcept;
2330b57cec5SDimitry Andric    bool compare_exchange_weak(T*& expc, T* desr,
2340b57cec5SDimitry Andric                               memory_order m = memory_order_seq_cst) noexcept;
2350b57cec5SDimitry Andric    bool compare_exchange_strong(T*& expc, T* desr,
2360b57cec5SDimitry Andric                                memory_order m = memory_order_seq_cst) volatile noexcept;
2370b57cec5SDimitry Andric    bool compare_exchange_strong(T*& expc, T* desr,
2380b57cec5SDimitry Andric                                 memory_order m = memory_order_seq_cst) noexcept;
2390b57cec5SDimitry Andric    T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;
2400b57cec5SDimitry Andric    T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;
2410b57cec5SDimitry Andric    T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;
2420b57cec5SDimitry Andric    T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;
2430b57cec5SDimitry Andric
2440b57cec5SDimitry Andric    T* operator++(int) volatile noexcept;
2450b57cec5SDimitry Andric    T* operator++(int) noexcept;
2460b57cec5SDimitry Andric    T* operator--(int) volatile noexcept;
2470b57cec5SDimitry Andric    T* operator--(int) noexcept;
2480b57cec5SDimitry Andric    T* operator++() volatile noexcept;
2490b57cec5SDimitry Andric    T* operator++() noexcept;
2500b57cec5SDimitry Andric    T* operator--() volatile noexcept;
2510b57cec5SDimitry Andric    T* operator--() noexcept;
2520b57cec5SDimitry Andric    T* operator+=(ptrdiff_t op) volatile noexcept;
2530b57cec5SDimitry Andric    T* operator+=(ptrdiff_t op) noexcept;
2540b57cec5SDimitry Andric    T* operator-=(ptrdiff_t op) volatile noexcept;
2550b57cec5SDimitry Andric    T* operator-=(ptrdiff_t op) noexcept;
2565ffd83dbSDimitry Andric
2575ffd83dbSDimitry Andric    void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept;
2585ffd83dbSDimitry Andric    void wait(T*, memory_order = memory_order::seq_cst) const noexcept;
2595ffd83dbSDimitry Andric    void notify_one() volatile noexcept;
2605ffd83dbSDimitry Andric    void notify_one() noexcept;
2615ffd83dbSDimitry Andric    void notify_all() volatile noexcept;
2625ffd83dbSDimitry Andric    void notify_all() noexcept;
2630b57cec5SDimitry Andric};
2640b57cec5SDimitry Andric
2650b57cec5SDimitry Andric
266*349cc55cSDimitry Andric// [atomics.nonmembers], non-member functions
2670b57cec5SDimitry Andrictemplate<class T>
268*349cc55cSDimitry Andric  bool atomic_is_lock_free(const volatile atomic<T>*) noexcept;
269*349cc55cSDimitry Andrictemplate<class T>
270*349cc55cSDimitry Andric  bool atomic_is_lock_free(const atomic<T>*) noexcept;
271*349cc55cSDimitry Andrictemplate<class T>
272*349cc55cSDimitry Andric  void atomic_store(volatile atomic<T>*, atomic<T>::value_type) noexcept;
273*349cc55cSDimitry Andrictemplate<class T>
274*349cc55cSDimitry Andric  void atomic_store(atomic<T>*, atomic<T>::value_type) noexcept;
275*349cc55cSDimitry Andrictemplate<class T>
276*349cc55cSDimitry Andric  void atomic_store_explicit(volatile atomic<T>*, atomic<T>::value_type,
277*349cc55cSDimitry Andric                             memory_order) noexcept;
278*349cc55cSDimitry Andrictemplate<class T>
279*349cc55cSDimitry Andric  void atomic_store_explicit(atomic<T>*, atomic<T>::value_type,
280*349cc55cSDimitry Andric                             memory_order) noexcept;
281*349cc55cSDimitry Andrictemplate<class T>
282*349cc55cSDimitry Andric  T atomic_load(const volatile atomic<T>*) noexcept;
283*349cc55cSDimitry Andrictemplate<class T>
284*349cc55cSDimitry Andric  T atomic_load(const atomic<T>*) noexcept;
285*349cc55cSDimitry Andrictemplate<class T>
286*349cc55cSDimitry Andric  T atomic_load_explicit(const volatile atomic<T>*, memory_order) noexcept;
287*349cc55cSDimitry Andrictemplate<class T>
288*349cc55cSDimitry Andric  T atomic_load_explicit(const atomic<T>*, memory_order) noexcept;
289*349cc55cSDimitry Andrictemplate<class T>
290*349cc55cSDimitry Andric  T atomic_exchange(volatile atomic<T>*, atomic<T>::value_type) noexcept;
291*349cc55cSDimitry Andrictemplate<class T>
292*349cc55cSDimitry Andric  T atomic_exchange(atomic<T>*, atomic<T>::value_type) noexcept;
293*349cc55cSDimitry Andrictemplate<class T>
294*349cc55cSDimitry Andric  T atomic_exchange_explicit(volatile atomic<T>*, atomic<T>::value_type,
295*349cc55cSDimitry Andric                             memory_order) noexcept;
296*349cc55cSDimitry Andrictemplate<class T>
297*349cc55cSDimitry Andric  T atomic_exchange_explicit(atomic<T>*, atomic<T>::value_type,
298*349cc55cSDimitry Andric                             memory_order) noexcept;
299*349cc55cSDimitry Andrictemplate<class T>
300*349cc55cSDimitry Andric  bool atomic_compare_exchange_weak(volatile atomic<T>*, atomic<T>::value_type*,
301*349cc55cSDimitry Andric                                    atomic<T>::value_type) noexcept;
302*349cc55cSDimitry Andrictemplate<class T>
303*349cc55cSDimitry Andric  bool atomic_compare_exchange_weak(atomic<T>*, atomic<T>::value_type*,
304*349cc55cSDimitry Andric                                    atomic<T>::value_type) noexcept;
305*349cc55cSDimitry Andrictemplate<class T>
306*349cc55cSDimitry Andric  bool atomic_compare_exchange_strong(volatile atomic<T>*, atomic<T>::value_type*,
307*349cc55cSDimitry Andric                                      atomic<T>::value_type) noexcept;
308*349cc55cSDimitry Andrictemplate<class T>
309*349cc55cSDimitry Andric  bool atomic_compare_exchange_strong(atomic<T>*, atomic<T>::value_type*,
310*349cc55cSDimitry Andric                                      atomic<T>::value_type) noexcept;
311*349cc55cSDimitry Andrictemplate<class T>
312*349cc55cSDimitry Andric  bool atomic_compare_exchange_weak_explicit(volatile atomic<T>*, atomic<T>::value_type*,
313*349cc55cSDimitry Andric                                             atomic<T>::value_type,
314*349cc55cSDimitry Andric                                             memory_order, memory_order) noexcept;
315*349cc55cSDimitry Andrictemplate<class T>
316*349cc55cSDimitry Andric  bool atomic_compare_exchange_weak_explicit(atomic<T>*, atomic<T>::value_type*,
317*349cc55cSDimitry Andric                                             atomic<T>::value_type,
318*349cc55cSDimitry Andric                                             memory_order, memory_order) noexcept;
319*349cc55cSDimitry Andrictemplate<class T>
320*349cc55cSDimitry Andric  bool atomic_compare_exchange_strong_explicit(volatile atomic<T>*, atomic<T>::value_type*,
321*349cc55cSDimitry Andric                                               atomic<T>::value_type,
322*349cc55cSDimitry Andric                                               memory_order, memory_order) noexcept;
323*349cc55cSDimitry Andrictemplate<class T>
324*349cc55cSDimitry Andric  bool atomic_compare_exchange_strong_explicit(atomic<T>*, atomic<T>::value_type*,
325*349cc55cSDimitry Andric                                               atomic<T>::value_type,
326*349cc55cSDimitry Andric                                               memory_order, memory_order) noexcept;
3270b57cec5SDimitry Andric
3280b57cec5SDimitry Andrictemplate<class T>
329*349cc55cSDimitry Andric  T atomic_fetch_add(volatile atomic<T>*, atomic<T>::difference_type) noexcept;
330*349cc55cSDimitry Andrictemplate<class T>
331*349cc55cSDimitry Andric  T atomic_fetch_add(atomic<T>*, atomic<T>::difference_type) noexcept;
332*349cc55cSDimitry Andrictemplate<class T>
333*349cc55cSDimitry Andric  T atomic_fetch_add_explicit(volatile atomic<T>*, atomic<T>::difference_type,
334*349cc55cSDimitry Andric                              memory_order) noexcept;
335*349cc55cSDimitry Andrictemplate<class T>
336*349cc55cSDimitry Andric  T atomic_fetch_add_explicit(atomic<T>*, atomic<T>::difference_type,
337*349cc55cSDimitry Andric                              memory_order) noexcept;
338*349cc55cSDimitry Andrictemplate<class T>
339*349cc55cSDimitry Andric  T atomic_fetch_sub(volatile atomic<T>*, atomic<T>::difference_type) noexcept;
340*349cc55cSDimitry Andrictemplate<class T>
341*349cc55cSDimitry Andric  T atomic_fetch_sub(atomic<T>*, atomic<T>::difference_type) noexcept;
342*349cc55cSDimitry Andrictemplate<class T>
343*349cc55cSDimitry Andric  T atomic_fetch_sub_explicit(volatile atomic<T>*, atomic<T>::difference_type,
344*349cc55cSDimitry Andric                              memory_order) noexcept;
345*349cc55cSDimitry Andrictemplate<class T>
346*349cc55cSDimitry Andric  T atomic_fetch_sub_explicit(atomic<T>*, atomic<T>::difference_type,
347*349cc55cSDimitry Andric                              memory_order) noexcept;
348*349cc55cSDimitry Andrictemplate<class T>
349*349cc55cSDimitry Andric  T atomic_fetch_and(volatile atomic<T>*, atomic<T>::value_type) noexcept;
350*349cc55cSDimitry Andrictemplate<class T>
351*349cc55cSDimitry Andric  T atomic_fetch_and(atomic<T>*, atomic<T>::value_type) noexcept;
352*349cc55cSDimitry Andrictemplate<class T>
353*349cc55cSDimitry Andric  T atomic_fetch_and_explicit(volatile atomic<T>*, atomic<T>::value_type,
354*349cc55cSDimitry Andric                              memory_order) noexcept;
355*349cc55cSDimitry Andrictemplate<class T>
356*349cc55cSDimitry Andric  T atomic_fetch_and_explicit(atomic<T>*, atomic<T>::value_type,
357*349cc55cSDimitry Andric                              memory_order) noexcept;
358*349cc55cSDimitry Andrictemplate<class T>
359*349cc55cSDimitry Andric  T atomic_fetch_or(volatile atomic<T>*, atomic<T>::value_type) noexcept;
360*349cc55cSDimitry Andrictemplate<class T>
361*349cc55cSDimitry Andric  T atomic_fetch_or(atomic<T>*, atomic<T>::value_type) noexcept;
362*349cc55cSDimitry Andrictemplate<class T>
363*349cc55cSDimitry Andric  T atomic_fetch_or_explicit(volatile atomic<T>*, atomic<T>::value_type,
364*349cc55cSDimitry Andric                             memory_order) noexcept;
365*349cc55cSDimitry Andrictemplate<class T>
366*349cc55cSDimitry Andric  T atomic_fetch_or_explicit(atomic<T>*, atomic<T>::value_type,
367*349cc55cSDimitry Andric                             memory_order) noexcept;
368*349cc55cSDimitry Andrictemplate<class T>
369*349cc55cSDimitry Andric  T atomic_fetch_xor(volatile atomic<T>*, atomic<T>::value_type) noexcept;
370*349cc55cSDimitry Andrictemplate<class T>
371*349cc55cSDimitry Andric  T atomic_fetch_xor(atomic<T>*, atomic<T>::value_type) noexcept;
372*349cc55cSDimitry Andrictemplate<class T>
373*349cc55cSDimitry Andric  T atomic_fetch_xor_explicit(volatile atomic<T>*, atomic<T>::value_type,
374*349cc55cSDimitry Andric                              memory_order) noexcept;
375*349cc55cSDimitry Andrictemplate<class T>
376*349cc55cSDimitry Andric  T atomic_fetch_xor_explicit(atomic<T>*, atomic<T>::value_type,
377*349cc55cSDimitry Andric                              memory_order) noexcept;
3780b57cec5SDimitry Andric
3790b57cec5SDimitry Andrictemplate<class T>
380*349cc55cSDimitry Andric  void atomic_wait(const volatile atomic<T>*, atomic<T>::value_type);
3810b57cec5SDimitry Andrictemplate<class T>
382*349cc55cSDimitry Andric  void atomic_wait(const atomic<T>*, atomic<T>::value_type);
3830b57cec5SDimitry Andrictemplate<class T>
384*349cc55cSDimitry Andric  void atomic_wait_explicit(const volatile atomic<T>*, atomic<T>::value_type,
385*349cc55cSDimitry Andric                            memory_order);
3860b57cec5SDimitry Andrictemplate<class T>
387*349cc55cSDimitry Andric  void atomic_wait_explicit(const atomic<T>*, atomic<T>::value_type,
388*349cc55cSDimitry Andric                            memory_order);
3890b57cec5SDimitry Andrictemplate<class T>
390*349cc55cSDimitry Andric  void atomic_notify_one(volatile atomic<T>*);
3910b57cec5SDimitry Andrictemplate<class T>
392*349cc55cSDimitry Andric  void atomic_notify_one(atomic<T>*);
3930b57cec5SDimitry Andrictemplate<class T>
394*349cc55cSDimitry Andric  void atomic_notify_all(volatile atomic<T>*);
3950b57cec5SDimitry Andrictemplate<class T>
396*349cc55cSDimitry Andric  void atomic_notify_all(atomic<T>*);
3970b57cec5SDimitry Andric
3980b57cec5SDimitry Andric// Atomics for standard typedef types
3990b57cec5SDimitry Andric
4000b57cec5SDimitry Andrictypedef atomic<bool>               atomic_bool;
4010b57cec5SDimitry Andrictypedef atomic<char>               atomic_char;
4020b57cec5SDimitry Andrictypedef atomic<signed char>        atomic_schar;
4030b57cec5SDimitry Andrictypedef atomic<unsigned char>      atomic_uchar;
4040b57cec5SDimitry Andrictypedef atomic<short>              atomic_short;
4050b57cec5SDimitry Andrictypedef atomic<unsigned short>     atomic_ushort;
4060b57cec5SDimitry Andrictypedef atomic<int>                atomic_int;
4070b57cec5SDimitry Andrictypedef atomic<unsigned int>       atomic_uint;
4080b57cec5SDimitry Andrictypedef atomic<long>               atomic_long;
4090b57cec5SDimitry Andrictypedef atomic<unsigned long>      atomic_ulong;
4100b57cec5SDimitry Andrictypedef atomic<long long>          atomic_llong;
4110b57cec5SDimitry Andrictypedef atomic<unsigned long long> atomic_ullong;
412e8d8bef9SDimitry Andrictypedef atomic<char8_t>            atomic_char8_t; // C++20
4130b57cec5SDimitry Andrictypedef atomic<char16_t>           atomic_char16_t;
4140b57cec5SDimitry Andrictypedef atomic<char32_t>           atomic_char32_t;
4150b57cec5SDimitry Andrictypedef atomic<wchar_t>            atomic_wchar_t;
4160b57cec5SDimitry Andric
4170b57cec5SDimitry Andrictypedef atomic<int_least8_t>   atomic_int_least8_t;
4180b57cec5SDimitry Andrictypedef atomic<uint_least8_t>  atomic_uint_least8_t;
4190b57cec5SDimitry Andrictypedef atomic<int_least16_t>  atomic_int_least16_t;
4200b57cec5SDimitry Andrictypedef atomic<uint_least16_t> atomic_uint_least16_t;
4210b57cec5SDimitry Andrictypedef atomic<int_least32_t>  atomic_int_least32_t;
4220b57cec5SDimitry Andrictypedef atomic<uint_least32_t> atomic_uint_least32_t;
4230b57cec5SDimitry Andrictypedef atomic<int_least64_t>  atomic_int_least64_t;
4240b57cec5SDimitry Andrictypedef atomic<uint_least64_t> atomic_uint_least64_t;
4250b57cec5SDimitry Andric
4260b57cec5SDimitry Andrictypedef atomic<int_fast8_t>   atomic_int_fast8_t;
4270b57cec5SDimitry Andrictypedef atomic<uint_fast8_t>  atomic_uint_fast8_t;
4280b57cec5SDimitry Andrictypedef atomic<int_fast16_t>  atomic_int_fast16_t;
4290b57cec5SDimitry Andrictypedef atomic<uint_fast16_t> atomic_uint_fast16_t;
4300b57cec5SDimitry Andrictypedef atomic<int_fast32_t>  atomic_int_fast32_t;
4310b57cec5SDimitry Andrictypedef atomic<uint_fast32_t> atomic_uint_fast32_t;
4320b57cec5SDimitry Andrictypedef atomic<int_fast64_t>  atomic_int_fast64_t;
4330b57cec5SDimitry Andrictypedef atomic<uint_fast64_t> atomic_uint_fast64_t;
4340b57cec5SDimitry Andric
4350b57cec5SDimitry Andrictypedef atomic<int8_t>   atomic_int8_t;
4360b57cec5SDimitry Andrictypedef atomic<uint8_t>  atomic_uint8_t;
4370b57cec5SDimitry Andrictypedef atomic<int16_t>  atomic_int16_t;
4380b57cec5SDimitry Andrictypedef atomic<uint16_t> atomic_uint16_t;
4390b57cec5SDimitry Andrictypedef atomic<int32_t>  atomic_int32_t;
4400b57cec5SDimitry Andrictypedef atomic<uint32_t> atomic_uint32_t;
4410b57cec5SDimitry Andrictypedef atomic<int64_t>  atomic_int64_t;
4420b57cec5SDimitry Andrictypedef atomic<uint64_t> atomic_uint64_t;
4430b57cec5SDimitry Andric
4440b57cec5SDimitry Andrictypedef atomic<intptr_t>  atomic_intptr_t;
4450b57cec5SDimitry Andrictypedef atomic<uintptr_t> atomic_uintptr_t;
4460b57cec5SDimitry Andrictypedef atomic<size_t>    atomic_size_t;
4470b57cec5SDimitry Andrictypedef atomic<ptrdiff_t> atomic_ptrdiff_t;
4480b57cec5SDimitry Andrictypedef atomic<intmax_t>  atomic_intmax_t;
4490b57cec5SDimitry Andrictypedef atomic<uintmax_t> atomic_uintmax_t;
4500b57cec5SDimitry Andric
4515ffd83dbSDimitry Andric// flag type and operations
4525ffd83dbSDimitry Andric
4535ffd83dbSDimitry Andrictypedef struct atomic_flag
4545ffd83dbSDimitry Andric{
455fe6060f1SDimitry Andric    atomic_flag() noexcept = default; // until C++20
456fe6060f1SDimitry Andric    constexpr atomic_flag() noexcept; // since C++20
4575ffd83dbSDimitry Andric    atomic_flag(const atomic_flag&) = delete;
4585ffd83dbSDimitry Andric    atomic_flag& operator=(const atomic_flag&) = delete;
4595ffd83dbSDimitry Andric    atomic_flag& operator=(const atomic_flag&) volatile = delete;
4605ffd83dbSDimitry Andric
4615ffd83dbSDimitry Andric    bool test(memory_order m = memory_order_seq_cst) volatile noexcept;
4625ffd83dbSDimitry Andric    bool test(memory_order m = memory_order_seq_cst) noexcept;
4635ffd83dbSDimitry Andric    bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept;
4645ffd83dbSDimitry Andric    bool test_and_set(memory_order m = memory_order_seq_cst) noexcept;
4655ffd83dbSDimitry Andric    void clear(memory_order m = memory_order_seq_cst) volatile noexcept;
4665ffd83dbSDimitry Andric    void clear(memory_order m = memory_order_seq_cst) noexcept;
4675ffd83dbSDimitry Andric
4685ffd83dbSDimitry Andric    void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept;
4695ffd83dbSDimitry Andric    void wait(bool, memory_order = memory_order::seq_cst) const noexcept;
4705ffd83dbSDimitry Andric    void notify_one() volatile noexcept;
4715ffd83dbSDimitry Andric    void notify_one() noexcept;
4725ffd83dbSDimitry Andric    void notify_all() volatile noexcept;
4735ffd83dbSDimitry Andric    void notify_all() noexcept;
4745ffd83dbSDimitry Andric} atomic_flag;
4755ffd83dbSDimitry Andric
4765ffd83dbSDimitry Andricbool atomic_flag_test(volatile atomic_flag* obj) noexcept;
4775ffd83dbSDimitry Andricbool atomic_flag_test(atomic_flag* obj) noexcept;
4785ffd83dbSDimitry Andricbool atomic_flag_test_explicit(volatile atomic_flag* obj,
4795ffd83dbSDimitry Andric                               memory_order m) noexcept;
4805ffd83dbSDimitry Andricbool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept;
4815ffd83dbSDimitry Andricbool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept;
4825ffd83dbSDimitry Andricbool atomic_flag_test_and_set(atomic_flag* obj) noexcept;
4835ffd83dbSDimitry Andricbool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj,
4845ffd83dbSDimitry Andric                                       memory_order m) noexcept;
4855ffd83dbSDimitry Andricbool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept;
4865ffd83dbSDimitry Andricvoid atomic_flag_clear(volatile atomic_flag* obj) noexcept;
4875ffd83dbSDimitry Andricvoid atomic_flag_clear(atomic_flag* obj) noexcept;
4885ffd83dbSDimitry Andricvoid atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept;
4895ffd83dbSDimitry Andricvoid atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept;
4905ffd83dbSDimitry Andric
4915ffd83dbSDimitry Andricvoid atomic_wait(const volatile atomic_flag* obj, T old) noexcept;
4925ffd83dbSDimitry Andricvoid atomic_wait(const atomic_flag* obj, T old) noexcept;
4935ffd83dbSDimitry Andricvoid atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept;
4945ffd83dbSDimitry Andricvoid atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept;
4955ffd83dbSDimitry Andricvoid atomic_one(volatile atomic_flag* obj) noexcept;
4965ffd83dbSDimitry Andricvoid atomic_one(atomic_flag* obj) noexcept;
4975ffd83dbSDimitry Andricvoid atomic_all(volatile atomic_flag* obj) noexcept;
4985ffd83dbSDimitry Andricvoid atomic_all(atomic_flag* obj) noexcept;
4995ffd83dbSDimitry Andric
5000b57cec5SDimitry Andric// fences
5010b57cec5SDimitry Andric
5020b57cec5SDimitry Andricvoid atomic_thread_fence(memory_order m) noexcept;
5030b57cec5SDimitry Andricvoid atomic_signal_fence(memory_order m) noexcept;
5040b57cec5SDimitry Andric
5055ffd83dbSDimitry Andric// deprecated
5065ffd83dbSDimitry Andric
5075ffd83dbSDimitry Andrictemplate <class T>
508*349cc55cSDimitry Andric  void atomic_init(volatile atomic<T>* obj, atomic<T>::value_type desr) noexcept;
5095ffd83dbSDimitry Andric
5105ffd83dbSDimitry Andrictemplate <class T>
511*349cc55cSDimitry Andric  void atomic_init(atomic<T>* obj, atomic<T>::value_type desr) noexcept;
5125ffd83dbSDimitry Andric
5135ffd83dbSDimitry Andric#define ATOMIC_VAR_INIT(value) see below
5145ffd83dbSDimitry Andric
5155ffd83dbSDimitry Andric#define ATOMIC_FLAG_INIT see below
5165ffd83dbSDimitry Andric
5170b57cec5SDimitry Andric}  // std
5180b57cec5SDimitry Andric
5190b57cec5SDimitry Andric*/
5200b57cec5SDimitry Andric
521e8d8bef9SDimitry Andric#include <__availability>
522fe6060f1SDimitry Andric#include <__config>
523*349cc55cSDimitry Andric#include <__thread/poll_with_backoff.h>
5240b57cec5SDimitry Andric#include <cstddef>
5250b57cec5SDimitry Andric#include <cstdint>
5265ffd83dbSDimitry Andric#include <cstring>
5270b57cec5SDimitry Andric#include <type_traits>
5280b57cec5SDimitry Andric#include <version>
5290b57cec5SDimitry Andric
530*349cc55cSDimitry Andric#ifndef _LIBCPP_HAS_NO_THREADS
531*349cc55cSDimitry Andric# include <__threading_support>
532*349cc55cSDimitry Andric#endif
533*349cc55cSDimitry Andric
5340b57cec5SDimitry Andric#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
5350b57cec5SDimitry Andric#pragma GCC system_header
5360b57cec5SDimitry Andric#endif
5370b57cec5SDimitry Andric
5380b57cec5SDimitry Andric#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER
5390b57cec5SDimitry Andric# error <atomic> is not implemented
5400b57cec5SDimitry Andric#endif
5410b57cec5SDimitry Andric#ifdef kill_dependency
5420b57cec5SDimitry Andric# error C++ standard library is incompatible with <stdatomic.h>
5430b57cec5SDimitry Andric#endif
5440b57cec5SDimitry Andric
5450b57cec5SDimitry Andric#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \
5460b57cec5SDimitry Andric  _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \
5470b57cec5SDimitry Andric                           __m == memory_order_acquire || \
5480b57cec5SDimitry Andric                           __m == memory_order_acq_rel,   \
5490b57cec5SDimitry Andric                        "memory order argument to atomic operation is invalid")
5500b57cec5SDimitry Andric
5510b57cec5SDimitry Andric#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \
5520b57cec5SDimitry Andric  _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \
5530b57cec5SDimitry Andric                           __m == memory_order_acq_rel,   \
5540b57cec5SDimitry Andric                        "memory order argument to atomic operation is invalid")
5550b57cec5SDimitry Andric
5560b57cec5SDimitry Andric#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \
5570b57cec5SDimitry Andric  _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \
5580b57cec5SDimitry Andric                           __f == memory_order_acq_rel,   \
5590b57cec5SDimitry Andric                        "memory order argument to atomic operation is invalid")
5600b57cec5SDimitry Andric
5610b57cec5SDimitry Andric_LIBCPP_BEGIN_NAMESPACE_STD
5620b57cec5SDimitry Andric
5630b57cec5SDimitry Andric// Figure out what the underlying type for `memory_order` would be if it were
5640b57cec5SDimitry Andric// declared as an unscoped enum (accounting for -fshort-enums). Use this result
5650b57cec5SDimitry Andric// to pin the underlying type in C++20.
5660b57cec5SDimitry Andricenum __legacy_memory_order {
5670b57cec5SDimitry Andric    __mo_relaxed,
5680b57cec5SDimitry Andric    __mo_consume,
5690b57cec5SDimitry Andric    __mo_acquire,
5700b57cec5SDimitry Andric    __mo_release,
5710b57cec5SDimitry Andric    __mo_acq_rel,
5720b57cec5SDimitry Andric    __mo_seq_cst
5730b57cec5SDimitry Andric};
5740b57cec5SDimitry Andric
5750b57cec5SDimitry Andrictypedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t;
5760b57cec5SDimitry Andric
5770b57cec5SDimitry Andric#if _LIBCPP_STD_VER > 17
5780b57cec5SDimitry Andric
5790b57cec5SDimitry Andricenum class memory_order : __memory_order_underlying_t {
5800b57cec5SDimitry Andric  relaxed = __mo_relaxed,
5810b57cec5SDimitry Andric  consume = __mo_consume,
5820b57cec5SDimitry Andric  acquire = __mo_acquire,
5830b57cec5SDimitry Andric  release = __mo_release,
5840b57cec5SDimitry Andric  acq_rel = __mo_acq_rel,
5850b57cec5SDimitry Andric  seq_cst = __mo_seq_cst
5860b57cec5SDimitry Andric};
5870b57cec5SDimitry Andric
5880b57cec5SDimitry Andricinline constexpr auto memory_order_relaxed = memory_order::relaxed;
5890b57cec5SDimitry Andricinline constexpr auto memory_order_consume = memory_order::consume;
5900b57cec5SDimitry Andricinline constexpr auto memory_order_acquire = memory_order::acquire;
5910b57cec5SDimitry Andricinline constexpr auto memory_order_release = memory_order::release;
5920b57cec5SDimitry Andricinline constexpr auto memory_order_acq_rel = memory_order::acq_rel;
5930b57cec5SDimitry Andricinline constexpr auto memory_order_seq_cst = memory_order::seq_cst;
5940b57cec5SDimitry Andric
5950b57cec5SDimitry Andric#else
5960b57cec5SDimitry Andric
5970b57cec5SDimitry Andrictypedef enum memory_order {
5980b57cec5SDimitry Andric  memory_order_relaxed = __mo_relaxed,
5990b57cec5SDimitry Andric  memory_order_consume = __mo_consume,
6000b57cec5SDimitry Andric  memory_order_acquire = __mo_acquire,
6010b57cec5SDimitry Andric  memory_order_release = __mo_release,
6020b57cec5SDimitry Andric  memory_order_acq_rel = __mo_acq_rel,
6030b57cec5SDimitry Andric  memory_order_seq_cst = __mo_seq_cst,
6040b57cec5SDimitry Andric} memory_order;
6050b57cec5SDimitry Andric
6060b57cec5SDimitry Andric#endif // _LIBCPP_STD_VER > 17
6070b57cec5SDimitry Andric
6085ffd83dbSDimitry Andrictemplate <typename _Tp> _LIBCPP_INLINE_VISIBILITY
6095ffd83dbSDimitry Andricbool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) {
610e8d8bef9SDimitry Andric    return _VSTD::memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0;
6115ffd83dbSDimitry Andric}
6125ffd83dbSDimitry Andric
6130b57cec5SDimitry Andricstatic_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value),
6140b57cec5SDimitry Andric  "unexpected underlying type for std::memory_order");
6150b57cec5SDimitry Andric
6160b57cec5SDimitry Andric#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \
6170b57cec5SDimitry Andric    defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS)
6180b57cec5SDimitry Andric
6190b57cec5SDimitry Andric// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because
6200b57cec5SDimitry Andric// the default operator= in an object is not volatile, a byte-by-byte copy
6210b57cec5SDimitry Andric// is required.
6220b57cec5SDimitry Andrictemplate <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY
6230b57cec5SDimitry Andrictypename enable_if<is_assignable<_Tp&, _Tv>::value>::type
6240b57cec5SDimitry Andric__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) {
6250b57cec5SDimitry Andric  __a_value = __val;
6260b57cec5SDimitry Andric}
6270b57cec5SDimitry Andrictemplate <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY
6280b57cec5SDimitry Andrictypename enable_if<is_assignable<_Tp&, _Tv>::value>::type
6290b57cec5SDimitry Andric__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) {
6300b57cec5SDimitry Andric  volatile char* __to = reinterpret_cast<volatile char*>(&__a_value);
6310b57cec5SDimitry Andric  volatile char* __end = __to + sizeof(_Tp);
6320b57cec5SDimitry Andric  volatile const char* __from = reinterpret_cast<volatile const char*>(&__val);
6330b57cec5SDimitry Andric  while (__to != __end)
6340b57cec5SDimitry Andric    *__to++ = *__from++;
6350b57cec5SDimitry Andric}
6360b57cec5SDimitry Andric
6370b57cec5SDimitry Andric#endif
6380b57cec5SDimitry Andric
6390b57cec5SDimitry Andric#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP)
6400b57cec5SDimitry Andric
6410b57cec5SDimitry Andrictemplate <typename _Tp>
6420b57cec5SDimitry Andricstruct __cxx_atomic_base_impl {
6430b57cec5SDimitry Andric
6440b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY
6450b57cec5SDimitry Andric#ifndef _LIBCPP_CXX03_LANG
6460b57cec5SDimitry Andric    __cxx_atomic_base_impl() _NOEXCEPT = default;
6470b57cec5SDimitry Andric#else
6480b57cec5SDimitry Andric    __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {}
6490b57cec5SDimitry Andric#endif // _LIBCPP_CXX03_LANG
6500b57cec5SDimitry Andric  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT
6510b57cec5SDimitry Andric    : __a_value(value) {}
6520b57cec5SDimitry Andric  _Tp __a_value;
6530b57cec5SDimitry Andric};
6540b57cec5SDimitry Andric
6550b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) {
6560b57cec5SDimitry Andric  // Avoid switch statement to make this a constexpr.
6570b57cec5SDimitry Andric  return __order == memory_order_relaxed ? __ATOMIC_RELAXED:
6580b57cec5SDimitry Andric         (__order == memory_order_acquire ? __ATOMIC_ACQUIRE:
6590b57cec5SDimitry Andric          (__order == memory_order_release ? __ATOMIC_RELEASE:
6600b57cec5SDimitry Andric           (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST:
6610b57cec5SDimitry Andric            (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL:
6620b57cec5SDimitry Andric              __ATOMIC_CONSUME))));
6630b57cec5SDimitry Andric}
6640b57cec5SDimitry Andric
6650b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) {
6660b57cec5SDimitry Andric  // Avoid switch statement to make this a constexpr.
6670b57cec5SDimitry Andric  return __order == memory_order_relaxed ? __ATOMIC_RELAXED:
6680b57cec5SDimitry Andric         (__order == memory_order_acquire ? __ATOMIC_ACQUIRE:
6690b57cec5SDimitry Andric          (__order == memory_order_release ? __ATOMIC_RELAXED:
6700b57cec5SDimitry Andric           (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST:
6710b57cec5SDimitry Andric            (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE:
6720b57cec5SDimitry Andric              __ATOMIC_CONSUME))));
6730b57cec5SDimitry Andric}
6740b57cec5SDimitry Andric
6750b57cec5SDimitry Andrictemplate <typename _Tp>
6760b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
6770b57cec5SDimitry Andricvoid __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a,  _Tp __val) {
6780b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, __val);
6790b57cec5SDimitry Andric}
6800b57cec5SDimitry Andric
6810b57cec5SDimitry Andrictemplate <typename _Tp>
6820b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
6830b57cec5SDimitry Andricvoid __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a,  _Tp __val) {
6840b57cec5SDimitry Andric  __a->__a_value = __val;
6850b57cec5SDimitry Andric}
6860b57cec5SDimitry Andric
6870b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline
6880b57cec5SDimitry Andricvoid __cxx_atomic_thread_fence(memory_order __order) {
6890b57cec5SDimitry Andric  __atomic_thread_fence(__to_gcc_order(__order));
6900b57cec5SDimitry Andric}
6910b57cec5SDimitry Andric
6920b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline
6930b57cec5SDimitry Andricvoid __cxx_atomic_signal_fence(memory_order __order) {
6940b57cec5SDimitry Andric  __atomic_signal_fence(__to_gcc_order(__order));
6950b57cec5SDimitry Andric}
6960b57cec5SDimitry Andric
6970b57cec5SDimitry Andrictemplate <typename _Tp>
6980b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
6990b57cec5SDimitry Andricvoid __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a,  _Tp __val,
7000b57cec5SDimitry Andric                        memory_order __order) {
7010b57cec5SDimitry Andric  __atomic_store(&__a->__a_value, &__val,
7020b57cec5SDimitry Andric                 __to_gcc_order(__order));
7030b57cec5SDimitry Andric}
7040b57cec5SDimitry Andric
7050b57cec5SDimitry Andrictemplate <typename _Tp>
7060b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7070b57cec5SDimitry Andricvoid __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a,  _Tp __val,
7080b57cec5SDimitry Andric                        memory_order __order) {
7090b57cec5SDimitry Andric  __atomic_store(&__a->__a_value, &__val,
7100b57cec5SDimitry Andric                 __to_gcc_order(__order));
7110b57cec5SDimitry Andric}
7120b57cec5SDimitry Andric
7130b57cec5SDimitry Andrictemplate <typename _Tp>
7140b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7150b57cec5SDimitry Andric_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a,
7160b57cec5SDimitry Andric                      memory_order __order) {
7170b57cec5SDimitry Andric  _Tp __ret;
7180b57cec5SDimitry Andric  __atomic_load(&__a->__a_value, &__ret,
7190b57cec5SDimitry Andric                __to_gcc_order(__order));
7200b57cec5SDimitry Andric  return __ret;
7210b57cec5SDimitry Andric}
7220b57cec5SDimitry Andric
7230b57cec5SDimitry Andrictemplate <typename _Tp>
7240b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7250b57cec5SDimitry Andric_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
7260b57cec5SDimitry Andric  _Tp __ret;
7270b57cec5SDimitry Andric  __atomic_load(&__a->__a_value, &__ret,
7280b57cec5SDimitry Andric                __to_gcc_order(__order));
7290b57cec5SDimitry Andric  return __ret;
7300b57cec5SDimitry Andric}
7310b57cec5SDimitry Andric
7320b57cec5SDimitry Andrictemplate <typename _Tp>
7330b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7340b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a,
7350b57cec5SDimitry Andric                          _Tp __value, memory_order __order) {
7360b57cec5SDimitry Andric  _Tp __ret;
7370b57cec5SDimitry Andric  __atomic_exchange(&__a->__a_value, &__value, &__ret,
7380b57cec5SDimitry Andric                    __to_gcc_order(__order));
7390b57cec5SDimitry Andric  return __ret;
7400b57cec5SDimitry Andric}
7410b57cec5SDimitry Andric
7420b57cec5SDimitry Andrictemplate <typename _Tp>
7430b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7440b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value,
7450b57cec5SDimitry Andric                          memory_order __order) {
7460b57cec5SDimitry Andric  _Tp __ret;
7470b57cec5SDimitry Andric  __atomic_exchange(&__a->__a_value, &__value, &__ret,
7480b57cec5SDimitry Andric                    __to_gcc_order(__order));
7490b57cec5SDimitry Andric  return __ret;
7500b57cec5SDimitry Andric}
7510b57cec5SDimitry Andric
7520b57cec5SDimitry Andrictemplate <typename _Tp>
7530b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7540b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(
7550b57cec5SDimitry Andric    volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value,
7560b57cec5SDimitry Andric    memory_order __success, memory_order __failure) {
7570b57cec5SDimitry Andric  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
7580b57cec5SDimitry Andric                                   false,
7590b57cec5SDimitry Andric                                   __to_gcc_order(__success),
7600b57cec5SDimitry Andric                                   __to_gcc_failure_order(__failure));
7610b57cec5SDimitry Andric}
7620b57cec5SDimitry Andric
7630b57cec5SDimitry Andrictemplate <typename _Tp>
7640b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7650b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(
7660b57cec5SDimitry Andric    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success,
7670b57cec5SDimitry Andric    memory_order __failure) {
7680b57cec5SDimitry Andric  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
7690b57cec5SDimitry Andric                                   false,
7700b57cec5SDimitry Andric                                   __to_gcc_order(__success),
7710b57cec5SDimitry Andric                                   __to_gcc_failure_order(__failure));
7720b57cec5SDimitry Andric}
7730b57cec5SDimitry Andric
7740b57cec5SDimitry Andrictemplate <typename _Tp>
7750b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7760b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(
7770b57cec5SDimitry Andric    volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value,
7780b57cec5SDimitry Andric    memory_order __success, memory_order __failure) {
7790b57cec5SDimitry Andric  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
7800b57cec5SDimitry Andric                                   true,
7810b57cec5SDimitry Andric                                   __to_gcc_order(__success),
7820b57cec5SDimitry Andric                                   __to_gcc_failure_order(__failure));
7830b57cec5SDimitry Andric}
7840b57cec5SDimitry Andric
7850b57cec5SDimitry Andrictemplate <typename _Tp>
7860b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
7870b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(
7880b57cec5SDimitry Andric    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success,
7890b57cec5SDimitry Andric    memory_order __failure) {
7900b57cec5SDimitry Andric  return __atomic_compare_exchange(&__a->__a_value, __expected, &__value,
7910b57cec5SDimitry Andric                                   true,
7920b57cec5SDimitry Andric                                   __to_gcc_order(__success),
7930b57cec5SDimitry Andric                                   __to_gcc_failure_order(__failure));
7940b57cec5SDimitry Andric}
7950b57cec5SDimitry Andric
7960b57cec5SDimitry Andrictemplate <typename _Tp>
7970b57cec5SDimitry Andricstruct __skip_amt { enum {value = 1}; };
7980b57cec5SDimitry Andric
7990b57cec5SDimitry Andrictemplate <typename _Tp>
8000b57cec5SDimitry Andricstruct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; };
8010b57cec5SDimitry Andric
8020b57cec5SDimitry Andric// FIXME: Haven't figured out what the spec says about using arrays with
8030b57cec5SDimitry Andric// atomic_fetch_add. Force a failure rather than creating bad behavior.
8040b57cec5SDimitry Andrictemplate <typename _Tp>
8050b57cec5SDimitry Andricstruct __skip_amt<_Tp[]> { };
8060b57cec5SDimitry Andrictemplate <typename _Tp, int n>
8070b57cec5SDimitry Andricstruct __skip_amt<_Tp[n]> { };
8080b57cec5SDimitry Andric
8090b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
8100b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8110b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a,
8120b57cec5SDimitry Andric                           _Td __delta, memory_order __order) {
8130b57cec5SDimitry Andric  return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
8140b57cec5SDimitry Andric                            __to_gcc_order(__order));
8150b57cec5SDimitry Andric}
8160b57cec5SDimitry Andric
8170b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
8180b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8190b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta,
8200b57cec5SDimitry Andric                           memory_order __order) {
8210b57cec5SDimitry Andric  return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
8220b57cec5SDimitry Andric                            __to_gcc_order(__order));
8230b57cec5SDimitry Andric}
8240b57cec5SDimitry Andric
8250b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
8260b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8270b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a,
8280b57cec5SDimitry Andric                           _Td __delta, memory_order __order) {
8290b57cec5SDimitry Andric  return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
8300b57cec5SDimitry Andric                            __to_gcc_order(__order));
8310b57cec5SDimitry Andric}
8320b57cec5SDimitry Andric
8330b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
8340b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8350b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta,
8360b57cec5SDimitry Andric                           memory_order __order) {
8370b57cec5SDimitry Andric  return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value,
8380b57cec5SDimitry Andric                            __to_gcc_order(__order));
8390b57cec5SDimitry Andric}
8400b57cec5SDimitry Andric
8410b57cec5SDimitry Andrictemplate <typename _Tp>
8420b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8430b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a,
8440b57cec5SDimitry Andric                           _Tp __pattern, memory_order __order) {
8450b57cec5SDimitry Andric  return __atomic_fetch_and(&__a->__a_value, __pattern,
8460b57cec5SDimitry Andric                            __to_gcc_order(__order));
8470b57cec5SDimitry Andric}
8480b57cec5SDimitry Andric
8490b57cec5SDimitry Andrictemplate <typename _Tp>
8500b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8510b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a,
8520b57cec5SDimitry Andric                           _Tp __pattern, memory_order __order) {
8530b57cec5SDimitry Andric  return __atomic_fetch_and(&__a->__a_value, __pattern,
8540b57cec5SDimitry Andric                            __to_gcc_order(__order));
8550b57cec5SDimitry Andric}
8560b57cec5SDimitry Andric
8570b57cec5SDimitry Andrictemplate <typename _Tp>
8580b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8590b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a,
8600b57cec5SDimitry Andric                          _Tp __pattern, memory_order __order) {
8610b57cec5SDimitry Andric  return __atomic_fetch_or(&__a->__a_value, __pattern,
8620b57cec5SDimitry Andric                           __to_gcc_order(__order));
8630b57cec5SDimitry Andric}
8640b57cec5SDimitry Andric
8650b57cec5SDimitry Andrictemplate <typename _Tp>
8660b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8670b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern,
8680b57cec5SDimitry Andric                          memory_order __order) {
8690b57cec5SDimitry Andric  return __atomic_fetch_or(&__a->__a_value, __pattern,
8700b57cec5SDimitry Andric                           __to_gcc_order(__order));
8710b57cec5SDimitry Andric}
8720b57cec5SDimitry Andric
8730b57cec5SDimitry Andrictemplate <typename _Tp>
8740b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8750b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a,
8760b57cec5SDimitry Andric                           _Tp __pattern, memory_order __order) {
8770b57cec5SDimitry Andric  return __atomic_fetch_xor(&__a->__a_value, __pattern,
8780b57cec5SDimitry Andric                            __to_gcc_order(__order));
8790b57cec5SDimitry Andric}
8800b57cec5SDimitry Andric
8810b57cec5SDimitry Andrictemplate <typename _Tp>
8820b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
8830b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern,
8840b57cec5SDimitry Andric                           memory_order __order) {
8850b57cec5SDimitry Andric  return __atomic_fetch_xor(&__a->__a_value, __pattern,
8860b57cec5SDimitry Andric                            __to_gcc_order(__order));
8870b57cec5SDimitry Andric}
8880b57cec5SDimitry Andric
8890b57cec5SDimitry Andric#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0)
8900b57cec5SDimitry Andric
8910b57cec5SDimitry Andric#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP)
8920b57cec5SDimitry Andric
8930b57cec5SDimitry Andrictemplate <typename _Tp>
8940b57cec5SDimitry Andricstruct __cxx_atomic_base_impl {
8950b57cec5SDimitry Andric
8960b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY
8970b57cec5SDimitry Andric#ifndef _LIBCPP_CXX03_LANG
8980b57cec5SDimitry Andric    __cxx_atomic_base_impl() _NOEXCEPT = default;
8990b57cec5SDimitry Andric#else
9000b57cec5SDimitry Andric    __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {}
9010b57cec5SDimitry Andric#endif // _LIBCPP_CXX03_LANG
9020b57cec5SDimitry Andric  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT
9030b57cec5SDimitry Andric    : __a_value(value) {}
904e40139ffSDimitry Andric  _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value;
9050b57cec5SDimitry Andric};
9060b57cec5SDimitry Andric
9070b57cec5SDimitry Andric#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
9080b57cec5SDimitry Andric
9090b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline
9100b57cec5SDimitry Andricvoid __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
9110b57cec5SDimitry Andric    __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
9120b57cec5SDimitry Andric}
9130b57cec5SDimitry Andric
9140b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline
9150b57cec5SDimitry Andricvoid __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
9160b57cec5SDimitry Andric    __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
9170b57cec5SDimitry Andric}
9180b57cec5SDimitry Andric
9190b57cec5SDimitry Andrictemplate<class _Tp>
9200b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9210b57cec5SDimitry Andricvoid __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
9220b57cec5SDimitry Andric    __c11_atomic_init(&__a->__a_value, __val);
9230b57cec5SDimitry Andric}
9240b57cec5SDimitry Andrictemplate<class _Tp>
9250b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9260b57cec5SDimitry Andricvoid __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT {
9270b57cec5SDimitry Andric    __c11_atomic_init(&__a->__a_value, __val);
9280b57cec5SDimitry Andric}
9290b57cec5SDimitry Andric
9300b57cec5SDimitry Andrictemplate<class _Tp>
9310b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9320b57cec5SDimitry Andricvoid __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
9330b57cec5SDimitry Andric    __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order));
9340b57cec5SDimitry Andric}
9350b57cec5SDimitry Andrictemplate<class _Tp>
9360b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9370b57cec5SDimitry Andricvoid __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT {
9380b57cec5SDimitry Andric    __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order));
9390b57cec5SDimitry Andric}
9400b57cec5SDimitry Andric
9410b57cec5SDimitry Andrictemplate<class _Tp>
9420b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9430b57cec5SDimitry Andric_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
9440b57cec5SDimitry Andric    using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*;
9450b57cec5SDimitry Andric    return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order));
9460b57cec5SDimitry Andric}
9470b57cec5SDimitry Andrictemplate<class _Tp>
9480b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9490b57cec5SDimitry Andric_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
9500b57cec5SDimitry Andric    using __ptr_type = typename remove_const<decltype(__a->__a_value)>::type*;
9510b57cec5SDimitry Andric    return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order));
9520b57cec5SDimitry Andric}
9530b57cec5SDimitry Andric
9540b57cec5SDimitry Andrictemplate<class _Tp>
9550b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9560b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
9570b57cec5SDimitry Andric    return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order));
9580b57cec5SDimitry Andric}
9590b57cec5SDimitry Andrictemplate<class _Tp>
9600b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9610b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT {
9620b57cec5SDimitry Andric    return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order));
9630b57cec5SDimitry Andric}
9640b57cec5SDimitry Andric
965fe6060f1SDimitry Andric_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
966fe6060f1SDimitry Andric  // Avoid switch statement to make this a constexpr.
967fe6060f1SDimitry Andric  return __order == memory_order_release ? memory_order_relaxed:
968fe6060f1SDimitry Andric         (__order == memory_order_acq_rel ? memory_order_acquire:
969fe6060f1SDimitry Andric             __order);
970fe6060f1SDimitry Andric}
971fe6060f1SDimitry Andric
9720b57cec5SDimitry Andrictemplate<class _Tp>
9730b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9740b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
975fe6060f1SDimitry Andric    return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
9760b57cec5SDimitry Andric}
9770b57cec5SDimitry Andrictemplate<class _Tp>
9780b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9790b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
980fe6060f1SDimitry Andric    return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
9810b57cec5SDimitry Andric}
9820b57cec5SDimitry Andric
9830b57cec5SDimitry Andrictemplate<class _Tp>
9840b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9850b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
986fe6060f1SDimitry Andric    return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
9870b57cec5SDimitry Andric}
9880b57cec5SDimitry Andrictemplate<class _Tp>
9890b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9900b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT {
991fe6060f1SDimitry Andric    return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value,  static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
9920b57cec5SDimitry Andric}
9930b57cec5SDimitry Andric
9940b57cec5SDimitry Andrictemplate<class _Tp>
9950b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
9960b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
9970b57cec5SDimitry Andric    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
9980b57cec5SDimitry Andric}
9990b57cec5SDimitry Andrictemplate<class _Tp>
10000b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10010b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT {
10020b57cec5SDimitry Andric    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10030b57cec5SDimitry Andric}
10040b57cec5SDimitry Andric
10050b57cec5SDimitry Andrictemplate<class _Tp>
10060b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10070b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
10080b57cec5SDimitry Andric    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10090b57cec5SDimitry Andric}
10100b57cec5SDimitry Andrictemplate<class _Tp>
10110b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10120b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
10130b57cec5SDimitry Andric    return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10140b57cec5SDimitry Andric}
10150b57cec5SDimitry Andric
10160b57cec5SDimitry Andrictemplate<class _Tp>
10170b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10180b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
10190b57cec5SDimitry Andric    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10200b57cec5SDimitry Andric}
10210b57cec5SDimitry Andrictemplate<class _Tp>
10220b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10230b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT {
10240b57cec5SDimitry Andric    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10250b57cec5SDimitry Andric}
10260b57cec5SDimitry Andrictemplate<class _Tp>
10270b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10280b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
10290b57cec5SDimitry Andric    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10300b57cec5SDimitry Andric}
10310b57cec5SDimitry Andrictemplate<class _Tp>
10320b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10330b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
10340b57cec5SDimitry Andric    return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order));
10350b57cec5SDimitry Andric}
10360b57cec5SDimitry Andric
10370b57cec5SDimitry Andrictemplate<class _Tp>
10380b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10390b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
10400b57cec5SDimitry Andric    return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
10410b57cec5SDimitry Andric}
10420b57cec5SDimitry Andrictemplate<class _Tp>
10430b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10440b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
10450b57cec5SDimitry Andric    return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
10460b57cec5SDimitry Andric}
10470b57cec5SDimitry Andric
10480b57cec5SDimitry Andrictemplate<class _Tp>
10490b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10500b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
10510b57cec5SDimitry Andric    return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
10520b57cec5SDimitry Andric}
10530b57cec5SDimitry Andrictemplate<class _Tp>
10540b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10550b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
10560b57cec5SDimitry Andric    return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
10570b57cec5SDimitry Andric}
10580b57cec5SDimitry Andric
10590b57cec5SDimitry Andrictemplate<class _Tp>
10600b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10610b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
10620b57cec5SDimitry Andric    return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
10630b57cec5SDimitry Andric}
10640b57cec5SDimitry Andrictemplate<class _Tp>
10650b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10660b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
10670b57cec5SDimitry Andric    return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order));
10680b57cec5SDimitry Andric}
10690b57cec5SDimitry Andric
10700b57cec5SDimitry Andric#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP
10710b57cec5SDimitry Andric
10720b57cec5SDimitry Andrictemplate <class _Tp>
10730b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
10740b57cec5SDimitry Andric_Tp kill_dependency(_Tp __y) _NOEXCEPT
10750b57cec5SDimitry Andric{
10760b57cec5SDimitry Andric    return __y;
10770b57cec5SDimitry Andric}
10780b57cec5SDimitry Andric
10790b57cec5SDimitry Andric#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE)
10800b57cec5SDimitry Andric# define ATOMIC_BOOL_LOCK_FREE      __CLANG_ATOMIC_BOOL_LOCK_FREE
10810b57cec5SDimitry Andric# define ATOMIC_CHAR_LOCK_FREE      __CLANG_ATOMIC_CHAR_LOCK_FREE
1082fe6060f1SDimitry Andric#ifndef _LIBCPP_HAS_NO_CHAR8_T
1083e8d8bef9SDimitry Andric# define ATOMIC_CHAR8_T_LOCK_FREE   __CLANG_ATOMIC_CHAR8_T_LOCK_FREE
1084e8d8bef9SDimitry Andric#endif
10850b57cec5SDimitry Andric# define ATOMIC_CHAR16_T_LOCK_FREE  __CLANG_ATOMIC_CHAR16_T_LOCK_FREE
10860b57cec5SDimitry Andric# define ATOMIC_CHAR32_T_LOCK_FREE  __CLANG_ATOMIC_CHAR32_T_LOCK_FREE
10870b57cec5SDimitry Andric# define ATOMIC_WCHAR_T_LOCK_FREE   __CLANG_ATOMIC_WCHAR_T_LOCK_FREE
10880b57cec5SDimitry Andric# define ATOMIC_SHORT_LOCK_FREE     __CLANG_ATOMIC_SHORT_LOCK_FREE
10890b57cec5SDimitry Andric# define ATOMIC_INT_LOCK_FREE       __CLANG_ATOMIC_INT_LOCK_FREE
10900b57cec5SDimitry Andric# define ATOMIC_LONG_LOCK_FREE      __CLANG_ATOMIC_LONG_LOCK_FREE
10910b57cec5SDimitry Andric# define ATOMIC_LLONG_LOCK_FREE     __CLANG_ATOMIC_LLONG_LOCK_FREE
10920b57cec5SDimitry Andric# define ATOMIC_POINTER_LOCK_FREE   __CLANG_ATOMIC_POINTER_LOCK_FREE
10930b57cec5SDimitry Andric#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE)
10940b57cec5SDimitry Andric# define ATOMIC_BOOL_LOCK_FREE      __GCC_ATOMIC_BOOL_LOCK_FREE
10950b57cec5SDimitry Andric# define ATOMIC_CHAR_LOCK_FREE      __GCC_ATOMIC_CHAR_LOCK_FREE
1096fe6060f1SDimitry Andric#ifndef _LIBCPP_HAS_NO_CHAR8_T
1097e8d8bef9SDimitry Andric# define ATOMIC_CHAR8_T_LOCK_FREE   __GCC_ATOMIC_CHAR8_T_LOCK_FREE
1098e8d8bef9SDimitry Andric#endif
10990b57cec5SDimitry Andric# define ATOMIC_CHAR16_T_LOCK_FREE  __GCC_ATOMIC_CHAR16_T_LOCK_FREE
11000b57cec5SDimitry Andric# define ATOMIC_CHAR32_T_LOCK_FREE  __GCC_ATOMIC_CHAR32_T_LOCK_FREE
11010b57cec5SDimitry Andric# define ATOMIC_WCHAR_T_LOCK_FREE   __GCC_ATOMIC_WCHAR_T_LOCK_FREE
11020b57cec5SDimitry Andric# define ATOMIC_SHORT_LOCK_FREE     __GCC_ATOMIC_SHORT_LOCK_FREE
11030b57cec5SDimitry Andric# define ATOMIC_INT_LOCK_FREE       __GCC_ATOMIC_INT_LOCK_FREE
11040b57cec5SDimitry Andric# define ATOMIC_LONG_LOCK_FREE      __GCC_ATOMIC_LONG_LOCK_FREE
11050b57cec5SDimitry Andric# define ATOMIC_LLONG_LOCK_FREE     __GCC_ATOMIC_LLONG_LOCK_FREE
11060b57cec5SDimitry Andric# define ATOMIC_POINTER_LOCK_FREE   __GCC_ATOMIC_POINTER_LOCK_FREE
11070b57cec5SDimitry Andric#endif
11080b57cec5SDimitry Andric
11090b57cec5SDimitry Andric#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS
11100b57cec5SDimitry Andric
11110b57cec5SDimitry Andrictemplate<typename _Tp>
11120b57cec5SDimitry Andricstruct __cxx_atomic_lock_impl {
11130b57cec5SDimitry Andric
11140b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY
11150b57cec5SDimitry Andric  __cxx_atomic_lock_impl() _NOEXCEPT
11160b57cec5SDimitry Andric    : __a_value(), __a_lock(0) {}
11170b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit
11180b57cec5SDimitry Andric  __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT
11190b57cec5SDimitry Andric    : __a_value(value), __a_lock(0) {}
11200b57cec5SDimitry Andric
11210b57cec5SDimitry Andric  _Tp __a_value;
11220b57cec5SDimitry Andric  mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock;
11230b57cec5SDimitry Andric
11240b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY void __lock() const volatile {
11250b57cec5SDimitry Andric    while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire))
11260b57cec5SDimitry Andric        /*spin*/;
11270b57cec5SDimitry Andric  }
11280b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY void __lock() const {
11290b57cec5SDimitry Andric    while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire))
11300b57cec5SDimitry Andric        /*spin*/;
11310b57cec5SDimitry Andric  }
11320b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile {
11330b57cec5SDimitry Andric    __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release);
11340b57cec5SDimitry Andric  }
11350b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY void __unlock() const {
11360b57cec5SDimitry Andric    __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release);
11370b57cec5SDimitry Andric  }
11380b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile {
11390b57cec5SDimitry Andric    __lock();
11400b57cec5SDimitry Andric    _Tp __old;
11410b57cec5SDimitry Andric    __cxx_atomic_assign_volatile(__old, __a_value);
11420b57cec5SDimitry Andric    __unlock();
11430b57cec5SDimitry Andric    return __old;
11440b57cec5SDimitry Andric  }
11450b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY _Tp __read() const {
11460b57cec5SDimitry Andric    __lock();
11470b57cec5SDimitry Andric    _Tp __old = __a_value;
11480b57cec5SDimitry Andric    __unlock();
11490b57cec5SDimitry Andric    return __old;
11500b57cec5SDimitry Andric  }
11510b57cec5SDimitry Andric};
11520b57cec5SDimitry Andric
11530b57cec5SDimitry Andrictemplate <typename _Tp>
11540b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11550b57cec5SDimitry Andricvoid __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val) {
11560b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, __val);
11570b57cec5SDimitry Andric}
11580b57cec5SDimitry Andrictemplate <typename _Tp>
11590b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11600b57cec5SDimitry Andricvoid __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val) {
11610b57cec5SDimitry Andric  __a->__a_value = __val;
11620b57cec5SDimitry Andric}
11630b57cec5SDimitry Andric
11640b57cec5SDimitry Andrictemplate <typename _Tp>
11650b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11660b57cec5SDimitry Andricvoid __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val, memory_order) {
11670b57cec5SDimitry Andric  __a->__lock();
11680b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, __val);
11690b57cec5SDimitry Andric  __a->__unlock();
11700b57cec5SDimitry Andric}
11710b57cec5SDimitry Andrictemplate <typename _Tp>
11720b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11730b57cec5SDimitry Andricvoid __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a,  _Tp __val, memory_order) {
11740b57cec5SDimitry Andric  __a->__lock();
11750b57cec5SDimitry Andric  __a->__a_value = __val;
11760b57cec5SDimitry Andric  __a->__unlock();
11770b57cec5SDimitry Andric}
11780b57cec5SDimitry Andric
11790b57cec5SDimitry Andrictemplate <typename _Tp>
11800b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11810b57cec5SDimitry Andric_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
11820b57cec5SDimitry Andric  return __a->__read();
11830b57cec5SDimitry Andric}
11840b57cec5SDimitry Andrictemplate <typename _Tp>
11850b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11860b57cec5SDimitry Andric_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
11870b57cec5SDimitry Andric  return __a->__read();
11880b57cec5SDimitry Andric}
11890b57cec5SDimitry Andric
11900b57cec5SDimitry Andrictemplate <typename _Tp>
11910b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
11920b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
11930b57cec5SDimitry Andric  __a->__lock();
11940b57cec5SDimitry Andric  _Tp __old;
11950b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
11960b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, __value);
11970b57cec5SDimitry Andric  __a->__unlock();
11980b57cec5SDimitry Andric  return __old;
11990b57cec5SDimitry Andric}
12000b57cec5SDimitry Andrictemplate <typename _Tp>
12010b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12020b57cec5SDimitry Andric_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
12030b57cec5SDimitry Andric  __a->__lock();
12040b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
12050b57cec5SDimitry Andric  __a->__a_value = __value;
12060b57cec5SDimitry Andric  __a->__unlock();
12070b57cec5SDimitry Andric  return __old;
12080b57cec5SDimitry Andric}
12090b57cec5SDimitry Andric
12100b57cec5SDimitry Andrictemplate <typename _Tp>
12110b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12120b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a,
12130b57cec5SDimitry Andric                                          _Tp* __expected, _Tp __value, memory_order, memory_order) {
12145ffd83dbSDimitry Andric  _Tp __temp;
1215e8d8bef9SDimitry Andric  __a->__lock();
12165ffd83dbSDimitry Andric  __cxx_atomic_assign_volatile(__temp, __a->__a_value);
1217e8d8bef9SDimitry Andric  bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
12180b57cec5SDimitry Andric  if(__ret)
12190b57cec5SDimitry Andric    __cxx_atomic_assign_volatile(__a->__a_value, __value);
12200b57cec5SDimitry Andric  else
12210b57cec5SDimitry Andric    __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
12220b57cec5SDimitry Andric  __a->__unlock();
12230b57cec5SDimitry Andric  return __ret;
12240b57cec5SDimitry Andric}
12250b57cec5SDimitry Andrictemplate <typename _Tp>
12260b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12270b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a,
12280b57cec5SDimitry Andric                                          _Tp* __expected, _Tp __value, memory_order, memory_order) {
12290b57cec5SDimitry Andric  __a->__lock();
1230e8d8bef9SDimitry Andric  bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
12310b57cec5SDimitry Andric  if(__ret)
1232e8d8bef9SDimitry Andric    _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
12330b57cec5SDimitry Andric  else
1234e8d8bef9SDimitry Andric    _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
12350b57cec5SDimitry Andric  __a->__unlock();
12360b57cec5SDimitry Andric  return __ret;
12370b57cec5SDimitry Andric}
12380b57cec5SDimitry Andric
12390b57cec5SDimitry Andrictemplate <typename _Tp>
12400b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12410b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a,
12420b57cec5SDimitry Andric                                        _Tp* __expected, _Tp __value, memory_order, memory_order) {
12435ffd83dbSDimitry Andric  _Tp __temp;
1244e8d8bef9SDimitry Andric  __a->__lock();
12455ffd83dbSDimitry Andric  __cxx_atomic_assign_volatile(__temp, __a->__a_value);
1246e8d8bef9SDimitry Andric  bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
12470b57cec5SDimitry Andric  if(__ret)
12480b57cec5SDimitry Andric    __cxx_atomic_assign_volatile(__a->__a_value, __value);
12490b57cec5SDimitry Andric  else
12500b57cec5SDimitry Andric    __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
12510b57cec5SDimitry Andric  __a->__unlock();
12520b57cec5SDimitry Andric  return __ret;
12530b57cec5SDimitry Andric}
12540b57cec5SDimitry Andrictemplate <typename _Tp>
12550b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12560b57cec5SDimitry Andricbool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a,
12570b57cec5SDimitry Andric                                        _Tp* __expected, _Tp __value, memory_order, memory_order) {
12580b57cec5SDimitry Andric  __a->__lock();
1259e8d8bef9SDimitry Andric  bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
12600b57cec5SDimitry Andric  if(__ret)
1261e8d8bef9SDimitry Andric    _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
12620b57cec5SDimitry Andric  else
1263e8d8bef9SDimitry Andric    _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
12640b57cec5SDimitry Andric  __a->__unlock();
12650b57cec5SDimitry Andric  return __ret;
12660b57cec5SDimitry Andric}
12670b57cec5SDimitry Andric
12680b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
12690b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12700b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a,
12710b57cec5SDimitry Andric                           _Td __delta, memory_order) {
12720b57cec5SDimitry Andric  __a->__lock();
12730b57cec5SDimitry Andric  _Tp __old;
12740b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
12750b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta));
12760b57cec5SDimitry Andric  __a->__unlock();
12770b57cec5SDimitry Andric  return __old;
12780b57cec5SDimitry Andric}
12790b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
12800b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12810b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a,
12820b57cec5SDimitry Andric                           _Td __delta, memory_order) {
12830b57cec5SDimitry Andric  __a->__lock();
12840b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
12850b57cec5SDimitry Andric  __a->__a_value += __delta;
12860b57cec5SDimitry Andric  __a->__unlock();
12870b57cec5SDimitry Andric  return __old;
12880b57cec5SDimitry Andric}
12890b57cec5SDimitry Andric
12900b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
12910b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
12920b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a,
12930b57cec5SDimitry Andric                           ptrdiff_t __delta, memory_order) {
12940b57cec5SDimitry Andric  __a->__lock();
12950b57cec5SDimitry Andric  _Tp* __old;
12960b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
12970b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta);
12980b57cec5SDimitry Andric  __a->__unlock();
12990b57cec5SDimitry Andric  return __old;
13000b57cec5SDimitry Andric}
13010b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
13020b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13030b57cec5SDimitry Andric_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a,
13040b57cec5SDimitry Andric                           ptrdiff_t __delta, memory_order) {
13050b57cec5SDimitry Andric  __a->__lock();
13060b57cec5SDimitry Andric  _Tp* __old = __a->__a_value;
13070b57cec5SDimitry Andric  __a->__a_value += __delta;
13080b57cec5SDimitry Andric  __a->__unlock();
13090b57cec5SDimitry Andric  return __old;
13100b57cec5SDimitry Andric}
13110b57cec5SDimitry Andric
13120b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
13130b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13140b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a,
13150b57cec5SDimitry Andric                           _Td __delta, memory_order) {
13160b57cec5SDimitry Andric  __a->__lock();
13170b57cec5SDimitry Andric  _Tp __old;
13180b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
13190b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta));
13200b57cec5SDimitry Andric  __a->__unlock();
13210b57cec5SDimitry Andric  return __old;
13220b57cec5SDimitry Andric}
13230b57cec5SDimitry Andrictemplate <typename _Tp, typename _Td>
13240b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13250b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a,
13260b57cec5SDimitry Andric                           _Td __delta, memory_order) {
13270b57cec5SDimitry Andric  __a->__lock();
13280b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
13290b57cec5SDimitry Andric  __a->__a_value -= __delta;
13300b57cec5SDimitry Andric  __a->__unlock();
13310b57cec5SDimitry Andric  return __old;
13320b57cec5SDimitry Andric}
13330b57cec5SDimitry Andric
13340b57cec5SDimitry Andrictemplate <typename _Tp>
13350b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13360b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a,
13370b57cec5SDimitry Andric                           _Tp __pattern, memory_order) {
13380b57cec5SDimitry Andric  __a->__lock();
13390b57cec5SDimitry Andric  _Tp __old;
13400b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
13410b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern));
13420b57cec5SDimitry Andric  __a->__unlock();
13430b57cec5SDimitry Andric  return __old;
13440b57cec5SDimitry Andric}
13450b57cec5SDimitry Andrictemplate <typename _Tp>
13460b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13470b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a,
13480b57cec5SDimitry Andric                           _Tp __pattern, memory_order) {
13490b57cec5SDimitry Andric  __a->__lock();
13500b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
13510b57cec5SDimitry Andric  __a->__a_value &= __pattern;
13520b57cec5SDimitry Andric  __a->__unlock();
13530b57cec5SDimitry Andric  return __old;
13540b57cec5SDimitry Andric}
13550b57cec5SDimitry Andric
13560b57cec5SDimitry Andrictemplate <typename _Tp>
13570b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13580b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a,
13590b57cec5SDimitry Andric                          _Tp __pattern, memory_order) {
13600b57cec5SDimitry Andric  __a->__lock();
13610b57cec5SDimitry Andric  _Tp __old;
13620b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
13630b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern));
13640b57cec5SDimitry Andric  __a->__unlock();
13650b57cec5SDimitry Andric  return __old;
13660b57cec5SDimitry Andric}
13670b57cec5SDimitry Andrictemplate <typename _Tp>
13680b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13690b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a,
13700b57cec5SDimitry Andric                          _Tp __pattern, memory_order) {
13710b57cec5SDimitry Andric  __a->__lock();
13720b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
13730b57cec5SDimitry Andric  __a->__a_value |= __pattern;
13740b57cec5SDimitry Andric  __a->__unlock();
13750b57cec5SDimitry Andric  return __old;
13760b57cec5SDimitry Andric}
13770b57cec5SDimitry Andric
13780b57cec5SDimitry Andrictemplate <typename _Tp>
13790b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13800b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a,
13810b57cec5SDimitry Andric                           _Tp __pattern, memory_order) {
13820b57cec5SDimitry Andric  __a->__lock();
13830b57cec5SDimitry Andric  _Tp __old;
13840b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__old, __a->__a_value);
13850b57cec5SDimitry Andric  __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern));
13860b57cec5SDimitry Andric  __a->__unlock();
13870b57cec5SDimitry Andric  return __old;
13880b57cec5SDimitry Andric}
13890b57cec5SDimitry Andrictemplate <typename _Tp>
13900b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
13910b57cec5SDimitry Andric_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a,
13920b57cec5SDimitry Andric                           _Tp __pattern, memory_order) {
13930b57cec5SDimitry Andric  __a->__lock();
13940b57cec5SDimitry Andric  _Tp __old = __a->__a_value;
13950b57cec5SDimitry Andric  __a->__a_value ^= __pattern;
13960b57cec5SDimitry Andric  __a->__unlock();
13970b57cec5SDimitry Andric  return __old;
13980b57cec5SDimitry Andric}
13990b57cec5SDimitry Andric
14000b57cec5SDimitry Andric#ifdef __cpp_lib_atomic_is_always_lock_free
14010b57cec5SDimitry Andric
14020b57cec5SDimitry Andrictemplate<typename _Tp> struct __cxx_is_always_lock_free {
14030b57cec5SDimitry Andric    enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; };
14040b57cec5SDimitry Andric
14050b57cec5SDimitry Andric#else
14060b57cec5SDimitry Andric
14070b57cec5SDimitry Andrictemplate<typename _Tp> struct __cxx_is_always_lock_free { enum { __value = false }; };
14080b57cec5SDimitry Andric// Implementations must match the C ATOMIC_*_LOCK_FREE macro values.
14090b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<bool> { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; };
14100b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; };
14110b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<signed char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; };
14120b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<unsigned char> { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; };
1413fe6060f1SDimitry Andric#ifndef _LIBCPP_HAS_NO_CHAR8_T
1414e8d8bef9SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<char8_t> { enum { __value = 2 == ATOMIC_CHAR8_T_LOCK_FREE }; };
1415e8d8bef9SDimitry Andric#endif
14160b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<char16_t> { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; };
14170b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<char32_t> { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; };
1418*349cc55cSDimitry Andric#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
14190b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<wchar_t> { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; };
1420*349cc55cSDimitry Andric#endif
14210b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; };
14220b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<unsigned short> { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; };
14230b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; };
14240b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<unsigned int> { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; };
14250b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; };
14260b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<unsigned long> { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; };
14270b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; };
14280b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<unsigned long long> { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; };
14290b57cec5SDimitry Andrictemplate<typename _Tp> struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; };
14300b57cec5SDimitry Andrictemplate<> struct __cxx_is_always_lock_free<std::nullptr_t> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; };
14310b57cec5SDimitry Andric
14320b57cec5SDimitry Andric#endif //__cpp_lib_atomic_is_always_lock_free
14330b57cec5SDimitry Andric
14340b57cec5SDimitry Andrictemplate <typename _Tp,
14350b57cec5SDimitry Andric          typename _Base = typename conditional<__cxx_is_always_lock_free<_Tp>::__value,
14360b57cec5SDimitry Andric                                                __cxx_atomic_base_impl<_Tp>,
14370b57cec5SDimitry Andric                                                __cxx_atomic_lock_impl<_Tp> >::type>
14380b57cec5SDimitry Andric#else
14390b57cec5SDimitry Andrictemplate <typename _Tp,
14400b57cec5SDimitry Andric          typename _Base = __cxx_atomic_base_impl<_Tp> >
14410b57cec5SDimitry Andric#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS
14420b57cec5SDimitry Andricstruct __cxx_atomic_impl : public _Base {
14430b57cec5SDimitry Andric    static_assert(is_trivially_copyable<_Tp>::value,
1444*349cc55cSDimitry Andric      "std::atomic<T> requires that 'T' be a trivially copyable type");
14450b57cec5SDimitry Andric
14460b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT
14470b57cec5SDimitry Andric  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT
14480b57cec5SDimitry Andric    : _Base(value) {}
14490b57cec5SDimitry Andric};
14500b57cec5SDimitry Andric
1451d061adc4SAdrian Chadd#if defined(__linux__) || (defined(__FreeBSD__) && defined(__mips__))
14525ffd83dbSDimitry Andric    using __cxx_contention_t = int32_t;
14535ffd83dbSDimitry Andric#else
14545ffd83dbSDimitry Andric    using __cxx_contention_t = int64_t;
1455d061adc4SAdrian Chadd#endif
14565ffd83dbSDimitry Andric
14575ffd83dbSDimitry Andricusing __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>;
14585ffd83dbSDimitry Andric
1459*349cc55cSDimitry Andric#if defined(_LIBCPP_HAS_NO_THREADS)
1460*349cc55cSDimitry Andric#   define _LIBCPP_HAS_NO_PLATFORM_WAIT
1461*349cc55cSDimitry Andric#endif
1462*349cc55cSDimitry Andric
1463*349cc55cSDimitry Andric// TODO:
1464*349cc55cSDimitry Andric// _LIBCPP_HAS_NO_PLATFORM_WAIT is currently a "dead" macro, in the sense that
1465*349cc55cSDimitry Andric// it is not tied anywhere into the build system or even documented. We should
1466*349cc55cSDimitry Andric// clean it up because it is technically never defined except when threads are
1467*349cc55cSDimitry Andric// disabled. We should clean it up in its own changeset in case we break "bad"
1468*349cc55cSDimitry Andric// users.
14695ffd83dbSDimitry Andric#ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT
14705ffd83dbSDimitry Andric
14715ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*);
14725ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*);
14735ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*);
14745ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t);
14755ffd83dbSDimitry Andric
14765ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*);
14775ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*);
14785ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*);
14795ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t);
14805ffd83dbSDimitry Andric
14815ffd83dbSDimitry Andrictemplate <class _Atp, class _Fn>
14825ffd83dbSDimitry Andricstruct __libcpp_atomic_wait_backoff_impl {
14835ffd83dbSDimitry Andric    _Atp* __a;
14845ffd83dbSDimitry Andric    _Fn __test_fn;
14855ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC
14865ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const
14875ffd83dbSDimitry Andric    {
14885ffd83dbSDimitry Andric        if(__elapsed > chrono::microseconds(64))
14895ffd83dbSDimitry Andric        {
14905ffd83dbSDimitry Andric            auto const __monitor = __libcpp_atomic_monitor(__a);
14915ffd83dbSDimitry Andric            if(__test_fn())
14925ffd83dbSDimitry Andric                return true;
14935ffd83dbSDimitry Andric            __libcpp_atomic_wait(__a, __monitor);
14945ffd83dbSDimitry Andric        }
14955ffd83dbSDimitry Andric        else if(__elapsed > chrono::microseconds(4))
14965ffd83dbSDimitry Andric            __libcpp_thread_yield();
14975ffd83dbSDimitry Andric        else
1498e8d8bef9SDimitry Andric            {} // poll
14995ffd83dbSDimitry Andric        return false;
15005ffd83dbSDimitry Andric    }
15015ffd83dbSDimitry Andric};
15025ffd83dbSDimitry Andric
15035ffd83dbSDimitry Andrictemplate <class _Atp, class _Fn>
15045ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC
15055ffd83dbSDimitry Andric_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn)
15065ffd83dbSDimitry Andric{
15075ffd83dbSDimitry Andric    __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn};
15085ffd83dbSDimitry Andric    return __libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn);
15095ffd83dbSDimitry Andric}
15105ffd83dbSDimitry Andric
15115ffd83dbSDimitry Andric#else // _LIBCPP_HAS_NO_PLATFORM_WAIT
15125ffd83dbSDimitry Andric
15135ffd83dbSDimitry Andrictemplate <class _Tp>
15145ffd83dbSDimitry Andric_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { }
15155ffd83dbSDimitry Andrictemplate <class _Tp>
15165ffd83dbSDimitry Andric_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { }
15175ffd83dbSDimitry Andrictemplate <class _Atp, class _Fn>
15185ffd83dbSDimitry Andric_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn)
15195ffd83dbSDimitry Andric{
1520*349cc55cSDimitry Andric#if defined(_LIBCPP_HAS_NO_THREADS)
1521*349cc55cSDimitry Andric    using _Policy = __spinning_backoff_policy;
1522*349cc55cSDimitry Andric#else
1523*349cc55cSDimitry Andric    using _Policy = __libcpp_timed_backoff_policy;
1524*349cc55cSDimitry Andric#endif
1525*349cc55cSDimitry Andric    return __libcpp_thread_poll_with_backoff(__test_fn, _Policy());
15265ffd83dbSDimitry Andric}
15275ffd83dbSDimitry Andric
15285ffd83dbSDimitry Andric#endif // _LIBCPP_HAS_NO_PLATFORM_WAIT
15295ffd83dbSDimitry Andric
15305ffd83dbSDimitry Andrictemplate <class _Atp, class _Tp>
15315ffd83dbSDimitry Andricstruct __cxx_atomic_wait_test_fn_impl {
15325ffd83dbSDimitry Andric    _Atp* __a;
15335ffd83dbSDimitry Andric    _Tp __val;
15345ffd83dbSDimitry Andric    memory_order __order;
15355ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY bool operator()() const
15365ffd83dbSDimitry Andric    {
15375ffd83dbSDimitry Andric        return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val);
15385ffd83dbSDimitry Andric    }
15395ffd83dbSDimitry Andric};
15405ffd83dbSDimitry Andric
15415ffd83dbSDimitry Andrictemplate <class _Atp, class _Tp>
15425ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC
15435ffd83dbSDimitry Andric_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order)
15445ffd83dbSDimitry Andric{
15455ffd83dbSDimitry Andric    __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order};
15465ffd83dbSDimitry Andric    return __cxx_atomic_wait(__a, __test_fn);
15475ffd83dbSDimitry Andric}
15485ffd83dbSDimitry Andric
15490b57cec5SDimitry Andric// general atomic<T>
15500b57cec5SDimitry Andric
15510b57cec5SDimitry Andrictemplate <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>
15520b57cec5SDimitry Andricstruct __atomic_base  // false
15530b57cec5SDimitry Andric{
15540b57cec5SDimitry Andric    mutable __cxx_atomic_impl<_Tp> __a_;
15550b57cec5SDimitry Andric
15560b57cec5SDimitry Andric#if defined(__cpp_lib_atomic_is_always_lock_free)
15570b57cec5SDimitry Andric  static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0);
15580b57cec5SDimitry Andric#endif
15590b57cec5SDimitry Andric
15600b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15610b57cec5SDimitry Andric    bool is_lock_free() const volatile _NOEXCEPT
15620b57cec5SDimitry Andric        {return __cxx_atomic_is_lock_free(sizeof(_Tp));}
15630b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15640b57cec5SDimitry Andric    bool is_lock_free() const _NOEXCEPT
15650b57cec5SDimitry Andric        {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();}
15660b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15670b57cec5SDimitry Andric    void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
15680b57cec5SDimitry Andric      _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
15690b57cec5SDimitry Andric        {__cxx_atomic_store(&__a_, __d, __m);}
15700b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15710b57cec5SDimitry Andric    void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
15720b57cec5SDimitry Andric      _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
15730b57cec5SDimitry Andric        {__cxx_atomic_store(&__a_, __d, __m);}
15740b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15750b57cec5SDimitry Andric    _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
15760b57cec5SDimitry Andric      _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
15770b57cec5SDimitry Andric        {return __cxx_atomic_load(&__a_, __m);}
15780b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15790b57cec5SDimitry Andric    _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
15800b57cec5SDimitry Andric      _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
15810b57cec5SDimitry Andric        {return __cxx_atomic_load(&__a_, __m);}
15820b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15830b57cec5SDimitry Andric    operator _Tp() const volatile _NOEXCEPT {return load();}
15840b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15850b57cec5SDimitry Andric    operator _Tp() const _NOEXCEPT          {return load();}
15860b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15870b57cec5SDimitry Andric    _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
15880b57cec5SDimitry Andric        {return __cxx_atomic_exchange(&__a_, __d, __m);}
15890b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15900b57cec5SDimitry Andric    _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
15910b57cec5SDimitry Andric        {return __cxx_atomic_exchange(&__a_, __d, __m);}
15920b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15930b57cec5SDimitry Andric    bool compare_exchange_weak(_Tp& __e, _Tp __d,
15940b57cec5SDimitry Andric                               memory_order __s, memory_order __f) volatile _NOEXCEPT
15950b57cec5SDimitry Andric      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
15960b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);}
15970b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
15980b57cec5SDimitry Andric    bool compare_exchange_weak(_Tp& __e, _Tp __d,
15990b57cec5SDimitry Andric                               memory_order __s, memory_order __f) _NOEXCEPT
16000b57cec5SDimitry Andric      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
16010b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);}
16020b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16030b57cec5SDimitry Andric    bool compare_exchange_strong(_Tp& __e, _Tp __d,
16040b57cec5SDimitry Andric                                 memory_order __s, memory_order __f) volatile _NOEXCEPT
16050b57cec5SDimitry Andric      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
16060b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);}
16070b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16080b57cec5SDimitry Andric    bool compare_exchange_strong(_Tp& __e, _Tp __d,
16090b57cec5SDimitry Andric                                 memory_order __s, memory_order __f) _NOEXCEPT
16100b57cec5SDimitry Andric      _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
16110b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);}
16120b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16130b57cec5SDimitry Andric    bool compare_exchange_weak(_Tp& __e, _Tp __d,
16140b57cec5SDimitry Andric                              memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
16150b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);}
16160b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16170b57cec5SDimitry Andric    bool compare_exchange_weak(_Tp& __e, _Tp __d,
16180b57cec5SDimitry Andric                               memory_order __m = memory_order_seq_cst) _NOEXCEPT
16190b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);}
16200b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16210b57cec5SDimitry Andric    bool compare_exchange_strong(_Tp& __e, _Tp __d,
16220b57cec5SDimitry Andric                              memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
16230b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);}
16240b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16250b57cec5SDimitry Andric    bool compare_exchange_strong(_Tp& __e, _Tp __d,
16260b57cec5SDimitry Andric                                 memory_order __m = memory_order_seq_cst) _NOEXCEPT
16270b57cec5SDimitry Andric        {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);}
16280b57cec5SDimitry Andric
16295ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
16305ffd83dbSDimitry Andric        {__cxx_atomic_wait(&__a_, __v, __m);}
16315ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT
16325ffd83dbSDimitry Andric        {__cxx_atomic_wait(&__a_, __v, __m);}
16335ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT
16345ffd83dbSDimitry Andric        {__cxx_atomic_notify_one(&__a_);}
16355ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT
16365ffd83dbSDimitry Andric        {__cxx_atomic_notify_one(&__a_);}
16375ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT
16385ffd83dbSDimitry Andric        {__cxx_atomic_notify_all(&__a_);}
16395ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT
16405ffd83dbSDimitry Andric        {__cxx_atomic_notify_all(&__a_);}
16415ffd83dbSDimitry Andric
1642fe6060f1SDimitry Andric#if _LIBCPP_STD_VER > 17
1643fe6060f1SDimitry Andric    _LIBCPP_INLINE_VISIBILITY constexpr
1644fe6060f1SDimitry Andric    __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {}
1645fe6060f1SDimitry Andric#else
16460b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16470b57cec5SDimitry Andric    __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT
1648fe6060f1SDimitry Andric#endif
16490b57cec5SDimitry Andric
16500b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
16510b57cec5SDimitry Andric    __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {}
16520b57cec5SDimitry Andric
16530b57cec5SDimitry Andric#ifndef _LIBCPP_CXX03_LANG
16540b57cec5SDimitry Andric    __atomic_base(const __atomic_base&) = delete;
16550b57cec5SDimitry Andric#else
16560b57cec5SDimitry Andricprivate:
16575ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16580b57cec5SDimitry Andric    __atomic_base(const __atomic_base&);
16590b57cec5SDimitry Andric#endif
16600b57cec5SDimitry Andric};
16610b57cec5SDimitry Andric
16620b57cec5SDimitry Andric#if defined(__cpp_lib_atomic_is_always_lock_free)
16630b57cec5SDimitry Andrictemplate <class _Tp, bool __b>
16640b57cec5SDimitry Andric_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free;
16650b57cec5SDimitry Andric#endif
16660b57cec5SDimitry Andric
16670b57cec5SDimitry Andric// atomic<Integral>
16680b57cec5SDimitry Andric
16690b57cec5SDimitry Andrictemplate <class _Tp>
16700b57cec5SDimitry Andricstruct __atomic_base<_Tp, true>
16710b57cec5SDimitry Andric    : public __atomic_base<_Tp, false>
16720b57cec5SDimitry Andric{
16730b57cec5SDimitry Andric    typedef __atomic_base<_Tp, false> __base;
1674fe6060f1SDimitry Andric
1675fe6060f1SDimitry Andric    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
16760b57cec5SDimitry Andric    __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT
1677fe6060f1SDimitry Andric
16780b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16790b57cec5SDimitry Andric    _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}
16800b57cec5SDimitry Andric
16810b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16820b57cec5SDimitry Andric    _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
16830b57cec5SDimitry Andric        {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);}
16840b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16850b57cec5SDimitry Andric    _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
16860b57cec5SDimitry Andric        {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);}
16870b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16880b57cec5SDimitry Andric    _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
16890b57cec5SDimitry Andric        {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);}
16900b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16910b57cec5SDimitry Andric    _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
16920b57cec5SDimitry Andric        {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);}
16930b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16940b57cec5SDimitry Andric    _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
16950b57cec5SDimitry Andric        {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);}
16960b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
16970b57cec5SDimitry Andric    _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
16980b57cec5SDimitry Andric        {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);}
16990b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17000b57cec5SDimitry Andric    _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
17010b57cec5SDimitry Andric        {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);}
17020b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17030b57cec5SDimitry Andric    _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
17040b57cec5SDimitry Andric        {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);}
17050b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17060b57cec5SDimitry Andric    _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
17070b57cec5SDimitry Andric        {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);}
17080b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17090b57cec5SDimitry Andric    _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT
17100b57cec5SDimitry Andric        {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);}
17110b57cec5SDimitry Andric
17120b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17130b57cec5SDimitry Andric    _Tp operator++(int) volatile _NOEXCEPT      {return fetch_add(_Tp(1));}
17140b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17150b57cec5SDimitry Andric    _Tp operator++(int) _NOEXCEPT               {return fetch_add(_Tp(1));}
17160b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17170b57cec5SDimitry Andric    _Tp operator--(int) volatile _NOEXCEPT      {return fetch_sub(_Tp(1));}
17180b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17190b57cec5SDimitry Andric    _Tp operator--(int) _NOEXCEPT               {return fetch_sub(_Tp(1));}
17200b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17210b57cec5SDimitry Andric    _Tp operator++() volatile _NOEXCEPT         {return fetch_add(_Tp(1)) + _Tp(1);}
17220b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17230b57cec5SDimitry Andric    _Tp operator++() _NOEXCEPT                  {return fetch_add(_Tp(1)) + _Tp(1);}
17240b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17250b57cec5SDimitry Andric    _Tp operator--() volatile _NOEXCEPT         {return fetch_sub(_Tp(1)) - _Tp(1);}
17260b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17270b57cec5SDimitry Andric    _Tp operator--() _NOEXCEPT                  {return fetch_sub(_Tp(1)) - _Tp(1);}
17280b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17290b57cec5SDimitry Andric    _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;}
17300b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17310b57cec5SDimitry Andric    _Tp operator+=(_Tp __op) _NOEXCEPT          {return fetch_add(__op) + __op;}
17320b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17330b57cec5SDimitry Andric    _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;}
17340b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17350b57cec5SDimitry Andric    _Tp operator-=(_Tp __op) _NOEXCEPT          {return fetch_sub(__op) - __op;}
17360b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17370b57cec5SDimitry Andric    _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;}
17380b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17390b57cec5SDimitry Andric    _Tp operator&=(_Tp __op) _NOEXCEPT          {return fetch_and(__op) & __op;}
17400b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17410b57cec5SDimitry Andric    _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;}
17420b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17430b57cec5SDimitry Andric    _Tp operator|=(_Tp __op) _NOEXCEPT          {return fetch_or(__op) | __op;}
17440b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17450b57cec5SDimitry Andric    _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;}
17460b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17470b57cec5SDimitry Andric    _Tp operator^=(_Tp __op) _NOEXCEPT          {return fetch_xor(__op) ^ __op;}
17480b57cec5SDimitry Andric};
17490b57cec5SDimitry Andric
17500b57cec5SDimitry Andric// atomic<T>
17510b57cec5SDimitry Andric
17520b57cec5SDimitry Andrictemplate <class _Tp>
17530b57cec5SDimitry Andricstruct atomic
17540b57cec5SDimitry Andric    : public __atomic_base<_Tp>
17550b57cec5SDimitry Andric{
17560b57cec5SDimitry Andric    typedef __atomic_base<_Tp> __base;
17575ffd83dbSDimitry Andric    typedef _Tp value_type;
1758e8d8bef9SDimitry Andric    typedef value_type difference_type;
1759fe6060f1SDimitry Andric
1760fe6060f1SDimitry Andric#if _LIBCPP_STD_VER > 17
1761fe6060f1SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
1762fe6060f1SDimitry Andric    atomic() = default;
1763fe6060f1SDimitry Andric#else
17640b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17650b57cec5SDimitry Andric    atomic() _NOEXCEPT _LIBCPP_DEFAULT
1766fe6060f1SDimitry Andric#endif
1767fe6060f1SDimitry Andric
17680b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17690b57cec5SDimitry Andric    _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {}
17700b57cec5SDimitry Andric
17710b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17720b57cec5SDimitry Andric    _Tp operator=(_Tp __d) volatile _NOEXCEPT
17730b57cec5SDimitry Andric        {__base::store(__d); return __d;}
17740b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17750b57cec5SDimitry Andric    _Tp operator=(_Tp __d) _NOEXCEPT
17760b57cec5SDimitry Andric        {__base::store(__d); return __d;}
1777fe6060f1SDimitry Andric
1778fe6060f1SDimitry Andric    atomic& operator=(const atomic&) = delete;
1779fe6060f1SDimitry Andric    atomic& operator=(const atomic&) volatile = delete;
17800b57cec5SDimitry Andric};
17810b57cec5SDimitry Andric
17820b57cec5SDimitry Andric// atomic<T*>
17830b57cec5SDimitry Andric
17840b57cec5SDimitry Andrictemplate <class _Tp>
17850b57cec5SDimitry Andricstruct atomic<_Tp*>
17860b57cec5SDimitry Andric    : public __atomic_base<_Tp*>
17870b57cec5SDimitry Andric{
17880b57cec5SDimitry Andric    typedef __atomic_base<_Tp*> __base;
17895ffd83dbSDimitry Andric    typedef _Tp* value_type;
1790e8d8bef9SDimitry Andric    typedef ptrdiff_t difference_type;
1791fe6060f1SDimitry Andric
17920b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17930b57cec5SDimitry Andric    atomic() _NOEXCEPT _LIBCPP_DEFAULT
1794fe6060f1SDimitry Andric
17950b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17960b57cec5SDimitry Andric    _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}
17970b57cec5SDimitry Andric
17980b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
17990b57cec5SDimitry Andric    _Tp* operator=(_Tp* __d) volatile _NOEXCEPT
18000b57cec5SDimitry Andric        {__base::store(__d); return __d;}
18010b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18020b57cec5SDimitry Andric    _Tp* operator=(_Tp* __d) _NOEXCEPT
18030b57cec5SDimitry Andric        {__base::store(__d); return __d;}
18040b57cec5SDimitry Andric
18050b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
1806*349cc55cSDimitry Andric    _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
1807*349cc55cSDimitry Andric        // __atomic_fetch_add accepts function pointers, guard against them.
1808*349cc55cSDimitry Andric        static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed");
1809*349cc55cSDimitry Andric        return __cxx_atomic_fetch_add(&this->__a_, __op, __m);
1810*349cc55cSDimitry Andric    }
1811*349cc55cSDimitry Andric
18120b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
1813*349cc55cSDimitry Andric    _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
1814*349cc55cSDimitry Andric        // __atomic_fetch_add accepts function pointers, guard against them.
1815*349cc55cSDimitry Andric        static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed");
1816*349cc55cSDimitry Andric        return __cxx_atomic_fetch_add(&this->__a_, __op, __m);
1817*349cc55cSDimitry Andric    }
1818*349cc55cSDimitry Andric
18190b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
1820*349cc55cSDimitry Andric    _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
1821*349cc55cSDimitry Andric        // __atomic_fetch_add accepts function pointers, guard against them.
1822*349cc55cSDimitry Andric        static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed");
1823*349cc55cSDimitry Andric        return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);
1824*349cc55cSDimitry Andric    }
1825*349cc55cSDimitry Andric
18260b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
1827*349cc55cSDimitry Andric    _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
1828*349cc55cSDimitry Andric        // __atomic_fetch_add accepts function pointers, guard against them.
1829*349cc55cSDimitry Andric        static_assert(!is_function<typename remove_pointer<_Tp>::type>::value, "Pointer to function isn't allowed");
1830*349cc55cSDimitry Andric        return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);
1831*349cc55cSDimitry Andric    }
18320b57cec5SDimitry Andric
18330b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18340b57cec5SDimitry Andric    _Tp* operator++(int) volatile _NOEXCEPT            {return fetch_add(1);}
18350b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18360b57cec5SDimitry Andric    _Tp* operator++(int) _NOEXCEPT                     {return fetch_add(1);}
18370b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18380b57cec5SDimitry Andric    _Tp* operator--(int) volatile _NOEXCEPT            {return fetch_sub(1);}
18390b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18400b57cec5SDimitry Andric    _Tp* operator--(int) _NOEXCEPT                     {return fetch_sub(1);}
18410b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18420b57cec5SDimitry Andric    _Tp* operator++() volatile _NOEXCEPT               {return fetch_add(1) + 1;}
18430b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18440b57cec5SDimitry Andric    _Tp* operator++() _NOEXCEPT                        {return fetch_add(1) + 1;}
18450b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18460b57cec5SDimitry Andric    _Tp* operator--() volatile _NOEXCEPT               {return fetch_sub(1) - 1;}
18470b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18480b57cec5SDimitry Andric    _Tp* operator--() _NOEXCEPT                        {return fetch_sub(1) - 1;}
18490b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18500b57cec5SDimitry Andric    _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;}
18510b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18520b57cec5SDimitry Andric    _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT          {return fetch_add(__op) + __op;}
18530b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18540b57cec5SDimitry Andric    _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;}
18550b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
18560b57cec5SDimitry Andric    _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT          {return fetch_sub(__op) - __op;}
1857fe6060f1SDimitry Andric
1858fe6060f1SDimitry Andric    atomic& operator=(const atomic&) = delete;
1859fe6060f1SDimitry Andric    atomic& operator=(const atomic&) volatile = delete;
18600b57cec5SDimitry Andric};
18610b57cec5SDimitry Andric
18620b57cec5SDimitry Andric// atomic_is_lock_free
18630b57cec5SDimitry Andric
18640b57cec5SDimitry Andrictemplate <class _Tp>
18650b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
18660b57cec5SDimitry Andricbool
18670b57cec5SDimitry Andricatomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT
18680b57cec5SDimitry Andric{
18690b57cec5SDimitry Andric    return __o->is_lock_free();
18700b57cec5SDimitry Andric}
18710b57cec5SDimitry Andric
18720b57cec5SDimitry Andrictemplate <class _Tp>
18730b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
18740b57cec5SDimitry Andricbool
18750b57cec5SDimitry Andricatomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT
18760b57cec5SDimitry Andric{
18770b57cec5SDimitry Andric    return __o->is_lock_free();
18780b57cec5SDimitry Andric}
18790b57cec5SDimitry Andric
18800b57cec5SDimitry Andric// atomic_init
18810b57cec5SDimitry Andric
18820b57cec5SDimitry Andrictemplate <class _Tp>
1883fe6060f1SDimitry Andric_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY
18840b57cec5SDimitry Andricvoid
1885e8d8bef9SDimitry Andricatomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
18860b57cec5SDimitry Andric{
18870b57cec5SDimitry Andric    __cxx_atomic_init(&__o->__a_, __d);
18880b57cec5SDimitry Andric}
18890b57cec5SDimitry Andric
18900b57cec5SDimitry Andrictemplate <class _Tp>
1891fe6060f1SDimitry Andric_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY
18920b57cec5SDimitry Andricvoid
1893e8d8bef9SDimitry Andricatomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
18940b57cec5SDimitry Andric{
18950b57cec5SDimitry Andric    __cxx_atomic_init(&__o->__a_, __d);
18960b57cec5SDimitry Andric}
18970b57cec5SDimitry Andric
18980b57cec5SDimitry Andric// atomic_store
18990b57cec5SDimitry Andric
19000b57cec5SDimitry Andrictemplate <class _Tp>
19010b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19020b57cec5SDimitry Andricvoid
1903e8d8bef9SDimitry Andricatomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
19040b57cec5SDimitry Andric{
19050b57cec5SDimitry Andric    __o->store(__d);
19060b57cec5SDimitry Andric}
19070b57cec5SDimitry Andric
19080b57cec5SDimitry Andrictemplate <class _Tp>
19090b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19100b57cec5SDimitry Andricvoid
1911e8d8bef9SDimitry Andricatomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
19120b57cec5SDimitry Andric{
19130b57cec5SDimitry Andric    __o->store(__d);
19140b57cec5SDimitry Andric}
19150b57cec5SDimitry Andric
19160b57cec5SDimitry Andric// atomic_store_explicit
19170b57cec5SDimitry Andric
19180b57cec5SDimitry Andrictemplate <class _Tp>
19190b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19200b57cec5SDimitry Andricvoid
1921e8d8bef9SDimitry Andricatomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
19220b57cec5SDimitry Andric  _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
19230b57cec5SDimitry Andric{
19240b57cec5SDimitry Andric    __o->store(__d, __m);
19250b57cec5SDimitry Andric}
19260b57cec5SDimitry Andric
19270b57cec5SDimitry Andrictemplate <class _Tp>
19280b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19290b57cec5SDimitry Andricvoid
1930e8d8bef9SDimitry Andricatomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
19310b57cec5SDimitry Andric  _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m)
19320b57cec5SDimitry Andric{
19330b57cec5SDimitry Andric    __o->store(__d, __m);
19340b57cec5SDimitry Andric}
19350b57cec5SDimitry Andric
19360b57cec5SDimitry Andric// atomic_load
19370b57cec5SDimitry Andric
19380b57cec5SDimitry Andrictemplate <class _Tp>
19390b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19400b57cec5SDimitry Andric_Tp
19410b57cec5SDimitry Andricatomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT
19420b57cec5SDimitry Andric{
19430b57cec5SDimitry Andric    return __o->load();
19440b57cec5SDimitry Andric}
19450b57cec5SDimitry Andric
19460b57cec5SDimitry Andrictemplate <class _Tp>
19470b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19480b57cec5SDimitry Andric_Tp
19490b57cec5SDimitry Andricatomic_load(const atomic<_Tp>* __o) _NOEXCEPT
19500b57cec5SDimitry Andric{
19510b57cec5SDimitry Andric    return __o->load();
19520b57cec5SDimitry Andric}
19530b57cec5SDimitry Andric
19540b57cec5SDimitry Andric// atomic_load_explicit
19550b57cec5SDimitry Andric
19560b57cec5SDimitry Andrictemplate <class _Tp>
19570b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19580b57cec5SDimitry Andric_Tp
19590b57cec5SDimitry Andricatomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
19600b57cec5SDimitry Andric  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
19610b57cec5SDimitry Andric{
19620b57cec5SDimitry Andric    return __o->load(__m);
19630b57cec5SDimitry Andric}
19640b57cec5SDimitry Andric
19650b57cec5SDimitry Andrictemplate <class _Tp>
19660b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19670b57cec5SDimitry Andric_Tp
19680b57cec5SDimitry Andricatomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
19690b57cec5SDimitry Andric  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
19700b57cec5SDimitry Andric{
19710b57cec5SDimitry Andric    return __o->load(__m);
19720b57cec5SDimitry Andric}
19730b57cec5SDimitry Andric
19740b57cec5SDimitry Andric// atomic_exchange
19750b57cec5SDimitry Andric
19760b57cec5SDimitry Andrictemplate <class _Tp>
19770b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19780b57cec5SDimitry Andric_Tp
1979e8d8bef9SDimitry Andricatomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
19800b57cec5SDimitry Andric{
19810b57cec5SDimitry Andric    return __o->exchange(__d);
19820b57cec5SDimitry Andric}
19830b57cec5SDimitry Andric
19840b57cec5SDimitry Andrictemplate <class _Tp>
19850b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19860b57cec5SDimitry Andric_Tp
1987e8d8bef9SDimitry Andricatomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT
19880b57cec5SDimitry Andric{
19890b57cec5SDimitry Andric    return __o->exchange(__d);
19900b57cec5SDimitry Andric}
19910b57cec5SDimitry Andric
19920b57cec5SDimitry Andric// atomic_exchange_explicit
19930b57cec5SDimitry Andric
19940b57cec5SDimitry Andrictemplate <class _Tp>
19950b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
19960b57cec5SDimitry Andric_Tp
1997e8d8bef9SDimitry Andricatomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
19980b57cec5SDimitry Andric{
19990b57cec5SDimitry Andric    return __o->exchange(__d, __m);
20000b57cec5SDimitry Andric}
20010b57cec5SDimitry Andric
20020b57cec5SDimitry Andrictemplate <class _Tp>
20030b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20040b57cec5SDimitry Andric_Tp
2005e8d8bef9SDimitry Andricatomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
20060b57cec5SDimitry Andric{
20070b57cec5SDimitry Andric    return __o->exchange(__d, __m);
20080b57cec5SDimitry Andric}
20090b57cec5SDimitry Andric
20100b57cec5SDimitry Andric// atomic_compare_exchange_weak
20110b57cec5SDimitry Andric
20120b57cec5SDimitry Andrictemplate <class _Tp>
20130b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20140b57cec5SDimitry Andricbool
2015e8d8bef9SDimitry Andricatomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT
20160b57cec5SDimitry Andric{
20170b57cec5SDimitry Andric    return __o->compare_exchange_weak(*__e, __d);
20180b57cec5SDimitry Andric}
20190b57cec5SDimitry Andric
20200b57cec5SDimitry Andrictemplate <class _Tp>
20210b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20220b57cec5SDimitry Andricbool
2023e8d8bef9SDimitry Andricatomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT
20240b57cec5SDimitry Andric{
20250b57cec5SDimitry Andric    return __o->compare_exchange_weak(*__e, __d);
20260b57cec5SDimitry Andric}
20270b57cec5SDimitry Andric
20280b57cec5SDimitry Andric// atomic_compare_exchange_strong
20290b57cec5SDimitry Andric
20300b57cec5SDimitry Andrictemplate <class _Tp>
20310b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20320b57cec5SDimitry Andricbool
2033e8d8bef9SDimitry Andricatomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT
20340b57cec5SDimitry Andric{
20350b57cec5SDimitry Andric    return __o->compare_exchange_strong(*__e, __d);
20360b57cec5SDimitry Andric}
20370b57cec5SDimitry Andric
20380b57cec5SDimitry Andrictemplate <class _Tp>
20390b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20400b57cec5SDimitry Andricbool
2041e8d8bef9SDimitry Andricatomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT
20420b57cec5SDimitry Andric{
20430b57cec5SDimitry Andric    return __o->compare_exchange_strong(*__e, __d);
20440b57cec5SDimitry Andric}
20450b57cec5SDimitry Andric
20460b57cec5SDimitry Andric// atomic_compare_exchange_weak_explicit
20470b57cec5SDimitry Andric
20480b57cec5SDimitry Andrictemplate <class _Tp>
20490b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20500b57cec5SDimitry Andricbool
2051e8d8bef9SDimitry Andricatomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e,
2052e8d8bef9SDimitry Andric                                      typename atomic<_Tp>::value_type __d,
20530b57cec5SDimitry Andric                                      memory_order __s, memory_order __f) _NOEXCEPT
20540b57cec5SDimitry Andric  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
20550b57cec5SDimitry Andric{
20560b57cec5SDimitry Andric    return __o->compare_exchange_weak(*__e, __d, __s, __f);
20570b57cec5SDimitry Andric}
20580b57cec5SDimitry Andric
20590b57cec5SDimitry Andrictemplate <class _Tp>
20600b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20610b57cec5SDimitry Andricbool
2062e8d8bef9SDimitry Andricatomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d,
20630b57cec5SDimitry Andric                                      memory_order __s, memory_order __f) _NOEXCEPT
20640b57cec5SDimitry Andric  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
20650b57cec5SDimitry Andric{
20660b57cec5SDimitry Andric    return __o->compare_exchange_weak(*__e, __d, __s, __f);
20670b57cec5SDimitry Andric}
20680b57cec5SDimitry Andric
20690b57cec5SDimitry Andric// atomic_compare_exchange_strong_explicit
20700b57cec5SDimitry Andric
20710b57cec5SDimitry Andrictemplate <class _Tp>
20720b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20730b57cec5SDimitry Andricbool
20740b57cec5SDimitry Andricatomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o,
2075e8d8bef9SDimitry Andric                                        typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d,
20760b57cec5SDimitry Andric                                        memory_order __s, memory_order __f) _NOEXCEPT
20770b57cec5SDimitry Andric  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
20780b57cec5SDimitry Andric{
20790b57cec5SDimitry Andric    return __o->compare_exchange_strong(*__e, __d, __s, __f);
20800b57cec5SDimitry Andric}
20810b57cec5SDimitry Andric
20820b57cec5SDimitry Andrictemplate <class _Tp>
20830b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
20840b57cec5SDimitry Andricbool
2085e8d8bef9SDimitry Andricatomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e,
2086e8d8bef9SDimitry Andric                                        typename atomic<_Tp>::value_type __d,
20870b57cec5SDimitry Andric                                        memory_order __s, memory_order __f) _NOEXCEPT
20880b57cec5SDimitry Andric  _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f)
20890b57cec5SDimitry Andric{
20900b57cec5SDimitry Andric    return __o->compare_exchange_strong(*__e, __d, __s, __f);
20910b57cec5SDimitry Andric}
20920b57cec5SDimitry Andric
20935ffd83dbSDimitry Andric// atomic_wait
20945ffd83dbSDimitry Andric
20955ffd83dbSDimitry Andrictemplate <class _Tp>
20965ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
20975ffd83dbSDimitry Andricvoid atomic_wait(const volatile atomic<_Tp>* __o,
20985ffd83dbSDimitry Andric                 typename atomic<_Tp>::value_type __v) _NOEXCEPT
20995ffd83dbSDimitry Andric{
21005ffd83dbSDimitry Andric    return __o->wait(__v);
21015ffd83dbSDimitry Andric}
21025ffd83dbSDimitry Andric
21035ffd83dbSDimitry Andrictemplate <class _Tp>
21045ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
21055ffd83dbSDimitry Andricvoid atomic_wait(const atomic<_Tp>* __o,
21065ffd83dbSDimitry Andric                 typename atomic<_Tp>::value_type __v) _NOEXCEPT
21075ffd83dbSDimitry Andric{
21085ffd83dbSDimitry Andric    return __o->wait(__v);
21095ffd83dbSDimitry Andric}
21105ffd83dbSDimitry Andric
21115ffd83dbSDimitry Andric// atomic_wait_explicit
21125ffd83dbSDimitry Andric
21135ffd83dbSDimitry Andrictemplate <class _Tp>
21145ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
21155ffd83dbSDimitry Andricvoid atomic_wait_explicit(const volatile atomic<_Tp>* __o,
21165ffd83dbSDimitry Andric                          typename atomic<_Tp>::value_type __v,
21175ffd83dbSDimitry Andric                          memory_order __m) _NOEXCEPT
21185ffd83dbSDimitry Andric  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
21195ffd83dbSDimitry Andric{
21205ffd83dbSDimitry Andric    return __o->wait(__v, __m);
21215ffd83dbSDimitry Andric}
21225ffd83dbSDimitry Andric
21235ffd83dbSDimitry Andrictemplate <class _Tp>
21245ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
21255ffd83dbSDimitry Andricvoid atomic_wait_explicit(const atomic<_Tp>* __o,
21265ffd83dbSDimitry Andric                          typename atomic<_Tp>::value_type __v,
21275ffd83dbSDimitry Andric                          memory_order __m) _NOEXCEPT
21285ffd83dbSDimitry Andric  _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m)
21295ffd83dbSDimitry Andric{
21305ffd83dbSDimitry Andric    return __o->wait(__v, __m);
21315ffd83dbSDimitry Andric}
21325ffd83dbSDimitry Andric
21335ffd83dbSDimitry Andric// atomic_notify_one
21345ffd83dbSDimitry Andric
21355ffd83dbSDimitry Andrictemplate <class _Tp>
21365ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
21375ffd83dbSDimitry Andricvoid atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT
21385ffd83dbSDimitry Andric{
21395ffd83dbSDimitry Andric    __o->notify_one();
21405ffd83dbSDimitry Andric}
21415ffd83dbSDimitry Andrictemplate <class _Tp>
21425ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
21435ffd83dbSDimitry Andricvoid atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT
21445ffd83dbSDimitry Andric{
21455ffd83dbSDimitry Andric    __o->notify_one();
21465ffd83dbSDimitry Andric}
21475ffd83dbSDimitry Andric
21485ffd83dbSDimitry Andric// atomic_notify_one
21495ffd83dbSDimitry Andric
21505ffd83dbSDimitry Andrictemplate <class _Tp>
21515ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
21525ffd83dbSDimitry Andricvoid atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT
21535ffd83dbSDimitry Andric{
21545ffd83dbSDimitry Andric    __o->notify_all();
21555ffd83dbSDimitry Andric}
21565ffd83dbSDimitry Andrictemplate <class _Tp>
21575ffd83dbSDimitry Andric_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
21585ffd83dbSDimitry Andricvoid atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT
21595ffd83dbSDimitry Andric{
21605ffd83dbSDimitry Andric    __o->notify_all();
21615ffd83dbSDimitry Andric}
21625ffd83dbSDimitry Andric
21630b57cec5SDimitry Andric// atomic_fetch_add
21640b57cec5SDimitry Andric
21650b57cec5SDimitry Andrictemplate <class _Tp>
21660b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
21670b57cec5SDimitry Andric_Tp
2168e8d8bef9SDimitry Andricatomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT
21690b57cec5SDimitry Andric{
21700b57cec5SDimitry Andric    return __o->fetch_add(__op);
21710b57cec5SDimitry Andric}
21720b57cec5SDimitry Andric
21730b57cec5SDimitry Andrictemplate <class _Tp>
21740b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
21750b57cec5SDimitry Andric_Tp
2176e8d8bef9SDimitry Andricatomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT
21770b57cec5SDimitry Andric{
21780b57cec5SDimitry Andric    return __o->fetch_add(__op);
21790b57cec5SDimitry Andric}
21800b57cec5SDimitry Andric
21810b57cec5SDimitry Andric// atomic_fetch_add_explicit
21820b57cec5SDimitry Andric
21830b57cec5SDimitry Andrictemplate <class _Tp>
21840b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
2185*349cc55cSDimitry Andric_Tp atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT
21860b57cec5SDimitry Andric{
21870b57cec5SDimitry Andric    return __o->fetch_add(__op, __m);
21880b57cec5SDimitry Andric}
21890b57cec5SDimitry Andric
21900b57cec5SDimitry Andrictemplate <class _Tp>
21910b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
2192*349cc55cSDimitry Andric_Tp atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT
21930b57cec5SDimitry Andric{
21940b57cec5SDimitry Andric    return __o->fetch_add(__op, __m);
21950b57cec5SDimitry Andric}
21960b57cec5SDimitry Andric
21970b57cec5SDimitry Andric// atomic_fetch_sub
21980b57cec5SDimitry Andric
21990b57cec5SDimitry Andrictemplate <class _Tp>
22000b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
2201*349cc55cSDimitry Andric_Tp atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT
22020b57cec5SDimitry Andric{
22030b57cec5SDimitry Andric    return __o->fetch_sub(__op);
22040b57cec5SDimitry Andric}
22050b57cec5SDimitry Andric
22060b57cec5SDimitry Andrictemplate <class _Tp>
22070b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
2208*349cc55cSDimitry Andric_Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT
22090b57cec5SDimitry Andric{
22100b57cec5SDimitry Andric    return __o->fetch_sub(__op);
22110b57cec5SDimitry Andric}
22120b57cec5SDimitry Andric
22130b57cec5SDimitry Andric// atomic_fetch_sub_explicit
22140b57cec5SDimitry Andric
22150b57cec5SDimitry Andrictemplate <class _Tp>
22160b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
2217*349cc55cSDimitry Andric_Tp atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT
22180b57cec5SDimitry Andric{
22190b57cec5SDimitry Andric    return __o->fetch_sub(__op, __m);
22200b57cec5SDimitry Andric}
22210b57cec5SDimitry Andric
22220b57cec5SDimitry Andrictemplate <class _Tp>
22230b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
2224*349cc55cSDimitry Andric_Tp atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT
22250b57cec5SDimitry Andric{
22260b57cec5SDimitry Andric    return __o->fetch_sub(__op, __m);
22270b57cec5SDimitry Andric}
22280b57cec5SDimitry Andric
22290b57cec5SDimitry Andric// atomic_fetch_and
22300b57cec5SDimitry Andric
22310b57cec5SDimitry Andrictemplate <class _Tp>
22320b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22330b57cec5SDimitry Andrictypename enable_if
22340b57cec5SDimitry Andric<
22350b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22360b57cec5SDimitry Andric    _Tp
22370b57cec5SDimitry Andric>::type
2238e8d8bef9SDimitry Andricatomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
22390b57cec5SDimitry Andric{
22400b57cec5SDimitry Andric    return __o->fetch_and(__op);
22410b57cec5SDimitry Andric}
22420b57cec5SDimitry Andric
22430b57cec5SDimitry Andrictemplate <class _Tp>
22440b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22450b57cec5SDimitry Andrictypename enable_if
22460b57cec5SDimitry Andric<
22470b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22480b57cec5SDimitry Andric    _Tp
22490b57cec5SDimitry Andric>::type
2250e8d8bef9SDimitry Andricatomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
22510b57cec5SDimitry Andric{
22520b57cec5SDimitry Andric    return __o->fetch_and(__op);
22530b57cec5SDimitry Andric}
22540b57cec5SDimitry Andric
22550b57cec5SDimitry Andric// atomic_fetch_and_explicit
22560b57cec5SDimitry Andric
22570b57cec5SDimitry Andrictemplate <class _Tp>
22580b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22590b57cec5SDimitry Andrictypename enable_if
22600b57cec5SDimitry Andric<
22610b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22620b57cec5SDimitry Andric    _Tp
22630b57cec5SDimitry Andric>::type
2264e8d8bef9SDimitry Andricatomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
22650b57cec5SDimitry Andric{
22660b57cec5SDimitry Andric    return __o->fetch_and(__op, __m);
22670b57cec5SDimitry Andric}
22680b57cec5SDimitry Andric
22690b57cec5SDimitry Andrictemplate <class _Tp>
22700b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22710b57cec5SDimitry Andrictypename enable_if
22720b57cec5SDimitry Andric<
22730b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22740b57cec5SDimitry Andric    _Tp
22750b57cec5SDimitry Andric>::type
2276e8d8bef9SDimitry Andricatomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
22770b57cec5SDimitry Andric{
22780b57cec5SDimitry Andric    return __o->fetch_and(__op, __m);
22790b57cec5SDimitry Andric}
22800b57cec5SDimitry Andric
22810b57cec5SDimitry Andric// atomic_fetch_or
22820b57cec5SDimitry Andric
22830b57cec5SDimitry Andrictemplate <class _Tp>
22840b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22850b57cec5SDimitry Andrictypename enable_if
22860b57cec5SDimitry Andric<
22870b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
22880b57cec5SDimitry Andric    _Tp
22890b57cec5SDimitry Andric>::type
2290e8d8bef9SDimitry Andricatomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
22910b57cec5SDimitry Andric{
22920b57cec5SDimitry Andric    return __o->fetch_or(__op);
22930b57cec5SDimitry Andric}
22940b57cec5SDimitry Andric
22950b57cec5SDimitry Andrictemplate <class _Tp>
22960b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
22970b57cec5SDimitry Andrictypename enable_if
22980b57cec5SDimitry Andric<
22990b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23000b57cec5SDimitry Andric    _Tp
23010b57cec5SDimitry Andric>::type
2302e8d8bef9SDimitry Andricatomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
23030b57cec5SDimitry Andric{
23040b57cec5SDimitry Andric    return __o->fetch_or(__op);
23050b57cec5SDimitry Andric}
23060b57cec5SDimitry Andric
23070b57cec5SDimitry Andric// atomic_fetch_or_explicit
23080b57cec5SDimitry Andric
23090b57cec5SDimitry Andrictemplate <class _Tp>
23100b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23110b57cec5SDimitry Andrictypename enable_if
23120b57cec5SDimitry Andric<
23130b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23140b57cec5SDimitry Andric    _Tp
23150b57cec5SDimitry Andric>::type
2316e8d8bef9SDimitry Andricatomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
23170b57cec5SDimitry Andric{
23180b57cec5SDimitry Andric    return __o->fetch_or(__op, __m);
23190b57cec5SDimitry Andric}
23200b57cec5SDimitry Andric
23210b57cec5SDimitry Andrictemplate <class _Tp>
23220b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23230b57cec5SDimitry Andrictypename enable_if
23240b57cec5SDimitry Andric<
23250b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23260b57cec5SDimitry Andric    _Tp
23270b57cec5SDimitry Andric>::type
2328e8d8bef9SDimitry Andricatomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
23290b57cec5SDimitry Andric{
23300b57cec5SDimitry Andric    return __o->fetch_or(__op, __m);
23310b57cec5SDimitry Andric}
23320b57cec5SDimitry Andric
23330b57cec5SDimitry Andric// atomic_fetch_xor
23340b57cec5SDimitry Andric
23350b57cec5SDimitry Andrictemplate <class _Tp>
23360b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23370b57cec5SDimitry Andrictypename enable_if
23380b57cec5SDimitry Andric<
23390b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23400b57cec5SDimitry Andric    _Tp
23410b57cec5SDimitry Andric>::type
2342e8d8bef9SDimitry Andricatomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
23430b57cec5SDimitry Andric{
23440b57cec5SDimitry Andric    return __o->fetch_xor(__op);
23450b57cec5SDimitry Andric}
23460b57cec5SDimitry Andric
23470b57cec5SDimitry Andrictemplate <class _Tp>
23480b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23490b57cec5SDimitry Andrictypename enable_if
23500b57cec5SDimitry Andric<
23510b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23520b57cec5SDimitry Andric    _Tp
23530b57cec5SDimitry Andric>::type
2354e8d8bef9SDimitry Andricatomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT
23550b57cec5SDimitry Andric{
23560b57cec5SDimitry Andric    return __o->fetch_xor(__op);
23570b57cec5SDimitry Andric}
23580b57cec5SDimitry Andric
23590b57cec5SDimitry Andric// atomic_fetch_xor_explicit
23600b57cec5SDimitry Andric
23610b57cec5SDimitry Andrictemplate <class _Tp>
23620b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23630b57cec5SDimitry Andrictypename enable_if
23640b57cec5SDimitry Andric<
23650b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23660b57cec5SDimitry Andric    _Tp
23670b57cec5SDimitry Andric>::type
2368e8d8bef9SDimitry Andricatomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
23690b57cec5SDimitry Andric{
23700b57cec5SDimitry Andric    return __o->fetch_xor(__op, __m);
23710b57cec5SDimitry Andric}
23720b57cec5SDimitry Andric
23730b57cec5SDimitry Andrictemplate <class _Tp>
23740b57cec5SDimitry Andric_LIBCPP_INLINE_VISIBILITY
23750b57cec5SDimitry Andrictypename enable_if
23760b57cec5SDimitry Andric<
23770b57cec5SDimitry Andric    is_integral<_Tp>::value && !is_same<_Tp, bool>::value,
23780b57cec5SDimitry Andric    _Tp
23790b57cec5SDimitry Andric>::type
2380e8d8bef9SDimitry Andricatomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT
23810b57cec5SDimitry Andric{
23820b57cec5SDimitry Andric    return __o->fetch_xor(__op, __m);
23830b57cec5SDimitry Andric}
23840b57cec5SDimitry Andric
23850b57cec5SDimitry Andric// flag type and operations
23860b57cec5SDimitry Andric
23870b57cec5SDimitry Andrictypedef struct atomic_flag
23880b57cec5SDimitry Andric{
23890b57cec5SDimitry Andric    __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_;
23900b57cec5SDimitry Andric
23910b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
23925ffd83dbSDimitry Andric    bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
23935ffd83dbSDimitry Andric        {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);}
23945ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
23955ffd83dbSDimitry Andric    bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
23965ffd83dbSDimitry Andric        {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);}
23975ffd83dbSDimitry Andric
23985ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
23990b57cec5SDimitry Andric    bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
24000b57cec5SDimitry Andric        {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);}
24010b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
24020b57cec5SDimitry Andric    bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT
24030b57cec5SDimitry Andric        {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);}
24040b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
24050b57cec5SDimitry Andric    void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
24060b57cec5SDimitry Andric        {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);}
24070b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
24080b57cec5SDimitry Andric    void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT
24090b57cec5SDimitry Andric        {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);}
24100b57cec5SDimitry Andric
24115ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
24125ffd83dbSDimitry Andric    void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
24135ffd83dbSDimitry Andric        {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);}
24145ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
24155ffd83dbSDimitry Andric    void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT
24165ffd83dbSDimitry Andric        {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);}
24175ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
24185ffd83dbSDimitry Andric    void notify_one() volatile _NOEXCEPT
24195ffd83dbSDimitry Andric        {__cxx_atomic_notify_one(&__a_);}
24205ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
24215ffd83dbSDimitry Andric    void notify_one() _NOEXCEPT
24225ffd83dbSDimitry Andric        {__cxx_atomic_notify_one(&__a_);}
24235ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
24245ffd83dbSDimitry Andric    void notify_all() volatile _NOEXCEPT
24255ffd83dbSDimitry Andric        {__cxx_atomic_notify_all(&__a_);}
24265ffd83dbSDimitry Andric    _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
24275ffd83dbSDimitry Andric    void notify_all() _NOEXCEPT
24285ffd83dbSDimitry Andric        {__cxx_atomic_notify_all(&__a_);}
24295ffd83dbSDimitry Andric
2430fe6060f1SDimitry Andric#if _LIBCPP_STD_VER > 17
2431fe6060f1SDimitry Andric    _LIBCPP_INLINE_VISIBILITY constexpr
2432fe6060f1SDimitry Andric    atomic_flag() _NOEXCEPT : __a_(false) {}
2433fe6060f1SDimitry Andric#else
24340b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY
24350b57cec5SDimitry Andric    atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT
2436fe6060f1SDimitry Andric#endif
24370b57cec5SDimitry Andric
24380b57cec5SDimitry Andric    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
24390b57cec5SDimitry Andric    atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION
24400b57cec5SDimitry Andric
24410b57cec5SDimitry Andric#ifndef _LIBCPP_CXX03_LANG
24420b57cec5SDimitry Andric    atomic_flag(const atomic_flag&) = delete;
24430b57cec5SDimitry Andric    atomic_flag& operator=(const atomic_flag&) = delete;
24440b57cec5SDimitry Andric    atomic_flag& operator=(const atomic_flag&) volatile = delete;
24450b57cec5SDimitry Andric#else
24460b57cec5SDimitry Andricprivate:
24475ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
24480b57cec5SDimitry Andric    atomic_flag(const atomic_flag&);
24495ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
24500b57cec5SDimitry Andric    atomic_flag& operator=(const atomic_flag&);
24515ffd83dbSDimitry Andric    _LIBCPP_INLINE_VISIBILITY
24520b57cec5SDimitry Andric    atomic_flag& operator=(const atomic_flag&) volatile;
24530b57cec5SDimitry Andric#endif
24540b57cec5SDimitry Andric} atomic_flag;
24550b57cec5SDimitry Andric
24565ffd83dbSDimitry Andric
24575ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
24585ffd83dbSDimitry Andricbool
24595ffd83dbSDimitry Andricatomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT
24605ffd83dbSDimitry Andric{
24615ffd83dbSDimitry Andric    return __o->test();
24625ffd83dbSDimitry Andric}
24635ffd83dbSDimitry Andric
24645ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
24655ffd83dbSDimitry Andricbool
24665ffd83dbSDimitry Andricatomic_flag_test(const atomic_flag* __o) _NOEXCEPT
24675ffd83dbSDimitry Andric{
24685ffd83dbSDimitry Andric    return __o->test();
24695ffd83dbSDimitry Andric}
24705ffd83dbSDimitry Andric
24715ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
24725ffd83dbSDimitry Andricbool
24735ffd83dbSDimitry Andricatomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
24745ffd83dbSDimitry Andric{
24755ffd83dbSDimitry Andric    return __o->test(__m);
24765ffd83dbSDimitry Andric}
24775ffd83dbSDimitry Andric
24785ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
24795ffd83dbSDimitry Andricbool
24805ffd83dbSDimitry Andricatomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT
24815ffd83dbSDimitry Andric{
24825ffd83dbSDimitry Andric    return __o->test(__m);
24835ffd83dbSDimitry Andric}
24845ffd83dbSDimitry Andric
24850b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
24860b57cec5SDimitry Andricbool
24870b57cec5SDimitry Andricatomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT
24880b57cec5SDimitry Andric{
24890b57cec5SDimitry Andric    return __o->test_and_set();
24900b57cec5SDimitry Andric}
24910b57cec5SDimitry Andric
24920b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
24930b57cec5SDimitry Andricbool
24940b57cec5SDimitry Andricatomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT
24950b57cec5SDimitry Andric{
24960b57cec5SDimitry Andric    return __o->test_and_set();
24970b57cec5SDimitry Andric}
24980b57cec5SDimitry Andric
24990b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
25000b57cec5SDimitry Andricbool
25010b57cec5SDimitry Andricatomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
25020b57cec5SDimitry Andric{
25030b57cec5SDimitry Andric    return __o->test_and_set(__m);
25040b57cec5SDimitry Andric}
25050b57cec5SDimitry Andric
25060b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
25070b57cec5SDimitry Andricbool
25080b57cec5SDimitry Andricatomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT
25090b57cec5SDimitry Andric{
25100b57cec5SDimitry Andric    return __o->test_and_set(__m);
25110b57cec5SDimitry Andric}
25120b57cec5SDimitry Andric
25130b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
25140b57cec5SDimitry Andricvoid
25150b57cec5SDimitry Andricatomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT
25160b57cec5SDimitry Andric{
25170b57cec5SDimitry Andric    __o->clear();
25180b57cec5SDimitry Andric}
25190b57cec5SDimitry Andric
25200b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
25210b57cec5SDimitry Andricvoid
25220b57cec5SDimitry Andricatomic_flag_clear(atomic_flag* __o) _NOEXCEPT
25230b57cec5SDimitry Andric{
25240b57cec5SDimitry Andric    __o->clear();
25250b57cec5SDimitry Andric}
25260b57cec5SDimitry Andric
25270b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
25280b57cec5SDimitry Andricvoid
25290b57cec5SDimitry Andricatomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT
25300b57cec5SDimitry Andric{
25310b57cec5SDimitry Andric    __o->clear(__m);
25320b57cec5SDimitry Andric}
25330b57cec5SDimitry Andric
25340b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
25350b57cec5SDimitry Andricvoid
25360b57cec5SDimitry Andricatomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT
25370b57cec5SDimitry Andric{
25380b57cec5SDimitry Andric    __o->clear(__m);
25390b57cec5SDimitry Andric}
25400b57cec5SDimitry Andric
25415ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
25425ffd83dbSDimitry Andricvoid
25435ffd83dbSDimitry Andricatomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT
25445ffd83dbSDimitry Andric{
25455ffd83dbSDimitry Andric    __o->wait(__v);
25465ffd83dbSDimitry Andric}
25475ffd83dbSDimitry Andric
25485ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
25495ffd83dbSDimitry Andricvoid
25505ffd83dbSDimitry Andricatomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT
25515ffd83dbSDimitry Andric{
25525ffd83dbSDimitry Andric    __o->wait(__v);
25535ffd83dbSDimitry Andric}
25545ffd83dbSDimitry Andric
25555ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
25565ffd83dbSDimitry Andricvoid
25575ffd83dbSDimitry Andricatomic_flag_wait_explicit(const volatile atomic_flag* __o,
25585ffd83dbSDimitry Andric                          bool __v, memory_order __m) _NOEXCEPT
25595ffd83dbSDimitry Andric{
25605ffd83dbSDimitry Andric    __o->wait(__v, __m);
25615ffd83dbSDimitry Andric}
25625ffd83dbSDimitry Andric
25635ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
25645ffd83dbSDimitry Andricvoid
25655ffd83dbSDimitry Andricatomic_flag_wait_explicit(const atomic_flag* __o,
25665ffd83dbSDimitry Andric                          bool __v, memory_order __m) _NOEXCEPT
25675ffd83dbSDimitry Andric{
25685ffd83dbSDimitry Andric    __o->wait(__v, __m);
25695ffd83dbSDimitry Andric}
25705ffd83dbSDimitry Andric
25715ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
25725ffd83dbSDimitry Andricvoid
25735ffd83dbSDimitry Andricatomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT
25745ffd83dbSDimitry Andric{
25755ffd83dbSDimitry Andric    __o->notify_one();
25765ffd83dbSDimitry Andric}
25775ffd83dbSDimitry Andric
25785ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
25795ffd83dbSDimitry Andricvoid
25805ffd83dbSDimitry Andricatomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT
25815ffd83dbSDimitry Andric{
25825ffd83dbSDimitry Andric    __o->notify_one();
25835ffd83dbSDimitry Andric}
25845ffd83dbSDimitry Andric
25855ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
25865ffd83dbSDimitry Andricvoid
25875ffd83dbSDimitry Andricatomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT
25885ffd83dbSDimitry Andric{
25895ffd83dbSDimitry Andric    __o->notify_all();
25905ffd83dbSDimitry Andric}
25915ffd83dbSDimitry Andric
25925ffd83dbSDimitry Andricinline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC
25935ffd83dbSDimitry Andricvoid
25945ffd83dbSDimitry Andricatomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT
25955ffd83dbSDimitry Andric{
25965ffd83dbSDimitry Andric    __o->notify_all();
25975ffd83dbSDimitry Andric}
25985ffd83dbSDimitry Andric
25990b57cec5SDimitry Andric// fences
26000b57cec5SDimitry Andric
26010b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
26020b57cec5SDimitry Andricvoid
26030b57cec5SDimitry Andricatomic_thread_fence(memory_order __m) _NOEXCEPT
26040b57cec5SDimitry Andric{
26050b57cec5SDimitry Andric    __cxx_atomic_thread_fence(__m);
26060b57cec5SDimitry Andric}
26070b57cec5SDimitry Andric
26080b57cec5SDimitry Andricinline _LIBCPP_INLINE_VISIBILITY
26090b57cec5SDimitry Andricvoid
26100b57cec5SDimitry Andricatomic_signal_fence(memory_order __m) _NOEXCEPT
26110b57cec5SDimitry Andric{
26120b57cec5SDimitry Andric    __cxx_atomic_signal_fence(__m);
26130b57cec5SDimitry Andric}
26140b57cec5SDimitry Andric
26150b57cec5SDimitry Andric// Atomics for standard typedef types
26160b57cec5SDimitry Andric
26170b57cec5SDimitry Andrictypedef atomic<bool>               atomic_bool;
26180b57cec5SDimitry Andrictypedef atomic<char>               atomic_char;
26190b57cec5SDimitry Andrictypedef atomic<signed char>        atomic_schar;
26200b57cec5SDimitry Andrictypedef atomic<unsigned char>      atomic_uchar;
26210b57cec5SDimitry Andrictypedef atomic<short>              atomic_short;
26220b57cec5SDimitry Andrictypedef atomic<unsigned short>     atomic_ushort;
26230b57cec5SDimitry Andrictypedef atomic<int>                atomic_int;
26240b57cec5SDimitry Andrictypedef atomic<unsigned int>       atomic_uint;
26250b57cec5SDimitry Andrictypedef atomic<long>               atomic_long;
26260b57cec5SDimitry Andrictypedef atomic<unsigned long>      atomic_ulong;
26270b57cec5SDimitry Andrictypedef atomic<long long>          atomic_llong;
26280b57cec5SDimitry Andrictypedef atomic<unsigned long long> atomic_ullong;
2629fe6060f1SDimitry Andric#ifndef _LIBCPP_HAS_NO_CHAR8_T
2630e8d8bef9SDimitry Andrictypedef atomic<char8_t>            atomic_char8_t;
2631e8d8bef9SDimitry Andric#endif
26320b57cec5SDimitry Andrictypedef atomic<char16_t>           atomic_char16_t;
26330b57cec5SDimitry Andrictypedef atomic<char32_t>           atomic_char32_t;
2634*349cc55cSDimitry Andric#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
26350b57cec5SDimitry Andrictypedef atomic<wchar_t>            atomic_wchar_t;
2636*349cc55cSDimitry Andric#endif
26370b57cec5SDimitry Andric
26380b57cec5SDimitry Andrictypedef atomic<int_least8_t>   atomic_int_least8_t;
26390b57cec5SDimitry Andrictypedef atomic<uint_least8_t>  atomic_uint_least8_t;
26400b57cec5SDimitry Andrictypedef atomic<int_least16_t>  atomic_int_least16_t;
26410b57cec5SDimitry Andrictypedef atomic<uint_least16_t> atomic_uint_least16_t;
26420b57cec5SDimitry Andrictypedef atomic<int_least32_t>  atomic_int_least32_t;
26430b57cec5SDimitry Andrictypedef atomic<uint_least32_t> atomic_uint_least32_t;
26440b57cec5SDimitry Andrictypedef atomic<int_least64_t>  atomic_int_least64_t;
26450b57cec5SDimitry Andrictypedef atomic<uint_least64_t> atomic_uint_least64_t;
26460b57cec5SDimitry Andric
26470b57cec5SDimitry Andrictypedef atomic<int_fast8_t>   atomic_int_fast8_t;
26480b57cec5SDimitry Andrictypedef atomic<uint_fast8_t>  atomic_uint_fast8_t;
26490b57cec5SDimitry Andrictypedef atomic<int_fast16_t>  atomic_int_fast16_t;
26500b57cec5SDimitry Andrictypedef atomic<uint_fast16_t> atomic_uint_fast16_t;
26510b57cec5SDimitry Andrictypedef atomic<int_fast32_t>  atomic_int_fast32_t;
26520b57cec5SDimitry Andrictypedef atomic<uint_fast32_t> atomic_uint_fast32_t;
26530b57cec5SDimitry Andrictypedef atomic<int_fast64_t>  atomic_int_fast64_t;
26540b57cec5SDimitry Andrictypedef atomic<uint_fast64_t> atomic_uint_fast64_t;
26550b57cec5SDimitry Andric
26560b57cec5SDimitry Andrictypedef atomic< int8_t>  atomic_int8_t;
26570b57cec5SDimitry Andrictypedef atomic<uint8_t>  atomic_uint8_t;
26580b57cec5SDimitry Andrictypedef atomic< int16_t> atomic_int16_t;
26590b57cec5SDimitry Andrictypedef atomic<uint16_t> atomic_uint16_t;
26600b57cec5SDimitry Andrictypedef atomic< int32_t> atomic_int32_t;
26610b57cec5SDimitry Andrictypedef atomic<uint32_t> atomic_uint32_t;
26620b57cec5SDimitry Andrictypedef atomic< int64_t> atomic_int64_t;
26630b57cec5SDimitry Andrictypedef atomic<uint64_t> atomic_uint64_t;
26640b57cec5SDimitry Andric
26650b57cec5SDimitry Andrictypedef atomic<intptr_t>  atomic_intptr_t;
26660b57cec5SDimitry Andrictypedef atomic<uintptr_t> atomic_uintptr_t;
26670b57cec5SDimitry Andrictypedef atomic<size_t>    atomic_size_t;
26680b57cec5SDimitry Andrictypedef atomic<ptrdiff_t> atomic_ptrdiff_t;
26690b57cec5SDimitry Andrictypedef atomic<intmax_t>  atomic_intmax_t;
26700b57cec5SDimitry Andrictypedef atomic<uintmax_t> atomic_uintmax_t;
26710b57cec5SDimitry Andric
26725ffd83dbSDimitry Andric// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type
26735ffd83dbSDimitry Andric
26745ffd83dbSDimitry Andric#ifdef __cpp_lib_atomic_is_always_lock_free
26755ffd83dbSDimitry Andric# define _LIBCPP_CONTENTION_LOCK_FREE __atomic_always_lock_free(sizeof(__cxx_contention_t), 0)
26765ffd83dbSDimitry Andric#else
26775ffd83dbSDimitry Andric# define _LIBCPP_CONTENTION_LOCK_FREE false
26785ffd83dbSDimitry Andric#endif
26795ffd83dbSDimitry Andric
26805ffd83dbSDimitry Andric#if ATOMIC_LLONG_LOCK_FREE == 2
26815ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long>::type          __libcpp_signed_lock_free;
26825ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long>::type __libcpp_unsigned_lock_free;
26835ffd83dbSDimitry Andric#elif ATOMIC_INT_LOCK_FREE == 2
26845ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int>::type                __libcpp_signed_lock_free;
26855ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int>::type       __libcpp_unsigned_lock_free;
26865ffd83dbSDimitry Andric#elif ATOMIC_SHORT_LOCK_FREE == 2
26875ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short>::type              __libcpp_signed_lock_free;
26885ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short>::type     __libcpp_unsigned_lock_free;
26895ffd83dbSDimitry Andric#elif ATOMIC_CHAR_LOCK_FREE == 2
26905ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char>::type               __libcpp_signed_lock_free;
26915ffd83dbSDimitry Andrictypedef conditional<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char>::type      __libcpp_unsigned_lock_free;
26925ffd83dbSDimitry Andric#else
26935ffd83dbSDimitry Andric    // No signed/unsigned lock-free types
26945ffd83dbSDimitry Andric#endif
26955ffd83dbSDimitry Andric
26965ffd83dbSDimitry Andrictypedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free;
26975ffd83dbSDimitry Andrictypedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free;
26985ffd83dbSDimitry Andric
26990b57cec5SDimitry Andric#define ATOMIC_FLAG_INIT {false}
27000b57cec5SDimitry Andric#define ATOMIC_VAR_INIT(__v) {__v}
27010b57cec5SDimitry Andric
27020b57cec5SDimitry Andric_LIBCPP_END_NAMESPACE_STD
27030b57cec5SDimitry Andric
27040b57cec5SDimitry Andric#endif // _LIBCPP_ATOMIC
2705