1// -*- C++ -*- 2//===----------------------------------------------------------------------===// 3// 4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5// See https://llvm.org/LICENSE.txt for license information. 6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7// 8//===----------------------------------------------------------------------===// 9 10#ifndef _LIBCPP_ATOMIC 11#define _LIBCPP_ATOMIC 12 13/* 14 atomic synopsis 15 16namespace std 17{ 18 19// feature test macro [version.syn] 20 21#define __cpp_lib_atomic_is_always_lock_free 22#define __cpp_lib_atomic_flag_test 23#define __cpp_lib_atomic_lock_free_type_aliases 24#define __cpp_lib_atomic_wait 25 26 // order and consistency 27 28 enum memory_order: unspecified // enum class in C++20 29 { 30 relaxed, 31 consume, // load-consume 32 acquire, // load-acquire 33 release, // store-release 34 acq_rel, // store-release load-acquire 35 seq_cst // store-release load-acquire 36 }; 37 38 inline constexpr auto memory_order_relaxed = memory_order::relaxed; 39 inline constexpr auto memory_order_consume = memory_order::consume; 40 inline constexpr auto memory_order_acquire = memory_order::acquire; 41 inline constexpr auto memory_order_release = memory_order::release; 42 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 43 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 44 45template <class T> T kill_dependency(T y) noexcept; 46 47// lock-free property 48 49#define ATOMIC_BOOL_LOCK_FREE unspecified 50#define ATOMIC_CHAR_LOCK_FREE unspecified 51#define ATOMIC_CHAR8_T_LOCK_FREE unspecified // C++20 52#define ATOMIC_CHAR16_T_LOCK_FREE unspecified 53#define ATOMIC_CHAR32_T_LOCK_FREE unspecified 54#define ATOMIC_WCHAR_T_LOCK_FREE unspecified 55#define ATOMIC_SHORT_LOCK_FREE unspecified 56#define ATOMIC_INT_LOCK_FREE unspecified 57#define ATOMIC_LONG_LOCK_FREE unspecified 58#define ATOMIC_LLONG_LOCK_FREE unspecified 59#define ATOMIC_POINTER_LOCK_FREE unspecified 60 61template <class T> 62struct atomic 63{ 64 using value_type = T; 65 66 static constexpr bool is_always_lock_free; 67 bool is_lock_free() const volatile noexcept; 68 bool is_lock_free() const noexcept; 69 70 atomic() noexcept = default; // until C++20 71 constexpr atomic() noexcept(is_nothrow_default_constructible_v<T>); // since C++20 72 constexpr atomic(T desr) noexcept; 73 atomic(const atomic&) = delete; 74 atomic& operator=(const atomic&) = delete; 75 atomic& operator=(const atomic&) volatile = delete; 76 77 T load(memory_order m = memory_order_seq_cst) const volatile noexcept; 78 T load(memory_order m = memory_order_seq_cst) const noexcept; 79 operator T() const volatile noexcept; 80 operator T() const noexcept; 81 void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 82 void store(T desr, memory_order m = memory_order_seq_cst) noexcept; 83 T operator=(T) volatile noexcept; 84 T operator=(T) noexcept; 85 86 T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept; 87 T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept; 88 bool compare_exchange_weak(T& expc, T desr, 89 memory_order s, memory_order f) volatile noexcept; 90 bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept; 91 bool compare_exchange_strong(T& expc, T desr, 92 memory_order s, memory_order f) volatile noexcept; 93 bool compare_exchange_strong(T& expc, T desr, 94 memory_order s, memory_order f) noexcept; 95 bool compare_exchange_weak(T& expc, T desr, 96 memory_order m = memory_order_seq_cst) volatile noexcept; 97 bool compare_exchange_weak(T& expc, T desr, 98 memory_order m = memory_order_seq_cst) noexcept; 99 bool compare_exchange_strong(T& expc, T desr, 100 memory_order m = memory_order_seq_cst) volatile noexcept; 101 bool compare_exchange_strong(T& expc, T desr, 102 memory_order m = memory_order_seq_cst) noexcept; 103 104 void wait(T, memory_order = memory_order::seq_cst) const volatile noexcept; 105 void wait(T, memory_order = memory_order::seq_cst) const noexcept; 106 void notify_one() volatile noexcept; 107 void notify_one() noexcept; 108 void notify_all() volatile noexcept; 109 void notify_all() noexcept; 110}; 111 112template <> 113struct atomic<integral> 114{ 115 using value_type = integral; 116 using difference_type = value_type; 117 118 static constexpr bool is_always_lock_free; 119 bool is_lock_free() const volatile noexcept; 120 bool is_lock_free() const noexcept; 121 122 atomic() noexcept = default; 123 constexpr atomic(integral desr) noexcept; 124 atomic(const atomic&) = delete; 125 atomic& operator=(const atomic&) = delete; 126 atomic& operator=(const atomic&) volatile = delete; 127 128 integral load(memory_order m = memory_order_seq_cst) const volatile noexcept; 129 integral load(memory_order m = memory_order_seq_cst) const noexcept; 130 operator integral() const volatile noexcept; 131 operator integral() const noexcept; 132 void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept; 133 void store(integral desr, memory_order m = memory_order_seq_cst) noexcept; 134 integral operator=(integral desr) volatile noexcept; 135 integral operator=(integral desr) noexcept; 136 137 integral exchange(integral desr, 138 memory_order m = memory_order_seq_cst) volatile noexcept; 139 integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept; 140 bool compare_exchange_weak(integral& expc, integral desr, 141 memory_order s, memory_order f) volatile noexcept; 142 bool compare_exchange_weak(integral& expc, integral desr, 143 memory_order s, memory_order f) noexcept; 144 bool compare_exchange_strong(integral& expc, integral desr, 145 memory_order s, memory_order f) volatile noexcept; 146 bool compare_exchange_strong(integral& expc, integral desr, 147 memory_order s, memory_order f) noexcept; 148 bool compare_exchange_weak(integral& expc, integral desr, 149 memory_order m = memory_order_seq_cst) volatile noexcept; 150 bool compare_exchange_weak(integral& expc, integral desr, 151 memory_order m = memory_order_seq_cst) noexcept; 152 bool compare_exchange_strong(integral& expc, integral desr, 153 memory_order m = memory_order_seq_cst) volatile noexcept; 154 bool compare_exchange_strong(integral& expc, integral desr, 155 memory_order m = memory_order_seq_cst) noexcept; 156 157 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 158 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept; 159 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 160 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept; 161 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 162 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept; 163 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 164 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept; 165 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept; 166 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept; 167 168 integral operator++(int) volatile noexcept; 169 integral operator++(int) noexcept; 170 integral operator--(int) volatile noexcept; 171 integral operator--(int) noexcept; 172 integral operator++() volatile noexcept; 173 integral operator++() noexcept; 174 integral operator--() volatile noexcept; 175 integral operator--() noexcept; 176 integral operator+=(integral op) volatile noexcept; 177 integral operator+=(integral op) noexcept; 178 integral operator-=(integral op) volatile noexcept; 179 integral operator-=(integral op) noexcept; 180 integral operator&=(integral op) volatile noexcept; 181 integral operator&=(integral op) noexcept; 182 integral operator|=(integral op) volatile noexcept; 183 integral operator|=(integral op) noexcept; 184 integral operator^=(integral op) volatile noexcept; 185 integral operator^=(integral op) noexcept; 186 187 void wait(integral, memory_order = memory_order::seq_cst) const volatile noexcept; 188 void wait(integral, memory_order = memory_order::seq_cst) const noexcept; 189 void notify_one() volatile noexcept; 190 void notify_one() noexcept; 191 void notify_all() volatile noexcept; 192 void notify_all() noexcept; 193}; 194 195template <class T> 196struct atomic<T*> 197{ 198 using value_type = T*; 199 using difference_type = ptrdiff_t; 200 201 static constexpr bool is_always_lock_free; 202 bool is_lock_free() const volatile noexcept; 203 bool is_lock_free() const noexcept; 204 205 atomic() noexcept = default; // until C++20 206 constexpr atomic() noexcept; // since C++20 207 constexpr atomic(T* desr) noexcept; 208 atomic(const atomic&) = delete; 209 atomic& operator=(const atomic&) = delete; 210 atomic& operator=(const atomic&) volatile = delete; 211 212 T* load(memory_order m = memory_order_seq_cst) const volatile noexcept; 213 T* load(memory_order m = memory_order_seq_cst) const noexcept; 214 operator T*() const volatile noexcept; 215 operator T*() const noexcept; 216 void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 217 void store(T* desr, memory_order m = memory_order_seq_cst) noexcept; 218 T* operator=(T*) volatile noexcept; 219 T* operator=(T*) noexcept; 220 221 T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept; 222 T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept; 223 bool compare_exchange_weak(T*& expc, T* desr, 224 memory_order s, memory_order f) volatile noexcept; 225 bool compare_exchange_weak(T*& expc, T* desr, 226 memory_order s, memory_order f) noexcept; 227 bool compare_exchange_strong(T*& expc, T* desr, 228 memory_order s, memory_order f) volatile noexcept; 229 bool compare_exchange_strong(T*& expc, T* desr, 230 memory_order s, memory_order f) noexcept; 231 bool compare_exchange_weak(T*& expc, T* desr, 232 memory_order m = memory_order_seq_cst) volatile noexcept; 233 bool compare_exchange_weak(T*& expc, T* desr, 234 memory_order m = memory_order_seq_cst) noexcept; 235 bool compare_exchange_strong(T*& expc, T* desr, 236 memory_order m = memory_order_seq_cst) volatile noexcept; 237 bool compare_exchange_strong(T*& expc, T* desr, 238 memory_order m = memory_order_seq_cst) noexcept; 239 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 240 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 241 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept; 242 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept; 243 244 T* operator++(int) volatile noexcept; 245 T* operator++(int) noexcept; 246 T* operator--(int) volatile noexcept; 247 T* operator--(int) noexcept; 248 T* operator++() volatile noexcept; 249 T* operator++() noexcept; 250 T* operator--() volatile noexcept; 251 T* operator--() noexcept; 252 T* operator+=(ptrdiff_t op) volatile noexcept; 253 T* operator+=(ptrdiff_t op) noexcept; 254 T* operator-=(ptrdiff_t op) volatile noexcept; 255 T* operator-=(ptrdiff_t op) noexcept; 256 257 void wait(T*, memory_order = memory_order::seq_cst) const volatile noexcept; 258 void wait(T*, memory_order = memory_order::seq_cst) const noexcept; 259 void notify_one() volatile noexcept; 260 void notify_one() noexcept; 261 void notify_all() volatile noexcept; 262 void notify_all() noexcept; 263}; 264 265 266// [atomics.nonmembers], non-member functions 267template<class T> 268 bool atomic_is_lock_free(const volatile atomic<T>*) noexcept; 269template<class T> 270 bool atomic_is_lock_free(const atomic<T>*) noexcept; 271template<class T> 272 void atomic_store(volatile atomic<T>*, atomic<T>::value_type) noexcept; 273template<class T> 274 void atomic_store(atomic<T>*, atomic<T>::value_type) noexcept; 275template<class T> 276 void atomic_store_explicit(volatile atomic<T>*, atomic<T>::value_type, 277 memory_order) noexcept; 278template<class T> 279 void atomic_store_explicit(atomic<T>*, atomic<T>::value_type, 280 memory_order) noexcept; 281template<class T> 282 T atomic_load(const volatile atomic<T>*) noexcept; 283template<class T> 284 T atomic_load(const atomic<T>*) noexcept; 285template<class T> 286 T atomic_load_explicit(const volatile atomic<T>*, memory_order) noexcept; 287template<class T> 288 T atomic_load_explicit(const atomic<T>*, memory_order) noexcept; 289template<class T> 290 T atomic_exchange(volatile atomic<T>*, atomic<T>::value_type) noexcept; 291template<class T> 292 T atomic_exchange(atomic<T>*, atomic<T>::value_type) noexcept; 293template<class T> 294 T atomic_exchange_explicit(volatile atomic<T>*, atomic<T>::value_type, 295 memory_order) noexcept; 296template<class T> 297 T atomic_exchange_explicit(atomic<T>*, atomic<T>::value_type, 298 memory_order) noexcept; 299template<class T> 300 bool atomic_compare_exchange_weak(volatile atomic<T>*, atomic<T>::value_type*, 301 atomic<T>::value_type) noexcept; 302template<class T> 303 bool atomic_compare_exchange_weak(atomic<T>*, atomic<T>::value_type*, 304 atomic<T>::value_type) noexcept; 305template<class T> 306 bool atomic_compare_exchange_strong(volatile atomic<T>*, atomic<T>::value_type*, 307 atomic<T>::value_type) noexcept; 308template<class T> 309 bool atomic_compare_exchange_strong(atomic<T>*, atomic<T>::value_type*, 310 atomic<T>::value_type) noexcept; 311template<class T> 312 bool atomic_compare_exchange_weak_explicit(volatile atomic<T>*, atomic<T>::value_type*, 313 atomic<T>::value_type, 314 memory_order, memory_order) noexcept; 315template<class T> 316 bool atomic_compare_exchange_weak_explicit(atomic<T>*, atomic<T>::value_type*, 317 atomic<T>::value_type, 318 memory_order, memory_order) noexcept; 319template<class T> 320 bool atomic_compare_exchange_strong_explicit(volatile atomic<T>*, atomic<T>::value_type*, 321 atomic<T>::value_type, 322 memory_order, memory_order) noexcept; 323template<class T> 324 bool atomic_compare_exchange_strong_explicit(atomic<T>*, atomic<T>::value_type*, 325 atomic<T>::value_type, 326 memory_order, memory_order) noexcept; 327 328template<class T> 329 T atomic_fetch_add(volatile atomic<T>*, atomic<T>::difference_type) noexcept; 330template<class T> 331 T atomic_fetch_add(atomic<T>*, atomic<T>::difference_type) noexcept; 332template<class T> 333 T atomic_fetch_add_explicit(volatile atomic<T>*, atomic<T>::difference_type, 334 memory_order) noexcept; 335template<class T> 336 T atomic_fetch_add_explicit(atomic<T>*, atomic<T>::difference_type, 337 memory_order) noexcept; 338template<class T> 339 T atomic_fetch_sub(volatile atomic<T>*, atomic<T>::difference_type) noexcept; 340template<class T> 341 T atomic_fetch_sub(atomic<T>*, atomic<T>::difference_type) noexcept; 342template<class T> 343 T atomic_fetch_sub_explicit(volatile atomic<T>*, atomic<T>::difference_type, 344 memory_order) noexcept; 345template<class T> 346 T atomic_fetch_sub_explicit(atomic<T>*, atomic<T>::difference_type, 347 memory_order) noexcept; 348template<class T> 349 T atomic_fetch_and(volatile atomic<T>*, atomic<T>::value_type) noexcept; 350template<class T> 351 T atomic_fetch_and(atomic<T>*, atomic<T>::value_type) noexcept; 352template<class T> 353 T atomic_fetch_and_explicit(volatile atomic<T>*, atomic<T>::value_type, 354 memory_order) noexcept; 355template<class T> 356 T atomic_fetch_and_explicit(atomic<T>*, atomic<T>::value_type, 357 memory_order) noexcept; 358template<class T> 359 T atomic_fetch_or(volatile atomic<T>*, atomic<T>::value_type) noexcept; 360template<class T> 361 T atomic_fetch_or(atomic<T>*, atomic<T>::value_type) noexcept; 362template<class T> 363 T atomic_fetch_or_explicit(volatile atomic<T>*, atomic<T>::value_type, 364 memory_order) noexcept; 365template<class T> 366 T atomic_fetch_or_explicit(atomic<T>*, atomic<T>::value_type, 367 memory_order) noexcept; 368template<class T> 369 T atomic_fetch_xor(volatile atomic<T>*, atomic<T>::value_type) noexcept; 370template<class T> 371 T atomic_fetch_xor(atomic<T>*, atomic<T>::value_type) noexcept; 372template<class T> 373 T atomic_fetch_xor_explicit(volatile atomic<T>*, atomic<T>::value_type, 374 memory_order) noexcept; 375template<class T> 376 T atomic_fetch_xor_explicit(atomic<T>*, atomic<T>::value_type, 377 memory_order) noexcept; 378 379template<class T> 380 void atomic_wait(const volatile atomic<T>*, atomic<T>::value_type) noexcept; 381template<class T> 382 void atomic_wait(const atomic<T>*, atomic<T>::value_type) noexcept; 383template<class T> 384 void atomic_wait_explicit(const volatile atomic<T>*, atomic<T>::value_type, 385 memory_order) noexcept; 386template<class T> 387 void atomic_wait_explicit(const atomic<T>*, atomic<T>::value_type, 388 memory_order) noexcept; 389template<class T> 390 void atomic_notify_one(volatile atomic<T>*) noexcept; 391template<class T> 392 void atomic_notify_one(atomic<T>*) noexcept; 393template<class T> 394 void atomic_notify_all(volatile atomic<T>*) noexcept; 395template<class T> 396 void atomic_notify_all(atomic<T>*) noexcept; 397 398// Atomics for standard typedef types 399 400typedef atomic<bool> atomic_bool; 401typedef atomic<char> atomic_char; 402typedef atomic<signed char> atomic_schar; 403typedef atomic<unsigned char> atomic_uchar; 404typedef atomic<short> atomic_short; 405typedef atomic<unsigned short> atomic_ushort; 406typedef atomic<int> atomic_int; 407typedef atomic<unsigned int> atomic_uint; 408typedef atomic<long> atomic_long; 409typedef atomic<unsigned long> atomic_ulong; 410typedef atomic<long long> atomic_llong; 411typedef atomic<unsigned long long> atomic_ullong; 412typedef atomic<char8_t> atomic_char8_t; // C++20 413typedef atomic<char16_t> atomic_char16_t; 414typedef atomic<char32_t> atomic_char32_t; 415typedef atomic<wchar_t> atomic_wchar_t; 416 417typedef atomic<int_least8_t> atomic_int_least8_t; 418typedef atomic<uint_least8_t> atomic_uint_least8_t; 419typedef atomic<int_least16_t> atomic_int_least16_t; 420typedef atomic<uint_least16_t> atomic_uint_least16_t; 421typedef atomic<int_least32_t> atomic_int_least32_t; 422typedef atomic<uint_least32_t> atomic_uint_least32_t; 423typedef atomic<int_least64_t> atomic_int_least64_t; 424typedef atomic<uint_least64_t> atomic_uint_least64_t; 425 426typedef atomic<int_fast8_t> atomic_int_fast8_t; 427typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 428typedef atomic<int_fast16_t> atomic_int_fast16_t; 429typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 430typedef atomic<int_fast32_t> atomic_int_fast32_t; 431typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 432typedef atomic<int_fast64_t> atomic_int_fast64_t; 433typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 434 435typedef atomic<int8_t> atomic_int8_t; 436typedef atomic<uint8_t> atomic_uint8_t; 437typedef atomic<int16_t> atomic_int16_t; 438typedef atomic<uint16_t> atomic_uint16_t; 439typedef atomic<int32_t> atomic_int32_t; 440typedef atomic<uint32_t> atomic_uint32_t; 441typedef atomic<int64_t> atomic_int64_t; 442typedef atomic<uint64_t> atomic_uint64_t; 443 444typedef atomic<intptr_t> atomic_intptr_t; 445typedef atomic<uintptr_t> atomic_uintptr_t; 446typedef atomic<size_t> atomic_size_t; 447typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 448typedef atomic<intmax_t> atomic_intmax_t; 449typedef atomic<uintmax_t> atomic_uintmax_t; 450 451// flag type and operations 452 453typedef struct atomic_flag 454{ 455 atomic_flag() noexcept = default; // until C++20 456 constexpr atomic_flag() noexcept; // since C++20 457 atomic_flag(const atomic_flag&) = delete; 458 atomic_flag& operator=(const atomic_flag&) = delete; 459 atomic_flag& operator=(const atomic_flag&) volatile = delete; 460 461 bool test(memory_order m = memory_order_seq_cst) volatile noexcept; 462 bool test(memory_order m = memory_order_seq_cst) noexcept; 463 bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept; 464 bool test_and_set(memory_order m = memory_order_seq_cst) noexcept; 465 void clear(memory_order m = memory_order_seq_cst) volatile noexcept; 466 void clear(memory_order m = memory_order_seq_cst) noexcept; 467 468 void wait(bool, memory_order = memory_order::seq_cst) const volatile noexcept; 469 void wait(bool, memory_order = memory_order::seq_cst) const noexcept; 470 void notify_one() volatile noexcept; 471 void notify_one() noexcept; 472 void notify_all() volatile noexcept; 473 void notify_all() noexcept; 474} atomic_flag; 475 476bool atomic_flag_test(volatile atomic_flag* obj) noexcept; 477bool atomic_flag_test(atomic_flag* obj) noexcept; 478bool atomic_flag_test_explicit(volatile atomic_flag* obj, 479 memory_order m) noexcept; 480bool atomic_flag_test_explicit(atomic_flag* obj, memory_order m) noexcept; 481bool atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept; 482bool atomic_flag_test_and_set(atomic_flag* obj) noexcept; 483bool atomic_flag_test_and_set_explicit(volatile atomic_flag* obj, 484 memory_order m) noexcept; 485bool atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept; 486void atomic_flag_clear(volatile atomic_flag* obj) noexcept; 487void atomic_flag_clear(atomic_flag* obj) noexcept; 488void atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept; 489void atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept; 490 491void atomic_wait(const volatile atomic_flag* obj, T old) noexcept; 492void atomic_wait(const atomic_flag* obj, T old) noexcept; 493void atomic_wait_explicit(const volatile atomic_flag* obj, T old, memory_order m) noexcept; 494void atomic_wait_explicit(const atomic_flag* obj, T old, memory_order m) noexcept; 495void atomic_one(volatile atomic_flag* obj) noexcept; 496void atomic_one(atomic_flag* obj) noexcept; 497void atomic_all(volatile atomic_flag* obj) noexcept; 498void atomic_all(atomic_flag* obj) noexcept; 499 500// fences 501 502void atomic_thread_fence(memory_order m) noexcept; 503void atomic_signal_fence(memory_order m) noexcept; 504 505// deprecated 506 507template <class T> 508 void atomic_init(volatile atomic<T>* obj, atomic<T>::value_type desr) noexcept; 509 510template <class T> 511 void atomic_init(atomic<T>* obj, atomic<T>::value_type desr) noexcept; 512 513#define ATOMIC_VAR_INIT(value) see below 514 515#define ATOMIC_FLAG_INIT see below 516 517} // std 518 519*/ 520 521#include <__assert> // all public C++ headers provide the assertion handler 522#include <__availability> 523#include <__chrono/duration.h> 524#include <__config> 525#include <__thread/poll_with_backoff.h> 526#include <__thread/timed_backoff_policy.h> 527#include <__type_traits/conditional.h> 528#include <__type_traits/decay.h> 529#include <__type_traits/is_assignable.h> 530#include <__type_traits/is_function.h> 531#include <__type_traits/is_nothrow_default_constructible.h> 532#include <__type_traits/is_same.h> 533#include <__type_traits/is_trivially_copyable.h> 534#include <__type_traits/remove_const.h> 535#include <__type_traits/remove_pointer.h> 536#include <__type_traits/underlying_type.h> 537#include <cstddef> 538#include <cstdint> 539#include <cstring> 540#include <version> 541 542#ifndef _LIBCPP_HAS_NO_THREADS 543# include <__threading_support> 544#endif 545 546#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 547# pragma GCC system_header 548#endif 549 550#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER 551# error <atomic> is not implemented 552#endif 553#ifdef kill_dependency 554# error <atomic> is incompatible with <stdatomic.h> before C++23. Please compile with -std=c++23. 555#endif 556 557#define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ 558 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_consume || \ 559 __m == memory_order_acquire || \ 560 __m == memory_order_acq_rel, \ 561 "memory order argument to atomic operation is invalid") 562 563#define _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) \ 564 _LIBCPP_DIAGNOSE_WARNING(__m == memory_order_release || \ 565 __m == memory_order_acq_rel, \ 566 "memory order argument to atomic operation is invalid") 567 568#define _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \ 569 _LIBCPP_DIAGNOSE_WARNING(__f == memory_order_release || \ 570 __f == memory_order_acq_rel, \ 571 "memory order argument to atomic operation is invalid") 572 573_LIBCPP_BEGIN_NAMESPACE_STD 574 575// Figure out what the underlying type for `memory_order` would be if it were 576// declared as an unscoped enum (accounting for -fshort-enums). Use this result 577// to pin the underlying type in C++20. 578enum __legacy_memory_order { 579 __mo_relaxed, 580 __mo_consume, 581 __mo_acquire, 582 __mo_release, 583 __mo_acq_rel, 584 __mo_seq_cst 585}; 586 587typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t; 588 589#if _LIBCPP_STD_VER > 17 590 591enum class memory_order : __memory_order_underlying_t { 592 relaxed = __mo_relaxed, 593 consume = __mo_consume, 594 acquire = __mo_acquire, 595 release = __mo_release, 596 acq_rel = __mo_acq_rel, 597 seq_cst = __mo_seq_cst 598}; 599 600inline constexpr auto memory_order_relaxed = memory_order::relaxed; 601inline constexpr auto memory_order_consume = memory_order::consume; 602inline constexpr auto memory_order_acquire = memory_order::acquire; 603inline constexpr auto memory_order_release = memory_order::release; 604inline constexpr auto memory_order_acq_rel = memory_order::acq_rel; 605inline constexpr auto memory_order_seq_cst = memory_order::seq_cst; 606 607#else 608 609typedef enum memory_order { 610 memory_order_relaxed = __mo_relaxed, 611 memory_order_consume = __mo_consume, 612 memory_order_acquire = __mo_acquire, 613 memory_order_release = __mo_release, 614 memory_order_acq_rel = __mo_acq_rel, 615 memory_order_seq_cst = __mo_seq_cst, 616} memory_order; 617 618#endif // _LIBCPP_STD_VER > 17 619 620template <typename _Tp> _LIBCPP_INLINE_VISIBILITY 621bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) { 622 return _VSTD::memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0; 623} 624 625static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value), 626 "unexpected underlying type for std::memory_order"); 627 628#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ 629 defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) 630 631// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because 632// the default operator= in an object is not volatile, a byte-by-byte copy 633// is required. 634template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 635typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 636__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { 637 __a_value = __val; 638} 639template <typename _Tp, typename _Tv> _LIBCPP_INLINE_VISIBILITY 640typename enable_if<is_assignable<_Tp&, _Tv>::value>::type 641__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { 642 volatile char* __to = reinterpret_cast<volatile char*>(&__a_value); 643 volatile char* __end = __to + sizeof(_Tp); 644 volatile const char* __from = reinterpret_cast<volatile const char*>(&__val); 645 while (__to != __end) 646 *__to++ = *__from++; 647} 648 649#endif 650 651#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) 652 653template <typename _Tp> 654struct __cxx_atomic_base_impl { 655 656 _LIBCPP_INLINE_VISIBILITY 657#ifndef _LIBCPP_CXX03_LANG 658 __cxx_atomic_base_impl() _NOEXCEPT = default; 659#else 660 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 661#endif // _LIBCPP_CXX03_LANG 662 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT 663 : __a_value(value) {} 664 _Tp __a_value; 665}; 666 667_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { 668 // Avoid switch statement to make this a constexpr. 669 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 670 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 671 (__order == memory_order_release ? __ATOMIC_RELEASE: 672 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 673 (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: 674 __ATOMIC_CONSUME)))); 675} 676 677_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { 678 // Avoid switch statement to make this a constexpr. 679 return __order == memory_order_relaxed ? __ATOMIC_RELAXED: 680 (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: 681 (__order == memory_order_release ? __ATOMIC_RELAXED: 682 (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: 683 (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: 684 __ATOMIC_CONSUME)))); 685} 686 687template <typename _Tp> 688_LIBCPP_INLINE_VISIBILITY 689void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 690 __cxx_atomic_assign_volatile(__a->__a_value, __val); 691} 692 693template <typename _Tp> 694_LIBCPP_INLINE_VISIBILITY 695void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { 696 __a->__a_value = __val; 697} 698 699_LIBCPP_INLINE_VISIBILITY inline 700void __cxx_atomic_thread_fence(memory_order __order) { 701 __atomic_thread_fence(__to_gcc_order(__order)); 702} 703 704_LIBCPP_INLINE_VISIBILITY inline 705void __cxx_atomic_signal_fence(memory_order __order) { 706 __atomic_signal_fence(__to_gcc_order(__order)); 707} 708 709template <typename _Tp> 710_LIBCPP_INLINE_VISIBILITY 711void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 712 memory_order __order) { 713 __atomic_store(&__a->__a_value, &__val, 714 __to_gcc_order(__order)); 715} 716 717template <typename _Tp> 718_LIBCPP_INLINE_VISIBILITY 719void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, 720 memory_order __order) { 721 __atomic_store(&__a->__a_value, &__val, 722 __to_gcc_order(__order)); 723} 724 725template <typename _Tp> 726_LIBCPP_INLINE_VISIBILITY 727_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, 728 memory_order __order) { 729 _Tp __ret; 730 __atomic_load(&__a->__a_value, &__ret, 731 __to_gcc_order(__order)); 732 return __ret; 733} 734 735template <typename _Tp> 736_LIBCPP_INLINE_VISIBILITY 737_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { 738 _Tp __ret; 739 __atomic_load(&__a->__a_value, &__ret, 740 __to_gcc_order(__order)); 741 return __ret; 742} 743 744template <typename _Tp> 745_LIBCPP_INLINE_VISIBILITY 746_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, 747 _Tp __value, memory_order __order) { 748 _Tp __ret; 749 __atomic_exchange(&__a->__a_value, &__value, &__ret, 750 __to_gcc_order(__order)); 751 return __ret; 752} 753 754template <typename _Tp> 755_LIBCPP_INLINE_VISIBILITY 756_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, 757 memory_order __order) { 758 _Tp __ret; 759 __atomic_exchange(&__a->__a_value, &__value, &__ret, 760 __to_gcc_order(__order)); 761 return __ret; 762} 763 764template <typename _Tp> 765_LIBCPP_INLINE_VISIBILITY 766bool __cxx_atomic_compare_exchange_strong( 767 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 768 memory_order __success, memory_order __failure) { 769 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 770 false, 771 __to_gcc_order(__success), 772 __to_gcc_failure_order(__failure)); 773} 774 775template <typename _Tp> 776_LIBCPP_INLINE_VISIBILITY 777bool __cxx_atomic_compare_exchange_strong( 778 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 779 memory_order __failure) { 780 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 781 false, 782 __to_gcc_order(__success), 783 __to_gcc_failure_order(__failure)); 784} 785 786template <typename _Tp> 787_LIBCPP_INLINE_VISIBILITY 788bool __cxx_atomic_compare_exchange_weak( 789 volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, 790 memory_order __success, memory_order __failure) { 791 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 792 true, 793 __to_gcc_order(__success), 794 __to_gcc_failure_order(__failure)); 795} 796 797template <typename _Tp> 798_LIBCPP_INLINE_VISIBILITY 799bool __cxx_atomic_compare_exchange_weak( 800 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, 801 memory_order __failure) { 802 return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, 803 true, 804 __to_gcc_order(__success), 805 __to_gcc_failure_order(__failure)); 806} 807 808template <typename _Tp> 809struct __skip_amt { enum {value = 1}; }; 810 811template <typename _Tp> 812struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; 813 814// FIXME: Haven't figured out what the spec says about using arrays with 815// atomic_fetch_add. Force a failure rather than creating bad behavior. 816template <typename _Tp> 817struct __skip_amt<_Tp[]> { }; 818template <typename _Tp, int n> 819struct __skip_amt<_Tp[n]> { }; 820 821template <typename _Tp, typename _Td> 822_LIBCPP_INLINE_VISIBILITY 823_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, 824 _Td __delta, memory_order __order) { 825 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 826 __to_gcc_order(__order)); 827} 828 829template <typename _Tp, typename _Td> 830_LIBCPP_INLINE_VISIBILITY 831_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 832 memory_order __order) { 833 return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 834 __to_gcc_order(__order)); 835} 836 837template <typename _Tp, typename _Td> 838_LIBCPP_INLINE_VISIBILITY 839_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, 840 _Td __delta, memory_order __order) { 841 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 842 __to_gcc_order(__order)); 843} 844 845template <typename _Tp, typename _Td> 846_LIBCPP_INLINE_VISIBILITY 847_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, 848 memory_order __order) { 849 return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, 850 __to_gcc_order(__order)); 851} 852 853template <typename _Tp> 854_LIBCPP_INLINE_VISIBILITY 855_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, 856 _Tp __pattern, memory_order __order) { 857 return __atomic_fetch_and(&__a->__a_value, __pattern, 858 __to_gcc_order(__order)); 859} 860 861template <typename _Tp> 862_LIBCPP_INLINE_VISIBILITY 863_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, 864 _Tp __pattern, memory_order __order) { 865 return __atomic_fetch_and(&__a->__a_value, __pattern, 866 __to_gcc_order(__order)); 867} 868 869template <typename _Tp> 870_LIBCPP_INLINE_VISIBILITY 871_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, 872 _Tp __pattern, memory_order __order) { 873 return __atomic_fetch_or(&__a->__a_value, __pattern, 874 __to_gcc_order(__order)); 875} 876 877template <typename _Tp> 878_LIBCPP_INLINE_VISIBILITY 879_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 880 memory_order __order) { 881 return __atomic_fetch_or(&__a->__a_value, __pattern, 882 __to_gcc_order(__order)); 883} 884 885template <typename _Tp> 886_LIBCPP_INLINE_VISIBILITY 887_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, 888 _Tp __pattern, memory_order __order) { 889 return __atomic_fetch_xor(&__a->__a_value, __pattern, 890 __to_gcc_order(__order)); 891} 892 893template <typename _Tp> 894_LIBCPP_INLINE_VISIBILITY 895_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, 896 memory_order __order) { 897 return __atomic_fetch_xor(&__a->__a_value, __pattern, 898 __to_gcc_order(__order)); 899} 900 901#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) 902 903#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) 904 905template <typename _Tp> 906struct __cxx_atomic_base_impl { 907 908 _LIBCPP_INLINE_VISIBILITY 909#ifndef _LIBCPP_CXX03_LANG 910 __cxx_atomic_base_impl() _NOEXCEPT = default; 911#else 912 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} 913#endif // _LIBCPP_CXX03_LANG 914 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT 915 : __a_value(__value) {} 916 _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; 917}; 918 919#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) 920 921_LIBCPP_INLINE_VISIBILITY inline 922void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { 923 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); 924} 925 926_LIBCPP_INLINE_VISIBILITY inline 927void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { 928 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); 929} 930 931template<class _Tp> 932_LIBCPP_INLINE_VISIBILITY 933void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { 934 __c11_atomic_init(&__a->__a_value, __val); 935} 936template<class _Tp> 937_LIBCPP_INLINE_VISIBILITY 938void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { 939 __c11_atomic_init(&__a->__a_value, __val); 940} 941 942template<class _Tp> 943_LIBCPP_INLINE_VISIBILITY 944void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { 945 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 946} 947template<class _Tp> 948_LIBCPP_INLINE_VISIBILITY 949void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { 950 __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); 951} 952 953template<class _Tp> 954_LIBCPP_INLINE_VISIBILITY 955_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { 956 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; 957 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 958} 959template<class _Tp> 960_LIBCPP_INLINE_VISIBILITY 961_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { 962 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*; 963 return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); 964} 965 966template<class _Tp> 967_LIBCPP_INLINE_VISIBILITY 968_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { 969 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 970} 971template<class _Tp> 972_LIBCPP_INLINE_VISIBILITY 973_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { 974 return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); 975} 976 977_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) { 978 // Avoid switch statement to make this a constexpr. 979 return __order == memory_order_release ? memory_order_relaxed: 980 (__order == memory_order_acq_rel ? memory_order_acquire: 981 __order); 982} 983 984template<class _Tp> 985_LIBCPP_INLINE_VISIBILITY 986bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 987 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 988} 989template<class _Tp> 990_LIBCPP_INLINE_VISIBILITY 991bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 992 return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 993} 994 995template<class _Tp> 996_LIBCPP_INLINE_VISIBILITY 997bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 998 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 999} 1000template<class _Tp> 1001_LIBCPP_INLINE_VISIBILITY 1002bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { 1003 return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__to_failure_order(__failure))); 1004} 1005 1006template<class _Tp> 1007_LIBCPP_INLINE_VISIBILITY 1008_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1009 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1010} 1011template<class _Tp> 1012_LIBCPP_INLINE_VISIBILITY 1013_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1014 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1015} 1016 1017template<class _Tp> 1018_LIBCPP_INLINE_VISIBILITY 1019_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1020 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1021} 1022template<class _Tp> 1023_LIBCPP_INLINE_VISIBILITY 1024_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1025 return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1026} 1027 1028template<class _Tp> 1029_LIBCPP_INLINE_VISIBILITY 1030_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1031 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1032} 1033template<class _Tp> 1034_LIBCPP_INLINE_VISIBILITY 1035_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { 1036 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1037} 1038template<class _Tp> 1039_LIBCPP_INLINE_VISIBILITY 1040_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1041 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1042} 1043template<class _Tp> 1044_LIBCPP_INLINE_VISIBILITY 1045_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { 1046 return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); 1047} 1048 1049template<class _Tp> 1050_LIBCPP_INLINE_VISIBILITY 1051_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1052 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1053} 1054template<class _Tp> 1055_LIBCPP_INLINE_VISIBILITY 1056_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1057 return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1058} 1059 1060template<class _Tp> 1061_LIBCPP_INLINE_VISIBILITY 1062_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1063 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1064} 1065template<class _Tp> 1066_LIBCPP_INLINE_VISIBILITY 1067_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1068 return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1069} 1070 1071template<class _Tp> 1072_LIBCPP_INLINE_VISIBILITY 1073_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1074 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1075} 1076template<class _Tp> 1077_LIBCPP_INLINE_VISIBILITY 1078_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { 1079 return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); 1080} 1081 1082#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP 1083 1084template <class _Tp> 1085_LIBCPP_INLINE_VISIBILITY 1086_Tp kill_dependency(_Tp __y) _NOEXCEPT 1087{ 1088 return __y; 1089} 1090 1091#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) 1092# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE 1093# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE 1094#ifndef _LIBCPP_HAS_NO_CHAR8_T 1095# define ATOMIC_CHAR8_T_LOCK_FREE __CLANG_ATOMIC_CHAR8_T_LOCK_FREE 1096#endif 1097# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE 1098# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE 1099# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE 1100# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE 1101# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE 1102# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE 1103# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE 1104# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE 1105#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) 1106# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 1107# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 1108#ifndef _LIBCPP_HAS_NO_CHAR8_T 1109# define ATOMIC_CHAR8_T_LOCK_FREE __GCC_ATOMIC_CHAR8_T_LOCK_FREE 1110#endif 1111# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 1112# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 1113# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 1114# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 1115# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 1116# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 1117# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 1118# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 1119#endif 1120 1121template <class _Tp> 1122struct __libcpp_is_always_lock_free { 1123 // __atomic_always_lock_free is available in all Standard modes 1124 static const bool __value = __atomic_always_lock_free(sizeof(_Tp), 0); 1125}; 1126 1127#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1128 1129template<typename _Tp> 1130struct __cxx_atomic_lock_impl { 1131 1132 _LIBCPP_INLINE_VISIBILITY 1133 __cxx_atomic_lock_impl() _NOEXCEPT 1134 : __a_value(), __a_lock(0) {} 1135 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit 1136 __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT 1137 : __a_value(value), __a_lock(0) {} 1138 1139 _Tp __a_value; 1140 mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; 1141 1142 _LIBCPP_INLINE_VISIBILITY void __lock() const volatile { 1143 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1144 /*spin*/; 1145 } 1146 _LIBCPP_INLINE_VISIBILITY void __lock() const { 1147 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) 1148 /*spin*/; 1149 } 1150 _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile { 1151 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1152 } 1153 _LIBCPP_INLINE_VISIBILITY void __unlock() const { 1154 __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); 1155 } 1156 _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile { 1157 __lock(); 1158 _Tp __old; 1159 __cxx_atomic_assign_volatile(__old, __a_value); 1160 __unlock(); 1161 return __old; 1162 } 1163 _LIBCPP_INLINE_VISIBILITY _Tp __read() const { 1164 __lock(); 1165 _Tp __old = __a_value; 1166 __unlock(); 1167 return __old; 1168 } 1169}; 1170 1171template <typename _Tp> 1172_LIBCPP_INLINE_VISIBILITY 1173void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1174 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1175} 1176template <typename _Tp> 1177_LIBCPP_INLINE_VISIBILITY 1178void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { 1179 __a->__a_value = __val; 1180} 1181 1182template <typename _Tp> 1183_LIBCPP_INLINE_VISIBILITY 1184void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1185 __a->__lock(); 1186 __cxx_atomic_assign_volatile(__a->__a_value, __val); 1187 __a->__unlock(); 1188} 1189template <typename _Tp> 1190_LIBCPP_INLINE_VISIBILITY 1191void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { 1192 __a->__lock(); 1193 __a->__a_value = __val; 1194 __a->__unlock(); 1195} 1196 1197template <typename _Tp> 1198_LIBCPP_INLINE_VISIBILITY 1199_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1200 return __a->__read(); 1201} 1202template <typename _Tp> 1203_LIBCPP_INLINE_VISIBILITY 1204_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { 1205 return __a->__read(); 1206} 1207 1208template <typename _Tp> 1209_LIBCPP_INLINE_VISIBILITY 1210_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1211 __a->__lock(); 1212 _Tp __old; 1213 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1214 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1215 __a->__unlock(); 1216 return __old; 1217} 1218template <typename _Tp> 1219_LIBCPP_INLINE_VISIBILITY 1220_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { 1221 __a->__lock(); 1222 _Tp __old = __a->__a_value; 1223 __a->__a_value = __value; 1224 __a->__unlock(); 1225 return __old; 1226} 1227 1228template <typename _Tp> 1229_LIBCPP_INLINE_VISIBILITY 1230bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1231 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1232 _Tp __temp; 1233 __a->__lock(); 1234 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1235 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1236 if(__ret) 1237 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1238 else 1239 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1240 __a->__unlock(); 1241 return __ret; 1242} 1243template <typename _Tp> 1244_LIBCPP_INLINE_VISIBILITY 1245bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, 1246 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1247 __a->__lock(); 1248 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1249 if(__ret) 1250 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1251 else 1252 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1253 __a->__unlock(); 1254 return __ret; 1255} 1256 1257template <typename _Tp> 1258_LIBCPP_INLINE_VISIBILITY 1259bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1260 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1261 _Tp __temp; 1262 __a->__lock(); 1263 __cxx_atomic_assign_volatile(__temp, __a->__a_value); 1264 bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0); 1265 if(__ret) 1266 __cxx_atomic_assign_volatile(__a->__a_value, __value); 1267 else 1268 __cxx_atomic_assign_volatile(*__expected, __a->__a_value); 1269 __a->__unlock(); 1270 return __ret; 1271} 1272template <typename _Tp> 1273_LIBCPP_INLINE_VISIBILITY 1274bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, 1275 _Tp* __expected, _Tp __value, memory_order, memory_order) { 1276 __a->__lock(); 1277 bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0); 1278 if(__ret) 1279 _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp)); 1280 else 1281 _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp)); 1282 __a->__unlock(); 1283 return __ret; 1284} 1285 1286template <typename _Tp, typename _Td> 1287_LIBCPP_INLINE_VISIBILITY 1288_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1289 _Td __delta, memory_order) { 1290 __a->__lock(); 1291 _Tp __old; 1292 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1293 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); 1294 __a->__unlock(); 1295 return __old; 1296} 1297template <typename _Tp, typename _Td> 1298_LIBCPP_INLINE_VISIBILITY 1299_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, 1300 _Td __delta, memory_order) { 1301 __a->__lock(); 1302 _Tp __old = __a->__a_value; 1303 __a->__a_value += __delta; 1304 __a->__unlock(); 1305 return __old; 1306} 1307 1308template <typename _Tp, typename _Td> 1309_LIBCPP_INLINE_VISIBILITY 1310_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, 1311 ptrdiff_t __delta, memory_order) { 1312 __a->__lock(); 1313 _Tp* __old; 1314 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1315 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); 1316 __a->__unlock(); 1317 return __old; 1318} 1319template <typename _Tp, typename _Td> 1320_LIBCPP_INLINE_VISIBILITY 1321_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, 1322 ptrdiff_t __delta, memory_order) { 1323 __a->__lock(); 1324 _Tp* __old = __a->__a_value; 1325 __a->__a_value += __delta; 1326 __a->__unlock(); 1327 return __old; 1328} 1329 1330template <typename _Tp, typename _Td> 1331_LIBCPP_INLINE_VISIBILITY 1332_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1333 _Td __delta, memory_order) { 1334 __a->__lock(); 1335 _Tp __old; 1336 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1337 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); 1338 __a->__unlock(); 1339 return __old; 1340} 1341template <typename _Tp, typename _Td> 1342_LIBCPP_INLINE_VISIBILITY 1343_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, 1344 _Td __delta, memory_order) { 1345 __a->__lock(); 1346 _Tp __old = __a->__a_value; 1347 __a->__a_value -= __delta; 1348 __a->__unlock(); 1349 return __old; 1350} 1351 1352template <typename _Tp> 1353_LIBCPP_INLINE_VISIBILITY 1354_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1355 _Tp __pattern, memory_order) { 1356 __a->__lock(); 1357 _Tp __old; 1358 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1359 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); 1360 __a->__unlock(); 1361 return __old; 1362} 1363template <typename _Tp> 1364_LIBCPP_INLINE_VISIBILITY 1365_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, 1366 _Tp __pattern, memory_order) { 1367 __a->__lock(); 1368 _Tp __old = __a->__a_value; 1369 __a->__a_value &= __pattern; 1370 __a->__unlock(); 1371 return __old; 1372} 1373 1374template <typename _Tp> 1375_LIBCPP_INLINE_VISIBILITY 1376_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1377 _Tp __pattern, memory_order) { 1378 __a->__lock(); 1379 _Tp __old; 1380 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1381 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); 1382 __a->__unlock(); 1383 return __old; 1384} 1385template <typename _Tp> 1386_LIBCPP_INLINE_VISIBILITY 1387_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, 1388 _Tp __pattern, memory_order) { 1389 __a->__lock(); 1390 _Tp __old = __a->__a_value; 1391 __a->__a_value |= __pattern; 1392 __a->__unlock(); 1393 return __old; 1394} 1395 1396template <typename _Tp> 1397_LIBCPP_INLINE_VISIBILITY 1398_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, 1399 _Tp __pattern, memory_order) { 1400 __a->__lock(); 1401 _Tp __old; 1402 __cxx_atomic_assign_volatile(__old, __a->__a_value); 1403 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); 1404 __a->__unlock(); 1405 return __old; 1406} 1407template <typename _Tp> 1408_LIBCPP_INLINE_VISIBILITY 1409_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, 1410 _Tp __pattern, memory_order) { 1411 __a->__lock(); 1412 _Tp __old = __a->__a_value; 1413 __a->__a_value ^= __pattern; 1414 __a->__unlock(); 1415 return __old; 1416} 1417 1418template <typename _Tp, 1419 typename _Base = typename conditional<__libcpp_is_always_lock_free<_Tp>::__value, 1420 __cxx_atomic_base_impl<_Tp>, 1421 __cxx_atomic_lock_impl<_Tp> >::type> 1422#else 1423template <typename _Tp, 1424 typename _Base = __cxx_atomic_base_impl<_Tp> > 1425#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS 1426struct __cxx_atomic_impl : public _Base { 1427 static_assert(is_trivially_copyable<_Tp>::value, 1428 "std::atomic<T> requires that 'T' be a trivially copyable type"); 1429 1430 _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT = default; 1431 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp __value) _NOEXCEPT 1432 : _Base(__value) {} 1433}; 1434 1435#if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__)) || \ 1436 (defined(__FreeBSD__) && defined(__mips__)) 1437 using __cxx_contention_t = int32_t; 1438#else 1439 using __cxx_contention_t = int64_t; 1440#endif // __linux__ || (_AIX && !__64BIT__) || (__FreeBSD__ && __mips__) 1441 1442using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>; 1443 1444#ifndef _LIBCPP_HAS_NO_THREADS 1445 1446_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); 1447_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*); 1448_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*); 1449_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t); 1450 1451_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*); 1452_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*); 1453_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); 1454_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); 1455 1456template <class _Atp, class _Fn> 1457struct __libcpp_atomic_wait_backoff_impl { 1458 _Atp* __a; 1459 _Fn __test_fn; 1460 _LIBCPP_AVAILABILITY_SYNC 1461 _LIBCPP_INLINE_VISIBILITY bool operator()(chrono::nanoseconds __elapsed) const 1462 { 1463 if(__elapsed > chrono::microseconds(64)) 1464 { 1465 auto const __monitor = std::__libcpp_atomic_monitor(__a); 1466 if(__test_fn()) 1467 return true; 1468 std::__libcpp_atomic_wait(__a, __monitor); 1469 } 1470 else if(__elapsed > chrono::microseconds(4)) 1471 __libcpp_thread_yield(); 1472 else 1473 {} // poll 1474 return false; 1475 } 1476}; 1477 1478template <class _Atp, class _Fn> 1479_LIBCPP_AVAILABILITY_SYNC 1480_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Fn && __test_fn) 1481{ 1482 __libcpp_atomic_wait_backoff_impl<_Atp, typename decay<_Fn>::type> __backoff_fn = {__a, __test_fn}; 1483 return std::__libcpp_thread_poll_with_backoff(__test_fn, __backoff_fn); 1484} 1485 1486#else // _LIBCPP_HAS_NO_THREADS 1487 1488template <class _Tp> 1489_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) { } 1490template <class _Tp> 1491_LIBCPP_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) { } 1492template <class _Atp, class _Fn> 1493_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp*, _Fn && __test_fn) 1494{ 1495 return __libcpp_thread_poll_with_backoff(__test_fn, __spinning_backoff_policy()); 1496} 1497 1498#endif // _LIBCPP_HAS_NO_THREADS 1499 1500template <class _Atp, class _Tp> 1501struct __cxx_atomic_wait_test_fn_impl { 1502 _Atp* __a; 1503 _Tp __val; 1504 memory_order __order; 1505 _LIBCPP_INLINE_VISIBILITY bool operator()() const 1506 { 1507 return !std::__cxx_nonatomic_compare_equal(std::__cxx_atomic_load(__a, __order), __val); 1508 } 1509}; 1510 1511template <class _Atp, class _Tp> 1512_LIBCPP_AVAILABILITY_SYNC 1513_LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) 1514{ 1515 __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; 1516 return std::__cxx_atomic_wait(__a, __test_fn); 1517} 1518 1519// general atomic<T> 1520 1521template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value> 1522struct __atomic_base // false 1523{ 1524 mutable __cxx_atomic_impl<_Tp> __a_; 1525 1526#if defined(__cpp_lib_atomic_is_always_lock_free) 1527 static _LIBCPP_CONSTEXPR bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value; 1528#endif 1529 1530 _LIBCPP_INLINE_VISIBILITY 1531 bool is_lock_free() const volatile _NOEXCEPT 1532 {return __cxx_atomic_is_lock_free(sizeof(_Tp));} 1533 _LIBCPP_INLINE_VISIBILITY 1534 bool is_lock_free() const _NOEXCEPT 1535 {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} 1536 _LIBCPP_INLINE_VISIBILITY 1537 void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1538 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1539 {std::__cxx_atomic_store(&__a_, __d, __m);} 1540 _LIBCPP_INLINE_VISIBILITY 1541 void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1542 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1543 {std::__cxx_atomic_store(&__a_, __d, __m);} 1544 _LIBCPP_INLINE_VISIBILITY 1545 _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1546 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1547 {return std::__cxx_atomic_load(&__a_, __m);} 1548 _LIBCPP_INLINE_VISIBILITY 1549 _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1550 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1551 {return std::__cxx_atomic_load(&__a_, __m);} 1552 _LIBCPP_INLINE_VISIBILITY 1553 operator _Tp() const volatile _NOEXCEPT {return load();} 1554 _LIBCPP_INLINE_VISIBILITY 1555 operator _Tp() const _NOEXCEPT {return load();} 1556 _LIBCPP_INLINE_VISIBILITY 1557 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1558 {return std::__cxx_atomic_exchange(&__a_, __d, __m);} 1559 _LIBCPP_INLINE_VISIBILITY 1560 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1561 {return std::__cxx_atomic_exchange(&__a_, __d, __m);} 1562 _LIBCPP_INLINE_VISIBILITY 1563 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1564 memory_order __s, memory_order __f) volatile _NOEXCEPT 1565 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1566 {return std::__cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1567 _LIBCPP_INLINE_VISIBILITY 1568 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1569 memory_order __s, memory_order __f) _NOEXCEPT 1570 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1571 {return std::__cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} 1572 _LIBCPP_INLINE_VISIBILITY 1573 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1574 memory_order __s, memory_order __f) volatile _NOEXCEPT 1575 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1576 {return std::__cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1577 _LIBCPP_INLINE_VISIBILITY 1578 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1579 memory_order __s, memory_order __f) _NOEXCEPT 1580 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 1581 {return std::__cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} 1582 _LIBCPP_INLINE_VISIBILITY 1583 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1584 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1585 {return std::__cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1586 _LIBCPP_INLINE_VISIBILITY 1587 bool compare_exchange_weak(_Tp& __e, _Tp __d, 1588 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1589 {return std::__cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} 1590 _LIBCPP_INLINE_VISIBILITY 1591 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1592 memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1593 {return std::__cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1594 _LIBCPP_INLINE_VISIBILITY 1595 bool compare_exchange_strong(_Tp& __e, _Tp __d, 1596 memory_order __m = memory_order_seq_cst) _NOEXCEPT 1597 {return std::__cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} 1598 1599 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 1600 {std::__cxx_atomic_wait(&__a_, __v, __m);} 1601 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 1602 {std::__cxx_atomic_wait(&__a_, __v, __m);} 1603 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() volatile _NOEXCEPT 1604 {std::__cxx_atomic_notify_one(&__a_);} 1605 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_one() _NOEXCEPT 1606 {std::__cxx_atomic_notify_one(&__a_);} 1607 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() volatile _NOEXCEPT 1608 {std::__cxx_atomic_notify_all(&__a_);} 1609 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY void notify_all() _NOEXCEPT 1610 {std::__cxx_atomic_notify_all(&__a_);} 1611 1612#if _LIBCPP_STD_VER > 17 1613 _LIBCPP_INLINE_VISIBILITY constexpr 1614 __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} 1615#else 1616 _LIBCPP_INLINE_VISIBILITY 1617 __atomic_base() _NOEXCEPT = default; 1618#endif 1619 1620 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 1621 __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} 1622 1623 __atomic_base(const __atomic_base&) = delete; 1624}; 1625 1626#if defined(__cpp_lib_atomic_is_always_lock_free) 1627template <class _Tp, bool __b> 1628_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; 1629#endif 1630 1631// atomic<Integral> 1632 1633template <class _Tp> 1634struct __atomic_base<_Tp, true> 1635 : public __atomic_base<_Tp, false> 1636{ 1637 typedef __atomic_base<_Tp, false> __base; 1638 1639 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20 1640 __atomic_base() _NOEXCEPT = default; 1641 1642 _LIBCPP_INLINE_VISIBILITY 1643 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} 1644 1645 _LIBCPP_INLINE_VISIBILITY 1646 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1647 {return std::__cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1648 _LIBCPP_INLINE_VISIBILITY 1649 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1650 {return std::__cxx_atomic_fetch_add(&this->__a_, __op, __m);} 1651 _LIBCPP_INLINE_VISIBILITY 1652 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1653 {return std::__cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1654 _LIBCPP_INLINE_VISIBILITY 1655 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1656 {return std::__cxx_atomic_fetch_sub(&this->__a_, __op, __m);} 1657 _LIBCPP_INLINE_VISIBILITY 1658 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1659 {return std::__cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1660 _LIBCPP_INLINE_VISIBILITY 1661 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1662 {return std::__cxx_atomic_fetch_and(&this->__a_, __op, __m);} 1663 _LIBCPP_INLINE_VISIBILITY 1664 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1665 {return std::__cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1666 _LIBCPP_INLINE_VISIBILITY 1667 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1668 {return std::__cxx_atomic_fetch_or(&this->__a_, __op, __m);} 1669 _LIBCPP_INLINE_VISIBILITY 1670 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 1671 {return std::__cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1672 _LIBCPP_INLINE_VISIBILITY 1673 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT 1674 {return std::__cxx_atomic_fetch_xor(&this->__a_, __op, __m);} 1675 1676 _LIBCPP_INLINE_VISIBILITY 1677 _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} 1678 _LIBCPP_INLINE_VISIBILITY 1679 _Tp operator++(int) _NOEXCEPT {return fetch_add(_Tp(1));} 1680 _LIBCPP_INLINE_VISIBILITY 1681 _Tp operator--(int) volatile _NOEXCEPT {return fetch_sub(_Tp(1));} 1682 _LIBCPP_INLINE_VISIBILITY 1683 _Tp operator--(int) _NOEXCEPT {return fetch_sub(_Tp(1));} 1684 _LIBCPP_INLINE_VISIBILITY 1685 _Tp operator++() volatile _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1686 _LIBCPP_INLINE_VISIBILITY 1687 _Tp operator++() _NOEXCEPT {return fetch_add(_Tp(1)) + _Tp(1);} 1688 _LIBCPP_INLINE_VISIBILITY 1689 _Tp operator--() volatile _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1690 _LIBCPP_INLINE_VISIBILITY 1691 _Tp operator--() _NOEXCEPT {return fetch_sub(_Tp(1)) - _Tp(1);} 1692 _LIBCPP_INLINE_VISIBILITY 1693 _Tp operator+=(_Tp __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1694 _LIBCPP_INLINE_VISIBILITY 1695 _Tp operator+=(_Tp __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1696 _LIBCPP_INLINE_VISIBILITY 1697 _Tp operator-=(_Tp __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1698 _LIBCPP_INLINE_VISIBILITY 1699 _Tp operator-=(_Tp __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1700 _LIBCPP_INLINE_VISIBILITY 1701 _Tp operator&=(_Tp __op) volatile _NOEXCEPT {return fetch_and(__op) & __op;} 1702 _LIBCPP_INLINE_VISIBILITY 1703 _Tp operator&=(_Tp __op) _NOEXCEPT {return fetch_and(__op) & __op;} 1704 _LIBCPP_INLINE_VISIBILITY 1705 _Tp operator|=(_Tp __op) volatile _NOEXCEPT {return fetch_or(__op) | __op;} 1706 _LIBCPP_INLINE_VISIBILITY 1707 _Tp operator|=(_Tp __op) _NOEXCEPT {return fetch_or(__op) | __op;} 1708 _LIBCPP_INLINE_VISIBILITY 1709 _Tp operator^=(_Tp __op) volatile _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1710 _LIBCPP_INLINE_VISIBILITY 1711 _Tp operator^=(_Tp __op) _NOEXCEPT {return fetch_xor(__op) ^ __op;} 1712}; 1713 1714// atomic<T> 1715 1716template <class _Tp> 1717struct atomic 1718 : public __atomic_base<_Tp> 1719{ 1720 typedef __atomic_base<_Tp> __base; 1721 typedef _Tp value_type; 1722 typedef value_type difference_type; 1723 1724#if _LIBCPP_STD_VER > 17 1725 _LIBCPP_INLINE_VISIBILITY 1726 atomic() = default; 1727#else 1728 _LIBCPP_INLINE_VISIBILITY 1729 atomic() _NOEXCEPT = default; 1730#endif 1731 1732 _LIBCPP_INLINE_VISIBILITY 1733 _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} 1734 1735 _LIBCPP_INLINE_VISIBILITY 1736 _Tp operator=(_Tp __d) volatile _NOEXCEPT 1737 {__base::store(__d); return __d;} 1738 _LIBCPP_INLINE_VISIBILITY 1739 _Tp operator=(_Tp __d) _NOEXCEPT 1740 {__base::store(__d); return __d;} 1741 1742 atomic& operator=(const atomic&) = delete; 1743 atomic& operator=(const atomic&) volatile = delete; 1744}; 1745 1746// atomic<T*> 1747 1748template <class _Tp> 1749struct atomic<_Tp*> 1750 : public __atomic_base<_Tp*> 1751{ 1752 typedef __atomic_base<_Tp*> __base; 1753 typedef _Tp* value_type; 1754 typedef ptrdiff_t difference_type; 1755 1756 _LIBCPP_INLINE_VISIBILITY 1757 atomic() _NOEXCEPT = default; 1758 1759 _LIBCPP_INLINE_VISIBILITY 1760 _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} 1761 1762 _LIBCPP_INLINE_VISIBILITY 1763 _Tp* operator=(_Tp* __d) volatile _NOEXCEPT 1764 {__base::store(__d); return __d;} 1765 _LIBCPP_INLINE_VISIBILITY 1766 _Tp* operator=(_Tp* __d) _NOEXCEPT 1767 {__base::store(__d); return __d;} 1768 1769 _LIBCPP_INLINE_VISIBILITY 1770 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 1771 // __atomic_fetch_add accepts function pointers, guard against them. 1772 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); 1773 return std::__cxx_atomic_fetch_add(&this->__a_, __op, __m); 1774 } 1775 1776 _LIBCPP_INLINE_VISIBILITY 1777 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 1778 // __atomic_fetch_add accepts function pointers, guard against them. 1779 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); 1780 return std::__cxx_atomic_fetch_add(&this->__a_, __op, __m); 1781 } 1782 1783 _LIBCPP_INLINE_VISIBILITY 1784 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT { 1785 // __atomic_fetch_add accepts function pointers, guard against them. 1786 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); 1787 return std::__cxx_atomic_fetch_sub(&this->__a_, __op, __m); 1788 } 1789 1790 _LIBCPP_INLINE_VISIBILITY 1791 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT { 1792 // __atomic_fetch_add accepts function pointers, guard against them. 1793 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed"); 1794 return std::__cxx_atomic_fetch_sub(&this->__a_, __op, __m); 1795 } 1796 1797 _LIBCPP_INLINE_VISIBILITY 1798 _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} 1799 _LIBCPP_INLINE_VISIBILITY 1800 _Tp* operator++(int) _NOEXCEPT {return fetch_add(1);} 1801 _LIBCPP_INLINE_VISIBILITY 1802 _Tp* operator--(int) volatile _NOEXCEPT {return fetch_sub(1);} 1803 _LIBCPP_INLINE_VISIBILITY 1804 _Tp* operator--(int) _NOEXCEPT {return fetch_sub(1);} 1805 _LIBCPP_INLINE_VISIBILITY 1806 _Tp* operator++() volatile _NOEXCEPT {return fetch_add(1) + 1;} 1807 _LIBCPP_INLINE_VISIBILITY 1808 _Tp* operator++() _NOEXCEPT {return fetch_add(1) + 1;} 1809 _LIBCPP_INLINE_VISIBILITY 1810 _Tp* operator--() volatile _NOEXCEPT {return fetch_sub(1) - 1;} 1811 _LIBCPP_INLINE_VISIBILITY 1812 _Tp* operator--() _NOEXCEPT {return fetch_sub(1) - 1;} 1813 _LIBCPP_INLINE_VISIBILITY 1814 _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_add(__op) + __op;} 1815 _LIBCPP_INLINE_VISIBILITY 1816 _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT {return fetch_add(__op) + __op;} 1817 _LIBCPP_INLINE_VISIBILITY 1818 _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT {return fetch_sub(__op) - __op;} 1819 _LIBCPP_INLINE_VISIBILITY 1820 _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT {return fetch_sub(__op) - __op;} 1821 1822 atomic& operator=(const atomic&) = delete; 1823 atomic& operator=(const atomic&) volatile = delete; 1824}; 1825 1826// atomic_is_lock_free 1827 1828template <class _Tp> 1829_LIBCPP_INLINE_VISIBILITY 1830bool 1831atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT 1832{ 1833 return __o->is_lock_free(); 1834} 1835 1836template <class _Tp> 1837_LIBCPP_INLINE_VISIBILITY 1838bool 1839atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT 1840{ 1841 return __o->is_lock_free(); 1842} 1843 1844// atomic_init 1845 1846template <class _Tp> 1847_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY 1848void 1849atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1850{ 1851 std::__cxx_atomic_init(&__o->__a_, __d); 1852} 1853 1854template <class _Tp> 1855_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_INLINE_VISIBILITY 1856void 1857atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1858{ 1859 std::__cxx_atomic_init(&__o->__a_, __d); 1860} 1861 1862// atomic_store 1863 1864template <class _Tp> 1865_LIBCPP_INLINE_VISIBILITY 1866void 1867atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1868{ 1869 __o->store(__d); 1870} 1871 1872template <class _Tp> 1873_LIBCPP_INLINE_VISIBILITY 1874void 1875atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1876{ 1877 __o->store(__d); 1878} 1879 1880// atomic_store_explicit 1881 1882template <class _Tp> 1883_LIBCPP_INLINE_VISIBILITY 1884void 1885atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1886 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1887{ 1888 __o->store(__d, __m); 1889} 1890 1891template <class _Tp> 1892_LIBCPP_INLINE_VISIBILITY 1893void 1894atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1895 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) 1896{ 1897 __o->store(__d, __m); 1898} 1899 1900// atomic_load 1901 1902template <class _Tp> 1903_LIBCPP_INLINE_VISIBILITY 1904_Tp 1905atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT 1906{ 1907 return __o->load(); 1908} 1909 1910template <class _Tp> 1911_LIBCPP_INLINE_VISIBILITY 1912_Tp 1913atomic_load(const atomic<_Tp>* __o) _NOEXCEPT 1914{ 1915 return __o->load(); 1916} 1917 1918// atomic_load_explicit 1919 1920template <class _Tp> 1921_LIBCPP_INLINE_VISIBILITY 1922_Tp 1923atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1924 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1925{ 1926 return __o->load(__m); 1927} 1928 1929template <class _Tp> 1930_LIBCPP_INLINE_VISIBILITY 1931_Tp 1932atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT 1933 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 1934{ 1935 return __o->load(__m); 1936} 1937 1938// atomic_exchange 1939 1940template <class _Tp> 1941_LIBCPP_INLINE_VISIBILITY 1942_Tp 1943atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1944{ 1945 return __o->exchange(__d); 1946} 1947 1948template <class _Tp> 1949_LIBCPP_INLINE_VISIBILITY 1950_Tp 1951atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1952{ 1953 return __o->exchange(__d); 1954} 1955 1956// atomic_exchange_explicit 1957 1958template <class _Tp> 1959_LIBCPP_INLINE_VISIBILITY 1960_Tp 1961atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1962{ 1963 return __o->exchange(__d, __m); 1964} 1965 1966template <class _Tp> 1967_LIBCPP_INLINE_VISIBILITY 1968_Tp 1969atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT 1970{ 1971 return __o->exchange(__d, __m); 1972} 1973 1974// atomic_compare_exchange_weak 1975 1976template <class _Tp> 1977_LIBCPP_INLINE_VISIBILITY 1978bool 1979atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1980{ 1981 return __o->compare_exchange_weak(*__e, __d); 1982} 1983 1984template <class _Tp> 1985_LIBCPP_INLINE_VISIBILITY 1986bool 1987atomic_compare_exchange_weak(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1988{ 1989 return __o->compare_exchange_weak(*__e, __d); 1990} 1991 1992// atomic_compare_exchange_strong 1993 1994template <class _Tp> 1995_LIBCPP_INLINE_VISIBILITY 1996bool 1997atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 1998{ 1999 return __o->compare_exchange_strong(*__e, __d); 2000} 2001 2002template <class _Tp> 2003_LIBCPP_INLINE_VISIBILITY 2004bool 2005atomic_compare_exchange_strong(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT 2006{ 2007 return __o->compare_exchange_strong(*__e, __d); 2008} 2009 2010// atomic_compare_exchange_weak_explicit 2011 2012template <class _Tp> 2013_LIBCPP_INLINE_VISIBILITY 2014bool 2015atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2016 typename atomic<_Tp>::value_type __d, 2017 memory_order __s, memory_order __f) _NOEXCEPT 2018 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2019{ 2020 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2021} 2022 2023template <class _Tp> 2024_LIBCPP_INLINE_VISIBILITY 2025bool 2026atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2027 memory_order __s, memory_order __f) _NOEXCEPT 2028 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2029{ 2030 return __o->compare_exchange_weak(*__e, __d, __s, __f); 2031} 2032 2033// atomic_compare_exchange_strong_explicit 2034 2035template <class _Tp> 2036_LIBCPP_INLINE_VISIBILITY 2037bool 2038atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, 2039 typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d, 2040 memory_order __s, memory_order __f) _NOEXCEPT 2041 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2042{ 2043 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2044} 2045 2046template <class _Tp> 2047_LIBCPP_INLINE_VISIBILITY 2048bool 2049atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, 2050 typename atomic<_Tp>::value_type __d, 2051 memory_order __s, memory_order __f) _NOEXCEPT 2052 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) 2053{ 2054 return __o->compare_exchange_strong(*__e, __d, __s, __f); 2055} 2056 2057// atomic_wait 2058 2059template <class _Tp> 2060_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2061void atomic_wait(const volatile atomic<_Tp>* __o, 2062 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2063{ 2064 return __o->wait(__v); 2065} 2066 2067template <class _Tp> 2068_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2069void atomic_wait(const atomic<_Tp>* __o, 2070 typename atomic<_Tp>::value_type __v) _NOEXCEPT 2071{ 2072 return __o->wait(__v); 2073} 2074 2075// atomic_wait_explicit 2076 2077template <class _Tp> 2078_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2079void atomic_wait_explicit(const volatile atomic<_Tp>* __o, 2080 typename atomic<_Tp>::value_type __v, 2081 memory_order __m) _NOEXCEPT 2082 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2083{ 2084 return __o->wait(__v, __m); 2085} 2086 2087template <class _Tp> 2088_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2089void atomic_wait_explicit(const atomic<_Tp>* __o, 2090 typename atomic<_Tp>::value_type __v, 2091 memory_order __m) _NOEXCEPT 2092 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) 2093{ 2094 return __o->wait(__v, __m); 2095} 2096 2097// atomic_notify_one 2098 2099template <class _Tp> 2100_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2101void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT 2102{ 2103 __o->notify_one(); 2104} 2105template <class _Tp> 2106_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2107void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT 2108{ 2109 __o->notify_one(); 2110} 2111 2112// atomic_notify_all 2113 2114template <class _Tp> 2115_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2116void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT 2117{ 2118 __o->notify_all(); 2119} 2120template <class _Tp> 2121_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2122void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT 2123{ 2124 __o->notify_all(); 2125} 2126 2127// atomic_fetch_add 2128 2129template <class _Tp> 2130_LIBCPP_INLINE_VISIBILITY 2131_Tp 2132atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2133{ 2134 return __o->fetch_add(__op); 2135} 2136 2137template <class _Tp> 2138_LIBCPP_INLINE_VISIBILITY 2139_Tp 2140atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2141{ 2142 return __o->fetch_add(__op); 2143} 2144 2145// atomic_fetch_add_explicit 2146 2147template <class _Tp> 2148_LIBCPP_INLINE_VISIBILITY 2149_Tp atomic_fetch_add_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2150{ 2151 return __o->fetch_add(__op, __m); 2152} 2153 2154template <class _Tp> 2155_LIBCPP_INLINE_VISIBILITY 2156_Tp atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2157{ 2158 return __o->fetch_add(__op, __m); 2159} 2160 2161// atomic_fetch_sub 2162 2163template <class _Tp> 2164_LIBCPP_INLINE_VISIBILITY 2165_Tp atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2166{ 2167 return __o->fetch_sub(__op); 2168} 2169 2170template <class _Tp> 2171_LIBCPP_INLINE_VISIBILITY 2172_Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT 2173{ 2174 return __o->fetch_sub(__op); 2175} 2176 2177// atomic_fetch_sub_explicit 2178 2179template <class _Tp> 2180_LIBCPP_INLINE_VISIBILITY 2181_Tp atomic_fetch_sub_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2182{ 2183 return __o->fetch_sub(__op, __m); 2184} 2185 2186template <class _Tp> 2187_LIBCPP_INLINE_VISIBILITY 2188_Tp atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT 2189{ 2190 return __o->fetch_sub(__op, __m); 2191} 2192 2193// atomic_fetch_and 2194 2195template <class _Tp> 2196_LIBCPP_INLINE_VISIBILITY 2197typename enable_if 2198< 2199 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2200 _Tp 2201>::type 2202atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2203{ 2204 return __o->fetch_and(__op); 2205} 2206 2207template <class _Tp> 2208_LIBCPP_INLINE_VISIBILITY 2209typename enable_if 2210< 2211 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2212 _Tp 2213>::type 2214atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2215{ 2216 return __o->fetch_and(__op); 2217} 2218 2219// atomic_fetch_and_explicit 2220 2221template <class _Tp> 2222_LIBCPP_INLINE_VISIBILITY 2223typename enable_if 2224< 2225 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2226 _Tp 2227>::type 2228atomic_fetch_and_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2229{ 2230 return __o->fetch_and(__op, __m); 2231} 2232 2233template <class _Tp> 2234_LIBCPP_INLINE_VISIBILITY 2235typename enable_if 2236< 2237 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2238 _Tp 2239>::type 2240atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2241{ 2242 return __o->fetch_and(__op, __m); 2243} 2244 2245// atomic_fetch_or 2246 2247template <class _Tp> 2248_LIBCPP_INLINE_VISIBILITY 2249typename enable_if 2250< 2251 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2252 _Tp 2253>::type 2254atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2255{ 2256 return __o->fetch_or(__op); 2257} 2258 2259template <class _Tp> 2260_LIBCPP_INLINE_VISIBILITY 2261typename enable_if 2262< 2263 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2264 _Tp 2265>::type 2266atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2267{ 2268 return __o->fetch_or(__op); 2269} 2270 2271// atomic_fetch_or_explicit 2272 2273template <class _Tp> 2274_LIBCPP_INLINE_VISIBILITY 2275typename enable_if 2276< 2277 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2278 _Tp 2279>::type 2280atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2281{ 2282 return __o->fetch_or(__op, __m); 2283} 2284 2285template <class _Tp> 2286_LIBCPP_INLINE_VISIBILITY 2287typename enable_if 2288< 2289 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2290 _Tp 2291>::type 2292atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2293{ 2294 return __o->fetch_or(__op, __m); 2295} 2296 2297// atomic_fetch_xor 2298 2299template <class _Tp> 2300_LIBCPP_INLINE_VISIBILITY 2301typename enable_if 2302< 2303 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2304 _Tp 2305>::type 2306atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2307{ 2308 return __o->fetch_xor(__op); 2309} 2310 2311template <class _Tp> 2312_LIBCPP_INLINE_VISIBILITY 2313typename enable_if 2314< 2315 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2316 _Tp 2317>::type 2318atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT 2319{ 2320 return __o->fetch_xor(__op); 2321} 2322 2323// atomic_fetch_xor_explicit 2324 2325template <class _Tp> 2326_LIBCPP_INLINE_VISIBILITY 2327typename enable_if 2328< 2329 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2330 _Tp 2331>::type 2332atomic_fetch_xor_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2333{ 2334 return __o->fetch_xor(__op, __m); 2335} 2336 2337template <class _Tp> 2338_LIBCPP_INLINE_VISIBILITY 2339typename enable_if 2340< 2341 is_integral<_Tp>::value && !is_same<_Tp, bool>::value, 2342 _Tp 2343>::type 2344atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT 2345{ 2346 return __o->fetch_xor(__op, __m); 2347} 2348 2349// flag type and operations 2350 2351typedef struct atomic_flag 2352{ 2353 __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; 2354 2355 _LIBCPP_INLINE_VISIBILITY 2356 bool test(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2357 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2358 _LIBCPP_INLINE_VISIBILITY 2359 bool test(memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2360 {return _LIBCPP_ATOMIC_FLAG_TYPE(true) == __cxx_atomic_load(&__a_, __m);} 2361 2362 _LIBCPP_INLINE_VISIBILITY 2363 bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2364 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2365 _LIBCPP_INLINE_VISIBILITY 2366 bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2367 {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} 2368 _LIBCPP_INLINE_VISIBILITY 2369 void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT 2370 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2371 _LIBCPP_INLINE_VISIBILITY 2372 void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT 2373 {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} 2374 2375 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2376 void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT 2377 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2378 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2379 void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT 2380 {__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);} 2381 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2382 void notify_one() volatile _NOEXCEPT 2383 {__cxx_atomic_notify_one(&__a_);} 2384 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2385 void notify_one() _NOEXCEPT 2386 {__cxx_atomic_notify_one(&__a_);} 2387 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2388 void notify_all() volatile _NOEXCEPT 2389 {__cxx_atomic_notify_all(&__a_);} 2390 _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY 2391 void notify_all() _NOEXCEPT 2392 {__cxx_atomic_notify_all(&__a_);} 2393 2394#if _LIBCPP_STD_VER > 17 2395 _LIBCPP_INLINE_VISIBILITY constexpr 2396 atomic_flag() _NOEXCEPT : __a_(false) {} 2397#else 2398 _LIBCPP_INLINE_VISIBILITY 2399 atomic_flag() _NOEXCEPT = default; 2400#endif 2401 2402 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR 2403 atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION 2404 2405 atomic_flag(const atomic_flag&) = delete; 2406 atomic_flag& operator=(const atomic_flag&) = delete; 2407 atomic_flag& operator=(const atomic_flag&) volatile = delete; 2408 2409} atomic_flag; 2410 2411 2412inline _LIBCPP_INLINE_VISIBILITY 2413bool 2414atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT 2415{ 2416 return __o->test(); 2417} 2418 2419inline _LIBCPP_INLINE_VISIBILITY 2420bool 2421atomic_flag_test(const atomic_flag* __o) _NOEXCEPT 2422{ 2423 return __o->test(); 2424} 2425 2426inline _LIBCPP_INLINE_VISIBILITY 2427bool 2428atomic_flag_test_explicit(const volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2429{ 2430 return __o->test(__m); 2431} 2432 2433inline _LIBCPP_INLINE_VISIBILITY 2434bool 2435atomic_flag_test_explicit(const atomic_flag* __o, memory_order __m) _NOEXCEPT 2436{ 2437 return __o->test(__m); 2438} 2439 2440inline _LIBCPP_INLINE_VISIBILITY 2441bool 2442atomic_flag_test_and_set(volatile atomic_flag* __o) _NOEXCEPT 2443{ 2444 return __o->test_and_set(); 2445} 2446 2447inline _LIBCPP_INLINE_VISIBILITY 2448bool 2449atomic_flag_test_and_set(atomic_flag* __o) _NOEXCEPT 2450{ 2451 return __o->test_and_set(); 2452} 2453 2454inline _LIBCPP_INLINE_VISIBILITY 2455bool 2456atomic_flag_test_and_set_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2457{ 2458 return __o->test_and_set(__m); 2459} 2460 2461inline _LIBCPP_INLINE_VISIBILITY 2462bool 2463atomic_flag_test_and_set_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2464{ 2465 return __o->test_and_set(__m); 2466} 2467 2468inline _LIBCPP_INLINE_VISIBILITY 2469void 2470atomic_flag_clear(volatile atomic_flag* __o) _NOEXCEPT 2471{ 2472 __o->clear(); 2473} 2474 2475inline _LIBCPP_INLINE_VISIBILITY 2476void 2477atomic_flag_clear(atomic_flag* __o) _NOEXCEPT 2478{ 2479 __o->clear(); 2480} 2481 2482inline _LIBCPP_INLINE_VISIBILITY 2483void 2484atomic_flag_clear_explicit(volatile atomic_flag* __o, memory_order __m) _NOEXCEPT 2485{ 2486 __o->clear(__m); 2487} 2488 2489inline _LIBCPP_INLINE_VISIBILITY 2490void 2491atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT 2492{ 2493 __o->clear(__m); 2494} 2495 2496inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2497void 2498atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT 2499{ 2500 __o->wait(__v); 2501} 2502 2503inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2504void 2505atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT 2506{ 2507 __o->wait(__v); 2508} 2509 2510inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2511void 2512atomic_flag_wait_explicit(const volatile atomic_flag* __o, 2513 bool __v, memory_order __m) _NOEXCEPT 2514{ 2515 __o->wait(__v, __m); 2516} 2517 2518inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2519void 2520atomic_flag_wait_explicit(const atomic_flag* __o, 2521 bool __v, memory_order __m) _NOEXCEPT 2522{ 2523 __o->wait(__v, __m); 2524} 2525 2526inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2527void 2528atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT 2529{ 2530 __o->notify_one(); 2531} 2532 2533inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2534void 2535atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT 2536{ 2537 __o->notify_one(); 2538} 2539 2540inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2541void 2542atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT 2543{ 2544 __o->notify_all(); 2545} 2546 2547inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_AVAILABILITY_SYNC 2548void 2549atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT 2550{ 2551 __o->notify_all(); 2552} 2553 2554// fences 2555 2556inline _LIBCPP_INLINE_VISIBILITY 2557void 2558atomic_thread_fence(memory_order __m) _NOEXCEPT 2559{ 2560 __cxx_atomic_thread_fence(__m); 2561} 2562 2563inline _LIBCPP_INLINE_VISIBILITY 2564void 2565atomic_signal_fence(memory_order __m) _NOEXCEPT 2566{ 2567 __cxx_atomic_signal_fence(__m); 2568} 2569 2570// Atomics for standard typedef types 2571 2572typedef atomic<bool> atomic_bool; 2573typedef atomic<char> atomic_char; 2574typedef atomic<signed char> atomic_schar; 2575typedef atomic<unsigned char> atomic_uchar; 2576typedef atomic<short> atomic_short; 2577typedef atomic<unsigned short> atomic_ushort; 2578typedef atomic<int> atomic_int; 2579typedef atomic<unsigned int> atomic_uint; 2580typedef atomic<long> atomic_long; 2581typedef atomic<unsigned long> atomic_ulong; 2582typedef atomic<long long> atomic_llong; 2583typedef atomic<unsigned long long> atomic_ullong; 2584#ifndef _LIBCPP_HAS_NO_CHAR8_T 2585typedef atomic<char8_t> atomic_char8_t; 2586#endif 2587typedef atomic<char16_t> atomic_char16_t; 2588typedef atomic<char32_t> atomic_char32_t; 2589#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS 2590typedef atomic<wchar_t> atomic_wchar_t; 2591#endif 2592 2593typedef atomic<int_least8_t> atomic_int_least8_t; 2594typedef atomic<uint_least8_t> atomic_uint_least8_t; 2595typedef atomic<int_least16_t> atomic_int_least16_t; 2596typedef atomic<uint_least16_t> atomic_uint_least16_t; 2597typedef atomic<int_least32_t> atomic_int_least32_t; 2598typedef atomic<uint_least32_t> atomic_uint_least32_t; 2599typedef atomic<int_least64_t> atomic_int_least64_t; 2600typedef atomic<uint_least64_t> atomic_uint_least64_t; 2601 2602typedef atomic<int_fast8_t> atomic_int_fast8_t; 2603typedef atomic<uint_fast8_t> atomic_uint_fast8_t; 2604typedef atomic<int_fast16_t> atomic_int_fast16_t; 2605typedef atomic<uint_fast16_t> atomic_uint_fast16_t; 2606typedef atomic<int_fast32_t> atomic_int_fast32_t; 2607typedef atomic<uint_fast32_t> atomic_uint_fast32_t; 2608typedef atomic<int_fast64_t> atomic_int_fast64_t; 2609typedef atomic<uint_fast64_t> atomic_uint_fast64_t; 2610 2611typedef atomic< int8_t> atomic_int8_t; 2612typedef atomic<uint8_t> atomic_uint8_t; 2613typedef atomic< int16_t> atomic_int16_t; 2614typedef atomic<uint16_t> atomic_uint16_t; 2615typedef atomic< int32_t> atomic_int32_t; 2616typedef atomic<uint32_t> atomic_uint32_t; 2617typedef atomic< int64_t> atomic_int64_t; 2618typedef atomic<uint64_t> atomic_uint64_t; 2619 2620typedef atomic<intptr_t> atomic_intptr_t; 2621typedef atomic<uintptr_t> atomic_uintptr_t; 2622typedef atomic<size_t> atomic_size_t; 2623typedef atomic<ptrdiff_t> atomic_ptrdiff_t; 2624typedef atomic<intmax_t> atomic_intmax_t; 2625typedef atomic<uintmax_t> atomic_uintmax_t; 2626 2627// atomic_*_lock_free : prefer the contention type most highly, then the largest lock-free type 2628 2629#ifdef __cpp_lib_atomic_is_always_lock_free 2630# define _LIBCPP_CONTENTION_LOCK_FREE ::std::__libcpp_is_always_lock_free<__cxx_contention_t>::__value 2631#else 2632# define _LIBCPP_CONTENTION_LOCK_FREE false 2633#endif 2634 2635#if ATOMIC_LLONG_LOCK_FREE == 2 2636typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, long long> __libcpp_signed_lock_free; 2637typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned long long> __libcpp_unsigned_lock_free; 2638#elif ATOMIC_INT_LOCK_FREE == 2 2639typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, int> __libcpp_signed_lock_free; 2640typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned int> __libcpp_unsigned_lock_free; 2641#elif ATOMIC_SHORT_LOCK_FREE == 2 2642typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, short> __libcpp_signed_lock_free; 2643typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned short> __libcpp_unsigned_lock_free; 2644#elif ATOMIC_CHAR_LOCK_FREE == 2 2645typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, char> __libcpp_signed_lock_free; 2646typedef __conditional_t<_LIBCPP_CONTENTION_LOCK_FREE, __cxx_contention_t, unsigned char> __libcpp_unsigned_lock_free; 2647#else 2648 // No signed/unsigned lock-free types 2649#define _LIBCPP_NO_LOCK_FREE_TYPES 2650#endif 2651 2652#if !defined(_LIBCPP_NO_LOCK_FREE_TYPES) 2653typedef atomic<__libcpp_signed_lock_free> atomic_signed_lock_free; 2654typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free; 2655#endif 2656 2657#define ATOMIC_FLAG_INIT {false} 2658#define ATOMIC_VAR_INIT(__v) {__v} 2659 2660#if _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS) 2661# if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1400 2662# pragma clang deprecated(ATOMIC_VAR_INIT) 2663# endif 2664#endif // _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS) 2665 2666_LIBCPP_END_NAMESPACE_STD 2667 2668#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20 2669# include <cmath> 2670# include <compare> 2671# include <type_traits> 2672#endif 2673 2674#endif // _LIBCPP_ATOMIC 2675