10b57cec5SDimitry Andric //===-- atomic_helpers.h ----------------------------------------*- C++ -*-===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric 90b57cec5SDimitry Andric #ifndef SCUDO_ATOMIC_H_ 100b57cec5SDimitry Andric #define SCUDO_ATOMIC_H_ 110b57cec5SDimitry Andric 120b57cec5SDimitry Andric #include "internal_defs.h" 130b57cec5SDimitry Andric 140b57cec5SDimitry Andric namespace scudo { 150b57cec5SDimitry Andric 160b57cec5SDimitry Andric enum memory_order { 170b57cec5SDimitry Andric memory_order_relaxed = 0, 180b57cec5SDimitry Andric memory_order_consume = 1, 190b57cec5SDimitry Andric memory_order_acquire = 2, 200b57cec5SDimitry Andric memory_order_release = 3, 210b57cec5SDimitry Andric memory_order_acq_rel = 4, 220b57cec5SDimitry Andric memory_order_seq_cst = 5 230b57cec5SDimitry Andric }; 24480093f4SDimitry Andric static_assert(memory_order_relaxed == __ATOMIC_RELAXED, ""); 25480093f4SDimitry Andric static_assert(memory_order_consume == __ATOMIC_CONSUME, ""); 26480093f4SDimitry Andric static_assert(memory_order_acquire == __ATOMIC_ACQUIRE, ""); 27480093f4SDimitry Andric static_assert(memory_order_release == __ATOMIC_RELEASE, ""); 28480093f4SDimitry Andric static_assert(memory_order_acq_rel == __ATOMIC_ACQ_REL, ""); 29480093f4SDimitry Andric static_assert(memory_order_seq_cst == __ATOMIC_SEQ_CST, ""); 300b57cec5SDimitry Andric 310b57cec5SDimitry Andric struct atomic_u8 { 320b57cec5SDimitry Andric typedef u8 Type; 330b57cec5SDimitry Andric volatile Type ValDoNotUse; 340b57cec5SDimitry Andric }; 350b57cec5SDimitry Andric 360b57cec5SDimitry Andric struct atomic_u16 { 370b57cec5SDimitry Andric typedef u16 Type; 380b57cec5SDimitry Andric volatile Type ValDoNotUse; 390b57cec5SDimitry Andric }; 400b57cec5SDimitry Andric 410b57cec5SDimitry Andric struct atomic_s32 { 420b57cec5SDimitry Andric typedef s32 Type; 430b57cec5SDimitry Andric volatile Type ValDoNotUse; 440b57cec5SDimitry Andric }; 450b57cec5SDimitry Andric 460b57cec5SDimitry Andric struct atomic_u32 { 470b57cec5SDimitry Andric typedef u32 Type; 480b57cec5SDimitry Andric volatile Type ValDoNotUse; 490b57cec5SDimitry Andric }; 500b57cec5SDimitry Andric 510b57cec5SDimitry Andric struct atomic_u64 { 520b57cec5SDimitry Andric typedef u64 Type; 530b57cec5SDimitry Andric // On 32-bit platforms u64 is not necessarily aligned on 8 bytes. 545ffd83dbSDimitry Andric alignas(8) volatile Type ValDoNotUse; 550b57cec5SDimitry Andric }; 560b57cec5SDimitry Andric 570b57cec5SDimitry Andric struct atomic_uptr { 580b57cec5SDimitry Andric typedef uptr Type; 590b57cec5SDimitry Andric volatile Type ValDoNotUse; 600b57cec5SDimitry Andric }; 610b57cec5SDimitry Andric 620b57cec5SDimitry Andric template <typename T> 63480093f4SDimitry Andric inline typename T::Type atomic_load(const volatile T *A, memory_order MO) { 640b57cec5SDimitry Andric DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); 650b57cec5SDimitry Andric typename T::Type V; 660b57cec5SDimitry Andric __atomic_load(&A->ValDoNotUse, &V, MO); 670b57cec5SDimitry Andric return V; 680b57cec5SDimitry Andric } 690b57cec5SDimitry Andric 700b57cec5SDimitry Andric template <typename T> 71480093f4SDimitry Andric inline void atomic_store(volatile T *A, typename T::Type V, memory_order MO) { 720b57cec5SDimitry Andric DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); 730b57cec5SDimitry Andric __atomic_store(&A->ValDoNotUse, &V, MO); 740b57cec5SDimitry Andric } 750b57cec5SDimitry Andric 76480093f4SDimitry Andric inline void atomic_thread_fence(memory_order) { __sync_synchronize(); } 770b57cec5SDimitry Andric 780b57cec5SDimitry Andric template <typename T> 79480093f4SDimitry Andric inline typename T::Type atomic_fetch_add(volatile T *A, typename T::Type V, 800b57cec5SDimitry Andric memory_order MO) { 810b57cec5SDimitry Andric DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); 820b57cec5SDimitry Andric return __atomic_fetch_add(&A->ValDoNotUse, V, MO); 830b57cec5SDimitry Andric } 840b57cec5SDimitry Andric 850b57cec5SDimitry Andric template <typename T> 86480093f4SDimitry Andric inline typename T::Type atomic_fetch_sub(volatile T *A, typename T::Type V, 870b57cec5SDimitry Andric memory_order MO) { 880b57cec5SDimitry Andric DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); 890b57cec5SDimitry Andric return __atomic_fetch_sub(&A->ValDoNotUse, V, MO); 900b57cec5SDimitry Andric } 910b57cec5SDimitry Andric 920b57cec5SDimitry Andric template <typename T> 93e8d8bef9SDimitry Andric inline typename T::Type atomic_fetch_and(volatile T *A, typename T::Type V, 94e8d8bef9SDimitry Andric memory_order MO) { 95e8d8bef9SDimitry Andric DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); 96e8d8bef9SDimitry Andric return __atomic_fetch_and(&A->ValDoNotUse, V, MO); 97e8d8bef9SDimitry Andric } 98e8d8bef9SDimitry Andric 99e8d8bef9SDimitry Andric template <typename T> 100e8d8bef9SDimitry Andric inline typename T::Type atomic_fetch_or(volatile T *A, typename T::Type V, 101e8d8bef9SDimitry Andric memory_order MO) { 102e8d8bef9SDimitry Andric DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); 103e8d8bef9SDimitry Andric return __atomic_fetch_or(&A->ValDoNotUse, V, MO); 104e8d8bef9SDimitry Andric } 105e8d8bef9SDimitry Andric 106e8d8bef9SDimitry Andric template <typename T> 107480093f4SDimitry Andric inline typename T::Type atomic_exchange(volatile T *A, typename T::Type V, 1080b57cec5SDimitry Andric memory_order MO) { 1090b57cec5SDimitry Andric DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A))); 1100b57cec5SDimitry Andric typename T::Type R; 1110b57cec5SDimitry Andric __atomic_exchange(&A->ValDoNotUse, &V, &R, MO); 1120b57cec5SDimitry Andric return R; 1130b57cec5SDimitry Andric } 1140b57cec5SDimitry Andric 1150b57cec5SDimitry Andric template <typename T> 116480093f4SDimitry Andric inline bool atomic_compare_exchange_strong(volatile T *A, typename T::Type *Cmp, 1170b57cec5SDimitry Andric typename T::Type Xchg, 1180b57cec5SDimitry Andric memory_order MO) { 1190b57cec5SDimitry Andric return __atomic_compare_exchange(&A->ValDoNotUse, Cmp, &Xchg, false, MO, 1200b57cec5SDimitry Andric __ATOMIC_RELAXED); 1210b57cec5SDimitry Andric } 1220b57cec5SDimitry Andric 1230b57cec5SDimitry Andric // Clutter-reducing helpers. 1240b57cec5SDimitry Andric 1250b57cec5SDimitry Andric template <typename T> 126480093f4SDimitry Andric inline typename T::Type atomic_load_relaxed(const volatile T *A) { 1270b57cec5SDimitry Andric return atomic_load(A, memory_order_relaxed); 1280b57cec5SDimitry Andric } 1290b57cec5SDimitry Andric 1300b57cec5SDimitry Andric template <typename T> 131480093f4SDimitry Andric inline void atomic_store_relaxed(volatile T *A, typename T::Type V) { 1320b57cec5SDimitry Andric atomic_store(A, V, memory_order_relaxed); 1330b57cec5SDimitry Andric } 1340b57cec5SDimitry Andric 1350b57cec5SDimitry Andric template <typename T> 136*5f757f3fSDimitry Andric inline typename T::Type 137*5f757f3fSDimitry Andric atomic_compare_exchange_strong(volatile T *A, typename T::Type Cmp, 138*5f757f3fSDimitry Andric typename T::Type Xchg, memory_order MO) { 139*5f757f3fSDimitry Andric atomic_compare_exchange_strong(A, &Cmp, Xchg, MO); 1400b57cec5SDimitry Andric return Cmp; 1410b57cec5SDimitry Andric } 1420b57cec5SDimitry Andric 1430b57cec5SDimitry Andric } // namespace scudo 1440b57cec5SDimitry Andric 1450b57cec5SDimitry Andric #endif // SCUDO_ATOMIC_H_ 146