1// SPDX-License-Identifier: GPL-2.0+ 2// 3// An earlier version of this file appeared in the companion webpage for 4// "Frightening small children and disconcerting grown-ups: Concurrency 5// in the Linux kernel" by Alglave, Maranget, McKenney, Parri, and Stern, 6// which appeared in ASPLOS 2018. 7 8// ONCE 9READ_ONCE(X) __load{ONCE}(X) 10WRITE_ONCE(X,V) { __store{ONCE}(X,V); } 11 12// Release Acquire and friends 13smp_store_release(X,V) { __store{RELEASE}(*X,V); } 14smp_load_acquire(X) __load{ACQUIRE}(*X) 15rcu_assign_pointer(X,V) { __store{RELEASE}(X,V); } 16rcu_dereference(X) __load{ONCE}(X) 17smp_store_mb(X,V) { __store{ONCE}(X,V); __fence{MB}; } 18 19// Fences 20smp_mb() { __fence{MB}; } 21smp_rmb() { __fence{rmb}; } 22smp_wmb() { __fence{wmb}; } 23smp_mb__before_atomic() { __fence{before-atomic}; } 24smp_mb__after_atomic() { __fence{after-atomic}; } 25smp_mb__after_spinlock() { __fence{after-spinlock}; } 26smp_mb__after_unlock_lock() { __fence{after-unlock-lock}; } 27smp_mb__after_srcu_read_unlock() { __fence{after-srcu-read-unlock}; } 28barrier() { __fence{barrier}; } 29 30// Exchange 31xchg(X,V) __xchg{MB}(X,V) 32xchg_relaxed(X,V) __xchg{ONCE}(X,V) 33xchg_release(X,V) __xchg{RELEASE}(X,V) 34xchg_acquire(X,V) __xchg{ACQUIRE}(X,V) 35cmpxchg(X,V,W) __cmpxchg{MB}(X,V,W) 36cmpxchg_relaxed(X,V,W) __cmpxchg{ONCE}(X,V,W) 37cmpxchg_acquire(X,V,W) __cmpxchg{ACQUIRE}(X,V,W) 38cmpxchg_release(X,V,W) __cmpxchg{RELEASE}(X,V,W) 39 40// Spinlocks 41spin_lock(X) { __lock(X); } 42spin_unlock(X) { __unlock(X); } 43spin_trylock(X) __trylock(X) 44spin_is_locked(X) __islocked(X) 45 46// RCU 47rcu_read_lock() { __fence{rcu-lock}; } 48rcu_read_unlock() { __fence{rcu-unlock}; } 49synchronize_rcu() { __fence{sync-rcu}; } 50synchronize_rcu_expedited() { __fence{sync-rcu}; } 51 52// SRCU 53srcu_read_lock(X) __load{srcu-lock}(*X) 54srcu_read_unlock(X,Y) { __store{srcu-unlock}(*X,Y); } 55srcu_down_read(X) __load{srcu-lock}(*X) 56srcu_up_read(X,Y) { __store{srcu-unlock}(*X,Y); } 57synchronize_srcu(X) { __srcu{sync-srcu}(X); } 58synchronize_srcu_expedited(X) { __srcu{sync-srcu}(X); } 59 60// Atomic 61atomic_read(X) READ_ONCE(*X) 62atomic_set(X,V) { WRITE_ONCE(*X,V); } 63atomic_read_acquire(X) smp_load_acquire(X) 64atomic_set_release(X,V) { smp_store_release(X,V); } 65 66atomic_add(V,X) { __atomic_op{NORETURN}(X,+,V); } 67atomic_sub(V,X) { __atomic_op{NORETURN}(X,-,V); } 68atomic_and(V,X) { __atomic_op{NORETURN}(X,&,V); } 69atomic_or(V,X) { __atomic_op{NORETURN}(X,|,V); } 70atomic_xor(V,X) { __atomic_op{NORETURN}(X,^,V); } 71atomic_inc(X) { __atomic_op{NORETURN}(X,+,1); } 72atomic_dec(X) { __atomic_op{NORETURN}(X,-,1); } 73atomic_andnot(V,X) { __atomic_op{NORETURN}(X,&~,V); } 74 75atomic_add_return(V,X) __atomic_op_return{MB}(X,+,V) 76atomic_add_return_relaxed(V,X) __atomic_op_return{ONCE}(X,+,V) 77atomic_add_return_acquire(V,X) __atomic_op_return{ACQUIRE}(X,+,V) 78atomic_add_return_release(V,X) __atomic_op_return{RELEASE}(X,+,V) 79atomic_fetch_add(V,X) __atomic_fetch_op{MB}(X,+,V) 80atomic_fetch_add_relaxed(V,X) __atomic_fetch_op{ONCE}(X,+,V) 81atomic_fetch_add_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,+,V) 82atomic_fetch_add_release(V,X) __atomic_fetch_op{RELEASE}(X,+,V) 83 84atomic_fetch_and(V,X) __atomic_fetch_op{MB}(X,&,V) 85atomic_fetch_and_relaxed(V,X) __atomic_fetch_op{ONCE}(X,&,V) 86atomic_fetch_and_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,&,V) 87atomic_fetch_and_release(V,X) __atomic_fetch_op{RELEASE}(X,&,V) 88 89atomic_fetch_or(V,X) __atomic_fetch_op{MB}(X,|,V) 90atomic_fetch_or_relaxed(V,X) __atomic_fetch_op{ONCE}(X,|,V) 91atomic_fetch_or_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,|,V) 92atomic_fetch_or_release(V,X) __atomic_fetch_op{RELEASE}(X,|,V) 93 94atomic_fetch_xor(V,X) __atomic_fetch_op{MB}(X,^,V) 95atomic_fetch_xor_relaxed(V,X) __atomic_fetch_op{ONCE}(X,^,V) 96atomic_fetch_xor_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,^,V) 97atomic_fetch_xor_release(V,X) __atomic_fetch_op{RELEASE}(X,^,V) 98 99atomic_inc_return(X) __atomic_op_return{MB}(X,+,1) 100atomic_inc_return_relaxed(X) __atomic_op_return{ONCE}(X,+,1) 101atomic_inc_return_acquire(X) __atomic_op_return{ACQUIRE}(X,+,1) 102atomic_inc_return_release(X) __atomic_op_return{RELEASE}(X,+,1) 103atomic_fetch_inc(X) __atomic_fetch_op{MB}(X,+,1) 104atomic_fetch_inc_relaxed(X) __atomic_fetch_op{ONCE}(X,+,1) 105atomic_fetch_inc_acquire(X) __atomic_fetch_op{ACQUIRE}(X,+,1) 106atomic_fetch_inc_release(X) __atomic_fetch_op{RELEASE}(X,+,1) 107 108atomic_sub_return(V,X) __atomic_op_return{MB}(X,-,V) 109atomic_sub_return_relaxed(V,X) __atomic_op_return{ONCE}(X,-,V) 110atomic_sub_return_acquire(V,X) __atomic_op_return{ACQUIRE}(X,-,V) 111atomic_sub_return_release(V,X) __atomic_op_return{RELEASE}(X,-,V) 112atomic_fetch_sub(V,X) __atomic_fetch_op{MB}(X,-,V) 113atomic_fetch_sub_relaxed(V,X) __atomic_fetch_op{ONCE}(X,-,V) 114atomic_fetch_sub_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,-,V) 115atomic_fetch_sub_release(V,X) __atomic_fetch_op{RELEASE}(X,-,V) 116 117atomic_dec_return(X) __atomic_op_return{MB}(X,-,1) 118atomic_dec_return_relaxed(X) __atomic_op_return{ONCE}(X,-,1) 119atomic_dec_return_acquire(X) __atomic_op_return{ACQUIRE}(X,-,1) 120atomic_dec_return_release(X) __atomic_op_return{RELEASE}(X,-,1) 121atomic_fetch_dec(X) __atomic_fetch_op{MB}(X,-,1) 122atomic_fetch_dec_relaxed(X) __atomic_fetch_op{ONCE}(X,-,1) 123atomic_fetch_dec_acquire(X) __atomic_fetch_op{ACQUIRE}(X,-,1) 124atomic_fetch_dec_release(X) __atomic_fetch_op{RELEASE}(X,-,1) 125 126atomic_xchg(X,V) __xchg{MB}(X,V) 127atomic_xchg_relaxed(X,V) __xchg{ONCE}(X,V) 128atomic_xchg_release(X,V) __xchg{RELEASE}(X,V) 129atomic_xchg_acquire(X,V) __xchg{ACQUIRE}(X,V) 130atomic_cmpxchg(X,V,W) __cmpxchg{MB}(X,V,W) 131atomic_cmpxchg_relaxed(X,V,W) __cmpxchg{ONCE}(X,V,W) 132atomic_cmpxchg_acquire(X,V,W) __cmpxchg{ACQUIRE}(X,V,W) 133atomic_cmpxchg_release(X,V,W) __cmpxchg{RELEASE}(X,V,W) 134 135atomic_sub_and_test(V,X) __atomic_op_return{MB}(X,-,V) == 0 136atomic_dec_and_test(X) __atomic_op_return{MB}(X,-,1) == 0 137atomic_inc_and_test(X) __atomic_op_return{MB}(X,+,1) == 0 138atomic_add_negative(V,X) __atomic_op_return{MB}(X,+,V) < 0 139atomic_add_negative_relaxed(V,X) __atomic_op_return{ONCE}(X,+,V) < 0 140atomic_add_negative_acquire(V,X) __atomic_op_return{ACQUIRE}(X,+,V) < 0 141atomic_add_negative_release(V,X) __atomic_op_return{RELEASE}(X,+,V) < 0 142 143atomic_fetch_andnot(V,X) __atomic_fetch_op{MB}(X,&~,V) 144atomic_fetch_andnot_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,&~,V) 145atomic_fetch_andnot_release(V,X) __atomic_fetch_op{RELEASE}(X,&~,V) 146atomic_fetch_andnot_relaxed(V,X) __atomic_fetch_op{ONCE}(X,&~,V) 147 148atomic_add_unless(X,V,W) __atomic_add_unless{MB}(X,V,W) 149