1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Queued spinlock 4 * 5 * A 'generic' spinlock implementation that is based on MCS locks. For an 6 * architecture that's looking for a 'generic' spinlock, please first consider 7 * ticket-lock.h and only come looking here when you've considered all the 8 * constraints below and can show your hardware does actually perform better 9 * with qspinlock. 10 * 11 * qspinlock relies on atomic_*_release()/atomic_*_acquire() to be RCsc (or no 12 * weaker than RCtso if you're power), where regular code only expects atomic_t 13 * to be RCpc. 14 * 15 * qspinlock relies on a far greater (compared to asm-generic/spinlock.h) set 16 * of atomic operations to behave well together, please audit them carefully to 17 * ensure they all have forward progress. Many atomic operations may default to 18 * cmpxchg() loops which will not have good forward progress properties on 19 * LL/SC architectures. 20 * 21 * One notable example is atomic_fetch_or_acquire(), which x86 cannot (cheaply) 22 * do. Carefully read the patches that introduced 23 * queued_fetch_set_pending_acquire(). 24 * 25 * qspinlock also heavily relies on mixed size atomic operations, in specific 26 * it requires architectures to have xchg16; something which many LL/SC 27 * architectures need to implement as a 32bit and+or in order to satisfy the 28 * forward progress guarantees mentioned above. 29 * 30 * Further reading on mixed size atomics that might be relevant: 31 * 32 * http://www.cl.cam.ac.uk/~pes20/popl17/mixed-size.pdf 33 * 34 * (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P. 35 * (C) Copyright 2015 Hewlett-Packard Enterprise Development LP 36 * 37 * Authors: Waiman Long <waiman.long@hpe.com> 38 */ 39 #ifndef __ASM_GENERIC_QSPINLOCK_H 40 #define __ASM_GENERIC_QSPINLOCK_H 41 42 #include <asm-generic/qspinlock_types.h> 43 #include <linux/atomic.h> 44 #include <linux/tracepoint-defs.h> 45 46 #ifndef queued_spin_is_locked 47 /** 48 * queued_spin_is_locked - is the spinlock locked? 49 * @lock: Pointer to queued spinlock structure 50 * Return: 1 if it is locked, 0 otherwise 51 */ 52 static __always_inline int queued_spin_is_locked(struct qspinlock *lock) 53 { 54 /* 55 * Any !0 state indicates it is locked, even if _Q_LOCKED_VAL 56 * isn't immediately observable. 57 */ 58 return atomic_read(&lock->val); 59 } 60 #endif 61 62 /** 63 * queued_spin_value_unlocked - is the spinlock structure unlocked? 64 * @lock: queued spinlock structure 65 * Return: 1 if it is unlocked, 0 otherwise 66 * 67 * N.B. Whenever there are tasks waiting for the lock, it is considered 68 * locked wrt the lockref code to avoid lock stealing by the lockref 69 * code and change things underneath the lock. This also allows some 70 * optimizations to be applied without conflict with lockref. 71 */ 72 static __always_inline int queued_spin_value_unlocked(struct qspinlock lock) 73 { 74 return !lock.val.counter; 75 } 76 77 /** 78 * queued_spin_is_contended - check if the lock is contended 79 * @lock : Pointer to queued spinlock structure 80 * Return: 1 if lock contended, 0 otherwise 81 */ 82 static __always_inline int queued_spin_is_contended(struct qspinlock *lock) 83 { 84 return atomic_read(&lock->val) & ~_Q_LOCKED_MASK; 85 } 86 /** 87 * queued_spin_trylock - try to acquire the queued spinlock 88 * @lock : Pointer to queued spinlock structure 89 * Return: 1 if lock acquired, 0 if failed 90 */ 91 static __always_inline int queued_spin_trylock(struct qspinlock *lock) 92 { 93 int val = atomic_read(&lock->val); 94 95 if (unlikely(val)) 96 return 0; 97 98 return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)); 99 } 100 101 extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 102 103 #ifndef queued_spin_lock 104 /** 105 * queued_spin_lock - acquire a queued spinlock 106 * @lock: Pointer to queued spinlock structure 107 */ 108 static __always_inline void queued_spin_lock(struct qspinlock *lock) 109 { 110 int val = 0; 111 112 if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL))) 113 return; 114 115 queued_spin_lock_slowpath(lock, val); 116 } 117 #endif 118 119 #ifndef queued_spin_release 120 /** 121 * queued_spin_release - release a queued spinlock 122 * @lock : Pointer to queued spinlock structure 123 */ 124 static __always_inline void queued_spin_release(struct qspinlock *lock) 125 { 126 /* 127 * unlock() needs release semantics: 128 */ 129 smp_store_release(&lock->locked, 0); 130 } 131 #endif 132 133 #ifndef queued_spin_unlock 134 135 DECLARE_TRACEPOINT(contended_release); 136 137 extern void queued_spin_release_traced(struct qspinlock *lock); 138 139 /** 140 * queued_spin_unlock - unlock a queued spinlock 141 * @lock : Pointer to queued spinlock structure 142 * 143 * Generic tracing wrapper around the arch-overridable 144 * queued_spin_release(). 145 */ 146 static __always_inline void queued_spin_unlock(struct qspinlock *lock) 147 { 148 /* 149 * Trace and release are combined in queued_spin_release_traced() so 150 * the compiler does not need to preserve the lock pointer across the 151 * function call, avoiding callee-saved register save/restore on the 152 * hot path. queued_spin_release() is therefore called both here and in 153 * queued_spin_release_traced(). Keep the two in sync. 154 */ 155 if (IS_ENABLED(CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE) && 156 tracepoint_enabled(contended_release)) { 157 queued_spin_release_traced(lock); 158 return; 159 } 160 queued_spin_release(lock); 161 } 162 #endif 163 164 #ifndef virt_spin_lock 165 static __always_inline bool virt_spin_lock(struct qspinlock *lock) 166 { 167 return false; 168 } 169 #endif 170 171 #ifndef __no_arch_spinlock_redefine 172 /* 173 * Remapping spinlock architecture specific functions to the corresponding 174 * queued spinlock functions. 175 */ 176 #define arch_spin_is_locked(l) queued_spin_is_locked(l) 177 #define arch_spin_is_contended(l) queued_spin_is_contended(l) 178 #define arch_spin_value_unlocked(l) queued_spin_value_unlocked(l) 179 #define arch_spin_lock(l) queued_spin_lock(l) 180 #define arch_spin_trylock(l) queued_spin_trylock(l) 181 #define arch_spin_unlock(l) queued_spin_unlock(l) 182 #endif 183 184 #endif /* __ASM_GENERIC_QSPINLOCK_H */ 185