1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * PREEMPT_RT substitution for spin/rw_locks 4 * 5 * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to 6 * resemble the non RT semantics: 7 * 8 * - Contrary to plain rtmutexes, spinlocks and rwlocks are state 9 * preserving. The task state is saved before blocking on the underlying 10 * rtmutex, and restored when the lock has been acquired. Regular wakeups 11 * during that time are redirected to the saved state so no wake up is 12 * missed. 13 * 14 * - Non RT spin/rwlocks disable preemption and eventually interrupts. 15 * Disabling preemption has the side effect of disabling migration and 16 * preventing RCU grace periods. 17 * 18 * The RT substitutions explicitly disable migration and take 19 * rcu_read_lock() across the lock held section. 20 */ 21 #include <linux/spinlock.h> 22 #include <linux/export.h> 23 24 #define RT_MUTEX_BUILD_SPINLOCKS 25 #include "rtmutex.c" 26 27 /* 28 * __might_resched() skips the state check as rtlocks are state 29 * preserving. Take RCU nesting into account as spin/read/write_lock() can 30 * legitimately nest into an RCU read side critical section. 31 */ 32 #define RTLOCK_RESCHED_OFFSETS \ 33 (rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT) 34 35 #define rtlock_might_resched() \ 36 __might_resched(__FILE__, __LINE__, RTLOCK_RESCHED_OFFSETS) 37 38 static __always_inline void rtlock_lock(struct rt_mutex_base *rtm) 39 { 40 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 41 rtlock_slowlock(rtm); 42 } 43 44 static __always_inline void __rt_spin_lock(spinlock_t *lock) 45 { 46 rtlock_might_resched(); 47 rtlock_lock(&lock->lock); 48 rcu_read_lock(); 49 migrate_disable(); 50 } 51 52 void __sched rt_spin_lock(spinlock_t *lock) 53 { 54 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); 55 __rt_spin_lock(lock); 56 } 57 EXPORT_SYMBOL(rt_spin_lock); 58 59 #ifdef CONFIG_DEBUG_LOCK_ALLOC 60 void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass) 61 { 62 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); 63 __rt_spin_lock(lock); 64 } 65 EXPORT_SYMBOL(rt_spin_lock_nested); 66 67 void __sched rt_spin_lock_nest_lock(spinlock_t *lock, 68 struct lockdep_map *nest_lock) 69 { 70 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); 71 __rt_spin_lock(lock); 72 } 73 EXPORT_SYMBOL(rt_spin_lock_nest_lock); 74 #endif 75 76 void __sched rt_spin_unlock(spinlock_t *lock) 77 { 78 spin_release(&lock->dep_map, _RET_IP_); 79 migrate_enable(); 80 rcu_read_unlock(); 81 82 if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL))) 83 rt_mutex_slowunlock(&lock->lock); 84 } 85 EXPORT_SYMBOL(rt_spin_unlock); 86 87 /* 88 * Wait for the lock to get unlocked: instead of polling for an unlock 89 * (like raw spinlocks do), lock and unlock, to force the kernel to 90 * schedule if there's contention: 91 */ 92 void __sched rt_spin_lock_unlock(spinlock_t *lock) 93 { 94 spin_lock(lock); 95 spin_unlock(lock); 96 } 97 EXPORT_SYMBOL(rt_spin_lock_unlock); 98 99 static __always_inline int __rt_spin_trylock(spinlock_t *lock) 100 { 101 int ret = 1; 102 103 if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current))) 104 ret = rt_mutex_slowtrylock(&lock->lock); 105 106 if (ret) { 107 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); 108 rcu_read_lock(); 109 migrate_disable(); 110 } 111 return ret; 112 } 113 114 int __sched rt_spin_trylock(spinlock_t *lock) 115 { 116 return __rt_spin_trylock(lock); 117 } 118 EXPORT_SYMBOL(rt_spin_trylock); 119 120 int __sched rt_spin_trylock_bh(spinlock_t *lock) 121 { 122 int ret; 123 124 local_bh_disable(); 125 ret = __rt_spin_trylock(lock); 126 if (!ret) 127 local_bh_enable(); 128 return ret; 129 } 130 EXPORT_SYMBOL(rt_spin_trylock_bh); 131 132 #ifdef CONFIG_DEBUG_LOCK_ALLOC 133 void __rt_spin_lock_init(spinlock_t *lock, const char *name, 134 struct lock_class_key *key, bool percpu) 135 { 136 u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL; 137 138 debug_check_no_locks_freed((void *)lock, sizeof(*lock)); 139 lockdep_init_map_type(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG, 140 LD_WAIT_INV, type); 141 } 142 EXPORT_SYMBOL(__rt_spin_lock_init); 143 #endif 144 145 /* 146 * RT-specific reader/writer locks 147 */ 148 #define rwbase_set_and_save_current_state(state) \ 149 current_save_and_set_rtlock_wait_state() 150 151 #define rwbase_restore_current_state() \ 152 current_restore_rtlock_saved_state() 153 154 static __always_inline int 155 rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state) 156 { 157 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 158 rtlock_slowlock(rtm); 159 return 0; 160 } 161 162 static __always_inline int 163 rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state) 164 { 165 rtlock_slowlock_locked(rtm); 166 return 0; 167 } 168 169 static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm) 170 { 171 if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL))) 172 return; 173 174 rt_mutex_slowunlock(rtm); 175 } 176 177 static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm) 178 { 179 if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 180 return 1; 181 182 return rt_mutex_slowtrylock(rtm); 183 } 184 185 #define rwbase_signal_pending_state(state, current) (0) 186 187 #define rwbase_schedule() \ 188 schedule_rtlock() 189 190 #include "rwbase_rt.c" 191 /* 192 * The common functions which get wrapped into the rwlock API. 193 */ 194 int __sched rt_read_trylock(rwlock_t *rwlock) 195 { 196 int ret; 197 198 ret = rwbase_read_trylock(&rwlock->rwbase); 199 if (ret) { 200 rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_); 201 rcu_read_lock(); 202 migrate_disable(); 203 } 204 return ret; 205 } 206 EXPORT_SYMBOL(rt_read_trylock); 207 208 int __sched rt_write_trylock(rwlock_t *rwlock) 209 { 210 int ret; 211 212 ret = rwbase_write_trylock(&rwlock->rwbase); 213 if (ret) { 214 rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); 215 rcu_read_lock(); 216 migrate_disable(); 217 } 218 return ret; 219 } 220 EXPORT_SYMBOL(rt_write_trylock); 221 222 void __sched rt_read_lock(rwlock_t *rwlock) 223 { 224 rtlock_might_resched(); 225 rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_); 226 rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 227 rcu_read_lock(); 228 migrate_disable(); 229 } 230 EXPORT_SYMBOL(rt_read_lock); 231 232 void __sched rt_write_lock(rwlock_t *rwlock) 233 { 234 rtlock_might_resched(); 235 rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); 236 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 237 rcu_read_lock(); 238 migrate_disable(); 239 } 240 EXPORT_SYMBOL(rt_write_lock); 241 242 #ifdef CONFIG_DEBUG_LOCK_ALLOC 243 void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass) 244 { 245 rtlock_might_resched(); 246 rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_); 247 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 248 rcu_read_lock(); 249 migrate_disable(); 250 } 251 EXPORT_SYMBOL(rt_write_lock_nested); 252 #endif 253 254 void __sched rt_read_unlock(rwlock_t *rwlock) 255 { 256 rwlock_release(&rwlock->dep_map, _RET_IP_); 257 migrate_enable(); 258 rcu_read_unlock(); 259 rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 260 } 261 EXPORT_SYMBOL(rt_read_unlock); 262 263 void __sched rt_write_unlock(rwlock_t *rwlock) 264 { 265 rwlock_release(&rwlock->dep_map, _RET_IP_); 266 rcu_read_unlock(); 267 migrate_enable(); 268 rwbase_write_unlock(&rwlock->rwbase); 269 } 270 EXPORT_SYMBOL(rt_write_unlock); 271 272 #ifdef CONFIG_DEBUG_LOCK_ALLOC 273 void __rt_rwlock_init(rwlock_t *rwlock, const char *name, 274 struct lock_class_key *key) 275 { 276 debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock)); 277 lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG); 278 } 279 EXPORT_SYMBOL(__rt_rwlock_init); 280 #endif 281