1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * PREEMPT_RT substitution for spin/rw_locks 4 * 5 * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to 6 * resemble the non RT semantics: 7 * 8 * - Contrary to plain rtmutexes, spinlocks and rwlocks are state 9 * preserving. The task state is saved before blocking on the underlying 10 * rtmutex, and restored when the lock has been acquired. Regular wakeups 11 * during that time are redirected to the saved state so no wake up is 12 * missed. 13 * 14 * - Non RT spin/rwlocks disable preemption and eventually interrupts. 15 * Disabling preemption has the side effect of disabling migration and 16 * preventing RCU grace periods. 17 * 18 * The RT substitutions explicitly disable migration and take 19 * rcu_read_lock() across the lock held section. 20 */ 21 #include <linux/spinlock.h> 22 #include <linux/export.h> 23 24 #define RT_MUTEX_BUILD_SPINLOCKS 25 #include "rtmutex.c" 26 27 /* 28 * __might_resched() skips the state check as rtlocks are state 29 * preserving. Take RCU nesting into account as spin/read/write_lock() can 30 * legitimately nest into an RCU read side critical section. 31 */ 32 #define RTLOCK_RESCHED_OFFSETS \ 33 (rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT) 34 35 #define rtlock_might_resched() \ 36 __might_resched(__FILE__, __LINE__, RTLOCK_RESCHED_OFFSETS) 37 38 static __always_inline void rtlock_lock(struct rt_mutex_base *rtm) 39 { 40 lockdep_assert(!current->pi_blocked_on); 41 42 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 43 rtlock_slowlock(rtm); 44 } 45 46 static __always_inline void __rt_spin_lock(spinlock_t *lock) 47 { 48 rtlock_might_resched(); 49 rtlock_lock(&lock->lock); 50 rcu_read_lock(); 51 migrate_disable(); 52 } 53 54 void __sched rt_spin_lock(spinlock_t *lock) __acquires(RCU) 55 { 56 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); 57 __rt_spin_lock(lock); 58 } 59 EXPORT_SYMBOL(rt_spin_lock); 60 61 #ifdef CONFIG_DEBUG_LOCK_ALLOC 62 void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass) 63 { 64 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); 65 __rt_spin_lock(lock); 66 } 67 EXPORT_SYMBOL(rt_spin_lock_nested); 68 69 void __sched rt_spin_lock_nest_lock(spinlock_t *lock, 70 struct lockdep_map *nest_lock) 71 { 72 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); 73 __rt_spin_lock(lock); 74 } 75 EXPORT_SYMBOL(rt_spin_lock_nest_lock); 76 #endif 77 78 void __sched rt_spin_unlock(spinlock_t *lock) __releases(RCU) 79 { 80 spin_release(&lock->dep_map, _RET_IP_); 81 migrate_enable(); 82 83 if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL))) 84 rt_mutex_slowunlock(&lock->lock); 85 86 /* 87 * This must be last to prevent the following UAF: 88 * 89 * T1 T2 90 * spin_lock(&p->lock); rcu_read_lock(); 91 * invalidate(p); p = rcu_dereference(ptr); 92 * rcu_assign_pointer(ptr, NULL); if (!p) return; 93 * spin_unlock(&p->lock); spin_lock(&p->lock); 94 * kfree_rcu(p); rcu_read_unlock(); 95 * .... 96 * spin_unlock(&p->lock) 97 * rcu_read_unlock(); // Ends grace period 98 * rcu_do_batch() 99 * kfree(p); 100 * UAF -> rt_mutex_cmpxchg_release(&p->lock.lock...) 101 */ 102 rcu_read_unlock(); 103 } 104 EXPORT_SYMBOL(rt_spin_unlock); 105 106 /* 107 * Wait for the lock to get unlocked: instead of polling for an unlock 108 * (like raw spinlocks do), lock and unlock, to force the kernel to 109 * schedule if there's contention: 110 */ 111 void __sched rt_spin_lock_unlock(spinlock_t *lock) 112 { 113 spin_lock(lock); 114 spin_unlock(lock); 115 } 116 EXPORT_SYMBOL(rt_spin_lock_unlock); 117 118 static __always_inline int __rt_spin_trylock(spinlock_t *lock) 119 { 120 int ret = 1; 121 122 if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current))) 123 ret = rt_mutex_slowtrylock(&lock->lock); 124 125 if (ret) { 126 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); 127 rcu_read_lock(); 128 migrate_disable(); 129 } 130 return ret; 131 } 132 133 int __sched rt_spin_trylock(spinlock_t *lock) 134 { 135 return __rt_spin_trylock(lock); 136 } 137 EXPORT_SYMBOL(rt_spin_trylock); 138 139 int __sched rt_spin_trylock_bh(spinlock_t *lock) 140 { 141 int ret; 142 143 local_bh_disable(); 144 ret = __rt_spin_trylock(lock); 145 if (!ret) 146 local_bh_enable(); 147 return ret; 148 } 149 EXPORT_SYMBOL(rt_spin_trylock_bh); 150 151 #ifdef CONFIG_DEBUG_LOCK_ALLOC 152 void __rt_spin_lock_init(spinlock_t *lock, const char *name, 153 struct lock_class_key *key, bool percpu) 154 { 155 u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL; 156 157 debug_check_no_locks_freed((void *)lock, sizeof(*lock)); 158 lockdep_init_map_type(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG, 159 LD_WAIT_INV, type); 160 } 161 EXPORT_SYMBOL(__rt_spin_lock_init); 162 #endif 163 164 /* 165 * RT-specific reader/writer locks 166 */ 167 #define rwbase_set_and_save_current_state(state) \ 168 current_save_and_set_rtlock_wait_state() 169 170 #define rwbase_restore_current_state() \ 171 current_restore_rtlock_saved_state() 172 173 static __always_inline int 174 rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state) 175 { 176 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 177 rtlock_slowlock(rtm); 178 return 0; 179 } 180 181 static __always_inline int 182 rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state, 183 struct wake_q_head *wake_q) 184 { 185 rtlock_slowlock_locked(rtm, wake_q); 186 return 0; 187 } 188 189 static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm) 190 { 191 if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL))) 192 return; 193 194 rt_mutex_slowunlock(rtm); 195 } 196 197 static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm) 198 { 199 if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 200 return 1; 201 202 return rt_mutex_slowtrylock(rtm); 203 } 204 205 #define rwbase_signal_pending_state(state, current) (0) 206 207 #define rwbase_pre_schedule() 208 209 #define rwbase_schedule() \ 210 schedule_rtlock() 211 212 #define rwbase_post_schedule() 213 214 #include "rwbase_rt.c" 215 /* 216 * The common functions which get wrapped into the rwlock API. 217 */ 218 int __sched rt_read_trylock(rwlock_t *rwlock) 219 { 220 int ret; 221 222 ret = rwbase_read_trylock(&rwlock->rwbase); 223 if (ret) { 224 rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_); 225 rcu_read_lock(); 226 migrate_disable(); 227 } 228 return ret; 229 } 230 EXPORT_SYMBOL(rt_read_trylock); 231 232 int __sched rt_write_trylock(rwlock_t *rwlock) 233 { 234 int ret; 235 236 ret = rwbase_write_trylock(&rwlock->rwbase); 237 if (ret) { 238 rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); 239 rcu_read_lock(); 240 migrate_disable(); 241 } 242 return ret; 243 } 244 EXPORT_SYMBOL(rt_write_trylock); 245 246 void __sched rt_read_lock(rwlock_t *rwlock) __acquires(RCU) 247 { 248 rtlock_might_resched(); 249 rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_); 250 rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 251 rcu_read_lock(); 252 migrate_disable(); 253 } 254 EXPORT_SYMBOL(rt_read_lock); 255 256 void __sched rt_write_lock(rwlock_t *rwlock) __acquires(RCU) 257 { 258 rtlock_might_resched(); 259 rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); 260 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 261 rcu_read_lock(); 262 migrate_disable(); 263 } 264 EXPORT_SYMBOL(rt_write_lock); 265 266 #ifdef CONFIG_DEBUG_LOCK_ALLOC 267 void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass) __acquires(RCU) 268 { 269 rtlock_might_resched(); 270 rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_); 271 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 272 rcu_read_lock(); 273 migrate_disable(); 274 } 275 EXPORT_SYMBOL(rt_write_lock_nested); 276 #endif 277 278 void __sched rt_read_unlock(rwlock_t *rwlock) __releases(RCU) 279 { 280 rwlock_release(&rwlock->dep_map, _RET_IP_); 281 migrate_enable(); 282 rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 283 284 /* This must be last. See comment in rt_spin_unlock() */ 285 rcu_read_unlock(); 286 } 287 EXPORT_SYMBOL(rt_read_unlock); 288 289 void __sched rt_write_unlock(rwlock_t *rwlock) __releases(RCU) 290 { 291 rwlock_release(&rwlock->dep_map, _RET_IP_); 292 migrate_enable(); 293 rwbase_write_unlock(&rwlock->rwbase); 294 295 /* This must be last. See comment in rt_spin_unlock() */ 296 rcu_read_unlock(); 297 } 298 EXPORT_SYMBOL(rt_write_unlock); 299 300 #ifdef CONFIG_DEBUG_LOCK_ALLOC 301 void __rt_rwlock_init(rwlock_t *rwlock, const char *name, 302 struct lock_class_key *key) 303 { 304 debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock)); 305 lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG); 306 } 307 EXPORT_SYMBOL(__rt_rwlock_init); 308 #endif 309