1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (2004) Linus Torvalds 4 * 5 * Author: Zwane Mwaikambo <zwane@fsmlabs.com> 6 * 7 * Copyright (2004, 2005) Ingo Molnar 8 * 9 * This file contains the spinlock/rwlock implementations for the 10 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them) 11 * 12 * Note that some architectures have special knowledge about the 13 * stack frames of these functions in their profile_pc. If you 14 * change anything significant here that could change the stack 15 * frame contact the architecture maintainers. 16 */ 17 18 #include <linux/linkage.h> 19 #include <linux/preempt.h> 20 #include <linux/spinlock.h> 21 #include <linux/interrupt.h> 22 #include <linux/debug_locks.h> 23 #include <linux/export.h> 24 25 #ifdef CONFIG_MMIOWB 26 #ifndef arch_mmiowb_state 27 DEFINE_PER_CPU(struct mmiowb_state, __mmiowb_state); 28 EXPORT_PER_CPU_SYMBOL(__mmiowb_state); 29 #endif 30 #endif 31 32 /* 33 * If lockdep is enabled then we use the non-preemption spin-ops 34 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are 35 * not re-enabled during lock-acquire (which the preempt-spin-ops do): 36 */ 37 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) 38 /* 39 * The __lock_function inlines are taken from 40 * spinlock : include/linux/spinlock_api_smp.h 41 * rwlock : include/linux/rwlock_api_smp.h 42 */ 43 #else 44 45 /* 46 * Some architectures can relax in favour of the CPU owning the lock. 47 */ 48 #ifndef arch_read_relax 49 # define arch_read_relax(l) cpu_relax() 50 #endif 51 #ifndef arch_write_relax 52 # define arch_write_relax(l) cpu_relax() 53 #endif 54 #ifndef arch_spin_relax 55 # define arch_spin_relax(l) cpu_relax() 56 #endif 57 58 /* 59 * We build the __lock_function inlines here. They are too large for 60 * inlining all over the place, but here is only one user per function 61 * which embeds them into the calling _lock_function below. 62 * 63 * This could be a long-held lock. We both prepare to spin for a long 64 * time (making _this_ CPU preemptible if possible), and we also signal 65 * towards that other CPU that it should break the lock ASAP. 66 */ 67 #define BUILD_LOCK_OPS(op, locktype, lock_ctx_op) \ 68 static void __lockfunc __raw_##op##_lock(locktype##_t *lock) \ 69 lock_ctx_op(lock) \ 70 { \ 71 for (;;) { \ 72 preempt_disable(); \ 73 if (likely(do_raw_##op##_trylock(lock))) \ 74 break; \ 75 preempt_enable(); \ 76 \ 77 arch_##op##_relax(&lock->raw_lock); \ 78 } \ 79 } \ 80 \ 81 static unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \ 82 lock_ctx_op(lock) \ 83 { \ 84 unsigned long flags; \ 85 \ 86 for (;;) { \ 87 preempt_disable(); \ 88 local_irq_save(flags); \ 89 if (likely(do_raw_##op##_trylock(lock))) \ 90 break; \ 91 local_irq_restore(flags); \ 92 preempt_enable(); \ 93 \ 94 arch_##op##_relax(&lock->raw_lock); \ 95 } \ 96 \ 97 return flags; \ 98 } \ 99 \ 100 static void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \ 101 lock_ctx_op(lock) \ 102 { \ 103 _raw_##op##_lock_irqsave(lock); \ 104 } \ 105 \ 106 static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ 107 lock_ctx_op(lock) \ 108 { \ 109 unsigned long flags; \ 110 \ 111 /* */ \ 112 /* Careful: we must exclude softirqs too, hence the */ \ 113 /* irq-disabling. We use the generic preemption-aware */ \ 114 /* function: */ \ 115 /**/ \ 116 flags = _raw_##op##_lock_irqsave(lock); \ 117 local_bh_disable(); \ 118 local_irq_restore(flags); \ 119 } \ 120 121 /* 122 * Build preemption-friendly versions of the following 123 * lock-spinning functions: 124 * 125 * __[spin|read|write]_lock() 126 * __[spin|read|write]_lock_irq() 127 * __[spin|read|write]_lock_irqsave() 128 * __[spin|read|write]_lock_bh() 129 */ 130 BUILD_LOCK_OPS(spin, raw_spinlock, __acquires); 131 132 /* No rwlock_t variants for now, so just build this function by hand */ 133 static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock) 134 { 135 for (;;) { 136 preempt_disable(); 137 local_interrupt_disable(); 138 if (likely(do_raw_spin_trylock(lock))) 139 break; 140 local_interrupt_enable(); 141 preempt_enable(); 142 143 arch_spin_relax(&lock->raw_lock); 144 } 145 } 146 147 #ifndef CONFIG_PREEMPT_RT 148 BUILD_LOCK_OPS(read, rwlock, __acquires_shared); 149 BUILD_LOCK_OPS(write, rwlock, __acquires); 150 #endif 151 152 #endif 153 154 #ifndef CONFIG_INLINE_SPIN_TRYLOCK 155 noinline int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) 156 { 157 return __raw_spin_trylock(lock); 158 } 159 EXPORT_SYMBOL(_raw_spin_trylock); 160 #endif 161 162 #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH 163 noinline int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) 164 { 165 return __raw_spin_trylock_bh(lock); 166 } 167 EXPORT_SYMBOL(_raw_spin_trylock_bh); 168 #endif 169 170 #ifndef CONFIG_INLINE_SPIN_LOCK 171 noinline void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) 172 { 173 __raw_spin_lock(lock); 174 } 175 EXPORT_SYMBOL(_raw_spin_lock); 176 #endif 177 178 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE 179 noinline unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) 180 { 181 return __raw_spin_lock_irqsave(lock); 182 } 183 EXPORT_SYMBOL(_raw_spin_lock_irqsave); 184 #endif 185 186 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ 187 noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) 188 { 189 __raw_spin_lock_irq(lock); 190 } 191 EXPORT_SYMBOL(_raw_spin_lock_irq); 192 #endif 193 194 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ 195 noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) 196 { 197 __raw_spin_lock_irq_disable(lock); 198 } 199 EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable); 200 #endif 201 202 #ifndef CONFIG_INLINE_SPIN_LOCK_BH 203 noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) 204 { 205 __raw_spin_lock_bh(lock); 206 } 207 EXPORT_SYMBOL(_raw_spin_lock_bh); 208 #endif 209 210 #ifdef CONFIG_UNINLINE_SPIN_UNLOCK 211 noinline void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) 212 { 213 __raw_spin_unlock(lock); 214 } 215 EXPORT_SYMBOL(_raw_spin_unlock); 216 #endif 217 218 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE 219 noinline void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) 220 { 221 __raw_spin_unlock_irqrestore(lock, flags); 222 } 223 EXPORT_SYMBOL(_raw_spin_unlock_irqrestore); 224 #endif 225 226 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ 227 noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) 228 { 229 __raw_spin_unlock_irq(lock); 230 } 231 EXPORT_SYMBOL(_raw_spin_unlock_irq); 232 #endif 233 234 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ 235 noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) 236 { 237 __raw_spin_unlock_irq_enable(lock); 238 } 239 EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable); 240 #endif 241 242 #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH 243 noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) 244 { 245 __raw_spin_unlock_bh(lock); 246 } 247 EXPORT_SYMBOL(_raw_spin_unlock_bh); 248 #endif 249 250 #ifndef CONFIG_PREEMPT_RT 251 252 #ifndef CONFIG_INLINE_READ_TRYLOCK 253 noinline int __lockfunc _raw_read_trylock(rwlock_t *lock) 254 { 255 return __raw_read_trylock(lock); 256 } 257 EXPORT_SYMBOL(_raw_read_trylock); 258 #endif 259 260 #ifndef CONFIG_INLINE_READ_LOCK 261 noinline void __lockfunc _raw_read_lock(rwlock_t *lock) 262 { 263 __raw_read_lock(lock); 264 } 265 EXPORT_SYMBOL(_raw_read_lock); 266 #endif 267 268 #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE 269 noinline unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) 270 { 271 return __raw_read_lock_irqsave(lock); 272 } 273 EXPORT_SYMBOL(_raw_read_lock_irqsave); 274 #endif 275 276 #ifndef CONFIG_INLINE_READ_LOCK_IRQ 277 noinline void __lockfunc _raw_read_lock_irq(rwlock_t *lock) 278 { 279 __raw_read_lock_irq(lock); 280 } 281 EXPORT_SYMBOL(_raw_read_lock_irq); 282 #endif 283 284 #ifndef CONFIG_INLINE_READ_LOCK_BH 285 noinline void __lockfunc _raw_read_lock_bh(rwlock_t *lock) 286 { 287 __raw_read_lock_bh(lock); 288 } 289 EXPORT_SYMBOL(_raw_read_lock_bh); 290 #endif 291 292 #ifndef CONFIG_INLINE_READ_UNLOCK 293 noinline void __lockfunc _raw_read_unlock(rwlock_t *lock) 294 { 295 __raw_read_unlock(lock); 296 } 297 EXPORT_SYMBOL(_raw_read_unlock); 298 #endif 299 300 #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE 301 noinline void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 302 { 303 __raw_read_unlock_irqrestore(lock, flags); 304 } 305 EXPORT_SYMBOL(_raw_read_unlock_irqrestore); 306 #endif 307 308 #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ 309 noinline void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) 310 { 311 __raw_read_unlock_irq(lock); 312 } 313 EXPORT_SYMBOL(_raw_read_unlock_irq); 314 #endif 315 316 #ifndef CONFIG_INLINE_READ_UNLOCK_BH 317 noinline void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) 318 { 319 __raw_read_unlock_bh(lock); 320 } 321 EXPORT_SYMBOL(_raw_read_unlock_bh); 322 #endif 323 324 #ifndef CONFIG_INLINE_WRITE_TRYLOCK 325 noinline int __lockfunc _raw_write_trylock(rwlock_t *lock) 326 { 327 return __raw_write_trylock(lock); 328 } 329 EXPORT_SYMBOL(_raw_write_trylock); 330 #endif 331 332 #ifndef CONFIG_INLINE_WRITE_LOCK 333 noinline void __lockfunc _raw_write_lock(rwlock_t *lock) 334 { 335 __raw_write_lock(lock); 336 } 337 EXPORT_SYMBOL(_raw_write_lock); 338 339 #ifndef CONFIG_DEBUG_LOCK_ALLOC 340 #define __raw_write_lock_nested(lock, subclass) __raw_write_lock(((void)(subclass), (lock))) 341 #endif 342 343 void __lockfunc _raw_write_lock_nested(rwlock_t *lock, int subclass) 344 { 345 __raw_write_lock_nested(lock, subclass); 346 } 347 EXPORT_SYMBOL(_raw_write_lock_nested); 348 #endif 349 350 #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE 351 noinline unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) 352 { 353 return __raw_write_lock_irqsave(lock); 354 } 355 EXPORT_SYMBOL(_raw_write_lock_irqsave); 356 #endif 357 358 #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ 359 noinline void __lockfunc _raw_write_lock_irq(rwlock_t *lock) 360 { 361 __raw_write_lock_irq(lock); 362 } 363 EXPORT_SYMBOL(_raw_write_lock_irq); 364 #endif 365 366 #ifndef CONFIG_INLINE_WRITE_LOCK_BH 367 noinline void __lockfunc _raw_write_lock_bh(rwlock_t *lock) 368 { 369 __raw_write_lock_bh(lock); 370 } 371 EXPORT_SYMBOL(_raw_write_lock_bh); 372 #endif 373 374 #ifndef CONFIG_INLINE_WRITE_UNLOCK 375 noinline void __lockfunc _raw_write_unlock(rwlock_t *lock) 376 { 377 __raw_write_unlock(lock); 378 } 379 EXPORT_SYMBOL(_raw_write_unlock); 380 #endif 381 382 #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE 383 noinline void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 384 { 385 __raw_write_unlock_irqrestore(lock, flags); 386 } 387 EXPORT_SYMBOL(_raw_write_unlock_irqrestore); 388 #endif 389 390 #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ 391 noinline void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) 392 { 393 __raw_write_unlock_irq(lock); 394 } 395 EXPORT_SYMBOL(_raw_write_unlock_irq); 396 #endif 397 398 #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH 399 noinline void __lockfunc _raw_write_unlock_bh(rwlock_t *lock) 400 { 401 __raw_write_unlock_bh(lock); 402 } 403 EXPORT_SYMBOL(_raw_write_unlock_bh); 404 #endif 405 406 #endif /* !CONFIG_PREEMPT_RT */ 407 408 #ifdef CONFIG_DEBUG_LOCK_ALLOC 409 410 void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) 411 { 412 preempt_disable(); 413 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); 414 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); 415 } 416 EXPORT_SYMBOL(_raw_spin_lock_nested); 417 418 unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock, 419 int subclass) 420 { 421 unsigned long flags; 422 423 local_irq_save(flags); 424 preempt_disable(); 425 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); 426 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); 427 return flags; 428 } 429 EXPORT_SYMBOL(_raw_spin_lock_irqsave_nested); 430 431 void __lockfunc _raw_spin_lock_nest_lock(raw_spinlock_t *lock, 432 struct lockdep_map *nest_lock) 433 { 434 preempt_disable(); 435 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); 436 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); 437 } 438 EXPORT_SYMBOL(_raw_spin_lock_nest_lock); 439 440 #endif 441 442 notrace int in_lock_functions(unsigned long addr) 443 { 444 /* Linker adds these: start and end of __lockfunc functions */ 445 extern char __lock_text_start[], __lock_text_end[]; 446 447 return addr >= (unsigned long)__lock_text_start 448 && addr < (unsigned long)__lock_text_end; 449 } 450 EXPORT_SYMBOL(in_lock_functions); 451 452 #if defined(CONFIG_PROVE_LOCKING) && defined(CONFIG_PREEMPT_RT) 453 void notrace lockdep_assert_in_softirq_func(void) 454 { 455 lockdep_assert_in_softirq(); 456 } 457 EXPORT_SYMBOL(lockdep_assert_in_softirq_func); 458 #endif 459