1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef __ASM_SPINLOCK_H 3 #define __ASM_SPINLOCK_H 4 #ifdef __KERNEL__ 5 6 /* 7 * Simple spin lock operations. 8 * 9 * Copyright (C) 2001-2004 Paul Mackerras <paulus@au.ibm.com>, IBM 10 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM 11 * Copyright (C) 2002 Dave Engebretsen <engebret@us.ibm.com>, IBM 12 * Rework to support virtual processors 13 * 14 * Type of int is used as a full 64b word is not necessary. 15 * 16 * (the type definitions are in asm/spinlock_types.h) 17 */ 18 #include <linux/irqflags.h> 19 #ifdef CONFIG_PPC64 20 #include <asm/paca.h> 21 #include <asm/hvcall.h> 22 #endif 23 #include <asm/synch.h> 24 #include <asm/ppc-opcode.h> 25 #include <asm/asm-405.h> 26 27 #ifdef CONFIG_PPC64 28 /* use 0x800000yy when locked, where yy == CPU number */ 29 #ifdef __BIG_ENDIAN__ 30 #define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token)) 31 #else 32 #define LOCK_TOKEN (*(u32 *)(&get_paca()->paca_index)) 33 #endif 34 #else 35 #define LOCK_TOKEN 1 36 #endif 37 38 #ifdef CONFIG_PPC_PSERIES 39 #define vcpu_is_preempted vcpu_is_preempted 40 static inline bool vcpu_is_preempted(int cpu) 41 { 42 if (!firmware_has_feature(FW_FEATURE_SPLPAR)) 43 return false; 44 return !!(be32_to_cpu(lppaca_of(cpu).yield_count) & 1); 45 } 46 #endif 47 48 static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) 49 { 50 return lock.slock == 0; 51 } 52 53 static inline int arch_spin_is_locked(arch_spinlock_t *lock) 54 { 55 smp_mb(); 56 return !arch_spin_value_unlocked(*lock); 57 } 58 59 /* 60 * This returns the old value in the lock, so we succeeded 61 * in getting the lock if the return value is 0. 62 */ 63 static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock) 64 { 65 unsigned long tmp, token; 66 67 token = LOCK_TOKEN; 68 __asm__ __volatile__( 69 "1: " PPC_LWARX(%0,0,%2,1) "\n\ 70 cmpwi 0,%0,0\n\ 71 bne- 2f\n\ 72 stwcx. %1,0,%2\n\ 73 bne- 1b\n" 74 PPC_ACQUIRE_BARRIER 75 "2:" 76 : "=&r" (tmp) 77 : "r" (token), "r" (&lock->slock) 78 : "cr0", "memory"); 79 80 return tmp; 81 } 82 83 static inline int arch_spin_trylock(arch_spinlock_t *lock) 84 { 85 return __arch_spin_trylock(lock) == 0; 86 } 87 88 /* 89 * On a system with shared processors (that is, where a physical 90 * processor is multiplexed between several virtual processors), 91 * there is no point spinning on a lock if the holder of the lock 92 * isn't currently scheduled on a physical processor. Instead 93 * we detect this situation and ask the hypervisor to give the 94 * rest of our timeslice to the lock holder. 95 * 96 * So that we can tell which virtual processor is holding a lock, 97 * we put 0x80000000 | smp_processor_id() in the lock when it is 98 * held. Conveniently, we have a word in the paca that holds this 99 * value. 100 */ 101 102 #if defined(CONFIG_PPC_SPLPAR) 103 /* We only yield to the hypervisor if we are in shared processor mode */ 104 void splpar_spin_yield(arch_spinlock_t *lock); 105 void splpar_rw_yield(arch_rwlock_t *lock); 106 #else /* SPLPAR */ 107 static inline void splpar_spin_yield(arch_spinlock_t *lock) {}; 108 static inline void splpar_rw_yield(arch_rwlock_t *lock) {}; 109 #endif 110 111 static inline bool is_shared_processor(void) 112 { 113 /* 114 * LPPACA is only available on Pseries so guard anything LPPACA related to 115 * allow other platforms (which include this common header) to compile. 116 */ 117 #ifdef CONFIG_PPC_PSERIES 118 return (IS_ENABLED(CONFIG_PPC_SPLPAR) && 119 lppaca_shared_proc(local_paca->lppaca_ptr)); 120 #else 121 return false; 122 #endif 123 } 124 125 static inline void spin_yield(arch_spinlock_t *lock) 126 { 127 if (is_shared_processor()) 128 splpar_spin_yield(lock); 129 else 130 barrier(); 131 } 132 133 static inline void rw_yield(arch_rwlock_t *lock) 134 { 135 if (is_shared_processor()) 136 splpar_rw_yield(lock); 137 else 138 barrier(); 139 } 140 141 static inline void arch_spin_lock(arch_spinlock_t *lock) 142 { 143 while (1) { 144 if (likely(__arch_spin_trylock(lock) == 0)) 145 break; 146 do { 147 HMT_low(); 148 if (is_shared_processor()) 149 splpar_spin_yield(lock); 150 } while (unlikely(lock->slock != 0)); 151 HMT_medium(); 152 } 153 } 154 155 static inline 156 void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long flags) 157 { 158 unsigned long flags_dis; 159 160 while (1) { 161 if (likely(__arch_spin_trylock(lock) == 0)) 162 break; 163 local_save_flags(flags_dis); 164 local_irq_restore(flags); 165 do { 166 HMT_low(); 167 if (is_shared_processor()) 168 splpar_spin_yield(lock); 169 } while (unlikely(lock->slock != 0)); 170 HMT_medium(); 171 local_irq_restore(flags_dis); 172 } 173 } 174 #define arch_spin_lock_flags arch_spin_lock_flags 175 176 static inline void arch_spin_unlock(arch_spinlock_t *lock) 177 { 178 __asm__ __volatile__("# arch_spin_unlock\n\t" 179 PPC_RELEASE_BARRIER: : :"memory"); 180 lock->slock = 0; 181 } 182 183 /* 184 * Read-write spinlocks, allowing multiple readers 185 * but only one writer. 186 * 187 * NOTE! it is quite common to have readers in interrupts 188 * but no interrupt writers. For those circumstances we 189 * can "mix" irq-safe locks - any writer needs to get a 190 * irq-safe write-lock, but readers can get non-irqsafe 191 * read-locks. 192 */ 193 194 #ifdef CONFIG_PPC64 195 #define __DO_SIGN_EXTEND "extsw %0,%0\n" 196 #define WRLOCK_TOKEN LOCK_TOKEN /* it's negative */ 197 #else 198 #define __DO_SIGN_EXTEND 199 #define WRLOCK_TOKEN (-1) 200 #endif 201 202 /* 203 * This returns the old value in the lock + 1, 204 * so we got a read lock if the return value is > 0. 205 */ 206 static inline long __arch_read_trylock(arch_rwlock_t *rw) 207 { 208 long tmp; 209 210 __asm__ __volatile__( 211 "1: " PPC_LWARX(%0,0,%1,1) "\n" 212 __DO_SIGN_EXTEND 213 " addic. %0,%0,1\n\ 214 ble- 2f\n" 215 PPC405_ERR77(0,%1) 216 " stwcx. %0,0,%1\n\ 217 bne- 1b\n" 218 PPC_ACQUIRE_BARRIER 219 "2:" : "=&r" (tmp) 220 : "r" (&rw->lock) 221 : "cr0", "xer", "memory"); 222 223 return tmp; 224 } 225 226 /* 227 * This returns the old value in the lock, 228 * so we got the write lock if the return value is 0. 229 */ 230 static inline long __arch_write_trylock(arch_rwlock_t *rw) 231 { 232 long tmp, token; 233 234 token = WRLOCK_TOKEN; 235 __asm__ __volatile__( 236 "1: " PPC_LWARX(%0,0,%2,1) "\n\ 237 cmpwi 0,%0,0\n\ 238 bne- 2f\n" 239 PPC405_ERR77(0,%1) 240 " stwcx. %1,0,%2\n\ 241 bne- 1b\n" 242 PPC_ACQUIRE_BARRIER 243 "2:" : "=&r" (tmp) 244 : "r" (token), "r" (&rw->lock) 245 : "cr0", "memory"); 246 247 return tmp; 248 } 249 250 static inline void arch_read_lock(arch_rwlock_t *rw) 251 { 252 while (1) { 253 if (likely(__arch_read_trylock(rw) > 0)) 254 break; 255 do { 256 HMT_low(); 257 if (is_shared_processor()) 258 splpar_rw_yield(rw); 259 } while (unlikely(rw->lock < 0)); 260 HMT_medium(); 261 } 262 } 263 264 static inline void arch_write_lock(arch_rwlock_t *rw) 265 { 266 while (1) { 267 if (likely(__arch_write_trylock(rw) == 0)) 268 break; 269 do { 270 HMT_low(); 271 if (is_shared_processor()) 272 splpar_rw_yield(rw); 273 } while (unlikely(rw->lock != 0)); 274 HMT_medium(); 275 } 276 } 277 278 static inline int arch_read_trylock(arch_rwlock_t *rw) 279 { 280 return __arch_read_trylock(rw) > 0; 281 } 282 283 static inline int arch_write_trylock(arch_rwlock_t *rw) 284 { 285 return __arch_write_trylock(rw) == 0; 286 } 287 288 static inline void arch_read_unlock(arch_rwlock_t *rw) 289 { 290 long tmp; 291 292 __asm__ __volatile__( 293 "# read_unlock\n\t" 294 PPC_RELEASE_BARRIER 295 "1: lwarx %0,0,%1\n\ 296 addic %0,%0,-1\n" 297 PPC405_ERR77(0,%1) 298 " stwcx. %0,0,%1\n\ 299 bne- 1b" 300 : "=&r"(tmp) 301 : "r"(&rw->lock) 302 : "cr0", "xer", "memory"); 303 } 304 305 static inline void arch_write_unlock(arch_rwlock_t *rw) 306 { 307 __asm__ __volatile__("# write_unlock\n\t" 308 PPC_RELEASE_BARRIER: : :"memory"); 309 rw->lock = 0; 310 } 311 312 #define arch_spin_relax(lock) spin_yield(lock) 313 #define arch_read_relax(lock) rw_yield(lock) 314 #define arch_write_relax(lock) rw_yield(lock) 315 316 /* See include/linux/spinlock.h */ 317 #define smp_mb__after_spinlock() smp_mb() 318 319 #endif /* __KERNEL__ */ 320 #endif /* __ASM_SPINLOCK_H */ 321