1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Out of line spinlock code. 4 * 5 * Copyright IBM Corp. 2004, 2006 6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) 7 */ 8 9 #include <linux/types.h> 10 #include <linux/export.h> 11 #include <linux/spinlock.h> 12 #include <linux/jiffies.h> 13 #include <linux/sysctl.h> 14 #include <linux/init.h> 15 #include <linux/smp.h> 16 #include <linux/percpu.h> 17 #include <linux/io.h> 18 #include <asm/alternative.h> 19 #include <asm/machine.h> 20 #include <asm/asm.h> 21 #include <trace/events/lock.h> 22 23 int spin_retry = -1; 24 25 static int __init spin_retry_init(void) 26 { 27 if (spin_retry < 0) 28 spin_retry = 1000; 29 return 0; 30 } 31 early_initcall(spin_retry_init); 32 33 /* 34 * spin_retry= parameter 35 */ 36 static int __init spin_retry_setup(char *str) 37 { 38 spin_retry = simple_strtoul(str, &str, 0); 39 return 1; 40 } 41 __setup("spin_retry=", spin_retry_setup); 42 43 static const struct ctl_table s390_spin_sysctl_table[] = { 44 { 45 .procname = "spin_retry", 46 .data = &spin_retry, 47 .maxlen = sizeof(int), 48 .mode = 0644, 49 .proc_handler = proc_dointvec, 50 }, 51 }; 52 53 static int __init init_s390_spin_sysctls(void) 54 { 55 register_sysctl_init("kernel", s390_spin_sysctl_table); 56 return 0; 57 } 58 arch_initcall(init_s390_spin_sysctls); 59 60 struct spin_wait { 61 struct spin_wait *next, *prev; 62 int node_id; 63 } __aligned(32); 64 65 static DEFINE_PER_CPU_ALIGNED(struct spin_wait, spin_wait[4]); 66 67 #define _Q_LOCK_CPU_OFFSET 0 68 #define _Q_LOCK_STEAL_OFFSET 16 69 #define _Q_TAIL_IDX_OFFSET 18 70 #define _Q_TAIL_CPU_OFFSET 20 71 72 #define _Q_LOCK_CPU_MASK 0x0000ffff 73 #define _Q_LOCK_STEAL_ADD 0x00010000 74 #define _Q_LOCK_STEAL_MASK 0x00030000 75 #define _Q_TAIL_IDX_MASK 0x000c0000 76 #define _Q_TAIL_CPU_MASK 0xfff00000 77 78 #define _Q_LOCK_MASK (_Q_LOCK_CPU_MASK | _Q_LOCK_STEAL_MASK) 79 #define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK) 80 81 void arch_spin_lock_setup(int cpu) 82 { 83 struct spin_wait *node; 84 int ix; 85 86 node = per_cpu_ptr(&spin_wait[0], cpu); 87 for (ix = 0; ix < 4; ix++, node++) { 88 memset(node, 0, sizeof(*node)); 89 node->node_id = ((cpu + 1) << _Q_TAIL_CPU_OFFSET) + 90 (ix << _Q_TAIL_IDX_OFFSET); 91 } 92 } 93 94 static inline int arch_load_niai4(int *lock) 95 { 96 int owner; 97 98 asm_inline volatile( 99 ALTERNATIVE("nop", ".insn rre,0xb2fa0000,4,0", ALT_FACILITY(49)) /* NIAI 4 */ 100 " l %[owner],%[lock]" 101 : [owner] "=d" (owner) : [lock] "R" (*lock) : "memory"); 102 return owner; 103 } 104 105 #ifdef __HAVE_ASM_FLAG_OUTPUTS__ 106 107 static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new) 108 { 109 int cc; 110 111 asm_inline volatile( 112 ALTERNATIVE("nop", ".insn rre,0xb2fa0000,8,0", ALT_FACILITY(49)) /* NIAI 8 */ 113 " cs %[old],%[new],%[lock]" 114 : [old] "+d" (old), [lock] "+Q" (*lock), "=@cc" (cc) 115 : [new] "d" (new) 116 : "memory"); 117 return cc == 0; 118 } 119 120 #else /* __HAVE_ASM_FLAG_OUTPUTS__ */ 121 122 static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new) 123 { 124 int expected = old; 125 126 asm_inline volatile( 127 ALTERNATIVE("nop", ".insn rre,0xb2fa0000,8,0", ALT_FACILITY(49)) /* NIAI 8 */ 128 " cs %[old],%[new],%[lock]" 129 : [old] "+d" (old), [lock] "+Q" (*lock) 130 : [new] "d" (new) 131 : "cc", "memory"); 132 return expected == old; 133 } 134 135 #endif /* __HAVE_ASM_FLAG_OUTPUTS__ */ 136 137 static inline struct spin_wait *arch_spin_decode_tail(int lock) 138 { 139 int ix, cpu; 140 141 ix = (lock & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET; 142 cpu = (lock & _Q_TAIL_CPU_MASK) >> _Q_TAIL_CPU_OFFSET; 143 return per_cpu_ptr(&spin_wait[ix], cpu - 1); 144 } 145 146 static inline int arch_spin_yield_target(int lock, struct spin_wait *node) 147 { 148 if (lock & _Q_LOCK_CPU_MASK) 149 return lock & _Q_LOCK_CPU_MASK; 150 if (node == NULL || node->prev == NULL) 151 return 0; /* 0 -> no target cpu */ 152 while (node->prev) 153 node = node->prev; 154 return node->node_id >> _Q_TAIL_CPU_OFFSET; 155 } 156 157 static inline void arch_spin_lock_queued(arch_spinlock_t *lp) 158 { 159 struct spin_wait *node, *next; 160 int lockval, ix, node_id, tail_id, old, new, owner, count; 161 162 ix = get_lowcore()->spinlock_index++; 163 barrier(); 164 lockval = spinlock_lockval(); /* cpu + 1 */ 165 node = this_cpu_ptr(&spin_wait[ix]); 166 node->prev = node->next = NULL; 167 node_id = node->node_id; 168 169 /* Enqueue the node for this CPU in the spinlock wait queue */ 170 old = READ_ONCE(lp->lock); 171 while (1) { 172 if ((old & _Q_LOCK_CPU_MASK) == 0 && 173 (old & _Q_LOCK_STEAL_MASK) != _Q_LOCK_STEAL_MASK) { 174 /* 175 * The lock is free but there may be waiters. 176 * With no waiters simply take the lock, if there 177 * are waiters try to steal the lock. The lock may 178 * be stolen three times before the next queued 179 * waiter will get the lock. 180 */ 181 new = (old ? (old + _Q_LOCK_STEAL_ADD) : 0) | lockval; 182 if (arch_try_cmpxchg(&lp->lock, &old, new)) 183 /* Got the lock */ 184 goto out; 185 /* lock passing in progress */ 186 continue; 187 } 188 /* Make the node of this CPU the new tail. */ 189 new = node_id | (old & _Q_LOCK_MASK); 190 if (arch_try_cmpxchg(&lp->lock, &old, new)) 191 break; 192 } 193 /* Set the 'next' pointer of the tail node in the queue */ 194 tail_id = old & _Q_TAIL_MASK; 195 if (tail_id != 0) { 196 node->prev = arch_spin_decode_tail(tail_id); 197 WRITE_ONCE(node->prev->next, node); 198 } 199 200 /* Pass the virtual CPU to the lock holder if it is not running */ 201 owner = arch_spin_yield_target(old, node); 202 if (owner && arch_vcpu_is_preempted(owner - 1)) 203 smp_yield_cpu(owner - 1); 204 205 /* Spin on the CPU local node->prev pointer */ 206 if (tail_id != 0) { 207 count = spin_retry; 208 while (READ_ONCE(node->prev) != NULL) { 209 if (count-- >= 0) 210 continue; 211 count = spin_retry; 212 /* Query running state of lock holder again. */ 213 owner = arch_spin_yield_target(old, node); 214 if (owner && arch_vcpu_is_preempted(owner - 1)) 215 smp_yield_cpu(owner - 1); 216 } 217 } 218 219 /* Spin on the lock value in the spinlock_t */ 220 count = spin_retry; 221 while (1) { 222 old = READ_ONCE(lp->lock); 223 owner = old & _Q_LOCK_CPU_MASK; 224 if (!owner) { 225 tail_id = old & _Q_TAIL_MASK; 226 new = ((tail_id != node_id) ? tail_id : 0) | lockval; 227 if (arch_try_cmpxchg(&lp->lock, &old, new)) 228 /* Got the lock */ 229 break; 230 continue; 231 } 232 if (count-- >= 0) 233 continue; 234 count = spin_retry; 235 if (!machine_is_lpar() || arch_vcpu_is_preempted(owner - 1)) 236 smp_yield_cpu(owner - 1); 237 } 238 239 /* Pass lock_spin job to next CPU in the queue */ 240 if (node_id && tail_id != node_id) { 241 /* Wait until the next CPU has set up the 'next' pointer */ 242 while ((next = READ_ONCE(node->next)) == NULL) 243 ; 244 next->prev = NULL; 245 } 246 247 out: 248 get_lowcore()->spinlock_index--; 249 } 250 251 static inline void arch_spin_lock_classic(arch_spinlock_t *lp) 252 { 253 int lockval, old, new, owner, count; 254 255 lockval = spinlock_lockval(); /* cpu + 1 */ 256 257 /* Pass the virtual CPU to the lock holder if it is not running */ 258 owner = arch_spin_yield_target(READ_ONCE(lp->lock), NULL); 259 if (owner && arch_vcpu_is_preempted(owner - 1)) 260 smp_yield_cpu(owner - 1); 261 262 count = spin_retry; 263 while (1) { 264 old = arch_load_niai4(&lp->lock); 265 owner = old & _Q_LOCK_CPU_MASK; 266 /* Try to get the lock if it is free. */ 267 if (!owner) { 268 new = (old & _Q_TAIL_MASK) | lockval; 269 if (arch_try_cmpxchg_niai8(&lp->lock, old, new)) { 270 /* Got the lock */ 271 return; 272 } 273 continue; 274 } 275 if (count-- >= 0) 276 continue; 277 count = spin_retry; 278 if (!machine_is_lpar() || arch_vcpu_is_preempted(owner - 1)) 279 smp_yield_cpu(owner - 1); 280 } 281 } 282 283 void arch_spin_lock_wait(arch_spinlock_t *lp) 284 { 285 trace_contention_begin(lp, LCB_F_SPIN); 286 if (test_cpu_flag(CIF_DEDICATED_CPU)) 287 arch_spin_lock_queued(lp); 288 else 289 arch_spin_lock_classic(lp); 290 trace_contention_end(lp, 0); 291 } 292 EXPORT_SYMBOL(arch_spin_lock_wait); 293 294 int arch_spin_trylock_retry(arch_spinlock_t *lp) 295 { 296 int cpu = spinlock_lockval(); 297 int owner, count; 298 299 for (count = spin_retry; count > 0; count--) { 300 owner = READ_ONCE(lp->lock); 301 /* Try to get the lock if it is free. */ 302 if (!owner) { 303 if (arch_try_cmpxchg(&lp->lock, &owner, cpu)) 304 return 1; 305 } 306 } 307 return 0; 308 } 309 EXPORT_SYMBOL(arch_spin_trylock_retry); 310 311 void arch_read_lock_wait(arch_rwlock_t *rw) 312 { 313 if (unlikely(in_interrupt())) { 314 while (READ_ONCE(rw->cnts) & 0x10000) 315 barrier(); 316 return; 317 } 318 319 /* Remove this reader again to allow recursive read locking */ 320 __atomic_add_const(-1, &rw->cnts); 321 /* Put the reader into the wait queue */ 322 arch_spin_lock(&rw->wait); 323 /* Now add this reader to the count value again */ 324 __atomic_add_const(1, &rw->cnts); 325 /* Loop until the writer is done */ 326 while (READ_ONCE(rw->cnts) & 0x10000) 327 barrier(); 328 arch_spin_unlock(&rw->wait); 329 } 330 EXPORT_SYMBOL(arch_read_lock_wait); 331 332 void arch_write_lock_wait(arch_rwlock_t *rw) 333 { 334 int old; 335 336 /* Add this CPU to the write waiters */ 337 __atomic_add(0x20000, &rw->cnts); 338 339 /* Put the writer into the wait queue */ 340 arch_spin_lock(&rw->wait); 341 342 while (1) { 343 old = READ_ONCE(rw->cnts); 344 if ((old & 0x1ffff) == 0 && 345 arch_try_cmpxchg(&rw->cnts, &old, old | 0x10000)) 346 /* Got the lock */ 347 break; 348 barrier(); 349 } 350 351 arch_spin_unlock(&rw->wait); 352 } 353 EXPORT_SYMBOL(arch_write_lock_wait); 354 355 void arch_spin_relax(arch_spinlock_t *lp) 356 { 357 int cpu; 358 359 cpu = READ_ONCE(lp->lock) & _Q_LOCK_CPU_MASK; 360 if (!cpu) 361 return; 362 if (machine_is_lpar() && !arch_vcpu_is_preempted(cpu - 1)) 363 return; 364 smp_yield_cpu(cpu - 1); 365 } 366 EXPORT_SYMBOL(arch_spin_relax); 367