1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Out of line spinlock code. 4 * 5 * Copyright IBM Corp. 2004, 2006 6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) 7 */ 8 9 #include <linux/types.h> 10 #include <linux/export.h> 11 #include <linux/spinlock.h> 12 #include <linux/jiffies.h> 13 #include <linux/sysctl.h> 14 #include <linux/init.h> 15 #include <linux/smp.h> 16 #include <linux/percpu.h> 17 #include <linux/io.h> 18 #include <asm/alternative.h> 19 #include <asm/machine.h> 20 #include <asm/asm.h> 21 22 int spin_retry = -1; 23 24 static int __init spin_retry_init(void) 25 { 26 if (spin_retry < 0) 27 spin_retry = 1000; 28 return 0; 29 } 30 early_initcall(spin_retry_init); 31 32 /* 33 * spin_retry= parameter 34 */ 35 static int __init spin_retry_setup(char *str) 36 { 37 spin_retry = simple_strtoul(str, &str, 0); 38 return 1; 39 } 40 __setup("spin_retry=", spin_retry_setup); 41 42 static const struct ctl_table s390_spin_sysctl_table[] = { 43 { 44 .procname = "spin_retry", 45 .data = &spin_retry, 46 .maxlen = sizeof(int), 47 .mode = 0644, 48 .proc_handler = proc_dointvec, 49 }, 50 }; 51 52 static int __init init_s390_spin_sysctls(void) 53 { 54 register_sysctl_init("kernel", s390_spin_sysctl_table); 55 return 0; 56 } 57 arch_initcall(init_s390_spin_sysctls); 58 59 struct spin_wait { 60 struct spin_wait *next, *prev; 61 int node_id; 62 } __aligned(32); 63 64 static DEFINE_PER_CPU_ALIGNED(struct spin_wait, spin_wait[4]); 65 66 #define _Q_LOCK_CPU_OFFSET 0 67 #define _Q_LOCK_STEAL_OFFSET 16 68 #define _Q_TAIL_IDX_OFFSET 18 69 #define _Q_TAIL_CPU_OFFSET 20 70 71 #define _Q_LOCK_CPU_MASK 0x0000ffff 72 #define _Q_LOCK_STEAL_ADD 0x00010000 73 #define _Q_LOCK_STEAL_MASK 0x00030000 74 #define _Q_TAIL_IDX_MASK 0x000c0000 75 #define _Q_TAIL_CPU_MASK 0xfff00000 76 77 #define _Q_LOCK_MASK (_Q_LOCK_CPU_MASK | _Q_LOCK_STEAL_MASK) 78 #define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK) 79 80 void arch_spin_lock_setup(int cpu) 81 { 82 struct spin_wait *node; 83 int ix; 84 85 node = per_cpu_ptr(&spin_wait[0], cpu); 86 for (ix = 0; ix < 4; ix++, node++) { 87 memset(node, 0, sizeof(*node)); 88 node->node_id = ((cpu + 1) << _Q_TAIL_CPU_OFFSET) + 89 (ix << _Q_TAIL_IDX_OFFSET); 90 } 91 } 92 93 static inline int arch_load_niai4(int *lock) 94 { 95 int owner; 96 97 asm_inline volatile( 98 ALTERNATIVE("nop", ".insn rre,0xb2fa0000,4,0", ALT_FACILITY(49)) /* NIAI 4 */ 99 " l %[owner],%[lock]\n" 100 : [owner] "=d" (owner) : [lock] "R" (*lock) : "memory"); 101 return owner; 102 } 103 104 #ifdef __HAVE_ASM_FLAG_OUTPUTS__ 105 106 static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new) 107 { 108 int cc; 109 110 asm_inline volatile( 111 ALTERNATIVE("nop", ".insn rre,0xb2fa0000,8,0", ALT_FACILITY(49)) /* NIAI 8 */ 112 " cs %[old],%[new],%[lock]\n" 113 : [old] "+d" (old), [lock] "+Q" (*lock), "=@cc" (cc) 114 : [new] "d" (new) 115 : "memory"); 116 return cc == 0; 117 } 118 119 #else /* __HAVE_ASM_FLAG_OUTPUTS__ */ 120 121 static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new) 122 { 123 int expected = old; 124 125 asm_inline volatile( 126 ALTERNATIVE("nop", ".insn rre,0xb2fa0000,8,0", ALT_FACILITY(49)) /* NIAI 8 */ 127 " cs %[old],%[new],%[lock]\n" 128 : [old] "+d" (old), [lock] "+Q" (*lock) 129 : [new] "d" (new) 130 : "cc", "memory"); 131 return expected == old; 132 } 133 134 #endif /* __HAVE_ASM_FLAG_OUTPUTS__ */ 135 136 static inline struct spin_wait *arch_spin_decode_tail(int lock) 137 { 138 int ix, cpu; 139 140 ix = (lock & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET; 141 cpu = (lock & _Q_TAIL_CPU_MASK) >> _Q_TAIL_CPU_OFFSET; 142 return per_cpu_ptr(&spin_wait[ix], cpu - 1); 143 } 144 145 static inline int arch_spin_yield_target(int lock, struct spin_wait *node) 146 { 147 if (lock & _Q_LOCK_CPU_MASK) 148 return lock & _Q_LOCK_CPU_MASK; 149 if (node == NULL || node->prev == NULL) 150 return 0; /* 0 -> no target cpu */ 151 while (node->prev) 152 node = node->prev; 153 return node->node_id >> _Q_TAIL_CPU_OFFSET; 154 } 155 156 static inline void arch_spin_lock_queued(arch_spinlock_t *lp) 157 { 158 struct spin_wait *node, *next; 159 int lockval, ix, node_id, tail_id, old, new, owner, count; 160 161 ix = get_lowcore()->spinlock_index++; 162 barrier(); 163 lockval = spinlock_lockval(); /* cpu + 1 */ 164 node = this_cpu_ptr(&spin_wait[ix]); 165 node->prev = node->next = NULL; 166 node_id = node->node_id; 167 168 /* Enqueue the node for this CPU in the spinlock wait queue */ 169 old = READ_ONCE(lp->lock); 170 while (1) { 171 if ((old & _Q_LOCK_CPU_MASK) == 0 && 172 (old & _Q_LOCK_STEAL_MASK) != _Q_LOCK_STEAL_MASK) { 173 /* 174 * The lock is free but there may be waiters. 175 * With no waiters simply take the lock, if there 176 * are waiters try to steal the lock. The lock may 177 * be stolen three times before the next queued 178 * waiter will get the lock. 179 */ 180 new = (old ? (old + _Q_LOCK_STEAL_ADD) : 0) | lockval; 181 if (arch_try_cmpxchg(&lp->lock, &old, new)) 182 /* Got the lock */ 183 goto out; 184 /* lock passing in progress */ 185 continue; 186 } 187 /* Make the node of this CPU the new tail. */ 188 new = node_id | (old & _Q_LOCK_MASK); 189 if (arch_try_cmpxchg(&lp->lock, &old, new)) 190 break; 191 } 192 /* Set the 'next' pointer of the tail node in the queue */ 193 tail_id = old & _Q_TAIL_MASK; 194 if (tail_id != 0) { 195 node->prev = arch_spin_decode_tail(tail_id); 196 WRITE_ONCE(node->prev->next, node); 197 } 198 199 /* Pass the virtual CPU to the lock holder if it is not running */ 200 owner = arch_spin_yield_target(old, node); 201 if (owner && arch_vcpu_is_preempted(owner - 1)) 202 smp_yield_cpu(owner - 1); 203 204 /* Spin on the CPU local node->prev pointer */ 205 if (tail_id != 0) { 206 count = spin_retry; 207 while (READ_ONCE(node->prev) != NULL) { 208 if (count-- >= 0) 209 continue; 210 count = spin_retry; 211 /* Query running state of lock holder again. */ 212 owner = arch_spin_yield_target(old, node); 213 if (owner && arch_vcpu_is_preempted(owner - 1)) 214 smp_yield_cpu(owner - 1); 215 } 216 } 217 218 /* Spin on the lock value in the spinlock_t */ 219 count = spin_retry; 220 while (1) { 221 old = READ_ONCE(lp->lock); 222 owner = old & _Q_LOCK_CPU_MASK; 223 if (!owner) { 224 tail_id = old & _Q_TAIL_MASK; 225 new = ((tail_id != node_id) ? tail_id : 0) | lockval; 226 if (arch_try_cmpxchg(&lp->lock, &old, new)) 227 /* Got the lock */ 228 break; 229 continue; 230 } 231 if (count-- >= 0) 232 continue; 233 count = spin_retry; 234 if (!machine_is_lpar() || arch_vcpu_is_preempted(owner - 1)) 235 smp_yield_cpu(owner - 1); 236 } 237 238 /* Pass lock_spin job to next CPU in the queue */ 239 if (node_id && tail_id != node_id) { 240 /* Wait until the next CPU has set up the 'next' pointer */ 241 while ((next = READ_ONCE(node->next)) == NULL) 242 ; 243 next->prev = NULL; 244 } 245 246 out: 247 get_lowcore()->spinlock_index--; 248 } 249 250 static inline void arch_spin_lock_classic(arch_spinlock_t *lp) 251 { 252 int lockval, old, new, owner, count; 253 254 lockval = spinlock_lockval(); /* cpu + 1 */ 255 256 /* Pass the virtual CPU to the lock holder if it is not running */ 257 owner = arch_spin_yield_target(READ_ONCE(lp->lock), NULL); 258 if (owner && arch_vcpu_is_preempted(owner - 1)) 259 smp_yield_cpu(owner - 1); 260 261 count = spin_retry; 262 while (1) { 263 old = arch_load_niai4(&lp->lock); 264 owner = old & _Q_LOCK_CPU_MASK; 265 /* Try to get the lock if it is free. */ 266 if (!owner) { 267 new = (old & _Q_TAIL_MASK) | lockval; 268 if (arch_try_cmpxchg_niai8(&lp->lock, old, new)) { 269 /* Got the lock */ 270 return; 271 } 272 continue; 273 } 274 if (count-- >= 0) 275 continue; 276 count = spin_retry; 277 if (!machine_is_lpar() || arch_vcpu_is_preempted(owner - 1)) 278 smp_yield_cpu(owner - 1); 279 } 280 } 281 282 void arch_spin_lock_wait(arch_spinlock_t *lp) 283 { 284 if (test_cpu_flag(CIF_DEDICATED_CPU)) 285 arch_spin_lock_queued(lp); 286 else 287 arch_spin_lock_classic(lp); 288 } 289 EXPORT_SYMBOL(arch_spin_lock_wait); 290 291 int arch_spin_trylock_retry(arch_spinlock_t *lp) 292 { 293 int cpu = spinlock_lockval(); 294 int owner, count; 295 296 for (count = spin_retry; count > 0; count--) { 297 owner = READ_ONCE(lp->lock); 298 /* Try to get the lock if it is free. */ 299 if (!owner) { 300 if (arch_try_cmpxchg(&lp->lock, &owner, cpu)) 301 return 1; 302 } 303 } 304 return 0; 305 } 306 EXPORT_SYMBOL(arch_spin_trylock_retry); 307 308 void arch_read_lock_wait(arch_rwlock_t *rw) 309 { 310 if (unlikely(in_interrupt())) { 311 while (READ_ONCE(rw->cnts) & 0x10000) 312 barrier(); 313 return; 314 } 315 316 /* Remove this reader again to allow recursive read locking */ 317 __atomic_add_const(-1, &rw->cnts); 318 /* Put the reader into the wait queue */ 319 arch_spin_lock(&rw->wait); 320 /* Now add this reader to the count value again */ 321 __atomic_add_const(1, &rw->cnts); 322 /* Loop until the writer is done */ 323 while (READ_ONCE(rw->cnts) & 0x10000) 324 barrier(); 325 arch_spin_unlock(&rw->wait); 326 } 327 EXPORT_SYMBOL(arch_read_lock_wait); 328 329 void arch_write_lock_wait(arch_rwlock_t *rw) 330 { 331 int old; 332 333 /* Add this CPU to the write waiters */ 334 __atomic_add(0x20000, &rw->cnts); 335 336 /* Put the writer into the wait queue */ 337 arch_spin_lock(&rw->wait); 338 339 while (1) { 340 old = READ_ONCE(rw->cnts); 341 if ((old & 0x1ffff) == 0 && 342 arch_try_cmpxchg(&rw->cnts, &old, old | 0x10000)) 343 /* Got the lock */ 344 break; 345 barrier(); 346 } 347 348 arch_spin_unlock(&rw->wait); 349 } 350 EXPORT_SYMBOL(arch_write_lock_wait); 351 352 void arch_spin_relax(arch_spinlock_t *lp) 353 { 354 int cpu; 355 356 cpu = READ_ONCE(lp->lock) & _Q_LOCK_CPU_MASK; 357 if (!cpu) 358 return; 359 if (machine_is_lpar() && !arch_vcpu_is_preempted(cpu - 1)) 360 return; 361 smp_yield_cpu(cpu - 1); 362 } 363 EXPORT_SYMBOL(arch_spin_relax); 364