1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Out of line spinlock code.
4 *
5 * Copyright IBM Corp. 2004, 2006
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
7 */
8
9 #include <linux/types.h>
10 #include <linux/export.h>
11 #include <linux/spinlock.h>
12 #include <linux/jiffies.h>
13 #include <linux/sysctl.h>
14 #include <linux/init.h>
15 #include <linux/smp.h>
16 #include <linux/percpu.h>
17 #include <linux/io.h>
18 #include <asm/alternative.h>
19 #include <asm/machine.h>
20 #include <asm/asm.h>
21 #include <trace/events/lock.h>
22
23 int spin_retry = -1;
24
spin_retry_init(void)25 static int __init spin_retry_init(void)
26 {
27 if (spin_retry < 0)
28 spin_retry = 1000;
29 return 0;
30 }
31 early_initcall(spin_retry_init);
32
33 /*
34 * spin_retry= parameter
35 */
spin_retry_setup(char * str)36 static int __init spin_retry_setup(char *str)
37 {
38 spin_retry = simple_strtoul(str, &str, 0);
39 return 1;
40 }
41 __setup("spin_retry=", spin_retry_setup);
42
43 static const struct ctl_table s390_spin_sysctl_table[] = {
44 {
45 .procname = "spin_retry",
46 .data = &spin_retry,
47 .maxlen = sizeof(int),
48 .mode = 0644,
49 .proc_handler = proc_dointvec,
50 },
51 };
52
init_s390_spin_sysctls(void)53 static int __init init_s390_spin_sysctls(void)
54 {
55 register_sysctl_init("kernel", s390_spin_sysctl_table);
56 return 0;
57 }
58 arch_initcall(init_s390_spin_sysctls);
59
60 struct spin_wait {
61 struct spin_wait *next, *prev;
62 int node_id;
63 } __aligned(32);
64
65 static DEFINE_PER_CPU_ALIGNED(struct spin_wait, spin_wait[4]);
66
67 #define _Q_LOCK_CPU_OFFSET 0
68 #define _Q_LOCK_STEAL_OFFSET 16
69 #define _Q_TAIL_IDX_OFFSET 18
70 #define _Q_TAIL_CPU_OFFSET 20
71
72 #define _Q_LOCK_CPU_MASK 0x0000ffff
73 #define _Q_LOCK_STEAL_ADD 0x00010000
74 #define _Q_LOCK_STEAL_MASK 0x00030000
75 #define _Q_TAIL_IDX_MASK 0x000c0000
76 #define _Q_TAIL_CPU_MASK 0xfff00000
77
78 #define _Q_LOCK_MASK (_Q_LOCK_CPU_MASK | _Q_LOCK_STEAL_MASK)
79 #define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
80
arch_spin_lock_setup(int cpu)81 void arch_spin_lock_setup(int cpu)
82 {
83 struct spin_wait *node;
84 int ix;
85
86 node = per_cpu_ptr(&spin_wait[0], cpu);
87 for (ix = 0; ix < 4; ix++, node++) {
88 memset(node, 0, sizeof(*node));
89 node->node_id = ((cpu + 1) << _Q_TAIL_CPU_OFFSET) +
90 (ix << _Q_TAIL_IDX_OFFSET);
91 }
92 }
93
arch_load_niai4(int * lock)94 static inline int arch_load_niai4(int *lock)
95 {
96 int owner;
97
98 asm_inline volatile(
99 ALTERNATIVE("nop", ".insn rre,0xb2fa0000,4,0", ALT_FACILITY(49)) /* NIAI 4 */
100 " l %[owner],%[lock]"
101 : [owner] "=d" (owner) : [lock] "R" (*lock) : "memory");
102 return owner;
103 }
104
105 #ifdef __HAVE_ASM_FLAG_OUTPUTS__
106
arch_try_cmpxchg_niai8(int * lock,int old,int new)107 static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new)
108 {
109 int cc;
110
111 asm_inline volatile(
112 ALTERNATIVE("nop", ".insn rre,0xb2fa0000,8,0", ALT_FACILITY(49)) /* NIAI 8 */
113 " cs %[old],%[new],%[lock]"
114 : [old] "+d" (old), [lock] "+Q" (*lock), "=@cc" (cc)
115 : [new] "d" (new)
116 : "memory");
117 return cc == 0;
118 }
119
120 #else /* __HAVE_ASM_FLAG_OUTPUTS__ */
121
arch_try_cmpxchg_niai8(int * lock,int old,int new)122 static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new)
123 {
124 int expected = old;
125
126 asm_inline volatile(
127 ALTERNATIVE("nop", ".insn rre,0xb2fa0000,8,0", ALT_FACILITY(49)) /* NIAI 8 */
128 " cs %[old],%[new],%[lock]"
129 : [old] "+d" (old), [lock] "+Q" (*lock)
130 : [new] "d" (new)
131 : "cc", "memory");
132 return expected == old;
133 }
134
135 #endif /* __HAVE_ASM_FLAG_OUTPUTS__ */
136
arch_spin_decode_tail(int lock)137 static inline struct spin_wait *arch_spin_decode_tail(int lock)
138 {
139 int ix, cpu;
140
141 ix = (lock & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
142 cpu = (lock & _Q_TAIL_CPU_MASK) >> _Q_TAIL_CPU_OFFSET;
143 return per_cpu_ptr(&spin_wait[ix], cpu - 1);
144 }
145
arch_spin_yield_target(int lock,struct spin_wait * node)146 static inline int arch_spin_yield_target(int lock, struct spin_wait *node)
147 {
148 if (lock & _Q_LOCK_CPU_MASK)
149 return lock & _Q_LOCK_CPU_MASK;
150 if (node == NULL || node->prev == NULL)
151 return 0; /* 0 -> no target cpu */
152 while (node->prev)
153 node = node->prev;
154 return node->node_id >> _Q_TAIL_CPU_OFFSET;
155 }
156
arch_spin_lock_queued(arch_spinlock_t * lp)157 static inline void arch_spin_lock_queued(arch_spinlock_t *lp)
158 {
159 struct spin_wait *node, *next;
160 int lockval, ix, node_id, tail_id, old, new, owner, count;
161
162 ix = get_lowcore()->spinlock_index++;
163 barrier();
164 lockval = spinlock_lockval(); /* cpu + 1 */
165 node = this_cpu_ptr(&spin_wait[ix]);
166 node->prev = node->next = NULL;
167 node_id = node->node_id;
168
169 /* Enqueue the node for this CPU in the spinlock wait queue */
170 old = READ_ONCE(lp->lock);
171 while (1) {
172 if ((old & _Q_LOCK_CPU_MASK) == 0 &&
173 (old & _Q_LOCK_STEAL_MASK) != _Q_LOCK_STEAL_MASK) {
174 /*
175 * The lock is free but there may be waiters.
176 * With no waiters simply take the lock, if there
177 * are waiters try to steal the lock. The lock may
178 * be stolen three times before the next queued
179 * waiter will get the lock.
180 */
181 new = (old ? (old + _Q_LOCK_STEAL_ADD) : 0) | lockval;
182 if (arch_try_cmpxchg(&lp->lock, &old, new))
183 /* Got the lock */
184 goto out;
185 /* lock passing in progress */
186 continue;
187 }
188 /* Make the node of this CPU the new tail. */
189 new = node_id | (old & _Q_LOCK_MASK);
190 if (arch_try_cmpxchg(&lp->lock, &old, new))
191 break;
192 }
193 /* Set the 'next' pointer of the tail node in the queue */
194 tail_id = old & _Q_TAIL_MASK;
195 if (tail_id != 0) {
196 node->prev = arch_spin_decode_tail(tail_id);
197 WRITE_ONCE(node->prev->next, node);
198 }
199
200 /* Pass the virtual CPU to the lock holder if it is not running */
201 owner = arch_spin_yield_target(old, node);
202 if (owner && arch_vcpu_is_preempted(owner - 1))
203 smp_yield_cpu(owner - 1);
204
205 /* Spin on the CPU local node->prev pointer */
206 if (tail_id != 0) {
207 count = spin_retry;
208 while (READ_ONCE(node->prev) != NULL) {
209 if (count-- >= 0)
210 continue;
211 count = spin_retry;
212 /* Query running state of lock holder again. */
213 owner = arch_spin_yield_target(old, node);
214 if (owner && arch_vcpu_is_preempted(owner - 1))
215 smp_yield_cpu(owner - 1);
216 }
217 }
218
219 /* Spin on the lock value in the spinlock_t */
220 count = spin_retry;
221 while (1) {
222 old = READ_ONCE(lp->lock);
223 owner = old & _Q_LOCK_CPU_MASK;
224 if (!owner) {
225 tail_id = old & _Q_TAIL_MASK;
226 new = ((tail_id != node_id) ? tail_id : 0) | lockval;
227 if (arch_try_cmpxchg(&lp->lock, &old, new))
228 /* Got the lock */
229 break;
230 continue;
231 }
232 if (count-- >= 0)
233 continue;
234 count = spin_retry;
235 if (!machine_is_lpar() || arch_vcpu_is_preempted(owner - 1))
236 smp_yield_cpu(owner - 1);
237 }
238
239 /* Pass lock_spin job to next CPU in the queue */
240 if (node_id && tail_id != node_id) {
241 /* Wait until the next CPU has set up the 'next' pointer */
242 while ((next = READ_ONCE(node->next)) == NULL)
243 ;
244 next->prev = NULL;
245 }
246
247 out:
248 get_lowcore()->spinlock_index--;
249 }
250
arch_spin_lock_classic(arch_spinlock_t * lp)251 static inline void arch_spin_lock_classic(arch_spinlock_t *lp)
252 {
253 int lockval, old, new, owner, count;
254
255 lockval = spinlock_lockval(); /* cpu + 1 */
256
257 /* Pass the virtual CPU to the lock holder if it is not running */
258 owner = arch_spin_yield_target(READ_ONCE(lp->lock), NULL);
259 if (owner && arch_vcpu_is_preempted(owner - 1))
260 smp_yield_cpu(owner - 1);
261
262 count = spin_retry;
263 while (1) {
264 old = arch_load_niai4(&lp->lock);
265 owner = old & _Q_LOCK_CPU_MASK;
266 /* Try to get the lock if it is free. */
267 if (!owner) {
268 new = (old & _Q_TAIL_MASK) | lockval;
269 if (arch_try_cmpxchg_niai8(&lp->lock, old, new)) {
270 /* Got the lock */
271 return;
272 }
273 continue;
274 }
275 if (count-- >= 0)
276 continue;
277 count = spin_retry;
278 if (!machine_is_lpar() || arch_vcpu_is_preempted(owner - 1))
279 smp_yield_cpu(owner - 1);
280 }
281 }
282
arch_spin_lock_wait(arch_spinlock_t * lp)283 void arch_spin_lock_wait(arch_spinlock_t *lp)
284 {
285 trace_contention_begin(lp, LCB_F_SPIN);
286 if (test_cpu_flag(CIF_DEDICATED_CPU))
287 arch_spin_lock_queued(lp);
288 else
289 arch_spin_lock_classic(lp);
290 trace_contention_end(lp, 0);
291 }
292 EXPORT_SYMBOL(arch_spin_lock_wait);
293
arch_spin_trylock_retry(arch_spinlock_t * lp)294 int arch_spin_trylock_retry(arch_spinlock_t *lp)
295 {
296 int cpu = spinlock_lockval();
297 int owner, count;
298
299 for (count = spin_retry; count > 0; count--) {
300 owner = READ_ONCE(lp->lock);
301 /* Try to get the lock if it is free. */
302 if (!owner) {
303 if (arch_try_cmpxchg(&lp->lock, &owner, cpu))
304 return 1;
305 }
306 }
307 return 0;
308 }
309 EXPORT_SYMBOL(arch_spin_trylock_retry);
310
arch_read_lock_wait(arch_rwlock_t * rw)311 void arch_read_lock_wait(arch_rwlock_t *rw)
312 {
313 if (unlikely(in_interrupt())) {
314 while (READ_ONCE(rw->cnts) & 0x10000)
315 barrier();
316 return;
317 }
318
319 /* Remove this reader again to allow recursive read locking */
320 __atomic_add_const(-1, &rw->cnts);
321 trace_contention_begin(rw, LCB_F_SPIN | LCB_F_READ);
322 /* Put the reader into the wait queue */
323 arch_spin_lock(&rw->wait);
324 /* Now add this reader to the count value again */
325 __atomic_add_const(1, &rw->cnts);
326 /* Loop until the writer is done */
327 while (READ_ONCE(rw->cnts) & 0x10000)
328 barrier();
329 arch_spin_unlock(&rw->wait);
330 trace_contention_end(rw, 0);
331 }
332 EXPORT_SYMBOL(arch_read_lock_wait);
333
arch_write_lock_wait(arch_rwlock_t * rw)334 void arch_write_lock_wait(arch_rwlock_t *rw)
335 {
336 int old;
337
338 trace_contention_begin(rw, LCB_F_SPIN | LCB_F_WRITE);
339 /* Add this CPU to the write waiters */
340 __atomic_add(0x20000, &rw->cnts);
341
342 /* Put the writer into the wait queue */
343 arch_spin_lock(&rw->wait);
344
345 while (1) {
346 old = READ_ONCE(rw->cnts);
347 if ((old & 0x1ffff) == 0 &&
348 arch_try_cmpxchg(&rw->cnts, &old, old | 0x10000))
349 /* Got the lock */
350 break;
351 barrier();
352 }
353
354 arch_spin_unlock(&rw->wait);
355 trace_contention_end(rw, 0);
356 }
357 EXPORT_SYMBOL(arch_write_lock_wait);
358
arch_spin_relax(arch_spinlock_t * lp)359 void arch_spin_relax(arch_spinlock_t *lp)
360 {
361 int cpu;
362
363 cpu = READ_ONCE(lp->lock) & _Q_LOCK_CPU_MASK;
364 if (!cpu)
365 return;
366 if (machine_is_lpar() && !arch_vcpu_is_preempted(cpu - 1))
367 return;
368 smp_yield_cpu(cpu - 1);
369 }
370 EXPORT_SYMBOL(arch_spin_relax);
371