xref: /linux/arch/mips/include/asm/spinlock.h (revision c4ee0af3fa0dc65f690fc908f02b8355f9576ea0) 
1 /*
2  * This file is subject to the terms and conditions of the GNU General Public
3  * License.  See the file "COPYING" in the main directory of this archive
4  * for more details.
5  *
6  * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8  */
9 #ifndef _ASM_SPINLOCK_H
10 #define _ASM_SPINLOCK_H
11 
12 #include <linux/compiler.h>
13 
14 #include <asm/barrier.h>
15 #include <asm/war.h>
16 
17 /*
18  * Your basic SMP spinlocks, allowing only a single CPU anywhere
19  *
20  * Simple spin lock operations.	 There are two variants, one clears IRQ's
21  * on the local processor, one does not.
22  *
23  * These are fair FIFO ticket locks
24  *
25  * (the type definitions are in asm/spinlock_types.h)
26  */
27 
28 
29 /*
30  * Ticket locks are conceptually two parts, one indicating the current head of
31  * the queue, and the other indicating the current tail. The lock is acquired
32  * by atomically noting the tail and incrementing it by one (thus adding
33  * ourself to the queue and noting our position), then waiting until the head
34  * becomes equal to the the initial value of the tail.
35  */
36 
37 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
38 {
39 	u32 counters = ACCESS_ONCE(lock->lock);
40 
41 	return ((counters >> 16) ^ counters) & 0xffff;
42 }
43 
44 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
45 #define arch_spin_unlock_wait(x) \
46 	while (arch_spin_is_locked(x)) { cpu_relax(); }
47 
48 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
49 {
50 	u32 counters = ACCESS_ONCE(lock->lock);
51 
52 	return (((counters >> 16) - counters) & 0xffff) > 1;
53 }
54 #define arch_spin_is_contended	arch_spin_is_contended
55 
56 static inline void arch_spin_lock(arch_spinlock_t *lock)
57 {
58 	int my_ticket;
59 	int tmp;
60 	int inc = 0x10000;
61 
62 	if (R10000_LLSC_WAR) {
63 		__asm__ __volatile__ (
64 		"	.set push		# arch_spin_lock	\n"
65 		"	.set noreorder					\n"
66 		"							\n"
67 		"1:	ll	%[ticket], %[ticket_ptr]		\n"
68 		"	addu	%[my_ticket], %[ticket], %[inc]		\n"
69 		"	sc	%[my_ticket], %[ticket_ptr]		\n"
70 		"	beqzl	%[my_ticket], 1b			\n"
71 		"	 nop						\n"
72 		"	srl	%[my_ticket], %[ticket], 16		\n"
73 		"	andi	%[ticket], %[ticket], 0xffff		\n"
74 		"	bne	%[ticket], %[my_ticket], 4f		\n"
75 		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
76 		"2:							\n"
77 		"	.subsection 2					\n"
78 		"4:	andi	%[ticket], %[ticket], 0xffff		\n"
79 		"	sll	%[ticket], 5				\n"
80 		"							\n"
81 		"6:	bnez	%[ticket], 6b				\n"
82 		"	 subu	%[ticket], 1				\n"
83 		"							\n"
84 		"	lhu	%[ticket], %[serving_now_ptr]		\n"
85 		"	beq	%[ticket], %[my_ticket], 2b		\n"
86 		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
87 		"	b	4b					\n"
88 		"	 subu	%[ticket], %[ticket], 1			\n"
89 		"	.previous					\n"
90 		"	.set pop					\n"
91 		: [ticket_ptr] "+m" (lock->lock),
92 		  [serving_now_ptr] "+m" (lock->h.serving_now),
93 		  [ticket] "=&r" (tmp),
94 		  [my_ticket] "=&r" (my_ticket)
95 		: [inc] "r" (inc));
96 	} else {
97 		__asm__ __volatile__ (
98 		"	.set push		# arch_spin_lock	\n"
99 		"	.set noreorder					\n"
100 		"							\n"
101 		"1:	ll	%[ticket], %[ticket_ptr]		\n"
102 		"	addu	%[my_ticket], %[ticket], %[inc]		\n"
103 		"	sc	%[my_ticket], %[ticket_ptr]		\n"
104 		"	beqz	%[my_ticket], 1b			\n"
105 		"	 srl	%[my_ticket], %[ticket], 16		\n"
106 		"	andi	%[ticket], %[ticket], 0xffff		\n"
107 		"	bne	%[ticket], %[my_ticket], 4f		\n"
108 		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
109 		"2:							\n"
110 		"	.subsection 2					\n"
111 		"4:	andi	%[ticket], %[ticket], 0x1fff		\n"
112 		"	sll	%[ticket], 5				\n"
113 		"							\n"
114 		"6:	bnez	%[ticket], 6b				\n"
115 		"	 subu	%[ticket], 1				\n"
116 		"							\n"
117 		"	lhu	%[ticket], %[serving_now_ptr]		\n"
118 		"	beq	%[ticket], %[my_ticket], 2b		\n"
119 		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
120 		"	b	4b					\n"
121 		"	 subu	%[ticket], %[ticket], 1			\n"
122 		"	.previous					\n"
123 		"	.set pop					\n"
124 		: [ticket_ptr] "+m" (lock->lock),
125 		  [serving_now_ptr] "+m" (lock->h.serving_now),
126 		  [ticket] "=&r" (tmp),
127 		  [my_ticket] "=&r" (my_ticket)
128 		: [inc] "r" (inc));
129 	}
130 
131 	smp_llsc_mb();
132 }
133 
134 static inline void arch_spin_unlock(arch_spinlock_t *lock)
135 {
136 	unsigned int serving_now = lock->h.serving_now + 1;
137 	wmb();
138 	lock->h.serving_now = (u16)serving_now;
139 	nudge_writes();
140 }
141 
142 static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
143 {
144 	int tmp, tmp2, tmp3;
145 	int inc = 0x10000;
146 
147 	if (R10000_LLSC_WAR) {
148 		__asm__ __volatile__ (
149 		"	.set push		# arch_spin_trylock	\n"
150 		"	.set noreorder					\n"
151 		"							\n"
152 		"1:	ll	%[ticket], %[ticket_ptr]		\n"
153 		"	srl	%[my_ticket], %[ticket], 16		\n"
154 		"	andi	%[now_serving], %[ticket], 0xffff	\n"
155 		"	bne	%[my_ticket], %[now_serving], 3f	\n"
156 		"	 addu	%[ticket], %[ticket], %[inc]		\n"
157 		"	sc	%[ticket], %[ticket_ptr]		\n"
158 		"	beqzl	%[ticket], 1b				\n"
159 		"	 li	%[ticket], 1				\n"
160 		"2:							\n"
161 		"	.subsection 2					\n"
162 		"3:	b	2b					\n"
163 		"	 li	%[ticket], 0				\n"
164 		"	.previous					\n"
165 		"	.set pop					\n"
166 		: [ticket_ptr] "+m" (lock->lock),
167 		  [ticket] "=&r" (tmp),
168 		  [my_ticket] "=&r" (tmp2),
169 		  [now_serving] "=&r" (tmp3)
170 		: [inc] "r" (inc));
171 	} else {
172 		__asm__ __volatile__ (
173 		"	.set push		# arch_spin_trylock	\n"
174 		"	.set noreorder					\n"
175 		"							\n"
176 		"1:	ll	%[ticket], %[ticket_ptr]		\n"
177 		"	srl	%[my_ticket], %[ticket], 16		\n"
178 		"	andi	%[now_serving], %[ticket], 0xffff	\n"
179 		"	bne	%[my_ticket], %[now_serving], 3f	\n"
180 		"	 addu	%[ticket], %[ticket], %[inc]		\n"
181 		"	sc	%[ticket], %[ticket_ptr]		\n"
182 		"	beqz	%[ticket], 1b				\n"
183 		"	 li	%[ticket], 1				\n"
184 		"2:							\n"
185 		"	.subsection 2					\n"
186 		"3:	b	2b					\n"
187 		"	 li	%[ticket], 0				\n"
188 		"	.previous					\n"
189 		"	.set pop					\n"
190 		: [ticket_ptr] "+m" (lock->lock),
191 		  [ticket] "=&r" (tmp),
192 		  [my_ticket] "=&r" (tmp2),
193 		  [now_serving] "=&r" (tmp3)
194 		: [inc] "r" (inc));
195 	}
196 
197 	smp_llsc_mb();
198 
199 	return tmp;
200 }
201 
202 /*
203  * Read-write spinlocks, allowing multiple readers but only one writer.
204  *
205  * NOTE! it is quite common to have readers in interrupts but no interrupt
206  * writers. For those circumstances we can "mix" irq-safe locks - any writer
207  * needs to get a irq-safe write-lock, but readers can get non-irqsafe
208  * read-locks.
209  */
210 
211 /*
212  * read_can_lock - would read_trylock() succeed?
213  * @lock: the rwlock in question.
214  */
215 #define arch_read_can_lock(rw)	((rw)->lock >= 0)
216 
217 /*
218  * write_can_lock - would write_trylock() succeed?
219  * @lock: the rwlock in question.
220  */
221 #define arch_write_can_lock(rw) (!(rw)->lock)
222 
223 static inline void arch_read_lock(arch_rwlock_t *rw)
224 {
225 	unsigned int tmp;
226 
227 	if (R10000_LLSC_WAR) {
228 		__asm__ __volatile__(
229 		"	.set	noreorder	# arch_read_lock	\n"
230 		"1:	ll	%1, %2					\n"
231 		"	bltz	%1, 1b					\n"
232 		"	 addu	%1, 1					\n"
233 		"	sc	%1, %0					\n"
234 		"	beqzl	%1, 1b					\n"
235 		"	 nop						\n"
236 		"	.set	reorder					\n"
237 		: "=m" (rw->lock), "=&r" (tmp)
238 		: "m" (rw->lock)
239 		: "memory");
240 	} else {
241 		do {
242 			__asm__ __volatile__(
243 			"1:	ll	%1, %2	# arch_read_lock	\n"
244 			"	bltz	%1, 1b				\n"
245 			"	 addu	%1, 1				\n"
246 			"2:	sc	%1, %0				\n"
247 			: "=m" (rw->lock), "=&r" (tmp)
248 			: "m" (rw->lock)
249 			: "memory");
250 		} while (unlikely(!tmp));
251 	}
252 
253 	smp_llsc_mb();
254 }
255 
256 /* Note the use of sub, not subu which will make the kernel die with an
257    overflow exception if we ever try to unlock an rwlock that is already
258    unlocked or is being held by a writer.  */
259 static inline void arch_read_unlock(arch_rwlock_t *rw)
260 {
261 	unsigned int tmp;
262 
263 	smp_mb__before_llsc();
264 
265 	if (R10000_LLSC_WAR) {
266 		__asm__ __volatile__(
267 		"1:	ll	%1, %2		# arch_read_unlock	\n"
268 		"	sub	%1, 1					\n"
269 		"	sc	%1, %0					\n"
270 		"	beqzl	%1, 1b					\n"
271 		: "=m" (rw->lock), "=&r" (tmp)
272 		: "m" (rw->lock)
273 		: "memory");
274 	} else {
275 		do {
276 			__asm__ __volatile__(
277 			"1:	ll	%1, %2	# arch_read_unlock	\n"
278 			"	sub	%1, 1				\n"
279 			"	sc	%1, %0				\n"
280 			: "=m" (rw->lock), "=&r" (tmp)
281 			: "m" (rw->lock)
282 			: "memory");
283 		} while (unlikely(!tmp));
284 	}
285 }
286 
287 static inline void arch_write_lock(arch_rwlock_t *rw)
288 {
289 	unsigned int tmp;
290 
291 	if (R10000_LLSC_WAR) {
292 		__asm__ __volatile__(
293 		"	.set	noreorder	# arch_write_lock	\n"
294 		"1:	ll	%1, %2					\n"
295 		"	bnez	%1, 1b					\n"
296 		"	 lui	%1, 0x8000				\n"
297 		"	sc	%1, %0					\n"
298 		"	beqzl	%1, 1b					\n"
299 		"	 nop						\n"
300 		"	.set	reorder					\n"
301 		: "=m" (rw->lock), "=&r" (tmp)
302 		: "m" (rw->lock)
303 		: "memory");
304 	} else {
305 		do {
306 			__asm__ __volatile__(
307 			"1:	ll	%1, %2	# arch_write_lock	\n"
308 			"	bnez	%1, 1b				\n"
309 			"	 lui	%1, 0x8000			\n"
310 			"2:	sc	%1, %0				\n"
311 			: "=m" (rw->lock), "=&r" (tmp)
312 			: "m" (rw->lock)
313 			: "memory");
314 		} while (unlikely(!tmp));
315 	}
316 
317 	smp_llsc_mb();
318 }
319 
320 static inline void arch_write_unlock(arch_rwlock_t *rw)
321 {
322 	smp_mb();
323 
324 	__asm__ __volatile__(
325 	"				# arch_write_unlock	\n"
326 	"	sw	$0, %0					\n"
327 	: "=m" (rw->lock)
328 	: "m" (rw->lock)
329 	: "memory");
330 }
331 
332 static inline int arch_read_trylock(arch_rwlock_t *rw)
333 {
334 	unsigned int tmp;
335 	int ret;
336 
337 	if (R10000_LLSC_WAR) {
338 		__asm__ __volatile__(
339 		"	.set	noreorder	# arch_read_trylock	\n"
340 		"	li	%2, 0					\n"
341 		"1:	ll	%1, %3					\n"
342 		"	bltz	%1, 2f					\n"
343 		"	 addu	%1, 1					\n"
344 		"	sc	%1, %0					\n"
345 		"	.set	reorder					\n"
346 		"	beqzl	%1, 1b					\n"
347 		"	 nop						\n"
348 		__WEAK_LLSC_MB
349 		"	li	%2, 1					\n"
350 		"2:							\n"
351 		: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
352 		: "m" (rw->lock)
353 		: "memory");
354 	} else {
355 		__asm__ __volatile__(
356 		"	.set	noreorder	# arch_read_trylock	\n"
357 		"	li	%2, 0					\n"
358 		"1:	ll	%1, %3					\n"
359 		"	bltz	%1, 2f					\n"
360 		"	 addu	%1, 1					\n"
361 		"	sc	%1, %0					\n"
362 		"	beqz	%1, 1b					\n"
363 		"	 nop						\n"
364 		"	.set	reorder					\n"
365 		__WEAK_LLSC_MB
366 		"	li	%2, 1					\n"
367 		"2:							\n"
368 		: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
369 		: "m" (rw->lock)
370 		: "memory");
371 	}
372 
373 	return ret;
374 }
375 
376 static inline int arch_write_trylock(arch_rwlock_t *rw)
377 {
378 	unsigned int tmp;
379 	int ret;
380 
381 	if (R10000_LLSC_WAR) {
382 		__asm__ __volatile__(
383 		"	.set	noreorder	# arch_write_trylock	\n"
384 		"	li	%2, 0					\n"
385 		"1:	ll	%1, %3					\n"
386 		"	bnez	%1, 2f					\n"
387 		"	 lui	%1, 0x8000				\n"
388 		"	sc	%1, %0					\n"
389 		"	beqzl	%1, 1b					\n"
390 		"	 nop						\n"
391 		__WEAK_LLSC_MB
392 		"	li	%2, 1					\n"
393 		"	.set	reorder					\n"
394 		"2:							\n"
395 		: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
396 		: "m" (rw->lock)
397 		: "memory");
398 	} else {
399 		do {
400 			__asm__ __volatile__(
401 			"	ll	%1, %3	# arch_write_trylock	\n"
402 			"	li	%2, 0				\n"
403 			"	bnez	%1, 2f				\n"
404 			"	lui	%1, 0x8000			\n"
405 			"	sc	%1, %0				\n"
406 			"	li	%2, 1				\n"
407 			"2:						\n"
408 			: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
409 			: "m" (rw->lock)
410 			: "memory");
411 		} while (unlikely(!tmp));
412 
413 		smp_llsc_mb();
414 	}
415 
416 	return ret;
417 }
418 
419 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
420 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
421 
422 #define arch_spin_relax(lock)	cpu_relax()
423 #define arch_read_relax(lock)	cpu_relax()
424 #define arch_write_relax(lock)	cpu_relax()
425 
426 #endif /* _ASM_SPINLOCK_H */
427