xref: /linux/arch/x86/include/asm/qspinlock.h (revision cc1e6315e83db0e517dd9279050b88adc83a7eba)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_QSPINLOCK_H
3 #define _ASM_X86_QSPINLOCK_H
4 
5 #include <linux/jump_label.h>
6 #include <asm/cpufeature.h>
7 #include <asm-generic/qspinlock_types.h>
8 #include <asm/paravirt.h>
9 
10 #define _Q_PENDING_LOOPS	(1 << 9)
11 
12 #ifdef CONFIG_PARAVIRT_SPINLOCKS
13 extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
14 extern void __pv_init_lock_hash(void);
15 extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
16 extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock);
17 
18 #define	queued_spin_unlock queued_spin_unlock
19 /**
20  * queued_spin_unlock - release a queued spinlock
21  * @lock : Pointer to queued spinlock structure
22  *
23  * A smp_store_release() on the least-significant byte.
24  */
25 static inline void native_queued_spin_unlock(struct qspinlock *lock)
26 {
27 	smp_store_release(&lock->locked, 0);
28 }
29 
30 static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
31 {
32 	pv_queued_spin_lock_slowpath(lock, val);
33 }
34 
35 static inline void queued_spin_unlock(struct qspinlock *lock)
36 {
37 	pv_queued_spin_unlock(lock);
38 }
39 
40 #define vcpu_is_preempted vcpu_is_preempted
41 static inline bool vcpu_is_preempted(long cpu)
42 {
43 	return pv_vcpu_is_preempted(cpu);
44 }
45 #endif
46 
47 #ifdef CONFIG_PARAVIRT
48 DECLARE_STATIC_KEY_TRUE(virt_spin_lock_key);
49 
50 void native_pv_lock_init(void) __init;
51 
52 #define virt_spin_lock virt_spin_lock
53 static inline bool virt_spin_lock(struct qspinlock *lock)
54 {
55 	if (!static_branch_likely(&virt_spin_lock_key))
56 		return false;
57 
58 	/*
59 	 * On hypervisors without PARAVIRT_SPINLOCKS support we fall
60 	 * back to a Test-and-Set spinlock, because fair locks have
61 	 * horrible lock 'holder' preemption issues.
62 	 */
63 
64 	do {
65 		while (atomic_read(&lock->val) != 0)
66 			cpu_relax();
67 	} while (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) != 0);
68 
69 	return true;
70 }
71 #else
72 static inline void native_pv_lock_init(void)
73 {
74 }
75 #endif /* CONFIG_PARAVIRT */
76 
77 #include <asm-generic/qspinlock.h>
78 
79 #endif /* _ASM_X86_QSPINLOCK_H */
80