1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 #ifndef _ASM_X86_PARAVIRT_SPINLOCK_H 3 #define _ASM_X86_PARAVIRT_SPINLOCK_H 4 5 #include <asm/paravirt_types.h> 6 7 #ifdef CONFIG_SMP 8 #include <asm/spinlock_types.h> 9 #endif 10 11 struct qspinlock; 12 13 struct pv_lock_ops { 14 void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); 15 struct paravirt_callee_save queued_spin_unlock; 16 17 void (*wait)(u8 *ptr, u8 val); 18 void (*kick)(int cpu); 19 20 struct paravirt_callee_save vcpu_is_preempted; 21 } __no_randomize_layout; 22 23 extern struct pv_lock_ops pv_ops_lock; 24 25 #ifdef CONFIG_PARAVIRT_SPINLOCKS 26 extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 27 extern void __pv_init_lock_hash(void); 28 extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 29 extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); 30 extern bool nopvspin; 31 32 static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, 33 u32 val) 34 { 35 PVOP_VCALL2(pv_ops_lock, queued_spin_lock_slowpath, lock, val); 36 } 37 38 static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) 39 { 40 PVOP_ALT_VCALLEE1(pv_ops_lock, queued_spin_unlock, lock, 41 "movb $0, (%%" _ASM_ARG1 ")", 42 ALT_NOT(X86_FEATURE_PVUNLOCK)); 43 } 44 45 static __always_inline bool pv_vcpu_is_preempted(long cpu) 46 { 47 return PVOP_ALT_CALLEE1(bool, pv_ops_lock, vcpu_is_preempted, cpu, 48 "xor %%eax, %%eax", 49 ALT_NOT(X86_FEATURE_VCPUPREEMPT)); 50 } 51 52 #define queued_spin_unlock queued_spin_unlock 53 /** 54 * queued_spin_unlock - release a queued spinlock 55 * @lock : Pointer to queued spinlock structure 56 * 57 * A smp_store_release() on the least-significant byte. 58 */ 59 static inline void native_queued_spin_unlock(struct qspinlock *lock) 60 { 61 smp_store_release(&lock->locked, 0); 62 } 63 64 static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) 65 { 66 pv_queued_spin_lock_slowpath(lock, val); 67 } 68 69 static inline void queued_spin_unlock(struct qspinlock *lock) 70 { 71 kcsan_release(); 72 pv_queued_spin_unlock(lock); 73 } 74 75 #define vcpu_is_preempted vcpu_is_preempted 76 static inline bool vcpu_is_preempted(long cpu) 77 { 78 return pv_vcpu_is_preempted(cpu); 79 } 80 81 static __always_inline void pv_wait(u8 *ptr, u8 val) 82 { 83 PVOP_VCALL2(pv_ops_lock, wait, ptr, val); 84 } 85 86 static __always_inline void pv_kick(int cpu) 87 { 88 PVOP_VCALL1(pv_ops_lock, kick, cpu); 89 } 90 91 void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); 92 bool __raw_callee_save___native_vcpu_is_preempted(long cpu); 93 #endif /* CONFIG_PARAVIRT_SPINLOCKS */ 94 95 void __init native_pv_lock_init(void); 96 __visible void __native_queued_spin_unlock(struct qspinlock *lock); 97 bool pv_is_native_spin_unlock(void); 98 __visible bool __native_vcpu_is_preempted(long cpu); 99 bool pv_is_native_vcpu_is_preempted(void); 100 101 /* 102 * virt_spin_lock_key - disables by default the virt_spin_lock() hijack. 103 * 104 * Native (and PV wanting native due to vCPU pinning) should keep this key 105 * disabled. Native does not touch the key. 106 * 107 * When in a guest then native_pv_lock_init() enables the key first and 108 * KVM/XEN might conditionally disable it later in the boot process again. 109 */ 110 DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key); 111 112 /* 113 * Shortcut for the queued_spin_lock_slowpath() function that allows 114 * virt to hijack it. 115 * 116 * Returns: 117 * true - lock has been negotiated, all done; 118 * false - queued_spin_lock_slowpath() will do its thing. 119 */ 120 #define virt_spin_lock virt_spin_lock 121 static inline bool virt_spin_lock(struct qspinlock *lock) 122 { 123 int val; 124 125 if (!static_branch_likely(&virt_spin_lock_key)) 126 return false; 127 128 /* 129 * On hypervisors without PARAVIRT_SPINLOCKS support we fall 130 * back to a Test-and-Set spinlock, because fair locks have 131 * horrible lock 'holder' preemption issues. 132 */ 133 134 __retry: 135 val = atomic_read(&lock->val); 136 137 if (val || !atomic_try_cmpxchg(&lock->val, &val, _Q_LOCKED_VAL)) { 138 cpu_relax(); 139 goto __retry; 140 } 141 142 return true; 143 } 144 145 #endif /* _ASM_X86_PARAVIRT_SPINLOCK_H */ 146