1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 #ifndef _ASM_X86_PARAVIRT_SPINLOCK_H 3 #define _ASM_X86_PARAVIRT_SPINLOCK_H 4 5 #include <asm/paravirt_types.h> 6 #include <linux/static_call_types.h> 7 8 #ifdef CONFIG_SMP 9 #include <asm/spinlock_types.h> 10 #endif 11 12 struct qspinlock; 13 14 struct pv_lock_ops { 15 void (*wait)(u8 *ptr, u8 val); 16 void (*kick)(int cpu); 17 18 struct paravirt_callee_save vcpu_is_preempted; 19 } __no_randomize_layout; 20 21 extern struct pv_lock_ops pv_ops_lock; 22 23 #ifdef CONFIG_PARAVIRT_SPINLOCKS 24 extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 25 extern void __pv_init_lock_hash(void); 26 extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 27 extern void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); 28 extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); 29 extern bool nopvspin; 30 31 DECLARE_STATIC_CALL(queued_spin_lock_slowpath, native_queued_spin_lock_slowpath); 32 DECLARE_STATIC_CALL(queued_spin_unlock, __raw_callee_save___native_queued_spin_unlock); 33 34 static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, 35 u32 val) 36 { 37 static_call_mod(queued_spin_lock_slowpath)(lock, val); 38 } 39 40 static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) 41 { 42 PVOP_CALL_ARGS; 43 __STATIC_CALL_MOD_ADDRESSABLE(queued_spin_unlock); 44 asm volatile ("call " STATIC_CALL_TRAMP_STR(queued_spin_unlock) 45 : PVOP_VCALLEE_CLOBBERS, ASM_CALL_CONSTRAINT 46 : PVOP_CALL_ARG1(lock) 47 : "memory", "cc"); 48 } 49 50 static __always_inline bool pv_vcpu_is_preempted(long cpu) 51 { 52 return PVOP_ALT_CALLEE1(bool, pv_ops_lock, vcpu_is_preempted, cpu, 53 "xor %%eax, %%eax", 54 ALT_NOT(X86_FEATURE_VCPUPREEMPT)); 55 } 56 57 #define queued_spin_unlock queued_spin_unlock 58 /** 59 * queued_spin_unlock - release a queued spinlock 60 * @lock : Pointer to queued spinlock structure 61 * 62 * A smp_store_release() on the least-significant byte. 63 */ 64 static inline void native_queued_spin_unlock(struct qspinlock *lock) 65 { 66 smp_store_release(&lock->locked, 0); 67 } 68 69 static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) 70 { 71 pv_queued_spin_lock_slowpath(lock, val); 72 } 73 74 static inline void queued_spin_unlock(struct qspinlock *lock) 75 { 76 kcsan_release(); 77 pv_queued_spin_unlock(lock); 78 } 79 80 #define vcpu_is_preempted vcpu_is_preempted 81 static inline bool vcpu_is_preempted(long cpu) 82 { 83 return pv_vcpu_is_preempted(cpu); 84 } 85 86 static __always_inline void pv_wait(u8 *ptr, u8 val) 87 { 88 PVOP_VCALL2(pv_ops_lock, wait, ptr, val); 89 } 90 91 static __always_inline void pv_kick(int cpu) 92 { 93 PVOP_VCALL1(pv_ops_lock, kick, cpu); 94 } 95 96 void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); 97 bool __raw_callee_save___native_vcpu_is_preempted(long cpu); 98 #endif /* CONFIG_PARAVIRT_SPINLOCKS */ 99 100 void __init native_pv_lock_init(void); 101 __visible void __native_queued_spin_unlock(struct qspinlock *lock); 102 __visible void native_queued_spin_unlock_traced(struct qspinlock *lock); 103 __visible void pv_queued_spin_unlock_traced(struct qspinlock *lock); 104 bool pv_is_native_spin_unlock(void); 105 __visible bool __native_vcpu_is_preempted(long cpu); 106 bool pv_is_native_vcpu_is_preempted(void); 107 108 /* 109 * virt_spin_lock_key - disables by default the virt_spin_lock() hijack. 110 * 111 * Native (and PV wanting native due to vCPU pinning) should keep this key 112 * disabled. Native does not touch the key. 113 * 114 * When in a guest then native_pv_lock_init() enables the key first and 115 * KVM/XEN might conditionally disable it later in the boot process again. 116 */ 117 DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key); 118 119 /* 120 * Shortcut for the queued_spin_lock_slowpath() function that allows 121 * virt to hijack it. 122 * 123 * Returns: 124 * true - lock has been negotiated, all done; 125 * false - queued_spin_lock_slowpath() will do its thing. 126 */ 127 #define virt_spin_lock virt_spin_lock 128 static inline bool virt_spin_lock(struct qspinlock *lock) 129 { 130 int val; 131 132 if (!static_branch_likely(&virt_spin_lock_key)) 133 return false; 134 135 /* 136 * On hypervisors without PARAVIRT_SPINLOCKS support we fall 137 * back to a Test-and-Set spinlock, because fair locks have 138 * horrible lock 'holder' preemption issues. 139 */ 140 141 __retry: 142 val = atomic_read(&lock->val); 143 144 if (val || !atomic_try_cmpxchg(&lock->val, &val, _Q_LOCKED_VAL)) { 145 cpu_relax(); 146 goto __retry; 147 } 148 149 return true; 150 } 151 152 #endif /* _ASM_X86_PARAVIRT_SPINLOCK_H */ 153