1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Split spinlock implementation out into its own file, so it can be 4 * compiled in a FTRACE-compatible way. 5 */ 6 #include <linux/static_call.h> 7 #include <linux/spinlock.h> 8 #include <linux/export.h> 9 #include <linux/jump_label.h> 10 #include <trace/events/lock.h> 11 12 DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); 13 14 #ifdef CONFIG_SMP 15 void __init native_pv_lock_init(void) 16 { 17 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 18 static_branch_enable(&virt_spin_lock_key); 19 } 20 #endif 21 22 #ifdef CONFIG_PARAVIRT_SPINLOCKS 23 __visible void __native_queued_spin_unlock(struct qspinlock *lock) 24 { 25 native_queued_spin_unlock(lock); 26 } 27 PV_CALLEE_SAVE_REGS_THUNK(__native_queued_spin_unlock); 28 29 DEFINE_STATIC_CALL(queued_spin_lock_slowpath, native_queued_spin_lock_slowpath); 30 EXPORT_STATIC_CALL_TRAMP(queued_spin_lock_slowpath); 31 DEFINE_STATIC_CALL(queued_spin_unlock, __raw_callee_save___native_queued_spin_unlock); 32 EXPORT_STATIC_CALL_TRAMP(queued_spin_unlock); 33 34 /* 35 * Traced unlock variants, swapped in via static_call while the 36 * contended_release tracepoint is enabled. Two of them, so each tail calls its 37 * own base directly. 38 */ 39 __visible void native_queued_spin_unlock_traced(struct qspinlock *lock) 40 { 41 if (queued_spin_is_contended(lock)) 42 trace_call__contended_release(lock); 43 native_queued_spin_unlock(lock); 44 } 45 PV_CALLEE_SAVE_REGS_THUNK(native_queued_spin_unlock_traced); 46 47 __visible void pv_queued_spin_unlock_traced(struct qspinlock *lock) 48 { 49 if (queued_spin_is_contended(lock)) 50 trace_call__contended_release(lock); 51 __raw_callee_save___pv_queued_spin_unlock(lock); 52 } 53 PV_CALLEE_SAVE_REGS_THUNK(pv_queued_spin_unlock_traced); 54 55 bool pv_is_native_spin_unlock(void) 56 { 57 void *unlock = static_call_query(queued_spin_unlock); 58 59 return unlock == __raw_callee_save___native_queued_spin_unlock || 60 unlock == __raw_callee_save_native_queued_spin_unlock_traced; 61 } 62 63 int arch_contended_release_trace_reg(void) 64 { 65 void *cur = static_call_query(queued_spin_unlock); 66 67 if (cur == __raw_callee_save___native_queued_spin_unlock) 68 static_call_update(queued_spin_unlock, 69 __raw_callee_save_native_queued_spin_unlock_traced); 70 else if (cur == __raw_callee_save___pv_queued_spin_unlock) 71 static_call_update(queued_spin_unlock, 72 __raw_callee_save_pv_queued_spin_unlock_traced); 73 return 0; 74 } 75 76 void arch_contended_release_trace_unreg(void) 77 { 78 void *cur = static_call_query(queued_spin_unlock); 79 80 if (cur == __raw_callee_save_native_queued_spin_unlock_traced) 81 static_call_update(queued_spin_unlock, 82 __raw_callee_save___native_queued_spin_unlock); 83 else if (cur == __raw_callee_save_pv_queued_spin_unlock_traced) 84 static_call_update(queued_spin_unlock, 85 __raw_callee_save___pv_queued_spin_unlock); 86 } 87 88 __visible bool __native_vcpu_is_preempted(long cpu) 89 { 90 return false; 91 } 92 PV_CALLEE_SAVE_REGS_THUNK(__native_vcpu_is_preempted); 93 94 bool pv_is_native_vcpu_is_preempted(void) 95 { 96 return pv_ops_lock.vcpu_is_preempted.func == 97 __raw_callee_save___native_vcpu_is_preempted; 98 } 99 100 void __init paravirt_set_cap(void) 101 { 102 if (!pv_is_native_vcpu_is_preempted()) 103 setup_force_cpu_cap(X86_FEATURE_VCPUPREEMPT); 104 } 105 106 struct pv_lock_ops pv_ops_lock = { 107 .wait = paravirt_nop, 108 .kick = paravirt_nop, 109 .vcpu_is_preempted = PV_CALLEE_SAVE(__native_vcpu_is_preempted), 110 }; 111 EXPORT_SYMBOL(pv_ops_lock); 112 #endif 113