1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef _ASM_POWERPC_PARAVIRT_H 3 #define _ASM_POWERPC_PARAVIRT_H 4 5 #include <linux/jump_label.h> 6 #include <asm/smp.h> 7 #ifdef CONFIG_PPC64 8 #include <asm/paca.h> 9 #include <asm/lppaca.h> 10 #include <asm/hvcall.h> 11 #endif 12 13 #ifdef CONFIG_PPC_SPLPAR 14 #include <linux/smp.h> 15 #include <asm/kvm_guest.h> 16 #include <asm/cputhreads.h> 17 18 DECLARE_STATIC_KEY_FALSE(shared_processor); 19 20 static inline bool is_shared_processor(void) 21 { 22 return static_branch_unlikely(&shared_processor); 23 } 24 25 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 26 extern struct static_key paravirt_steal_enabled; 27 extern struct static_key paravirt_steal_rq_enabled; 28 29 u64 pseries_paravirt_steal_clock(int cpu); 30 31 static inline u64 paravirt_steal_clock(int cpu) 32 { 33 return pseries_paravirt_steal_clock(cpu); 34 } 35 #endif 36 37 /* If bit 0 is set, the cpu has been ceded, conferred, or preempted */ 38 static inline u32 yield_count_of(int cpu) 39 { 40 __be32 yield_count = READ_ONCE(lppaca_of(cpu).yield_count); 41 return be32_to_cpu(yield_count); 42 } 43 44 /* 45 * Spinlock code confers and prods, so don't trace the hcalls because the 46 * tracing code takes spinlocks which can cause recursion deadlocks. 47 * 48 * These calls are made while the lock is not held: the lock slowpath yields if 49 * it can not acquire the lock, and unlock slow path might prod if a waiter has 50 * yielded). So this may not be a problem for simple spin locks because the 51 * tracing does not technically recurse on the lock, but we avoid it anyway. 52 * 53 * However the queued spin lock contended path is more strictly ordered: the 54 * H_CONFER hcall is made after the task has queued itself on the lock, so then 55 * recursing on that lock will cause the task to then queue up again behind the 56 * first instance (or worse: queued spinlocks use tricks that assume a context 57 * never waits on more than one spinlock, so such recursion may cause random 58 * corruption in the lock code). 59 */ 60 static inline void yield_to_preempted(int cpu, u32 yield_count) 61 { 62 plpar_hcall_norets_notrace(H_CONFER, get_hard_smp_processor_id(cpu), yield_count); 63 } 64 65 static inline void prod_cpu(int cpu) 66 { 67 plpar_hcall_norets_notrace(H_PROD, get_hard_smp_processor_id(cpu)); 68 } 69 70 static inline void yield_to_any(void) 71 { 72 plpar_hcall_norets_notrace(H_CONFER, -1, 0); 73 } 74 #else 75 static inline bool is_shared_processor(void) 76 { 77 return false; 78 } 79 80 static inline u32 yield_count_of(int cpu) 81 { 82 return 0; 83 } 84 85 extern void ___bad_yield_to_preempted(void); 86 static inline void yield_to_preempted(int cpu, u32 yield_count) 87 { 88 ___bad_yield_to_preempted(); /* This would be a bug */ 89 } 90 91 extern void ___bad_yield_to_any(void); 92 static inline void yield_to_any(void) 93 { 94 ___bad_yield_to_any(); /* This would be a bug */ 95 } 96 97 extern void ___bad_prod_cpu(void); 98 static inline void prod_cpu(int cpu) 99 { 100 ___bad_prod_cpu(); /* This would be a bug */ 101 } 102 103 #endif 104 105 #define vcpu_is_preempted vcpu_is_preempted 106 static inline bool vcpu_is_preempted(int cpu) 107 { 108 /* 109 * The dispatch/yield bit alone is an imperfect indicator of 110 * whether the hypervisor has dispatched @cpu to run on a physical 111 * processor. When it is clear, @cpu is definitely not preempted. 112 * But when it is set, it means only that it *might* be, subject to 113 * other conditions. So we check other properties of the VM and 114 * @cpu first, resorting to the yield count last. 115 */ 116 117 /* 118 * Hypervisor preemption isn't possible in dedicated processor 119 * mode by definition. 120 */ 121 if (!is_shared_processor()) 122 return false; 123 124 #ifdef CONFIG_PPC_SPLPAR 125 if (!is_kvm_guest()) { 126 int first_cpu; 127 128 /* 129 * The result of vcpu_is_preempted() is used in a 130 * speculative way, and is always subject to invalidation 131 * by events internal and external to Linux. While we can 132 * be called in preemptable context (in the Linux sense), 133 * we're not accessing per-cpu resources in a way that can 134 * race destructively with Linux scheduler preemption and 135 * migration, and callers can tolerate the potential for 136 * error introduced by sampling the CPU index without 137 * pinning the task to it. So it is permissible to use 138 * raw_smp_processor_id() here to defeat the preempt debug 139 * warnings that can arise from using smp_processor_id() 140 * in arbitrary contexts. 141 */ 142 first_cpu = cpu_first_thread_sibling(raw_smp_processor_id()); 143 144 /* 145 * The PowerVM hypervisor dispatches VMs on a whole core 146 * basis. So we know that a thread sibling of the local CPU 147 * cannot have been preempted by the hypervisor, even if it 148 * has called H_CONFER, which will set the yield bit. 149 */ 150 if (cpu_first_thread_sibling(cpu) == first_cpu) 151 return false; 152 } 153 #endif 154 155 if (yield_count_of(cpu) & 1) 156 return true; 157 return false; 158 } 159 160 static inline bool pv_is_native_spin_unlock(void) 161 { 162 return !is_shared_processor(); 163 } 164 165 #endif /* _ASM_POWERPC_PARAVIRT_H */ 166