1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef _ASM_POWERPC_PARAVIRT_H 3 #define _ASM_POWERPC_PARAVIRT_H 4 5 #include <linux/jump_label.h> 6 #include <asm/smp.h> 7 #ifdef CONFIG_PPC64 8 #include <asm/paca.h> 9 #include <asm/lppaca.h> 10 #include <asm/hvcall.h> 11 #endif 12 13 #ifdef CONFIG_PPC_SPLPAR 14 #include <linux/smp.h> 15 #include <asm/kvm_guest.h> 16 #include <asm/cputhreads.h> 17 18 DECLARE_STATIC_KEY_FALSE(shared_processor); 19 20 static inline bool is_shared_processor(void) 21 { 22 return static_branch_unlikely(&shared_processor); 23 } 24 25 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 26 extern struct static_key paravirt_steal_enabled; 27 extern struct static_key paravirt_steal_rq_enabled; 28 29 u64 pseries_paravirt_steal_clock(int cpu); 30 31 static inline u64 paravirt_steal_clock(int cpu) 32 { 33 return pseries_paravirt_steal_clock(cpu); 34 } 35 #endif 36 37 /* If bit 0 is set, the cpu has been ceded, conferred, or preempted */ 38 static inline u32 yield_count_of(int cpu) 39 { 40 __be32 yield_count = READ_ONCE(lppaca_of(cpu).yield_count); 41 return be32_to_cpu(yield_count); 42 } 43 44 /* 45 * Spinlock code confers and prods, so don't trace the hcalls because the 46 * tracing code takes spinlocks which can cause recursion deadlocks. 47 * 48 * These calls are made while the lock is not held: the lock slowpath yields if 49 * it can not acquire the lock, and unlock slow path might prod if a waiter has 50 * yielded). So this may not be a problem for simple spin locks because the 51 * tracing does not technically recurse on the lock, but we avoid it anyway. 52 * 53 * However the queued spin lock contended path is more strictly ordered: the 54 * H_CONFER hcall is made after the task has queued itself on the lock, so then 55 * recursing on that lock will cause the task to then queue up again behind the 56 * first instance (or worse: queued spinlocks use tricks that assume a context 57 * never waits on more than one spinlock, so such recursion may cause random 58 * corruption in the lock code). 59 */ 60 static inline void yield_to_preempted(int cpu, u32 yield_count) 61 { 62 plpar_hcall_norets_notrace(H_CONFER, get_hard_smp_processor_id(cpu), yield_count); 63 } 64 65 static inline void prod_cpu(int cpu) 66 { 67 plpar_hcall_norets_notrace(H_PROD, get_hard_smp_processor_id(cpu)); 68 } 69 70 static inline void yield_to_any(void) 71 { 72 plpar_hcall_norets_notrace(H_CONFER, -1, 0); 73 } 74 75 static inline bool is_vcpu_idle(int vcpu) 76 { 77 return lppaca_of(vcpu).idle; 78 } 79 80 static inline bool vcpu_is_dispatched(int vcpu) 81 { 82 /* 83 * This is the yield_count. An "odd" value (low bit on) means that 84 * the processor is yielded (either because of an OS yield or a 85 * hypervisor preempt). An even value implies that the processor is 86 * currently executing. 87 */ 88 return (!(yield_count_of(vcpu) & 1)); 89 } 90 #else 91 static inline bool is_shared_processor(void) 92 { 93 return false; 94 } 95 96 static inline u32 yield_count_of(int cpu) 97 { 98 return 0; 99 } 100 101 extern void ___bad_yield_to_preempted(void); 102 static inline void yield_to_preempted(int cpu, u32 yield_count) 103 { 104 ___bad_yield_to_preempted(); /* This would be a bug */ 105 } 106 107 extern void ___bad_yield_to_any(void); 108 static inline void yield_to_any(void) 109 { 110 ___bad_yield_to_any(); /* This would be a bug */ 111 } 112 113 extern void ___bad_prod_cpu(void); 114 static inline void prod_cpu(int cpu) 115 { 116 ___bad_prod_cpu(); /* This would be a bug */ 117 } 118 119 static inline bool is_vcpu_idle(int vcpu) 120 { 121 return false; 122 } 123 static inline bool vcpu_is_dispatched(int vcpu) 124 { 125 return true; 126 } 127 #endif 128 129 #define vcpu_is_preempted vcpu_is_preempted 130 static inline bool vcpu_is_preempted(int cpu) 131 { 132 /* 133 * The dispatch/yield bit alone is an imperfect indicator of 134 * whether the hypervisor has dispatched @cpu to run on a physical 135 * processor. When it is clear, @cpu is definitely not preempted. 136 * But when it is set, it means only that it *might* be, subject to 137 * other conditions. So we check other properties of the VM and 138 * @cpu first, resorting to the yield count last. 139 */ 140 141 /* 142 * Hypervisor preemption isn't possible in dedicated processor 143 * mode by definition. 144 */ 145 if (!is_shared_processor()) 146 return false; 147 148 /* 149 * If the hypervisor has dispatched the target CPU on a physical 150 * processor, then the target CPU is definitely not preempted. 151 */ 152 if (vcpu_is_dispatched(cpu)) 153 return false; 154 155 /* 156 * if the target CPU is not dispatched and the guest OS 157 * has not marked the CPU idle, then it is hypervisor preempted. 158 */ 159 if (!is_vcpu_idle(cpu)) 160 return true; 161 162 #ifdef CONFIG_PPC_SPLPAR 163 if (!is_kvm_guest()) { 164 int first_cpu, i; 165 166 /* 167 * The result of vcpu_is_preempted() is used in a 168 * speculative way, and is always subject to invalidation 169 * by events internal and external to Linux. While we can 170 * be called in preemptable context (in the Linux sense), 171 * we're not accessing per-cpu resources in a way that can 172 * race destructively with Linux scheduler preemption and 173 * migration, and callers can tolerate the potential for 174 * error introduced by sampling the CPU index without 175 * pinning the task to it. So it is permissible to use 176 * raw_smp_processor_id() here to defeat the preempt debug 177 * warnings that can arise from using smp_processor_id() 178 * in arbitrary contexts. 179 */ 180 first_cpu = cpu_first_thread_sibling(raw_smp_processor_id()); 181 182 /* 183 * The PowerVM hypervisor dispatches VMs on a whole core 184 * basis. So we know that a thread sibling of the executing CPU 185 * cannot have been preempted by the hypervisor, even if it 186 * has called H_CONFER, which will set the yield bit. 187 */ 188 if (cpu_first_thread_sibling(cpu) == first_cpu) 189 return false; 190 191 /* 192 * The specific target CPU was marked by guest OS as idle, but 193 * then also check all other cpus in the core for PowerVM 194 * because it does core scheduling and one of the vcpu 195 * of the core getting preempted by hypervisor implies 196 * other vcpus can also be considered preempted. 197 */ 198 first_cpu = cpu_first_thread_sibling(cpu); 199 for (i = first_cpu; i < first_cpu + threads_per_core; i++) { 200 if (i == cpu) 201 continue; 202 if (vcpu_is_dispatched(i)) 203 return false; 204 if (!is_vcpu_idle(i)) 205 return true; 206 } 207 } 208 #endif 209 210 /* 211 * None of the threads in target CPU's core are running but none of 212 * them were preempted too. Hence assume the target CPU to be 213 * non-preempted. 214 */ 215 return false; 216 } 217 218 static inline bool pv_is_native_spin_unlock(void) 219 { 220 return !is_shared_processor(); 221 } 222 223 #endif /* _ASM_POWERPC_PARAVIRT_H */ 224