1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef _ASM_POWERPC_PARAVIRT_H 3 #define _ASM_POWERPC_PARAVIRT_H 4 5 #include <linux/jump_label.h> 6 #include <asm/smp.h> 7 #ifdef CONFIG_PPC64 8 #include <asm/paca.h> 9 #include <asm/lppaca.h> 10 #include <asm/hvcall.h> 11 #endif 12 13 #ifdef CONFIG_PPC_SPLPAR 14 #include <linux/smp.h> 15 #include <asm/kvm_guest.h> 16 #include <asm/cputhreads.h> 17 18 DECLARE_STATIC_KEY_FALSE(shared_processor); 19 20 static inline bool is_shared_processor(void) 21 { 22 return static_branch_unlikely(&shared_processor); 23 } 24 25 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 26 u64 pseries_paravirt_steal_clock(int cpu); 27 28 static inline u64 paravirt_steal_clock(int cpu) 29 { 30 return pseries_paravirt_steal_clock(cpu); 31 } 32 #endif 33 34 /* If bit 0 is set, the cpu has been ceded, conferred, or preempted */ 35 static inline u32 yield_count_of(int cpu) 36 { 37 __be32 yield_count = READ_ONCE(lppaca_of(cpu).yield_count); 38 return be32_to_cpu(yield_count); 39 } 40 41 /* 42 * Spinlock code confers and prods, so don't trace the hcalls because the 43 * tracing code takes spinlocks which can cause recursion deadlocks. 44 * 45 * These calls are made while the lock is not held: the lock slowpath yields if 46 * it can not acquire the lock, and unlock slow path might prod if a waiter has 47 * yielded). So this may not be a problem for simple spin locks because the 48 * tracing does not technically recurse on the lock, but we avoid it anyway. 49 * 50 * However the queued spin lock contended path is more strictly ordered: the 51 * H_CONFER hcall is made after the task has queued itself on the lock, so then 52 * recursing on that lock will cause the task to then queue up again behind the 53 * first instance (or worse: queued spinlocks use tricks that assume a context 54 * never waits on more than one spinlock, so such recursion may cause random 55 * corruption in the lock code). 56 */ 57 static inline void yield_to_preempted(int cpu, u32 yield_count) 58 { 59 plpar_hcall_norets_notrace(H_CONFER, get_hard_smp_processor_id(cpu), yield_count); 60 } 61 62 static inline void prod_cpu(int cpu) 63 { 64 plpar_hcall_norets_notrace(H_PROD, get_hard_smp_processor_id(cpu)); 65 } 66 67 static inline void yield_to_any(void) 68 { 69 plpar_hcall_norets_notrace(H_CONFER, -1, 0); 70 } 71 72 static inline bool is_vcpu_idle(int vcpu) 73 { 74 return lppaca_of(vcpu).idle; 75 } 76 77 static inline bool vcpu_is_dispatched(int vcpu) 78 { 79 /* 80 * This is the yield_count. An "odd" value (low bit on) means that 81 * the processor is yielded (either because of an OS yield or a 82 * hypervisor preempt). An even value implies that the processor is 83 * currently executing. 84 */ 85 return (!(yield_count_of(vcpu) & 1)); 86 } 87 #else 88 static inline bool is_shared_processor(void) 89 { 90 return false; 91 } 92 93 static inline u32 yield_count_of(int cpu) 94 { 95 return 0; 96 } 97 98 extern void ___bad_yield_to_preempted(void); 99 static inline void yield_to_preempted(int cpu, u32 yield_count) 100 { 101 ___bad_yield_to_preempted(); /* This would be a bug */ 102 } 103 104 extern void ___bad_yield_to_any(void); 105 static inline void yield_to_any(void) 106 { 107 ___bad_yield_to_any(); /* This would be a bug */ 108 } 109 110 extern void ___bad_prod_cpu(void); 111 static inline void prod_cpu(int cpu) 112 { 113 ___bad_prod_cpu(); /* This would be a bug */ 114 } 115 116 static inline bool is_vcpu_idle(int vcpu) 117 { 118 return false; 119 } 120 static inline bool vcpu_is_dispatched(int vcpu) 121 { 122 return true; 123 } 124 #endif 125 126 #define vcpu_is_preempted vcpu_is_preempted 127 static inline bool vcpu_is_preempted(int cpu) 128 { 129 /* 130 * The dispatch/yield bit alone is an imperfect indicator of 131 * whether the hypervisor has dispatched @cpu to run on a physical 132 * processor. When it is clear, @cpu is definitely not preempted. 133 * But when it is set, it means only that it *might* be, subject to 134 * other conditions. So we check other properties of the VM and 135 * @cpu first, resorting to the yield count last. 136 */ 137 138 /* 139 * Hypervisor preemption isn't possible in dedicated processor 140 * mode by definition. 141 */ 142 if (!is_shared_processor()) 143 return false; 144 145 /* 146 * If the hypervisor has dispatched the target CPU on a physical 147 * processor, then the target CPU is definitely not preempted. 148 */ 149 if (vcpu_is_dispatched(cpu)) 150 return false; 151 152 /* 153 * if the target CPU is not dispatched and the guest OS 154 * has not marked the CPU idle, then it is hypervisor preempted. 155 */ 156 if (!is_vcpu_idle(cpu)) 157 return true; 158 159 #ifdef CONFIG_PPC_SPLPAR 160 if (!is_kvm_guest()) { 161 int first_cpu, i; 162 163 /* 164 * The result of vcpu_is_preempted() is used in a 165 * speculative way, and is always subject to invalidation 166 * by events internal and external to Linux. While we can 167 * be called in preemptable context (in the Linux sense), 168 * we're not accessing per-cpu resources in a way that can 169 * race destructively with Linux scheduler preemption and 170 * migration, and callers can tolerate the potential for 171 * error introduced by sampling the CPU index without 172 * pinning the task to it. So it is permissible to use 173 * raw_smp_processor_id() here to defeat the preempt debug 174 * warnings that can arise from using smp_processor_id() 175 * in arbitrary contexts. 176 */ 177 first_cpu = cpu_first_thread_sibling(raw_smp_processor_id()); 178 179 /* 180 * The PowerVM hypervisor dispatches VMs on a whole core 181 * basis. So we know that a thread sibling of the executing CPU 182 * cannot have been preempted by the hypervisor, even if it 183 * has called H_CONFER, which will set the yield bit. 184 */ 185 if (cpu_first_thread_sibling(cpu) == first_cpu) 186 return false; 187 188 /* 189 * The specific target CPU was marked by guest OS as idle, but 190 * then also check all other cpus in the core for PowerVM 191 * because it does core scheduling and one of the vcpu 192 * of the core getting preempted by hypervisor implies 193 * other vcpus can also be considered preempted. 194 */ 195 first_cpu = cpu_first_thread_sibling(cpu); 196 for (i = first_cpu; i < first_cpu + threads_per_core; i++) { 197 if (i == cpu) 198 continue; 199 if (vcpu_is_dispatched(i)) 200 return false; 201 if (!is_vcpu_idle(i)) 202 return true; 203 } 204 } 205 #endif 206 207 /* 208 * None of the threads in target CPU's core are running but none of 209 * them were preempted too. Hence assume the target CPU to be 210 * non-preempted. 211 */ 212 return false; 213 } 214 215 static inline bool pv_is_native_spin_unlock(void) 216 { 217 return !is_shared_processor(); 218 } 219 220 #endif /* _ASM_POWERPC_PARAVIRT_H */ 221