1 // SPDX-License-Identifier: GPL-2.0-only 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include <linux/kvm_host.h> 5 #include <linux/kvm_irqfd.h> 6 7 #include <asm/irq_remapping.h> 8 #include <asm/cpu.h> 9 10 #include "lapic.h" 11 #include "irq.h" 12 #include "posted_intr.h" 13 #include "trace.h" 14 #include "vmx.h" 15 #include "tdx.h" 16 17 #pragma GCC poison to_vmx 18 19 /* 20 * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler() 21 * when a WAKEUP_VECTOR interrupted is posted. vCPUs are added to the list when 22 * the vCPU is scheduled out and is blocking (e.g. in HLT) with IRQs enabled. 23 * The vCPUs posted interrupt descriptor is updated at the same time to set its 24 * notification vector to WAKEUP_VECTOR, so that posted interrupt from devices 25 * wake the target vCPUs. vCPUs are removed from the list and the notification 26 * vector is reset when the vCPU is scheduled in. 27 */ 28 static DEFINE_PER_CPU(struct list_head, wakeup_vcpus_on_cpu); 29 /* 30 * Protect the per-CPU list with a per-CPU spinlock to handle task migration. 31 * When a blocking vCPU is awakened _and_ migrated to a different pCPU, the 32 * ->sched_in() path will need to take the vCPU off the list of the _previous_ 33 * CPU. IRQs must be disabled when taking this lock, otherwise deadlock will 34 * occur if a wakeup IRQ arrives and attempts to acquire the lock. 35 */ 36 static DEFINE_PER_CPU(raw_spinlock_t, wakeup_vcpus_on_cpu_lock); 37 38 #define PI_LOCK_SCHED_OUT SINGLE_DEPTH_NESTING 39 40 static struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu) 41 { 42 return &(to_vt(vcpu)->pi_desc); 43 } 44 45 static int pi_try_set_control(struct pi_desc *pi_desc, u64 *pold, u64 new) 46 { 47 /* 48 * PID.ON can be set at any time by a different vCPU or by hardware, 49 * e.g. a device. PID.control must be written atomically, and the 50 * update must be retried with a fresh snapshot an ON change causes 51 * the cmpxchg to fail. 52 */ 53 if (!try_cmpxchg64(&pi_desc->control, pold, new)) 54 return -EBUSY; 55 56 return 0; 57 } 58 59 void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) 60 { 61 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); 62 struct vcpu_vt *vt = to_vt(vcpu); 63 struct pi_desc old, new; 64 unsigned long flags; 65 unsigned int dest; 66 67 /* 68 * To simplify hot-plug and dynamic toggling of APICv, keep PI.NDST and 69 * PI.SN up-to-date even if there is no assigned device or if APICv is 70 * deactivated due to a dynamic inhibit bit, e.g. for Hyper-V's SyncIC. 71 */ 72 if (!enable_apicv || !lapic_in_kernel(vcpu)) 73 return; 74 75 /* 76 * If the vCPU wasn't on the wakeup list and wasn't migrated, then the 77 * full update can be skipped as neither the vector nor the destination 78 * needs to be changed. Clear SN even if there is no assigned device, 79 * again for simplicity. 80 */ 81 if (pi_desc->nv != POSTED_INTR_WAKEUP_VECTOR && vcpu->cpu == cpu) { 82 if (pi_test_and_clear_sn(pi_desc)) 83 goto after_clear_sn; 84 return; 85 } 86 87 local_irq_save(flags); 88 89 /* 90 * If the vCPU was waiting for wakeup, remove the vCPU from the wakeup 91 * list of the _previous_ pCPU, which will not be the same as the 92 * current pCPU if the task was migrated. 93 */ 94 if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR) { 95 raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu); 96 97 /* 98 * In addition to taking the wakeup lock for the regular/IRQ 99 * context, tell lockdep it is being taken for the "sched out" 100 * context as well. vCPU loads happens in task context, and 101 * this is taking the lock of the *previous* CPU, i.e. can race 102 * with both the scheduler and the wakeup handler. 103 */ 104 raw_spin_lock(spinlock); 105 spin_acquire(&spinlock->dep_map, PI_LOCK_SCHED_OUT, 0, _RET_IP_); 106 list_del(&vt->pi_wakeup_list); 107 spin_release(&spinlock->dep_map, _RET_IP_); 108 raw_spin_unlock(spinlock); 109 } 110 111 dest = cpu_physical_id(cpu); 112 if (!x2apic_mode) 113 dest = (dest << 8) & 0xFF00; 114 115 old.control = READ_ONCE(pi_desc->control); 116 do { 117 new.control = old.control; 118 119 /* 120 * Clear SN (as above) and refresh the destination APIC ID to 121 * handle task migration (@cpu != vcpu->cpu). 122 */ 123 new.ndst = dest; 124 __pi_clear_sn(&new); 125 126 /* 127 * Restore the notification vector; in the blocking case, the 128 * descriptor was modified on "put" to use the wakeup vector. 129 */ 130 new.nv = POSTED_INTR_VECTOR; 131 } while (pi_try_set_control(pi_desc, &old.control, new.control)); 132 133 local_irq_restore(flags); 134 135 after_clear_sn: 136 137 /* 138 * Clear SN before reading the bitmap. The VT-d firmware 139 * writes the bitmap and reads SN atomically (5.2.3 in the 140 * spec), so it doesn't really have a memory barrier that 141 * pairs with this, but we cannot do that and we need one. 142 */ 143 smp_mb__after_atomic(); 144 145 if (!pi_is_pir_empty(pi_desc)) 146 pi_set_on(pi_desc); 147 } 148 149 static bool vt_can_use_vtd_pi(struct kvm *kvm) 150 { 151 /* 152 * Note, reading the number of possible bypass IRQs can race with a 153 * bypass IRQ being attached to the VM. vt_pi_start_bypass() ensures 154 * blockng vCPUs will see an elevated count or get KVM_REQ_UNBLOCK. 155 */ 156 return irqchip_in_kernel(kvm) && kvm_arch_has_irq_bypass() && 157 READ_ONCE(kvm->arch.nr_possible_bypass_irqs); 158 } 159 160 /* 161 * Put the vCPU on this pCPU's list of vCPUs that needs to be awakened and set 162 * WAKEUP as the notification vector in the PI descriptor. 163 */ 164 static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu) 165 { 166 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); 167 struct vcpu_vt *vt = to_vt(vcpu); 168 struct pi_desc old, new; 169 170 lockdep_assert_irqs_disabled(); 171 172 /* 173 * Acquire the wakeup lock using the "sched out" context to workaround 174 * a lockdep false positive. When this is called, schedule() holds 175 * various per-CPU scheduler locks. When the wakeup handler runs, it 176 * holds this CPU's wakeup lock while calling try_to_wake_up(), which 177 * can eventually take the aforementioned scheduler locks, which causes 178 * lockdep to assume there is deadlock. 179 * 180 * Deadlock can't actually occur because IRQs are disabled for the 181 * entirety of the sched_out critical section, i.e. the wakeup handler 182 * can't run while the scheduler locks are held. 183 */ 184 raw_spin_lock_nested(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu), 185 PI_LOCK_SCHED_OUT); 186 list_add_tail(&vt->pi_wakeup_list, 187 &per_cpu(wakeup_vcpus_on_cpu, vcpu->cpu)); 188 raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu)); 189 190 WARN(pi_test_sn(pi_desc), "PI descriptor SN field set before blocking"); 191 192 old.control = READ_ONCE(pi_desc->control); 193 do { 194 /* set 'NV' to 'wakeup vector' */ 195 new.control = old.control; 196 new.nv = POSTED_INTR_WAKEUP_VECTOR; 197 } while (pi_try_set_control(pi_desc, &old.control, new.control)); 198 199 /* 200 * Send a wakeup IPI to this CPU if an interrupt may have been posted 201 * before the notification vector was updated, in which case the IRQ 202 * will arrive on the non-wakeup vector. An IPI is needed as calling 203 * try_to_wake_up() from ->sched_out() isn't allowed (IRQs are not 204 * enabled until it is safe to call try_to_wake_up() on the task being 205 * scheduled out). 206 */ 207 if (pi_test_on(&new)) 208 __apic_send_IPI_self(POSTED_INTR_WAKEUP_VECTOR); 209 } 210 211 static bool vt_needs_pi_wakeup(struct kvm_vcpu *vcpu) 212 { 213 /* 214 * The default posted interrupt vector does nothing when 215 * invoked outside guest mode. Return whether a blocked vCPU 216 * can be the target of posted interrupts, as is the case when 217 * using either IPI virtualization or VT-d PI, so that the 218 * notification vector is switched to the one that calls 219 * back to the pi_wakeup_handler() function. 220 */ 221 return (vmx_can_use_ipiv(vcpu) && !is_td_vcpu(vcpu)) || 222 vt_can_use_vtd_pi(vcpu->kvm); 223 } 224 225 void vt_vcpu_pi_put(struct kvm_vcpu *vcpu) 226 { 227 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); 228 229 if (!vt_needs_pi_wakeup(vcpu)) 230 return; 231 232 /* 233 * If the vCPU is blocking with IRQs enabled and ISN'T being preempted, 234 * enable the wakeup handler so that notification IRQ wakes the vCPU as 235 * expected. There is no need to enable the wakeup handler if the vCPU 236 * is preempted between setting its wait state and manually scheduling 237 * out, as the task is still runnable, i.e. doesn't need a wake event 238 * from KVM to be scheduled in. 239 * 240 * If the wakeup handler isn't being enabled, Suppress Notifications as 241 * the cost of propagating PIR.IRR to PID.ON is negligible compared to 242 * the cost of a spurious IRQ, and vCPU put/load is a slow path. 243 */ 244 if (!vcpu->preempted && kvm_vcpu_is_blocking(vcpu) && 245 ((is_td_vcpu(vcpu) && tdx_interrupt_allowed(vcpu)) || 246 (!is_td_vcpu(vcpu) && !vmx_interrupt_blocked(vcpu)))) 247 pi_enable_wakeup_handler(vcpu); 248 else 249 pi_set_sn(pi_desc); 250 } 251 252 /* 253 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR. 254 */ 255 void pi_wakeup_handler(void) 256 { 257 int cpu = smp_processor_id(); 258 struct list_head *wakeup_list = &per_cpu(wakeup_vcpus_on_cpu, cpu); 259 raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, cpu); 260 struct vcpu_vt *vt; 261 262 raw_spin_lock(spinlock); 263 list_for_each_entry(vt, wakeup_list, pi_wakeup_list) { 264 265 if (pi_test_on(&vt->pi_desc)) 266 kvm_vcpu_wake_up(vt_to_vcpu(vt)); 267 } 268 raw_spin_unlock(spinlock); 269 } 270 271 void __init pi_init_cpu(int cpu) 272 { 273 INIT_LIST_HEAD(&per_cpu(wakeup_vcpus_on_cpu, cpu)); 274 raw_spin_lock_init(&per_cpu(wakeup_vcpus_on_cpu_lock, cpu)); 275 } 276 277 void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu) 278 { 279 struct pi_desc *pi = vcpu_to_pi_desc(vcpu); 280 281 pi_clear_on(pi); 282 memset(pi->pir, 0, sizeof(pi->pir)); 283 } 284 285 bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu) 286 { 287 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); 288 289 return pi_test_on(pi_desc) || 290 (pi_test_sn(pi_desc) && !pi_is_pir_empty(pi_desc)); 291 } 292 293 294 /* 295 * Kick all vCPUs when the first possible bypass IRQ is attached to a VM, as 296 * blocking vCPUs may scheduled out without reconfiguring PID.NV to the wakeup 297 * vector, i.e. if the bypass IRQ came along after vt_vcpu_pi_put(). 298 */ 299 void vt_pi_start_bypass(struct kvm *kvm) 300 { 301 if (WARN_ON_ONCE(!vt_can_use_vtd_pi(kvm))) 302 return; 303 304 kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK); 305 } 306 307 int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, 308 unsigned int host_irq, uint32_t guest_irq, 309 struct kvm_vcpu *vcpu, u32 vector) 310 { 311 if (vcpu) { 312 struct intel_iommu_pi_data pi_data = { 313 .pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu)), 314 .vector = vector, 315 }; 316 317 return irq_set_vcpu_affinity(host_irq, &pi_data); 318 } else { 319 return irq_set_vcpu_affinity(host_irq, NULL); 320 } 321 } 322