1 // SPDX-License-Identifier: GPL-2.0-only
11 #include "irq.h"
18  * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler()
23  * wake the target vCPUs.  vCPUs are removed from the list and the notification
28  * Protect the per-CPU list with a per-CPU spinlock to handle task migration.
30  * ->sched_in() path will need to take the vCPU off the list of the _previous_
32  * occur if a wakeup IRQ arrives and attempts to acquire the lock.
40 	return &(to_vt(vcpu)->pi_desc);  in vcpu_to_pi_desc()
46 	 * PID.ON can be set at any time by a different vCPU or by hardware,  in pi_try_set_control()
51 	if (!try_cmpxchg64(&pi_desc->control, pold, new))  in pi_try_set_control()
52 		return -EBUSY;  in pi_try_set_control()
66 	 * To simplify hot-plug and dynamic toggling of APICv, keep PI.NDST and  in vmx_vcpu_pi_load()
67 	 * PI.SN up-to-date even if there is no assigned device or if APICv is  in vmx_vcpu_pi_load()
68 	 * deactivated due to a dynamic inhibit bit, e.g. for Hyper-V's SyncIC.  in vmx_vcpu_pi_load()
75 	 * full update can be skipped as neither the vector nor the destination  in vmx_vcpu_pi_load()
79 	if (pi_desc->nv != POSTED_INTR_WAKEUP_VECTOR && vcpu->cpu == cpu) {  in vmx_vcpu_pi_load()
92 	if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR) {  in vmx_vcpu_pi_load()
93 		raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu);  in vmx_vcpu_pi_load()
96 		 * In addition to taking the wakeup lock for the regular/IRQ  in vmx_vcpu_pi_load()
99 		 * this is taking the lock of the *previous* CPU, i.e. can race  in vmx_vcpu_pi_load()
103 		spin_acquire(&spinlock->dep_map, PI_LOCK_SCHED_OUT, 0, _RET_IP_);  in vmx_vcpu_pi_load()
104 		list_del(&vt->pi_wakeup_list);  in vmx_vcpu_pi_load()
105 		spin_release(&spinlock->dep_map, _RET_IP_);  in vmx_vcpu_pi_load()
113 	old.control = READ_ONCE(pi_desc->control);  in vmx_vcpu_pi_load()
119 		 * handle task migration (@cpu != vcpu->cpu).  in vmx_vcpu_pi_load()
136 	 * Clear SN before reading the bitmap.  The VT-d firmware  in vmx_vcpu_pi_load()
150 	 * Note, reading the number of possible bypass IRQs can race with a  in vmx_can_use_vtd_pi()
151 	 * bypass IRQ being attached to the VM.  vmx_pi_start_bypass() ensures  in vmx_can_use_vtd_pi()
155 	       READ_ONCE(kvm->arch.nr_possible_bypass_irqs);  in vmx_can_use_vtd_pi()
173 	 * various per-CPU scheduler locks.  When the wakeup handler runs, it  in pi_enable_wakeup_handler()
175 	 * can eventually take the aforementioned scheduler locks, which causes  in pi_enable_wakeup_handler()
178 	 * Deadlock can't actually occur because IRQs are disabled for the  in pi_enable_wakeup_handler()
180 	 * can't run while the scheduler locks are held.  in pi_enable_wakeup_handler()
182 	raw_spin_lock_nested(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu),  in pi_enable_wakeup_handler()
184 	list_add_tail(&vt->pi_wakeup_list,  in pi_enable_wakeup_handler()
185 		      &per_cpu(wakeup_vcpus_on_cpu, vcpu->cpu));  in pi_enable_wakeup_handler()
186 	raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));  in pi_enable_wakeup_handler()
190 	old.control = READ_ONCE(pi_desc->control);  in pi_enable_wakeup_handler()
199 	 * before the notification vector was updated, in which case the IRQ  in pi_enable_wakeup_handler()
200 	 * will arrive on the non-wakeup vector.  An IPI is needed as calling  in pi_enable_wakeup_handler()
201 	 * try_to_wake_up() from ->sched_out() isn't allowed (IRQs are not  in pi_enable_wakeup_handler()
214 	 * can be the target of posted interrupts, as is the case when  in vmx_needs_pi_wakeup()
215 	 * using either IPI virtualization or VT-d PI, so that the  in vmx_needs_pi_wakeup()
220 		vmx_can_use_vtd_pi(vcpu->kvm);  in vmx_needs_pi_wakeup()
232 	 * enable the wakeup handler so that notification IRQ wakes the vCPU as  in vmx_vcpu_pi_put()
235 	 * out, as the task is still runnable, i.e. doesn't need a wake event  in vmx_vcpu_pi_put()
240 	 * the cost of a spurious IRQ, and vCPU put/load is a slow path.  in vmx_vcpu_pi_put()
242 	if (!vcpu->preempted && kvm_vcpu_is_blocking(vcpu) &&  in vmx_vcpu_pi_put()
263 		if (pi_test_on(&vt->pi_desc))  in pi_wakeup_handler()
280 	memset(pi->pir, 0, sizeof(pi->pir));  in pi_apicv_pre_state_restore()
293  * Kick all vCPUs when the first possible bypass IRQ is attached to a VM, as
295  * vector, i.e. if the bypass IRQ came along after vmx_vcpu_pi_put().