1 // SPDX-License-Identifier: GPL-2.0-only
18  * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler()
28  * Protect the per-CPU list with a per-CPU spinlock to handle task migration.
30  * ->sched_in() path will need to take the vCPU off the list of the _previous_
40 	return &(to_vt(vcpu)->pi_desc);  in vcpu_to_pi_desc()
51 	if (!try_cmpxchg64(&pi_desc->control, pold, new))  in pi_try_set_control()
52 		return -EBUSY;  in pi_try_set_control()
66 	 * To simplify hot-plug and dynamic toggling of APICv, keep PI.NDST and  in vmx_vcpu_pi_load()
67 	 * PI.SN up-to-date even if there is no assigned device or if APICv is  in vmx_vcpu_pi_load()
68 	 * deactivated due to a dynamic inhibit bit, e.g. for Hyper-V's SyncIC.  in vmx_vcpu_pi_load()
79 	if (pi_desc->nv != POSTED_INTR_WAKEUP_VECTOR && vcpu->cpu == cpu) {  in vmx_vcpu_pi_load()
92 	if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR) {  in vmx_vcpu_pi_load()
93 		raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu);  in vmx_vcpu_pi_load()
103 		spin_acquire(&spinlock->dep_map, PI_LOCK_SCHED_OUT, 0, _RET_IP_);  in vmx_vcpu_pi_load()
104 		list_del(&vt->pi_wakeup_list);  in vmx_vcpu_pi_load()
105 		spin_release(&spinlock->dep_map, _RET_IP_);  in vmx_vcpu_pi_load()
113 	old.control = READ_ONCE(pi_desc->control);  in vmx_vcpu_pi_load()
119 		 * handle task migration (@cpu != vcpu->cpu).  in vmx_vcpu_pi_load()
136 	 * Clear SN before reading the bitmap.  The VT-d firmware  in vmx_vcpu_pi_load()
155 	       READ_ONCE(kvm->arch.nr_possible_bypass_irqs);  in vmx_can_use_vtd_pi()
173 	 * various per-CPU scheduler locks.  When the wakeup handler runs, it  in pi_enable_wakeup_handler()
182 	raw_spin_lock_nested(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu),  in pi_enable_wakeup_handler()
184 	list_add_tail(&vt->pi_wakeup_list,  in pi_enable_wakeup_handler()
185 		      &per_cpu(wakeup_vcpus_on_cpu, vcpu->cpu));  in pi_enable_wakeup_handler()
186 	raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));  in pi_enable_wakeup_handler()
190 	old.control = READ_ONCE(pi_desc->control);  in pi_enable_wakeup_handler()
198 	 * Send a wakeup IPI to this CPU if an interrupt may have been posted  in pi_enable_wakeup_handler()
200 	 * will arrive on the non-wakeup vector.  An IPI is needed as calling  in pi_enable_wakeup_handler()
201 	 * try_to_wake_up() from ->sched_out() isn't allowed (IRQs are not  in pi_enable_wakeup_handler()
215 	 * using either IPI virtualization or VT-d PI, so that the  in vmx_needs_pi_wakeup()
220 		vmx_can_use_vtd_pi(vcpu->kvm);  in vmx_needs_pi_wakeup()
242 	if (!vcpu->preempted && kvm_vcpu_is_blocking(vcpu) &&  in vmx_vcpu_pi_put()
263 		if (pi_test_on(&vt->pi_desc))  in pi_wakeup_handler()
280 	memset(pi->pir, 0, sizeof(pi->pir));  in pi_apicv_pre_state_restore()