7 #include <linux/irq.h>
62  * Index the VM's xarray of mapped LPIs and return a reference to the IRQ
64  * finished with the IRQ.
69 	struct vgic_irq *irq = NULL;  in vgic_get_lpi()  local
73 	irq = xa_load(&dist->lpi_xa, intid);  in vgic_get_lpi()
74 	if (!vgic_try_get_irq_ref(irq))  in vgic_get_lpi()
75 		irq = NULL;  in vgic_get_lpi()
79 	return irq;  in vgic_get_lpi()
85  * to call vgic_put_irq() once it's finished with this IRQ.
117 static void vgic_release_lpi_locked(struct vgic_dist *dist, struct vgic_irq *irq)  in vgic_release_lpi_locked()  argument
120 	__xa_erase(&dist->lpi_xa, irq->intid);  in vgic_release_lpi_locked()
121 	kfree_rcu(irq, rcu);  in vgic_release_lpi_locked()
124 static __must_check bool __vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)  in __vgic_put_irq()  argument
126 	if (irq->intid < VGIC_MIN_LPI)  in __vgic_put_irq()
129 	return refcount_dec_and_test(&irq->refcount);  in __vgic_put_irq()
132 static __must_check bool vgic_put_irq_norelease(struct kvm *kvm, struct vgic_irq *irq)  in vgic_put_irq_norelease()  argument
134 	if (!__vgic_put_irq(kvm, irq))  in vgic_put_irq_norelease()
137 	irq->pending_release = true;  in vgic_put_irq_norelease()
141 void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)  in vgic_put_irq()  argument
145 	if (irq->intid >= VGIC_MIN_LPI)  in vgic_put_irq()
148 	if (!__vgic_put_irq(kvm, irq))  in vgic_put_irq()
152 	vgic_release_lpi_locked(dist, irq);  in vgic_put_irq()
160 	struct vgic_irq *irq;  in vgic_release_deleted_lpis()  local
164 	xa_for_each(&dist->lpi_xa, intid, irq) {  in vgic_release_deleted_lpis()
165 		if (irq->pending_release)  in vgic_release_deleted_lpis()
166 			vgic_release_lpi_locked(dist, irq);  in vgic_release_deleted_lpis()
175 	struct vgic_irq *irq, *tmp;  in vgic_flush_pending_lpis()  local
181 	list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {  in vgic_flush_pending_lpis()
182 		if (irq->intid >= VGIC_MIN_LPI) {  in vgic_flush_pending_lpis()
183 			raw_spin_lock(&irq->irq_lock);  in vgic_flush_pending_lpis()
184 			list_del(&irq->ap_list);  in vgic_flush_pending_lpis()
185 			irq->vcpu = NULL;  in vgic_flush_pending_lpis()
186 			raw_spin_unlock(&irq->irq_lock);  in vgic_flush_pending_lpis()
187 			deleted |= vgic_put_irq_norelease(vcpu->kvm, irq);  in vgic_flush_pending_lpis()
197 void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending)  in vgic_irq_set_phys_pending()  argument
199 	WARN_ON(irq_set_irqchip_state(irq->host_irq,  in vgic_irq_set_phys_pending()
204 bool vgic_get_phys_line_level(struct vgic_irq *irq)  in vgic_get_phys_line_level()  argument
208 	BUG_ON(!irq->hw);  in vgic_get_phys_line_level()
210 	if (irq->ops && irq->ops->get_input_level)  in vgic_get_phys_line_level()
211 		return irq->ops->get_input_level(irq->intid);  in vgic_get_phys_line_level()
213 	WARN_ON(irq_get_irqchip_state(irq->host_irq,  in vgic_get_phys_line_level()
220 void vgic_irq_set_phys_active(struct vgic_irq *irq, bool active)  in vgic_irq_set_phys_active()  argument
223 	BUG_ON(!irq->hw);  in vgic_irq_set_phys_active()
224 	WARN_ON(irq_set_irqchip_state(irq->host_irq,  in vgic_irq_set_phys_active()
230  * vgic_target_oracle - compute the target vcpu for an irq
232  * @irq:	The irq to route. Must be already locked.
238  * Requires the IRQ lock to be held.
240 static struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq)  in vgic_target_oracle()  argument
242 	lockdep_assert_held(&irq->irq_lock);  in vgic_target_oracle()
245 	if (irq->active)  in vgic_target_oracle()
246 		return irq->vcpu ? : irq->target_vcpu;  in vgic_target_oracle()
249 	 * If the IRQ is not active but enabled and pending, we should direct  in vgic_target_oracle()
254 	if (irq->enabled && irq_is_pending(irq)) {  in vgic_target_oracle()
255 		if (unlikely(irq->target_vcpu &&  in vgic_target_oracle()
256 			     !irq->target_vcpu->kvm->arch.vgic.enabled))  in vgic_target_oracle()
259 		return irq->target_vcpu;  in vgic_target_oracle()
262 	/* If neither active nor pending and enabled, then this IRQ should not  in vgic_target_oracle()
334  * rising edge, and in-kernel connected IRQ lines can only be controlled by
337 static bool vgic_validate_injection(struct vgic_irq *irq, bool level, void *owner)  in vgic_validate_injection()  argument
339 	if (irq->owner != owner)  in vgic_validate_injection()
342 	switch (irq->config) {  in vgic_validate_injection()
344 		return irq->line_level != level;  in vgic_validate_injection()
353  * Check whether an IRQ needs to (and can) be queued to a VCPU's ap list.
355  * Returns true when the IRQ was queued, false otherwise.
357  * Needs to be entered with the IRQ lock already held, but will return
360 bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,  in vgic_queue_irq_unlock()  argument
361 			   unsigned long flags) __releases(&irq->irq_lock)  in vgic_queue_irq_unlock()
365 	lockdep_assert_held(&irq->irq_lock);  in vgic_queue_irq_unlock()
368 	vcpu = vgic_target_oracle(irq);  in vgic_queue_irq_unlock()
369 	if (irq->vcpu || !vcpu) {  in vgic_queue_irq_unlock()
371 		 * If this IRQ is already on a VCPU's ap_list, then it  in vgic_queue_irq_unlock()
375 		 * Otherwise, if the irq is not pending and enabled, it does  in vgic_queue_irq_unlock()
379 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_queue_irq_unlock()
385 		 * while the IRQ is already on the VCPU's AP list, the  in vgic_queue_irq_unlock()
398 	 * We must unlock the irq lock to take the ap_list_lock where  in vgic_queue_irq_unlock()
401 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_queue_irq_unlock()
406 	raw_spin_lock(&irq->irq_lock);  in vgic_queue_irq_unlock()
412 	 * 1) The irq lost its pending state or was disabled behind our  in vgic_queue_irq_unlock()
414 	 * 2) Someone changed the affinity on this irq behind our  in vgic_queue_irq_unlock()
420 	if (unlikely(irq->vcpu || vcpu != vgic_target_oracle(irq))) {  in vgic_queue_irq_unlock()
421 		raw_spin_unlock(&irq->irq_lock);  in vgic_queue_irq_unlock()
425 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_queue_irq_unlock()
430 	 * Grab a reference to the irq to reflect the fact that it is  in vgic_queue_irq_unlock()
432 	 * reference on the irq.  in vgic_queue_irq_unlock()
434 	vgic_get_irq_ref(irq);  in vgic_queue_irq_unlock()
435 	list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head);  in vgic_queue_irq_unlock()
436 	irq->vcpu = vcpu;  in vgic_queue_irq_unlock()
438 	raw_spin_unlock(&irq->irq_lock);  in vgic_queue_irq_unlock()
448  * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic
456  * @owner:   The opaque pointer to the owner of the IRQ being raised to verify
457  *           that the caller is allowed to inject this IRQ.  Userspace
467 	struct vgic_irq *irq;  in kvm_vgic_inject_irq()  local
481 		irq = vgic_get_vcpu_irq(vcpu, intid);  in kvm_vgic_inject_irq()
483 		irq = vgic_get_irq(kvm, intid);  in kvm_vgic_inject_irq()
484 	if (!irq)  in kvm_vgic_inject_irq()
487 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_inject_irq()
489 	if (!vgic_validate_injection(irq, level, owner)) {  in kvm_vgic_inject_irq()
491 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_inject_irq()
492 		vgic_put_irq(kvm, irq);  in kvm_vgic_inject_irq()
496 	if (irq->config == VGIC_CONFIG_LEVEL)  in kvm_vgic_inject_irq()
497 		irq->line_level = level;  in kvm_vgic_inject_irq()
499 		irq->pending_latch = true;  in kvm_vgic_inject_irq()
501 	vgic_queue_irq_unlock(kvm, irq, flags);  in kvm_vgic_inject_irq()
502 	vgic_put_irq(kvm, irq);  in kvm_vgic_inject_irq()
507 /* @irq->irq_lock must be held */
508 static int kvm_vgic_map_irq(struct kvm_vcpu *vcpu, struct vgic_irq *irq,  in kvm_vgic_map_irq()  argument
516 	 * Find the physical IRQ number corresponding to @host_irq  in kvm_vgic_map_irq()
527 	irq->hw = true;  in kvm_vgic_map_irq()
528 	irq->host_irq = host_irq;  in kvm_vgic_map_irq()
529 	irq->hwintid = data->hwirq;  in kvm_vgic_map_irq()
530 	irq->ops = ops;  in kvm_vgic_map_irq()
534 /* @irq->irq_lock must be held */
535 static inline void kvm_vgic_unmap_irq(struct vgic_irq *irq)  in kvm_vgic_unmap_irq()  argument
537 	irq->hw = false;  in kvm_vgic_unmap_irq()
538 	irq->hwintid = 0;  in kvm_vgic_unmap_irq()
539 	irq->ops = NULL;  in kvm_vgic_unmap_irq()
545 	struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, vintid);  in kvm_vgic_map_phys_irq()  local
549 	BUG_ON(!irq);  in kvm_vgic_map_phys_irq()
551 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_map_phys_irq()
552 	ret = kvm_vgic_map_irq(vcpu, irq, host_irq, ops);  in kvm_vgic_map_phys_irq()
553 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_map_phys_irq()
554 	vgic_put_irq(vcpu->kvm, irq);  in kvm_vgic_map_phys_irq()
560  * kvm_vgic_reset_mapped_irq - Reset a mapped IRQ
570 	struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, vintid);  in kvm_vgic_reset_mapped_irq()  local
573 	if (!irq->hw)  in kvm_vgic_reset_mapped_irq()
576 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_reset_mapped_irq()
577 	irq->active = false;  in kvm_vgic_reset_mapped_irq()
578 	irq->pending_latch = false;  in kvm_vgic_reset_mapped_irq()
579 	irq->line_level = false;  in kvm_vgic_reset_mapped_irq()
580 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_reset_mapped_irq()
582 	vgic_put_irq(vcpu->kvm, irq);  in kvm_vgic_reset_mapped_irq()
587 	struct vgic_irq *irq;  in kvm_vgic_unmap_phys_irq()  local
593 	irq = vgic_get_vcpu_irq(vcpu, vintid);  in kvm_vgic_unmap_phys_irq()
594 	BUG_ON(!irq);  in kvm_vgic_unmap_phys_irq()
596 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_unmap_phys_irq()
597 	kvm_vgic_unmap_irq(irq);  in kvm_vgic_unmap_phys_irq()
598 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_unmap_phys_irq()
599 	vgic_put_irq(vcpu->kvm, irq);  in kvm_vgic_unmap_phys_irq()
606 	struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, vintid);  in kvm_vgic_get_map()  local
610 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_get_map()
611 	if (irq->hw)  in kvm_vgic_get_map()
612 		ret = irq->hwintid;  in kvm_vgic_get_map()
613 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_get_map()
615 	vgic_put_irq(vcpu->kvm, irq);  in kvm_vgic_get_map()
631 	struct vgic_irq *irq;  in kvm_vgic_set_owner()  local
642 	irq = vgic_get_vcpu_irq(vcpu, intid);  in kvm_vgic_set_owner()
643 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_set_owner()
644 	if (irq->owner && irq->owner != owner)  in kvm_vgic_set_owner()
647 		irq->owner = owner;  in kvm_vgic_set_owner()
648 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_set_owner()
664 	struct vgic_irq *irq, *tmp;  in vgic_prune_ap_list()  local
672 	list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {  in vgic_prune_ap_list()
676 		raw_spin_lock(&irq->irq_lock);  in vgic_prune_ap_list()
678 		BUG_ON(vcpu != irq->vcpu);  in vgic_prune_ap_list()
680 		target_vcpu = vgic_target_oracle(irq);  in vgic_prune_ap_list()
687 			list_del(&irq->ap_list);  in vgic_prune_ap_list()
688 			irq->vcpu = NULL;  in vgic_prune_ap_list()
689 			raw_spin_unlock(&irq->irq_lock);  in vgic_prune_ap_list()
695 			 * we remove the irq from the list, we drop  in vgic_prune_ap_list()
698 			deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq);  in vgic_prune_ap_list()
704 			raw_spin_unlock(&irq->irq_lock);  in vgic_prune_ap_list()
710 		raw_spin_unlock(&irq->irq_lock);  in vgic_prune_ap_list()
728 		raw_spin_lock(&irq->irq_lock);  in vgic_prune_ap_list()
739 		if (target_vcpu == vgic_target_oracle(irq)) {  in vgic_prune_ap_list()
742 			list_del(&irq->ap_list);  in vgic_prune_ap_list()
743 			irq->vcpu = target_vcpu;  in vgic_prune_ap_list()
744 			list_add_tail(&irq->ap_list, &new_cpu->ap_list_head);  in vgic_prune_ap_list()
748 		raw_spin_unlock(&irq->irq_lock);  in vgic_prune_ap_list()
776 				    struct vgic_irq *irq, int lr)  in vgic_populate_lr()  argument
778 	lockdep_assert_held(&irq->irq_lock);  in vgic_populate_lr()
781 		vgic_v2_populate_lr(vcpu, irq, lr);  in vgic_populate_lr()
783 		vgic_v3_populate_lr(vcpu, irq, lr);  in vgic_populate_lr()
807 	struct vgic_irq *irq;  in compute_ap_list_depth()  local
814 	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {  in compute_ap_list_depth()
817 		raw_spin_lock(&irq->irq_lock);  in compute_ap_list_depth()
819 		w = vgic_irq_get_lr_count(irq);  in compute_ap_list_depth()
820 		raw_spin_unlock(&irq->irq_lock);  in compute_ap_list_depth()
832 	struct vgic_irq *irq;  in vgic_flush_lr_state()  local
846 	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {  in vgic_flush_lr_state()
847 		raw_spin_lock(&irq->irq_lock);  in vgic_flush_lr_state()
851 		 * guarantee that they are all seen before any IRQ of  in vgic_flush_lr_state()
856 		if (multi_sgi && irq->priority > prio) {  in vgic_flush_lr_state()
857 			raw_spin_unlock(&irq->irq_lock);  in vgic_flush_lr_state()
861 		if (likely(vgic_target_oracle(irq) == vcpu)) {  in vgic_flush_lr_state()
862 			vgic_populate_lr(vcpu, irq, count++);  in vgic_flush_lr_state()
864 			if (irq->source)  in vgic_flush_lr_state()
865 				prio = irq->priority;  in vgic_flush_lr_state()
868 		raw_spin_unlock(&irq->irq_lock);  in vgic_flush_lr_state()
871 			if (!list_is_last(&irq->ap_list,  in vgic_flush_lr_state()
951 	 * - If we have any pending IRQ for the guest and the guest  in kvm_vgic_flush_hwstate()
954 	 *   virtual EL2 mode, then we have to emulate an IRQ  in kvm_vgic_flush_hwstate()
1039 	struct vgic_irq *irq;  in kvm_vgic_vcpu_pending_irq()  local
1054 	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {  in kvm_vgic_vcpu_pending_irq()
1055 		raw_spin_lock(&irq->irq_lock);  in kvm_vgic_vcpu_pending_irq()
1056 		pending = irq_is_pending(irq) && irq->enabled &&  in kvm_vgic_vcpu_pending_irq()
1057 			  !irq->active &&  in kvm_vgic_vcpu_pending_irq()
1058 			  irq->priority < vmcr.pmr;  in kvm_vgic_vcpu_pending_irq()
1059 		raw_spin_unlock(&irq->irq_lock);  in kvm_vgic_vcpu_pending_irq()
1089 	struct vgic_irq *irq;  in kvm_vgic_map_is_active()  local
1096 	irq = vgic_get_vcpu_irq(vcpu, vintid);  in kvm_vgic_map_is_active()
1097 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_map_is_active()
1098 	map_is_active = irq->hw && irq->active;  in kvm_vgic_map_is_active()
1099 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_map_is_active()
1100 	vgic_put_irq(vcpu->kvm, irq);  in kvm_vgic_map_is_active()
1125 void vgic_irq_handle_resampling(struct vgic_irq *irq,  in vgic_irq_handle_resampling()  argument
1128 	if (vgic_irq_is_mapped_level(irq)) {  in vgic_irq_handle_resampling()
1131 		if (unlikely(vgic_irq_needs_resampling(irq))) {  in vgic_irq_handle_resampling()
1132 			resample = !(irq->active || irq->pending_latch);  in vgic_irq_handle_resampling()
1133 		} else if (lr_pending || (lr_deactivated && irq->line_level)) {  in vgic_irq_handle_resampling()
1134 			irq->line_level = vgic_get_phys_line_level(irq);  in vgic_irq_handle_resampling()
1135 			resample = !irq->line_level;  in vgic_irq_handle_resampling()
1139 			vgic_irq_set_phys_active(irq, false);  in vgic_irq_handle_resampling()