Lines Matching +full:vp +full:- +full:p
1 // SPDX-License-Identifier: GPL-2.0-only
33 mutex_lock(&partition->pt_irq_lock); in mshv_register_irq_ack_notifier()
34 hlist_add_head_rcu(&mian->link, &partition->irq_ack_notifier_list); in mshv_register_irq_ack_notifier()
35 mutex_unlock(&partition->pt_irq_lock); in mshv_register_irq_ack_notifier()
41 mutex_lock(&partition->pt_irq_lock); in mshv_unregister_irq_ack_notifier()
42 hlist_del_init_rcu(&mian->link); in mshv_unregister_irq_ack_notifier()
43 mutex_unlock(&partition->pt_irq_lock); in mshv_unregister_irq_ack_notifier()
53 hlist_for_each_entry_rcu(mian, &partition->irq_ack_notifier_list, in mshv_notify_acked_gsi()
55 if (mian->irq_ack_gsi == gsi) { in mshv_notify_acked_gsi()
56 mian->irq_acked(mian); in mshv_notify_acked_gsi()
86 partition = resampler->rsmplr_partn; in mshv_irqfd_resampler_ack()
88 idx = srcu_read_lock(&partition->pt_irq_srcu); in mshv_irqfd_resampler_ack()
90 hlist_for_each_entry_rcu(irqfd, &resampler->rsmplr_irqfd_list, in mshv_irqfd_resampler_ack()
92 if (hv_should_clear_interrupt(irqfd->irqfd_lapic_irq.lapic_control.interrupt_type)) in mshv_irqfd_resampler_ack()
93 hv_call_clear_virtual_interrupt(partition->pt_id); in mshv_irqfd_resampler_ack()
95 eventfd_signal(irqfd->irqfd_resamplefd); in mshv_irqfd_resampler_ack()
98 srcu_read_unlock(&partition->pt_irq_srcu, idx); in mshv_irqfd_resampler_ack()
116 static int mshv_vp_irq_try_set_vector(struct mshv_vp *vp, u32 vector) in mshv_vp_irq_try_set_vector() argument
120 iv = vp->vp_register_page->interrupt_vectors; in mshv_vp_irq_try_set_vector()
127 return -ENOSPC; in mshv_vp_irq_try_set_vector()
131 if (cmpxchg(&vp->vp_register_page->interrupt_vectors.as_uint64, in mshv_vp_irq_try_set_vector()
133 return -EAGAIN; in mshv_vp_irq_try_set_vector()
138 static int mshv_vp_irq_set_vector(struct mshv_vp *vp, u32 vector) in mshv_vp_irq_set_vector() argument
143 ret = mshv_vp_irq_try_set_vector(vp, vector); in mshv_vp_irq_set_vector()
144 } while (ret == -EAGAIN && !need_resched()); in mshv_vp_irq_set_vector()
155 struct mshv_partition *partition = irqfd->irqfd_partn; in mshv_try_assert_irq_fast()
156 struct mshv_lapic_irq *irq = &irqfd->irqfd_lapic_irq; in mshv_try_assert_irq_fast()
157 struct mshv_vp *vp; in mshv_try_assert_irq_fast() local
161 return -EOPNOTSUPP; in mshv_try_assert_irq_fast()
164 return -EOPNOTSUPP; in mshv_try_assert_irq_fast()
166 if (irq->lapic_control.logical_dest_mode) in mshv_try_assert_irq_fast()
167 return -EOPNOTSUPP; in mshv_try_assert_irq_fast()
169 vp = partition->pt_vp_array[irq->lapic_apic_id]; in mshv_try_assert_irq_fast()
171 if (!vp->vp_register_page) in mshv_try_assert_irq_fast()
172 return -EOPNOTSUPP; in mshv_try_assert_irq_fast()
174 if (mshv_vp_irq_set_vector(vp, irq->lapic_vector)) in mshv_try_assert_irq_fast()
175 return -EINVAL; in mshv_try_assert_irq_fast()
177 if (vp->run.flags.root_sched_dispatched && in mshv_try_assert_irq_fast()
178 vp->vp_register_page->interrupt_vectors.as_uint64) in mshv_try_assert_irq_fast()
179 return -EBUSY; in mshv_try_assert_irq_fast()
181 wake_up(&vp->run.vp_suspend_queue); in mshv_try_assert_irq_fast()
188 return -EOPNOTSUPP; in mshv_try_assert_irq_fast()
194 struct mshv_partition *partition = irqfd->irqfd_partn; in mshv_assert_irq_slow()
195 struct mshv_lapic_irq *irq = &irqfd->irqfd_lapic_irq; in mshv_assert_irq_slow()
199 WARN_ON(irqfd->irqfd_resampler && in mshv_assert_irq_slow()
200 !irq->lapic_control.level_triggered); in mshv_assert_irq_slow()
202 idx = srcu_read_lock(&partition->pt_irq_srcu); in mshv_assert_irq_slow()
203 if (irqfd->irqfd_girq_ent.guest_irq_num) { in mshv_assert_irq_slow()
204 if (!irqfd->irqfd_girq_ent.girq_entry_valid) { in mshv_assert_irq_slow()
205 srcu_read_unlock(&partition->pt_irq_srcu, idx); in mshv_assert_irq_slow()
210 seq = read_seqcount_begin(&irqfd->irqfd_irqe_sc); in mshv_assert_irq_slow()
211 } while (read_seqcount_retry(&irqfd->irqfd_irqe_sc, seq)); in mshv_assert_irq_slow()
214 hv_call_assert_virtual_interrupt(irqfd->irqfd_partn->pt_id, in mshv_assert_irq_slow()
215 irq->lapic_vector, irq->lapic_apic_id, in mshv_assert_irq_slow()
216 irq->lapic_control); in mshv_assert_irq_slow()
217 srcu_read_unlock(&partition->pt_irq_srcu, idx); in mshv_assert_irq_slow()
222 struct mshv_irqfd_resampler *rp = irqfd->irqfd_resampler; in mshv_irqfd_resampler_shutdown()
223 struct mshv_partition *pt = rp->rsmplr_partn; in mshv_irqfd_resampler_shutdown()
225 mutex_lock(&pt->irqfds_resampler_lock); in mshv_irqfd_resampler_shutdown()
227 hlist_del_rcu(&irqfd->irqfd_resampler_hnode); in mshv_irqfd_resampler_shutdown()
228 synchronize_srcu(&pt->pt_irq_srcu); in mshv_irqfd_resampler_shutdown()
230 if (hlist_empty(&rp->rsmplr_irqfd_list)) { in mshv_irqfd_resampler_shutdown()
231 hlist_del(&rp->rsmplr_hnode); in mshv_irqfd_resampler_shutdown()
232 mshv_unregister_irq_ack_notifier(pt, &rp->rsmplr_notifier); in mshv_irqfd_resampler_shutdown()
236 mutex_unlock(&pt->irqfds_resampler_lock); in mshv_irqfd_resampler_shutdown()
240 * Race-free decouple logic (ordering is critical)
248 * Synchronize with the wait-queue and unhook ourselves to prevent in mshv_irqfd_shutdown()
251 remove_wait_queue(irqfd->irqfd_wqh, &irqfd->irqfd_wait); in mshv_irqfd_shutdown()
253 if (irqfd->irqfd_resampler) { in mshv_irqfd_shutdown()
255 eventfd_ctx_put(irqfd->irqfd_resamplefd); in mshv_irqfd_shutdown()
261 eventfd_ctx_put(irqfd->irqfd_eventfd_ctx); in mshv_irqfd_shutdown()
265 /* assumes partition->pt_irqfds_lock is held */
268 return !hlist_unhashed(&irqfd->irqfd_hnode); in mshv_irqfd_is_active()
274 * assumes partition->pt_irqfds_lock is held
281 hlist_del(&irqfd->irqfd_hnode); in mshv_irqfd_deactivate()
283 queue_work(irqfd_cleanup_wq, &irqfd->irqfd_shutdown); in mshv_irqfd_deactivate()
287 * Called with wqh->lock held and interrupts disabled
297 struct mshv_partition *pt = irqfd->irqfd_partn; in mshv_irqfd_wakeup()
303 eventfd_ctx_do_read(irqfd->irqfd_eventfd_ctx, &cnt); in mshv_irqfd_wakeup()
304 idx = srcu_read_lock(&pt->pt_irq_srcu); in mshv_irqfd_wakeup()
306 seq = read_seqcount_begin(&irqfd->irqfd_irqe_sc); in mshv_irqfd_wakeup()
307 } while (read_seqcount_retry(&irqfd->irqfd_irqe_sc, seq)); in mshv_irqfd_wakeup()
314 srcu_read_unlock(&pt->pt_irq_srcu, idx); in mshv_irqfd_wakeup()
323 spin_lock_irqsave(&pt->pt_irqfds_lock, flags); in mshv_irqfd_wakeup()
329 * the wait-queue. If it is already deactivated, we can in mshv_irqfd_wakeup()
332 * other side is required to acquire wqh->lock, which we hold in mshv_irqfd_wakeup()
337 spin_unlock_irqrestore(&pt->pt_irqfds_lock, flags); in mshv_irqfd_wakeup()
347 write_seqcount_begin(&irqfd->irqfd_irqe_sc); in mshv_irqfd_update()
348 irqfd->irqfd_girq_ent = mshv_ret_girq_entry(pt, in mshv_irqfd_update()
349 irqfd->irqfd_irqnum); in mshv_irqfd_update()
350 mshv_copy_girq_info(&irqfd->irqfd_girq_ent, &irqfd->irqfd_lapic_irq); in mshv_irqfd_update()
351 write_seqcount_end(&irqfd->irqfd_irqe_sc); in mshv_irqfd_update()
358 spin_lock_irq(&pt->pt_irqfds_lock); in mshv_irqfd_routing_update()
359 hlist_for_each_entry(irqfd, &pt->pt_irqfds_list, irqfd_hnode) in mshv_irqfd_routing_update()
361 spin_unlock_irq(&pt->pt_irqfds_lock); in mshv_irqfd_routing_update()
370 irqfd->irqfd_wqh = wqh; in mshv_irqfd_queue_proc()
378 irqfd->irqfd_wait.flags |= WQ_FLAG_EXCLUSIVE; in mshv_irqfd_queue_proc()
379 add_wait_queue_priority(wqh, &irqfd->irqfd_wait); in mshv_irqfd_queue_proc()
391 CLASS(fd, f)(args->fd); in mshv_irqfd_assign()
395 return -ENOMEM; in mshv_irqfd_assign()
397 irqfd->irqfd_partn = pt; in mshv_irqfd_assign()
398 irqfd->irqfd_irqnum = args->gsi; in mshv_irqfd_assign()
399 INIT_WORK(&irqfd->irqfd_shutdown, mshv_irqfd_shutdown); in mshv_irqfd_assign()
400 seqcount_spinlock_init(&irqfd->irqfd_irqe_sc, &pt->pt_irqfds_lock); in mshv_irqfd_assign()
403 ret = -EBADF; in mshv_irqfd_assign()
413 irqfd->irqfd_eventfd_ctx = eventfd; in mshv_irqfd_assign()
415 if (args->flags & BIT(MSHV_IRQFD_BIT_RESAMPLE)) { in mshv_irqfd_assign()
418 resamplefd = eventfd_ctx_fdget(args->resamplefd); in mshv_irqfd_assign()
424 irqfd->irqfd_resamplefd = resamplefd; in mshv_irqfd_assign()
426 mutex_lock(&pt->irqfds_resampler_lock); in mshv_irqfd_assign()
428 hlist_for_each_entry(rp, &pt->irqfds_resampler_list, in mshv_irqfd_assign()
430 if (rp->rsmplr_notifier.irq_ack_gsi == in mshv_irqfd_assign()
431 irqfd->irqfd_irqnum) { in mshv_irqfd_assign()
432 irqfd->irqfd_resampler = rp; in mshv_irqfd_assign()
437 if (!irqfd->irqfd_resampler) { in mshv_irqfd_assign()
440 ret = -ENOMEM; in mshv_irqfd_assign()
441 mutex_unlock(&pt->irqfds_resampler_lock); in mshv_irqfd_assign()
445 rp->rsmplr_partn = pt; in mshv_irqfd_assign()
446 INIT_HLIST_HEAD(&rp->rsmplr_irqfd_list); in mshv_irqfd_assign()
447 rp->rsmplr_notifier.irq_ack_gsi = irqfd->irqfd_irqnum; in mshv_irqfd_assign()
448 rp->rsmplr_notifier.irq_acked = in mshv_irqfd_assign()
451 hlist_add_head(&rp->rsmplr_hnode, in mshv_irqfd_assign()
452 &pt->irqfds_resampler_list); in mshv_irqfd_assign()
454 &rp->rsmplr_notifier); in mshv_irqfd_assign()
455 irqfd->irqfd_resampler = rp; in mshv_irqfd_assign()
458 hlist_add_head_rcu(&irqfd->irqfd_resampler_hnode, in mshv_irqfd_assign()
459 &irqfd->irqfd_resampler->rsmplr_irqfd_list); in mshv_irqfd_assign()
461 mutex_unlock(&pt->irqfds_resampler_lock); in mshv_irqfd_assign()
465 * Install our own custom wake-up handling so we are notified via in mshv_irqfd_assign()
468 init_waitqueue_func_entry(&irqfd->irqfd_wait, mshv_irqfd_wakeup); in mshv_irqfd_assign()
469 init_poll_funcptr(&irqfd->irqfd_polltbl, mshv_irqfd_queue_proc); in mshv_irqfd_assign()
471 spin_lock_irq(&pt->pt_irqfds_lock); in mshv_irqfd_assign()
472 if (args->flags & BIT(MSHV_IRQFD_BIT_RESAMPLE) && in mshv_irqfd_assign()
473 !irqfd->irqfd_lapic_irq.lapic_control.level_triggered) { in mshv_irqfd_assign()
478 spin_unlock_irq(&pt->pt_irqfds_lock); in mshv_irqfd_assign()
479 ret = -EINVAL; in mshv_irqfd_assign()
483 hlist_for_each_entry(tmp, &pt->pt_irqfds_list, irqfd_hnode) { in mshv_irqfd_assign()
484 if (irqfd->irqfd_eventfd_ctx != tmp->irqfd_eventfd_ctx) in mshv_irqfd_assign()
487 ret = -EBUSY; in mshv_irqfd_assign()
488 spin_unlock_irq(&pt->pt_irqfds_lock); in mshv_irqfd_assign()
492 idx = srcu_read_lock(&pt->pt_irq_srcu); in mshv_irqfd_assign()
494 hlist_add_head(&irqfd->irqfd_hnode, &pt->pt_irqfds_list); in mshv_irqfd_assign()
495 spin_unlock_irq(&pt->pt_irqfds_lock); in mshv_irqfd_assign()
501 events = vfs_poll(fd_file(f), &irqfd->irqfd_polltbl); in mshv_irqfd_assign()
506 srcu_read_unlock(&pt->pt_irq_srcu, idx); in mshv_irqfd_assign()
510 if (irqfd->irqfd_resampler) in mshv_irqfd_assign()
534 eventfd = eventfd_ctx_fdget(args->fd); in mshv_irqfd_deassign()
538 hlist_for_each_entry_safe(irqfd, n, &pt->pt_irqfds_list, in mshv_irqfd_deassign()
540 if (irqfd->irqfd_eventfd_ctx == eventfd && in mshv_irqfd_deassign()
541 irqfd->irqfd_irqnum == args->gsi) in mshv_irqfd_deassign()
561 if (args->flags & ~MSHV_IRQFD_FLAGS_MASK) in mshv_set_unset_irqfd()
562 return -EINVAL; in mshv_set_unset_irqfd()
564 if (args->flags & BIT(MSHV_IRQFD_BIT_DEASSIGN)) in mshv_set_unset_irqfd()
579 spin_lock_irq(&pt->pt_irqfds_lock); in mshv_irqfd_release()
581 hlist_for_each_entry_safe(irqfd, n, &pt->pt_irqfds_list, irqfd_hnode) in mshv_irqfd_release()
584 spin_unlock_irq(&pt->pt_irqfds_lock); in mshv_irqfd_release()
595 irqfd_cleanup_wq = alloc_workqueue("mshv-irqfd-cleanup", 0, 0); in mshv_irqfd_wq_init()
597 return -ENOMEM; in mshv_irqfd_wq_init()
608 * --------------------------------------------------------------------
613 * --------------------------------------------------------------------
616 static void ioeventfd_release(struct mshv_ioeventfd *p, u64 partition_id) in ioeventfd_release() argument
618 if (p->iovntfd_doorbell_id > 0) in ioeventfd_release()
619 mshv_unregister_doorbell(partition_id, p->iovntfd_doorbell_id); in ioeventfd_release()
620 eventfd_ctx_put(p->iovntfd_eventfd); in ioeventfd_release()
621 kfree(p); in ioeventfd_release()
628 struct mshv_ioeventfd *p; in ioeventfd_mmio_write() local
631 hlist_for_each_entry_rcu(p, &partition->ioeventfds_list, iovntfd_hnode) in ioeventfd_mmio_write()
632 if (p->iovntfd_doorbell_id == doorbell_id) { in ioeventfd_mmio_write()
633 eventfd_signal(p->iovntfd_eventfd); in ioeventfd_mmio_write()
641 struct mshv_ioeventfd *p) in ioeventfd_check_collision() argument
642 __must_hold(&pt->mutex) in ioeventfd_check_collision()
646 hlist_for_each_entry(_p, &pt->ioeventfds_list, iovntfd_hnode) in ioeventfd_check_collision()
647 if (_p->iovntfd_addr == p->iovntfd_addr && in ioeventfd_check_collision()
648 _p->iovntfd_length == p->iovntfd_length && in ioeventfd_check_collision()
649 (_p->iovntfd_wildcard || p->iovntfd_wildcard || in ioeventfd_check_collision()
650 _p->iovntfd_datamatch == p->iovntfd_datamatch)) in ioeventfd_check_collision()
658 __must_hold(&pt->mutex) in mshv_assign_ioeventfd()
660 struct mshv_ioeventfd *p; in mshv_assign_ioeventfd() local
666 WARN_ON_ONCE(!mutex_is_locked(&pt->pt_mutex)); in mshv_assign_ioeventfd()
668 if (args->flags & BIT(MSHV_IOEVENTFD_BIT_PIO)) in mshv_assign_ioeventfd()
669 return -EOPNOTSUPP; in mshv_assign_ioeventfd()
671 /* must be natural-word sized */ in mshv_assign_ioeventfd()
672 switch (args->len) { in mshv_assign_ioeventfd()
689 return -EINVAL; in mshv_assign_ioeventfd()
693 if (args->addr + args->len < args->addr) in mshv_assign_ioeventfd()
694 return -EINVAL; in mshv_assign_ioeventfd()
697 if (args->flags & ~MSHV_IOEVENTFD_FLAGS_MASK) in mshv_assign_ioeventfd()
698 return -EINVAL; in mshv_assign_ioeventfd()
700 eventfd = eventfd_ctx_fdget(args->fd); in mshv_assign_ioeventfd()
704 p = kzalloc(sizeof(*p), GFP_KERNEL); in mshv_assign_ioeventfd()
705 if (!p) { in mshv_assign_ioeventfd()
706 ret = -ENOMEM; in mshv_assign_ioeventfd()
710 p->iovntfd_addr = args->addr; in mshv_assign_ioeventfd()
711 p->iovntfd_length = args->len; in mshv_assign_ioeventfd()
712 p->iovntfd_eventfd = eventfd; in mshv_assign_ioeventfd()
715 if (args->flags & BIT(MSHV_IOEVENTFD_BIT_DATAMATCH)) { in mshv_assign_ioeventfd()
716 p->iovntfd_datamatch = args->datamatch; in mshv_assign_ioeventfd()
718 p->iovntfd_wildcard = true; in mshv_assign_ioeventfd()
722 if (ioeventfd_check_collision(pt, p)) { in mshv_assign_ioeventfd()
723 ret = -EEXIST; in mshv_assign_ioeventfd()
727 ret = mshv_register_doorbell(pt->pt_id, ioeventfd_mmio_write, in mshv_assign_ioeventfd()
728 (void *)pt, p->iovntfd_addr, in mshv_assign_ioeventfd()
729 p->iovntfd_datamatch, doorbell_flags); in mshv_assign_ioeventfd()
733 p->iovntfd_doorbell_id = ret; in mshv_assign_ioeventfd()
735 hlist_add_head_rcu(&p->iovntfd_hnode, &pt->ioeventfds_list); in mshv_assign_ioeventfd()
740 kfree(p); in mshv_assign_ioeventfd()
750 __must_hold(&pt->mutex) in mshv_deassign_ioeventfd()
752 struct mshv_ioeventfd *p; in mshv_deassign_ioeventfd() local
755 int ret = -ENOENT; in mshv_deassign_ioeventfd()
758 WARN_ON_ONCE(!mutex_is_locked(&pt->pt_mutex)); in mshv_deassign_ioeventfd()
760 eventfd = eventfd_ctx_fdget(args->fd); in mshv_deassign_ioeventfd()
764 hlist_for_each_entry_safe(p, n, &pt->ioeventfds_list, iovntfd_hnode) { in mshv_deassign_ioeventfd()
765 bool wildcard = !(args->flags & BIT(MSHV_IOEVENTFD_BIT_DATAMATCH)); in mshv_deassign_ioeventfd()
767 if (p->iovntfd_eventfd != eventfd || in mshv_deassign_ioeventfd()
768 p->iovntfd_addr != args->addr || in mshv_deassign_ioeventfd()
769 p->iovntfd_length != args->len || in mshv_deassign_ioeventfd()
770 p->iovntfd_wildcard != wildcard) in mshv_deassign_ioeventfd()
773 if (!p->iovntfd_wildcard && in mshv_deassign_ioeventfd()
774 p->iovntfd_datamatch != args->datamatch) in mshv_deassign_ioeventfd()
777 hlist_del_rcu(&p->iovntfd_hnode); in mshv_deassign_ioeventfd()
779 ioeventfd_release(p, pt->pt_id); in mshv_deassign_ioeventfd()
791 __must_hold(&pt->mutex) in mshv_set_unset_ioeventfd()
793 if ((args->flags & ~MSHV_IOEVENTFD_FLAGS_MASK) || in mshv_set_unset_ioeventfd()
795 return -EINVAL; in mshv_set_unset_ioeventfd()
798 if (args->flags & BIT(MSHV_IOEVENTFD_BIT_PIO)) in mshv_set_unset_ioeventfd()
799 return -EOPNOTSUPP; in mshv_set_unset_ioeventfd()
801 if (args->flags & BIT(MSHV_IOEVENTFD_BIT_DEASSIGN)) in mshv_set_unset_ioeventfd()
809 spin_lock_init(&pt->pt_irqfds_lock); in mshv_eventfd_init()
810 INIT_HLIST_HEAD(&pt->pt_irqfds_list); in mshv_eventfd_init()
812 INIT_HLIST_HEAD(&pt->irqfds_resampler_list); in mshv_eventfd_init()
813 mutex_init(&pt->irqfds_resampler_lock); in mshv_eventfd_init()
815 INIT_HLIST_HEAD(&pt->ioeventfds_list); in mshv_eventfd_init()
822 struct mshv_ioeventfd *p; in mshv_eventfd_release() local
824 hlist_move_list(&pt->ioeventfds_list, &items); in mshv_eventfd_release()
827 hlist_for_each_entry_safe(p, n, &items, iovntfd_hnode) { in mshv_eventfd_release()
828 hlist_del(&p->iovntfd_hnode); in mshv_eventfd_release()
829 ioeventfd_release(p, pt->pt_id); in mshv_eventfd_release()