1 // SPDX-License-Identifier: GPL-2.0-only
3  * kvm eventfd support - use eventfd objects to signal various KVM events
17 #include <linux/wait.h>
46 	struct kvm *kvm = irqfd->kvm;  in irqfd_inject()
48 	if (!irqfd->resampler) {  in irqfd_inject()
49 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,  in irqfd_inject()
51 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,  in irqfd_inject()
55 			    irqfd->gsi, 1, false);  in irqfd_inject()
62 	list_for_each_entry_srcu(irqfd, &resampler->list, resampler_link,  in irqfd_resampler_notify()
63 				 srcu_read_lock_held(&resampler->kvm->irq_srcu))  in irqfd_resampler_notify()
64 		eventfd_signal(irqfd->resamplefd);  in irqfd_resampler_notify()
68  * Since resampler irqfds share an IRQ source ID, we de-assert once
70  * do multiple de-asserts or we risk racing with incoming re-asserts.
81 	kvm = resampler->kvm;  in irqfd_resampler_ack()
84 		    resampler->notifier.gsi, 0, false);  in irqfd_resampler_ack()
86 	idx = srcu_read_lock(&kvm->irq_srcu);  in irqfd_resampler_ack()
88 	srcu_read_unlock(&kvm->irq_srcu, idx);  in irqfd_resampler_ack()
94 	struct kvm_kernel_irqfd_resampler *resampler = irqfd->resampler;  in irqfd_resampler_shutdown()
95 	struct kvm *kvm = resampler->kvm;  in irqfd_resampler_shutdown()
97 	mutex_lock(&kvm->irqfds.resampler_lock);  in irqfd_resampler_shutdown()
99 	list_del_rcu(&irqfd->resampler_link);  in irqfd_resampler_shutdown()
101 	if (list_empty(&resampler->list)) {  in irqfd_resampler_shutdown()
102 		list_del_rcu(&resampler->link);  in irqfd_resampler_shutdown()
103 		kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier);  in irqfd_resampler_shutdown()
105 		 * synchronize_srcu_expedited(&kvm->irq_srcu) already called  in irqfd_resampler_shutdown()
109 			    resampler->notifier.gsi, 0, false);  in irqfd_resampler_shutdown()
112 		synchronize_srcu_expedited(&kvm->irq_srcu);  in irqfd_resampler_shutdown()
115 	mutex_unlock(&kvm->irqfds.resampler_lock);  in irqfd_resampler_shutdown()
119  * Race-free decouple logic (ordering is critical)
126 	struct kvm *kvm = irqfd->kvm;  in irqfd_shutdown()
130 	synchronize_srcu_expedited(&kvm->irq_srcu);  in irqfd_shutdown()
133 	 * Synchronize with the wait-queue and unhook ourselves to prevent  in irqfd_shutdown()
136 	eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt);  in irqfd_shutdown()
142 	flush_work(&irqfd->inject);  in irqfd_shutdown()
144 	if (irqfd->resampler) {  in irqfd_shutdown()
146 		eventfd_ctx_put(irqfd->resamplefd);  in irqfd_shutdown()
153 	irq_bypass_unregister_consumer(&irqfd->consumer);  in irqfd_shutdown()
155 	eventfd_ctx_put(irqfd->eventfd);  in irqfd_shutdown()
160 /* assumes kvm->irqfds.lock is held */
164 	return list_empty(&irqfd->list) ? false : true;  in irqfd_is_active()
170  * assumes kvm->irqfds.lock is held
177 	list_del_init(&irqfd->list);  in irqfd_deactivate()
179 	queue_work(irqfd_cleanup_wq, &irqfd->shutdown);  in irqfd_deactivate()
188 	return -EWOULDBLOCK;  in kvm_arch_set_irq_inatomic()
192  * Called with wqh->lock held and interrupts disabled
195 irqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)  in irqfd_wakeup()  argument
198 		container_of(wait, struct kvm_kernel_irqfd, wait);  in irqfd_wakeup()
201 	struct kvm *kvm = irqfd->kvm;  in irqfd_wakeup()
208 		eventfd_ctx_do_read(irqfd->eventfd, &cnt);  in irqfd_wakeup()
210 		idx = srcu_read_lock(&kvm->irq_srcu);  in irqfd_wakeup()
212 			seq = read_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_wakeup()
213 			irq = irqfd->irq_entry;  in irqfd_wakeup()
214 		} while (read_seqcount_retry(&irqfd->irq_entry_sc, seq));  in irqfd_wakeup()
218 					      false) == -EWOULDBLOCK)  in irqfd_wakeup()
219 			schedule_work(&irqfd->inject);  in irqfd_wakeup()
220 		srcu_read_unlock(&kvm->irq_srcu, idx);  in irqfd_wakeup()
228 		spin_lock_irqsave(&kvm->irqfds.lock, iflags);  in irqfd_wakeup()
234 		 * the wait-queue.  If it is already deactivated, we can  in irqfd_wakeup()
237 		 * other side is required to acquire wqh->lock, which we hold  in irqfd_wakeup()
242 		spin_unlock_irqrestore(&kvm->irqfds.lock, iflags);  in irqfd_wakeup()
254 	add_wait_queue_priority(wqh, &irqfd->wait);  in irqfd_ptable_queue_proc()
264 	n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);  in irqfd_update()
266 	write_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_update()
270 		irqfd->irq_entry = *e;  in irqfd_update()
272 		irqfd->irq_entry.type = 0;  in irqfd_update()
274 	write_seqcount_end(&irqfd->irq_entry_sc);  in irqfd_update()
313 		return -EAGAIN;  in kvm_irqfd_assign()
316 		return -EINVAL;  in kvm_irqfd_assign()
320 		return -ENOMEM;  in kvm_irqfd_assign()
322 	irqfd->kvm = kvm;  in kvm_irqfd_assign()
323 	irqfd->gsi = args->gsi;  in kvm_irqfd_assign()
324 	INIT_LIST_HEAD(&irqfd->list);  in kvm_irqfd_assign()
325 	INIT_WORK(&irqfd->inject, irqfd_inject);  in kvm_irqfd_assign()
326 	INIT_WORK(&irqfd->shutdown, irqfd_shutdown);  in kvm_irqfd_assign()
327 	seqcount_spinlock_init(&irqfd->irq_entry_sc, &kvm->irqfds.lock);  in kvm_irqfd_assign()
329 	CLASS(fd, f)(args->fd);  in kvm_irqfd_assign()
331 		ret = -EBADF;  in kvm_irqfd_assign()
341 	irqfd->eventfd = eventfd;  in kvm_irqfd_assign()
343 	if (args->flags & KVM_IRQFD_FLAG_RESAMPLE) {  in kvm_irqfd_assign()
346 		resamplefd = eventfd_ctx_fdget(args->resamplefd);  in kvm_irqfd_assign()
352 		irqfd->resamplefd = resamplefd;  in kvm_irqfd_assign()
353 		INIT_LIST_HEAD(&irqfd->resampler_link);  in kvm_irqfd_assign()
355 		mutex_lock(&kvm->irqfds.resampler_lock);  in kvm_irqfd_assign()
358 				    &kvm->irqfds.resampler_list, link) {  in kvm_irqfd_assign()
359 			if (resampler->notifier.gsi == irqfd->gsi) {  in kvm_irqfd_assign()
360 				irqfd->resampler = resampler;  in kvm_irqfd_assign()
365 		if (!irqfd->resampler) {  in kvm_irqfd_assign()
369 				ret = -ENOMEM;  in kvm_irqfd_assign()
370 				mutex_unlock(&kvm->irqfds.resampler_lock);  in kvm_irqfd_assign()
374 			resampler->kvm = kvm;  in kvm_irqfd_assign()
375 			INIT_LIST_HEAD(&resampler->list);  in kvm_irqfd_assign()
376 			resampler->notifier.gsi = irqfd->gsi;  in kvm_irqfd_assign()
377 			resampler->notifier.irq_acked = irqfd_resampler_ack;  in kvm_irqfd_assign()
378 			INIT_LIST_HEAD(&resampler->link);  in kvm_irqfd_assign()
380 			list_add_rcu(&resampler->link, &kvm->irqfds.resampler_list);  in kvm_irqfd_assign()
382 						      &resampler->notifier);  in kvm_irqfd_assign()
383 			irqfd->resampler = resampler;  in kvm_irqfd_assign()
386 		list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);  in kvm_irqfd_assign()
387 		synchronize_srcu_expedited(&kvm->irq_srcu);  in kvm_irqfd_assign()
389 		mutex_unlock(&kvm->irqfds.resampler_lock);  in kvm_irqfd_assign()
393 	 * Install our own custom wake-up handling so we are notified via  in kvm_irqfd_assign()
396 	init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup);  in kvm_irqfd_assign()
397 	init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc);  in kvm_irqfd_assign()
399 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irqfd_assign()
402 	list_for_each_entry(tmp, &kvm->irqfds.items, list) {  in kvm_irqfd_assign()
403 		if (irqfd->eventfd != tmp->eventfd)  in kvm_irqfd_assign()
406 		ret = -EBUSY;  in kvm_irqfd_assign()
407 		spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_assign()
411 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_irqfd_assign()
414 	list_add_tail(&irqfd->list, &kvm->irqfds.items);  in kvm_irqfd_assign()
416 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_assign()
422 	events = vfs_poll(fd_file(f), &irqfd->pt);  in kvm_irqfd_assign()
425 		schedule_work(&irqfd->inject);  in kvm_irqfd_assign()
429 		irqfd->consumer.token = (void *)irqfd->eventfd;  in kvm_irqfd_assign()
430 		irqfd->consumer.add_producer = kvm_arch_irq_bypass_add_producer;  in kvm_irqfd_assign()
431 		irqfd->consumer.del_producer = kvm_arch_irq_bypass_del_producer;  in kvm_irqfd_assign()
432 		irqfd->consumer.stop = kvm_arch_irq_bypass_stop;  in kvm_irqfd_assign()
433 		irqfd->consumer.start = kvm_arch_irq_bypass_start;  in kvm_irqfd_assign()
434 		ret = irq_bypass_register_consumer(&irqfd->consumer);  in kvm_irqfd_assign()
437 				irqfd->consumer.token, ret);  in kvm_irqfd_assign()
441 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_irqfd_assign()
445 	if (irqfd->resampler)  in kvm_irqfd_assign()
459 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin)  in kvm_irq_has_notifier()  argument
464 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_irq_has_notifier()
465 	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);  in kvm_irq_has_notifier()
466 	if (gsi != -1)  in kvm_irq_has_notifier()
467 		hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,  in kvm_irq_has_notifier()
468 					  link, srcu_read_lock_held(&kvm->irq_srcu))  in kvm_irq_has_notifier()
469 			if (kian->gsi == gsi) {  in kvm_irq_has_notifier()
470 				srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_irq_has_notifier()
474 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_irq_has_notifier()
484 	hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,  in kvm_notify_acked_gsi()
485 				  link, srcu_read_lock_held(&kvm->irq_srcu))  in kvm_notify_acked_gsi()
486 		if (kian->gsi == gsi)  in kvm_notify_acked_gsi()
487 			kian->irq_acked(kian);  in kvm_notify_acked_gsi()
490 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)  in kvm_notify_acked_irq()  argument
494 	trace_kvm_ack_irq(irqchip, pin);  in kvm_notify_acked_irq()
496 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_notify_acked_irq()
497 	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);  in kvm_notify_acked_irq()
498 	if (gsi != -1)  in kvm_notify_acked_irq()
500 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_notify_acked_irq()
506 	mutex_lock(&kvm->irq_lock);  in kvm_register_irq_ack_notifier()
507 	hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list);  in kvm_register_irq_ack_notifier()
508 	mutex_unlock(&kvm->irq_lock);  in kvm_register_irq_ack_notifier()
515 	mutex_lock(&kvm->irq_lock);  in kvm_unregister_irq_ack_notifier()
516 	hlist_del_init_rcu(&kian->link);  in kvm_unregister_irq_ack_notifier()
517 	mutex_unlock(&kvm->irq_lock);  in kvm_unregister_irq_ack_notifier()
518 	synchronize_srcu_expedited(&kvm->irq_srcu);  in kvm_unregister_irq_ack_notifier()
531 	eventfd = eventfd_ctx_fdget(args->fd);  in kvm_irqfd_deassign()
535 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irqfd_deassign()
537 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {  in kvm_irqfd_deassign()
538 		if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) {  in kvm_irqfd_deassign()
545 			write_seqcount_begin(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
546 			irqfd->irq_entry.type = 0;  in kvm_irqfd_deassign()
547 			write_seqcount_end(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
552 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_deassign()
568 	if (args->flags & ~(KVM_IRQFD_FLAG_DEASSIGN | KVM_IRQFD_FLAG_RESAMPLE))  in kvm_irqfd()
569 		return -EINVAL;  in kvm_irqfd()
571 	if (args->flags & KVM_IRQFD_FLAG_DEASSIGN)  in kvm_irqfd()
586 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irqfd_release()
588 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)  in kvm_irqfd_release()
591 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irqfd_release()
603  * Caller must invoke synchronize_srcu_expedited(&kvm->irq_srcu) afterwards.
609 	spin_lock_irq(&kvm->irqfds.lock);  in kvm_irq_routing_update()
611 	list_for_each_entry(irqfd, &kvm->irqfds.items, list) {  in kvm_irq_routing_update()
614 		struct kvm_kernel_irq_routing_entry old = irqfd->irq_entry;  in kvm_irq_routing_update()
620 		if (irqfd->producer &&  in kvm_irq_routing_update()
621 		    kvm_arch_irqfd_route_changed(&old, &irqfd->irq_entry)) {  in kvm_irq_routing_update()
623 					irqfd->kvm, irqfd->producer->irq,  in kvm_irq_routing_update()
624 					irqfd->gsi, 1);  in kvm_irq_routing_update()
630 	spin_unlock_irq(&kvm->irqfds.lock);  in kvm_irq_routing_update()
635 				unsigned int pin)  in kvm_notify_irqfd_resampler()  argument
640 	idx = srcu_read_lock(&kvm->irq_srcu);  in kvm_notify_irqfd_resampler()
641 	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);  in kvm_notify_irqfd_resampler()
642 	if (gsi != -1) {  in kvm_notify_irqfd_resampler()
644 					 &kvm->irqfds.resampler_list, link,  in kvm_notify_irqfd_resampler()
645 					 srcu_read_lock_held(&kvm->irq_srcu)) {  in kvm_notify_irqfd_resampler()
646 			if (resampler->notifier.gsi == gsi) {  in kvm_notify_irqfd_resampler()
648 				srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_notify_irqfd_resampler()
653 	srcu_read_unlock(&kvm->irq_srcu, idx);  in kvm_notify_irqfd_resampler()
659  * create a host-wide workqueue for issuing deferred shutdown requests
665 	irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", 0, 0);  in kvm_irqfd_init()
667 		return -ENOMEM;  in kvm_irqfd_init()
679  * --------------------------------------------------------------------
684  * --------------------------------------------------------------------
707 	eventfd_ctx_put(p->eventfd);  in ioeventfd_release()
708 	list_del(&p->list);  in ioeventfd_release()
717 	if (addr != p->addr)  in ioeventfd_in_range()
721 	if (!p->length)  in ioeventfd_in_range()
725 	if (len != p->length)  in ioeventfd_in_range()
726 		/* address-range must be precise for a hit */  in ioeventfd_in_range()
729 	if (p->wildcard)  in ioeventfd_in_range()
754 	return _val == p->datamatch;  in ioeventfd_in_range()
765 		return -EOPNOTSUPP;  in ioeventfd_write()
767 	eventfd_signal(p->eventfd);  in ioeventfd_write()
788 /* assumes kvm->slots_lock held */
794 	list_for_each_entry(_p, &kvm->ioeventfds, list)  in ioeventfd_check_collision()
795 		if (_p->bus_idx == p->bus_idx &&  in ioeventfd_check_collision()
796 		    _p->addr == p->addr &&  in ioeventfd_check_collision()
797 		    (!_p->length || !p->length ||  in ioeventfd_check_collision()
798 		     (_p->length == p->length &&  in ioeventfd_check_collision()
799 		      (_p->wildcard || p->wildcard ||  in ioeventfd_check_collision()
800 		       _p->datamatch == p->datamatch))))  in ioeventfd_check_collision()
824 	eventfd = eventfd_ctx_fdget(args->fd);  in kvm_assign_ioeventfd_idx()
830 		ret = -ENOMEM;  in kvm_assign_ioeventfd_idx()
834 	INIT_LIST_HEAD(&p->list);  in kvm_assign_ioeventfd_idx()
835 	p->addr    = args->addr;  in kvm_assign_ioeventfd_idx()
836 	p->bus_idx = bus_idx;  in kvm_assign_ioeventfd_idx()
837 	p->length  = args->len;  in kvm_assign_ioeventfd_idx()
838 	p->eventfd = eventfd;  in kvm_assign_ioeventfd_idx()
841 	if (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH)  in kvm_assign_ioeventfd_idx()
842 		p->datamatch = args->datamatch;  in kvm_assign_ioeventfd_idx()
844 		p->wildcard = true;  in kvm_assign_ioeventfd_idx()
846 	mutex_lock(&kvm->slots_lock);  in kvm_assign_ioeventfd_idx()
850 		ret = -EEXIST;  in kvm_assign_ioeventfd_idx()
854 	kvm_iodevice_init(&p->dev, &ioeventfd_ops);  in kvm_assign_ioeventfd_idx()
856 	ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length,  in kvm_assign_ioeventfd_idx()
857 				      &p->dev);  in kvm_assign_ioeventfd_idx()
861 	kvm_get_bus(kvm, bus_idx)->ioeventfd_count++;  in kvm_assign_ioeventfd_idx()
862 	list_add_tail(&p->list, &kvm->ioeventfds);  in kvm_assign_ioeventfd_idx()
864 	mutex_unlock(&kvm->slots_lock);  in kvm_assign_ioeventfd_idx()
869 	mutex_unlock(&kvm->slots_lock);  in kvm_assign_ioeventfd_idx()
885 	int                       ret = -ENOENT;  in kvm_deassign_ioeventfd_idx()
888 	eventfd = eventfd_ctx_fdget(args->fd);  in kvm_deassign_ioeventfd_idx()
892 	wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH);  in kvm_deassign_ioeventfd_idx()
894 	mutex_lock(&kvm->slots_lock);  in kvm_deassign_ioeventfd_idx()
896 	list_for_each_entry(p, &kvm->ioeventfds, list) {  in kvm_deassign_ioeventfd_idx()
897 		if (p->bus_idx != bus_idx ||  in kvm_deassign_ioeventfd_idx()
898 		    p->eventfd != eventfd  ||  in kvm_deassign_ioeventfd_idx()
899 		    p->addr != args->addr  ||  in kvm_deassign_ioeventfd_idx()
900 		    p->length != args->len ||  in kvm_deassign_ioeventfd_idx()
901 		    p->wildcard != wildcard)  in kvm_deassign_ioeventfd_idx()
904 		if (!p->wildcard && p->datamatch != args->datamatch)  in kvm_deassign_ioeventfd_idx()
907 		kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);  in kvm_deassign_ioeventfd_idx()
910 			bus->ioeventfd_count--;  in kvm_deassign_ioeventfd_idx()
915 	mutex_unlock(&kvm->slots_lock);  in kvm_deassign_ioeventfd_idx()
924 	enum kvm_bus bus_idx = ioeventfd_bus_from_flags(args->flags);  in kvm_deassign_ioeventfd()
927 	if (!args->len && bus_idx == KVM_MMIO_BUS)  in kvm_deassign_ioeventfd()
939 	bus_idx = ioeventfd_bus_from_flags(args->flags);  in kvm_assign_ioeventfd()
940 	/* must be natural-word sized, or 0 to ignore length */  in kvm_assign_ioeventfd()
941 	switch (args->len) {  in kvm_assign_ioeventfd()
949 		return -EINVAL;  in kvm_assign_ioeventfd()
953 	if (args->addr + args->len < args->addr)  in kvm_assign_ioeventfd()
954 		return -EINVAL;  in kvm_assign_ioeventfd()
957 	if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK)  in kvm_assign_ioeventfd()
958 		return -EINVAL;  in kvm_assign_ioeventfd()
961 	if (!args->len && (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH))  in kvm_assign_ioeventfd()
962 		return -EINVAL;  in kvm_assign_ioeventfd()
971 	if (!args->len && bus_idx == KVM_MMIO_BUS) {  in kvm_assign_ioeventfd()
988 	if (args->flags & KVM_IOEVENTFD_FLAG_DEASSIGN)  in kvm_ioeventfd()
998 	spin_lock_init(&kvm->irqfds.lock);  in kvm_eventfd_init()
999 	INIT_LIST_HEAD(&kvm->irqfds.items);  in kvm_eventfd_init()
1000 	INIT_LIST_HEAD(&kvm->irqfds.resampler_list);  in kvm_eventfd_init()
1001 	mutex_init(&kvm->irqfds.resampler_lock);  in kvm_eventfd_init()
1003 	INIT_LIST_HEAD(&kvm->ioeventfds);  in kvm_eventfd_init()