1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * VFIO generic eventfd code for IRQFD support.
4 * Derived from drivers/vfio/pci/vfio_pci_intrs.c
5 *
6 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
7 * Author: Alex Williamson <alex.williamson@redhat.com>
8 */
9
10 #include <linux/vfio.h>
11 #include <linux/eventfd.h>
12 #include <linux/file.h>
13 #include <linux/module.h>
14 #include <linux/slab.h>
15 #include "vfio.h"
16
17 static struct workqueue_struct *vfio_irqfd_cleanup_wq;
18 static DEFINE_SPINLOCK(virqfd_lock);
19
vfio_virqfd_init(void)20 int __init vfio_virqfd_init(void)
21 {
22 vfio_irqfd_cleanup_wq =
23 create_singlethread_workqueue("vfio-irqfd-cleanup");
24 if (!vfio_irqfd_cleanup_wq)
25 return -ENOMEM;
26
27 return 0;
28 }
29
vfio_virqfd_exit(void)30 void vfio_virqfd_exit(void)
31 {
32 destroy_workqueue(vfio_irqfd_cleanup_wq);
33 }
34
virqfd_deactivate(struct virqfd * virqfd)35 static void virqfd_deactivate(struct virqfd *virqfd)
36 {
37 queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown);
38 }
39
virqfd_wakeup(wait_queue_entry_t * wait,unsigned mode,int sync,void * key)40 static int virqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
41 {
42 struct virqfd *virqfd = container_of(wait, struct virqfd, wait);
43 __poll_t flags = key_to_poll(key);
44
45 if (flags & EPOLLIN) {
46 u64 cnt;
47 eventfd_ctx_do_read(virqfd->eventfd, &cnt);
48
49 /* An event has been signaled, call function */
50 if ((!virqfd->handler ||
51 virqfd->handler(virqfd->opaque, virqfd->data)) &&
52 virqfd->thread)
53 schedule_work(&virqfd->inject);
54 }
55
56 if (flags & EPOLLHUP) {
57 unsigned long flags;
58 spin_lock_irqsave(&virqfd_lock, flags);
59
60 /*
61 * The eventfd is closing, if the virqfd has not yet been
62 * queued for release, as determined by testing whether the
63 * virqfd pointer to it is still valid, queue it now. As
64 * with kvm irqfds, we know we won't race against the virqfd
65 * going away because we hold the lock to get here.
66 */
67 if (*(virqfd->pvirqfd) == virqfd) {
68 *(virqfd->pvirqfd) = NULL;
69 virqfd_deactivate(virqfd);
70 }
71
72 spin_unlock_irqrestore(&virqfd_lock, flags);
73 }
74
75 return 0;
76 }
77
virqfd_ptable_queue_proc(struct file * file,wait_queue_head_t * wqh,poll_table * pt)78 static void virqfd_ptable_queue_proc(struct file *file,
79 wait_queue_head_t *wqh, poll_table *pt)
80 {
81 struct virqfd *virqfd = container_of(pt, struct virqfd, pt);
82 add_wait_queue(wqh, &virqfd->wait);
83 }
84
virqfd_shutdown(struct work_struct * work)85 static void virqfd_shutdown(struct work_struct *work)
86 {
87 struct virqfd *virqfd = container_of(work, struct virqfd, shutdown);
88 u64 cnt;
89
90 eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt);
91 flush_work(&virqfd->inject);
92 eventfd_ctx_put(virqfd->eventfd);
93
94 kfree(virqfd);
95 }
96
virqfd_inject(struct work_struct * work)97 static void virqfd_inject(struct work_struct *work)
98 {
99 struct virqfd *virqfd = container_of(work, struct virqfd, inject);
100 if (virqfd->thread)
101 virqfd->thread(virqfd->opaque, virqfd->data);
102 }
103
virqfd_flush_inject(struct work_struct * work)104 static void virqfd_flush_inject(struct work_struct *work)
105 {
106 struct virqfd *virqfd = container_of(work, struct virqfd, flush_inject);
107
108 flush_work(&virqfd->inject);
109 }
110
vfio_virqfd_enable(void * opaque,int (* handler)(void *,void *),void (* thread)(void *,void *),void * data,struct virqfd ** pvirqfd,int fd)111 int vfio_virqfd_enable(void *opaque,
112 int (*handler)(void *, void *),
113 void (*thread)(void *, void *),
114 void *data, struct virqfd **pvirqfd, int fd)
115 {
116 struct eventfd_ctx *ctx;
117 struct virqfd *virqfd;
118 int ret = 0;
119 __poll_t events;
120
121 virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL_ACCOUNT);
122 if (!virqfd)
123 return -ENOMEM;
124
125 virqfd->pvirqfd = pvirqfd;
126 virqfd->opaque = opaque;
127 virqfd->handler = handler;
128 virqfd->thread = thread;
129 virqfd->data = data;
130
131 INIT_WORK(&virqfd->shutdown, virqfd_shutdown);
132 INIT_WORK(&virqfd->inject, virqfd_inject);
133 INIT_WORK(&virqfd->flush_inject, virqfd_flush_inject);
134
135 CLASS(fd, irqfd)(fd);
136 if (fd_empty(irqfd)) {
137 ret = -EBADF;
138 goto err_fd;
139 }
140
141 ctx = eventfd_ctx_fileget(fd_file(irqfd));
142 if (IS_ERR(ctx)) {
143 ret = PTR_ERR(ctx);
144 goto err_fd;
145 }
146
147 virqfd->eventfd = ctx;
148
149 /*
150 * virqfds can be released by closing the eventfd or directly
151 * through ioctl. These are both done through a workqueue, so
152 * we update the pointer to the virqfd under lock to avoid
153 * pushing multiple jobs to release the same virqfd.
154 */
155 spin_lock_irq(&virqfd_lock);
156
157 if (*pvirqfd) {
158 spin_unlock_irq(&virqfd_lock);
159 ret = -EBUSY;
160 goto err_busy;
161 }
162 *pvirqfd = virqfd;
163
164 spin_unlock_irq(&virqfd_lock);
165
166 /*
167 * Install our own custom wake-up handling so we are notified via
168 * a callback whenever someone signals the underlying eventfd.
169 */
170 init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup);
171 init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc);
172
173 events = vfs_poll(fd_file(irqfd), &virqfd->pt);
174
175 /*
176 * Check if there was an event already pending on the eventfd
177 * before we registered and trigger it as if we didn't miss it.
178 */
179 if (events & EPOLLIN) {
180 if ((!handler || handler(opaque, data)) && thread)
181 schedule_work(&virqfd->inject);
182 }
183 return 0;
184 err_busy:
185 eventfd_ctx_put(ctx);
186 err_fd:
187 kfree(virqfd);
188
189 return ret;
190 }
191 EXPORT_SYMBOL_GPL(vfio_virqfd_enable);
192
vfio_virqfd_disable(struct virqfd ** pvirqfd)193 void vfio_virqfd_disable(struct virqfd **pvirqfd)
194 {
195 unsigned long flags;
196
197 spin_lock_irqsave(&virqfd_lock, flags);
198
199 if (*pvirqfd) {
200 virqfd_deactivate(*pvirqfd);
201 *pvirqfd = NULL;
202 }
203
204 spin_unlock_irqrestore(&virqfd_lock, flags);
205
206 /*
207 * Block until we know all outstanding shutdown jobs have completed.
208 * Even if we don't queue the job, flush the wq to be sure it's
209 * been released.
210 */
211 flush_workqueue(vfio_irqfd_cleanup_wq);
212 }
213 EXPORT_SYMBOL_GPL(vfio_virqfd_disable);
214
vfio_virqfd_flush_thread(struct virqfd ** pvirqfd)215 void vfio_virqfd_flush_thread(struct virqfd **pvirqfd)
216 {
217 unsigned long flags;
218
219 spin_lock_irqsave(&virqfd_lock, flags);
220 if (*pvirqfd && (*pvirqfd)->thread)
221 queue_work(vfio_irqfd_cleanup_wq, &(*pvirqfd)->flush_inject);
222 spin_unlock_irqrestore(&virqfd_lock, flags);
223
224 flush_workqueue(vfio_irqfd_cleanup_wq);
225 }
226 EXPORT_SYMBOL_GPL(vfio_virqfd_flush_thread);
227