1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/mm.h>
5 #include <linux/slab.h>
6 #include <linux/eventfd.h>
7 #include <linux/eventpoll.h>
8 #include <linux/io_uring.h>
9 #include <linux/io_uring_types.h>
10
11 #include "io-wq.h"
12 #include "eventfd.h"
13
14 struct io_ev_fd {
15 struct eventfd_ctx *cq_ev_fd;
16 unsigned int eventfd_async;
17 /* protected by ->completion_lock */
18 unsigned last_cq_tail;
19 refcount_t refs;
20 atomic_t ops;
21 struct rcu_head rcu;
22 };
23
24 enum {
25 IO_EVENTFD_OP_SIGNAL_BIT,
26 };
27
io_eventfd_free(struct rcu_head * rcu)28 static void io_eventfd_free(struct rcu_head *rcu)
29 {
30 struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
31
32 eventfd_ctx_put(ev_fd->cq_ev_fd);
33 kfree(ev_fd);
34 }
35
io_eventfd_put(struct io_ev_fd * ev_fd)36 static void io_eventfd_put(struct io_ev_fd *ev_fd)
37 {
38 if (refcount_dec_and_test(&ev_fd->refs))
39 call_rcu(&ev_fd->rcu, io_eventfd_free);
40 }
41
io_eventfd_do_signal(struct rcu_head * rcu)42 static void io_eventfd_do_signal(struct rcu_head *rcu)
43 {
44 struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
45
46 eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
47 io_eventfd_put(ev_fd);
48 }
49
io_eventfd_release(struct io_ev_fd * ev_fd,bool put_ref)50 static void io_eventfd_release(struct io_ev_fd *ev_fd, bool put_ref)
51 {
52 if (put_ref)
53 io_eventfd_put(ev_fd);
54 rcu_read_unlock();
55 }
56
57 /*
58 * Returns true if the caller should put the ev_fd reference, false if not.
59 */
__io_eventfd_signal(struct io_ev_fd * ev_fd)60 static bool __io_eventfd_signal(struct io_ev_fd *ev_fd)
61 {
62 if (eventfd_signal_allowed()) {
63 eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
64 return true;
65 }
66 if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) {
67 call_rcu_hurry(&ev_fd->rcu, io_eventfd_do_signal);
68 return false;
69 }
70 return true;
71 }
72
73 /*
74 * Trigger if eventfd_async isn't set, or if it's set and the caller is
75 * an async worker. If ev_fd isn't valid, obviously return false.
76 */
io_eventfd_trigger(struct io_ev_fd * ev_fd)77 static bool io_eventfd_trigger(struct io_ev_fd *ev_fd)
78 {
79 if (ev_fd)
80 return !ev_fd->eventfd_async || io_wq_current_is_worker();
81 return false;
82 }
83
84 /*
85 * On success, returns with an ev_fd reference grabbed and the RCU read
86 * lock held.
87 */
io_eventfd_grab(struct io_ring_ctx * ctx)88 static struct io_ev_fd *io_eventfd_grab(struct io_ring_ctx *ctx)
89 {
90 struct io_ev_fd *ev_fd;
91
92 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
93 return NULL;
94
95 rcu_read_lock();
96
97 /*
98 * rcu_dereference ctx->io_ev_fd once and use it for both for checking
99 * and eventfd_signal
100 */
101 ev_fd = rcu_dereference(ctx->io_ev_fd);
102
103 /*
104 * Check again if ev_fd exists in case an io_eventfd_unregister call
105 * completed between the NULL check of ctx->io_ev_fd at the start of
106 * the function and rcu_read_lock.
107 */
108 if (io_eventfd_trigger(ev_fd) && refcount_inc_not_zero(&ev_fd->refs))
109 return ev_fd;
110
111 rcu_read_unlock();
112 return NULL;
113 }
114
io_eventfd_signal(struct io_ring_ctx * ctx)115 void io_eventfd_signal(struct io_ring_ctx *ctx)
116 {
117 struct io_ev_fd *ev_fd;
118
119 ev_fd = io_eventfd_grab(ctx);
120 if (ev_fd)
121 io_eventfd_release(ev_fd, __io_eventfd_signal(ev_fd));
122 }
123
io_eventfd_flush_signal(struct io_ring_ctx * ctx)124 void io_eventfd_flush_signal(struct io_ring_ctx *ctx)
125 {
126 struct io_ev_fd *ev_fd;
127
128 ev_fd = io_eventfd_grab(ctx);
129 if (ev_fd) {
130 bool skip, put_ref = true;
131
132 /*
133 * Eventfd should only get triggered when at least one event
134 * has been posted. Some applications rely on the eventfd
135 * notification count only changing IFF a new CQE has been
136 * added to the CQ ring. There's no dependency on 1:1
137 * relationship between how many times this function is called
138 * (and hence the eventfd count) and number of CQEs posted to
139 * the CQ ring.
140 */
141 spin_lock(&ctx->completion_lock);
142 skip = ctx->cached_cq_tail == ev_fd->last_cq_tail;
143 ev_fd->last_cq_tail = ctx->cached_cq_tail;
144 spin_unlock(&ctx->completion_lock);
145
146 if (!skip)
147 put_ref = __io_eventfd_signal(ev_fd);
148
149 io_eventfd_release(ev_fd, put_ref);
150 }
151 }
152
io_eventfd_register(struct io_ring_ctx * ctx,void __user * arg,unsigned int eventfd_async)153 int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
154 unsigned int eventfd_async)
155 {
156 struct io_ev_fd *ev_fd;
157 __s32 __user *fds = arg;
158 int fd;
159
160 ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
161 lockdep_is_held(&ctx->uring_lock));
162 if (ev_fd)
163 return -EBUSY;
164
165 if (copy_from_user(&fd, fds, sizeof(*fds)))
166 return -EFAULT;
167
168 ev_fd = kmalloc(sizeof(*ev_fd), GFP_KERNEL);
169 if (!ev_fd)
170 return -ENOMEM;
171
172 ev_fd->cq_ev_fd = eventfd_ctx_fdget(fd);
173 if (IS_ERR(ev_fd->cq_ev_fd)) {
174 int ret = PTR_ERR(ev_fd->cq_ev_fd);
175
176 kfree(ev_fd);
177 return ret;
178 }
179
180 spin_lock(&ctx->completion_lock);
181 ev_fd->last_cq_tail = ctx->cached_cq_tail;
182 spin_unlock(&ctx->completion_lock);
183
184 ev_fd->eventfd_async = eventfd_async;
185 ctx->has_evfd = true;
186 refcount_set(&ev_fd->refs, 1);
187 atomic_set(&ev_fd->ops, 0);
188 rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
189 return 0;
190 }
191
io_eventfd_unregister(struct io_ring_ctx * ctx)192 int io_eventfd_unregister(struct io_ring_ctx *ctx)
193 {
194 struct io_ev_fd *ev_fd;
195
196 ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
197 lockdep_is_held(&ctx->uring_lock));
198 if (ev_fd) {
199 ctx->has_evfd = false;
200 rcu_assign_pointer(ctx->io_ev_fd, NULL);
201 io_eventfd_put(ev_fd);
202 return 0;
203 }
204
205 return -ENXIO;
206 }
207