1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/mm.h> 5 #include <linux/slab.h> 6 #include <linux/eventfd.h> 7 #include <linux/eventpoll.h> 8 #include <linux/io_uring.h> 9 #include <linux/io_uring_types.h> 10 11 #include "io-wq.h" 12 #include "eventfd.h" 13 14 struct io_ev_fd { 15 struct eventfd_ctx *cq_ev_fd; 16 unsigned int eventfd_async; 17 /* protected by ->completion_lock */ 18 unsigned last_cq_tail; 19 refcount_t refs; 20 atomic_t ops; 21 struct rcu_head rcu; 22 }; 23 24 enum { 25 IO_EVENTFD_OP_SIGNAL_BIT, 26 }; 27 28 static void io_eventfd_free(struct rcu_head *rcu) 29 { 30 struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu); 31 32 eventfd_ctx_put(ev_fd->cq_ev_fd); 33 kfree(ev_fd); 34 } 35 36 static void io_eventfd_put(struct io_ev_fd *ev_fd) 37 { 38 if (refcount_dec_and_test(&ev_fd->refs)) 39 call_rcu(&ev_fd->rcu, io_eventfd_free); 40 } 41 42 static void io_eventfd_do_signal(struct rcu_head *rcu) 43 { 44 struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu); 45 46 eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); 47 io_eventfd_put(ev_fd); 48 } 49 50 /* 51 * Returns true if the caller should put the ev_fd reference, false if not. 52 */ 53 static bool __io_eventfd_signal(struct io_ev_fd *ev_fd) 54 { 55 if (eventfd_signal_allowed()) { 56 eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); 57 return true; 58 } 59 if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) { 60 call_rcu_hurry(&ev_fd->rcu, io_eventfd_do_signal); 61 return false; 62 } 63 return true; 64 } 65 66 /* 67 * Trigger if eventfd_async isn't set, or if it's set and the caller is 68 * an async worker. 69 */ 70 static bool io_eventfd_trigger(struct io_ev_fd *ev_fd) 71 { 72 return !ev_fd->eventfd_async || io_wq_current_is_worker(); 73 } 74 75 void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) 76 { 77 bool skip = false; 78 struct io_ev_fd *ev_fd; 79 80 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) 81 return; 82 83 guard(rcu)(); 84 ev_fd = rcu_dereference(ctx->io_ev_fd); 85 /* 86 * Check again if ev_fd exists in case an io_eventfd_unregister call 87 * completed between the NULL check of ctx->io_ev_fd at the start of 88 * the function and rcu_read_lock. 89 */ 90 if (!ev_fd) 91 return; 92 if (!io_eventfd_trigger(ev_fd) || !refcount_inc_not_zero(&ev_fd->refs)) 93 return; 94 95 if (cqe_event) { 96 /* 97 * Eventfd should only get triggered when at least one event 98 * has been posted. Some applications rely on the eventfd 99 * notification count only changing IFF a new CQE has been 100 * added to the CQ ring. There's no dependency on 1:1 101 * relationship between how many times this function is called 102 * (and hence the eventfd count) and number of CQEs posted to 103 * the CQ ring. 104 */ 105 spin_lock(&ctx->completion_lock); 106 skip = ctx->cached_cq_tail == ev_fd->last_cq_tail; 107 ev_fd->last_cq_tail = ctx->cached_cq_tail; 108 spin_unlock(&ctx->completion_lock); 109 } 110 111 if (skip || __io_eventfd_signal(ev_fd)) 112 io_eventfd_put(ev_fd); 113 } 114 115 int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, 116 unsigned int eventfd_async) 117 { 118 struct io_ev_fd *ev_fd; 119 __s32 __user *fds = arg; 120 int fd; 121 122 ev_fd = rcu_dereference_protected(ctx->io_ev_fd, 123 lockdep_is_held(&ctx->uring_lock)); 124 if (ev_fd) 125 return -EBUSY; 126 127 if (copy_from_user(&fd, fds, sizeof(*fds))) 128 return -EFAULT; 129 130 ev_fd = kmalloc(sizeof(*ev_fd), GFP_KERNEL); 131 if (!ev_fd) 132 return -ENOMEM; 133 134 ev_fd->cq_ev_fd = eventfd_ctx_fdget(fd); 135 if (IS_ERR(ev_fd->cq_ev_fd)) { 136 int ret = PTR_ERR(ev_fd->cq_ev_fd); 137 138 kfree(ev_fd); 139 return ret; 140 } 141 142 spin_lock(&ctx->completion_lock); 143 ev_fd->last_cq_tail = ctx->cached_cq_tail; 144 spin_unlock(&ctx->completion_lock); 145 146 ev_fd->eventfd_async = eventfd_async; 147 ctx->has_evfd = true; 148 refcount_set(&ev_fd->refs, 1); 149 atomic_set(&ev_fd->ops, 0); 150 rcu_assign_pointer(ctx->io_ev_fd, ev_fd); 151 return 0; 152 } 153 154 int io_eventfd_unregister(struct io_ring_ctx *ctx) 155 { 156 struct io_ev_fd *ev_fd; 157 158 ev_fd = rcu_dereference_protected(ctx->io_ev_fd, 159 lockdep_is_held(&ctx->uring_lock)); 160 if (ev_fd) { 161 ctx->has_evfd = false; 162 rcu_assign_pointer(ctx->io_ev_fd, NULL); 163 io_eventfd_put(ev_fd); 164 return 0; 165 } 166 167 return -ENXIO; 168 } 169