30 	struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);  in io_eventfd_free()  local
32 	eventfd_ctx_put(ev_fd->cq_ev_fd);  in io_eventfd_free()
33 	kfree(ev_fd);  in io_eventfd_free()
36 static void io_eventfd_put(struct io_ev_fd *ev_fd)  in io_eventfd_put()  argument
38 	if (refcount_dec_and_test(&ev_fd->refs))  in io_eventfd_put()
39 		call_rcu(&ev_fd->rcu, io_eventfd_free);  in io_eventfd_put()
44 	struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);  in io_eventfd_do_signal()  local
46 	atomic_andnot(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops);  in io_eventfd_do_signal()
47 	eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);  in io_eventfd_do_signal()
48 	io_eventfd_put(ev_fd);  in io_eventfd_do_signal()
54 static bool __io_eventfd_signal(struct io_ev_fd *ev_fd)  in __io_eventfd_signal()  argument
57 		eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);  in __io_eventfd_signal()
60 	if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) {  in __io_eventfd_signal()
61 		call_rcu_hurry(&ev_fd->rcu, io_eventfd_do_signal);  in __io_eventfd_signal()
71 static bool io_eventfd_trigger(struct io_ev_fd *ev_fd)  in io_eventfd_trigger()  argument
73 	return !ev_fd->eventfd_async || io_wq_current_is_worker();  in io_eventfd_trigger()
79 	struct io_ev_fd *ev_fd;  in io_eventfd_signal()  local
89 	ev_fd = rcu_dereference(ctx->io_ev_fd);  in io_eventfd_signal()
95 	if (!ev_fd)  in io_eventfd_signal()
97 	if (!io_eventfd_trigger(ev_fd) || !refcount_inc_not_zero(&ev_fd->refs))  in io_eventfd_signal()
111 		skip = ctx->cached_cq_tail == ev_fd->last_cq_tail;  in io_eventfd_signal()
112 		ev_fd->last_cq_tail = ctx->cached_cq_tail;  in io_eventfd_signal()
116 	if (skip || __io_eventfd_signal(ev_fd))  in io_eventfd_signal()
117 		io_eventfd_put(ev_fd);  in io_eventfd_signal()
123 	struct io_ev_fd *ev_fd;  in io_eventfd_register()  local
127 	ev_fd = rcu_dereference_protected(ctx->io_ev_fd,  in io_eventfd_register()
129 	if (ev_fd)  in io_eventfd_register()
135 	ev_fd = kmalloc_obj(*ev_fd);  in io_eventfd_register()
136 	if (!ev_fd)  in io_eventfd_register()
139 	ev_fd->cq_ev_fd = eventfd_ctx_fdget(fd);  in io_eventfd_register()
140 	if (IS_ERR(ev_fd->cq_ev_fd)) {  in io_eventfd_register()
141 		int ret = PTR_ERR(ev_fd->cq_ev_fd);  in io_eventfd_register()
143 		kfree(ev_fd);  in io_eventfd_register()
148 	ev_fd->last_cq_tail = ctx->cached_cq_tail;  in io_eventfd_register()
151 	ev_fd->eventfd_async = eventfd_async;  in io_eventfd_register()
153 	refcount_set(&ev_fd->refs, 1);  in io_eventfd_register()
154 	atomic_set(&ev_fd->ops, 0);  in io_eventfd_register()
155 	rcu_assign_pointer(ctx->io_ev_fd, ev_fd);  in io_eventfd_register()
161 	struct io_ev_fd *ev_fd;  in io_eventfd_unregister()  local
163 	ev_fd = rcu_dereference_protected(ctx->io_ev_fd,  in io_eventfd_unregister()
165 	if (ev_fd) {  in io_eventfd_unregister()
168 		io_eventfd_put(ev_fd);  in io_eventfd_unregister()