36 	struct io_kiocb *req;  member
73 static bool io_poll_get_ownership_slowpath(struct io_kiocb *req)  in io_poll_get_ownership_slowpath()  argument
82 	v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs);  in io_poll_get_ownership_slowpath()
85 	return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);  in io_poll_get_ownership_slowpath()
94 static inline bool io_poll_get_ownership(struct io_kiocb *req)  in io_poll_get_ownership()  argument
96 	if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS))  in io_poll_get_ownership()
97 		return io_poll_get_ownership_slowpath(req);  in io_poll_get_ownership()
98 	return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);  in io_poll_get_ownership()
101 static void io_poll_mark_cancelled(struct io_kiocb *req)  in io_poll_mark_cancelled()  argument
103 	atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs);  in io_poll_mark_cancelled()
106 static struct io_poll *io_poll_get_double(struct io_kiocb *req)  in io_poll_get_double()  argument
109 	if (req->opcode == IORING_OP_POLL_ADD)  in io_poll_get_double()
110 		return req->async_data;  in io_poll_get_double()
111 	return req->apoll->double_poll;  in io_poll_get_double()
114 static struct io_poll *io_poll_get_single(struct io_kiocb *req)  in io_poll_get_single()  argument
116 	if (req->opcode == IORING_OP_POLL_ADD)  in io_poll_get_single()
117 		return io_kiocb_to_cmd(req, struct io_poll);  in io_poll_get_single()
118 	return &req->apoll->poll;  in io_poll_get_single()
121 static void io_poll_req_insert(struct io_kiocb *req)  in io_poll_req_insert()  argument
123 	struct io_hash_table *table = &req->ctx->cancel_table;  in io_poll_req_insert()
124 	u32 index = hash_long(req->cqe.user_data, table->hash_bits);  in io_poll_req_insert()
126 	lockdep_assert_held(&req->ctx->uring_lock);  in io_poll_req_insert()
128 	hlist_add_head(&req->hash_node, &table->hbs[index].list);  in io_poll_req_insert()
171 static void io_poll_remove_entries(struct io_kiocb *req)  in io_poll_remove_entries()  argument
177 	if (!(req->flags & (REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL)))  in io_poll_remove_entries()
196 	if (req->flags & REQ_F_SINGLE_POLL)  in io_poll_remove_entries()
197 		io_poll_remove_entry(io_poll_get_single(req));  in io_poll_remove_entries()
198 	if (req->flags & REQ_F_DOUBLE_POLL)  in io_poll_remove_entries()
199 		io_poll_remove_entry(io_poll_get_double(req));  in io_poll_remove_entries()
211 static void __io_poll_execute(struct io_kiocb *req, int mask)  in __io_poll_execute()  argument
215 	io_req_set_res(req, mask, 0);  in __io_poll_execute()
216 	req->io_task_work.func = io_poll_task_func;  in __io_poll_execute()
218 	trace_io_uring_task_add(req, mask);  in __io_poll_execute()
220 	if (!(req->flags & REQ_F_POLL_NO_LAZY))  in __io_poll_execute()
222 	__io_req_task_work_add(req, flags);  in __io_poll_execute()
225 static inline void io_poll_execute(struct io_kiocb *req, int res)  in io_poll_execute()  argument
227 	if (io_poll_get_ownership(req))  in io_poll_execute()
228 		__io_poll_execute(req, res);  in io_poll_execute()
241 static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)  in io_poll_check_events()  argument
249 		v = atomic_read(&req->poll_refs);  in io_poll_check_events()
263 				req->cqe.res = 0;  in io_poll_check_events()
266 				req->cqe.res = 0;  in io_poll_check_events()
272 				atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs);  in io_poll_check_events()
278 		if (!req->cqe.res) {  in io_poll_check_events()
279 			struct poll_table_struct pt = { ._key = req->apoll_events };  in io_poll_check_events()
280 			req->cqe.res = vfs_poll(req->file, &pt) & req->apoll_events;  in io_poll_check_events()
287 			if (unlikely(!req->cqe.res)) {  in io_poll_check_events()
289 				if (!(req->apoll_events & EPOLLONESHOT))  in io_poll_check_events()
294 		if (req->apoll_events & EPOLLONESHOT)  in io_poll_check_events()
298 		if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {  in io_poll_check_events()
299 			__poll_t mask = mangle_poll(req->cqe.res &  in io_poll_check_events()
300 						    req->apoll_events);  in io_poll_check_events()
302 			if (!io_req_post_cqe(req, mask, IORING_CQE_F_MORE)) {  in io_poll_check_events()
303 				io_req_set_res(req, mask, 0);  in io_poll_check_events()
307 			int ret = io_poll_issue(req, tw);  in io_poll_check_events()
318 		req->cqe.res = 0;  in io_poll_check_events()
325 	} while (atomic_sub_return(v, &req->poll_refs) & IO_POLL_REF_MASK);  in io_poll_check_events()
327 	io_napi_add(req);  in io_poll_check_events()
333 	struct io_kiocb *req = tw_req.req;  in io_poll_task_func()  local
336 	ret = io_poll_check_events(req, tw);  in io_poll_task_func()
340 		__io_poll_execute(req, 0);  in io_poll_task_func()
343 	io_poll_remove_entries(req);  in io_poll_task_func()
345 	hash_del(&req->hash_node);  in io_poll_task_func()
347 	if (req->opcode == IORING_OP_POLL_ADD) {  in io_poll_task_func()
351 			poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_task_func()
352 			req->cqe.res = mangle_poll(req->cqe.res & poll->events);  in io_poll_task_func()
357 			req->cqe.res = ret;  in io_poll_task_func()
358 			req_set_fail(req);  in io_poll_task_func()
361 		io_req_set_res(req, req->cqe.res, 0);  in io_poll_task_func()
364 		io_tw_lock(req->ctx, tw);  in io_poll_task_func()
371 			io_req_defer_failed(req, ret);  in io_poll_task_func()
375 static void io_poll_cancel_req(struct io_kiocb *req)  in io_poll_cancel_req()  argument
377 	io_poll_mark_cancelled(req);  in io_poll_cancel_req()
379 	io_poll_execute(req, 0);  in io_poll_cancel_req()
384 static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll)  in io_pollfree_wake()  argument
386 	io_poll_mark_cancelled(req);  in io_pollfree_wake()
388 	io_poll_execute(req, 0);  in io_pollfree_wake()
396 	struct io_kiocb *req = wqe_to_req(wait);  in io_poll_wake()  local
401 		return io_pollfree_wake(req, poll);  in io_poll_wake()
407 	if (io_poll_get_ownership(req)) {  in io_poll_wake()
420 				req->flags &= ~REQ_F_DOUBLE_POLL;  in io_poll_wake()
422 				req->flags &= ~REQ_F_SINGLE_POLL;  in io_poll_wake()
424 		__io_poll_execute(req, mask);  in io_poll_wake()
430 static bool io_poll_double_prepare(struct io_kiocb *req)  in io_poll_double_prepare()  argument
433 	struct io_poll *poll = io_poll_get_single(req);  in io_poll_double_prepare()
446 		req->flags |= REQ_F_DOUBLE_POLL;  in io_poll_double_prepare()
447 		if (req->opcode == IORING_OP_POLL_ADD)  in io_poll_double_prepare()
448 			req->flags |= REQ_F_ASYNC_DATA;  in io_poll_double_prepare()
459 	struct io_kiocb *req = pt->req;  in __io_queue_proc()  local
460 	unsigned long wqe_private = (unsigned long) req;  in __io_queue_proc()
490 		if (!io_poll_double_prepare(req)) {  in __io_queue_proc()
498 		req->flags |= REQ_F_SINGLE_POLL;  in __io_queue_proc()
516 	struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll);  in io_poll_queue_proc()
519 			(struct io_poll **) &pt->req->async_data);  in io_poll_queue_proc()
522 static bool io_poll_can_finish_inline(struct io_kiocb *req,  in io_poll_can_finish_inline()  argument
525 	return pt->owning || io_poll_get_ownership(req);  in io_poll_can_finish_inline()
528 static void io_poll_add_hash(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_add_hash()  argument
530 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_add_hash()
533 	io_poll_req_insert(req);  in io_poll_add_hash()
543 static int __io_arm_poll_handler(struct io_kiocb *req,  in __io_arm_poll_handler()  argument
548 	INIT_HLIST_NODE(&req->hash_node);  in __io_arm_poll_handler()
550 	poll->file = req->file;  in __io_arm_poll_handler()
551 	req->apoll_events = poll->events;  in __io_arm_poll_handler()
554 	ipt->req = req;  in __io_arm_poll_handler()
569 	atomic_set(&req->poll_refs, (int)ipt->owning);  in __io_arm_poll_handler()
579 		req->flags |= REQ_F_POLL_NO_LAZY;  in __io_arm_poll_handler()
581 	mask = vfs_poll(req->file, &ipt->pt) & poll->events;  in __io_arm_poll_handler()
584 		io_poll_remove_entries(req);  in __io_arm_poll_handler()
586 		if (!io_poll_can_finish_inline(req, ipt)) {  in __io_arm_poll_handler()
587 			io_poll_mark_cancelled(req);  in __io_arm_poll_handler()
598 		if (!io_poll_can_finish_inline(req, ipt)) {  in __io_arm_poll_handler()
599 			io_poll_add_hash(req, issue_flags);  in __io_arm_poll_handler()
602 		io_poll_remove_entries(req);  in __io_arm_poll_handler()
608 	io_poll_add_hash(req, issue_flags);  in __io_arm_poll_handler()
611 	    io_poll_can_finish_inline(req, ipt)) {  in __io_arm_poll_handler()
612 		__io_poll_execute(req, mask);  in __io_arm_poll_handler()
615 	io_napi_add(req);  in __io_arm_poll_handler()
622 		if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)  in __io_arm_poll_handler()
623 			__io_poll_execute(req, 0);  in __io_arm_poll_handler()
632 	struct async_poll *apoll = pt->req->apoll;  in io_async_queue_proc()
645 static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req,  in io_req_alloc_apoll()  argument
648 	struct io_ring_ctx *ctx = req->ctx;  in io_req_alloc_apoll()
651 	if (req->flags & REQ_F_POLLED) {  in io_req_alloc_apoll()
652 		apoll = req->apoll;  in io_req_alloc_apoll()
664 	req->apoll = apoll;  in io_req_alloc_apoll()
670 int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask)  in io_arm_apoll()  argument
677 	if (!io_file_can_poll(req))  in io_arm_apoll()
679 	if (!(req->flags & REQ_F_APOLL_MULTISHOT))  in io_arm_apoll()
682 	apoll = io_req_alloc_apoll(req, issue_flags);  in io_arm_apoll()
685 	req->flags &= ~(REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL);  in io_arm_apoll()
686 	req->flags |= REQ_F_POLLED;  in io_arm_apoll()
689 	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags);  in io_arm_apoll()
692 	trace_io_uring_poll_arm(req, mask, apoll->poll.events);  in io_arm_apoll()
696 int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)  in io_arm_poll_handler()  argument
698 	const struct io_issue_def *def = &io_issue_defs[req->opcode];  in io_arm_poll_handler()
703 	if (!io_file_can_poll(req))  in io_arm_poll_handler()
710 		if (req->flags & REQ_F_CLEAR_POLLIN)  in io_arm_poll_handler()
718 	return io_arm_apoll(req, issue_flags, mask);  in io_arm_poll_handler()
729 	struct io_kiocb *req;  in io_poll_remove_all()  local
738 		hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) {  in io_poll_remove_all()
739 			if (io_match_task_safe(req, tctx, cancel_all)) {  in io_poll_remove_all()
740 				hlist_del_init(&req->hash_node);  in io_poll_remove_all()
741 				io_poll_cancel_req(req);  in io_poll_remove_all()
752 	struct io_kiocb *req;  in io_poll_find()  local
756 	hlist_for_each_entry(req, &hb->list, hash_node) {  in io_poll_find()
757 		if (cd->data != req->cqe.user_data)  in io_poll_find()
759 		if (poll_only && req->opcode != IORING_OP_POLL_ADD)  in io_poll_find()
762 			if (io_cancel_match_sequence(req, cd->seq))  in io_poll_find()
765 		return req;  in io_poll_find()
774 	struct io_kiocb *req;  in io_poll_file_find()  local
780 		hlist_for_each_entry(req, &hb->list, hash_node) {  in io_poll_file_find()
781 			if (io_cancel_req_match(req, cd))  in io_poll_file_find()
782 				return req;  in io_poll_file_find()
788 static int io_poll_disarm(struct io_kiocb *req)  in io_poll_disarm()  argument
790 	if (!req)  in io_poll_disarm()
792 	if (!io_poll_get_ownership(req))  in io_poll_disarm()
794 	io_poll_remove_entries(req);  in io_poll_disarm()
795 	hash_del(&req->hash_node);  in io_poll_disarm()
801 	struct io_kiocb *req;  in __io_poll_cancel()  local
805 		req = io_poll_file_find(ctx, cd);  in __io_poll_cancel()
807 		req = io_poll_find(ctx, false, cd);  in __io_poll_cancel()
809 	if (req) {  in __io_poll_cancel()
810 		io_poll_cancel_req(req);  in __io_poll_cancel()
844 int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_poll_remove_prep()  argument
846 	struct io_poll_update *upd = io_kiocb_to_cmd(req, struct io_poll_update);  in io_poll_remove_prep()
874 int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_poll_add_prep()  argument
876 	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_add_prep()
884 	if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP))  in io_poll_add_prep()
891 int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_add()  argument
893 	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);  in io_poll_add()
899 	ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags);  in io_poll_add()
901 		io_req_set_res(req, ipt.result_mask, 0);  in io_poll_add()
907 int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)  in io_poll_remove()  argument
909 	struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update);  in io_poll_remove()
910 	struct io_ring_ctx *ctx = req->ctx;  in io_poll_remove()
957 		req_set_fail(req);  in io_poll_remove()
961 	io_req_set_res(req, ret, 0);  in io_poll_remove()