Lines Matching +full:wait +full:- +full:retry +full:- +full:us
1 // SPDX-License-Identifier: GPL-2.0
49 * We usually have 1-2 refs taken, 128 is more than enough and we want to
56 static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
61 unsigned long priv = (unsigned long)wqe->private; in wqe_to_req()
68 unsigned long priv = (unsigned long)wqe->private; in wqe_is_double()
79 * grabbing the ownership. Instead of incrementing set a retry flag in io_poll_get_ownership_slowpath()
82 v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs); in io_poll_get_ownership_slowpath()
85 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); in io_poll_get_ownership_slowpath()
89 * If refs part of ->poll_refs (see IO_POLL_REF_MASK) is 0, it's free. We can
96 if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS)) in io_poll_get_ownership()
98 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); in io_poll_get_ownership()
103 atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs); in io_poll_mark_cancelled()
108 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */ in io_poll_get_double()
109 if (req->opcode == IORING_OP_POLL_ADD) in io_poll_get_double()
110 return req->async_data; in io_poll_get_double()
111 return req->apoll->double_poll; in io_poll_get_double()
116 if (req->opcode == IORING_OP_POLL_ADD) in io_poll_get_single()
118 return &req->apoll->poll; in io_poll_get_single()
123 struct io_hash_table *table = &req->ctx->cancel_table; in io_poll_req_insert()
124 u32 index = hash_long(req->cqe.user_data, table->hash_bits); in io_poll_req_insert()
125 struct io_hash_bucket *hb = &table->hbs[index]; in io_poll_req_insert()
127 spin_lock(&hb->lock); in io_poll_req_insert()
128 hlist_add_head(&req->hash_node, &hb->list); in io_poll_req_insert()
129 spin_unlock(&hb->lock); in io_poll_req_insert()
134 struct io_hash_table *table = &req->ctx->cancel_table; in io_poll_req_delete()
135 u32 index = hash_long(req->cqe.user_data, table->hash_bits); in io_poll_req_delete()
136 spinlock_t *lock = &table->hbs[index].lock; in io_poll_req_delete()
139 hash_del(&req->hash_node); in io_poll_req_delete()
145 struct io_hash_table *table = &req->ctx->cancel_table_locked; in io_poll_req_insert_locked()
146 u32 index = hash_long(req->cqe.user_data, table->hash_bits); in io_poll_req_insert_locked()
148 lockdep_assert_held(&req->ctx->uring_lock); in io_poll_req_insert_locked()
150 hlist_add_head(&req->hash_node, &table->hbs[index].list); in io_poll_req_insert_locked()
155 struct io_ring_ctx *ctx = req->ctx; in io_poll_tw_hash_eject()
157 if (req->flags & REQ_F_HASH_LOCKED) { in io_poll_tw_hash_eject()
159 * ->cancel_table_locked is protected by ->uring_lock in in io_poll_tw_hash_eject()
161 * already grabbed the mutex for us, but there is a chance it in io_poll_tw_hash_eject()
165 hash_del(&req->hash_node); in io_poll_tw_hash_eject()
166 req->flags &= ~REQ_F_HASH_LOCKED; in io_poll_tw_hash_eject()
174 poll->head = NULL; in io_init_poll_iocb()
177 poll->events = events | IO_POLL_UNMASK; in io_init_poll_iocb()
178 INIT_LIST_HEAD(&poll->wait.entry); in io_init_poll_iocb()
179 init_waitqueue_func_entry(&poll->wait, io_poll_wake); in io_init_poll_iocb()
184 struct wait_queue_head *head = smp_load_acquire(&poll->head); in io_poll_remove_entry()
187 spin_lock_irq(&head->lock); in io_poll_remove_entry()
188 list_del_init(&poll->wait.entry); in io_poll_remove_entry()
189 poll->head = NULL; in io_poll_remove_entry()
190 spin_unlock_irq(&head->lock); in io_poll_remove_entry()
200 if (!(req->flags & (REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL))) in io_poll_remove_entries()
205 * wake_up_pollfree() will wait for us. However, taking the waitqueue in io_poll_remove_entries()
209 * all users of wake_up_pollfree() will RCU-delay the actual free. If in io_poll_remove_entries()
211 * non-NULL, we can then lock it without the memory being freed out from in io_poll_remove_entries()
212 * under us. in io_poll_remove_entries()
219 if (req->flags & REQ_F_SINGLE_POLL) in io_poll_remove_entries()
221 if (req->flags & REQ_F_DOUBLE_POLL) in io_poll_remove_entries()
239 req->io_task_work.func = io_poll_task_func; in __io_poll_execute()
243 if (!(req->flags & REQ_F_POLL_NO_LAZY)) in __io_poll_execute()
261 * req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot
262 * poll and that the result is stored in req->cqe.
268 /* req->task == current here, checking PF_EXITING is safe */ in io_poll_check_events()
269 if (unlikely(req->task->flags & PF_EXITING)) in io_poll_check_events()
270 return -ECANCELED; in io_poll_check_events()
273 v = atomic_read(&req->poll_refs); in io_poll_check_events()
280 return -ECANCELED; in io_poll_check_events()
287 req->cqe.res = 0; in io_poll_check_events()
290 req->cqe.res = 0; in io_poll_check_events()
296 atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs); in io_poll_check_events()
302 if (!req->cqe.res) { in io_poll_check_events()
303 struct poll_table_struct pt = { ._key = req->apoll_events }; in io_poll_check_events()
304 req->cqe.res = vfs_poll(req->file, &pt) & req->apoll_events; in io_poll_check_events()
307 * it first. The above vfs_poll() doesn't add us back in io_poll_check_events()
311 if (unlikely(!req->cqe.res)) { in io_poll_check_events()
313 if (!(req->apoll_events & EPOLLONESHOT)) in io_poll_check_events()
318 if (req->apoll_events & EPOLLONESHOT) in io_poll_check_events()
322 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { in io_poll_check_events()
323 __poll_t mask = mangle_poll(req->cqe.res & in io_poll_check_events()
324 req->apoll_events); in io_poll_check_events()
341 req->cqe.res = 0; in io_poll_check_events()
344 * Release all references, retry if someone tried to restart in io_poll_check_events()
348 } while (atomic_sub_return(v, &req->poll_refs) & IO_POLL_REF_MASK); in io_poll_check_events()
368 if (req->opcode == IORING_OP_POLL_ADD) { in io_poll_task_func()
373 req->cqe.res = mangle_poll(req->cqe.res & poll->events); in io_poll_task_func()
378 req->cqe.res = ret; in io_poll_task_func()
382 io_req_set_res(req, req->cqe.res, 0); in io_poll_task_func()
385 io_tw_lock(req->ctx, ts); in io_poll_task_func()
418 list_del_init(&poll->wait.entry); in io_pollfree_wake()
422 * as req->head is NULL'ed out, the request can be in io_pollfree_wake()
426 smp_store_release(&poll->head, NULL); in io_pollfree_wake()
430 static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, in io_poll_wake() argument
433 struct io_kiocb *req = wqe_to_req(wait); in io_poll_wake()
434 struct io_poll *poll = container_of(wait, struct io_poll, wait); in io_poll_wake()
441 if (mask && !(mask & (poll->events & ~IO_ASYNC_POLL_COMMON))) in io_poll_wake()
451 poll->events |= EPOLLONESHOT; in io_poll_wake()
454 if (mask && poll->events & EPOLLONESHOT) { in io_poll_wake()
455 list_del_init(&poll->wait.entry); in io_poll_wake()
456 poll->head = NULL; in io_poll_wake()
457 if (wqe_is_double(wait)) in io_poll_wake()
458 req->flags &= ~REQ_F_DOUBLE_POLL; in io_poll_wake()
460 req->flags &= ~REQ_F_SINGLE_POLL; in io_poll_wake()
475 head = smp_load_acquire(&poll->head); in io_poll_double_prepare()
477 * poll arm might not hold ownership and so race for req->flags with in io_poll_double_prepare()
483 spin_lock_irq(&head->lock); in io_poll_double_prepare()
484 req->flags |= REQ_F_DOUBLE_POLL; in io_poll_double_prepare()
485 if (req->opcode == IORING_OP_POLL_ADD) in io_poll_double_prepare()
486 req->flags |= REQ_F_ASYNC_DATA; in io_poll_double_prepare()
487 spin_unlock_irq(&head->lock); in io_poll_double_prepare()
497 struct io_kiocb *req = pt->req; in __io_queue_proc()
505 if (unlikely(pt->nr_entries)) { in __io_queue_proc()
509 if (first->head == head) in __io_queue_proc()
513 if ((*poll_ptr)->head == head) in __io_queue_proc()
515 pt->error = -EINVAL; in __io_queue_proc()
521 pt->error = -ENOMEM; in __io_queue_proc()
527 io_init_poll_iocb(poll, first->events); in __io_queue_proc()
535 /* fine to modify, there is no poll queued to race with us */ in __io_queue_proc()
536 req->flags |= REQ_F_SINGLE_POLL; in __io_queue_proc()
539 pt->nr_entries++; in __io_queue_proc()
540 poll->head = head; in __io_queue_proc()
541 poll->wait.private = (void *) wqe_private; in __io_queue_proc()
543 if (poll->events & EPOLLEXCLUSIVE) { in __io_queue_proc()
544 add_wait_queue_exclusive(head, &poll->wait); in __io_queue_proc()
546 add_wait_queue(head, &poll->wait); in __io_queue_proc()
554 struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll); in io_poll_queue_proc()
557 (struct io_poll **) &pt->req->async_data); in io_poll_queue_proc()
563 return pt->owning || io_poll_get_ownership(req); in io_poll_can_finish_inline()
568 if (req->flags & REQ_F_HASH_LOCKED) in io_poll_add_hash()
576 * it returns non-zero, but otherwise should not touch it. Negative values
585 INIT_HLIST_NODE(&req->hash_node); in __io_arm_poll_handler()
587 poll->file = req->file; in __io_arm_poll_handler()
588 req->apoll_events = poll->events; in __io_arm_poll_handler()
590 ipt->pt._key = mask; in __io_arm_poll_handler()
591 ipt->req = req; in __io_arm_poll_handler()
592 ipt->error = 0; in __io_arm_poll_handler()
593 ipt->nr_entries = 0; in __io_arm_poll_handler()
597 * the same task. When it's io-wq, take the ownership to prevent tw in __io_arm_poll_handler()
605 ipt->owning = issue_flags & IO_URING_F_UNLOCKED; in __io_arm_poll_handler()
606 atomic_set(&req->poll_refs, (int)ipt->owning); in __io_arm_poll_handler()
608 /* io-wq doesn't hold uring_lock */ in __io_arm_poll_handler()
610 req->flags &= ~REQ_F_HASH_LOCKED; in __io_arm_poll_handler()
616 * wake if someone does wait(events > 1). Ensure we don't do in __io_arm_poll_handler()
620 if (poll->events & EPOLLEXCLUSIVE) in __io_arm_poll_handler()
621 req->flags |= REQ_F_POLL_NO_LAZY; in __io_arm_poll_handler()
623 mask = vfs_poll(req->file, &ipt->pt) & poll->events; in __io_arm_poll_handler()
625 if (unlikely(ipt->error || !ipt->nr_entries)) { in __io_arm_poll_handler()
631 } else if (mask && (poll->events & EPOLLET)) { in __io_arm_poll_handler()
632 ipt->result_mask = mask; in __io_arm_poll_handler()
635 return ipt->error ?: -EINVAL; in __io_arm_poll_handler()
639 ((poll->events & (EPOLLET|EPOLLONESHOT)) == (EPOLLET|EPOLLONESHOT))) { in __io_arm_poll_handler()
645 ipt->result_mask = mask; in __io_arm_poll_handler()
652 if (mask && (poll->events & EPOLLET) && in __io_arm_poll_handler()
659 if (ipt->owning) { in __io_arm_poll_handler()
664 if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) in __io_arm_poll_handler()
674 struct async_poll *apoll = pt->req->apoll; in io_async_queue_proc()
676 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll); in io_async_queue_proc()
690 struct io_ring_ctx *ctx = req->ctx; in io_req_alloc_apoll()
693 if (req->flags & REQ_F_POLLED) { in io_req_alloc_apoll()
694 apoll = req->apoll; in io_req_alloc_apoll()
695 kfree(apoll->double_poll); in io_req_alloc_apoll()
697 apoll = io_alloc_cache_get(&ctx->apoll_cache); in io_req_alloc_apoll()
700 apoll->poll.retries = APOLL_MAX_RETRY; in io_req_alloc_apoll()
706 apoll->poll.retries = APOLL_MAX_RETRY; in io_req_alloc_apoll()
708 apoll->double_poll = NULL; in io_req_alloc_apoll()
709 req->apoll = apoll; in io_req_alloc_apoll()
710 if (unlikely(!--apoll->poll.retries)) in io_req_alloc_apoll()
717 const struct io_issue_def *def = &io_issue_defs[req->opcode]; in io_arm_poll_handler()
725 * so removal from the mutex-backed hash is free, use it by default. in io_arm_poll_handler()
727 req->flags |= REQ_F_HASH_LOCKED; in io_arm_poll_handler()
729 if (!def->pollin && !def->pollout) in io_arm_poll_handler()
733 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) in io_arm_poll_handler()
736 if (def->pollin) { in io_arm_poll_handler()
740 if (req->flags & REQ_F_CLEAR_POLLIN) in io_arm_poll_handler()
745 if (def->poll_exclusive) in io_arm_poll_handler()
751 req->flags &= ~(REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL); in io_arm_poll_handler()
752 req->flags |= REQ_F_POLLED; in io_arm_poll_handler()
757 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags); in io_arm_poll_handler()
760 trace_io_uring_poll_arm(req, mask, apoll->poll.events); in io_arm_poll_handler()
768 unsigned nr_buckets = 1U << table->hash_bits; in io_poll_remove_all_table()
775 struct io_hash_bucket *hb = &table->hbs[i]; in io_poll_remove_all_table()
777 spin_lock(&hb->lock); in io_poll_remove_all_table()
778 hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { in io_poll_remove_all_table()
780 hlist_del_init(&req->hash_node); in io_poll_remove_all_table()
785 spin_unlock(&hb->lock); in io_poll_remove_all_table()
795 __must_hold(&ctx->uring_lock) in io_poll_remove_all()
799 ret = io_poll_remove_all_table(tsk, &ctx->cancel_table, cancel_all); in io_poll_remove_all()
800 ret |= io_poll_remove_all_table(tsk, &ctx->cancel_table_locked, cancel_all); in io_poll_remove_all()
810 u32 index = hash_long(cd->data, table->hash_bits); in io_poll_find()
811 struct io_hash_bucket *hb = &table->hbs[index]; in io_poll_find()
815 spin_lock(&hb->lock); in io_poll_find()
816 hlist_for_each_entry(req, &hb->list, hash_node) { in io_poll_find()
817 if (cd->data != req->cqe.user_data) in io_poll_find()
819 if (poll_only && req->opcode != IORING_OP_POLL_ADD) in io_poll_find()
821 if (cd->flags & IORING_ASYNC_CANCEL_ALL) { in io_poll_find()
822 if (io_cancel_match_sequence(req, cd->seq)) in io_poll_find()
828 spin_unlock(&hb->lock); in io_poll_find()
837 unsigned nr_buckets = 1U << table->hash_bits; in io_poll_file_find()
844 struct io_hash_bucket *hb = &table->hbs[i]; in io_poll_file_find()
846 spin_lock(&hb->lock); in io_poll_file_find()
847 hlist_for_each_entry(req, &hb->list, hash_node) { in io_poll_file_find()
853 spin_unlock(&hb->lock); in io_poll_file_find()
861 return -ENOENT; in io_poll_disarm()
863 return -EALREADY; in io_poll_disarm()
865 hash_del(&req->hash_node); in io_poll_disarm()
875 if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP | in __io_poll_cancel()
884 spin_unlock(&bucket->lock); in __io_poll_cancel()
885 return req ? 0 : -ENOENT; in __io_poll_cancel()
893 ret = __io_poll_cancel(ctx, cd, &ctx->cancel_table); in io_poll_cancel()
894 if (ret != -ENOENT) in io_poll_cancel()
898 ret = __io_poll_cancel(ctx, cd, &ctx->cancel_table_locked); in io_poll_cancel()
908 events = READ_ONCE(sqe->poll32_events); in io_poll_parse_events()
925 if (sqe->buf_index || sqe->splice_fd_in) in io_poll_remove_prep()
926 return -EINVAL; in io_poll_remove_prep()
927 flags = READ_ONCE(sqe->len); in io_poll_remove_prep()
930 return -EINVAL; in io_poll_remove_prep()
933 return -EINVAL; in io_poll_remove_prep()
935 upd->old_user_data = READ_ONCE(sqe->addr); in io_poll_remove_prep()
936 upd->update_events = flags & IORING_POLL_UPDATE_EVENTS; in io_poll_remove_prep()
937 upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA; in io_poll_remove_prep()
939 upd->new_user_data = READ_ONCE(sqe->off); in io_poll_remove_prep()
940 if (!upd->update_user_data && upd->new_user_data) in io_poll_remove_prep()
941 return -EINVAL; in io_poll_remove_prep()
942 if (upd->update_events) in io_poll_remove_prep()
943 upd->events = io_poll_parse_events(sqe, flags); in io_poll_remove_prep()
944 else if (sqe->poll32_events) in io_poll_remove_prep()
945 return -EINVAL; in io_poll_remove_prep()
955 if (sqe->buf_index || sqe->off || sqe->addr) in io_poll_add_prep()
956 return -EINVAL; in io_poll_add_prep()
957 flags = READ_ONCE(sqe->len); in io_poll_add_prep()
959 return -EINVAL; in io_poll_add_prep()
960 if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP)) in io_poll_add_prep()
961 return -EINVAL; in io_poll_add_prep()
963 poll->events = io_poll_parse_events(sqe, flags); in io_poll_add_prep()
976 * If sqpoll or single issuer, there is no contention for ->uring_lock in io_poll_add()
979 if (req->ctx->flags & (IORING_SETUP_SQPOLL|IORING_SETUP_SINGLE_ISSUER)) in io_poll_add()
980 req->flags |= REQ_F_HASH_LOCKED; in io_poll_add()
982 ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags); in io_poll_add()
993 struct io_ring_ctx *ctx = req->ctx; in io_poll_remove()
994 struct io_cancel_data cd = { .ctx = ctx, .data = poll_update->old_user_data, }; in io_poll_remove()
1000 preq = io_poll_find(ctx, true, &cd, &ctx->cancel_table, &bucket); in io_poll_remove()
1003 spin_unlock(&bucket->lock); in io_poll_remove()
1006 if (ret2 != -ENOENT) { in io_poll_remove()
1011 preq = io_poll_find(ctx, true, &cd, &ctx->cancel_table_locked, &bucket); in io_poll_remove()
1014 spin_unlock(&bucket->lock); in io_poll_remove()
1021 if (WARN_ON_ONCE(preq->opcode != IORING_OP_POLL_ADD)) { in io_poll_remove()
1022 ret = -EFAULT; in io_poll_remove()
1026 if (poll_update->update_events || poll_update->update_user_data) { in io_poll_remove()
1028 if (poll_update->update_events) { in io_poll_remove()
1031 poll->events &= ~0xffff; in io_poll_remove()
1032 poll->events |= poll_update->events & 0xffff; in io_poll_remove()
1033 poll->events |= IO_POLL_UNMASK; in io_poll_remove()
1035 if (poll_update->update_user_data) in io_poll_remove()
1036 preq->cqe.user_data = poll_update->new_user_data; in io_poll_remove()
1040 if (!ret2 || ret2 == -EIOCBQUEUED) in io_poll_remove()
1045 io_req_set_res(preq, -ECANCELED, 0); in io_poll_remove()
1046 preq->io_task_work.func = io_req_task_complete; in io_poll_remove()