1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/fs.h> 5 #include <linux/file.h> 6 #include <linux/mm.h> 7 #include <linux/slab.h> 8 #include <linux/poll.h> 9 #include <linux/hashtable.h> 10 #include <linux/io_uring.h> 11 12 #include <trace/events/io_uring.h> 13 14 #include <uapi/linux/io_uring.h> 15 16 #include "io_uring.h" 17 #include "alloc_cache.h" 18 #include "refs.h" 19 #include "napi.h" 20 #include "opdef.h" 21 #include "kbuf.h" 22 #include "poll.h" 23 #include "cancel.h" 24 25 struct io_poll_update { 26 struct file *file; 27 u64 old_user_data; 28 u64 new_user_data; 29 __poll_t events; 30 bool update_events; 31 bool update_user_data; 32 }; 33 34 struct io_poll_table { 35 struct poll_table_struct pt; 36 struct io_kiocb *req; 37 int nr_entries; 38 int error; 39 bool owning; 40 /* output value, set only if arm poll returns >0 */ 41 __poll_t result_mask; 42 }; 43 44 #define IO_POLL_CANCEL_FLAG BIT(31) 45 #define IO_POLL_RETRY_FLAG BIT(30) 46 #define IO_POLL_REF_MASK GENMASK(29, 0) 47 48 /* 49 * We usually have 1-2 refs taken, 128 is more than enough and we want to 50 * maximise the margin between this amount and the moment when it overflows. 51 */ 52 #define IO_POLL_REF_BIAS 128 53 54 #define IO_WQE_F_DOUBLE 1 55 56 static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, 57 void *key); 58 59 static inline struct io_kiocb *wqe_to_req(struct wait_queue_entry *wqe) 60 { 61 unsigned long priv = (unsigned long)wqe->private; 62 63 return (struct io_kiocb *)(priv & ~IO_WQE_F_DOUBLE); 64 } 65 66 static inline bool wqe_is_double(struct wait_queue_entry *wqe) 67 { 68 unsigned long priv = (unsigned long)wqe->private; 69 70 return priv & IO_WQE_F_DOUBLE; 71 } 72 73 static bool io_poll_get_ownership_slowpath(struct io_kiocb *req) 74 { 75 int v; 76 77 /* 78 * poll_refs are already elevated and we don't have much hope for 79 * grabbing the ownership. Instead of incrementing set a retry flag 80 * to notify the loop that there might have been some change. 81 */ 82 v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs); 83 if (v & IO_POLL_REF_MASK) 84 return false; 85 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); 86 } 87 88 /* 89 * If refs part of ->poll_refs (see IO_POLL_REF_MASK) is 0, it's free. We can 90 * bump it and acquire ownership. It's disallowed to modify requests while not 91 * owning it, that prevents from races for enqueueing task_work's and b/w 92 * arming poll and wakeups. 93 */ 94 static inline bool io_poll_get_ownership(struct io_kiocb *req) 95 { 96 if (unlikely((unsigned int)atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS)) 97 return io_poll_get_ownership_slowpath(req); 98 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); 99 } 100 101 static void io_poll_mark_cancelled(struct io_kiocb *req) 102 { 103 atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs); 104 } 105 106 static struct io_poll *io_poll_get_double(struct io_kiocb *req) 107 { 108 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */ 109 if (req->opcode == IORING_OP_POLL_ADD) 110 return req->async_data; 111 return req->apoll->double_poll; 112 } 113 114 static struct io_poll *io_poll_get_single(struct io_kiocb *req) 115 { 116 if (req->opcode == IORING_OP_POLL_ADD) 117 return io_kiocb_to_cmd(req, struct io_poll); 118 return &req->apoll->poll; 119 } 120 121 static void io_poll_req_insert(struct io_kiocb *req) 122 { 123 struct io_hash_table *table = &req->ctx->cancel_table; 124 u32 index = hash_long(req->cqe.user_data, table->hash_bits); 125 126 lockdep_assert_held(&req->ctx->uring_lock); 127 128 hlist_add_head(&req->hash_node, &table->hbs[index].list); 129 } 130 131 static void io_init_poll_iocb(struct io_poll *poll, __poll_t events) 132 { 133 poll->head = NULL; 134 #define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP) 135 /* mask in events that we always want/need */ 136 poll->events = events | IO_POLL_UNMASK; 137 INIT_LIST_HEAD(&poll->wait.entry); 138 init_waitqueue_func_entry(&poll->wait, io_poll_wake); 139 } 140 141 static void io_poll_remove_waitq(struct io_poll *poll) 142 { 143 /* 144 * If the waitqueue is being freed early but someone is already holds 145 * ownership over it, we have to tear down the request as best we can. 146 * That means immediately removing the request from its waitqueue and 147 * preventing all further accesses to the waitqueue via the request. 148 */ 149 list_del_init(&poll->wait.entry); 150 151 /* 152 * Careful: this *must* be the last step, since as soon as req->head is 153 * NULL'ed out, the request can be completed and freed, since 154 * io_poll_remove_entry() will no longer need to take the waitqueue 155 * lock. 156 */ 157 smp_store_release(&poll->head, NULL); 158 } 159 160 static inline void io_poll_remove_entry(struct io_poll *poll) 161 { 162 struct wait_queue_head *head = smp_load_acquire(&poll->head); 163 164 if (head) { 165 spin_lock_irq(&head->lock); 166 io_poll_remove_waitq(poll); 167 spin_unlock_irq(&head->lock); 168 } 169 } 170 171 static void io_poll_remove_entries(struct io_kiocb *req) 172 { 173 /* 174 * Nothing to do if neither of those flags are set. Avoid dipping 175 * into the poll/apoll/double cachelines if we can. 176 */ 177 if (!(req->flags & (REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL))) 178 return; 179 180 /* 181 * While we hold the waitqueue lock and the waitqueue is nonempty, 182 * wake_up_pollfree() will wait for us. However, taking the waitqueue 183 * lock in the first place can race with the waitqueue being freed. 184 * 185 * We solve this as eventpoll does: by taking advantage of the fact that 186 * all users of wake_up_pollfree() will RCU-delay the actual free. If 187 * we enter rcu_read_lock() and see that the pointer to the queue is 188 * non-NULL, we can then lock it without the memory being freed out from 189 * under us. 190 * 191 * Keep holding rcu_read_lock() as long as we hold the queue lock, in 192 * case the caller deletes the entry from the queue, leaving it empty. 193 * In that case, only RCU prevents the queue memory from being freed. 194 */ 195 rcu_read_lock(); 196 if (req->flags & REQ_F_SINGLE_POLL) 197 io_poll_remove_entry(io_poll_get_single(req)); 198 if (req->flags & REQ_F_DOUBLE_POLL) 199 io_poll_remove_entry(io_poll_get_double(req)); 200 rcu_read_unlock(); 201 } 202 203 enum { 204 IOU_POLL_DONE = 0, 205 IOU_POLL_NO_ACTION = 1, 206 IOU_POLL_REMOVE_POLL_USE_RES = 2, 207 IOU_POLL_REISSUE = 3, 208 IOU_POLL_REQUEUE = 4, 209 }; 210 211 static void __io_poll_execute(struct io_kiocb *req, int mask, unsigned tw_flags) 212 { 213 unsigned flags = tw_flags; 214 215 io_req_set_res(req, mask, 0); 216 req->io_task_work.func = io_poll_task_func; 217 218 trace_io_uring_task_add(req, mask); 219 220 if (!(req->flags & REQ_F_POLL_NO_LAZY)) 221 flags |= IOU_F_TWQ_LAZY_WAKE; 222 __io_req_task_work_add(req, flags); 223 } 224 225 static inline void io_poll_execute(struct io_kiocb *req, int res, 226 unsigned tw_flags) 227 { 228 if (io_poll_get_ownership(req)) 229 __io_poll_execute(req, res, tw_flags); 230 } 231 232 /* 233 * All poll tw should go through this. Checks for poll events, manages 234 * references, does rewait, etc. 235 * 236 * Returns a negative error on failure. IOU_POLL_NO_ACTION when no action 237 * require, which is either spurious wakeup or multishot CQE is served. 238 * IOU_POLL_DONE when it's done with the request, then the mask is stored in 239 * req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot 240 * poll and that the result is stored in req->cqe. 241 */ 242 static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) 243 { 244 int v; 245 246 if (unlikely(tw.cancel)) 247 return -ECANCELED; 248 249 do { 250 v = atomic_read(&req->poll_refs); 251 252 if (unlikely(v != 1)) { 253 /* tw should be the owner and so have some refs */ 254 if (WARN_ON_ONCE(!(v & IO_POLL_REF_MASK))) 255 return IOU_POLL_NO_ACTION; 256 if (v & IO_POLL_CANCEL_FLAG) 257 return -ECANCELED; 258 /* 259 * cqe.res contains only events of the first wake up 260 * and all others are to be lost. Redo vfs_poll() to get 261 * up to date state. 262 */ 263 if ((v & IO_POLL_REF_MASK) != 1) 264 req->cqe.res = 0; 265 266 if (v & IO_POLL_RETRY_FLAG) { 267 req->cqe.res = 0; 268 /* 269 * We won't find new events that came in between 270 * vfs_poll and the ref put unless we clear the 271 * flag in advance. 272 */ 273 atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs); 274 v &= ~IO_POLL_RETRY_FLAG; 275 } 276 v &= IO_POLL_REF_MASK; 277 } 278 279 /* the mask was stashed in __io_poll_execute */ 280 if (!req->cqe.res) { 281 __poll_t events = req->apoll_events; 282 struct poll_table_struct pt = { ._key = events }; 283 284 req->cqe.res = vfs_poll(req->file, &pt) & events; 285 /* 286 * We got woken with a mask, but someone else got to 287 * it first. The above vfs_poll() doesn't add us back 288 * to the waitqueue, so if we get nothing back, we 289 * should be safe and attempt a reissue. 290 */ 291 if (unlikely(!req->cqe.res)) { 292 /* Multishot armed need not reissue */ 293 if (!(events & EPOLLONESHOT)) 294 continue; 295 return IOU_POLL_REISSUE; 296 } 297 } 298 if (req->apoll_events & EPOLLONESHOT) 299 return IOU_POLL_DONE; 300 301 /* multishot, just fill a CQE and proceed */ 302 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 303 __poll_t mask = mangle_poll(req->cqe.res & 304 req->apoll_events); 305 306 if (!io_req_post_cqe(req, mask, IORING_CQE_F_MORE)) { 307 io_req_set_res(req, mask, 0); 308 return IOU_POLL_REMOVE_POLL_USE_RES; 309 } 310 } else { 311 int ret; 312 313 /* multiple refs and HUP, ensure we loop once more */ 314 if ((req->cqe.res & (POLLHUP | POLLRDHUP)) && v != 1) 315 v--; 316 317 ret = io_poll_issue(req, tw); 318 if (ret == IOU_COMPLETE) 319 return IOU_POLL_REMOVE_POLL_USE_RES; 320 else if (ret == IOU_REQUEUE) 321 return IOU_POLL_REQUEUE; 322 if (ret != IOU_RETRY && ret < 0) 323 return ret; 324 } 325 326 /* force the next iteration to vfs_poll() */ 327 req->cqe.res = 0; 328 329 /* 330 * Release all references, retry if someone tried to restart 331 * task_work while we were executing it. 332 */ 333 } while (atomic_sub_return(v, &req->poll_refs) & IO_POLL_REF_MASK); 334 335 io_napi_add(req); 336 return IOU_POLL_NO_ACTION; 337 } 338 339 void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw) 340 { 341 struct io_kiocb *req = tw_req.req; 342 int ret; 343 344 ret = io_poll_check_events(req, tw); 345 if (ret == IOU_POLL_NO_ACTION) { 346 return; 347 } else if (ret == IOU_POLL_REQUEUE) { 348 __io_poll_execute(req, 0, 0); 349 return; 350 } 351 io_poll_remove_entries(req); 352 /* task_work always has ->uring_lock held */ 353 hash_del(&req->hash_node); 354 355 if (req->opcode == IORING_OP_POLL_ADD) { 356 if (ret == IOU_POLL_DONE) { 357 struct io_poll *poll; 358 359 poll = io_kiocb_to_cmd(req, struct io_poll); 360 req->cqe.res = mangle_poll(req->cqe.res & poll->events); 361 } else if (ret == IOU_POLL_REISSUE) { 362 io_req_task_submit(tw_req, tw); 363 return; 364 } else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) { 365 req->cqe.res = ret; 366 req_set_fail(req); 367 } 368 369 io_req_set_res(req, req->cqe.res, 0); 370 io_req_task_complete(tw_req, tw); 371 } else { 372 io_tw_lock(req->ctx, tw); 373 374 if (ret == IOU_POLL_REMOVE_POLL_USE_RES) 375 io_req_task_complete(tw_req, tw); 376 else if (ret == IOU_POLL_DONE || ret == IOU_POLL_REISSUE) 377 io_req_task_submit(tw_req, tw); 378 else 379 io_req_defer_failed(req, ret); 380 } 381 } 382 383 static void io_poll_cancel_req(struct io_kiocb *req) 384 { 385 io_poll_mark_cancelled(req); 386 /* kick tw, which should complete the request */ 387 io_poll_execute(req, 0, 0); 388 } 389 390 #define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI) 391 392 static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) 393 { 394 io_poll_mark_cancelled(req); 395 /* we have to kick tw in case it's not already */ 396 io_poll_execute(req, 0, IOU_F_TWQ_IN_WAKE); 397 io_poll_remove_waitq(poll); 398 return 1; 399 } 400 401 static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, 402 void *key) 403 { 404 struct io_kiocb *req = wqe_to_req(wait); 405 struct io_poll *poll = container_of(wait, struct io_poll, wait); 406 __poll_t mask = key_to_poll(key); 407 408 if (unlikely(mask & POLLFREE)) 409 return io_pollfree_wake(req, poll); 410 411 /* for instances that support it check for an event match first */ 412 if (mask && !(mask & (poll->events & ~IO_ASYNC_POLL_COMMON))) 413 return 0; 414 415 if (io_poll_get_ownership(req)) { 416 /* 417 * If we trigger a multishot poll off our own wakeup path, 418 * disable multishot as there is a circular dependency between 419 * CQ posting and triggering the event. 420 */ 421 if (mask & EPOLL_URING_WAKE) { 422 poll->events |= EPOLLONESHOT; 423 req->apoll_events |= EPOLLONESHOT; 424 } 425 426 /* optional, saves extra locking for removal in tw handler */ 427 if (mask && poll->events & EPOLLONESHOT) { 428 io_poll_remove_waitq(poll); 429 if (wqe_is_double(wait)) 430 req->flags &= ~REQ_F_DOUBLE_POLL; 431 else 432 req->flags &= ~REQ_F_SINGLE_POLL; 433 } 434 __io_poll_execute(req, mask, IOU_F_TWQ_IN_WAKE); 435 } 436 return 1; 437 } 438 439 /* fails only when polling is already completing by the first entry */ 440 static bool io_poll_double_prepare(struct io_kiocb *req) 441 { 442 struct wait_queue_head *head; 443 struct io_poll *poll = io_poll_get_single(req); 444 445 /* head is RCU protected, see io_poll_remove_entries() comments */ 446 rcu_read_lock(); 447 head = smp_load_acquire(&poll->head); 448 /* 449 * poll arm might not hold ownership and so race for req->flags with 450 * io_poll_wake(). There is only one poll entry queued, serialise with 451 * it by taking its head lock. As we're still arming the tw hanlder 452 * is not going to be run, so there are no races with it. 453 */ 454 if (head) { 455 spin_lock_irq(&head->lock); 456 req->flags |= REQ_F_DOUBLE_POLL; 457 if (req->opcode == IORING_OP_POLL_ADD) 458 req->flags |= REQ_F_ASYNC_DATA; 459 spin_unlock_irq(&head->lock); 460 } 461 rcu_read_unlock(); 462 return !!head; 463 } 464 465 static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt, 466 struct wait_queue_head *head, 467 struct io_poll **poll_ptr) 468 { 469 struct io_kiocb *req = pt->req; 470 unsigned long wqe_private = (unsigned long) req; 471 472 /* 473 * The file being polled uses multiple waitqueues for poll handling 474 * (e.g. one for read, one for write). Setup a separate io_poll 475 * if this happens. 476 */ 477 if (unlikely(pt->nr_entries)) { 478 struct io_poll *first = poll; 479 480 /* double add on the same waitqueue head, ignore */ 481 if (first->head == head) 482 return; 483 /* already have a 2nd entry, fail a third attempt */ 484 if (*poll_ptr) { 485 if ((*poll_ptr)->head == head) 486 return; 487 pt->error = -EINVAL; 488 return; 489 } 490 491 poll = kmalloc_obj(*poll, GFP_ATOMIC); 492 if (!poll) { 493 pt->error = -ENOMEM; 494 return; 495 } 496 497 /* mark as double wq entry */ 498 wqe_private |= IO_WQE_F_DOUBLE; 499 io_init_poll_iocb(poll, first->events); 500 if (!io_poll_double_prepare(req)) { 501 /* the request is completing, just back off */ 502 kfree(poll); 503 return; 504 } 505 *poll_ptr = poll; 506 } else { 507 /* fine to modify, there is no poll queued to race with us */ 508 req->flags |= REQ_F_SINGLE_POLL; 509 } 510 511 pt->nr_entries++; 512 poll->head = head; 513 poll->wait.private = (void *) wqe_private; 514 515 if (poll->events & EPOLLEXCLUSIVE) { 516 add_wait_queue_exclusive(head, &poll->wait); 517 } else { 518 add_wait_queue(head, &poll->wait); 519 } 520 } 521 522 static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head, 523 struct poll_table_struct *p) 524 { 525 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); 526 struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll); 527 528 __io_queue_proc(poll, pt, head, 529 (struct io_poll **) &pt->req->async_data); 530 } 531 532 static bool io_poll_can_finish_inline(struct io_kiocb *req, 533 struct io_poll_table *pt) 534 { 535 return pt->owning || io_poll_get_ownership(req); 536 } 537 538 static void io_poll_add_hash(struct io_kiocb *req, unsigned int issue_flags) 539 { 540 struct io_ring_ctx *ctx = req->ctx; 541 542 io_ring_submit_lock(ctx, issue_flags); 543 io_poll_req_insert(req); 544 io_ring_submit_unlock(ctx, issue_flags); 545 } 546 547 /* 548 * Returns 0 when it's handed over for polling. The caller owns the requests if 549 * it returns non-zero, but otherwise should not touch it. Negative values 550 * contain an error code. When the result is >0, the polling has completed 551 * inline and ipt.result_mask is set to the mask. 552 */ 553 static int __io_arm_poll_handler(struct io_kiocb *req, 554 struct io_poll *poll, 555 struct io_poll_table *ipt, __poll_t mask, 556 unsigned issue_flags) 557 { 558 INIT_HLIST_NODE(&req->hash_node); 559 io_init_poll_iocb(poll, mask); 560 poll->file = req->file; 561 req->apoll_events = poll->events; 562 563 ipt->pt._key = mask; 564 ipt->req = req; 565 ipt->error = 0; 566 ipt->nr_entries = 0; 567 /* 568 * Polling is either completed here or via task_work, so if we're in the 569 * task context we're naturally serialised with tw by merit of running 570 * the same task. When it's io-wq, take the ownership to prevent tw 571 * from running. However, when we're in the task context, skip taking 572 * it as an optimisation. 573 * 574 * Note: even though the request won't be completed/freed, without 575 * ownership we still can race with io_poll_wake(). 576 * io_poll_can_finish_inline() tries to deal with that. 577 */ 578 ipt->owning = issue_flags & IO_URING_F_UNLOCKED; 579 atomic_set(&req->poll_refs, (int)ipt->owning); 580 581 /* 582 * Exclusive waits may only wake a limited amount of entries 583 * rather than all of them, this may interfere with lazy 584 * wake if someone does wait(events > 1). Ensure we don't do 585 * lazy wake for those, as we need to process each one as they 586 * come in. 587 */ 588 if (poll->events & EPOLLEXCLUSIVE) 589 req->flags |= REQ_F_POLL_NO_LAZY; 590 591 mask = vfs_poll(req->file, &ipt->pt) & poll->events; 592 593 if (unlikely(ipt->error || !ipt->nr_entries)) { 594 io_poll_remove_entries(req); 595 596 if (!io_poll_can_finish_inline(req, ipt)) { 597 io_poll_mark_cancelled(req); 598 return 0; 599 } else if (mask && (poll->events & EPOLLET)) { 600 ipt->result_mask = mask; 601 return 1; 602 } 603 return ipt->error ?: -EINVAL; 604 } 605 606 if (mask && 607 ((poll->events & (EPOLLET|EPOLLONESHOT)) == (EPOLLET|EPOLLONESHOT))) { 608 if (!io_poll_can_finish_inline(req, ipt)) { 609 io_poll_add_hash(req, issue_flags); 610 return 0; 611 } 612 io_poll_remove_entries(req); 613 ipt->result_mask = mask; 614 /* no one else has access to the req, forget about the ref */ 615 return 1; 616 } 617 618 io_poll_add_hash(req, issue_flags); 619 620 if (mask && (poll->events & EPOLLET) && 621 io_poll_can_finish_inline(req, ipt)) { 622 __io_poll_execute(req, mask, 0); 623 return 0; 624 } 625 io_napi_add(req); 626 627 if (ipt->owning) { 628 /* 629 * Try to release ownership. If we see a change of state, e.g. 630 * poll was waken up, queue up a tw, it'll deal with it. 631 */ 632 if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) 633 __io_poll_execute(req, 0, 0); 634 } 635 return 0; 636 } 637 638 static void io_async_queue_proc(struct file *file, struct wait_queue_head *head, 639 struct poll_table_struct *p) 640 { 641 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); 642 struct async_poll *apoll = pt->req->apoll; 643 644 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll); 645 } 646 647 /* 648 * We can't reliably detect loops in repeated poll triggers and issue 649 * subsequently failing. But rather than fail these immediately, allow a 650 * certain amount of retries before we give up. Given that this condition 651 * should _rarely_ trigger even once, we should be fine with a larger value. 652 */ 653 #define APOLL_MAX_RETRY 128 654 655 static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req, 656 unsigned issue_flags) 657 { 658 struct io_ring_ctx *ctx = req->ctx; 659 struct async_poll *apoll; 660 661 if (req->flags & REQ_F_POLLED) { 662 apoll = req->apoll; 663 kfree(apoll->double_poll); 664 } else { 665 if (!(issue_flags & IO_URING_F_UNLOCKED)) 666 apoll = io_cache_alloc(&ctx->apoll_cache, GFP_ATOMIC); 667 else 668 apoll = kmalloc_obj(*apoll, GFP_ATOMIC); 669 if (!apoll) 670 return NULL; 671 apoll->poll.retries = APOLL_MAX_RETRY; 672 } 673 apoll->double_poll = NULL; 674 req->apoll = apoll; 675 if (unlikely(!--apoll->poll.retries)) 676 return NULL; 677 return apoll; 678 } 679 680 int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask) 681 { 682 struct async_poll *apoll; 683 struct io_poll_table ipt; 684 int ret; 685 686 mask |= EPOLLET; 687 if (!io_file_can_poll(req)) 688 return IO_APOLL_ABORTED; 689 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) 690 mask |= EPOLLONESHOT; 691 692 apoll = io_req_alloc_apoll(req, issue_flags); 693 if (!apoll) 694 return IO_APOLL_ABORTED; 695 req->flags &= ~(REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL); 696 req->flags |= REQ_F_POLLED; 697 ipt.pt._qproc = io_async_queue_proc; 698 699 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags); 700 if (ret) 701 return ret > 0 ? IO_APOLL_READY : IO_APOLL_ABORTED; 702 trace_io_uring_poll_arm(req, mask, apoll->poll.events); 703 return IO_APOLL_OK; 704 } 705 706 int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) 707 { 708 const struct io_issue_def *def = &io_issue_defs[req->opcode]; 709 __poll_t mask = POLLPRI | POLLERR; 710 711 if (!def->pollin && !def->pollout) 712 return IO_APOLL_ABORTED; 713 if (!io_file_can_poll(req)) 714 return IO_APOLL_ABORTED; 715 716 if (def->pollin) { 717 mask |= EPOLLIN | EPOLLRDNORM; 718 719 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */ 720 if (req->flags & REQ_F_CLEAR_POLLIN) 721 mask &= ~EPOLLIN; 722 } else { 723 mask |= EPOLLOUT | EPOLLWRNORM; 724 } 725 if (def->poll_exclusive) 726 mask |= EPOLLEXCLUSIVE; 727 728 return io_arm_apoll(req, issue_flags, mask); 729 } 730 731 /* 732 * Returns true if we found and killed one or more poll requests 733 */ 734 __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, 735 bool cancel_all) 736 { 737 unsigned nr_buckets = 1U << ctx->cancel_table.hash_bits; 738 struct hlist_node *tmp; 739 struct io_kiocb *req; 740 bool found = false; 741 int i; 742 743 lockdep_assert_held(&ctx->uring_lock); 744 745 for (i = 0; i < nr_buckets; i++) { 746 struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; 747 748 hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) { 749 if (io_match_task_safe(req, tctx, cancel_all)) { 750 hlist_del_init(&req->hash_node); 751 io_poll_cancel_req(req); 752 found = true; 753 } 754 } 755 } 756 return found; 757 } 758 759 static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, 760 struct io_cancel_data *cd) 761 { 762 struct io_kiocb *req; 763 u32 index = hash_long(cd->data, ctx->cancel_table.hash_bits); 764 struct io_hash_bucket *hb = &ctx->cancel_table.hbs[index]; 765 766 hlist_for_each_entry(req, &hb->list, hash_node) { 767 if (cd->data != req->cqe.user_data) 768 continue; 769 if (poll_only && req->opcode != IORING_OP_POLL_ADD) 770 continue; 771 if (cd->flags & IORING_ASYNC_CANCEL_ALL) { 772 if (io_cancel_match_sequence(req, cd->seq)) 773 continue; 774 } 775 return req; 776 } 777 return NULL; 778 } 779 780 static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, 781 struct io_cancel_data *cd) 782 { 783 unsigned nr_buckets = 1U << ctx->cancel_table.hash_bits; 784 struct io_kiocb *req; 785 int i; 786 787 for (i = 0; i < nr_buckets; i++) { 788 struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; 789 790 hlist_for_each_entry(req, &hb->list, hash_node) { 791 if (io_cancel_req_match(req, cd)) 792 return req; 793 } 794 } 795 return NULL; 796 } 797 798 static int io_poll_disarm(struct io_kiocb *req) 799 { 800 if (!req) 801 return -ENOENT; 802 if (!io_poll_get_ownership(req)) 803 return -EALREADY; 804 io_poll_remove_entries(req); 805 hash_del(&req->hash_node); 806 return 0; 807 } 808 809 static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) 810 { 811 struct io_kiocb *req; 812 813 if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP | 814 IORING_ASYNC_CANCEL_ANY)) 815 req = io_poll_file_find(ctx, cd); 816 else 817 req = io_poll_find(ctx, false, cd); 818 819 if (req) { 820 io_poll_cancel_req(req); 821 return 0; 822 } 823 return -ENOENT; 824 } 825 826 int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, 827 unsigned issue_flags) 828 { 829 int ret; 830 831 io_ring_submit_lock(ctx, issue_flags); 832 ret = __io_poll_cancel(ctx, cd); 833 io_ring_submit_unlock(ctx, issue_flags); 834 return ret; 835 } 836 837 static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe, 838 unsigned int flags) 839 { 840 u32 events; 841 842 events = READ_ONCE(sqe->poll32_events); 843 #ifdef __BIG_ENDIAN 844 events = swahw32(events); 845 #endif 846 if (!(flags & IORING_POLL_ADD_MULTI)) 847 events |= EPOLLONESHOT; 848 if (!(flags & IORING_POLL_ADD_LEVEL)) 849 events |= EPOLLET; 850 return demangle_poll(events) | 851 (events & (EPOLLEXCLUSIVE|EPOLLONESHOT|EPOLLET)); 852 } 853 854 int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 855 { 856 struct io_poll_update *upd = io_kiocb_to_cmd(req, struct io_poll_update); 857 u32 flags; 858 859 if (sqe->buf_index || sqe->splice_fd_in) 860 return -EINVAL; 861 flags = READ_ONCE(sqe->len); 862 if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA | 863 IORING_POLL_ADD_MULTI)) 864 return -EINVAL; 865 /* meaningless without update */ 866 if (flags == IORING_POLL_ADD_MULTI) 867 return -EINVAL; 868 869 upd->old_user_data = READ_ONCE(sqe->addr); 870 upd->update_events = flags & IORING_POLL_UPDATE_EVENTS; 871 upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA; 872 873 upd->new_user_data = READ_ONCE(sqe->off); 874 if (!upd->update_user_data && upd->new_user_data) 875 return -EINVAL; 876 if (upd->update_events) 877 upd->events = io_poll_parse_events(sqe, flags); 878 else if (sqe->poll32_events) 879 return -EINVAL; 880 881 return 0; 882 } 883 884 int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 885 { 886 struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); 887 u32 flags; 888 889 if (sqe->buf_index || sqe->off || sqe->addr) 890 return -EINVAL; 891 flags = READ_ONCE(sqe->len); 892 if (flags & ~IORING_POLL_ADD_MULTI) 893 return -EINVAL; 894 if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP)) 895 return -EINVAL; 896 897 poll->events = io_poll_parse_events(sqe, flags); 898 return 0; 899 } 900 901 int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) 902 { 903 struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); 904 struct io_poll_table ipt; 905 int ret; 906 907 ipt.pt._qproc = io_poll_queue_proc; 908 909 ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags); 910 if (ret > 0) { 911 io_req_set_res(req, ipt.result_mask, 0); 912 return IOU_COMPLETE; 913 } 914 return ret ?: IOU_ISSUE_SKIP_COMPLETE; 915 } 916 917 int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) 918 { 919 struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update); 920 struct io_ring_ctx *ctx = req->ctx; 921 struct io_cancel_data cd = { .ctx = ctx, .data = poll_update->old_user_data, }; 922 struct io_kiocb *preq; 923 int ret2, ret = 0; 924 925 io_ring_submit_lock(ctx, issue_flags); 926 preq = io_poll_find(ctx, true, &cd); 927 ret2 = io_poll_disarm(preq); 928 if (ret2) { 929 ret = ret2; 930 goto out; 931 } 932 if (WARN_ON_ONCE(preq->opcode != IORING_OP_POLL_ADD)) { 933 ret = -EFAULT; 934 goto out; 935 } 936 937 if (poll_update->update_events || poll_update->update_user_data) { 938 /* only mask one event flags, keep behavior flags */ 939 if (poll_update->update_events) { 940 struct io_poll *poll = io_kiocb_to_cmd(preq, struct io_poll); 941 942 poll->events &= ~0xffff; 943 poll->events |= poll_update->events & 0xffff; 944 poll->events |= IO_POLL_UNMASK; 945 } 946 if (poll_update->update_user_data) 947 preq->cqe.user_data = poll_update->new_user_data; 948 949 ret2 = io_poll_add(preq, issue_flags & ~IO_URING_F_UNLOCKED); 950 /* successfully updated, don't complete poll request */ 951 if (ret2 == IOU_ISSUE_SKIP_COMPLETE) 952 goto out; 953 /* request completed as part of the update, complete it */ 954 else if (ret2 == IOU_COMPLETE) 955 goto complete; 956 } 957 958 io_req_set_res(preq, -ECANCELED, 0); 959 complete: 960 if (preq->cqe.res < 0) 961 req_set_fail(preq); 962 preq->io_task_work.func = io_req_task_complete; 963 io_req_task_work_add(preq); 964 out: 965 io_ring_submit_unlock(ctx, issue_flags); 966 if (ret < 0) { 967 req_set_fail(req); 968 return ret; 969 } 970 /* complete update request, we're done with it */ 971 io_req_set_res(req, ret, 0); 972 return IOU_COMPLETE; 973 } 974