1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/io_uring.h> 6 7 #include <trace/events/io_uring.h> 8 9 #include <uapi/linux/io_uring.h> 10 11 #include "io_uring.h" 12 #include "refs.h" 13 #include "cancel.h" 14 #include "timeout.h" 15 16 struct io_timeout { 17 struct file *file; 18 u32 off; 19 u32 target_seq; 20 u32 repeats; 21 struct list_head list; 22 /* head of the link, used by linked timeouts only */ 23 struct io_kiocb *head; 24 /* for linked completions */ 25 struct io_kiocb *prev; 26 }; 27 28 struct io_timeout_rem { 29 struct file *file; 30 u64 addr; 31 32 /* timeout update */ 33 struct timespec64 ts; 34 u32 flags; 35 bool ltimeout; 36 }; 37 38 static inline bool io_is_timeout_noseq(struct io_kiocb *req) 39 { 40 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 41 struct io_timeout_data *data = req->async_data; 42 43 return !timeout->off || data->flags & IORING_TIMEOUT_MULTISHOT; 44 } 45 46 static inline void io_put_req(struct io_kiocb *req) 47 { 48 if (req_ref_put_and_test(req)) { 49 io_queue_next(req); 50 io_free_req(req); 51 } 52 } 53 54 static inline bool io_timeout_finish(struct io_timeout *timeout, 55 struct io_timeout_data *data) 56 { 57 if (!(data->flags & IORING_TIMEOUT_MULTISHOT)) 58 return true; 59 60 if (!timeout->off || (timeout->repeats && --timeout->repeats)) 61 return false; 62 63 return true; 64 } 65 66 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer); 67 68 static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts) 69 { 70 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 71 struct io_timeout_data *data = req->async_data; 72 struct io_ring_ctx *ctx = req->ctx; 73 74 if (!io_timeout_finish(timeout, data)) { 75 if (io_req_post_cqe(req, -ETIME, IORING_CQE_F_MORE)) { 76 /* re-arm timer */ 77 spin_lock_irq(&ctx->timeout_lock); 78 list_add(&timeout->list, ctx->timeout_list.prev); 79 data->timer.function = io_timeout_fn; 80 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); 81 spin_unlock_irq(&ctx->timeout_lock); 82 return; 83 } 84 } 85 86 io_req_task_complete(req, ts); 87 } 88 89 static bool io_kill_timeout(struct io_kiocb *req, int status) 90 __must_hold(&req->ctx->timeout_lock) 91 { 92 struct io_timeout_data *io = req->async_data; 93 94 if (hrtimer_try_to_cancel(&io->timer) != -1) { 95 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 96 97 if (status) 98 req_set_fail(req); 99 atomic_set(&req->ctx->cq_timeouts, 100 atomic_read(&req->ctx->cq_timeouts) + 1); 101 list_del_init(&timeout->list); 102 io_req_queue_tw_complete(req, status); 103 return true; 104 } 105 return false; 106 } 107 108 __cold void io_flush_timeouts(struct io_ring_ctx *ctx) 109 { 110 u32 seq; 111 struct io_timeout *timeout, *tmp; 112 113 spin_lock_irq(&ctx->timeout_lock); 114 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); 115 116 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { 117 struct io_kiocb *req = cmd_to_io_kiocb(timeout); 118 u32 events_needed, events_got; 119 120 if (io_is_timeout_noseq(req)) 121 break; 122 123 /* 124 * Since seq can easily wrap around over time, subtract 125 * the last seq at which timeouts were flushed before comparing. 126 * Assuming not more than 2^31-1 events have happened since, 127 * these subtractions won't have wrapped, so we can check if 128 * target is in [last_seq, current_seq] by comparing the two. 129 */ 130 events_needed = timeout->target_seq - ctx->cq_last_tm_flush; 131 events_got = seq - ctx->cq_last_tm_flush; 132 if (events_got < events_needed) 133 break; 134 135 io_kill_timeout(req, 0); 136 } 137 ctx->cq_last_tm_flush = seq; 138 spin_unlock_irq(&ctx->timeout_lock); 139 } 140 141 static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts) 142 { 143 io_tw_lock(link->ctx, ts); 144 while (link) { 145 struct io_kiocb *nxt = link->link; 146 long res = -ECANCELED; 147 148 if (link->flags & REQ_F_FAIL) 149 res = link->cqe.res; 150 link->link = NULL; 151 io_req_set_res(link, res, 0); 152 io_req_task_complete(link, ts); 153 link = nxt; 154 } 155 } 156 157 static void io_fail_links(struct io_kiocb *req) 158 __must_hold(&req->ctx->completion_lock) 159 { 160 struct io_kiocb *link = req->link; 161 bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES; 162 163 if (!link) 164 return; 165 166 while (link) { 167 if (ignore_cqes) 168 link->flags |= REQ_F_CQE_SKIP; 169 else 170 link->flags &= ~REQ_F_CQE_SKIP; 171 trace_io_uring_fail_link(req, link); 172 link = link->link; 173 } 174 175 link = req->link; 176 link->io_task_work.func = io_req_tw_fail_links; 177 io_req_task_work_add(link); 178 req->link = NULL; 179 } 180 181 static inline void io_remove_next_linked(struct io_kiocb *req) 182 { 183 struct io_kiocb *nxt = req->link; 184 185 req->link = nxt->link; 186 nxt->link = NULL; 187 } 188 189 void io_disarm_next(struct io_kiocb *req) 190 __must_hold(&req->ctx->completion_lock) 191 { 192 struct io_kiocb *link = NULL; 193 194 if (req->flags & REQ_F_ARM_LTIMEOUT) { 195 link = req->link; 196 req->flags &= ~REQ_F_ARM_LTIMEOUT; 197 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { 198 io_remove_next_linked(req); 199 io_req_queue_tw_complete(link, -ECANCELED); 200 } 201 } else if (req->flags & REQ_F_LINK_TIMEOUT) { 202 struct io_ring_ctx *ctx = req->ctx; 203 204 spin_lock_irq(&ctx->timeout_lock); 205 link = io_disarm_linked_timeout(req); 206 spin_unlock_irq(&ctx->timeout_lock); 207 if (link) 208 io_req_queue_tw_complete(link, -ECANCELED); 209 } 210 if (unlikely((req->flags & REQ_F_FAIL) && 211 !(req->flags & REQ_F_HARDLINK))) 212 io_fail_links(req); 213 } 214 215 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, 216 struct io_kiocb *link) 217 __must_hold(&req->ctx->completion_lock) 218 __must_hold(&req->ctx->timeout_lock) 219 { 220 struct io_timeout_data *io = link->async_data; 221 struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout); 222 223 io_remove_next_linked(req); 224 timeout->head = NULL; 225 if (hrtimer_try_to_cancel(&io->timer) != -1) { 226 list_del(&timeout->list); 227 return link; 228 } 229 230 return NULL; 231 } 232 233 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) 234 { 235 struct io_timeout_data *data = container_of(timer, 236 struct io_timeout_data, timer); 237 struct io_kiocb *req = data->req; 238 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 239 struct io_ring_ctx *ctx = req->ctx; 240 unsigned long flags; 241 242 spin_lock_irqsave(&ctx->timeout_lock, flags); 243 list_del_init(&timeout->list); 244 atomic_set(&req->ctx->cq_timeouts, 245 atomic_read(&req->ctx->cq_timeouts) + 1); 246 spin_unlock_irqrestore(&ctx->timeout_lock, flags); 247 248 if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS)) 249 req_set_fail(req); 250 251 io_req_set_res(req, -ETIME, 0); 252 req->io_task_work.func = io_timeout_complete; 253 io_req_task_work_add(req); 254 return HRTIMER_NORESTART; 255 } 256 257 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, 258 struct io_cancel_data *cd) 259 __must_hold(&ctx->timeout_lock) 260 { 261 struct io_timeout *timeout; 262 struct io_timeout_data *io; 263 struct io_kiocb *req = NULL; 264 265 list_for_each_entry(timeout, &ctx->timeout_list, list) { 266 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); 267 268 if (io_cancel_req_match(tmp, cd)) { 269 req = tmp; 270 break; 271 } 272 } 273 if (!req) 274 return ERR_PTR(-ENOENT); 275 276 io = req->async_data; 277 if (hrtimer_try_to_cancel(&io->timer) == -1) 278 return ERR_PTR(-EALREADY); 279 timeout = io_kiocb_to_cmd(req, struct io_timeout); 280 list_del_init(&timeout->list); 281 return req; 282 } 283 284 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) 285 __must_hold(&ctx->completion_lock) 286 { 287 struct io_kiocb *req; 288 289 spin_lock_irq(&ctx->timeout_lock); 290 req = io_timeout_extract(ctx, cd); 291 spin_unlock_irq(&ctx->timeout_lock); 292 293 if (IS_ERR(req)) 294 return PTR_ERR(req); 295 io_req_task_queue_fail(req, -ECANCELED); 296 return 0; 297 } 298 299 static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts) 300 { 301 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 302 struct io_kiocb *prev = timeout->prev; 303 int ret; 304 305 if (prev) { 306 if (!io_should_terminate_tw()) { 307 struct io_cancel_data cd = { 308 .ctx = req->ctx, 309 .data = prev->cqe.user_data, 310 }; 311 312 ret = io_try_cancel(req->tctx, &cd, 0); 313 } else { 314 ret = -ECANCELED; 315 } 316 io_req_set_res(req, ret ?: -ETIME, 0); 317 io_req_task_complete(req, ts); 318 io_put_req(prev); 319 } else { 320 io_req_set_res(req, -ETIME, 0); 321 io_req_task_complete(req, ts); 322 } 323 } 324 325 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) 326 { 327 struct io_timeout_data *data = container_of(timer, 328 struct io_timeout_data, timer); 329 struct io_kiocb *prev, *req = data->req; 330 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 331 struct io_ring_ctx *ctx = req->ctx; 332 unsigned long flags; 333 334 spin_lock_irqsave(&ctx->timeout_lock, flags); 335 prev = timeout->head; 336 timeout->head = NULL; 337 338 /* 339 * We don't expect the list to be empty, that will only happen if we 340 * race with the completion of the linked work. 341 */ 342 if (prev) { 343 io_remove_next_linked(prev); 344 if (!req_ref_inc_not_zero(prev)) 345 prev = NULL; 346 } 347 list_del(&timeout->list); 348 timeout->prev = prev; 349 spin_unlock_irqrestore(&ctx->timeout_lock, flags); 350 351 req->io_task_work.func = io_req_task_link_timeout; 352 io_req_task_work_add(req); 353 return HRTIMER_NORESTART; 354 } 355 356 static clockid_t io_timeout_get_clock(struct io_timeout_data *data) 357 { 358 switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) { 359 case IORING_TIMEOUT_BOOTTIME: 360 return CLOCK_BOOTTIME; 361 case IORING_TIMEOUT_REALTIME: 362 return CLOCK_REALTIME; 363 default: 364 /* can't happen, vetted at prep time */ 365 WARN_ON_ONCE(1); 366 fallthrough; 367 case 0: 368 return CLOCK_MONOTONIC; 369 } 370 } 371 372 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, 373 struct timespec64 *ts, enum hrtimer_mode mode) 374 __must_hold(&ctx->timeout_lock) 375 { 376 struct io_timeout_data *io; 377 struct io_timeout *timeout; 378 struct io_kiocb *req = NULL; 379 380 list_for_each_entry(timeout, &ctx->ltimeout_list, list) { 381 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); 382 383 if (user_data == tmp->cqe.user_data) { 384 req = tmp; 385 break; 386 } 387 } 388 if (!req) 389 return -ENOENT; 390 391 io = req->async_data; 392 if (hrtimer_try_to_cancel(&io->timer) == -1) 393 return -EALREADY; 394 hrtimer_init(&io->timer, io_timeout_get_clock(io), mode); 395 io->timer.function = io_link_timeout_fn; 396 hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode); 397 return 0; 398 } 399 400 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, 401 struct timespec64 *ts, enum hrtimer_mode mode) 402 __must_hold(&ctx->timeout_lock) 403 { 404 struct io_cancel_data cd = { .ctx = ctx, .data = user_data, }; 405 struct io_kiocb *req = io_timeout_extract(ctx, &cd); 406 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 407 struct io_timeout_data *data; 408 409 if (IS_ERR(req)) 410 return PTR_ERR(req); 411 412 timeout->off = 0; /* noseq */ 413 data = req->async_data; 414 list_add_tail(&timeout->list, &ctx->timeout_list); 415 hrtimer_init(&data->timer, io_timeout_get_clock(data), mode); 416 data->timer.function = io_timeout_fn; 417 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode); 418 return 0; 419 } 420 421 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 422 { 423 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); 424 425 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) 426 return -EINVAL; 427 if (sqe->buf_index || sqe->len || sqe->splice_fd_in) 428 return -EINVAL; 429 430 tr->ltimeout = false; 431 tr->addr = READ_ONCE(sqe->addr); 432 tr->flags = READ_ONCE(sqe->timeout_flags); 433 if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) { 434 if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1) 435 return -EINVAL; 436 if (tr->flags & IORING_LINK_TIMEOUT_UPDATE) 437 tr->ltimeout = true; 438 if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS)) 439 return -EINVAL; 440 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2))) 441 return -EFAULT; 442 if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0) 443 return -EINVAL; 444 } else if (tr->flags) { 445 /* timeout removal doesn't support flags */ 446 return -EINVAL; 447 } 448 449 return 0; 450 } 451 452 static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) 453 { 454 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS 455 : HRTIMER_MODE_REL; 456 } 457 458 /* 459 * Remove or update an existing timeout command 460 */ 461 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) 462 { 463 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); 464 struct io_ring_ctx *ctx = req->ctx; 465 int ret; 466 467 if (!(tr->flags & IORING_TIMEOUT_UPDATE)) { 468 struct io_cancel_data cd = { .ctx = ctx, .data = tr->addr, }; 469 470 spin_lock(&ctx->completion_lock); 471 ret = io_timeout_cancel(ctx, &cd); 472 spin_unlock(&ctx->completion_lock); 473 } else { 474 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags); 475 476 spin_lock_irq(&ctx->timeout_lock); 477 if (tr->ltimeout) 478 ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode); 479 else 480 ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode); 481 spin_unlock_irq(&ctx->timeout_lock); 482 } 483 484 if (ret < 0) 485 req_set_fail(req); 486 io_req_set_res(req, ret, 0); 487 return IOU_OK; 488 } 489 490 static int __io_timeout_prep(struct io_kiocb *req, 491 const struct io_uring_sqe *sqe, 492 bool is_timeout_link) 493 { 494 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 495 struct io_timeout_data *data; 496 unsigned flags; 497 u32 off = READ_ONCE(sqe->off); 498 499 if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in) 500 return -EINVAL; 501 if (off && is_timeout_link) 502 return -EINVAL; 503 flags = READ_ONCE(sqe->timeout_flags); 504 if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK | 505 IORING_TIMEOUT_ETIME_SUCCESS | 506 IORING_TIMEOUT_MULTISHOT)) 507 return -EINVAL; 508 /* more than one clock specified is invalid, obviously */ 509 if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1) 510 return -EINVAL; 511 /* multishot requests only make sense with rel values */ 512 if (!(~flags & (IORING_TIMEOUT_MULTISHOT | IORING_TIMEOUT_ABS))) 513 return -EINVAL; 514 515 INIT_LIST_HEAD(&timeout->list); 516 timeout->off = off; 517 if (unlikely(off && !req->ctx->off_timeout_used)) 518 req->ctx->off_timeout_used = true; 519 /* 520 * for multishot reqs w/ fixed nr of repeats, repeats tracks the 521 * remaining nr 522 */ 523 timeout->repeats = 0; 524 if ((flags & IORING_TIMEOUT_MULTISHOT) && off > 0) 525 timeout->repeats = off; 526 527 if (WARN_ON_ONCE(req_has_async_data(req))) 528 return -EFAULT; 529 if (io_alloc_async_data(req)) 530 return -ENOMEM; 531 532 data = req->async_data; 533 data->req = req; 534 data->flags = flags; 535 536 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) 537 return -EFAULT; 538 539 if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0) 540 return -EINVAL; 541 542 data->mode = io_translate_timeout_mode(flags); 543 hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode); 544 545 if (is_timeout_link) { 546 struct io_submit_link *link = &req->ctx->submit_state.link; 547 548 if (!link->head) 549 return -EINVAL; 550 if (link->last->opcode == IORING_OP_LINK_TIMEOUT) 551 return -EINVAL; 552 timeout->head = link->last; 553 link->last->flags |= REQ_F_ARM_LTIMEOUT; 554 } 555 return 0; 556 } 557 558 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 559 { 560 return __io_timeout_prep(req, sqe, false); 561 } 562 563 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 564 { 565 return __io_timeout_prep(req, sqe, true); 566 } 567 568 int io_timeout(struct io_kiocb *req, unsigned int issue_flags) 569 { 570 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 571 struct io_ring_ctx *ctx = req->ctx; 572 struct io_timeout_data *data = req->async_data; 573 struct list_head *entry; 574 u32 tail, off = timeout->off; 575 576 spin_lock_irq(&ctx->timeout_lock); 577 578 /* 579 * sqe->off holds how many events that need to occur for this 580 * timeout event to be satisfied. If it isn't set, then this is 581 * a pure timeout request, sequence isn't used. 582 */ 583 if (io_is_timeout_noseq(req)) { 584 entry = ctx->timeout_list.prev; 585 goto add; 586 } 587 588 tail = data_race(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts); 589 timeout->target_seq = tail + off; 590 591 /* Update the last seq here in case io_flush_timeouts() hasn't. 592 * This is safe because ->completion_lock is held, and submissions 593 * and completions are never mixed in the same ->completion_lock section. 594 */ 595 ctx->cq_last_tm_flush = tail; 596 597 /* 598 * Insertion sort, ensuring the first entry in the list is always 599 * the one we need first. 600 */ 601 list_for_each_prev(entry, &ctx->timeout_list) { 602 struct io_timeout *nextt = list_entry(entry, struct io_timeout, list); 603 struct io_kiocb *nxt = cmd_to_io_kiocb(nextt); 604 605 if (io_is_timeout_noseq(nxt)) 606 continue; 607 /* nxt.seq is behind @tail, otherwise would've been completed */ 608 if (off >= nextt->target_seq - tail) 609 break; 610 } 611 add: 612 list_add(&timeout->list, entry); 613 data->timer.function = io_timeout_fn; 614 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); 615 spin_unlock_irq(&ctx->timeout_lock); 616 return IOU_ISSUE_SKIP_COMPLETE; 617 } 618 619 void io_queue_linked_timeout(struct io_kiocb *req) 620 { 621 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 622 struct io_ring_ctx *ctx = req->ctx; 623 624 spin_lock_irq(&ctx->timeout_lock); 625 /* 626 * If the back reference is NULL, then our linked request finished 627 * before we got a chance to setup the timer 628 */ 629 if (timeout->head) { 630 struct io_timeout_data *data = req->async_data; 631 632 data->timer.function = io_link_timeout_fn; 633 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), 634 data->mode); 635 list_add_tail(&timeout->list, &ctx->ltimeout_list); 636 } 637 spin_unlock_irq(&ctx->timeout_lock); 638 /* drop submission reference */ 639 io_put_req(req); 640 } 641 642 static bool io_match_task(struct io_kiocb *head, struct io_uring_task *tctx, 643 bool cancel_all) 644 __must_hold(&head->ctx->timeout_lock) 645 { 646 struct io_kiocb *req; 647 648 if (tctx && head->tctx != tctx) 649 return false; 650 if (cancel_all) 651 return true; 652 653 io_for_each_link(req, head) { 654 if (req->flags & REQ_F_INFLIGHT) 655 return true; 656 } 657 return false; 658 } 659 660 /* Returns true if we found and killed one or more timeouts */ 661 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct io_uring_task *tctx, 662 bool cancel_all) 663 { 664 struct io_timeout *timeout, *tmp; 665 int canceled = 0; 666 667 /* 668 * completion_lock is needed for io_match_task(). Take it before 669 * timeout_lockfirst to keep locking ordering. 670 */ 671 spin_lock(&ctx->completion_lock); 672 spin_lock_irq(&ctx->timeout_lock); 673 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { 674 struct io_kiocb *req = cmd_to_io_kiocb(timeout); 675 676 if (io_match_task(req, tctx, cancel_all) && 677 io_kill_timeout(req, -ECANCELED)) 678 canceled++; 679 } 680 spin_unlock_irq(&ctx->timeout_lock); 681 spin_unlock(&ctx->completion_lock); 682 return canceled != 0; 683 } 684