1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/io_uring.h>
6
7 #include <trace/events/io_uring.h>
8
9 #include <uapi/linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "refs.h"
13 #include "cancel.h"
14 #include "timeout.h"
15
16 struct io_timeout {
17 struct file *file;
18 u32 off;
19 u32 target_seq;
20 u32 repeats;
21 struct list_head list;
22 /* head of the link, used by linked timeouts only */
23 struct io_kiocb *head;
24 /* for linked completions */
25 struct io_kiocb *prev;
26 };
27
28 struct io_timeout_rem {
29 struct file *file;
30 u64 addr;
31
32 /* timeout update */
33 struct timespec64 ts;
34 u32 flags;
35 bool ltimeout;
36 };
37
io_is_timeout_noseq(struct io_kiocb * req)38 static inline bool io_is_timeout_noseq(struct io_kiocb *req)
39 {
40 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
41 struct io_timeout_data *data = req->async_data;
42
43 return !timeout->off || data->flags & IORING_TIMEOUT_MULTISHOT;
44 }
45
io_put_req(struct io_kiocb * req)46 static inline void io_put_req(struct io_kiocb *req)
47 {
48 if (req_ref_put_and_test(req)) {
49 io_queue_next(req);
50 io_free_req(req);
51 }
52 }
53
io_timeout_finish(struct io_timeout * timeout,struct io_timeout_data * data)54 static inline bool io_timeout_finish(struct io_timeout *timeout,
55 struct io_timeout_data *data)
56 {
57 if (!(data->flags & IORING_TIMEOUT_MULTISHOT))
58 return true;
59
60 if (!timeout->off || (timeout->repeats && --timeout->repeats))
61 return false;
62
63 return true;
64 }
65
66 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer);
67
io_timeout_complete(struct io_kiocb * req,struct io_tw_state * ts)68 static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)
69 {
70 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
71 struct io_timeout_data *data = req->async_data;
72 struct io_ring_ctx *ctx = req->ctx;
73
74 if (!io_timeout_finish(timeout, data)) {
75 if (io_req_post_cqe(req, -ETIME, IORING_CQE_F_MORE)) {
76 /* re-arm timer */
77 raw_spin_lock_irq(&ctx->timeout_lock);
78 list_add(&timeout->list, ctx->timeout_list.prev);
79 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
80 raw_spin_unlock_irq(&ctx->timeout_lock);
81 return;
82 }
83 }
84
85 io_req_task_complete(req, ts);
86 }
87
io_kill_timeout(struct io_kiocb * req,int status)88 static bool io_kill_timeout(struct io_kiocb *req, int status)
89 __must_hold(&req->ctx->timeout_lock)
90 {
91 struct io_timeout_data *io = req->async_data;
92
93 if (hrtimer_try_to_cancel(&io->timer) != -1) {
94 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
95
96 if (status)
97 req_set_fail(req);
98 atomic_set(&req->ctx->cq_timeouts,
99 atomic_read(&req->ctx->cq_timeouts) + 1);
100 list_del_init(&timeout->list);
101 io_req_queue_tw_complete(req, status);
102 return true;
103 }
104 return false;
105 }
106
io_flush_timeouts(struct io_ring_ctx * ctx)107 __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
108 {
109 u32 seq;
110 struct io_timeout *timeout, *tmp;
111
112 raw_spin_lock_irq(&ctx->timeout_lock);
113 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
114
115 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
116 struct io_kiocb *req = cmd_to_io_kiocb(timeout);
117 u32 events_needed, events_got;
118
119 if (io_is_timeout_noseq(req))
120 break;
121
122 /*
123 * Since seq can easily wrap around over time, subtract
124 * the last seq at which timeouts were flushed before comparing.
125 * Assuming not more than 2^31-1 events have happened since,
126 * these subtractions won't have wrapped, so we can check if
127 * target is in [last_seq, current_seq] by comparing the two.
128 */
129 events_needed = timeout->target_seq - ctx->cq_last_tm_flush;
130 events_got = seq - ctx->cq_last_tm_flush;
131 if (events_got < events_needed)
132 break;
133
134 io_kill_timeout(req, 0);
135 }
136 ctx->cq_last_tm_flush = seq;
137 raw_spin_unlock_irq(&ctx->timeout_lock);
138 }
139
io_req_tw_fail_links(struct io_kiocb * link,struct io_tw_state * ts)140 static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts)
141 {
142 io_tw_lock(link->ctx, ts);
143 while (link) {
144 struct io_kiocb *nxt = link->link;
145 long res = -ECANCELED;
146
147 if (link->flags & REQ_F_FAIL)
148 res = link->cqe.res;
149 link->link = NULL;
150 io_req_set_res(link, res, 0);
151 io_req_task_complete(link, ts);
152 link = nxt;
153 }
154 }
155
io_fail_links(struct io_kiocb * req)156 static void io_fail_links(struct io_kiocb *req)
157 __must_hold(&req->ctx->completion_lock)
158 {
159 struct io_kiocb *link = req->link;
160 bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES;
161
162 if (!link)
163 return;
164
165 while (link) {
166 if (ignore_cqes)
167 link->flags |= REQ_F_CQE_SKIP;
168 else
169 link->flags &= ~REQ_F_CQE_SKIP;
170 trace_io_uring_fail_link(req, link);
171 link = link->link;
172 }
173
174 link = req->link;
175 link->io_task_work.func = io_req_tw_fail_links;
176 io_req_task_work_add(link);
177 req->link = NULL;
178 }
179
io_remove_next_linked(struct io_kiocb * req)180 static inline void io_remove_next_linked(struct io_kiocb *req)
181 {
182 struct io_kiocb *nxt = req->link;
183
184 req->link = nxt->link;
185 nxt->link = NULL;
186 }
187
io_disarm_next(struct io_kiocb * req)188 void io_disarm_next(struct io_kiocb *req)
189 __must_hold(&req->ctx->completion_lock)
190 {
191 struct io_kiocb *link = NULL;
192
193 if (req->flags & REQ_F_ARM_LTIMEOUT) {
194 link = req->link;
195 req->flags &= ~REQ_F_ARM_LTIMEOUT;
196 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
197 io_remove_next_linked(req);
198 io_req_queue_tw_complete(link, -ECANCELED);
199 }
200 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
201 struct io_ring_ctx *ctx = req->ctx;
202
203 raw_spin_lock_irq(&ctx->timeout_lock);
204 link = io_disarm_linked_timeout(req);
205 raw_spin_unlock_irq(&ctx->timeout_lock);
206 if (link)
207 io_req_queue_tw_complete(link, -ECANCELED);
208 }
209 if (unlikely((req->flags & REQ_F_FAIL) &&
210 !(req->flags & REQ_F_HARDLINK)))
211 io_fail_links(req);
212 }
213
__io_disarm_linked_timeout(struct io_kiocb * req,struct io_kiocb * link)214 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
215 struct io_kiocb *link)
216 __must_hold(&req->ctx->completion_lock)
217 __must_hold(&req->ctx->timeout_lock)
218 {
219 struct io_timeout_data *io = link->async_data;
220 struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout);
221
222 io_remove_next_linked(req);
223 timeout->head = NULL;
224 if (hrtimer_try_to_cancel(&io->timer) != -1) {
225 list_del(&timeout->list);
226 return link;
227 }
228
229 return NULL;
230 }
231
io_timeout_fn(struct hrtimer * timer)232 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
233 {
234 struct io_timeout_data *data = container_of(timer,
235 struct io_timeout_data, timer);
236 struct io_kiocb *req = data->req;
237 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
238 struct io_ring_ctx *ctx = req->ctx;
239 unsigned long flags;
240
241 raw_spin_lock_irqsave(&ctx->timeout_lock, flags);
242 list_del_init(&timeout->list);
243 atomic_set(&req->ctx->cq_timeouts,
244 atomic_read(&req->ctx->cq_timeouts) + 1);
245 raw_spin_unlock_irqrestore(&ctx->timeout_lock, flags);
246
247 if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS))
248 req_set_fail(req);
249
250 io_req_set_res(req, -ETIME, 0);
251 req->io_task_work.func = io_timeout_complete;
252 io_req_task_work_add(req);
253 return HRTIMER_NORESTART;
254 }
255
io_timeout_extract(struct io_ring_ctx * ctx,struct io_cancel_data * cd)256 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
257 struct io_cancel_data *cd)
258 __must_hold(&ctx->timeout_lock)
259 {
260 struct io_timeout *timeout;
261 struct io_timeout_data *io;
262 struct io_kiocb *req = NULL;
263
264 list_for_each_entry(timeout, &ctx->timeout_list, list) {
265 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
266
267 if (io_cancel_req_match(tmp, cd)) {
268 req = tmp;
269 break;
270 }
271 }
272 if (!req)
273 return ERR_PTR(-ENOENT);
274
275 io = req->async_data;
276 if (hrtimer_try_to_cancel(&io->timer) == -1)
277 return ERR_PTR(-EALREADY);
278 timeout = io_kiocb_to_cmd(req, struct io_timeout);
279 list_del_init(&timeout->list);
280 return req;
281 }
282
io_timeout_cancel(struct io_ring_ctx * ctx,struct io_cancel_data * cd)283 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
284 __must_hold(&ctx->completion_lock)
285 {
286 struct io_kiocb *req;
287
288 raw_spin_lock_irq(&ctx->timeout_lock);
289 req = io_timeout_extract(ctx, cd);
290 raw_spin_unlock_irq(&ctx->timeout_lock);
291
292 if (IS_ERR(req))
293 return PTR_ERR(req);
294 io_req_task_queue_fail(req, -ECANCELED);
295 return 0;
296 }
297
io_req_task_link_timeout(struct io_kiocb * req,struct io_tw_state * ts)298 static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts)
299 {
300 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
301 struct io_kiocb *prev = timeout->prev;
302 int ret;
303
304 if (prev) {
305 if (!io_should_terminate_tw()) {
306 struct io_cancel_data cd = {
307 .ctx = req->ctx,
308 .data = prev->cqe.user_data,
309 };
310
311 ret = io_try_cancel(req->tctx, &cd, 0);
312 } else {
313 ret = -ECANCELED;
314 }
315 io_req_set_res(req, ret ?: -ETIME, 0);
316 io_req_task_complete(req, ts);
317 io_put_req(prev);
318 } else {
319 io_req_set_res(req, -ETIME, 0);
320 io_req_task_complete(req, ts);
321 }
322 }
323
io_link_timeout_fn(struct hrtimer * timer)324 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
325 {
326 struct io_timeout_data *data = container_of(timer,
327 struct io_timeout_data, timer);
328 struct io_kiocb *prev, *req = data->req;
329 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
330 struct io_ring_ctx *ctx = req->ctx;
331 unsigned long flags;
332
333 raw_spin_lock_irqsave(&ctx->timeout_lock, flags);
334 prev = timeout->head;
335 timeout->head = NULL;
336
337 /*
338 * We don't expect the list to be empty, that will only happen if we
339 * race with the completion of the linked work.
340 */
341 if (prev) {
342 io_remove_next_linked(prev);
343 if (!req_ref_inc_not_zero(prev))
344 prev = NULL;
345 }
346 list_del(&timeout->list);
347 timeout->prev = prev;
348 raw_spin_unlock_irqrestore(&ctx->timeout_lock, flags);
349
350 req->io_task_work.func = io_req_task_link_timeout;
351 io_req_task_work_add(req);
352 return HRTIMER_NORESTART;
353 }
354
io_timeout_get_clock(struct io_timeout_data * data)355 static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
356 {
357 switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) {
358 case IORING_TIMEOUT_BOOTTIME:
359 return CLOCK_BOOTTIME;
360 case IORING_TIMEOUT_REALTIME:
361 return CLOCK_REALTIME;
362 default:
363 /* can't happen, vetted at prep time */
364 WARN_ON_ONCE(1);
365 fallthrough;
366 case 0:
367 return CLOCK_MONOTONIC;
368 }
369 }
370
io_linked_timeout_update(struct io_ring_ctx * ctx,__u64 user_data,struct timespec64 * ts,enum hrtimer_mode mode)371 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
372 struct timespec64 *ts, enum hrtimer_mode mode)
373 __must_hold(&ctx->timeout_lock)
374 {
375 struct io_timeout_data *io;
376 struct io_timeout *timeout;
377 struct io_kiocb *req = NULL;
378
379 list_for_each_entry(timeout, &ctx->ltimeout_list, list) {
380 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
381
382 if (user_data == tmp->cqe.user_data) {
383 req = tmp;
384 break;
385 }
386 }
387 if (!req)
388 return -ENOENT;
389
390 io = req->async_data;
391 if (hrtimer_try_to_cancel(&io->timer) == -1)
392 return -EALREADY;
393 hrtimer_init(&io->timer, io_timeout_get_clock(io), mode);
394 io->timer.function = io_link_timeout_fn;
395 hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode);
396 return 0;
397 }
398
io_timeout_update(struct io_ring_ctx * ctx,__u64 user_data,struct timespec64 * ts,enum hrtimer_mode mode)399 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
400 struct timespec64 *ts, enum hrtimer_mode mode)
401 __must_hold(&ctx->timeout_lock)
402 {
403 struct io_cancel_data cd = { .ctx = ctx, .data = user_data, };
404 struct io_kiocb *req = io_timeout_extract(ctx, &cd);
405 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
406 struct io_timeout_data *data;
407
408 if (IS_ERR(req))
409 return PTR_ERR(req);
410
411 timeout->off = 0; /* noseq */
412 data = req->async_data;
413 list_add_tail(&timeout->list, &ctx->timeout_list);
414 hrtimer_init(&data->timer, io_timeout_get_clock(data), mode);
415 data->timer.function = io_timeout_fn;
416 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
417 return 0;
418 }
419
io_timeout_remove_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)420 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
421 {
422 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);
423
424 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
425 return -EINVAL;
426 if (sqe->buf_index || sqe->len || sqe->splice_fd_in)
427 return -EINVAL;
428
429 tr->ltimeout = false;
430 tr->addr = READ_ONCE(sqe->addr);
431 tr->flags = READ_ONCE(sqe->timeout_flags);
432 if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) {
433 if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
434 return -EINVAL;
435 if (tr->flags & IORING_LINK_TIMEOUT_UPDATE)
436 tr->ltimeout = true;
437 if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS))
438 return -EINVAL;
439 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
440 return -EFAULT;
441 if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0)
442 return -EINVAL;
443 } else if (tr->flags) {
444 /* timeout removal doesn't support flags */
445 return -EINVAL;
446 }
447
448 return 0;
449 }
450
io_translate_timeout_mode(unsigned int flags)451 static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
452 {
453 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
454 : HRTIMER_MODE_REL;
455 }
456
457 /*
458 * Remove or update an existing timeout command
459 */
io_timeout_remove(struct io_kiocb * req,unsigned int issue_flags)460 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
461 {
462 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);
463 struct io_ring_ctx *ctx = req->ctx;
464 int ret;
465
466 if (!(tr->flags & IORING_TIMEOUT_UPDATE)) {
467 struct io_cancel_data cd = { .ctx = ctx, .data = tr->addr, };
468
469 spin_lock(&ctx->completion_lock);
470 ret = io_timeout_cancel(ctx, &cd);
471 spin_unlock(&ctx->completion_lock);
472 } else {
473 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);
474
475 raw_spin_lock_irq(&ctx->timeout_lock);
476 if (tr->ltimeout)
477 ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode);
478 else
479 ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode);
480 raw_spin_unlock_irq(&ctx->timeout_lock);
481 }
482
483 if (ret < 0)
484 req_set_fail(req);
485 io_req_set_res(req, ret, 0);
486 return IOU_OK;
487 }
488
__io_timeout_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe,bool is_timeout_link)489 static int __io_timeout_prep(struct io_kiocb *req,
490 const struct io_uring_sqe *sqe,
491 bool is_timeout_link)
492 {
493 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
494 struct io_timeout_data *data;
495 unsigned flags;
496 u32 off = READ_ONCE(sqe->off);
497
498 if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in)
499 return -EINVAL;
500 if (off && is_timeout_link)
501 return -EINVAL;
502 flags = READ_ONCE(sqe->timeout_flags);
503 if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK |
504 IORING_TIMEOUT_ETIME_SUCCESS |
505 IORING_TIMEOUT_MULTISHOT))
506 return -EINVAL;
507 /* more than one clock specified is invalid, obviously */
508 if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
509 return -EINVAL;
510 /* multishot requests only make sense with rel values */
511 if (!(~flags & (IORING_TIMEOUT_MULTISHOT | IORING_TIMEOUT_ABS)))
512 return -EINVAL;
513
514 INIT_LIST_HEAD(&timeout->list);
515 timeout->off = off;
516 if (unlikely(off && !req->ctx->off_timeout_used))
517 req->ctx->off_timeout_used = true;
518 /*
519 * for multishot reqs w/ fixed nr of repeats, repeats tracks the
520 * remaining nr
521 */
522 timeout->repeats = 0;
523 if ((flags & IORING_TIMEOUT_MULTISHOT) && off > 0)
524 timeout->repeats = off;
525
526 if (WARN_ON_ONCE(req_has_async_data(req)))
527 return -EFAULT;
528 if (io_alloc_async_data(req))
529 return -ENOMEM;
530
531 data = req->async_data;
532 data->req = req;
533 data->flags = flags;
534
535 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
536 return -EFAULT;
537
538 if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0)
539 return -EINVAL;
540
541 data->mode = io_translate_timeout_mode(flags);
542 hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode);
543
544 if (is_timeout_link) {
545 struct io_submit_link *link = &req->ctx->submit_state.link;
546
547 if (!link->head)
548 return -EINVAL;
549 if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
550 return -EINVAL;
551 timeout->head = link->last;
552 link->last->flags |= REQ_F_ARM_LTIMEOUT;
553 }
554 return 0;
555 }
556
io_timeout_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)557 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
558 {
559 return __io_timeout_prep(req, sqe, false);
560 }
561
io_link_timeout_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)562 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
563 {
564 return __io_timeout_prep(req, sqe, true);
565 }
566
io_timeout(struct io_kiocb * req,unsigned int issue_flags)567 int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
568 {
569 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
570 struct io_ring_ctx *ctx = req->ctx;
571 struct io_timeout_data *data = req->async_data;
572 struct list_head *entry;
573 u32 tail, off = timeout->off;
574
575 raw_spin_lock_irq(&ctx->timeout_lock);
576
577 /*
578 * sqe->off holds how many events that need to occur for this
579 * timeout event to be satisfied. If it isn't set, then this is
580 * a pure timeout request, sequence isn't used.
581 */
582 if (io_is_timeout_noseq(req)) {
583 entry = ctx->timeout_list.prev;
584 goto add;
585 }
586
587 tail = data_race(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts);
588 timeout->target_seq = tail + off;
589
590 /* Update the last seq here in case io_flush_timeouts() hasn't.
591 * This is safe because ->completion_lock is held, and submissions
592 * and completions are never mixed in the same ->completion_lock section.
593 */
594 ctx->cq_last_tm_flush = tail;
595
596 /*
597 * Insertion sort, ensuring the first entry in the list is always
598 * the one we need first.
599 */
600 list_for_each_prev(entry, &ctx->timeout_list) {
601 struct io_timeout *nextt = list_entry(entry, struct io_timeout, list);
602 struct io_kiocb *nxt = cmd_to_io_kiocb(nextt);
603
604 if (io_is_timeout_noseq(nxt))
605 continue;
606 /* nxt.seq is behind @tail, otherwise would've been completed */
607 if (off >= nextt->target_seq - tail)
608 break;
609 }
610 add:
611 list_add(&timeout->list, entry);
612 data->timer.function = io_timeout_fn;
613 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
614 raw_spin_unlock_irq(&ctx->timeout_lock);
615 return IOU_ISSUE_SKIP_COMPLETE;
616 }
617
io_queue_linked_timeout(struct io_kiocb * req)618 void io_queue_linked_timeout(struct io_kiocb *req)
619 {
620 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
621 struct io_ring_ctx *ctx = req->ctx;
622
623 raw_spin_lock_irq(&ctx->timeout_lock);
624 /*
625 * If the back reference is NULL, then our linked request finished
626 * before we got a chance to setup the timer
627 */
628 if (timeout->head) {
629 struct io_timeout_data *data = req->async_data;
630
631 data->timer.function = io_link_timeout_fn;
632 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
633 data->mode);
634 list_add_tail(&timeout->list, &ctx->ltimeout_list);
635 }
636 raw_spin_unlock_irq(&ctx->timeout_lock);
637 /* drop submission reference */
638 io_put_req(req);
639 }
640
io_match_task(struct io_kiocb * head,struct io_uring_task * tctx,bool cancel_all)641 static bool io_match_task(struct io_kiocb *head, struct io_uring_task *tctx,
642 bool cancel_all)
643 __must_hold(&head->ctx->timeout_lock)
644 {
645 struct io_kiocb *req;
646
647 if (tctx && head->tctx != tctx)
648 return false;
649 if (cancel_all)
650 return true;
651
652 io_for_each_link(req, head) {
653 if (req->flags & REQ_F_INFLIGHT)
654 return true;
655 }
656 return false;
657 }
658
659 /* Returns true if we found and killed one or more timeouts */
io_kill_timeouts(struct io_ring_ctx * ctx,struct io_uring_task * tctx,bool cancel_all)660 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
661 bool cancel_all)
662 {
663 struct io_timeout *timeout, *tmp;
664 int canceled = 0;
665
666 /*
667 * completion_lock is needed for io_match_task(). Take it before
668 * timeout_lockfirst to keep locking ordering.
669 */
670 spin_lock(&ctx->completion_lock);
671 raw_spin_lock_irq(&ctx->timeout_lock);
672 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
673 struct io_kiocb *req = cmd_to_io_kiocb(timeout);
674
675 if (io_match_task(req, tctx, cancel_all) &&
676 io_kill_timeout(req, -ECANCELED))
677 canceled++;
678 }
679 raw_spin_unlock_irq(&ctx->timeout_lock);
680 spin_unlock(&ctx->completion_lock);
681 return canceled != 0;
682 }
683