1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include "io_uring.h"
3 #include "wait.h"
4 #include "loop.h"
5 #include "tctx.h"
6
io_loop_nr_cqes(const struct io_ring_ctx * ctx,const struct iou_loop_params * lp)7 static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx,
8 const struct iou_loop_params *lp)
9 {
10 return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail);
11 }
12
io_loop_wait_start(struct io_ring_ctx * ctx,unsigned nr_wait)13 static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait)
14 {
15 atomic_set(&ctx->cq_wait_nr, nr_wait);
16 set_current_state(TASK_INTERRUPTIBLE);
17 }
18
io_loop_wait_finish(struct io_ring_ctx * ctx)19 static inline void io_loop_wait_finish(struct io_ring_ctx *ctx)
20 {
21 __set_current_state(TASK_RUNNING);
22 atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
23 }
24
io_loop_wait(struct io_ring_ctx * ctx,struct iou_loop_params * lp,unsigned nr_wait)25 static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp,
26 unsigned nr_wait)
27 {
28 io_loop_wait_start(ctx, nr_wait);
29
30 if (unlikely(io_local_work_pending(ctx) ||
31 io_loop_nr_cqes(ctx, lp) <= 0) ||
32 READ_ONCE(ctx->check_cq)) {
33 io_loop_wait_finish(ctx);
34 return;
35 }
36
37 mutex_unlock(&ctx->uring_lock);
38 schedule();
39 io_loop_wait_finish(ctx);
40 mutex_lock(&ctx->uring_lock);
41 }
42
__io_run_loop(struct io_ring_ctx * ctx)43 static int __io_run_loop(struct io_ring_ctx *ctx)
44 {
45 struct iou_loop_params lp = {};
46
47 while (true) {
48 int nr_wait, step_res;
49
50 if (unlikely(!ctx->loop_step))
51 return -EFAULT;
52
53 step_res = ctx->loop_step(io_loop_mangle_ctx(ctx), &lp);
54 if (step_res == IOU_LOOP_STOP)
55 break;
56 if (step_res != IOU_LOOP_CONTINUE)
57 return -EINVAL;
58
59 nr_wait = io_loop_nr_cqes(ctx, &lp);
60 if (nr_wait > 0)
61 io_loop_wait(ctx, &lp, nr_wait);
62 else
63 nr_wait = 0;
64
65 if (task_work_pending(current)) {
66 mutex_unlock(&ctx->uring_lock);
67 io_run_task_work();
68 mutex_lock(&ctx->uring_lock);
69 }
70 if (unlikely(task_sigpending(current)))
71 return -EINTR;
72 io_run_local_work_locked(ctx, nr_wait);
73
74 if (READ_ONCE(ctx->check_cq) & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
75 io_cqring_overflow_flush_locked(ctx);
76 }
77
78 return 0;
79 }
80
io_run_loop(struct io_ring_ctx * ctx)81 int io_run_loop(struct io_ring_ctx *ctx)
82 {
83 int ret;
84
85 ret = io_uring_add_tctx_node(ctx);
86 if (unlikely(ret))
87 return ret;
88
89 if (!io_allowed_run_tw(ctx))
90 return -EEXIST;
91
92 mutex_lock(&ctx->uring_lock);
93 ret = __io_run_loop(ctx);
94 mutex_unlock(&ctx->uring_lock);
95 return ret;
96 }
97