1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #include "io_uring.h" 3 #include "wait.h" 4 #include "loop.h" 5 #include "tctx.h" 6 7 static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx, 8 const struct iou_loop_params *lp) 9 { 10 return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail); 11 } 12 13 static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait) 14 { 15 atomic_set(&ctx->cq_wait_nr, nr_wait); 16 set_current_state(TASK_INTERRUPTIBLE); 17 } 18 19 static inline void io_loop_wait_finish(struct io_ring_ctx *ctx) 20 { 21 __set_current_state(TASK_RUNNING); 22 atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT); 23 } 24 25 static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp, 26 unsigned nr_wait) 27 { 28 io_loop_wait_start(ctx, nr_wait); 29 30 if (unlikely(io_local_work_pending(ctx) || 31 io_loop_nr_cqes(ctx, lp) <= 0) || 32 READ_ONCE(ctx->check_cq)) { 33 io_loop_wait_finish(ctx); 34 return; 35 } 36 37 mutex_unlock(&ctx->uring_lock); 38 schedule(); 39 io_loop_wait_finish(ctx); 40 mutex_lock(&ctx->uring_lock); 41 } 42 43 static int __io_run_loop(struct io_ring_ctx *ctx) 44 { 45 struct iou_loop_params lp = {}; 46 47 while (true) { 48 int nr_wait, step_res; 49 50 if (unlikely(!ctx->loop_step)) 51 return -EFAULT; 52 53 step_res = ctx->loop_step(io_loop_mangle_ctx(ctx), &lp); 54 if (step_res == IOU_LOOP_STOP) 55 break; 56 if (step_res != IOU_LOOP_CONTINUE) 57 return -EINVAL; 58 59 nr_wait = io_loop_nr_cqes(ctx, &lp); 60 if (nr_wait > 0) 61 io_loop_wait(ctx, &lp, nr_wait); 62 else 63 nr_wait = 0; 64 65 if (task_work_pending(current)) { 66 mutex_unlock(&ctx->uring_lock); 67 io_run_task_work(); 68 mutex_lock(&ctx->uring_lock); 69 } 70 if (unlikely(task_sigpending(current))) 71 return -EINTR; 72 io_run_local_work_locked(ctx, nr_wait); 73 74 if (READ_ONCE(ctx->check_cq) & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) 75 io_cqring_overflow_flush_locked(ctx); 76 } 77 78 return 0; 79 } 80 81 int io_run_loop(struct io_ring_ctx *ctx) 82 { 83 int ret; 84 85 ret = io_uring_add_tctx_node(ctx); 86 if (unlikely(ret)) 87 return ret; 88 89 if (!io_allowed_run_tw(ctx)) 90 return -EEXIST; 91 92 mutex_lock(&ctx->uring_lock); 93 ret = __io_run_loop(ctx); 94 mutex_unlock(&ctx->uring_lock); 95 return ret; 96 } 97