1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #include "io_uring.h" 3 #include "wait.h" 4 #include "loop.h" 5 6 static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx, 7 const struct iou_loop_params *lp) 8 { 9 return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail); 10 } 11 12 static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait) 13 { 14 atomic_set(&ctx->cq_wait_nr, nr_wait); 15 set_current_state(TASK_INTERRUPTIBLE); 16 } 17 18 static inline void io_loop_wait_finish(struct io_ring_ctx *ctx) 19 { 20 __set_current_state(TASK_RUNNING); 21 atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT); 22 } 23 24 static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp, 25 unsigned nr_wait) 26 { 27 io_loop_wait_start(ctx, nr_wait); 28 29 if (unlikely(io_local_work_pending(ctx) || 30 io_loop_nr_cqes(ctx, lp) <= 0) || 31 READ_ONCE(ctx->check_cq)) { 32 io_loop_wait_finish(ctx); 33 return; 34 } 35 36 mutex_unlock(&ctx->uring_lock); 37 schedule(); 38 io_loop_wait_finish(ctx); 39 mutex_lock(&ctx->uring_lock); 40 } 41 42 static int __io_run_loop(struct io_ring_ctx *ctx) 43 { 44 struct iou_loop_params lp = {}; 45 46 while (true) { 47 int nr_wait, step_res; 48 49 if (unlikely(!ctx->loop_step)) 50 return -EFAULT; 51 52 step_res = ctx->loop_step(ctx, &lp); 53 if (step_res == IOU_LOOP_STOP) 54 break; 55 if (step_res != IOU_LOOP_CONTINUE) 56 return -EINVAL; 57 58 nr_wait = io_loop_nr_cqes(ctx, &lp); 59 if (nr_wait > 0) 60 io_loop_wait(ctx, &lp, nr_wait); 61 else 62 nr_wait = 0; 63 64 if (task_work_pending(current)) { 65 mutex_unlock(&ctx->uring_lock); 66 io_run_task_work(); 67 mutex_lock(&ctx->uring_lock); 68 } 69 if (unlikely(task_sigpending(current))) 70 return -EINTR; 71 io_run_local_work_locked(ctx, nr_wait); 72 73 if (READ_ONCE(ctx->check_cq) & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) 74 io_cqring_overflow_flush_locked(ctx); 75 } 76 77 return 0; 78 } 79 80 int io_run_loop(struct io_ring_ctx *ctx) 81 { 82 int ret; 83 84 if (!io_allowed_run_tw(ctx)) 85 return -EEXIST; 86 87 mutex_lock(&ctx->uring_lock); 88 ret = __io_run_loop(ctx); 89 mutex_unlock(&ctx->uring_lock); 90 return ret; 91 } 92