1*033af2b3SPavel Begunkov /* SPDX-License-Identifier: GPL-2.0 */ 2*033af2b3SPavel Begunkov #include "io_uring.h" 3*033af2b3SPavel Begunkov #include "wait.h" 4*033af2b3SPavel Begunkov #include "loop.h" 5*033af2b3SPavel Begunkov 6*033af2b3SPavel Begunkov static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx, 7*033af2b3SPavel Begunkov const struct iou_loop_params *lp) 8*033af2b3SPavel Begunkov { 9*033af2b3SPavel Begunkov return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail); 10*033af2b3SPavel Begunkov } 11*033af2b3SPavel Begunkov 12*033af2b3SPavel Begunkov static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait) 13*033af2b3SPavel Begunkov { 14*033af2b3SPavel Begunkov atomic_set(&ctx->cq_wait_nr, nr_wait); 15*033af2b3SPavel Begunkov set_current_state(TASK_INTERRUPTIBLE); 16*033af2b3SPavel Begunkov } 17*033af2b3SPavel Begunkov 18*033af2b3SPavel Begunkov static inline void io_loop_wait_finish(struct io_ring_ctx *ctx) 19*033af2b3SPavel Begunkov { 20*033af2b3SPavel Begunkov __set_current_state(TASK_RUNNING); 21*033af2b3SPavel Begunkov atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT); 22*033af2b3SPavel Begunkov } 23*033af2b3SPavel Begunkov 24*033af2b3SPavel Begunkov static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp, 25*033af2b3SPavel Begunkov unsigned nr_wait) 26*033af2b3SPavel Begunkov { 27*033af2b3SPavel Begunkov io_loop_wait_start(ctx, nr_wait); 28*033af2b3SPavel Begunkov 29*033af2b3SPavel Begunkov if (unlikely(io_local_work_pending(ctx) || 30*033af2b3SPavel Begunkov io_loop_nr_cqes(ctx, lp) <= 0) || 31*033af2b3SPavel Begunkov READ_ONCE(ctx->check_cq)) { 32*033af2b3SPavel Begunkov io_loop_wait_finish(ctx); 33*033af2b3SPavel Begunkov return; 34*033af2b3SPavel Begunkov } 35*033af2b3SPavel Begunkov 36*033af2b3SPavel Begunkov mutex_unlock(&ctx->uring_lock); 37*033af2b3SPavel Begunkov schedule(); 38*033af2b3SPavel Begunkov io_loop_wait_finish(ctx); 39*033af2b3SPavel Begunkov mutex_lock(&ctx->uring_lock); 40*033af2b3SPavel Begunkov } 41*033af2b3SPavel Begunkov 42*033af2b3SPavel Begunkov static int __io_run_loop(struct io_ring_ctx *ctx) 43*033af2b3SPavel Begunkov { 44*033af2b3SPavel Begunkov struct iou_loop_params lp = {}; 45*033af2b3SPavel Begunkov 46*033af2b3SPavel Begunkov while (true) { 47*033af2b3SPavel Begunkov int nr_wait, step_res; 48*033af2b3SPavel Begunkov 49*033af2b3SPavel Begunkov if (unlikely(!ctx->loop_step)) 50*033af2b3SPavel Begunkov return -EFAULT; 51*033af2b3SPavel Begunkov 52*033af2b3SPavel Begunkov step_res = ctx->loop_step(ctx, &lp); 53*033af2b3SPavel Begunkov if (step_res == IOU_LOOP_STOP) 54*033af2b3SPavel Begunkov break; 55*033af2b3SPavel Begunkov if (step_res != IOU_LOOP_CONTINUE) 56*033af2b3SPavel Begunkov return -EINVAL; 57*033af2b3SPavel Begunkov 58*033af2b3SPavel Begunkov nr_wait = io_loop_nr_cqes(ctx, &lp); 59*033af2b3SPavel Begunkov if (nr_wait > 0) 60*033af2b3SPavel Begunkov io_loop_wait(ctx, &lp, nr_wait); 61*033af2b3SPavel Begunkov else 62*033af2b3SPavel Begunkov nr_wait = 0; 63*033af2b3SPavel Begunkov 64*033af2b3SPavel Begunkov if (task_work_pending(current)) { 65*033af2b3SPavel Begunkov mutex_unlock(&ctx->uring_lock); 66*033af2b3SPavel Begunkov io_run_task_work(); 67*033af2b3SPavel Begunkov mutex_lock(&ctx->uring_lock); 68*033af2b3SPavel Begunkov } 69*033af2b3SPavel Begunkov if (unlikely(task_sigpending(current))) 70*033af2b3SPavel Begunkov return -EINTR; 71*033af2b3SPavel Begunkov io_run_local_work_locked(ctx, nr_wait); 72*033af2b3SPavel Begunkov 73*033af2b3SPavel Begunkov if (READ_ONCE(ctx->check_cq) & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) 74*033af2b3SPavel Begunkov io_cqring_overflow_flush_locked(ctx); 75*033af2b3SPavel Begunkov } 76*033af2b3SPavel Begunkov 77*033af2b3SPavel Begunkov return 0; 78*033af2b3SPavel Begunkov } 79*033af2b3SPavel Begunkov 80*033af2b3SPavel Begunkov int io_run_loop(struct io_ring_ctx *ctx) 81*033af2b3SPavel Begunkov { 82*033af2b3SPavel Begunkov int ret; 83*033af2b3SPavel Begunkov 84*033af2b3SPavel Begunkov if (!io_allowed_run_tw(ctx)) 85*033af2b3SPavel Begunkov return -EEXIST; 86*033af2b3SPavel Begunkov 87*033af2b3SPavel Begunkov mutex_lock(&ctx->uring_lock); 88*033af2b3SPavel Begunkov ret = __io_run_loop(ctx); 89*033af2b3SPavel Begunkov mutex_unlock(&ctx->uring_lock); 90*033af2b3SPavel Begunkov return ret; 91*033af2b3SPavel Begunkov } 92