21 	struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, wq);  in io_wake_function()  local
27 	if (io_should_wake(iowq) || io_has_work(iowq->ctx))  in io_wake_function()
57 	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);  in io_cqring_timer_wakeup()  local
59 	WRITE_ONCE(iowq->hit_timeout, 1);  in io_cqring_timer_wakeup()
60 	iowq->min_timeout = 0;  in io_cqring_timer_wakeup()
61 	wake_up_process(iowq->wq.private);  in io_cqring_timer_wakeup()
72 	struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);  in io_cqring_min_timer_wakeup()  local
73 	struct io_ring_ctx *ctx = iowq->ctx;  in io_cqring_min_timer_wakeup()
76 	if (iowq->timeout == KTIME_MAX ||  in io_cqring_min_timer_wakeup()
77 	    ktime_compare(iowq->min_timeout, iowq->timeout) >= 0)  in io_cqring_min_timer_wakeup()
86 		if (iowq->cq_min_tail != READ_ONCE(rings->cq.tail))  in io_cqring_min_timer_wakeup()
106 	iowq->cq_tail = iowq->cq_min_tail;  in io_cqring_min_timer_wakeup()
108 	hrtimer_update_function(&iowq->t, io_cqring_timer_wakeup);  in io_cqring_min_timer_wakeup()
109 	hrtimer_set_expires(timer, iowq->timeout);  in io_cqring_min_timer_wakeup()
115 static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,  in io_cqring_schedule_timeout()  argument
120 	if (iowq->min_timeout) {  in io_cqring_schedule_timeout()
121 		timeout = ktime_add_ns(iowq->min_timeout, start_time);  in io_cqring_schedule_timeout()
122 		hrtimer_setup_on_stack(&iowq->t, io_cqring_min_timer_wakeup, clock_id,  in io_cqring_schedule_timeout()
125 		timeout = iowq->timeout;  in io_cqring_schedule_timeout()
126 		hrtimer_setup_on_stack(&iowq->t, io_cqring_timer_wakeup, clock_id,  in io_cqring_schedule_timeout()
130 	hrtimer_set_expires_range_ns(&iowq->t, timeout, 0);  in io_cqring_schedule_timeout()
131 	hrtimer_start_expires(&iowq->t, HRTIMER_MODE_ABS);  in io_cqring_schedule_timeout()
133 	if (!READ_ONCE(iowq->hit_timeout))  in io_cqring_schedule_timeout()
136 	hrtimer_cancel(&iowq->t);  in io_cqring_schedule_timeout()
137 	destroy_hrtimer_on_stack(&iowq->t);  in io_cqring_schedule_timeout()
140 	return READ_ONCE(iowq->hit_timeout) ? -ETIME : 0;  in io_cqring_schedule_timeout()
144 				     struct io_wait_queue *iowq,  in __io_cqring_wait_schedule()  argument
157 	if (iowq->timeout != KTIME_MAX || iowq->min_timeout)  in __io_cqring_wait_schedule()
158 		ret = io_cqring_schedule_timeout(iowq, ctx->clockid, start_time);  in __io_cqring_wait_schedule()
167 					  struct io_wait_queue *iowq,  in io_cqring_wait_schedule()  argument
179 	if (unlikely(io_should_wake(iowq)))  in io_cqring_wait_schedule()
182 	return __io_cqring_wait_schedule(ctx, iowq, ext_arg, start_time);  in io_cqring_wait_schedule()
192 	struct io_wait_queue iowq;  in io_cqring_wait()  local
216 	init_waitqueue_func_entry(&iowq.wq, io_wake_function);  in io_cqring_wait()
217 	iowq.wq.private = current;  in io_cqring_wait()
218 	INIT_LIST_HEAD(&iowq.wq.entry);  in io_cqring_wait()
219 	iowq.ctx = ctx;  in io_cqring_wait()
220 	iowq.cq_tail = READ_ONCE(rings->cq.head) + min_events;  in io_cqring_wait()
221 	iowq.cq_min_tail = READ_ONCE(rings->cq.tail);  in io_cqring_wait()
222 	nr_wait = (int) iowq.cq_tail - READ_ONCE(rings->cq.tail);  in io_cqring_wait()
225 	iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);  in io_cqring_wait()
226 	iowq.hit_timeout = 0;  in io_cqring_wait()
227 	iowq.min_timeout = ext_arg->min_time;  in io_cqring_wait()
228 	iowq.timeout = KTIME_MAX;  in io_cqring_wait()
232 		iowq.timeout = timespec64_to_ktime(ext_arg->ts);  in io_cqring_wait()
234 			iowq.timeout = timens_ktime_to_host(ctx->clockid,  in io_cqring_wait()
235 							    iowq.timeout);  in io_cqring_wait()
237 			iowq.timeout = ktime_add(iowq.timeout, start_time);  in io_cqring_wait()
253 	io_napi_busy_loop(ctx, &iowq);  in io_cqring_wait()
263 			prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,  in io_cqring_wait()
267 		ret = io_cqring_wait_schedule(ctx, &iowq, ext_arg, start_time);  in io_cqring_wait()
303 		if (io_should_wake(&iowq)) {  in io_cqring_wait()
310 		if (!iowq.hit_timeout)  in io_cqring_wait()
312 				nr_wait = (int) iowq.cq_tail -  in io_cqring_wait()
319 		finish_wait(&ctx->cq_wait, &iowq.wq);  in io_cqring_wait()