xref: /linux/io_uring/io_uring.c (revision cf6542493e35ab9e72ac07da10a178ac78db5a16)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Shared application/kernel submission and completion ring pairs, for
4  * supporting fast/efficient IO.
5  *
6  * A note on the read/write ordering memory barriers that are matched between
7  * the application and kernel side.
8  *
9  * After the application reads the CQ ring tail, it must use an
10  * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11  * before writing the tail (using smp_load_acquire to read the tail will
12  * do). It also needs a smp_mb() before updating CQ head (ordering the
13  * entry load(s) with the head store), pairing with an implicit barrier
14  * through a control-dependency in io_get_cqe (smp_store_release to
15  * store head will do). Failure to do so could lead to reading invalid
16  * CQ entries.
17  *
18  * Likewise, the application must use an appropriate smp_wmb() before
19  * writing the SQ tail (ordering SQ entry stores with the tail store),
20  * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21  * to store the tail will do). And it needs a barrier ordering the SQ
22  * head load before writing new SQ entries (smp_load_acquire to read
23  * head will do).
24  *
25  * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26  * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27  * updating the SQ tail; a full memory barrier smp_mb() is needed
28  * between.
29  *
30  * Also see the examples in the liburing library:
31  *
32  *	git://git.kernel.org/pub/scm/linux/kernel/git/axboe/liburing.git
33  *
34  * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35  * from data shared between the kernel and application. This is done both
36  * for ordering purposes, but also to ensure that once a value is loaded from
37  * data that the application could potentially modify, it remains stable.
38  *
39  * Copyright (C) 2018-2019 Jens Axboe
40  * Copyright (c) 2018-2019 Christoph Hellwig
41  */
42 #include <linux/kernel.h>
43 #include <linux/errno.h>
44 #include <linux/syscalls.h>
45 #include <linux/refcount.h>
46 #include <linux/bits.h>
47 
48 #include <linux/sched/signal.h>
49 #include <linux/fs.h>
50 #include <linux/mm.h>
51 #include <linux/percpu.h>
52 #include <linux/slab.h>
53 #include <linux/anon_inodes.h>
54 #include <linux/uaccess.h>
55 #include <linux/nospec.h>
56 #include <linux/task_work.h>
57 #include <linux/io_uring.h>
58 #include <linux/io_uring/cmd.h>
59 #include <linux/audit.h>
60 #include <linux/security.h>
61 #include <linux/jump_label.h>
62 #include <linux/kcov.h>
63 
64 #define CREATE_TRACE_POINTS
65 #include <trace/events/io_uring.h>
66 
67 #include <uapi/linux/io_uring.h>
68 
69 #include "io-wq.h"
70 
71 #include "filetable.h"
72 #include "io_uring.h"
73 #include "opdef.h"
74 #include "refs.h"
75 #include "tctx.h"
76 #include "register.h"
77 #include "sqpoll.h"
78 #include "fdinfo.h"
79 #include "kbuf.h"
80 #include "rsrc.h"
81 #include "cancel.h"
82 #include "net.h"
83 #include "notif.h"
84 #include "waitid.h"
85 #include "futex.h"
86 #include "napi.h"
87 #include "uring_cmd.h"
88 #include "msg_ring.h"
89 #include "memmap.h"
90 #include "zcrx.h"
91 #include "bpf-ops.h"
92 
93 #include "timeout.h"
94 #include "poll.h"
95 #include "rw.h"
96 #include "alloc_cache.h"
97 #include "eventfd.h"
98 #include "wait.h"
99 #include "bpf_filter.h"
100 #include "loop.h"
101 
102 #define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \
103 			  IOSQE_IO_HARDLINK | IOSQE_ASYNC)
104 
105 #define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK)
106 
107 #define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
108 				REQ_F_INFLIGHT | REQ_F_CREDS | REQ_F_ASYNC_DATA)
109 
110 #define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT | IO_REQ_LINK_FLAGS | \
111 				 REQ_F_REISSUE | REQ_F_POLLED | \
112 				 IO_REQ_CLEAN_FLAGS)
113 
114 #define IO_TCTX_REFS_CACHE_NR	(1U << 10)
115 
116 #define IO_COMPL_BATCH			32
117 #define IO_REQ_ALLOC_BATCH		8
118 
119 /* requests with any of those set should undergo io_disarm_next() */
120 #define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
121 
122 static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags);
123 static void __io_req_caches_free(struct io_ring_ctx *ctx);
124 
125 static __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(io_key_has_sqarray, HZ);
126 
127 struct kmem_cache *req_cachep;
128 static struct workqueue_struct *iou_wq __ro_after_init;
129 
130 static int __read_mostly sysctl_io_uring_disabled;
131 static int __read_mostly sysctl_io_uring_group = -1;
132 
133 #ifdef CONFIG_SYSCTL
134 static const struct ctl_table kernel_io_uring_disabled_table[] = {
135 	{
136 		.procname	= "io_uring_disabled",
137 		.data		= &sysctl_io_uring_disabled,
138 		.maxlen		= sizeof(sysctl_io_uring_disabled),
139 		.mode		= 0644,
140 		.proc_handler	= proc_dointvec_minmax,
141 		.extra1		= SYSCTL_ZERO,
142 		.extra2		= SYSCTL_TWO,
143 	},
144 	{
145 		.procname	= "io_uring_group",
146 		.data		= &sysctl_io_uring_group,
147 		.maxlen		= sizeof(gid_t),
148 		.mode		= 0644,
149 		.proc_handler	= proc_dointvec,
150 	},
151 };
152 #endif
153 
154 static void io_poison_cached_req(struct io_kiocb *req)
155 {
156 	req->ctx = IO_URING_PTR_POISON;
157 	req->tctx = IO_URING_PTR_POISON;
158 	req->file = IO_URING_PTR_POISON;
159 	req->creds = IO_URING_PTR_POISON;
160 	req->io_task_work.func = IO_URING_PTR_POISON;
161 	req->apoll = IO_URING_PTR_POISON;
162 }
163 
164 void io_poison_req(struct io_kiocb *req)
165 {
166 	io_poison_cached_req(req);
167 	req->async_data = IO_URING_PTR_POISON;
168 	req->kbuf = IO_URING_PTR_POISON;
169 	req->comp_list.next = IO_URING_PTR_POISON;
170 	req->file_node = IO_URING_PTR_POISON;
171 	req->link = IO_URING_PTR_POISON;
172 }
173 
174 static inline void req_fail_link_node(struct io_kiocb *req, int res)
175 {
176 	req_set_fail(req);
177 	io_req_set_res(req, res, 0);
178 }
179 
180 static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx)
181 {
182 	if (IS_ENABLED(CONFIG_KASAN))
183 		io_poison_cached_req(req);
184 	wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list);
185 }
186 
187 static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref)
188 {
189 	struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
190 
191 	complete(&ctx->ref_comp);
192 }
193 
194 static int io_alloc_hash_table(struct io_hash_table *table, unsigned bits)
195 {
196 	unsigned int hash_buckets;
197 	int i;
198 
199 	do {
200 		hash_buckets = 1U << bits;
201 		table->hbs = kvmalloc_objs(table->hbs[0], hash_buckets,
202 					   GFP_KERNEL_ACCOUNT);
203 		if (table->hbs)
204 			break;
205 		if (bits == 1)
206 			return -ENOMEM;
207 		bits--;
208 	} while (1);
209 
210 	table->hash_bits = bits;
211 	for (i = 0; i < hash_buckets; i++)
212 		INIT_HLIST_HEAD(&table->hbs[i].list);
213 	return 0;
214 }
215 
216 static void io_free_alloc_caches(struct io_ring_ctx *ctx)
217 {
218 	io_alloc_cache_free(&ctx->apoll_cache, kfree);
219 	io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
220 	io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free);
221 	io_alloc_cache_free(&ctx->cmd_cache, io_cmd_cache_free);
222 	io_futex_cache_free(ctx);
223 	io_rsrc_cache_free(ctx);
224 }
225 
226 static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
227 {
228 	struct io_ring_ctx *ctx;
229 	int hash_bits;
230 	bool ret;
231 
232 	ctx = kzalloc_obj(*ctx);
233 	if (!ctx)
234 		return NULL;
235 
236 	xa_init(&ctx->io_bl_xa);
237 	xa_init(&ctx->hpage_acct);
238 
239 	/*
240 	 * Use 5 bits less than the max cq entries, that should give us around
241 	 * 32 entries per hash list if totally full and uniformly spread, but
242 	 * don't keep too many buckets to not overconsume memory.
243 	 */
244 	hash_bits = ilog2(p->cq_entries) - 5;
245 	hash_bits = clamp(hash_bits, 1, 8);
246 	if (io_alloc_hash_table(&ctx->cancel_table, hash_bits))
247 		goto err;
248 	if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
249 			    0, GFP_KERNEL))
250 		goto err;
251 
252 	ctx->flags = p->flags;
253 	ctx->hybrid_poll_time = LLONG_MAX;
254 	atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
255 	init_waitqueue_head(&ctx->sqo_sq_wait);
256 	INIT_LIST_HEAD(&ctx->sqd_list);
257 	INIT_LIST_HEAD(&ctx->cq_overflow_list);
258 	ret = io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX,
259 			    sizeof(struct async_poll), 0);
260 	ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
261 			    sizeof(struct io_async_msghdr),
262 			    offsetof(struct io_async_msghdr, clear));
263 	ret |= io_alloc_cache_init(&ctx->rw_cache, IO_ALLOC_CACHE_MAX,
264 			    sizeof(struct io_async_rw),
265 			    offsetof(struct io_async_rw, clear));
266 	ret |= io_alloc_cache_init(&ctx->cmd_cache, IO_ALLOC_CACHE_MAX,
267 			    sizeof(struct io_async_cmd),
268 			    sizeof(struct io_async_cmd));
269 	ret |= io_futex_cache_init(ctx);
270 	ret |= io_rsrc_cache_init(ctx);
271 	if (ret)
272 		goto free_ref;
273 	init_completion(&ctx->ref_comp);
274 	xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1 | XA_FLAGS_ACCOUNT);
275 	mutex_init(&ctx->uring_lock);
276 	init_waitqueue_head(&ctx->cq_wait);
277 	init_waitqueue_head(&ctx->poll_wq);
278 	spin_lock_init(&ctx->completion_lock);
279 	raw_spin_lock_init(&ctx->timeout_lock);
280 	INIT_LIST_HEAD(&ctx->iopoll_list);
281 	INIT_LIST_HEAD(&ctx->defer_list);
282 	INIT_LIST_HEAD(&ctx->timeout_list);
283 	INIT_LIST_HEAD(&ctx->ltimeout_list);
284 	mpscq_init(&ctx->work_list, &ctx->work_head);
285 	INIT_LIST_HEAD(&ctx->tctx_list);
286 	mutex_init(&ctx->tctx_lock);
287 	ctx->submit_state.free_list.next = NULL;
288 	INIT_HLIST_HEAD(&ctx->waitid_list);
289 	xa_init_flags(&ctx->zcrx_ctxs, XA_FLAGS_ALLOC);
290 #ifdef CONFIG_FUTEX
291 	INIT_HLIST_HEAD(&ctx->futex_list);
292 #endif
293 	INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
294 	INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd);
295 	io_napi_init(ctx);
296 	mutex_init(&ctx->mmap_lock);
297 	ctx->kcov_handle = kcov_common_handle();
298 
299 	return ctx;
300 
301 free_ref:
302 	percpu_ref_exit(&ctx->refs);
303 err:
304 	io_free_alloc_caches(ctx);
305 	kvfree(ctx->cancel_table.hbs);
306 	xa_destroy(&ctx->io_bl_xa);
307 	xa_destroy(&ctx->hpage_acct);
308 	kfree(ctx);
309 	return NULL;
310 }
311 
312 static void io_clean_op(struct io_kiocb *req)
313 {
314 	if (unlikely(req->flags & REQ_F_BUFFER_SELECTED))
315 		io_kbuf_drop_legacy(req);
316 
317 	if (req->flags & REQ_F_NEED_CLEANUP) {
318 		const struct io_cold_def *def = &io_cold_defs[req->opcode];
319 
320 		if (def->cleanup)
321 			def->cleanup(req);
322 	}
323 	if (req->flags & REQ_F_INFLIGHT)
324 		atomic_dec(&req->tctx->inflight_tracked);
325 	if (req->flags & REQ_F_CREDS)
326 		put_cred(req->creds);
327 	if (req->flags & REQ_F_ASYNC_DATA) {
328 		kfree(req->async_data);
329 		req->async_data = NULL;
330 	}
331 	req->flags &= ~IO_REQ_CLEAN_FLAGS;
332 }
333 
334 /*
335  * Mark the request as inflight, so that file cancelation will find it.
336  * Can be used if the file is an io_uring instance, or if the request itself
337  * relies on ->mm being alive for the duration of the request.
338  */
339 inline void io_req_track_inflight(struct io_kiocb *req)
340 {
341 	if (!(req->flags & REQ_F_INFLIGHT)) {
342 		req->flags |= REQ_F_INFLIGHT;
343 		atomic_inc(&req->tctx->inflight_tracked);
344 	}
345 }
346 
347 static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)
348 {
349 	if (WARN_ON_ONCE(!req->link))
350 		return NULL;
351 
352 	req->flags &= ~REQ_F_ARM_LTIMEOUT;
353 	req->flags |= REQ_F_LINK_TIMEOUT;
354 
355 	/* linked timeouts should have two refs once prep'ed */
356 	io_req_set_refcount(req);
357 	__io_req_set_refcount(req->link, 2);
358 	return req->link;
359 }
360 
361 static void io_prep_async_work(struct io_kiocb *req)
362 {
363 	const struct io_issue_def *def = &io_issue_defs[req->opcode];
364 
365 	if (!(req->flags & REQ_F_CREDS)) {
366 		req->flags |= REQ_F_CREDS;
367 		req->creds = get_current_cred();
368 	}
369 
370 	req->work.list.next = NULL;
371 	atomic_set(&req->work.flags, 0);
372 	if (req->flags & REQ_F_FORCE_ASYNC)
373 		atomic_or(IO_WQ_WORK_CONCURRENT, &req->work.flags);
374 
375 	if (req->file && !(req->flags & REQ_F_FIXED_FILE))
376 		req->flags |= io_file_get_flags(req->file);
377 
378 	if (req->file && (req->flags & REQ_F_ISREG)) {
379 		bool should_hash = def->hash_reg_file;
380 
381 		/* don't serialize this request if the fs doesn't need it */
382 		if (should_hash && (req->file->f_flags & O_DIRECT) &&
383 		    (req->file->f_op->fop_flags & FOP_DIO_PARALLEL_WRITE))
384 			should_hash = false;
385 		if (should_hash || (req->flags & REQ_F_IOPOLL))
386 			io_wq_hash_work(&req->work, file_inode(req->file));
387 	} else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
388 		if (def->unbound_nonreg_file)
389 			atomic_or(IO_WQ_WORK_UNBOUND, &req->work.flags);
390 	}
391 }
392 
393 static void io_prep_async_link(struct io_kiocb *req)
394 {
395 	struct io_kiocb *cur;
396 
397 	if (req->flags & REQ_F_LINK_TIMEOUT) {
398 		struct io_ring_ctx *ctx = req->ctx;
399 
400 		raw_spin_lock_irq(&ctx->timeout_lock);
401 		io_for_each_link(cur, req)
402 			io_prep_async_work(cur);
403 		raw_spin_unlock_irq(&ctx->timeout_lock);
404 	} else {
405 		io_for_each_link(cur, req)
406 			io_prep_async_work(cur);
407 	}
408 }
409 
410 /* Called for every request that is issued: keep it inline */
411 static __always_inline bool io_assign_file(struct io_kiocb *req,
412 					   const struct io_issue_def *def,
413 					   unsigned int issue_flags)
414 {
415 	if (req->file || !def->needs_file)
416 		return true;
417 
418 	if (req->flags & REQ_F_FIXED_FILE)
419 		req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags);
420 	else
421 		req->file = io_file_get_normal(req, req->cqe.fd);
422 
423 	return !!req->file;
424 }
425 
426 static void io_req_queue_iowq_tw(struct io_tw_req tw_req, io_tw_token_t tw)
427 {
428 	struct io_kiocb *req = tw_req.req;
429 
430 	io_tw_lock(req->ctx, tw);
431 	if (unlikely(tw.cancel))
432 		io_req_defer_failed(req, -ECANCELED);
433 	else
434 		io_queue_iowq(req);
435 }
436 
437 void io_queue_iowq(struct io_kiocb *req)
438 {
439 	struct io_uring_task *tctx = req->tctx;
440 
441 	BUG_ON(!tctx);
442 	lockdep_assert_held(&req->ctx->uring_lock);
443 
444 	if (current->flags & PF_KTHREAD)
445 		goto cancel;
446 
447 	/* Only the submitter can safely access its io-wq during task exit. */
448 	if (unlikely(current != tctx->task)) {
449 		req->io_task_work.func = io_req_queue_iowq_tw;
450 		io_req_task_work_add(req);
451 		return;
452 	}
453 	if (!tctx->io_wq)
454 		goto cancel;
455 
456 	/*
457 	 * The file is looked up as late as possible, as it may be the result
458 	 * of an earlier request of the link. This one is up now, and what kind
459 	 * of work it is depends on the file. If that fails, leave the error
460 	 * handling to the worker.
461 	 *
462 	 * Not for a splice: whatever it writes to, it may wait for its input
463 	 * for ever. It has to remain unbound work that isn't hashed.
464 	 */
465 	if (req->opcode != IORING_OP_SPLICE)
466 		io_assign_file(req, &io_issue_defs[req->opcode], 0);
467 
468 	/* init ->work of the whole link before punting */
469 	io_prep_async_link(req);
470 
471 	/*
472 	 * Not expected to happen, but if we do have a bug where this _can_
473 	 * happen, catch it here and ensure the request is marked as
474 	 * canceled. That will make io-wq go through the usual work cancel
475 	 * procedure rather than attempt to run this request (or create a new
476 	 * worker for it).
477 	 */
478 	if (WARN_ON_ONCE(!same_thread_group(tctx->task, current)))
479 		atomic_or(IO_WQ_WORK_CANCEL, &req->work.flags);
480 
481 	trace_io_uring_queue_async_work(req, io_wq_is_hashed(&req->work));
482 	io_wq_enqueue(tctx->io_wq, &req->work);
483 	return;
484 cancel:
485 	io_req_task_queue_fail(req, -ECANCELED);
486 }
487 
488 unsigned io_linked_nr(struct io_kiocb *req)
489 {
490 	struct io_kiocb *tmp;
491 	unsigned nr = 0;
492 
493 	io_for_each_link(tmp, req)
494 		nr++;
495 	return nr;
496 }
497 
498 static __cold noinline bool io_queue_deferred(struct io_ring_ctx *ctx)
499 {
500 	bool drain_seen = false, first = true, drain_queued = false;
501 
502 	lockdep_assert_held(&ctx->uring_lock);
503 	__io_req_caches_free(ctx);
504 	/* nothing outside of ->defer_list is alive */
505 	if (ctx->nr_req_allocated == ctx->nr_drained)
506 		ctx->drain_req = NULL;
507 
508 	while (!list_empty(&ctx->defer_list)) {
509 		struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
510 						struct io_defer_entry, list);
511 
512 		drain_seen |= de->req->flags & REQ_F_IO_DRAIN;
513 		if ((drain_seen || first) && ctx->nr_req_allocated != ctx->nr_drained)
514 			return drain_queued;
515 
516 		list_del_init(&de->list);
517 		ctx->nr_drained -= io_linked_nr(de->req);
518 		if (de->req->flags & REQ_F_IO_DRAIN) {
519 			drain_queued = true;
520 			/* it runs alone, track its completion unless it's a link */
521 			if (!de->req->link)
522 				ctx->drain_req = de->req;
523 		}
524 		io_req_task_queue(de->req);
525 		kfree(de);
526 		first = false;
527 	}
528 	return drain_queued;
529 }
530 
531 void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
532 {
533 	if (ctx->int_flags & IO_RING_F_POLL_ACTIVATED)
534 		io_poll_wq_wake(ctx);
535 	if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED)
536 		io_flush_timeouts(ctx);
537 	if (ctx->int_flags & IO_RING_F_HAS_EVFD)
538 		io_eventfd_signal(ctx, true, false);
539 }
540 
541 static inline void __io_cq_lock(struct io_ring_ctx *ctx)
542 {
543 	if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ))
544 		spin_lock(&ctx->completion_lock);
545 }
546 
547 static inline void io_cq_lock(struct io_ring_ctx *ctx)
548 	__acquires(ctx->completion_lock)
549 {
550 	spin_lock(&ctx->completion_lock);
551 }
552 
553 static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx)
554 {
555 	io_commit_cqring(ctx);
556 	if (!(ctx->int_flags & IO_RING_F_TASK_COMPLETE)) {
557 		if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ))
558 			spin_unlock(&ctx->completion_lock);
559 		/* IOPOLL rings only need to wake up if it's also SQPOLL */
560 		if (!(ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL))
561 			io_cqring_wake(ctx);
562 	}
563 	io_commit_cqring_flush(ctx);
564 }
565 
566 static void io_cq_unlock_post(struct io_ring_ctx *ctx)
567 	__releases(ctx->completion_lock)
568 {
569 	io_commit_cqring(ctx);
570 	spin_unlock(&ctx->completion_lock);
571 	io_cqring_wake(ctx);
572 	io_commit_cqring_flush(ctx);
573 }
574 
575 static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool dying)
576 {
577 	lockdep_assert_held(&ctx->uring_lock);
578 
579 	/* don't abort if we're dying, entries must get freed */
580 	if (!dying && __io_cqring_events(ctx) == ctx->cq_entries)
581 		return;
582 
583 	io_cq_lock(ctx);
584 	while (!list_empty(&ctx->cq_overflow_list)) {
585 		size_t cqe_size = sizeof(struct io_uring_cqe);
586 		struct io_uring_cqe *cqe;
587 		struct io_overflow_cqe *ocqe;
588 		bool is_cqe32 = false;
589 
590 		ocqe = list_first_entry(&ctx->cq_overflow_list,
591 					struct io_overflow_cqe, list);
592 		if (ocqe->cqe.flags & IORING_CQE_F_32 ||
593 		    ctx->flags & IORING_SETUP_CQE32) {
594 			is_cqe32 = true;
595 			cqe_size <<= 1;
596 		}
597 		if (ctx->flags & IORING_SETUP_CQE32)
598 			is_cqe32 = false;
599 
600 		if (!dying) {
601 			if (!io_get_cqe_overflow(ctx, &cqe, true, is_cqe32))
602 				break;
603 			memcpy(cqe, &ocqe->cqe, cqe_size);
604 		}
605 		list_del(&ocqe->list);
606 		kfree(ocqe);
607 
608 		/*
609 		 * For silly syzbot cases that deliberately overflow by huge
610 		 * amounts, check if we need to resched and drop and
611 		 * reacquire the locks if so. Nothing real would ever hit this.
612 		 * Ideally we'd have a non-posting unlock for this, but hard
613 		 * to care for a non-real case.
614 		 */
615 		if (need_resched()) {
616 			ctx->cqe_sentinel = ctx->cqe_cached;
617 			io_cq_unlock_post(ctx);
618 			mutex_unlock(&ctx->uring_lock);
619 			cond_resched();
620 			mutex_lock(&ctx->uring_lock);
621 			io_cq_lock(ctx);
622 		}
623 	}
624 
625 	if (list_empty(&ctx->cq_overflow_list)) {
626 		clear_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
627 		atomic_andnot(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
628 	}
629 	io_cq_unlock_post(ctx);
630 }
631 
632 static void io_cqring_overflow_kill(struct io_ring_ctx *ctx)
633 {
634 	if (ctx->rings)
635 		__io_cqring_overflow_flush(ctx, true);
636 }
637 
638 void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx)
639 {
640 	mutex_lock(&ctx->uring_lock);
641 	__io_cqring_overflow_flush(ctx, false);
642 	mutex_unlock(&ctx->uring_lock);
643 }
644 
645 void io_cqring_overflow_flush_locked(struct io_ring_ctx *ctx)
646 {
647 	__io_cqring_overflow_flush(ctx, false);
648 }
649 
650 /* must to be called somewhat shortly after putting a request */
651 static inline void io_put_task(struct io_kiocb *req)
652 {
653 	struct io_uring_task *tctx = req->tctx;
654 
655 	if (likely(tctx->task == current)) {
656 		tctx->cached_refs++;
657 	} else {
658 		guard(rcu)();
659 
660 		percpu_counter_sub(&tctx->inflight, 1);
661 		if (unlikely(atomic_read(&tctx->in_cancel)))
662 			wake_up(&tctx->wait);
663 		put_task_struct(tctx->task);
664 	}
665 }
666 
667 void io_task_refs_refill(struct io_uring_task *tctx)
668 {
669 	unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;
670 
671 	percpu_counter_add(&tctx->inflight, refill);
672 	refcount_add(refill, &current->usage);
673 	tctx->cached_refs += refill;
674 }
675 
676 __cold void io_uring_drop_tctx_refs(struct task_struct *task)
677 {
678 	struct io_uring_task *tctx = task->io_uring;
679 	unsigned int refs = tctx->cached_refs;
680 
681 	if (refs) {
682 		tctx->cached_refs = 0;
683 		percpu_counter_sub(&tctx->inflight, refs);
684 		put_task_struct_many(task, refs);
685 	}
686 }
687 
688 static __cold bool io_cqring_add_overflow(struct io_ring_ctx *ctx,
689 					  struct io_overflow_cqe *ocqe)
690 {
691 	lockdep_assert_held(&ctx->completion_lock);
692 
693 	if (!ocqe) {
694 		struct io_rings *r = ctx->rings;
695 
696 		/*
697 		 * If we're in ring overflow flush mode, or in task cancel mode,
698 		 * or cannot allocate an overflow entry, then we need to drop it
699 		 * on the floor.
700 		 */
701 		WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1);
702 		set_bit(IO_CHECK_CQ_DROPPED_BIT, &ctx->check_cq);
703 		return false;
704 	}
705 	if (list_empty(&ctx->cq_overflow_list)) {
706 		set_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
707 		atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
708 
709 	}
710 	list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
711 	return true;
712 }
713 
714 static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
715 					     struct io_cqe *cqe,
716 					     struct io_big_cqe *big_cqe, gfp_t gfp)
717 {
718 	struct io_overflow_cqe *ocqe;
719 	size_t ocq_size = sizeof(struct io_overflow_cqe);
720 	bool is_cqe32 = false;
721 
722 	if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32) {
723 		is_cqe32 = true;
724 		ocq_size += sizeof(struct io_uring_cqe);
725 	}
726 
727 	ocqe = kzalloc(ocq_size, gfp | __GFP_ACCOUNT);
728 	trace_io_uring_cqe_overflow(ctx, cqe->user_data, cqe->res, cqe->flags, ocqe);
729 	if (ocqe) {
730 		ocqe->cqe.user_data = cqe->user_data;
731 		ocqe->cqe.res = cqe->res;
732 		ocqe->cqe.flags = cqe->flags;
733 		if (is_cqe32 && big_cqe) {
734 			ocqe->cqe.big_cqe[0] = big_cqe->extra1;
735 			ocqe->cqe.big_cqe[1] = big_cqe->extra2;
736 		}
737 	}
738 	if (big_cqe)
739 		big_cqe->extra1 = big_cqe->extra2 = 0;
740 	return ocqe;
741 }
742 
743 /*
744  * Compute queued CQEs for free-space calculation, clamped to cq_entries.
745  */
746 static unsigned int io_cqring_queued(struct io_ring_ctx *ctx)
747 {
748 	struct io_rings *rings = io_get_rings(ctx);
749 	int diff;
750 
751 	diff = (int)(ctx->cached_cq_tail - READ_ONCE(rings->cq.head));
752 	if (diff >= 0)
753 		return min((unsigned int)diff, ctx->cq_entries);
754 	return 0;
755 }
756 
757 /*
758  * Fill an empty dummy CQE, in case alignment is off for posting a 32b CQE
759  * because the ring is a single 16b entry away from wrapping.
760  */
761 static bool io_fill_nop_cqe(struct io_ring_ctx *ctx, unsigned int off)
762 {
763 	if (io_cqring_queued(ctx) < ctx->cq_entries) {
764 		struct io_uring_cqe *cqe = &ctx->rings->cqes[off];
765 
766 		cqe->user_data = 0;
767 		cqe->res = 0;
768 		cqe->flags = IORING_CQE_F_SKIP;
769 		ctx->cached_cq_tail++;
770 		return true;
771 	}
772 	return false;
773 }
774 
775 /*
776  * writes to the cq entry need to come after reading head; the
777  * control dependency is enough as we're using WRITE_ONCE to
778  * fill the cq entry
779  */
780 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32)
781 {
782 	struct io_rings *rings = ctx->rings;
783 	unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
784 	unsigned int free, len;
785 
786 	/*
787 	 * Posting into the CQ when there are pending overflowed CQEs may break
788 	 * ordering guarantees, which will affect links, F_MORE users and more.
789 	 * Force overflow the completion.
790 	 */
791 	if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)))
792 		return false;
793 
794 	/*
795 	 * A single entry may be left if a 32b CQE is asked for. It must not be
796 	 * handed out later on if this fails, as the CQE is going to overflow.
797 	 */
798 	ctx->cqe_cached = NULL;
799 	ctx->cqe_sentinel = NULL;
800 
801 	/*
802 	 * Post dummy CQE if a 32b CQE is needed and there's only room for a
803 	 * 16b CQE before the ring wraps.
804 	 */
805 	if (cqe32 && (ctx->flags & IORING_SETUP_CQE_MIXED) &&
806 	    off + 1 == ctx->cq_entries) {
807 		/* one for the dummy CQE, two for the 32b CQE itself */
808 		if (ctx->cq_entries - io_cqring_queued(ctx) < 3)
809 			return false;
810 		if (!io_fill_nop_cqe(ctx, off))
811 			return false;
812 		off = 0;
813 	}
814 
815 	free = ctx->cq_entries - io_cqring_queued(ctx);
816 	/* we need a contiguous range, limit based on the current array offset */
817 	len = min(free, ctx->cq_entries - off);
818 	if (ctx->flags & IORING_SETUP_CQE32) {
819 		if (!len)
820 			return false;
821 		off <<= 1;
822 		len <<= 1;
823 	} else if (len < (cqe32 + 1)) {
824 		return false;
825 	}
826 
827 	ctx->cqe_cached = &rings->cqes[off];
828 	ctx->cqe_sentinel = ctx->cqe_cached + len;
829 	return true;
830 }
831 
832 static bool io_fill_cqe_aux32(struct io_ring_ctx *ctx,
833 			      struct io_uring_cqe src_cqe[2])
834 {
835 	struct io_uring_cqe *cqe;
836 
837 	if (WARN_ON_ONCE(!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))))
838 		return false;
839 	if (unlikely(!io_get_cqe(ctx, &cqe, true)))
840 		return false;
841 
842 	memcpy(cqe, src_cqe, 2 * sizeof(*cqe));
843 	trace_io_uring_complete(ctx, NULL, cqe);
844 	return true;
845 }
846 
847 static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
848 			      u32 cflags)
849 {
850 	bool cqe32 = cflags & IORING_CQE_F_32;
851 	struct io_uring_cqe *cqe;
852 
853 	if (likely(io_get_cqe(ctx, &cqe, cqe32))) {
854 		WRITE_ONCE(cqe->user_data, user_data);
855 		WRITE_ONCE(cqe->res, res);
856 		WRITE_ONCE(cqe->flags, cflags);
857 
858 		if (cqe32 || (ctx->flags & IORING_SETUP_CQE32)) {
859 			WRITE_ONCE(cqe->big_cqe[0], 0);
860 			WRITE_ONCE(cqe->big_cqe[1], 0);
861 		}
862 
863 		trace_io_uring_complete(ctx, NULL, cqe);
864 		return true;
865 	}
866 	return false;
867 }
868 
869 static inline struct io_cqe io_init_cqe(u64 user_data, s32 res, u32 cflags)
870 {
871 	return (struct io_cqe) { .user_data = user_data, .res = res, .flags = cflags };
872 }
873 
874 static __cold void io_cqe_overflow(struct io_ring_ctx *ctx, struct io_cqe *cqe,
875 				   struct io_big_cqe *big_cqe)
876 {
877 	struct io_overflow_cqe *ocqe;
878 
879 	ocqe = io_alloc_ocqe(ctx, cqe, big_cqe, GFP_KERNEL);
880 	spin_lock(&ctx->completion_lock);
881 	io_cqring_add_overflow(ctx, ocqe);
882 	spin_unlock(&ctx->completion_lock);
883 }
884 
885 static __cold bool io_cqe_overflow_locked(struct io_ring_ctx *ctx,
886 					  struct io_cqe *cqe,
887 					  struct io_big_cqe *big_cqe)
888 {
889 	struct io_overflow_cqe *ocqe;
890 
891 	ocqe = io_alloc_ocqe(ctx, cqe, big_cqe, GFP_NOWAIT);
892 	return io_cqring_add_overflow(ctx, ocqe);
893 }
894 
895 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
896 {
897 	bool filled;
898 
899 	io_cq_lock(ctx);
900 	filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
901 	if (unlikely(!filled)) {
902 		struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
903 
904 		filled = io_cqe_overflow_locked(ctx, &cqe, NULL);
905 	}
906 	io_cq_unlock_post(ctx);
907 	return filled;
908 }
909 
910 /*
911  * Must be called from inline task_work so we know a flush will happen later,
912  * and obviously with ctx->uring_lock held (tw always has that).
913  */
914 void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
915 {
916 	lockdep_assert_held(&ctx->uring_lock);
917 	lockdep_assert(ctx->int_flags & IO_RING_F_LOCKLESS_CQ);
918 
919 	if (!io_fill_cqe_aux(ctx, user_data, res, cflags)) {
920 		struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
921 
922 		io_cqe_overflow(ctx, &cqe, NULL);
923 	}
924 	ctx->submit_state.cq_flush = true;
925 }
926 
927 /*
928  * A helper for multishot requests posting additional CQEs.
929  * Should only be used from a task_work including IO_URING_F_MULTISHOT.
930  */
931 bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags)
932 {
933 	struct io_ring_ctx *ctx = req->ctx;
934 	bool posted;
935 
936 	/*
937 	 * If multishot has already posted deferred completions, ensure that
938 	 * those are flushed first before posting this one. If not, CQEs
939 	 * could get reordered.
940 	 */
941 	if (!wq_list_empty(&ctx->submit_state.compl_reqs))
942 		__io_submit_flush_completions(ctx);
943 
944 	lockdep_assert(!io_wq_current_is_worker());
945 	lockdep_assert_held(&ctx->uring_lock);
946 
947 	if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ)) {
948 		spin_lock(&ctx->completion_lock);
949 		posted = io_fill_cqe_aux(ctx, req->cqe.user_data, res, cflags);
950 		spin_unlock(&ctx->completion_lock);
951 	} else {
952 		posted = io_fill_cqe_aux(ctx, req->cqe.user_data, res, cflags);
953 	}
954 
955 	ctx->submit_state.cq_flush = true;
956 	return posted;
957 }
958 
959 /*
960  * A helper for multishot requests posting additional CQEs.
961  * Should only be used from a task_work including IO_URING_F_MULTISHOT.
962  */
963 bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2])
964 {
965 	struct io_ring_ctx *ctx = req->ctx;
966 	bool posted;
967 
968 	lockdep_assert(!io_wq_current_is_worker());
969 	lockdep_assert_held(&ctx->uring_lock);
970 
971 	cqe[0].user_data = req->cqe.user_data;
972 	if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ)) {
973 		spin_lock(&ctx->completion_lock);
974 		posted = io_fill_cqe_aux32(ctx, cqe);
975 		spin_unlock(&ctx->completion_lock);
976 	} else {
977 		posted = io_fill_cqe_aux32(ctx, cqe);
978 	}
979 
980 	ctx->submit_state.cq_flush = true;
981 	return posted;
982 }
983 
984 /* drop the request file before the CQE is posted, not deferred after it */
985 static void io_req_put_file(struct io_kiocb *req, bool sync)
986 {
987 	struct file *file = req->file;
988 
989 	if (!file || (req->flags & (REQ_F_FIXED_FILE | REQ_F_REISSUE)))
990 		return;
991 
992 	WRITE_ONCE(req->file, NULL);
993 	/* releasing a ring may wait on other rings, keep that deferred */
994 	if (sync && !io_is_uring_fops(file))
995 		__fput_sync(file);
996 	else
997 		fput(file);
998 }
999 
1000 static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
1001 {
1002 	struct io_ring_ctx *ctx = req->ctx;
1003 	bool completed = true;
1004 
1005 	/*
1006 	 * All execution paths but io-wq use the deferred completions by
1007 	 * passing IO_URING_F_COMPLETE_DEFER and thus should not end up here.
1008 	 */
1009 	if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_IOWQ)))
1010 		return;
1011 
1012 	io_req_put_file(req, true);
1013 
1014 	/*
1015 	 * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
1016 	 * the submitter task context, IOPOLL protects with uring_lock. A drain
1017 	 * request ends the drain under uring_lock as its CQE is posted.
1018 	 */
1019 	if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) ||
1020 	    (req->flags & (REQ_F_REISSUE | REQ_F_IO_DRAIN))) {
1021 defer_complete:
1022 		req->io_task_work.func = io_req_task_complete;
1023 		io_req_task_work_add(req);
1024 		return;
1025 	}
1026 
1027 	io_cq_lock(ctx);
1028 	if (!(req->flags & REQ_F_CQE_SKIP))
1029 		completed = io_fill_cqe_req(ctx, req);
1030 	io_cq_unlock_post(ctx);
1031 
1032 	if (!completed)
1033 		goto defer_complete;
1034 
1035 	/*
1036 	 * Request not freed here because we know it's called from io-wq only,
1037 	 * which holds a reference. Hence it can't be the last put. The CQE
1038 	 * has been posted, last put frees it.
1039 	 */
1040 	req->flags |= REQ_F_CQE_SKIP;
1041 	req_ref_put(req);
1042 }
1043 
1044 void io_req_defer_failed(struct io_kiocb *req, s32 res)
1045 	__must_hold(&ctx->uring_lock)
1046 {
1047 	const struct io_cold_def *def = &io_cold_defs[req->opcode];
1048 
1049 	lockdep_assert_held(&req->ctx->uring_lock);
1050 
1051 	req_set_fail(req);
1052 	io_req_set_res(req, res, io_put_kbuf(req, res, NULL));
1053 	if (def->fail)
1054 		def->fail(req);
1055 	io_req_complete_defer(req);
1056 }
1057 
1058 /*
1059  * A request might get retired back into the request caches even before opcode
1060  * handlers and io_issue_sqe() are done with it, e.g. inline completion path.
1061  * Because of that, io_alloc_req() should be called only under ->uring_lock
1062  * and with extra caution to not get a request that is still worked on.
1063  */
1064 __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)
1065 	__must_hold(&ctx->uring_lock)
1066 {
1067 	gfp_t gfp = GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO;
1068 	void *reqs[IO_REQ_ALLOC_BATCH];
1069 	int nr_reqs = ARRAY_SIZE(reqs);
1070 
1071 	/*
1072 	 * Bulk alloc is all-or-nothing. If we fail to get a batch, retry a
1073 	 * single allocation to be on the safe side.
1074 	 */
1075 	if (!kmem_cache_alloc_bulk(req_cachep, gfp, nr_reqs, reqs)) {
1076 		reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1077 		if (!reqs[0])
1078 			return false;
1079 		nr_reqs = 1;
1080 	}
1081 
1082 	percpu_ref_get_many(&ctx->refs, nr_reqs);
1083 	ctx->nr_req_allocated += nr_reqs;
1084 
1085 	while (nr_reqs--)
1086 		io_req_add_to_cache(reqs[nr_reqs], ctx);
1087 	return true;
1088 }
1089 
1090 __cold void io_free_req(struct io_kiocb *req)
1091 {
1092 	/* refs were already put, restore them for io_req_task_complete() */
1093 	req->flags &= ~REQ_F_REFCOUNT;
1094 	req->io_task_work.func = io_req_task_complete;
1095 	io_req_task_work_add(req);
1096 }
1097 
1098 static void __io_req_find_next_prep(struct io_kiocb *req)
1099 {
1100 	struct io_ring_ctx *ctx = req->ctx;
1101 
1102 	spin_lock(&ctx->completion_lock);
1103 	io_disarm_next(req);
1104 	spin_unlock(&ctx->completion_lock);
1105 }
1106 
1107 static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
1108 {
1109 	struct io_kiocb *nxt;
1110 
1111 	/*
1112 	 * If LINK is set, we have dependent requests in this chain. If we
1113 	 * didn't fail this request, queue the first one up, moving any other
1114 	 * dependencies to the next request. In case of failure, fail the rest
1115 	 * of the chain.
1116 	 */
1117 	if (unlikely(req->flags & IO_DISARM_MASK))
1118 		__io_req_find_next_prep(req);
1119 	nxt = req->link;
1120 	req->link = NULL;
1121 	return nxt;
1122 }
1123 
1124 static void io_req_task_cancel(struct io_tw_req tw_req, io_tw_token_t tw)
1125 {
1126 	struct io_kiocb *req = tw_req.req;
1127 
1128 	io_tw_lock(req->ctx, tw);
1129 	io_req_defer_failed(req, req->cqe.res);
1130 }
1131 
1132 void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw)
1133 {
1134 	struct io_kiocb *req = tw_req.req;
1135 	struct io_ring_ctx *ctx = req->ctx;
1136 
1137 	io_tw_lock(ctx, tw);
1138 	if (unlikely(tw.cancel))
1139 		io_req_defer_failed(req, -EFAULT);
1140 	else if (req->flags & REQ_F_FORCE_ASYNC)
1141 		io_queue_iowq(req);
1142 	else
1143 		io_queue_sqe(req, 0);
1144 }
1145 
1146 void io_req_task_queue_fail(struct io_kiocb *req, int ret)
1147 {
1148 	io_req_set_res(req, ret, 0);
1149 	req->io_task_work.func = io_req_task_cancel;
1150 	io_req_task_work_add(req);
1151 }
1152 
1153 void io_req_task_queue(struct io_kiocb *req)
1154 {
1155 	req->io_task_work.func = io_req_task_submit;
1156 	io_req_task_work_add(req);
1157 }
1158 
1159 void io_queue_next(struct io_kiocb *req)
1160 {
1161 	struct io_kiocb *nxt = io_req_find_next(req);
1162 
1163 	if (nxt)
1164 		io_req_task_queue(nxt);
1165 }
1166 
1167 static inline void io_req_put_rsrc_nodes(struct io_kiocb *req)
1168 {
1169 	struct io_ring_ctx *ctx = req->ctx;
1170 
1171 	if (req->file_node) {
1172 		io_put_rsrc_node(ctx, req->file_node);
1173 		req->file_node = NULL;
1174 	}
1175 	if (req->flags & REQ_F_BUF_NODE)
1176 		io_put_rsrc_node(ctx, req->buf_node);
1177 }
1178 
1179 static void io_free_batch_list(struct io_ring_ctx *ctx,
1180 			       struct io_wq_work_node *node)
1181 	__must_hold(&ctx->uring_lock)
1182 {
1183 	do {
1184 		struct io_kiocb *req = container_of(node, struct io_kiocb,
1185 						    comp_list);
1186 
1187 		if (unlikely(req->flags & IO_REQ_CLEAN_SLOW_FLAGS)) {
1188 			if (req->flags & REQ_F_REISSUE) {
1189 				node = req->comp_list.next;
1190 				req->flags &= ~REQ_F_REISSUE;
1191 				io_queue_iowq(req);
1192 				continue;
1193 			}
1194 			if (req->flags & REQ_F_REFCOUNT) {
1195 				node = req->comp_list.next;
1196 				/* CQE posted, the last put only frees */
1197 				req->flags |= REQ_F_CQE_SKIP;
1198 				if (!req_ref_put_and_test(req))
1199 					continue;
1200 			}
1201 			if ((req->flags & REQ_F_POLLED) && req->apoll) {
1202 				struct async_poll *apoll = req->apoll;
1203 
1204 				if (apoll->double_poll)
1205 					kfree(apoll->double_poll);
1206 				io_cache_free(&ctx->apoll_cache, apoll);
1207 				req->flags &= ~REQ_F_POLLED;
1208 			}
1209 			if (req->flags & IO_REQ_LINK_FLAGS)
1210 				io_queue_next(req);
1211 			if (unlikely(req->flags & IO_REQ_CLEAN_FLAGS))
1212 				io_clean_op(req);
1213 		}
1214 		io_put_file(req);
1215 		io_req_put_rsrc_nodes(req);
1216 		io_put_task(req);
1217 
1218 		node = req->comp_list.next;
1219 		io_req_add_to_cache(req, ctx);
1220 	} while (node);
1221 }
1222 
1223 /*
1224  * The dispatched drain request ran alone, and anything submitted after it was
1225  * deferred. Once it completed, new requests may run if nothing is waiting.
1226  */
1227 static __cold bool io_drain_req_completed(struct io_ring_ctx *ctx)
1228 {
1229 	struct io_wq_work_node *node;
1230 
1231 	__wq_list_for_each(node, &ctx->submit_state.compl_reqs) {
1232 		struct io_kiocb *req = container_of(node, struct io_kiocb,
1233 					    comp_list);
1234 
1235 		/* a requeued request hasn't completed */
1236 		if (req == ctx->drain_req && !(req->flags & REQ_F_REISSUE)) {
1237 			ctx->drain_req = NULL;
1238 			return true;
1239 		}
1240 	}
1241 	return false;
1242 }
1243 
1244 void __io_submit_flush_completions(struct io_ring_ctx *ctx)
1245 	__must_hold(&ctx->uring_lock)
1246 {
1247 	struct io_submit_state *state = &ctx->submit_state;
1248 	struct io_wq_work_node *node;
1249 	bool drain_done = false;
1250 
1251 	/*
1252 	 * Drop the files before the CQEs are posted, so they're released by
1253 	 * the time the completions are visible. Not for requests that are
1254 	 * requeued or still referenced, those aren't freed below.
1255 	 */
1256 	__wq_list_for_each(node, &state->compl_reqs) {
1257 		struct io_kiocb *req = container_of(node, struct io_kiocb,
1258 					    comp_list);
1259 
1260 		if (!io_req_shared(req))
1261 			io_req_put_file(req, true);
1262 	}
1263 
1264 	__io_cq_lock(ctx);
1265 	__wq_list_for_each(node, &state->compl_reqs) {
1266 		struct io_kiocb *req = container_of(node, struct io_kiocb,
1267 					    comp_list);
1268 
1269 		/*
1270 		 * Requests marked with REQUEUE should not post a CQE, they
1271 		 * will go through the io-wq retry machinery and post one
1272 		 * later.
1273 		 */
1274 		if (!(req->flags & (REQ_F_CQE_SKIP | REQ_F_REISSUE)) &&
1275 		    unlikely(!io_fill_cqe_req(ctx, req))) {
1276 			if (ctx->int_flags & IO_RING_F_LOCKLESS_CQ)
1277 				io_cqe_overflow(ctx, &req->cqe, &req->big_cqe);
1278 			else
1279 				io_cqe_overflow_locked(ctx, &req->cqe, &req->big_cqe);
1280 		}
1281 	}
1282 	__io_cq_unlock_post(ctx);
1283 
1284 	if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE) && ctx->drain_req)
1285 		drain_done = io_drain_req_completed(ctx);
1286 
1287 	if (!wq_list_empty(&state->compl_reqs)) {
1288 		io_free_batch_list(ctx, state->compl_reqs.first);
1289 		INIT_WQ_LIST(&state->compl_reqs);
1290 	}
1291 
1292 	if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) {
1293 		/* a drain request started from the list keeps the drain going */
1294 		if (io_queue_deferred(ctx))
1295 			drain_done = false;
1296 		if (drain_done && list_empty(&ctx->defer_list))
1297 			ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE;
1298 	}
1299 
1300 	ctx->submit_state.cq_flush = false;
1301 }
1302 
1303 /*
1304  * We can't just wait for polled events to come to us, we have to actively
1305  * find and complete them.
1306  */
1307 __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
1308 {
1309 	if (!(ctx->flags & IORING_SETUP_IOPOLL))
1310 		return;
1311 
1312 	mutex_lock(&ctx->uring_lock);
1313 	while (!list_empty(&ctx->iopoll_list)) {
1314 		/* let it sleep and repeat later if can't complete a request */
1315 		if (io_do_iopoll(ctx, true) == 0)
1316 			break;
1317 		/*
1318 		 * Ensure we allow local-to-the-cpu processing to take place,
1319 		 * in this case we need to ensure that we reap all events.
1320 		 * Also let task_work, etc. to progress by releasing the mutex
1321 		 */
1322 		if (need_resched()) {
1323 			mutex_unlock(&ctx->uring_lock);
1324 			cond_resched();
1325 			mutex_lock(&ctx->uring_lock);
1326 		}
1327 	}
1328 	mutex_unlock(&ctx->uring_lock);
1329 
1330 	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
1331 		io_cancel_local_task_work(ctx);
1332 }
1333 
1334 static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)
1335 {
1336 	unsigned long check_cq;
1337 
1338 	min_events = min(min_events, ctx->cq_entries);
1339 
1340 	lockdep_assert_held(&ctx->uring_lock);
1341 
1342 	if (!io_allowed_run_tw(ctx))
1343 		return -EEXIST;
1344 
1345 	check_cq = READ_ONCE(ctx->check_cq);
1346 	if (unlikely(check_cq)) {
1347 		if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
1348 			__io_cqring_overflow_flush(ctx, false);
1349 		/*
1350 		 * Similarly do not spin if we have not informed the user of any
1351 		 * dropped CQE.
1352 		 */
1353 		if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT))
1354 			return -EBADR;
1355 	}
1356 	/*
1357 	 * Don't enter poll loop if we already have events pending.
1358 	 * If we do, we can potentially be spinning for commands that
1359 	 * already triggered a CQE (eg in error).
1360 	 */
1361 	if (io_cqring_events(ctx))
1362 		return 0;
1363 
1364 	do {
1365 		int ret = 0;
1366 
1367 		/*
1368 		 * If a submit got punted to a workqueue, we can have the
1369 		 * application entering polling for a command before it gets
1370 		 * issued. That app will hold the uring_lock for the duration
1371 		 * of the poll right here, so we need to take a breather every
1372 		 * now and then to ensure that the issue has a chance to add
1373 		 * the poll to the issued list. Otherwise we can spin here
1374 		 * forever, while the workqueue is stuck trying to acquire the
1375 		 * very same mutex.
1376 		 */
1377 		if (list_empty(&ctx->iopoll_list) || io_task_work_pending(ctx)) {
1378 			(void) io_run_local_work_locked(ctx, min_events);
1379 
1380 			if (task_work_pending(current) || list_empty(&ctx->iopoll_list)) {
1381 				mutex_unlock(&ctx->uring_lock);
1382 				io_run_task_work();
1383 				mutex_lock(&ctx->uring_lock);
1384 			}
1385 			/* some requests don't go through iopoll_list */
1386 			if (list_empty(&ctx->iopoll_list))
1387 				break;
1388 		}
1389 		ret = io_do_iopoll(ctx, !min_events);
1390 		if (unlikely(ret < 0))
1391 			return ret;
1392 
1393 		if (task_sigpending(current))
1394 			return -EINTR;
1395 		if (need_resched())
1396 			break;
1397 	} while (io_cqring_events(ctx) < min_events);
1398 
1399 	return 0;
1400 }
1401 
1402 void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw)
1403 {
1404 	struct io_kiocb *req = tw_req.req;
1405 
1406 	if (io_req_complete_ready(req))
1407 		io_req_complete_defer(req);
1408 }
1409 
1410 /*
1411  * After the iocb has been issued, it's safe to be found on the poll list.
1412  * Adding the kiocb to the list AFTER submission ensures that we don't
1413  * find it from a io_do_iopoll() thread before the issuer is done
1414  * accessing the kiocb cookie.
1415  */
1416 static void io_iopoll_req_issued(struct io_kiocb *req, unsigned int issue_flags)
1417 {
1418 	struct io_ring_ctx *ctx = req->ctx;
1419 	const bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
1420 
1421 	/* workqueue context doesn't hold uring_lock, grab it now */
1422 	if (unlikely(needs_lock))
1423 		mutex_lock(&ctx->uring_lock);
1424 
1425 	/*
1426 	 * Track whether we have multiple files in our lists. This will impact
1427 	 * how we do polling eventually, not spinning if we're on potentially
1428 	 * different devices.
1429 	 */
1430 	if (list_empty(&ctx->iopoll_list)) {
1431 		ctx->poll_multi_queue = false;
1432 	} else if (!ctx->poll_multi_queue) {
1433 		struct io_kiocb *list_req;
1434 
1435 		list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb, iopoll_node);
1436 		if (list_req->file != req->file)
1437 			ctx->poll_multi_queue = true;
1438 	}
1439 
1440 	list_add_tail(&req->iopoll_node, &ctx->iopoll_list);
1441 
1442 	if (unlikely(needs_lock)) {
1443 		/*
1444 		 * If IORING_SETUP_SQPOLL is enabled, sqes are either handle
1445 		 * in sq thread task context or in io worker task context. If
1446 		 * current task context is sq thread, we don't need to check
1447 		 * whether should wake up sq thread.
1448 		 */
1449 		if ((ctx->flags & IORING_SETUP_SQPOLL) &&
1450 		    wq_has_sleeper(&ctx->sq_data->wait))
1451 			wake_up(&ctx->sq_data->wait);
1452 
1453 		mutex_unlock(&ctx->uring_lock);
1454 	}
1455 }
1456 
1457 io_req_flags_t io_file_get_flags(struct file *file)
1458 {
1459 	io_req_flags_t res = 0;
1460 
1461 	BUILD_BUG_ON(REQ_F_ISREG_BIT != REQ_F_SUPPORT_NOWAIT_BIT + 1);
1462 
1463 	if (S_ISREG(file_inode(file)->i_mode))
1464 		res |= REQ_F_ISREG;
1465 	if ((file->f_flags & O_NONBLOCK) || (file->f_mode & FMODE_NOWAIT))
1466 		res |= REQ_F_SUPPORT_NOWAIT;
1467 	return res;
1468 }
1469 
1470 static __cold void io_drain_req(struct io_kiocb *req)
1471 	__must_hold(&ctx->uring_lock)
1472 {
1473 	struct io_ring_ctx *ctx = req->ctx;
1474 	bool drain = req->flags & IOSQE_IO_DRAIN;
1475 	struct io_defer_entry *de;
1476 
1477 	de = kmalloc_obj(*de, GFP_KERNEL_ACCOUNT);
1478 	if (!de) {
1479 		io_req_defer_failed(req, -ENOMEM);
1480 		return;
1481 	}
1482 
1483 	io_prep_async_link(req);
1484 	trace_io_uring_defer(req);
1485 	de->req = req;
1486 
1487 	ctx->nr_drained += io_linked_nr(req);
1488 	list_add_tail(&de->list, &ctx->defer_list);
1489 	io_queue_deferred(ctx);
1490 	if (!drain && list_empty(&ctx->defer_list))
1491 		ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE;
1492 }
1493 
1494 #define REQ_ISSUE_SLOW_FLAGS	(REQ_F_CREDS | REQ_F_ARM_LTIMEOUT)
1495 
1496 static inline int __io_issue_sqe(struct io_kiocb *req,
1497 				 unsigned int issue_flags,
1498 				 const struct io_issue_def *def)
1499 {
1500 	const struct cred *creds = NULL;
1501 	struct io_kiocb *link = NULL;
1502 	int ret;
1503 
1504 	if (unlikely(req->flags & REQ_ISSUE_SLOW_FLAGS)) {
1505 		if ((req->flags & REQ_F_CREDS) && req->creds != current_cred())
1506 			creds = override_creds(req->creds);
1507 		if (req->flags & REQ_F_ARM_LTIMEOUT)
1508 			link = __io_prep_linked_timeout(req);
1509 	}
1510 
1511 	if (!def->audit_skip)
1512 		audit_uring_entry(req->opcode);
1513 
1514 	ret = def->issue(req, issue_flags);
1515 
1516 	if (!def->audit_skip)
1517 		audit_uring_exit(!ret, ret);
1518 
1519 	if (unlikely(creds || link)) {
1520 		if (creds)
1521 			revert_creds(creds);
1522 		if (link)
1523 			io_queue_linked_timeout(link);
1524 	}
1525 
1526 	return ret;
1527 }
1528 
1529 static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
1530 {
1531 	const struct io_issue_def *def = &io_issue_defs[req->opcode];
1532 	int ret;
1533 
1534 	if (unlikely(!io_assign_file(req, def, issue_flags)))
1535 		return -EBADF;
1536 
1537 	ret = __io_issue_sqe(req, issue_flags, def);
1538 
1539 	if (ret == IOU_COMPLETE) {
1540 		/* not all handlers take care of the internal restart codes */
1541 		if (unlikely(req->cqe.res <= -ERESTARTSYS))
1542 			req->cqe.res = io_fixup_restart_res(req->cqe.res);
1543 		if (issue_flags & IO_URING_F_COMPLETE_DEFER)
1544 			io_req_complete_defer(req);
1545 		else
1546 			io_req_complete_post(req, issue_flags);
1547 
1548 		return 0;
1549 	}
1550 
1551 	if (ret == IOU_ISSUE_SKIP_COMPLETE) {
1552 		ret = 0;
1553 
1554 		if (req->flags & REQ_F_IOPOLL)
1555 			io_iopoll_req_issued(req, issue_flags);
1556 	}
1557 	if (unlikely(ret <= -ERESTARTSYS))
1558 		ret = io_fixup_restart_res(ret);
1559 	return ret;
1560 }
1561 
1562 int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)
1563 {
1564 	const unsigned int issue_flags = IO_URING_F_NONBLOCK |
1565 					 IO_URING_F_MULTISHOT |
1566 					 IO_URING_F_COMPLETE_DEFER;
1567 	int ret;
1568 
1569 	io_tw_lock(req->ctx, tw);
1570 
1571 	WARN_ON_ONCE(!req->file);
1572 	if (WARN_ON_ONCE(req->flags & REQ_F_IOPOLL))
1573 		return -EFAULT;
1574 
1575 	ret = __io_issue_sqe(req, issue_flags, &io_issue_defs[req->opcode]);
1576 
1577 	WARN_ON_ONCE(ret == IOU_ISSUE_SKIP_COMPLETE);
1578 	return ret;
1579 }
1580 
1581 /*
1582  * io-wq is done with @work. Returns true if it held the last reference, and
1583  * hands out the next request of the link then. The caller has to make sure
1584  * that cancellations can find the latter before it calls io_wq_free_work().
1585  */
1586 bool io_wq_put_work(struct io_wq_work *work, struct io_wq_work **linked)
1587 {
1588 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
1589 	struct io_kiocb *nxt = NULL;
1590 
1591 	*linked = NULL;
1592 	if (!req_ref_put_and_test_atomic(req))
1593 		return false;
1594 
1595 	/*
1596 	 * Only continue the link from here if the completion has been posted.
1597 	 * If it's still pending, io_free_req() completes the request and the
1598 	 * next link follows from there, after the CQE.
1599 	 */
1600 	if ((req->flags & IO_REQ_LINK_FLAGS) && (req->flags & REQ_F_CQE_SKIP)) {
1601 		struct io_ring_ctx *ctx = req->ctx;
1602 
1603 		mutex_lock(&ctx->uring_lock);
1604 		nxt = io_req_find_next(req);
1605 		mutex_unlock(&ctx->uring_lock);
1606 	}
1607 	if (nxt)
1608 		*linked = &nxt->work;
1609 	return true;
1610 }
1611 
1612 void io_wq_free_work(struct io_wq_work *work)
1613 {
1614 	io_free_req(container_of(work, struct io_kiocb, work));
1615 }
1616 
1617 void io_wq_submit_work(struct io_wq_work *work)
1618 {
1619 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
1620 	const struct io_issue_def *def = &io_issue_defs[req->opcode];
1621 	unsigned int issue_flags = IO_URING_F_UNLOCKED | IO_URING_F_IOWQ;
1622 	bool needs_poll = false;
1623 	int ret = 0, err = -ECANCELED;
1624 
1625 	/* one will be dropped by io_wq_put_work() after returning to io-wq */
1626 	if (!(req->flags & REQ_F_REFCOUNT))
1627 		__io_req_set_refcount(req, 2);
1628 	else
1629 		req_ref_get(req);
1630 
1631 	/* either cancelled or io-wq is dying, so don't touch tctx->iowq */
1632 	if (atomic_read(&work->flags) & IO_WQ_WORK_CANCEL) {
1633 fail:
1634 		io_req_put_file(req, false);
1635 		io_req_task_queue_fail(req, err);
1636 		return;
1637 	}
1638 	if (!io_assign_file(req, def, issue_flags)) {
1639 		err = -EBADF;
1640 		atomic_or(IO_WQ_WORK_CANCEL, &work->flags);
1641 		goto fail;
1642 	}
1643 
1644 	/*
1645 	 * If DEFER_TASKRUN is set, it's only allowed to post CQEs from the
1646 	 * submitter task context. Final request completions are handed to the
1647 	 * right context, however this is not the case of auxiliary CQEs,
1648 	 * which is the main mean of operation for multishot requests.
1649 	 * Don't allow any multishot execution from io-wq. It's more restrictive
1650 	 * than necessary and also cleaner.
1651 	 */
1652 	if (req->flags & (REQ_F_MULTISHOT|REQ_F_APOLL_MULTISHOT)) {
1653 		err = -EBADFD;
1654 		if (!io_file_can_poll(req))
1655 			goto fail;
1656 		if (req->file->f_flags & O_NONBLOCK ||
1657 		    req->file->f_mode & FMODE_NOWAIT) {
1658 			err = -ECANCELED;
1659 			if (io_arm_poll_handler(req, issue_flags) != IO_APOLL_OK)
1660 				goto fail;
1661 			return;
1662 		} else {
1663 			req->flags &= ~(REQ_F_APOLL_MULTISHOT|REQ_F_MULTISHOT);
1664 		}
1665 	}
1666 
1667 	if (req->flags & REQ_F_FORCE_ASYNC) {
1668 		bool opcode_poll = def->pollin || def->pollout;
1669 
1670 		if (opcode_poll && io_file_can_poll(req)) {
1671 			needs_poll = true;
1672 			issue_flags |= IO_URING_F_NONBLOCK;
1673 		}
1674 	}
1675 
1676 	do {
1677 		ret = io_issue_sqe(req, issue_flags);
1678 		if (ret != -EAGAIN)
1679 			break;
1680 
1681 		/*
1682 		 * If REQ_F_NOWAIT is set, then don't wait or retry with
1683 		 * poll. -EAGAIN is final for that case.
1684 		 */
1685 		if (req->flags & REQ_F_NOWAIT)
1686 			break;
1687 
1688 		/*
1689 		 * We can get EAGAIN for iopolled IO even though we're
1690 		 * forcing a sync submission from here, since we can't
1691 		 * wait for request slots on the block side.
1692 		 */
1693 		if (!needs_poll) {
1694 			if (!(req->flags & REQ_F_IOPOLL))
1695 				break;
1696 			if (io_wq_worker_stopped())
1697 				break;
1698 			cond_resched();
1699 			continue;
1700 		}
1701 
1702 		if (io_arm_poll_handler(req, issue_flags) == IO_APOLL_OK)
1703 			return;
1704 		/* aborted or ready, in either case retry blocking */
1705 		needs_poll = false;
1706 		issue_flags &= ~IO_URING_F_NONBLOCK;
1707 	} while (1);
1708 
1709 	/* avoid locking problems by failing it from a clean context */
1710 	if (ret) {
1711 		io_req_put_file(req, true);
1712 		io_req_task_queue_fail(req, ret);
1713 	}
1714 }
1715 
1716 inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
1717 				      unsigned int issue_flags)
1718 {
1719 	struct io_ring_ctx *ctx = req->ctx;
1720 	struct io_rsrc_node *node;
1721 	struct file *file = NULL;
1722 
1723 	io_ring_submit_lock(ctx, issue_flags);
1724 	node = io_rsrc_node_lookup(&ctx->file_table.data, fd);
1725 	if (node) {
1726 		node->refs++;
1727 		req->file_node = node;
1728 		req->flags |= io_slot_flags(node);
1729 		file = io_slot_file(node);
1730 	}
1731 	io_ring_submit_unlock(ctx, issue_flags);
1732 	return file;
1733 }
1734 
1735 struct file *io_file_get_normal(struct io_kiocb *req, int fd)
1736 {
1737 	struct file *file = fget(fd);
1738 
1739 	trace_io_uring_file_get(req, fd);
1740 
1741 	/* we don't allow fixed io_uring files */
1742 	if (file && io_is_uring_fops(file))
1743 		io_req_track_inflight(req);
1744 	return file;
1745 }
1746 
1747 static int io_req_sqe_copy(struct io_kiocb *req, unsigned int issue_flags)
1748 {
1749 	const struct io_cold_def *def = &io_cold_defs[req->opcode];
1750 
1751 	if (req->flags & REQ_F_SQE_COPIED)
1752 		return 0;
1753 	req->flags |= REQ_F_SQE_COPIED;
1754 	if (!def->sqe_copy)
1755 		return 0;
1756 	if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_INLINE)))
1757 		return -EFAULT;
1758 	def->sqe_copy(req);
1759 	return 0;
1760 }
1761 
1762 static void io_queue_async(struct io_kiocb *req, unsigned int issue_flags, int ret)
1763 	__must_hold(&req->ctx->uring_lock)
1764 {
1765 	if (ret != -EAGAIN || (req->flags & REQ_F_NOWAIT)) {
1766 fail:
1767 		io_req_defer_failed(req, ret);
1768 		return;
1769 	}
1770 
1771 	ret = io_req_sqe_copy(req, issue_flags);
1772 	if (unlikely(ret))
1773 		goto fail;
1774 
1775 	switch (io_arm_poll_handler(req, 0)) {
1776 	case IO_APOLL_READY:
1777 		io_req_task_queue(req);
1778 		break;
1779 	case IO_APOLL_ABORTED:
1780 		io_queue_iowq(req);
1781 		break;
1782 	case IO_APOLL_OK:
1783 		break;
1784 	}
1785 }
1786 
1787 static inline void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags)
1788 	__must_hold(&req->ctx->uring_lock)
1789 {
1790 	unsigned int issue_flags = IO_URING_F_NONBLOCK |
1791 				   IO_URING_F_COMPLETE_DEFER | extra_flags;
1792 	int ret;
1793 
1794 	ret = io_issue_sqe(req, issue_flags);
1795 
1796 	/*
1797 	 * We async punt it if the file wasn't marked NOWAIT, or if the file
1798 	 * doesn't support non-blocking read/write attempts
1799 	 */
1800 	if (unlikely(ret))
1801 		io_queue_async(req, issue_flags, ret);
1802 }
1803 
1804 static void io_queue_sqe_fallback(struct io_kiocb *req)
1805 	__must_hold(&req->ctx->uring_lock)
1806 {
1807 	if (unlikely(req->flags & REQ_F_FAIL)) {
1808 		/*
1809 		 * We don't submit, fail them all, for that replace hardlinks
1810 		 * with normal links. Extra REQ_F_LINK is tolerated.
1811 		 */
1812 		req->flags &= ~REQ_F_HARDLINK;
1813 		req->flags |= REQ_F_LINK;
1814 		io_req_defer_failed(req, req->cqe.res);
1815 	} else {
1816 		/* can't fail with IO_URING_F_INLINE */
1817 		io_req_sqe_copy(req, IO_URING_F_INLINE);
1818 		if (unlikely(req->ctx->int_flags & IO_RING_F_DRAIN_ACTIVE))
1819 			io_drain_req(req);
1820 		else
1821 			io_queue_iowq(req);
1822 	}
1823 }
1824 
1825 /*
1826  * Check SQE restrictions (opcode and flags).
1827  *
1828  * Returns 'true' if SQE is allowed, 'false' otherwise.
1829  */
1830 static inline bool io_check_restriction(struct io_ring_ctx *ctx,
1831 					struct io_kiocb *req,
1832 					unsigned int sqe_flags)
1833 {
1834 	if (!(ctx->int_flags & IO_RING_F_OP_RESTRICTED))
1835 		return true;
1836 	if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
1837 		return false;
1838 
1839 	if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
1840 	    ctx->restrictions.sqe_flags_required)
1841 		return false;
1842 
1843 	if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
1844 			  ctx->restrictions.sqe_flags_required))
1845 		return false;
1846 
1847 	return true;
1848 }
1849 
1850 static void io_init_drain(struct io_ring_ctx *ctx)
1851 {
1852 	struct io_kiocb *head = ctx->submit_state.link.head;
1853 
1854 	ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE;
1855 	if (head) {
1856 		/*
1857 		 * If we need to drain a request in the middle of a link, drain
1858 		 * the head request and the next request/link after the current
1859 		 * link. Considering sequential execution of links,
1860 		 * REQ_F_IO_DRAIN will be maintained for every request of our
1861 		 * link.
1862 		 */
1863 		head->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
1864 		ctx->int_flags |= IO_RING_F_DRAIN_NEXT;
1865 	}
1866 }
1867 
1868 static __cold int io_init_fail_req(struct io_kiocb *req, int err)
1869 {
1870 	/* ensure per-opcode data is cleared if we fail before prep */
1871 	memset(&req->cmd.data, 0, sizeof(req->cmd.data));
1872 	return err;
1873 }
1874 
1875 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
1876 		       const struct io_uring_sqe *sqe, unsigned int *left)
1877 	__must_hold(&ctx->uring_lock)
1878 {
1879 	const struct io_issue_def *def;
1880 	unsigned int sqe_flags;
1881 	int personality;
1882 
1883 	req->ctx = ctx;
1884 	req->opcode = READ_ONCE(sqe->opcode);
1885 	/* same numerical values with corresponding REQ_F_*, safe to copy */
1886 	sqe_flags = READ_ONCE(sqe->flags);
1887 	req->flags = (__force io_req_flags_t) sqe_flags;
1888 	req->cqe.user_data = READ_ONCE(sqe->user_data);
1889 	req->file = NULL;
1890 	req->tctx = current->io_uring;
1891 	req->cancel_seq_set = false;
1892 	req->async_data = NULL;
1893 
1894 	if (unlikely(req->opcode >= IORING_OP_LAST)) {
1895 		req->opcode = 0;
1896 		return io_init_fail_req(req, -EINVAL);
1897 	}
1898 	req->opcode = array_index_nospec(req->opcode, IORING_OP_LAST);
1899 
1900 	def = &io_issue_defs[req->opcode];
1901 	if (def->is_128 && !(ctx->flags & IORING_SETUP_SQE128)) {
1902 		/*
1903 		 * A 128b op on a non-128b SQ requires mixed SQE support as
1904 		 * well as 2 contiguous entries.
1905 		 */
1906 		if (!(ctx->flags & IORING_SETUP_SQE_MIXED) ||
1907 		    (unsigned)(sqe - ctx->sq_sqes) >= ctx->sq_entries - 1)
1908 			return io_init_fail_req(req, -EINVAL);
1909 		if (*left >= 2) {
1910 			/*
1911 			 * A 128b operation on a mixed SQ uses two entries, so
1912 			 * we have to increment the head and cached refs, and
1913 			 * decrement what's left.
1914 			 */
1915 			current->io_uring->cached_refs++;
1916 			(*left)--;
1917 		} else if (!(ctx->flags & IORING_SETUP_SQPOLL) ||
1918 			   !__io_sqring_entries(ctx)) {
1919 			return io_init_fail_req(req, -EINVAL);
1920 		}
1921 		/*
1922 		 * else: the SQPOLL thread caps the number of entries it submits
1923 		 * in one go, which is none of the application's business. Take
1924 		 * the 2nd entry on top.
1925 		 */
1926 		ctx->cached_sq_head++;
1927 	}
1928 
1929 	if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) {
1930 		/* enforce forwards compatibility on users */
1931 		if (sqe_flags & ~SQE_VALID_FLAGS)
1932 			return io_init_fail_req(req, -EINVAL);
1933 		if (sqe_flags & IOSQE_BUFFER_SELECT) {
1934 			if (!def->buffer_select)
1935 				return io_init_fail_req(req, -EOPNOTSUPP);
1936 			req->buf_index = READ_ONCE(sqe->buf_group);
1937 		}
1938 		if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS)
1939 			ctx->int_flags |= IO_RING_F_DRAIN_DISABLED;
1940 		if (sqe_flags & IOSQE_IO_DRAIN) {
1941 			if (ctx->int_flags & IO_RING_F_DRAIN_DISABLED)
1942 				return io_init_fail_req(req, -EOPNOTSUPP);
1943 			io_init_drain(ctx);
1944 		}
1945 	}
1946 	if (unlikely(ctx->int_flags & (IO_RING_F_OP_RESTRICTED | IO_RING_F_DRAIN_ACTIVE | IO_RING_F_DRAIN_NEXT))) {
1947 		if (!io_check_restriction(ctx, req, sqe_flags))
1948 			return io_init_fail_req(req, -EACCES);
1949 		/* knock it to the slow queue path, will be drained there */
1950 		if (ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)
1951 			req->flags |= REQ_F_FORCE_ASYNC;
1952 		/* if there is no link, we're at "next" request and need to drain */
1953 		if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_NEXT) && !ctx->submit_state.link.head) {
1954 			ctx->int_flags &= ~IO_RING_F_DRAIN_NEXT;
1955 			ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE;
1956 			req->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
1957 		}
1958 	}
1959 
1960 	if (!def->ioprio && sqe->ioprio)
1961 		return io_init_fail_req(req, -EINVAL);
1962 	if (!def->iopoll && (ctx->flags & IORING_SETUP_IOPOLL))
1963 		return io_init_fail_req(req, -EINVAL);
1964 
1965 	if (def->needs_file) {
1966 		struct io_submit_state *state = &ctx->submit_state;
1967 
1968 		req->cqe.fd = READ_ONCE(sqe->fd);
1969 
1970 		/*
1971 		 * Plug now if we have more than 2 IO left after this, and the
1972 		 * target is potentially a read/write to block based storage.
1973 		 */
1974 		if (state->need_plug && def->plug) {
1975 			state->plug_started = true;
1976 			state->need_plug = false;
1977 			blk_start_plug_nr_ios(&state->plug, state->submit_nr);
1978 		}
1979 	}
1980 
1981 	personality = READ_ONCE(sqe->personality);
1982 	if (personality) {
1983 		int ret;
1984 
1985 		req->creds = xa_load(&ctx->personalities, personality);
1986 		if (!req->creds)
1987 			return io_init_fail_req(req, -EINVAL);
1988 		get_cred(req->creds);
1989 		ret = security_uring_override_creds(req->creds);
1990 		if (ret) {
1991 			put_cred(req->creds);
1992 			return io_init_fail_req(req, ret);
1993 		}
1994 		req->flags |= REQ_F_CREDS;
1995 	}
1996 
1997 	return def->prep(req, sqe);
1998 }
1999 
2000 static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,
2001 				      struct io_kiocb *req, int ret)
2002 {
2003 	struct io_ring_ctx *ctx = req->ctx;
2004 	struct io_submit_link *link = &ctx->submit_state.link;
2005 	struct io_kiocb *head = link->head;
2006 
2007 	trace_io_uring_req_failed(sqe, req, ret);
2008 
2009 	/*
2010 	 * Avoid breaking links in the middle as it renders links with SQPOLL
2011 	 * unusable. Instead of failing eagerly, continue assembling the link if
2012 	 * applicable and mark the head with REQ_F_FAIL. The link flushing code
2013 	 * should find the flag and handle the rest.
2014 	 */
2015 	req_fail_link_node(req, ret);
2016 	if (head && !(head->flags & REQ_F_FAIL))
2017 		req_fail_link_node(head, -ECANCELED);
2018 
2019 	if (!(req->flags & IO_REQ_LINK_FLAGS)) {
2020 		if (head) {
2021 			link->last->link = req;
2022 			link->head = NULL;
2023 			req = head;
2024 		}
2025 		io_queue_sqe_fallback(req);
2026 		return ret;
2027 	}
2028 
2029 	if (head)
2030 		link->last->link = req;
2031 	else
2032 		link->head = req;
2033 	link->last = req;
2034 	return 0;
2035 }
2036 
2037 static inline int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
2038 			 const struct io_uring_sqe *sqe, unsigned int *left)
2039 	__must_hold(&ctx->uring_lock)
2040 {
2041 	struct io_submit_link *link = &ctx->submit_state.link;
2042 	int ret;
2043 
2044 	ret = io_init_req(ctx, req, sqe, left);
2045 	if (unlikely(ret))
2046 		return io_submit_fail_init(sqe, req, ret);
2047 
2048 	if (unlikely(ctx->bpf_filters)) {
2049 		ret = io_uring_run_bpf_filters(ctx->bpf_filters, req);
2050 		if (ret)
2051 			return io_submit_fail_init(sqe, req, ret);
2052 	}
2053 
2054 	trace_io_uring_submit_req(req);
2055 
2056 	/*
2057 	 * If we already have a head request, queue this one for async
2058 	 * submittal once the head completes. If we don't have a head but
2059 	 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
2060 	 * submitted sync once the chain is complete. If none of those
2061 	 * conditions are true (normal request), then just queue it.
2062 	 */
2063 	if (unlikely(link->head)) {
2064 		trace_io_uring_link(req, link->last);
2065 		io_req_sqe_copy(req, IO_URING_F_INLINE);
2066 		link->last->link = req;
2067 		link->last = req;
2068 
2069 		if (req->flags & IO_REQ_LINK_FLAGS)
2070 			return 0;
2071 		/* last request of the link, flush it */
2072 		req = link->head;
2073 		link->head = NULL;
2074 		if (req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL))
2075 			goto fallback;
2076 
2077 	} else if (unlikely(req->flags & (IO_REQ_LINK_FLAGS |
2078 					  REQ_F_FORCE_ASYNC | REQ_F_FAIL))) {
2079 		if (req->flags & IO_REQ_LINK_FLAGS) {
2080 			link->head = req;
2081 			link->last = req;
2082 		} else {
2083 fallback:
2084 			io_queue_sqe_fallback(req);
2085 		}
2086 		return 0;
2087 	}
2088 
2089 	io_queue_sqe(req, IO_URING_F_INLINE);
2090 	return 0;
2091 }
2092 
2093 /*
2094  * Batched submission is done, ensure local IO is flushed out.
2095  */
2096 static void io_submit_state_end(struct io_ring_ctx *ctx)
2097 {
2098 	struct io_submit_state *state = &ctx->submit_state;
2099 
2100 	if (unlikely(state->link.head))
2101 		io_queue_sqe_fallback(state->link.head);
2102 	/* flush only after queuing links as they can generate completions */
2103 	io_submit_flush_completions(ctx);
2104 	if (state->plug_started)
2105 		blk_finish_plug(&state->plug);
2106 }
2107 
2108 /*
2109  * Start submission side cache.
2110  */
2111 static void io_submit_state_start(struct io_submit_state *state,
2112 				  unsigned int max_ios)
2113 {
2114 	state->plug_started = false;
2115 	state->need_plug = max_ios > 2;
2116 	state->submit_nr = max_ios;
2117 	/* set only head, no need to init link_last in advance */
2118 	state->link.head = NULL;
2119 }
2120 
2121 static void io_commit_sqring(struct io_ring_ctx *ctx)
2122 {
2123 	struct io_rings *rings = ctx->rings;
2124 
2125 	if (ctx->flags & IORING_SETUP_SQ_REWIND) {
2126 		ctx->cached_sq_head = 0;
2127 	} else {
2128 		/*
2129 		 * Ensure any loads from the SQEs are done at this point,
2130 		 * since once we write the new head, the application could
2131 		 * write new data to them.
2132 		 */
2133 		smp_store_release(&rings->sq.head, ctx->cached_sq_head);
2134 	}
2135 }
2136 
2137 /*
2138  * Fetch an sqe, if one is available. Note this returns a pointer to memory
2139  * that is mapped by userspace. This means that care needs to be taken to
2140  * ensure that reads are stable, as we cannot rely on userspace always
2141  * being a good citizen. If members of the sqe are validated and then later
2142  * used, it's important that those reads are done through READ_ONCE() to
2143  * prevent a re-load down the line.
2144  */
2145 static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe)
2146 {
2147 	unsigned mask = ctx->sq_entries - 1;
2148 	unsigned head = ctx->cached_sq_head++ & mask;
2149 
2150 	if (static_branch_unlikely(&io_key_has_sqarray.key) &&
2151 	    (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) {
2152 		head = READ_ONCE(ctx->sq_array[head]);
2153 		if (unlikely(head >= ctx->sq_entries)) {
2154 			WRITE_ONCE(ctx->rings->sq_dropped,
2155 				   READ_ONCE(ctx->rings->sq_dropped) + 1);
2156 			return false;
2157 		}
2158 		head = array_index_nospec(head, ctx->sq_entries);
2159 	}
2160 
2161 	/*
2162 	 * The cached sq head (or cq tail) serves two purposes:
2163 	 *
2164 	 * 1) allows us to batch the cost of updating the user visible
2165 	 *    head updates.
2166 	 * 2) allows the kernel side to track the head on its own, even
2167 	 *    though the application is the one updating it.
2168 	 */
2169 
2170 	/* double index for 128-byte SQEs, twice as long */
2171 	if (ctx->flags & IORING_SETUP_SQE128)
2172 		head <<= 1;
2173 	*sqe = &ctx->sq_sqes[head];
2174 	return true;
2175 }
2176 
2177 int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
2178 	__must_hold(&ctx->uring_lock)
2179 {
2180 	unsigned int entries;
2181 	unsigned int left;
2182 	int ret;
2183 
2184 	if (ctx->flags & IORING_SETUP_SQ_REWIND)
2185 		entries = ctx->sq_entries;
2186 	else
2187 		entries = __io_sqring_entries(ctx);
2188 
2189 	entries = min(nr, entries);
2190 	if (unlikely(!entries))
2191 		return 0;
2192 
2193 	ret = left = entries;
2194 	io_get_task_refs(left);
2195 	io_submit_state_start(&ctx->submit_state, left);
2196 
2197 	do {
2198 		const struct io_uring_sqe *sqe;
2199 		struct io_kiocb *req;
2200 
2201 		if (unlikely(!io_alloc_req(ctx, &req)))
2202 			break;
2203 		if (unlikely(!io_get_sqe(ctx, &sqe))) {
2204 			io_req_add_to_cache(req, ctx);
2205 			break;
2206 		}
2207 
2208 		/*
2209 		 * Continue submitting even for sqe failure if the
2210 		 * ring was setup with IORING_SETUP_SUBMIT_ALL
2211 		 */
2212 		if (unlikely(io_submit_sqe(ctx, req, sqe, &left)) &&
2213 		    !(ctx->flags & IORING_SETUP_SUBMIT_ALL)) {
2214 			left--;
2215 			break;
2216 		}
2217 	} while (--left);
2218 
2219 	if (unlikely(left)) {
2220 		ret -= left;
2221 		/* try again if it submitted nothing and can't allocate a req */
2222 		if (!ret && io_req_cache_empty(ctx))
2223 			ret = -EAGAIN;
2224 		current->io_uring->cached_refs += left;
2225 	}
2226 
2227 	io_submit_state_end(ctx);
2228 	 /* Commit SQ ring head once we've consumed and submitted all SQEs */
2229 	io_commit_sqring(ctx);
2230 	return ret;
2231 }
2232 
2233 static void io_rings_free(struct io_ring_ctx *ctx)
2234 {
2235 	/* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */
2236 	io_free_region(NULL, &ctx->sq_region);
2237 	io_free_region(NULL, &ctx->ring_region);
2238 	ctx->rings = NULL;
2239 	RCU_INIT_POINTER(ctx->rings_rcu, NULL);
2240 	ctx->sq_sqes = NULL;
2241 }
2242 
2243 static int rings_size(unsigned int flags, unsigned int sq_entries,
2244 		      unsigned int cq_entries, struct io_rings_layout *rl)
2245 {
2246 	struct io_rings *rings;
2247 	size_t sqe_size;
2248 	size_t off;
2249 
2250 	if (flags & IORING_SETUP_CQE_MIXED) {
2251 		if (cq_entries < 2)
2252 			return -EOVERFLOW;
2253 	}
2254 	if (flags & IORING_SETUP_SQE_MIXED) {
2255 		if (sq_entries < 2)
2256 			return -EOVERFLOW;
2257 	}
2258 
2259 	rl->sq_array_offset = SIZE_MAX;
2260 
2261 	sqe_size = sizeof(struct io_uring_sqe);
2262 	if (flags & IORING_SETUP_SQE128)
2263 		sqe_size *= 2;
2264 
2265 	rl->sq_size = array_size(sqe_size, sq_entries);
2266 	if (rl->sq_size == SIZE_MAX)
2267 		return -EOVERFLOW;
2268 
2269 	off = struct_size(rings, cqes, cq_entries);
2270 	if (flags & IORING_SETUP_CQE32)
2271 		off = size_mul(off, 2);
2272 	if (off == SIZE_MAX)
2273 		return -EOVERFLOW;
2274 
2275 #ifdef CONFIG_SMP
2276 	off = ALIGN(off, SMP_CACHE_BYTES);
2277 	if (off == 0)
2278 		return -EOVERFLOW;
2279 #endif
2280 
2281 	if (!(flags & IORING_SETUP_NO_SQARRAY)) {
2282 		size_t sq_array_size;
2283 
2284 		rl->sq_array_offset = off;
2285 
2286 		sq_array_size = array_size(sizeof(u32), sq_entries);
2287 		off = size_add(off, sq_array_size);
2288 		if (off == SIZE_MAX)
2289 			return -EOVERFLOW;
2290 	}
2291 
2292 	rl->rings_size = off;
2293 	return 0;
2294 }
2295 
2296 static __cold void __io_req_caches_free(struct io_ring_ctx *ctx)
2297 {
2298 	struct io_kiocb *req;
2299 	int nr = 0;
2300 
2301 	while (!io_req_cache_empty(ctx)) {
2302 		req = io_extract_req(ctx);
2303 		io_poison_req(req);
2304 		kmem_cache_free(req_cachep, req);
2305 		nr++;
2306 	}
2307 	if (nr) {
2308 		ctx->nr_req_allocated -= nr;
2309 		percpu_ref_put_many(&ctx->refs, nr);
2310 	}
2311 }
2312 
2313 static __cold void io_req_caches_free(struct io_ring_ctx *ctx)
2314 {
2315 	guard(mutex)(&ctx->uring_lock);
2316 	__io_req_caches_free(ctx);
2317 }
2318 
2319 static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
2320 {
2321 	io_unregister_bpf_ops(ctx);
2322 	io_sq_thread_finish(ctx);
2323 
2324 	mutex_lock(&ctx->uring_lock);
2325 	io_unregister_zcrx(ctx);
2326 	io_cqring_overflow_kill(ctx);
2327 	io_eventfd_unregister(ctx);
2328 	io_free_alloc_caches(ctx);
2329 	io_destroy_buffers(ctx);
2330 	io_free_region(ctx->user, &ctx->param_region);
2331 	mutex_unlock(&ctx->uring_lock);
2332 	if (ctx->sq_creds)
2333 		put_cred(ctx->sq_creds);
2334 	if (ctx->submitter_task)
2335 		put_task_struct(ctx->submitter_task);
2336 
2337 	WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list));
2338 
2339 	if (ctx->mm_account) {
2340 		mmdrop(ctx->mm_account);
2341 		ctx->mm_account = NULL;
2342 	}
2343 	io_rings_free(ctx);
2344 
2345 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
2346 		static_branch_slow_dec_deferred(&io_key_has_sqarray);
2347 
2348 	percpu_ref_exit(&ctx->refs);
2349 	free_uid(ctx->user);
2350 	io_req_caches_free(ctx);
2351 
2352 	if (ctx->restrictions.bpf_filters) {
2353 		WARN_ON_ONCE(ctx->bpf_filters !=
2354 			     ctx->restrictions.bpf_filters->filters);
2355 	} else {
2356 		WARN_ON_ONCE(ctx->bpf_filters);
2357 	}
2358 	io_put_bpf_filters(&ctx->restrictions);
2359 
2360 	WARN_ON_ONCE(ctx->nr_req_allocated);
2361 
2362 	if (ctx->hash_map)
2363 		io_wq_put_hash(ctx->hash_map);
2364 	io_napi_free(ctx);
2365 	kvfree(ctx->cancel_table.hbs);
2366 	xa_destroy(&ctx->io_bl_xa);
2367 	xa_destroy(&ctx->hpage_acct);
2368 	xa_destroy(&ctx->personalities);
2369 	kfree(ctx);
2370 }
2371 
2372 static __cold void io_activate_pollwq_cb(struct callback_head *cb)
2373 {
2374 	struct io_ring_ctx *ctx = container_of(cb, struct io_ring_ctx,
2375 					       poll_wq_task_work);
2376 
2377 	mutex_lock(&ctx->uring_lock);
2378 	ctx->int_flags |= IO_RING_F_POLL_ACTIVATED;
2379 	mutex_unlock(&ctx->uring_lock);
2380 
2381 	/*
2382 	 * Wake ups for some events between start of polling and activation
2383 	 * might've been lost due to loose synchronisation.
2384 	 */
2385 	wake_up_all(&ctx->poll_wq);
2386 	percpu_ref_put(&ctx->refs);
2387 }
2388 
2389 __cold void io_activate_pollwq(struct io_ring_ctx *ctx)
2390 {
2391 	spin_lock(&ctx->completion_lock);
2392 	/* already activated or in progress */
2393 	if ((ctx->int_flags & IO_RING_F_POLL_ACTIVATED) || ctx->poll_wq_task_work.func)
2394 		goto out;
2395 	if (WARN_ON_ONCE(!(ctx->int_flags & IO_RING_F_TASK_COMPLETE)))
2396 		goto out;
2397 	if (!ctx->submitter_task)
2398 		goto out;
2399 	/*
2400 	 * with ->submitter_task only the submitter task completes requests, we
2401 	 * only need to sync with it, which is done by injecting a tw
2402 	 */
2403 	init_task_work(&ctx->poll_wq_task_work, io_activate_pollwq_cb);
2404 	percpu_ref_get(&ctx->refs);
2405 	if (task_work_add(ctx->submitter_task, &ctx->poll_wq_task_work, TWA_SIGNAL))
2406 		percpu_ref_put(&ctx->refs);
2407 out:
2408 	spin_unlock(&ctx->completion_lock);
2409 }
2410 
2411 static __poll_t io_uring_poll(struct file *file, poll_table *wait)
2412 {
2413 	struct io_ring_ctx *ctx = file->private_data;
2414 	__poll_t mask = 0;
2415 
2416 	if (unlikely(!(data_race(ctx->int_flags) & IO_RING_F_POLL_ACTIVATED)))
2417 		io_activate_pollwq(ctx);
2418 	/*
2419 	 * provides mb() which pairs with barrier from wq_has_sleeper
2420 	 * call in io_commit_cqring
2421 	 */
2422 	poll_wait(file, &ctx->poll_wq, wait);
2423 
2424 	rcu_read_lock();
2425 
2426 	if (!__io_sqring_full(ctx))
2427 		mask |= EPOLLOUT | EPOLLWRNORM;
2428 
2429 	/*
2430 	 * Don't flush cqring overflow list here, just do a simple check.
2431 	 * Otherwise there could possible be ABBA deadlock:
2432 	 *      CPU0                    CPU1
2433 	 *      ----                    ----
2434 	 * lock(&ctx->uring_lock);
2435 	 *                              lock(&ep->mtx);
2436 	 *                              lock(&ctx->uring_lock);
2437 	 * lock(&ep->mtx);
2438 	 *
2439 	 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
2440 	 * pushes them to do the flush.
2441 	 */
2442 
2443 	if (__io_cqring_events_user(ctx) || io_has_work(ctx))
2444 		mask |= EPOLLIN | EPOLLRDNORM;
2445 
2446 	rcu_read_unlock();
2447 	return mask;
2448 }
2449 
2450 struct io_tctx_exit {
2451 	struct callback_head		task_work;
2452 	struct completion		completion;
2453 	struct io_ring_ctx		*ctx;
2454 	bool				in_cancel;
2455 };
2456 
2457 static __cold void io_tctx_exit_cb(struct callback_head *cb)
2458 {
2459 	struct io_uring_task *tctx = current->io_uring;
2460 	struct io_tctx_exit *work;
2461 
2462 	work = container_of(cb, struct io_tctx_exit, task_work);
2463 	/*
2464 	 * When @in_cancel, we're in cancellation and it's racy to remove the
2465 	 * node. It'll be removed by the end of cancellation, just ignore it.
2466 	 * tctx can be NULL if the queueing of this task_work raced with
2467 	 * work cancelation off the exec path.
2468 	 */
2469 	work->in_cancel = tctx && atomic_read(&tctx->in_cancel);
2470 	if (tctx && !work->in_cancel)
2471 		io_uring_del_tctx_node((unsigned long)work->ctx);
2472 	complete(&work->completion);
2473 }
2474 
2475 /*
2476  * Cancel what can be canceled on a dying ring and reap what has completed.
2477  * Only waits on polled I/O, never anything else.
2478  */
2479 static __cold void io_ring_ctx_cancel(struct io_ring_ctx *ctx)
2480 {
2481 	struct io_sq_data *sqd = ctx->sq_data;
2482 
2483 	if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) {
2484 		scoped_guard(mutex, &ctx->uring_lock)
2485 			io_cqring_overflow_kill(ctx);
2486 	}
2487 
2488 	/*
2489 	 * If we're doing polled IO and end up having requests being
2490 	 * submitted async (out-of-line), then completions can come in while
2491 	 * we're waiting for refs to drop. We need to reap these manually,
2492 	 * as nobody else will be looking for them.
2493 	 */
2494 	do {
2495 		if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
2496 			io_cancel_local_task_work(ctx);
2497 		cond_resched();
2498 	} while (io_uring_try_cancel_requests(ctx, NULL, IO_CANCEL_ALL));
2499 
2500 	if (sqd) {
2501 		struct task_struct *tsk;
2502 
2503 		io_sq_thread_park(sqd);
2504 		tsk = sqpoll_task_locked(sqd);
2505 		if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
2506 			io_wq_cancel_cb(tsk->io_uring->io_wq,
2507 					io_cancel_ctx_cb, ctx, true);
2508 		io_sq_thread_unpark(sqd);
2509 	}
2510 
2511 	io_req_caches_free(ctx);
2512 }
2513 
2514 /* Number of requests that should be waited for */
2515 static __cold unsigned int io_ring_ctx_inflight(struct io_ring_ctx *ctx)
2516 {
2517 	guard(mutex)(&ctx->uring_lock);
2518 	__io_req_caches_free(ctx);
2519 	return ctx->nr_req_allocated - ctx->nr_notifs;
2520 }
2521 
2522 /*
2523  * Run task_work completions for current. Only do so if the io_uring callback
2524  * itself can get pruned first, otherwise we risk recursing.
2525  */
2526 static __cold bool io_ring_run_own_completions(struct io_uring_task *tctx)
2527 {
2528 	unsigned int count = 0;
2529 
2530 	if (!tctx || mpscq_empty(&tctx->task_list))
2531 		return true;
2532 	if (!task_work_cancel(current, &tctx->task_work))
2533 		return false;
2534 	tctx_task_work_run(tctx, UINT_MAX, &count);
2535 	return true;
2536 }
2537 
2538 /*
2539  * Requests may remain after cancelations have been run, as not all requests
2540  * are cancelable. Storage I/O is an example. Wait for those so that once
2541  * close(2) returns, files pinned by these requests have been released.
2542  */
2543 static __cold void io_ring_ctx_wait_inflight(struct io_ring_ctx *ctx)
2544 {
2545 	struct io_uring_task *tctx = current->io_uring;
2546 	bool ran_own = true;
2547 
2548 	if (current->flags & (PF_KTHREAD | PF_EXITING))
2549 		return;
2550 	/* Only wait if this task has attached to the ring */
2551 	if (!tctx || !xa_load(&tctx->xa, (unsigned long)ctx))
2552 		return;
2553 	if (atomic_read(&tctx->in_cancel))
2554 		return;
2555 
2556 	while (io_ring_ctx_inflight(ctx) && !fatal_signal_pending(current)) {
2557 		unsigned int state;
2558 
2559 		if (test_thread_flag(TIF_NOTIFY_SIGNAL)) {
2560 			clear_notify_signal();
2561 			if (task_work_pending(current))
2562 				set_notify_resume(current);
2563 		}
2564 		state = TASK_INTERRUPTIBLE;
2565 		if (signal_pending(current))
2566 			state = TASK_KILLABLE;
2567 		set_current_state(state | TASK_FREEZABLE);
2568 		/* don't sleep on work that's already there and that we can run */
2569 		if (ran_own && ((tctx && !mpscq_empty(&tctx->task_list)) ||
2570 		    io_local_work_pending(ctx)))
2571 			__set_current_state(TASK_RUNNING);
2572 		else
2573 			schedule_timeout(1);
2574 
2575 		/* completions may be queued behind us */
2576 		if (!io_ring_run_own_completions(tctx)) {
2577 			if (!ran_own)
2578 				break;
2579 			ran_own = false;
2580 		} else {
2581 			ran_own = true;
2582 		}
2583 		io_ring_ctx_cancel(ctx);
2584 	}
2585 }
2586 
2587 static __cold void io_ring_exit_work(struct work_struct *work)
2588 {
2589 	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
2590 	unsigned long timeout = jiffies + IO_URING_EXIT_WAIT_MAX;
2591 	unsigned long interval = HZ / 20;
2592 	struct io_tctx_exit exit;
2593 	struct io_tctx_node *node;
2594 	int ret;
2595 
2596 	do {
2597 		io_ring_ctx_cancel(ctx);
2598 
2599 		if (WARN_ON_ONCE(time_after(jiffies, timeout))) {
2600 			/* there is little hope left, don't run it too often */
2601 			interval = HZ * 60;
2602 		}
2603 		/*
2604 		 * This is really an uninterruptible wait, as it has to be
2605 		 * complete. But it's also run from a kworker, which doesn't
2606 		 * take signals, so it's fine to make it interruptible. This
2607 		 * avoids scenarios where we knowingly can wait much longer
2608 		 * on completions, for example if someone does a SIGSTOP on
2609 		 * a task that needs to finish task_work to make this loop
2610 		 * complete. That's a synthetic situation that should not
2611 		 * cause a stuck task backtrace, and hence a potential panic
2612 		 * on stuck tasks if that is enabled.
2613 		 */
2614 	} while (!wait_for_completion_interruptible_timeout(&ctx->ref_comp, interval));
2615 
2616 	init_completion(&exit.completion);
2617 	init_task_work(&exit.task_work, io_tctx_exit_cb);
2618 	exit.ctx = ctx;
2619 
2620 	mutex_lock(&ctx->uring_lock);
2621 	mutex_lock(&ctx->tctx_lock);
2622 	while (!list_empty(&ctx->tctx_list)) {
2623 		WARN_ON_ONCE(time_after(jiffies, timeout));
2624 
2625 		node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
2626 					ctx_node);
2627 		/* don't spin on a single task if cancellation failed */
2628 		list_rotate_left(&ctx->tctx_list);
2629 		ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
2630 		if (WARN_ON_ONCE(ret))
2631 			continue;
2632 
2633 		mutex_unlock(&ctx->tctx_lock);
2634 		mutex_unlock(&ctx->uring_lock);
2635 		/*
2636 		 * See comment above for
2637 		 * wait_for_completion_interruptible_timeout() on why this
2638 		 * wait is marked as interruptible.
2639 		 */
2640 		wait_for_completion_interruptible(&exit.completion);
2641 		/*
2642 		 * The task removes the node when it is done with its
2643 		 * cancellation. Give it the time for that: it doesn't get back
2644 		 * to it as long as it finds new task_work each time it has run
2645 		 * the old one, and we may be one of many who send it.
2646 		 */
2647 		if (exit.in_cancel)
2648 			schedule_timeout_idle(HZ / 20);
2649 		mutex_lock(&ctx->uring_lock);
2650 		mutex_lock(&ctx->tctx_lock);
2651 	}
2652 	mutex_unlock(&ctx->tctx_lock);
2653 	mutex_unlock(&ctx->uring_lock);
2654 	spin_lock(&ctx->completion_lock);
2655 	spin_unlock(&ctx->completion_lock);
2656 
2657 	/* pairs with the RCU read sections in the task_work add paths */
2658 	if (ctx->flags & (IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SQPOLL))
2659 		synchronize_rcu();
2660 
2661 	io_ring_ctx_free(ctx);
2662 }
2663 
2664 static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
2665 {
2666 	unsigned long index;
2667 	struct cred *creds;
2668 
2669 	mutex_lock(&ctx->uring_lock);
2670 	percpu_ref_kill(&ctx->refs);
2671 	xa_for_each(&ctx->personalities, index, creds)
2672 		io_unregister_personality(ctx, index);
2673 	io_terminate_zcrx(ctx);
2674 	/* Drop these now, rather than async, to unpin the files */
2675 	io_sqe_buffers_unregister(ctx);
2676 	io_sqe_files_unregister(ctx);
2677 	mutex_unlock(&ctx->uring_lock);
2678 
2679 	/*
2680 	 * Do the first round of cancelations upfront rather than leaving it
2681 	 * to to exit_work. Anything cancelable is then already on its way
2682 	 * out, and for requests owned by the task closing the ring, this
2683 	 * ensures any held files are put before close(2) returns.
2684 	 */
2685 	if (!(current->flags & PF_IO_WORKER)) {
2686 		io_ring_ctx_cancel(ctx);
2687 		io_ring_ctx_wait_inflight(ctx);
2688 	}
2689 
2690 	INIT_WORK(&ctx->exit_work, io_ring_exit_work);
2691 	/*
2692 	 * Use system_dfl_wq to avoid spawning tons of event kworkers
2693 	 * if we're exiting a ton of rings at the same time. It just adds
2694 	 * noise and overhead, there's no discernable change in runtime
2695 	 * over using system_percpu_wq.
2696 	 */
2697 	queue_work(iou_wq, &ctx->exit_work);
2698 }
2699 
2700 static int io_uring_release(struct inode *inode, struct file *file)
2701 {
2702 	struct io_ring_ctx *ctx = file->private_data;
2703 
2704 	file->private_data = NULL;
2705 	io_ring_ctx_wait_and_kill(ctx);
2706 	return 0;
2707 }
2708 
2709 static struct io_uring_reg_wait *io_get_ext_arg_reg(struct io_ring_ctx *ctx,
2710 			const struct io_uring_getevents_arg __user *uarg)
2711 {
2712 	unsigned long size = sizeof(struct io_uring_reg_wait);
2713 	unsigned long offset = (uintptr_t)uarg;
2714 	unsigned long end;
2715 
2716 	if (unlikely(offset % sizeof(long)))
2717 		return ERR_PTR(-EFAULT);
2718 
2719 	/* also protects from NULL ->cq_wait_arg as the size would be 0 */
2720 	if (unlikely(check_add_overflow(offset, size, &end) ||
2721 		     end > ctx->cq_wait_size))
2722 		return ERR_PTR(-EFAULT);
2723 
2724 	offset = array_index_nospec(offset, ctx->cq_wait_size - size);
2725 	return ctx->cq_wait_arg + offset;
2726 }
2727 
2728 static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
2729 			       const void __user *argp, size_t argsz)
2730 {
2731 	struct io_uring_getevents_arg arg;
2732 
2733 	if (!(flags & IORING_ENTER_EXT_ARG))
2734 		return 0;
2735 	if (flags & IORING_ENTER_EXT_ARG_REG)
2736 		return -EINVAL;
2737 	if (argsz != sizeof(arg))
2738 		return -EINVAL;
2739 	if (copy_from_user(&arg, argp, sizeof(arg)))
2740 		return -EFAULT;
2741 	return 0;
2742 }
2743 
2744 static int io_get_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
2745 			  const void __user *argp, struct ext_arg *ext_arg)
2746 {
2747 	const struct io_uring_getevents_arg __user *uarg = argp;
2748 	struct io_uring_getevents_arg arg;
2749 
2750 	ext_arg->iowait = !(flags & IORING_ENTER_NO_IOWAIT);
2751 
2752 	/*
2753 	 * If EXT_ARG isn't set, then we have no timespec and the argp pointer
2754 	 * is just a pointer to the sigset_t.
2755 	 */
2756 	if (!(flags & IORING_ENTER_EXT_ARG)) {
2757 		ext_arg->sig = (const sigset_t __user *) argp;
2758 		return 0;
2759 	}
2760 
2761 	if (flags & IORING_ENTER_EXT_ARG_REG) {
2762 		struct io_uring_reg_wait *w;
2763 
2764 		if (ext_arg->argsz != sizeof(struct io_uring_reg_wait))
2765 			return -EINVAL;
2766 		w = io_get_ext_arg_reg(ctx, argp);
2767 		if (IS_ERR(w))
2768 			return PTR_ERR(w);
2769 
2770 		if (w->flags & ~IORING_REG_WAIT_TS)
2771 			return -EINVAL;
2772 		ext_arg->min_time = READ_ONCE(w->min_wait_usec) * NSEC_PER_USEC;
2773 		ext_arg->sig = u64_to_user_ptr(READ_ONCE(w->sigmask));
2774 		ext_arg->argsz = READ_ONCE(w->sigmask_sz);
2775 		if (w->flags & IORING_REG_WAIT_TS) {
2776 			ext_arg->ts.tv_sec = READ_ONCE(w->ts.tv_sec);
2777 			ext_arg->ts.tv_nsec = READ_ONCE(w->ts.tv_nsec);
2778 			ext_arg->ts_set = true;
2779 		}
2780 		return 0;
2781 	}
2782 
2783 	/*
2784 	 * EXT_ARG is set - ensure we agree on the size of it and copy in our
2785 	 * timespec and sigset_t pointers if good.
2786 	 */
2787 	if (ext_arg->argsz != sizeof(arg))
2788 		return -EINVAL;
2789 #ifdef CONFIG_64BIT
2790 	if (!user_access_begin(uarg, sizeof(*uarg)))
2791 		return -EFAULT;
2792 	unsafe_get_user(arg.sigmask, &uarg->sigmask, uaccess_end);
2793 	unsafe_get_user(arg.sigmask_sz, &uarg->sigmask_sz, uaccess_end);
2794 	unsafe_get_user(arg.min_wait_usec, &uarg->min_wait_usec, uaccess_end);
2795 	unsafe_get_user(arg.ts, &uarg->ts, uaccess_end);
2796 	user_access_end();
2797 #else
2798 	if (copy_from_user(&arg, uarg, sizeof(arg)))
2799 		return -EFAULT;
2800 #endif
2801 	ext_arg->min_time = arg.min_wait_usec * NSEC_PER_USEC;
2802 	ext_arg->sig = u64_to_user_ptr(arg.sigmask);
2803 	ext_arg->argsz = arg.sigmask_sz;
2804 	if (arg.ts) {
2805 		if (get_timespec64(&ext_arg->ts, u64_to_user_ptr(arg.ts)))
2806 			return -EFAULT;
2807 		ext_arg->ts_set = true;
2808 	}
2809 	return 0;
2810 #ifdef CONFIG_64BIT
2811 uaccess_end:
2812 	user_access_end();
2813 	return -EFAULT;
2814 #endif
2815 }
2816 
2817 /*
2818  * Given an 'fd' value, return the ctx associated with if. If 'registered' is
2819  * true, then the registered index is used. Otherwise, the normal fd table.
2820  * Caller must call fput() on the returned file if it isn't a registered file,
2821  * unless it's an ERR_PTR.
2822  */
2823 struct file *io_uring_ctx_get_file(unsigned int fd, bool registered)
2824 {
2825 	struct file *file;
2826 
2827 	if (registered) {
2828 		/*
2829 		 * Ring fd has been registered via IORING_REGISTER_RING_FDS, we
2830 		 * need only dereference our task private array to find it.
2831 		 */
2832 		struct io_uring_task *tctx = current->io_uring;
2833 
2834 		if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))
2835 			return ERR_PTR(-EINVAL);
2836 		fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
2837 		file = tctx->registered_rings[fd];
2838 	} else {
2839 		file = fget(fd);
2840 	}
2841 
2842 	if (unlikely(!file))
2843 		return ERR_PTR(-EBADF);
2844 	if (io_is_uring_fops(file))
2845 		return file;
2846 	if (!registered)
2847 		fput(file);
2848 	return ERR_PTR(-EOPNOTSUPP);
2849 }
2850 
2851 
2852 SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
2853 		u32, min_complete, u32, flags, const void __user *, argp,
2854 		size_t, argsz)
2855 {
2856 	struct io_ring_ctx *ctx;
2857 	struct file *file;
2858 	long ret;
2859 
2860 	if (unlikely(flags & ~IORING_ENTER_FLAGS))
2861 		return -EINVAL;
2862 
2863 	file = io_uring_ctx_get_file(fd, flags & IORING_ENTER_REGISTERED_RING);
2864 	if (IS_ERR(file))
2865 		return PTR_ERR(file);
2866 	ctx = file->private_data;
2867 	ret = -EBADFD;
2868 	/*
2869 	 * Keep IORING_SETUP_R_DISABLED check before submitter_task load
2870 	 * in io_uring_add_tctx_node() -> __io_uring_add_tctx_node_from_submit()
2871 	 */
2872 	if (unlikely(smp_load_acquire(&ctx->flags) & IORING_SETUP_R_DISABLED))
2873 		goto out;
2874 
2875 	if (io_has_loop_ops(ctx)) {
2876 		ret = io_run_loop(ctx);
2877 		goto out;
2878 	}
2879 
2880 	/*
2881 	 * For SQ polling, the thread will do all submissions and completions.
2882 	 * Just return the requested submit count, and wake the thread if
2883 	 * we were asked to.
2884 	 */
2885 	ret = 0;
2886 	if (ctx->flags & IORING_SETUP_SQPOLL) {
2887 		if (unlikely(ctx->sq_data->thread == NULL)) {
2888 			ret = -EOWNERDEAD;
2889 			goto out;
2890 		}
2891 		if (flags & IORING_ENTER_SQ_WAKEUP)
2892 			wake_up(&ctx->sq_data->wait);
2893 		if (flags & IORING_ENTER_SQ_WAIT)
2894 			io_sqpoll_wait_sq(ctx);
2895 
2896 		ret = to_submit;
2897 	} else if (to_submit) {
2898 		ret = io_uring_add_tctx_node(ctx);
2899 		if (unlikely(ret))
2900 			goto out;
2901 
2902 		mutex_lock(&ctx->uring_lock);
2903 		ret = io_submit_sqes(ctx, to_submit);
2904 		if (ret != to_submit) {
2905 			mutex_unlock(&ctx->uring_lock);
2906 			goto out;
2907 		}
2908 		if (flags & IORING_ENTER_GETEVENTS) {
2909 			if (ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL)
2910 				goto iopoll_locked;
2911 			/*
2912 			 * Ignore errors, we'll soon call io_cqring_wait() and
2913 			 * it should handle ownership problems if any.
2914 			 */
2915 			if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
2916 				(void)io_run_local_work_locked(ctx, min_complete);
2917 		}
2918 		mutex_unlock(&ctx->uring_lock);
2919 	}
2920 
2921 	if (flags & IORING_ENTER_GETEVENTS) {
2922 		int ret2;
2923 
2924 		if (ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL) {
2925 			/*
2926 			 * We disallow the app entering submit/complete with
2927 			 * polling, but we still need to lock the ring to
2928 			 * prevent racing with polled issue that got punted to
2929 			 * a workqueue.
2930 			 */
2931 			mutex_lock(&ctx->uring_lock);
2932 iopoll_locked:
2933 			ret2 = io_validate_ext_arg(ctx, flags, argp, argsz);
2934 			if (likely(!ret2))
2935 				ret2 = io_iopoll_check(ctx, min_complete);
2936 			mutex_unlock(&ctx->uring_lock);
2937 		} else {
2938 			struct ext_arg ext_arg = { .argsz = argsz };
2939 
2940 			ret2 = io_get_ext_arg(ctx, flags, argp, &ext_arg);
2941 			if (likely(!ret2))
2942 				ret2 = io_cqring_wait(ctx, min_complete, flags,
2943 						      &ext_arg);
2944 		}
2945 
2946 		if (!ret) {
2947 			ret = ret2;
2948 
2949 			/*
2950 			 * EBADR indicates that one or more CQE were dropped.
2951 			 * Once the user has been informed we can clear the bit
2952 			 * as they are obviously ok with those drops.
2953 			 */
2954 			if (unlikely(ret2 == -EBADR))
2955 				clear_bit(IO_CHECK_CQ_DROPPED_BIT,
2956 					  &ctx->check_cq);
2957 		}
2958 	}
2959 out:
2960 	if (!(flags & IORING_ENTER_REGISTERED_RING))
2961 		fput(file);
2962 	return ret;
2963 }
2964 
2965 static const struct file_operations io_uring_fops = {
2966 	.release	= io_uring_release,
2967 	.mmap		= io_uring_mmap,
2968 	.get_unmapped_area = io_uring_get_unmapped_area,
2969 #ifndef CONFIG_MMU
2970 	.mmap_capabilities = io_uring_nommu_mmap_capabilities,
2971 #endif
2972 	.poll		= io_uring_poll,
2973 #ifdef CONFIG_PROC_FS
2974 	.show_fdinfo	= io_uring_show_fdinfo,
2975 #endif
2976 };
2977 
2978 bool io_is_uring_fops(struct file *file)
2979 {
2980 	return file->f_op == &io_uring_fops;
2981 }
2982 
2983 static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
2984 					 struct io_ctx_config *config)
2985 {
2986 	struct io_uring_params *p = &config->p;
2987 	struct io_rings_layout *rl = &config->layout;
2988 	struct io_uring_region_desc rd;
2989 	struct io_rings *rings;
2990 	int ret;
2991 
2992 	/* make sure these are sane, as we already accounted them */
2993 	ctx->sq_entries = p->sq_entries;
2994 	ctx->cq_entries = p->cq_entries;
2995 
2996 	memset(&rd, 0, sizeof(rd));
2997 	rd.size = PAGE_ALIGN(rl->rings_size);
2998 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
2999 		rd.user_addr = p->cq_off.user_addr;
3000 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
3001 	}
3002 	/* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */
3003 	ret = io_create_region(NULL, &ctx->ring_region, &rd, IORING_OFF_CQ_RING);
3004 	if (ret)
3005 		return ret;
3006 	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
3007 	rcu_assign_pointer(ctx->rings_rcu, rings);
3008 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
3009 		ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
3010 
3011 	memset(&rd, 0, sizeof(rd));
3012 	rd.size = PAGE_ALIGN(rl->sq_size);
3013 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
3014 		rd.user_addr = p->sq_off.user_addr;
3015 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
3016 	}
3017 	ret = io_create_region(NULL, &ctx->sq_region, &rd, IORING_OFF_SQES);
3018 	if (ret) {
3019 		io_rings_free(ctx);
3020 		return ret;
3021 	}
3022 	ctx->sq_sqes = io_region_get_ptr(&ctx->sq_region);
3023 
3024 	memset(rings, 0, sizeof(*rings));
3025 	WRITE_ONCE(rings->sq_ring_mask, ctx->sq_entries - 1);
3026 	WRITE_ONCE(rings->cq_ring_mask, ctx->cq_entries - 1);
3027 	WRITE_ONCE(rings->sq_ring_entries, ctx->sq_entries);
3028 	WRITE_ONCE(rings->cq_ring_entries, ctx->cq_entries);
3029 	return 0;
3030 }
3031 
3032 static int io_uring_install_fd(struct file *file)
3033 {
3034 	int fd;
3035 
3036 	fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
3037 	if (fd < 0)
3038 		return fd;
3039 	fd_install(fd, file);
3040 	return fd;
3041 }
3042 
3043 /*
3044  * Allocate an anonymous fd, this is what constitutes the application
3045  * visible backing of an io_uring instance. The application mmaps this
3046  * fd to gain access to the SQ/CQ ring details.
3047  */
3048 static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
3049 {
3050 	/* Create a new inode so that the LSM can block the creation.  */
3051 	return anon_inode_create_getfile("[io_uring]", &io_uring_fops, ctx,
3052 					 O_RDWR | O_CLOEXEC, NULL);
3053 }
3054 
3055 static int io_uring_sanitise_params(struct io_uring_params *p)
3056 {
3057 	unsigned flags = p->flags;
3058 
3059 	if (flags & ~IORING_SETUP_FLAGS)
3060 		return -EINVAL;
3061 
3062 	if (flags & IORING_SETUP_SQ_REWIND) {
3063 		if ((flags & IORING_SETUP_SQPOLL) ||
3064 		    !(flags & IORING_SETUP_NO_SQARRAY))
3065 			return -EINVAL;
3066 	}
3067 
3068 	/* There is no way to mmap rings without a real fd */
3069 	if ((flags & IORING_SETUP_REGISTERED_FD_ONLY) &&
3070 	    !(flags & IORING_SETUP_NO_MMAP))
3071 		return -EINVAL;
3072 
3073 	if (flags & IORING_SETUP_SQPOLL) {
3074 		/* IPI related flags don't make sense with SQPOLL */
3075 		if (flags & (IORING_SETUP_COOP_TASKRUN |
3076 			     IORING_SETUP_TASKRUN_FLAG |
3077 			     IORING_SETUP_DEFER_TASKRUN))
3078 			return -EINVAL;
3079 	}
3080 
3081 	if (flags & IORING_SETUP_TASKRUN_FLAG) {
3082 		if (!(flags & (IORING_SETUP_COOP_TASKRUN |
3083 			       IORING_SETUP_DEFER_TASKRUN)))
3084 			return -EINVAL;
3085 	}
3086 
3087 	/* HYBRID_IOPOLL only valid with IOPOLL */
3088 	if ((flags & IORING_SETUP_HYBRID_IOPOLL) && !(flags & IORING_SETUP_IOPOLL))
3089 		return -EINVAL;
3090 
3091 	/*
3092 	 * For DEFER_TASKRUN we require the completion task to be the same as
3093 	 * the submission task. This implies that there is only one submitter.
3094 	 */
3095 	if ((flags & IORING_SETUP_DEFER_TASKRUN) &&
3096 	    !(flags & IORING_SETUP_SINGLE_ISSUER))
3097 		return -EINVAL;
3098 
3099 	/*
3100 	 * Nonsensical to ask for CQE32 and mixed CQE support, it's not
3101 	 * supported to post 16b CQEs on a ring setup with CQE32.
3102 	 */
3103 	if ((flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)) ==
3104 	    (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))
3105 		return -EINVAL;
3106 	/*
3107 	 * Nonsensical to ask for SQE128 and mixed SQE support, it's not
3108 	 * supported to post 64b SQEs on a ring setup with SQE128.
3109 	 */
3110 	if ((flags & (IORING_SETUP_SQE128|IORING_SETUP_SQE_MIXED)) ==
3111 	    (IORING_SETUP_SQE128|IORING_SETUP_SQE_MIXED))
3112 		return -EINVAL;
3113 
3114 	return 0;
3115 }
3116 
3117 static int io_uring_fill_params(struct io_uring_params *p)
3118 {
3119 	unsigned entries = p->sq_entries;
3120 
3121 	if (!entries)
3122 		return -EINVAL;
3123 	if (entries > IORING_MAX_ENTRIES) {
3124 		if (!(p->flags & IORING_SETUP_CLAMP))
3125 			return -EINVAL;
3126 		entries = IORING_MAX_ENTRIES;
3127 	}
3128 
3129 	/*
3130 	 * Use twice as many entries for the CQ ring. It's possible for the
3131 	 * application to drive a higher depth than the size of the SQ ring,
3132 	 * since the sqes are only used at submission time. This allows for
3133 	 * some flexibility in overcommitting a bit. If the application has
3134 	 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
3135 	 * of CQ ring entries manually.
3136 	 */
3137 	p->sq_entries = roundup_pow_of_two(entries);
3138 	if (p->flags & IORING_SETUP_CQSIZE) {
3139 		/*
3140 		 * If IORING_SETUP_CQSIZE is set, we do the same roundup
3141 		 * to a power-of-two, if it isn't already. We do NOT impose
3142 		 * any cq vs sq ring sizing.
3143 		 */
3144 		if (!p->cq_entries)
3145 			return -EINVAL;
3146 		if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
3147 			if (!(p->flags & IORING_SETUP_CLAMP))
3148 				return -EINVAL;
3149 			p->cq_entries = IORING_MAX_CQ_ENTRIES;
3150 		}
3151 		p->cq_entries = roundup_pow_of_two(p->cq_entries);
3152 		if (p->cq_entries < p->sq_entries)
3153 			return -EINVAL;
3154 	} else {
3155 		p->cq_entries = 2 * p->sq_entries;
3156 	}
3157 
3158 	return 0;
3159 }
3160 
3161 int io_prepare_config(struct io_ctx_config *config)
3162 {
3163 	struct io_uring_params *p = &config->p;
3164 	int ret;
3165 
3166 	ret = io_uring_sanitise_params(p);
3167 	if (ret)
3168 		return ret;
3169 
3170 	ret = io_uring_fill_params(p);
3171 	if (ret)
3172 		return ret;
3173 
3174 	ret = rings_size(p->flags, p->sq_entries, p->cq_entries,
3175 			 &config->layout);
3176 	if (ret)
3177 		return ret;
3178 
3179 	p->sq_off.head = offsetof(struct io_rings, sq.head);
3180 	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
3181 	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
3182 	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
3183 	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
3184 	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
3185 	p->sq_off.resv1 = 0;
3186 	if (!(p->flags & IORING_SETUP_NO_MMAP))
3187 		p->sq_off.user_addr = 0;
3188 
3189 	p->cq_off.head = offsetof(struct io_rings, cq.head);
3190 	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
3191 	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
3192 	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
3193 	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
3194 	p->cq_off.cqes = offsetof(struct io_rings, cqes);
3195 	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
3196 	p->cq_off.resv1 = 0;
3197 	if (!(p->flags & IORING_SETUP_NO_MMAP))
3198 		p->cq_off.user_addr = 0;
3199 	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
3200 		p->sq_off.array = config->layout.sq_array_offset;
3201 
3202 	return 0;
3203 }
3204 
3205 void io_restriction_clone(struct io_restriction *dst, struct io_restriction *src)
3206 {
3207 	memcpy(&dst->register_op, &src->register_op, sizeof(dst->register_op));
3208 	memcpy(&dst->sqe_op, &src->sqe_op, sizeof(dst->sqe_op));
3209 	dst->sqe_flags_allowed = src->sqe_flags_allowed;
3210 	dst->sqe_flags_required = src->sqe_flags_required;
3211 	dst->op_registered = src->op_registered;
3212 	dst->reg_registered = src->reg_registered;
3213 
3214 	io_bpf_filter_clone(dst, src);
3215 }
3216 
3217 static void io_ctx_restriction_clone(struct io_ring_ctx *ctx,
3218 				     struct io_restriction *src)
3219 {
3220 	struct io_restriction *dst = &ctx->restrictions;
3221 
3222 	io_restriction_clone(dst, src);
3223 	if (dst->bpf_filters)
3224 		WRITE_ONCE(ctx->bpf_filters, dst->bpf_filters->filters);
3225 	if (dst->op_registered)
3226 		ctx->int_flags |= IO_RING_F_OP_RESTRICTED;
3227 	if (dst->reg_registered)
3228 		ctx->int_flags |= IO_RING_F_REG_RESTRICTED |
3229 				  IO_RING_F_RESTRICT_INHERITED;
3230 }
3231 
3232 static __cold int io_uring_create(struct io_ctx_config *config)
3233 {
3234 	struct io_uring_params *p = &config->p;
3235 	struct io_ring_ctx *ctx;
3236 	struct io_uring_task *tctx;
3237 	struct file *file;
3238 	int ret;
3239 
3240 	ret = io_prepare_config(config);
3241 	if (ret)
3242 		return ret;
3243 
3244 	ctx = io_ring_ctx_alloc(p);
3245 	if (!ctx)
3246 		return -ENOMEM;
3247 
3248 	ctx->clockid = CLOCK_MONOTONIC;
3249 	ctx->clock_offset = 0;
3250 
3251 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
3252 		static_branch_deferred_inc(&io_key_has_sqarray);
3253 
3254 	if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
3255 	    !(ctx->flags & IORING_SETUP_IOPOLL))
3256 		ctx->int_flags |= IO_RING_F_TASK_COMPLETE;
3257 
3258 	if ((ctx->int_flags & IO_RING_F_TASK_COMPLETE) ||
3259 	    (ctx->flags & IORING_SETUP_IOPOLL))
3260 		ctx->int_flags |= IO_RING_F_LOCKLESS_CQ;
3261 
3262 	/*
3263 	 * lazy poll_wq activation relies on ->task_complete for synchronisation
3264 	 * purposes, see io_activate_pollwq()
3265 	 */
3266 	if (!(ctx->int_flags & IO_RING_F_TASK_COMPLETE))
3267 		ctx->int_flags |= IO_RING_F_POLL_ACTIVATED;
3268 
3269 	/*
3270 	 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
3271 	 * space applications don't need to do io completion events
3272 	 * polling again, they can rely on io_sq_thread to do polling
3273 	 * work, which can reduce cpu usage and uring_lock contention.
3274 	 */
3275 	if (ctx->flags & IORING_SETUP_IOPOLL &&
3276 	    !(ctx->flags & IORING_SETUP_SQPOLL))
3277 		ctx->int_flags |= IO_RING_F_SYSCALL_IOPOLL;
3278 
3279 	if (in_compat_syscall())
3280 		ctx->int_flags |= IO_RING_F_COMPAT;
3281 	if (!ns_capable_noaudit(&init_user_ns, CAP_IPC_LOCK))
3282 		ctx->user = get_uid(current_user());
3283 
3284 	/*
3285 	 * For SQPOLL, we just need a wakeup, always. For !SQPOLL, if
3286 	 * COOP_TASKRUN is set, then IPIs are never needed by the app.
3287 	 */
3288 	if (ctx->flags & (IORING_SETUP_SQPOLL|IORING_SETUP_COOP_TASKRUN))
3289 		ctx->notify_method = TWA_SIGNAL_NO_IPI;
3290 	else
3291 		ctx->notify_method = TWA_SIGNAL;
3292 
3293 	/*
3294 	 * If the current task has restrictions enabled, then copy them to
3295 	 * our newly created ring and mark it as registered.
3296 	 */
3297 	if (current->io_uring_restrict)
3298 		io_ctx_restriction_clone(ctx, current->io_uring_restrict);
3299 
3300 	/*
3301 	 * This is just grabbed for accounting purposes. When a process exits,
3302 	 * the mm is exited and dropped before the files, hence we need to hang
3303 	 * on to this mm purely for the purposes of being able to unaccount
3304 	 * memory (locked/pinned vm). It's not used for anything else.
3305 	 */
3306 	mmgrab(current->mm);
3307 	ctx->mm_account = current->mm;
3308 
3309 	ret = io_allocate_scq_urings(ctx, config);
3310 	if (ret)
3311 		goto err;
3312 
3313 	ret = io_sq_offload_create(ctx, p);
3314 	if (ret)
3315 		goto err;
3316 
3317 	p->features = IORING_FEAT_FLAGS;
3318 
3319 	if (copy_to_user(config->uptr, p, sizeof(*p))) {
3320 		ret = -EFAULT;
3321 		goto err;
3322 	}
3323 
3324 	if (ctx->flags & IORING_SETUP_SINGLE_ISSUER
3325 	    && !(ctx->flags & IORING_SETUP_R_DISABLED))
3326 		ctx->submitter_task = get_task_struct(current);
3327 
3328 	file = io_uring_get_file(ctx);
3329 	if (IS_ERR(file)) {
3330 		ret = PTR_ERR(file);
3331 		goto err;
3332 	}
3333 
3334 	ret = __io_uring_add_tctx_node(ctx);
3335 	if (ret)
3336 		goto err_fput;
3337 	tctx = current->io_uring;
3338 
3339 	/*
3340 	 * Install ring fd as the very last thing, so we don't risk someone
3341 	 * having closed it before we finish setup
3342 	 */
3343 	if (p->flags & IORING_SETUP_REGISTERED_FD_ONLY)
3344 		ret = io_ring_add_registered_file(tctx, file, 0, IO_RINGFD_REG_MAX);
3345 	else
3346 		ret = io_uring_install_fd(file);
3347 	if (ret < 0)
3348 		goto err_fput;
3349 
3350 	trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
3351 	return ret;
3352 err:
3353 	io_ring_ctx_wait_and_kill(ctx);
3354 	return ret;
3355 err_fput:
3356 	fput(file);
3357 	return ret;
3358 }
3359 
3360 /*
3361  * Sets up an aio uring context, and returns the fd. Applications asks for a
3362  * ring size, we return the actual sq/cq ring sizes (among other things) in the
3363  * params structure passed in.
3364  */
3365 static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
3366 {
3367 	struct io_ctx_config config;
3368 
3369 	memset(&config, 0, sizeof(config));
3370 
3371 	if (copy_from_user(&config.p, params, sizeof(config.p)))
3372 		return -EFAULT;
3373 
3374 	if (!mem_is_zero(&config.p.resv, sizeof(config.p.resv)))
3375 		return -EINVAL;
3376 
3377 	config.p.sq_entries = entries;
3378 	config.uptr = params;
3379 	return io_uring_create(&config);
3380 }
3381 
3382 static inline int io_uring_allowed(void)
3383 {
3384 	int disabled = READ_ONCE(sysctl_io_uring_disabled);
3385 	kgid_t io_uring_group;
3386 
3387 	if (disabled == 2)
3388 		return -EPERM;
3389 
3390 	if (disabled == 0 || capable(CAP_SYS_ADMIN))
3391 		goto allowed_lsm;
3392 
3393 	io_uring_group = make_kgid(&init_user_ns, sysctl_io_uring_group);
3394 	if (!gid_valid(io_uring_group))
3395 		return -EPERM;
3396 
3397 	if (!in_group_p(io_uring_group))
3398 		return -EPERM;
3399 
3400 allowed_lsm:
3401 	return security_uring_allowed();
3402 }
3403 
3404 SYSCALL_DEFINE2(io_uring_setup, u32, entries,
3405 		struct io_uring_params __user *, params)
3406 {
3407 	int ret;
3408 
3409 	ret = io_uring_allowed();
3410 	if (ret)
3411 		return ret;
3412 
3413 	return io_uring_setup(entries, params);
3414 }
3415 
3416 static int __init io_uring_init(void)
3417 {
3418 	struct kmem_cache_args kmem_args = {
3419 		.useroffset = offsetof(struct io_kiocb, cmd.data),
3420 		.usersize = sizeof_field(struct io_kiocb, cmd.data),
3421 		.freeptr_offset = offsetof(struct io_kiocb, work),
3422 		.use_freeptr_offset = true,
3423 	};
3424 
3425 #define __BUILD_BUG_VERIFY_OFFSET_SIZE(stype, eoffset, esize, ename) do { \
3426 	BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
3427 	BUILD_BUG_ON(sizeof_field(stype, ename) != esize); \
3428 } while (0)
3429 
3430 #define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
3431 	__BUILD_BUG_VERIFY_OFFSET_SIZE(struct io_uring_sqe, eoffset, sizeof(etype), ename)
3432 #define BUILD_BUG_SQE_ELEM_SIZE(eoffset, esize, ename) \
3433 	__BUILD_BUG_VERIFY_OFFSET_SIZE(struct io_uring_sqe, eoffset, esize, ename)
3434 	BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
3435 	BUILD_BUG_SQE_ELEM(0,  __u8,   opcode);
3436 	BUILD_BUG_SQE_ELEM(1,  __u8,   flags);
3437 	BUILD_BUG_SQE_ELEM(2,  __u16,  ioprio);
3438 	BUILD_BUG_SQE_ELEM(4,  __s32,  fd);
3439 	BUILD_BUG_SQE_ELEM(8,  __u64,  off);
3440 	BUILD_BUG_SQE_ELEM(8,  __u64,  addr2);
3441 	BUILD_BUG_SQE_ELEM(8,  __u32,  cmd_op);
3442 	BUILD_BUG_SQE_ELEM(12, __u32, __pad1);
3443 	BUILD_BUG_SQE_ELEM(16, __u64,  addr);
3444 	BUILD_BUG_SQE_ELEM(16, __u64,  splice_off_in);
3445 	BUILD_BUG_SQE_ELEM(24, __u32,  len);
3446 	BUILD_BUG_SQE_ELEM(28,     __kernel_rwf_t, rw_flags);
3447 	BUILD_BUG_SQE_ELEM(28, /* compat */   int, rw_flags);
3448 	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
3449 	BUILD_BUG_SQE_ELEM(28, __u32,  fsync_flags);
3450 	BUILD_BUG_SQE_ELEM(28, /* compat */ __u16,  poll_events);
3451 	BUILD_BUG_SQE_ELEM(28, __u32,  poll32_events);
3452 	BUILD_BUG_SQE_ELEM(28, __u32,  sync_range_flags);
3453 	BUILD_BUG_SQE_ELEM(28, __u32,  msg_flags);
3454 	BUILD_BUG_SQE_ELEM(28, __u32,  timeout_flags);
3455 	BUILD_BUG_SQE_ELEM(28, __u32,  accept_flags);
3456 	BUILD_BUG_SQE_ELEM(28, __u32,  cancel_flags);
3457 	BUILD_BUG_SQE_ELEM(28, __u32,  open_flags);
3458 	BUILD_BUG_SQE_ELEM(28, __u32,  statx_flags);
3459 	BUILD_BUG_SQE_ELEM(28, __u32,  fadvise_advice);
3460 	BUILD_BUG_SQE_ELEM(28, __u32,  splice_flags);
3461 	BUILD_BUG_SQE_ELEM(28, __u32,  rename_flags);
3462 	BUILD_BUG_SQE_ELEM(28, __u32,  unlink_flags);
3463 	BUILD_BUG_SQE_ELEM(28, __u32,  hardlink_flags);
3464 	BUILD_BUG_SQE_ELEM(28, __u32,  xattr_flags);
3465 	BUILD_BUG_SQE_ELEM(28, __u32,  msg_ring_flags);
3466 	BUILD_BUG_SQE_ELEM(32, __u64,  user_data);
3467 	BUILD_BUG_SQE_ELEM(40, __u16,  buf_index);
3468 	BUILD_BUG_SQE_ELEM(40, __u16,  buf_group);
3469 	BUILD_BUG_SQE_ELEM(42, __u16,  personality);
3470 	BUILD_BUG_SQE_ELEM(44, __s32,  splice_fd_in);
3471 	BUILD_BUG_SQE_ELEM(44, __u32,  file_index);
3472 	BUILD_BUG_SQE_ELEM(44, __u16,  addr_len);
3473 	BUILD_BUG_SQE_ELEM(44, __u8,   write_stream);
3474 	BUILD_BUG_SQE_ELEM(45, __u8,   __pad4[0]);
3475 	BUILD_BUG_SQE_ELEM(46, __u16,  __pad3[0]);
3476 	BUILD_BUG_SQE_ELEM(48, __u64,  addr3);
3477 	BUILD_BUG_SQE_ELEM_SIZE(48, 0, cmd);
3478 	BUILD_BUG_SQE_ELEM(48, __u64, attr_ptr);
3479 	BUILD_BUG_SQE_ELEM(56, __u64, attr_type_mask);
3480 	BUILD_BUG_SQE_ELEM(56, __u64,  __pad2);
3481 
3482 	BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
3483 		     sizeof(struct io_uring_rsrc_update));
3484 	BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
3485 		     sizeof(struct io_uring_rsrc_update2));
3486 
3487 	/* ->buf_index is u16 */
3488 	BUILD_BUG_ON(offsetof(struct io_uring_buf_ring, bufs) != 0);
3489 	BUILD_BUG_ON(offsetof(struct io_uring_buf, resv) !=
3490 		     offsetof(struct io_uring_buf_ring, tail));
3491 
3492 	/* should fit into one byte */
3493 	BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));
3494 	BUILD_BUG_ON(SQE_COMMON_FLAGS >= (1 << 8));
3495 	BUILD_BUG_ON((SQE_VALID_FLAGS | SQE_COMMON_FLAGS) != SQE_VALID_FLAGS);
3496 
3497 	BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof_field(struct io_kiocb, flags));
3498 
3499 	BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));
3500 
3501 	/* top 8bits are for internal use */
3502 	BUILD_BUG_ON((IORING_URING_CMD_MASK & 0xff000000) != 0);
3503 
3504 	io_uring_optable_init();
3505 
3506 	/* imu->dir is u8 */
3507 	BUILD_BUG_ON((IO_BUF_DEST | IO_BUF_SOURCE) > U8_MAX);
3508 
3509 	/*
3510 	 * Allow user copy in the per-command field, which starts after the
3511 	 * file in io_kiocb and until the opcode field. The openat2 handling
3512 	 * requires copying in user memory into the io_kiocb object in that
3513 	 * range, and HARDENED_USERCOPY will complain if we haven't
3514 	 * correctly annotated this range.
3515 	 */
3516 	req_cachep = kmem_cache_create("io_kiocb", sizeof(struct io_kiocb), &kmem_args,
3517 				SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT |
3518 				SLAB_TYPESAFE_BY_RCU);
3519 
3520 	iou_wq = alloc_workqueue("iou_exit", WQ_UNBOUND, 64);
3521 	BUG_ON(!iou_wq);
3522 
3523 #ifdef CONFIG_SYSCTL
3524 	register_sysctl_init("kernel", kernel_io_uring_disabled_table);
3525 #endif
3526 
3527 	return 0;
3528 };
3529 __initcall(io_uring_init);
3530