121 static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags);
153 static void io_poison_cached_req(struct io_kiocb *req)  in io_poison_cached_req()
163 static void io_poison_req(struct io_kiocb *req)  in io_poison_req()
173 static inline void req_fail_link_node(struct io_kiocb *req, int res)  in req_fail_link_node()
179 static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx)  in io_req_add_to_cache()
309 static void io_clean_op(struct io_kiocb *req)  in io_clean_op()
336 inline void io_req_track_inflight(struct io_kiocb *req)  in io_req_track_inflight()
344 static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)  in __io_prep_linked_timeout()
358 static void io_prep_async_work(struct io_kiocb *req)  in io_prep_async_work()
390 static void io_prep_async_link(struct io_kiocb *req)  in io_prep_async_link()
392 	struct io_kiocb *cur;  in io_prep_async_link()
407 static void io_queue_iowq(struct io_kiocb *req)  in io_queue_iowq()
440 void io_req_queue_iowq(struct io_kiocb *req)  in io_req_queue_iowq()
446 unsigned io_linked_nr(struct io_kiocb *req)  in io_linked_nr()
448 	struct io_kiocb *tmp;  in io_linked_nr()
599 static inline void io_put_task(struct io_kiocb *req)  in io_put_task()
853 bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags)  in io_req_post_cqe()
885 bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2])  in io_req_post_cqe32()
906 static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)  in io_req_complete_post()
944 void io_req_defer_failed(struct io_kiocb *req, s32 res)  in io_req_defer_failed()
988 		struct io_kiocb *req = reqs[ret];  in __io_alloc_req_refill()
995 __cold void io_free_req(struct io_kiocb *req)  in io_free_req()
1005 static void __io_req_find_next_prep(struct io_kiocb *req)  in __io_req_find_next_prep()
1014 static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)  in io_req_find_next()
1016 	struct io_kiocb *nxt;  in io_req_find_next()
1033 	struct io_kiocb *req = tw_req.req;  in io_req_task_cancel()
1041 	struct io_kiocb *req = tw_req.req;  in io_req_task_submit()
1053 void io_req_task_queue_fail(struct io_kiocb *req, int ret)  in io_req_task_queue_fail()
1060 void io_req_task_queue(struct io_kiocb *req)  in io_req_task_queue()
1066 void io_queue_next(struct io_kiocb *req)  in io_queue_next()
1068 	struct io_kiocb *nxt = io_req_find_next(req);  in io_queue_next()
1074 static inline void io_req_put_rsrc_nodes(struct io_kiocb *req)  in io_req_put_rsrc_nodes()
1091 		struct io_kiocb *req = container_of(node, struct io_kiocb,  in io_free_batch_list()
1136 		struct io_kiocb *req = container_of(node, struct io_kiocb,  in __io_submit_flush_completions()
1275 static void io_iopoll_req_issued(struct io_kiocb *req, unsigned int issue_flags)  in io_iopoll_req_issued()
1292 		struct io_kiocb *list_req;  in io_iopoll_req_issued()
1294 		list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb, iopoll_node);  in io_iopoll_req_issued()
1329 static __cold void io_drain_req(struct io_kiocb *req)  in io_drain_req()
1353 static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,  in io_assign_file()
1369 static inline int __io_issue_sqe(struct io_kiocb *req,  in __io_issue_sqe()
1374 	struct io_kiocb *link = NULL;  in __io_issue_sqe()
1402 static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)  in io_issue_sqe()
1430 int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)  in io_poll_issue()
1451 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_wq_free_work()
1452 	struct io_kiocb *nxt = NULL;  in io_wq_free_work()
1464 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);  in io_wq_submit_work()
1558 inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,  in io_file_get_fixed()
1577 struct file *io_file_get_normal(struct io_kiocb *req, int fd)  in io_file_get_normal()
1589 static int io_req_sqe_copy(struct io_kiocb *req, unsigned int issue_flags)  in io_req_sqe_copy()
1604 static void io_queue_async(struct io_kiocb *req, unsigned int issue_flags, int ret)  in io_queue_async()
1629 static inline void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags)  in io_queue_sqe()
1646 static void io_queue_sqe_fallback(struct io_kiocb *req)  in io_queue_sqe_fallback()
1673 					struct io_kiocb *req,  in io_check_restriction()
1694 	struct io_kiocb *head = ctx->submit_state.link.head;  in io_init_drain()
1710 static __cold int io_init_fail_req(struct io_kiocb *req, int err)  in io_init_fail_req()
1717 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_init_req()
1834 				      struct io_kiocb *req, int ret)  in io_submit_fail_init()
1838 	struct io_kiocb *head = link->head;  in io_submit_fail_init()
1870 static inline int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,  in io_submit_sqe()
2032 		struct io_kiocb *req;  in io_submit_sqes()
2130 	struct io_kiocb *req;  in __io_req_caches_free()
3149 		.useroffset = offsetof(struct io_kiocb, cmd.data),  in io_uring_init()
3150 		.usersize = sizeof_field(struct io_kiocb, cmd.data),  in io_uring_init()
3151 		.freeptr_offset = offsetof(struct io_kiocb, work),  in io_uring_init()
3227 	BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof_field(struct io_kiocb, flags));  in io_uring_init()
3246 	req_cachep = kmem_cache_create("io_kiocb", sizeof(struct io_kiocb), &kmem_args,  in io_uring_init()