26 static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)  in io_req_uring_cleanup()  argument
28 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);  in io_req_uring_cleanup()
29 	struct io_async_cmd *ac = req->async_data;  in io_req_uring_cleanup()
38 	if (io_alloc_cache_put(&req->ctx->cmd_cache, ac)) {  in io_req_uring_cleanup()
40 		io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);  in io_req_uring_cleanup()
46 void io_uring_cmd_cleanup(struct io_kiocb *req)  in io_uring_cmd_cleanup()  argument
48 	io_req_uring_cleanup(req, 0);  in io_uring_cmd_cleanup()
55 	struct io_kiocb *req;  in io_uring_try_cancel_uring_cmd()  local
60 	hlist_for_each_entry_safe(req, tmp, &ctx->cancelable_uring_cmd,  in io_uring_try_cancel_uring_cmd()
62 		struct io_uring_cmd *cmd = io_kiocb_to_cmd(req,  in io_uring_try_cancel_uring_cmd()
64 		struct file *file = req->file;  in io_uring_try_cancel_uring_cmd()
66 		if (!cancel_all && req->tctx != tctx)  in io_uring_try_cancel_uring_cmd()
82 	struct io_kiocb *req = cmd_to_io_kiocb(cmd);  in io_uring_cmd_del_cancelable()  local
83 	struct io_ring_ctx *ctx = req->ctx;  in io_uring_cmd_del_cancelable()
90 	hlist_del(&req->hash_node);  in io_uring_cmd_del_cancelable()
106 	struct io_kiocb *req = cmd_to_io_kiocb(cmd);  in io_uring_cmd_mark_cancelable()  local
107 	struct io_ring_ctx *ctx = req->ctx;  in io_uring_cmd_mark_cancelable()
115 	if (req->flags & REQ_F_IOPOLL)  in io_uring_cmd_mark_cancelable()
121 		hlist_add_head(&req->hash_node, &ctx->cancelable_uring_cmd);  in io_uring_cmd_mark_cancelable()
131 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in __io_uring_cmd_do_in_task()  local
133 	if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))  in __io_uring_cmd_do_in_task()
136 	req->io_task_work.func = task_work_cb;  in __io_uring_cmd_do_in_task()
137 	__io_req_task_work_add(req, flags);  in __io_uring_cmd_do_in_task()
141 static inline void io_req_set_cqe32_extra(struct io_kiocb *req,  in io_req_set_cqe32_extra()  argument
144 	req->big_cqe.extra1 = extra1;  in io_req_set_cqe32_extra()
145 	req->big_cqe.extra2 = extra2;  in io_req_set_cqe32_extra()
155 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in __io_uring_cmd_done()  local
157 	if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))  in __io_uring_cmd_done()
163 		req_set_fail(req);  in __io_uring_cmd_done()
165 	io_req_set_res(req, ret, 0);  in __io_uring_cmd_done()
167 		if (req->ctx->flags & IORING_SETUP_CQE_MIXED)  in __io_uring_cmd_done()
168 			req->cqe.flags |= IORING_CQE_F_32;  in __io_uring_cmd_done()
169 		io_req_set_cqe32_extra(req, res2, 0);  in __io_uring_cmd_done()
171 	io_req_uring_cleanup(req, issue_flags);  in __io_uring_cmd_done()
172 	if (req->flags & REQ_F_IOPOLL) {  in __io_uring_cmd_done()
174 		smp_store_release(&req->iopoll_completed, 1);  in __io_uring_cmd_done()
178 		io_req_complete_defer(req);  in __io_uring_cmd_done()
180 		req->io_task_work.func = io_req_task_complete;  in __io_uring_cmd_done()
181 		io_req_task_work_add(req);  in __io_uring_cmd_done()
186 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_uring_cmd_prep()  argument
188 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);  in io_uring_cmd_prep()
201 		req->buf_index = READ_ONCE(sqe->buf_index);  in io_uring_cmd_prep()
205 	    !!(req->flags & REQ_F_BUFFER_SELECT))  in io_uring_cmd_prep()
210 	ac = io_uring_alloc_async_data(&req->ctx->cmd_cache, req);  in io_uring_cmd_prep()
214 		req->flags |= REQ_F_NEED_CLEANUP;  in io_uring_cmd_prep()
223 static inline size_t uring_sqe_size(struct io_kiocb *req)  in uring_sqe_size()  argument
225 	if (req->ctx->flags & IORING_SETUP_SQE128 ||  in uring_sqe_size()
226 	    req->opcode == IORING_OP_URING_CMD128)  in uring_sqe_size()
231 void io_uring_cmd_sqe_copy(struct io_kiocb *req)  in io_uring_cmd_sqe_copy()  argument
233 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);  in io_uring_cmd_sqe_copy()
234 	struct io_async_cmd *ac = req->async_data;  in io_uring_cmd_sqe_copy()
239 	memcpy(ac->sqes, ioucmd->sqe, uring_sqe_size(req));  in io_uring_cmd_sqe_copy()
243 int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)  in io_uring_cmd()  argument
245 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);  in io_uring_cmd()
246 	struct io_ring_ctx *ctx = req->ctx;  in io_uring_cmd()
247 	struct file *file = req->file;  in io_uring_cmd()
258 	    req->opcode == IORING_OP_URING_CMD128)  in io_uring_cmd()
265 		req->flags |= REQ_F_IOPOLL;  in io_uring_cmd()
267 		req->iopoll_completed = 0;  in io_uring_cmd()
270 			req->flags &= ~REQ_F_IOPOLL_STATE;  in io_uring_cmd()
271 			req->iopoll_start = ktime_get_ns();  in io_uring_cmd()
283 		req_set_fail(req);  in io_uring_cmd()
284 	io_req_uring_cleanup(req, issue_flags);  in io_uring_cmd()
285 	io_req_set_res(req, ret, 0);  in io_uring_cmd()
294 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in io_uring_cmd_import_fixed()  local
299 	return io_import_reg_buf(req, iter, ubuf, len, rw, issue_flags);  in io_uring_cmd_import_fixed()
309 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in io_uring_cmd_import_fixed_vec()  local
310 	struct io_async_cmd *ac = req->async_data;  in io_uring_cmd_import_fixed_vec()
316 	ret = io_prep_reg_iovec(req, &ac->vec, uvec, uvec_segs);  in io_uring_cmd_import_fixed_vec()
320 	return io_import_reg_vec(ddir, iter, req, &ac->vec, uvec_segs,  in io_uring_cmd_import_fixed_vec()
327 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in io_uring_cmd_issue_blocking()  local
329 	io_queue_iowq(req);  in io_uring_cmd_issue_blocking()
335 	struct io_kiocb *req = cmd_to_io_kiocb(cmd);  in io_cmd_poll_multishot()  local
338 	if (likely(req->flags & REQ_F_APOLL_MULTISHOT))  in io_cmd_poll_multishot()
341 	req->flags |= REQ_F_APOLL_MULTISHOT;  in io_cmd_poll_multishot()
344 	ret = io_arm_apoll(req, issue_flags, mask);  in io_cmd_poll_multishot()
352 	struct io_kiocb *req = cmd_to_io_kiocb(cmd);  in io_uring_cmd_post_mshot_cqe32()  local
356 	return io_req_post_cqe32(req, cqe);  in io_uring_cmd_post_mshot_cqe32()
367 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in io_uring_cmd_buffer_select()  local
372 	if (WARN_ON_ONCE(!io_do_buffer_select(req)))  in io_uring_cmd_buffer_select()
375 	return io_buffer_select(req, len, buf_group, issue_flags);  in io_uring_cmd_buffer_select()
390 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);  in io_uring_mshot_cmd_post_cqe()  local
397 		cflags = io_put_kbuf(req, sel->val, sel->buf_list);  in io_uring_mshot_cmd_post_cqe()
398 		if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE))  in io_uring_mshot_cmd_post_cqe()
402 	io_kbuf_recycle(req, sel->buf_list, issue_flags);  in io_uring_mshot_cmd_post_cqe()
404 		req_set_fail(req);  in io_uring_mshot_cmd_post_cqe()
405 	io_req_set_res(req, sel->val, cflags);  in io_uring_mshot_cmd_post_cqe()