1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/io_uring/cmd.h>
6 #include <linux/security.h>
7 #include <linux/nospec.h>
8
9 #include <uapi/linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "alloc_cache.h"
13 #include "rsrc.h"
14 #include "kbuf.h"
15 #include "uring_cmd.h"
16 #include "poll.h"
17
io_cmd_cache_free(const void * entry)18 void io_cmd_cache_free(const void *entry)
19 {
20 struct io_async_cmd *ac = (struct io_async_cmd *)entry;
21
22 io_vec_free(&ac->vec);
23 kfree(ac);
24 }
25
io_req_uring_cleanup(struct io_kiocb * req,unsigned int issue_flags)26 static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)
27 {
28 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
29 struct io_async_cmd *ac = req->async_data;
30
31 if (issue_flags & IO_URING_F_UNLOCKED)
32 return;
33
34 io_alloc_cache_vec_kasan(&ac->vec);
35 if (ac->vec.nr > IO_VEC_CACHE_SOFT_CAP)
36 io_vec_free(&ac->vec);
37
38 if (io_alloc_cache_put(&req->ctx->cmd_cache, ac)) {
39 ioucmd->sqe = NULL;
40 io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);
41 } else {
42 io_vec_free(&ac->vec);
43 }
44 }
45
io_uring_cmd_cleanup(struct io_kiocb * req)46 void io_uring_cmd_cleanup(struct io_kiocb *req)
47 {
48 io_req_uring_cleanup(req, 0);
49 }
50
io_uring_try_cancel_uring_cmd(struct io_ring_ctx * ctx,struct io_uring_task * tctx,bool cancel_all)51 bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
52 struct io_uring_task *tctx, bool cancel_all)
53 {
54 struct hlist_node *tmp;
55 struct io_kiocb *req;
56 bool ret = false;
57
58 lockdep_assert_held(&ctx->uring_lock);
59
60 hlist_for_each_entry_safe(req, tmp, &ctx->cancelable_uring_cmd,
61 hash_node) {
62 struct io_uring_cmd *cmd = io_kiocb_to_cmd(req,
63 struct io_uring_cmd);
64 struct file *file = req->file;
65
66 if (!cancel_all && req->tctx != tctx)
67 continue;
68
69 if (cmd->flags & IORING_URING_CMD_CANCELABLE) {
70 file->f_op->uring_cmd(cmd, IO_URING_F_CANCEL |
71 IO_URING_F_COMPLETE_DEFER);
72 ret = true;
73 }
74 }
75 io_submit_flush_completions(ctx);
76 return ret;
77 }
78
io_uring_cmd_del_cancelable(struct io_uring_cmd * cmd,unsigned int issue_flags)79 static void io_uring_cmd_del_cancelable(struct io_uring_cmd *cmd,
80 unsigned int issue_flags)
81 {
82 struct io_kiocb *req = cmd_to_io_kiocb(cmd);
83 struct io_ring_ctx *ctx = req->ctx;
84
85 if (!(cmd->flags & IORING_URING_CMD_CANCELABLE))
86 return;
87
88 cmd->flags &= ~IORING_URING_CMD_CANCELABLE;
89 io_ring_submit_lock(ctx, issue_flags);
90 hlist_del(&req->hash_node);
91 io_ring_submit_unlock(ctx, issue_flags);
92 }
93
94 /*
95 * Mark this command as cancelable, then io_uring_try_cancel_uring_cmd()
96 * will try to cancel this issued command by sending ->uring_cmd() with
97 * issue_flags of IO_URING_F_CANCEL.
98 *
99 * The command is guaranteed to not be done when calling ->uring_cmd()
100 * with IO_URING_F_CANCEL, but it is driver's responsibility to deal
101 * with race between io_uring canceling and normal completion.
102 */
io_uring_cmd_mark_cancelable(struct io_uring_cmd * cmd,unsigned int issue_flags)103 void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
104 unsigned int issue_flags)
105 {
106 struct io_kiocb *req = cmd_to_io_kiocb(cmd);
107 struct io_ring_ctx *ctx = req->ctx;
108
109 /*
110 * Doing cancelations on IOPOLL requests are not supported. Both
111 * because they can't get canceled in the block stack, but also
112 * because iopoll completion data overlaps with the hash_node used
113 * for tracking.
114 */
115 if (req->flags & REQ_F_IOPOLL)
116 return;
117
118 if (!(cmd->flags & IORING_URING_CMD_CANCELABLE)) {
119 cmd->flags |= IORING_URING_CMD_CANCELABLE;
120 io_ring_submit_lock(ctx, issue_flags);
121 hlist_add_head(&req->hash_node, &ctx->cancelable_uring_cmd);
122 io_ring_submit_unlock(ctx, issue_flags);
123 }
124 }
125 EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
126
__io_uring_cmd_do_in_task(struct io_uring_cmd * ioucmd,io_req_tw_func_t task_work_cb,unsigned flags)127 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
128 io_req_tw_func_t task_work_cb,
129 unsigned flags)
130 {
131 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
132
133 if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
134 return;
135
136 req->io_task_work.func = task_work_cb;
137 __io_req_task_work_add(req, flags);
138 }
139 EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
140
io_req_set_cqe32_extra(struct io_kiocb * req,u64 extra1,u64 extra2)141 static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
142 u64 extra1, u64 extra2)
143 {
144 req->big_cqe.extra1 = extra1;
145 req->big_cqe.extra2 = extra2;
146 }
147
148 /*
149 * Called by consumers of io_uring_cmd, if they originally returned
150 * -EIOCBQUEUED upon receiving the command.
151 */
__io_uring_cmd_done(struct io_uring_cmd * ioucmd,s32 ret,u64 res2,unsigned issue_flags,bool is_cqe32)152 void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
153 unsigned issue_flags, bool is_cqe32)
154 {
155 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
156
157 if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
158 return;
159
160 io_uring_cmd_del_cancelable(ioucmd, issue_flags);
161
162 if (ret < 0)
163 req_set_fail(req);
164
165 io_req_set_res(req, ret, 0);
166 if (is_cqe32) {
167 if (req->ctx->flags & IORING_SETUP_CQE_MIXED)
168 req->cqe.flags |= IORING_CQE_F_32;
169 io_req_set_cqe32_extra(req, res2, 0);
170 }
171 io_req_uring_cleanup(req, issue_flags);
172 if (req->flags & REQ_F_IOPOLL) {
173 /* order with io_do_iopoll() checking ->iopoll_completed */
174 smp_store_release(&req->iopoll_completed, 1);
175 } else if (issue_flags & IO_URING_F_COMPLETE_DEFER) {
176 if (WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED))
177 return;
178 io_req_complete_defer(req);
179 } else {
180 req->io_task_work.func = io_req_task_complete;
181 io_req_task_work_add(req);
182 }
183 }
184 EXPORT_SYMBOL_GPL(__io_uring_cmd_done);
185
io_uring_cmd_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)186 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
187 {
188 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
189 struct io_async_cmd *ac;
190
191 if (sqe->__pad1)
192 return -EINVAL;
193
194 ioucmd->flags = READ_ONCE(sqe->uring_cmd_flags);
195 if (ioucmd->flags & ~IORING_URING_CMD_MASK)
196 return -EINVAL;
197
198 if (ioucmd->flags & IORING_URING_CMD_FIXED) {
199 if (ioucmd->flags & IORING_URING_CMD_MULTISHOT)
200 return -EINVAL;
201 req->buf_index = READ_ONCE(sqe->buf_index);
202 }
203
204 if (!!(ioucmd->flags & IORING_URING_CMD_MULTISHOT) !=
205 !!(req->flags & REQ_F_BUFFER_SELECT))
206 return -EINVAL;
207
208 ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
209
210 ac = io_uring_alloc_async_data(&req->ctx->cmd_cache, req);
211 if (!ac)
212 return -ENOMEM;
213 if (ac->vec.iovec)
214 req->flags |= REQ_F_NEED_CLEANUP;
215 ioucmd->sqe = sqe;
216 return 0;
217 }
218
219 /*
220 * IORING_SETUP_SQE128 contexts allocate twice the normal SQE size for each
221 * slot.
222 */
uring_sqe_size(struct io_kiocb * req)223 static inline size_t uring_sqe_size(struct io_kiocb *req)
224 {
225 if (req->ctx->flags & IORING_SETUP_SQE128 ||
226 req->opcode == IORING_OP_URING_CMD128)
227 return 2 * sizeof(struct io_uring_sqe);
228 return sizeof(struct io_uring_sqe);
229 }
230
io_uring_cmd_sqe_copy(struct io_kiocb * req)231 void io_uring_cmd_sqe_copy(struct io_kiocb *req)
232 {
233 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
234 struct io_async_cmd *ac = req->async_data;
235
236 /* Should not happen, as REQ_F_SQE_COPIED covers this */
237 if (WARN_ON_ONCE(ioucmd->sqe == ac->sqes))
238 return;
239 memcpy(ac->sqes, ioucmd->sqe, uring_sqe_size(req));
240 ioucmd->sqe = ac->sqes;
241 }
242
io_uring_cmd(struct io_kiocb * req,unsigned int issue_flags)243 int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
244 {
245 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
246 struct io_ring_ctx *ctx = req->ctx;
247 struct file *file = req->file;
248 int ret;
249
250 if (!file->f_op->uring_cmd)
251 return -EOPNOTSUPP;
252
253 ret = security_uring_cmd(ioucmd);
254 if (ret)
255 return ret;
256
257 if (ctx->flags & IORING_SETUP_SQE128 ||
258 req->opcode == IORING_OP_URING_CMD128)
259 issue_flags |= IO_URING_F_SQE128;
260 if (ctx->flags & (IORING_SETUP_CQE32 | IORING_SETUP_CQE_MIXED))
261 issue_flags |= IO_URING_F_CQE32;
262 if (io_is_compat(ctx))
263 issue_flags |= IO_URING_F_COMPAT;
264 if (ctx->flags & IORING_SETUP_IOPOLL && file->f_op->uring_cmd_iopoll) {
265 req->flags |= REQ_F_IOPOLL;
266 issue_flags |= IO_URING_F_IOPOLL;
267 req->iopoll_completed = 0;
268 if (ctx->flags & IORING_SETUP_HYBRID_IOPOLL) {
269 /* make sure every req only blocks once */
270 req->flags &= ~REQ_F_IOPOLL_STATE;
271 req->iopoll_start = ktime_get_ns();
272 }
273 }
274
275 ret = file->f_op->uring_cmd(ioucmd, issue_flags);
276 if (ret == -EAGAIN) {
277 ioucmd->flags |= IORING_URING_CMD_REISSUE;
278 return ret;
279 }
280 if (ret == -EIOCBQUEUED)
281 return ret;
282 if (ret < 0)
283 req_set_fail(req);
284 io_req_uring_cleanup(req, issue_flags);
285 io_req_set_res(req, ret, 0);
286 return IOU_COMPLETE;
287 }
288
io_uring_cmd_import_fixed(u64 ubuf,unsigned long len,int rw,struct iov_iter * iter,struct io_uring_cmd * ioucmd,unsigned int issue_flags)289 int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
290 struct iov_iter *iter,
291 struct io_uring_cmd *ioucmd,
292 unsigned int issue_flags)
293 {
294 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
295
296 if (WARN_ON_ONCE(!(ioucmd->flags & IORING_URING_CMD_FIXED)))
297 return -EINVAL;
298
299 return io_import_reg_buf(req, iter, ubuf, len, rw, issue_flags);
300 }
301 EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed);
302
io_uring_cmd_import_fixed_vec(struct io_uring_cmd * ioucmd,const struct iovec __user * uvec,size_t uvec_segs,int ddir,struct iov_iter * iter,unsigned issue_flags)303 int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
304 const struct iovec __user *uvec,
305 size_t uvec_segs,
306 int ddir, struct iov_iter *iter,
307 unsigned issue_flags)
308 {
309 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
310 struct io_async_cmd *ac = req->async_data;
311 int ret;
312
313 if (WARN_ON_ONCE(!(ioucmd->flags & IORING_URING_CMD_FIXED)))
314 return -EINVAL;
315
316 ret = io_prep_reg_iovec(req, &ac->vec, uvec, uvec_segs);
317 if (ret)
318 return ret;
319
320 return io_import_reg_vec(ddir, iter, req, &ac->vec, uvec_segs,
321 issue_flags);
322 }
323 EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_vec);
324
io_uring_cmd_issue_blocking(struct io_uring_cmd * ioucmd)325 void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd)
326 {
327 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
328
329 io_queue_iowq(req);
330 }
331
io_cmd_poll_multishot(struct io_uring_cmd * cmd,unsigned int issue_flags,__poll_t mask)332 int io_cmd_poll_multishot(struct io_uring_cmd *cmd,
333 unsigned int issue_flags, __poll_t mask)
334 {
335 struct io_kiocb *req = cmd_to_io_kiocb(cmd);
336 int ret;
337
338 if (likely(req->flags & REQ_F_APOLL_MULTISHOT))
339 return 0;
340
341 req->flags |= REQ_F_APOLL_MULTISHOT;
342 mask &= ~EPOLLONESHOT;
343
344 ret = io_arm_apoll(req, issue_flags, mask);
345 return ret == IO_APOLL_OK ? -EIOCBQUEUED : -ECANCELED;
346 }
347
io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd * cmd,unsigned int issue_flags,struct io_uring_cqe cqe[2])348 bool io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd *cmd,
349 unsigned int issue_flags,
350 struct io_uring_cqe cqe[2])
351 {
352 struct io_kiocb *req = cmd_to_io_kiocb(cmd);
353
354 if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_MULTISHOT)))
355 return false;
356 return io_req_post_cqe32(req, cqe);
357 }
358
359 /*
360 * Work with io_uring_mshot_cmd_post_cqe() together for committing the
361 * provided buffer upfront
362 */
io_uring_cmd_buffer_select(struct io_uring_cmd * ioucmd,unsigned buf_group,size_t * len,unsigned int issue_flags)363 struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd,
364 unsigned buf_group, size_t *len,
365 unsigned int issue_flags)
366 {
367 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
368
369 if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT))
370 return (struct io_br_sel) { .val = -EINVAL };
371
372 if (WARN_ON_ONCE(!io_do_buffer_select(req)))
373 return (struct io_br_sel) { .val = -EINVAL };
374
375 return io_buffer_select(req, len, buf_group, issue_flags);
376 }
377 EXPORT_SYMBOL_GPL(io_uring_cmd_buffer_select);
378
379 /*
380 * Return true if this multishot uring_cmd needs to be completed, otherwise
381 * the event CQE is posted successfully.
382 *
383 * This function must use `struct io_br_sel` returned from
384 * io_uring_cmd_buffer_select() for committing the buffer in the same
385 * uring_cmd submission context.
386 */
io_uring_mshot_cmd_post_cqe(struct io_uring_cmd * ioucmd,struct io_br_sel * sel,unsigned int issue_flags)387 bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,
388 struct io_br_sel *sel, unsigned int issue_flags)
389 {
390 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
391 unsigned int cflags = 0;
392
393 if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT))
394 return true;
395
396 if (sel->val > 0) {
397 cflags = io_put_kbuf(req, sel->val, sel->buf_list);
398 if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE))
399 return false;
400 }
401
402 io_kbuf_recycle(req, sel->buf_list, issue_flags);
403 if (sel->val < 0)
404 req_set_fail(req);
405 io_req_set_res(req, sel->val, cflags);
406 return true;
407 }
408 EXPORT_SYMBOL_GPL(io_uring_mshot_cmd_post_cqe);
409