1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef _LINUX_IO_URING_CMD_H 3 #define _LINUX_IO_URING_CMD_H 4 5 #include <uapi/linux/io_uring.h> 6 #include <linux/io_uring_types.h> 7 #include <linux/blk-mq.h> 8 9 /* only top 8 bits of sqe->uring_cmd_flags for kernel internal use */ 10 #define IORING_URING_CMD_CANCELABLE (1U << 30) 11 /* io_uring_cmd is being issued again */ 12 #define IORING_URING_CMD_REISSUE (1U << 31) 13 14 struct io_uring_cmd { 15 struct file *file; 16 const struct io_uring_sqe *sqe; 17 u32 cmd_op; 18 u32 flags; 19 u8 pdu[32]; /* available inline for free use */ 20 u8 unused[8]; 21 }; 22 23 #define io_uring_sqe128_cmd(sqe, type) ({ \ 24 BUILD_BUG_ON(sizeof(type) > ((2 * sizeof(struct io_uring_sqe)) - \ 25 offsetof(struct io_uring_sqe, cmd))); \ 26 (const type *)(sqe)->cmd; \ 27 }) 28 29 #define io_uring_sqe_cmd(sqe, type) ({ \ 30 BUILD_BUG_ON(sizeof(type) > (sizeof(struct io_uring_sqe) - \ 31 offsetof(struct io_uring_sqe, cmd))); \ 32 (const type *)(sqe)->cmd; \ 33 }) 34 35 static inline void io_uring_cmd_private_sz_check(size_t cmd_sz) 36 { 37 BUILD_BUG_ON(cmd_sz > sizeof_field(struct io_uring_cmd, pdu)); 38 } 39 #define io_uring_cmd_to_pdu(cmd, pdu_type) ( \ 40 io_uring_cmd_private_sz_check(sizeof(pdu_type)), \ 41 ((pdu_type *)&(cmd)->pdu) \ 42 ) 43 44 #if defined(CONFIG_IO_URING) 45 int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, 46 struct iov_iter *iter, 47 struct io_uring_cmd *ioucmd, 48 unsigned int issue_flags); 49 int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, 50 const struct iovec __user *uvec, 51 size_t uvec_segs, 52 int ddir, struct iov_iter *iter, 53 unsigned issue_flags); 54 55 /* 56 * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd 57 * and the corresponding io_uring request. 58 * 59 * Note: the caller should never hard code @issue_flags and is only allowed 60 * to pass the mask provided by the core io_uring code. 61 */ 62 void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2, 63 unsigned issue_flags, bool is_cqe32); 64 65 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, 66 io_req_tw_func_t task_work_cb, 67 unsigned flags); 68 69 /* 70 * Note: the caller should never hard code @issue_flags and only use the 71 * mask provided by the core io_uring code. 72 */ 73 void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd, 74 unsigned int issue_flags); 75 76 /* Execute the request from a blocking context */ 77 void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd); 78 79 /* 80 * Select a buffer from the provided buffer group for multishot uring_cmd. 81 * Returns the selected buffer address and size. 82 */ 83 struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd, 84 unsigned buf_group, size_t *len, 85 unsigned int issue_flags); 86 87 /* 88 * Complete a multishot uring_cmd event. This will post a CQE to the completion 89 * queue and update the provided buffer. 90 */ 91 bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, 92 struct io_br_sel *sel, unsigned int issue_flags); 93 94 #else 95 static inline int 96 io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, 97 struct iov_iter *iter, struct io_uring_cmd *ioucmd, 98 unsigned int issue_flags) 99 { 100 return -EOPNOTSUPP; 101 } 102 static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, 103 const struct iovec __user *uvec, 104 size_t uvec_segs, 105 int ddir, struct iov_iter *iter, 106 unsigned issue_flags) 107 { 108 return -EOPNOTSUPP; 109 } 110 static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, 111 u64 ret2, unsigned issue_flags, bool is_cqe32) 112 { 113 } 114 static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, 115 io_req_tw_func_t task_work_cb, unsigned flags) 116 { 117 } 118 static inline void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd, 119 unsigned int issue_flags) 120 { 121 } 122 static inline void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd) 123 { 124 } 125 static inline struct io_br_sel 126 io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd, unsigned buf_group, 127 size_t *len, unsigned int issue_flags) 128 { 129 return (struct io_br_sel) { .val = -EOPNOTSUPP }; 130 } 131 static inline bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, 132 struct io_br_sel *sel, unsigned int issue_flags) 133 { 134 return true; 135 } 136 #endif 137 138 static inline struct io_uring_cmd *io_uring_cmd_from_tw(struct io_tw_req tw_req) 139 { 140 return io_kiocb_to_cmd(tw_req.req, struct io_uring_cmd); 141 } 142 143 /* task_work executor checks the deferred list completion */ 144 #define IO_URING_CMD_TASK_WORK_ISSUE_FLAGS IO_URING_F_COMPLETE_DEFER 145 146 /* users must follow the IOU_F_TWQ_LAZY_WAKE semantics */ 147 static inline void io_uring_cmd_do_in_task_lazy(struct io_uring_cmd *ioucmd, 148 io_req_tw_func_t task_work_cb) 149 { 150 __io_uring_cmd_do_in_task(ioucmd, task_work_cb, IOU_F_TWQ_LAZY_WAKE); 151 } 152 153 static inline void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd, 154 io_req_tw_func_t task_work_cb) 155 { 156 __io_uring_cmd_do_in_task(ioucmd, task_work_cb, 0); 157 } 158 159 static inline struct task_struct *io_uring_cmd_get_task(struct io_uring_cmd *cmd) 160 { 161 return cmd_to_io_kiocb(cmd)->tctx->task; 162 } 163 164 /* 165 * Return uring_cmd's context reference as its context handle for driver to 166 * track per-context resource, such as registered kernel IO buffer 167 */ 168 static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd) 169 { 170 return cmd_to_io_kiocb(cmd)->ctx; 171 } 172 173 static inline void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, 174 unsigned issue_flags) 175 { 176 return __io_uring_cmd_done(ioucmd, ret, 0, issue_flags, false); 177 } 178 179 static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret, 180 u64 res2, unsigned issue_flags) 181 { 182 return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true); 183 } 184 185 int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq, 186 void (*release)(void *), unsigned int index, 187 unsigned int issue_flags); 188 int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, 189 unsigned int issue_flags); 190 191 #endif /* _LINUX_IO_URING_CMD_H */ 192