| /linux/io_uring/ |
| H A D | rw.h | 36 int io_prep_read_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe);
37 int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe);
38 int io_prep_readv_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe);
39 int io_prep_writev_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe);
40 int io_prep_readv(struct io_kiocb *req, const struct io_uring_sqe *sqe);
41 int io_prep_writev(struct io_kiocb *req, const struct io_uring_sqe *sqe);
42 int io_prep_read(struct io_kiocb *req, const struct io_uring_sqe *sqe);
43 int io_prep_write(struct io_kiocb *req, const struct io_uring_sqe *sqe);
44 int io_read(struct io_kiocb *req, unsigned int issue_flags);
45 int io_write(struct io_kiocb *req, unsigned int issue_flags);
[all …]
|
| H A D | fs.h | 3 int io_renameat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
4 int io_renameat(struct io_kiocb *req, unsigned int issue_flags);
5 void io_renameat_cleanup(struct io_kiocb *req);
7 int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
8 int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags);
9 void io_unlinkat_cleanup(struct io_kiocb *req);
11 int io_mkdirat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
12 int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags);
13 void io_mkdirat_cleanup(struct io_kiocb *req);
15 int io_symlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
[all …]
|
| H A D | kbuf.h | 70 struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len,
72 int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,
74 int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg,
78 int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
79 int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
80 int io_manage_buffers_legacy(struct io_kiocb *req, unsigned int issue_flags);
86 bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
87 void io_kbuf_drop_legacy(struct io_kiocb *req);
89 unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl,
91 bool io_kbuf_commit(struct io_kiocb *req,
[all …]
|
| H A D | xattr.h | 3 void io_xattr_cleanup(struct io_kiocb *req);
5 int io_fsetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
6 int io_fsetxattr(struct io_kiocb *req, unsigned int issue_flags);
8 int io_setxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
9 int io_setxattr(struct io_kiocb *req, unsigned int issue_flags);
11 int io_fgetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
12 int io_fgetxattr(struct io_kiocb *req, unsigned int issue_flags);
14 int io_getxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
15 int io_getxattr(struct io_kiocb *req, unsigned int issue_flags);
|
| H A D | io_uring.h | 126 struct io_kiocb *req;
177 void io_req_defer_failed(struct io_kiocb *req, s32 res);
180 bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags);
181 bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe src_cqe[2]);
184 unsigned io_linked_nr(struct io_kiocb *req);
185 void io_req_track_inflight(struct io_kiocb *req);
186 struct file *io_file_get_normal(struct io_kiocb *req, int fd);
187 struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
191 void io_req_task_queue(struct io_kiocb *req);
193 void io_req_task_queue_fail(struct io_kiocb *req, int ret);
[all …]
|
| H A D | timeout.c | 24 struct io_kiocb *head;
26 struct io_kiocb *prev;
77 static struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
78 struct io_kiocb *link);
80 static inline bool io_is_timeout_noseq(struct io_kiocb *req) in io_is_timeout_noseq()
88 static inline void io_put_req(struct io_kiocb *req) in io_put_req()
112 struct io_kiocb *req = tw_req.req; in io_timeout_complete()
138 struct io_kiocb *req; in io_flush_killed_timeouts()
151 static void io_kill_timeout(struct io_kiocb *req, struct list_head *list) in io_kill_timeout()
175 struct io_kiocb *req = cmd_to_io_kiocb(timeout); in io_flush_timeouts()
[all …]
|
| H A D | sync.h | 3 int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
4 int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags);
6 int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
7 int io_fsync(struct io_kiocb *req, unsigned int issue_flags);
9 int io_fallocate(struct io_kiocb *req, unsigned int issue_flags);
10 int io_fallocate_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
|
| H A D | uring_cmd.c | 26 static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags) in io_req_uring_cleanup()
46 void io_uring_cmd_cleanup(struct io_kiocb *req) in io_uring_cmd_cleanup()
55 struct io_kiocb *req; in io_uring_try_cancel_uring_cmd()
82 struct io_kiocb *req = cmd_to_io_kiocb(cmd); in io_uring_cmd_del_cancelable()
106 struct io_kiocb *req = cmd_to_io_kiocb(cmd); in io_uring_cmd_mark_cancelable()
131 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); in __io_uring_cmd_do_in_task()
141 static inline void io_req_set_cqe32_extra(struct io_kiocb *req, in io_req_set_cqe32_extra()
155 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); in __io_uring_cmd_done()
186 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_uring_cmd_prep()
223 static inline size_t uring_sqe_size(struct io_kiocb *req) in uring_sqe_size()
[all …]
|
| H A D | poll.c | 36 struct io_kiocb *req;
59 static inline struct io_kiocb *wqe_to_req(struct wait_queue_entry *wqe) in wqe_to_req()
63 return (struct io_kiocb *)(priv & ~IO_WQE_F_DOUBLE); in wqe_to_req()
73 static bool io_poll_get_ownership_slowpath(struct io_kiocb *req) in io_poll_get_ownership_slowpath()
94 static inline bool io_poll_get_ownership(struct io_kiocb *req) in io_poll_get_ownership()
101 static void io_poll_mark_cancelled(struct io_kiocb *req) in io_poll_mark_cancelled()
106 static struct io_poll *io_poll_get_double(struct io_kiocb *req) in io_poll_get_double()
114 static struct io_poll *io_poll_get_single(struct io_kiocb *req) in io_poll_get_single()
121 static void io_poll_req_insert(struct io_kiocb *req) in io_poll_req_insert()
171 static void io_poll_remove_entries(struct io_kiocb *req) in io_poll_remove_entries()
[all …]
|
| H A D | rw.c | 37 static bool io_file_supports_nowait(struct io_kiocb *req, __poll_t mask) in io_file_supports_nowait()
52 static int io_iov_buffer_select_prep(struct io_kiocb *req) in io_iov_buffer_select_prep()
69 static int io_import_vec(int ddir, struct io_kiocb *req, in io_import_vec()
96 static int __io_import_rw_buffer(int ddir, struct io_kiocb *req, in __io_import_rw_buffer()
118 static inline int io_import_rw_buffer(int rw, struct io_kiocb *req, in io_import_rw_buffer()
133 static bool io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags) in io_rw_recycle()
151 static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags) in io_req_rw_cleanup()
190 static int io_rw_alloc_async(struct io_kiocb *req) in io_rw_alloc_async()
218 static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, in io_prep_rw_pi()
245 static int __io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, in __io_prep_rw()
[all …]
|
| H A D | xattr.c | 25 void io_xattr_cleanup(struct io_kiocb *req) in io_xattr_cleanup()
34 static void io_xattr_finish(struct io_kiocb *req, int ret) in io_xattr_finish()
42 static int __io_getxattr_prep(struct io_kiocb *req, in __io_getxattr_prep()
74 int io_fgetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_fgetxattr_prep()
79 int io_getxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_getxattr_prep()
97 int io_fgetxattr(struct io_kiocb *req, unsigned int issue_flags) in io_fgetxattr()
109 int io_getxattr(struct io_kiocb *req, unsigned int issue_flags) in io_getxattr()
122 static int __io_setxattr_prep(struct io_kiocb *req, in __io_setxattr_prep()
151 int io_setxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_setxattr_prep()
169 int io_fsetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_fsetxattr_prep()
[all …]
|
| H A D | futex.h | 5 int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
6 int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
7 int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
8 int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags);
9 int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags);
10 int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags);
|
| H A D | fs.c | 50 int io_renameat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_renameat_prep()
82 int io_renameat(struct io_kiocb *req, unsigned int issue_flags) in io_renameat()
99 void io_renameat_cleanup(struct io_kiocb *req) in io_renameat_cleanup()
107 int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_unlinkat_prep()
135 int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) in io_unlinkat()
153 void io_unlinkat_cleanup(struct io_kiocb *req) in io_unlinkat_cleanup()
160 int io_mkdirat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_mkdirat_prep()
184 int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags) in io_mkdirat()
199 void io_mkdirat_cleanup(struct io_kiocb *req) in io_mkdirat_cleanup()
206 int io_symlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_symlinkat_prep()
[all …]
|
| H A D | openclose.c | 53 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in __io_openat_prep()
85 void io_openat_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req) in io_openat_bpf_populate()
94 int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_openat_prep()
104 int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_openat2_prep()
123 int io_openat2(struct io_kiocb *req, unsigned int issue_flags) in io_openat2()
188 int io_openat(struct io_kiocb *req, unsigned int issue_flags) in io_openat()
193 void io_open_cleanup(struct io_kiocb *req) in io_open_cleanup()
212 static inline int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) in io_close_fixed()
219 int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_close_prep()
236 int io_close(struct io_kiocb *req, unsigned int issue_flags) in io_close()
[all …]
|
| H A D | advise.h | 3 int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
4 int io_madvise(struct io_kiocb *req, unsigned int issue_flags);
6 int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
7 int io_fadvise(struct io_kiocb *req, unsigned int issue_flags);
|
| H A D | futex.c | 27 struct io_kiocb *req;
56 struct io_kiocb *req = tw_req.req; in io_futex_complete()
67 struct io_kiocb *req = tw_req.req; in io_futexv_complete()
92 static bool __io_futex_cancel(struct io_kiocb *req) in __io_futex_cancel()
127 int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_futex_prep()
155 int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_futex_wait_prep()
172 struct io_kiocb *req = q->wake_data; in io_futex_wakev_fn()
187 int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_futexv_prep()
233 struct io_kiocb *req = ifd->req; in io_futex_wake_fn()
243 int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) in io_futexv_wait()
[all …]
|
| H A D | uring_cmd.h | 11 int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags);
12 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
13 void io_uring_cmd_sqe_copy(struct io_kiocb *req);
14 void io_uring_cmd_cleanup(struct io_kiocb *req);
|
| H A D | msg_ring.c | 57 void io_msg_ring_cleanup(struct io_kiocb *req) in io_msg_ring_cleanup()
75 struct io_kiocb *req = tw_req.req; in io_msg_tw_complete()
83 static void io_msg_remote_post(struct io_ring_ctx *ctx, struct io_kiocb *req, in io_msg_remote_post()
116 struct io_kiocb *target; in io_msg_data_remote()
168 static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags) in io_msg_ring_data()
176 static int io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags) in io_msg_grab_file()
196 static int io_msg_install_complete(struct io_kiocb *req, unsigned int issue_flags) in io_msg_install_complete()
231 struct io_kiocb *req = cmd_to_io_kiocb(msg); in io_msg_tw_fd_complete()
241 static int io_msg_fd_remote(struct io_kiocb *req) in io_msg_fd_remote()
254 static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) in io_msg_send_fd()
[all …]
|
| H A D | tw.c | 72 struct io_kiocb *req; in tctx_task_work_run()
88 req = container_of(node, struct io_kiocb, io_task_work.node); in tctx_task_work_run()
151 void io_req_local_work_add(struct io_kiocb *req, unsigned flags) in io_req_local_work_add()
208 void io_req_normal_work_add(struct io_kiocb *req) in io_req_normal_work_add()
236 void io_req_task_work_add_remote(struct io_kiocb *req, unsigned flags) in io_req_task_work_add_remote()
259 struct io_kiocb *req; in io_cancel_local_task_work()
267 req = container_of(node, struct io_kiocb, io_task_work.node); in io_cancel_local_task_work()
293 struct io_kiocb *req; in __io_run_local_work_loop()
297 req = container_of(node, struct io_kiocb, io_task_work.node); in __io_run_local_work_loop()
|
| H A D | waitid.c | 35 static void io_waitid_free(struct io_kiocb *req) in io_waitid_free()
68 static bool io_waitid_copy_si(struct io_kiocb *req, int signo) in io_waitid_copy_si()
97 static int io_waitid_finish(struct io_kiocb *req, int ret) in io_waitid_finish()
112 static void io_waitid_remove_wq(struct io_kiocb *req) in io_waitid_remove_wq()
128 static void io_waitid_complete(struct io_kiocb *req, int ret, bool copy_si) in io_waitid_complete()
149 static bool __io_waitid_cancel(struct io_kiocb *req, bool copy_si) in __io_waitid_cancel()
170 static bool io_waitid_cancel_cb(struct io_kiocb *req) in io_waitid_cancel()
175 static bool io_waitid_cancel_nocopy_cb(struct io_kiocb *req) in io_waitid_remove_all()
193 static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req) in io_waitid_drop_issue_ref()
213 struct io_kiocb *re in io_waitid_cb()
[all...] |
| H A D | notif.h | 26 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx);
30 static inline struct io_notif_data *io_notif_to_data(struct io_kiocb *notif) in io_notif_to_data()
35 static inline void io_notif_flush(struct io_kiocb *notif) in io_notif_flush()
43 static inline int io_notif_account_mem(struct io_kiocb *notif, unsigned len) in io_notif_account_mem()
|
| H A D | io_uring.c | 122 static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags);
154 static void io_poison_cached_req(struct io_kiocb *req) in io_poison_cached_req()
164 void io_poison_req(struct io_kiocb *req) in io_poison_req()
174 static inline void req_fail_link_node(struct io_kiocb *req, int res) in req_fail_link_node()
180 static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx) in io_req_add_to_cache()
312 static void io_clean_op(struct io_kiocb *req) in io_clean_op()
339 inline void io_req_track_inflight(struct io_kiocb *req) in io_req_track_inflight()
347 static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req) in __io_prep_linked_timeout()
361 static void io_prep_async_work(struct io_kiocb *re
[all...] |
| H A D | cancel.c | 39 bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd) in io_cancel_req_match()
72 struct io_kiocb *req = container_of(work, struct io_kiocb, work); in io_cancel_cb()
140 int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_async_cancel_prep()
210 int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) in io_async_cancel()
359 bool (*cancel)(struct io_kiocb *)) in io_cancel_remove_all() argument
362 struct io_kiocb *req; in io_cancel_remove_all()
380 bool (*cancel)(struct io_kiocb *)) in io_cancel_remove() argument
383 struct io_kiocb *req; in io_cancel_remove()
399 static bool io_match_linked(struct io_kiocb *head) in io_match_linked()
401 struct io_kiocb *req; in io_match_linked()
[all …]
|
| H A D | statx.h | 3 int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
4 int io_statx(struct io_kiocb *req, unsigned int issue_flags);
5 void io_statx_cleanup(struct io_kiocb *req);
|
| H A D | rsrc.h | 62 struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
64 int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
68 struct io_kiocb *req, struct iou_vec *vec,
70 int io_prep_reg_iovec(struct io_kiocb *req, struct iou_vec *iv,
124 int io_files_update(struct io_kiocb *req, unsigned int issue_flags); in io_reset_rsrc_node()
125 int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); in io_reset_rsrc_node()
|