| /linux/io_uring/ |
| H A D | napi.h | 12 void io_napi_init(struct io_ring_ctx *ctx);
13 void io_napi_free(struct io_ring_ctx *ctx);
15 int io_register_napi(struct io_ring_ctx *ctx, void __user *arg);
16 int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg);
18 int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id,
21 void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq);
22 int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx);
24 static inline bool io_napi(struct io_ring_ctx *ctx) in io_napi()
29 static inline void io_napi_busy_loop(struct io_ring_ctx *ctx, in io_napi_busy_loop()
45 struct io_ring_ctx *ctx = req->ctx; in io_napi_add()
[all …]
|
| H A D | io_uring.h | 130 struct io_ring_ctx *ctx;
145 static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx) in io_get_rings()
154 struct io_ring_ctx *ctx = iowq->ctx; in io_should_wake()
175 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
177 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
178 void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
181 void __io_commit_cqring_flush(struct io_ring_ctx *ctx);
201 int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
202 int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
203 __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx);
[all …]
|
| H A D | rsrc.h | 59 bool io_rsrc_cache_init(struct io_ring_ctx *ctx);
60 void io_rsrc_cache_free(struct io_ring_ctx *ctx);
61 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type);
62 void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node);
63 void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data);
77 int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg);
78 int io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
79 int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
81 int io_sqe_files_unregister(struct io_ring_ctx *ctx);
82 int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
[all …]
|
| H A D | wait.h | 24 int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
26 int io_run_task_work_sig(struct io_ring_ctx *ctx);
27 void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx);
28 void io_cqring_overflow_flush_locked(struct io_ring_ctx *ctx);
30 static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) in __io_cqring_events()
36 static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx) in __io_cqring_events_user()
47 static inline unsigned io_cqring_events(struct io_ring_ctx *ctx) in io_cqring_events()
|
| H A D | napi.c | 41 int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id, in __io_napi_add_id()
90 static int __io_napi_del_id(struct io_ring_ctx *ctx, unsigned int napi_id) in __io_napi_del_id()
111 static void __io_napi_remove_stale(struct io_ring_ctx *ctx) in __io_napi_remove_stale()
131 static inline void io_napi_remove_stale(struct io_ring_ctx *ctx, bool is_stale) in io_napi_remove_stale()
169 static bool static_tracking_do_busy_loop(struct io_ring_ctx *ctx, in static_tracking_do_busy_loop()
182 dynamic_tracking_do_busy_loop(struct io_ring_ctx *ctx, in dynamic_tracking_do_busy_loop()
201 __io_napi_do_busy_loop(struct io_ring_ctx *ctx, in __io_napi_do_busy_loop()
215 static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, in io_napi_blocking_busy_loop()
248 void io_napi_init(struct io_ring_ctx *ctx) in io_napi_init()
265 void io_napi_free(struct io_ring_ctx *ctx) in io_napi_free()
[all …]
|
| H A D | tw.c | 20 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, in io_fallback_req_func()
36 static void ctx_flush_and_put(struct io_ring_ctx *ctx, io_tw_token_t tw) in ctx_flush_and_put()
57 struct io_ring_ctx *ctx = NULL; in io_handle_tw_list()
90 struct io_ring_ctx *last_ctx = NULL; in __io_fallback_tw()
159 static void io_ctx_mark_taskrun(struct io_ring_ctx *ctx) in io_ctx_mark_taskrun()
172 struct io_ring_ctx *ctx = req->ctx; in io_req_local_work_add()
242 struct io_ring_ctx *ctx = req->ctx; in io_req_normal_work_add()
274 void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) in io_move_task_work_from_local()
293 static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events, in io_run_local_work_continue()
326 static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw, in __io_run_local_work()
[all …]
|
| H A D | io_uring.c | 122 static void __io_req_caches_free(struct io_ring_ctx *ctx);
179 static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx) in io_req_add_to_cache()
188 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); in io_ring_ctx_ref_free()
215 static void io_free_alloc_caches(struct io_ring_ctx *ctx) in io_free_alloc_caches()
225 static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) in io_ring_ctx_alloc()
227 struct io_ring_ctx *ctx; in io_ring_ctx_alloc()
395 struct io_ring_ctx *ctx = req->ctx; in io_prep_async_link()
456 static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) in io_queue_deferred()
479 void __io_commit_cqring_flush(struct io_ring_ctx *ctx) in __io_commit_cqring_flush()
489 static inline void __io_cq_lock(struct io_ring_ctx *ctx) in __io_cq_lock()
[all …]
|
| H A D | sqpoll.h | 24 int io_sq_offload_create(struct io_ring_ctx *ctx, struct io_uring_params *p);
25 void io_sq_thread_finish(struct io_ring_ctx *ctx);
30 void io_sqpoll_wait_sq(struct io_ring_ctx *ctx);
31 int io_sqpoll_wq_cpu_affinity(struct io_ring_ctx *ctx, cpumask_var_t mask);
|
| H A D | rsrc.c | 30 static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
111 static struct io_mapped_ubuf *io_alloc_imu(struct io_ring_ctx *ctx, in io_alloc_imu()
119 static void io_free_imu(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu) in io_free_imu()
127 static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu) in io_buffer_unmap()
140 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type) in io_rsrc_node_alloc()
154 bool io_rsrc_cache_init(struct io_ring_ctx *ctx) in io_rsrc_cache_init()
168 void io_rsrc_cache_free(struct io_ring_ctx *ctx) in io_rsrc_cache_free()
186 __cold void io_rsrc_data_free(struct io_ring_ctx *ctx, in io_rsrc_data_free()
211 static int __io_sqe_files_update(struct io_ring_ctx *ctx, in __io_sqe_files_update()
279 static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, in __io_sqe_buffers_update()
[all …]
|
| H A D | kbuf.h | 76 void io_destroy_buffers(struct io_ring_ctx *ctx);
82 int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
83 int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
84 int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg);
94 struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx,
|
| H A D | kbuf.c | 78 static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx, in io_buffer_get_list()
86 static int io_buffer_add_list(struct io_ring_ctx *ctx, in io_buffer_add_list()
110 struct io_ring_ctx *ctx = req->ctx; in io_kbuf_recycle_legacy()
229 struct io_ring_ctx *ctx = req->ctx; in io_buffer_select()
334 struct io_ring_ctx *ctx = req->ctx; in io_buffers_select()
370 struct io_ring_ctx *ctx = req->ctx; in io_buffers_peek()
424 static int io_remove_buffers_legacy(struct io_ring_ctx *ctx, in io_remove_buffers_legacy()
445 static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) in io_put_bl()
455 void io_destroy_buffers(struct io_ring_ctx *ctx) in io_destroy_buffers()
473 static void io_destroy_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) in io_destroy_bl()
[all …]
|
| H A D | memmap.c | 184 int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, in io_create_region()
233 static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx, in io_mmap_get_region()
258 static void *io_region_validate_mmap(struct io_ring_ctx *ctx, in io_region_validate_mmap()
273 struct io_ring_ctx *ctx = file->private_data; in io_uring_validate_mmap_request()
284 static int io_region_mmap(struct io_ring_ctx *ctx, in io_region_mmap()
297 struct io_ring_ctx *ctx = file->private_data; in io_uring_mmap()
325 struct io_ring_ctx *ctx = filp->private_data; in io_uring_get_unmapped_area()
390 struct io_ring_ctx *ctx = file->private_data; in io_uring_mmap()
427 struct io_ring_ctx *ctx = file->private_data; in io_uring_get_unmapped_area()
|
| H A D | futex.c | 37 bool io_futex_cache_init(struct io_ring_ctx *ctx) in io_futex_cache_init()
43 void io_futex_cache_free(struct io_ring_ctx *ctx) in io_futex_cache_free()
57 struct io_ring_ctx *ctx = req->ctx; in io_futex_complete()
115 int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, in io_futex_cancel()
121 bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, in io_futex_remove_all()
227 struct io_ring_ctx *ctx = req->ctx; in io_futexv_wait()
279 struct io_ring_ctx *ctx = req->ctx; in io_futex_wait()
|
| H A D | timeout.c | 115 struct io_ring_ctx *ctx = req->ctx; in io_timeout_complete()
165 __cold void io_flush_timeouts(struct io_ring_ctx *ctx) in io_flush_timeouts()
263 struct io_ring_ctx *ctx = req->ctx; in io_disarm_next()
302 struct io_ring_ctx *ctx = req->ctx; in io_timeout_fn()
320 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, in io_timeout_extract()
347 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) in io_timeout_cancel()
395 struct io_ring_ctx *ctx = req->ctx; in io_link_timeout_fn()
425 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, in io_linked_timeout_update()
452 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, in io_timeout_update()
521 struct io_ring_ctx *ctx = req->ctx; in io_timeout_remove()
[all …]
|
| H A D | tctx.c | 16 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx, in io_init_wq_offload()
78 struct io_ring_ctx *ctx) in io_uring_alloc_task_context()
111 static int io_tctx_install_node(struct io_ring_ctx *ctx, in io_tctx_install_node()
139 int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) in __io_uring_add_tctx_node()
188 int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx) in __io_uring_add_tctx_node_from_submit()
308 int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg, in io_ringfd_register()
367 int io_ringfd_unregister(struct io_ring_ctx *ctx, void __user *__arg, in io_ringfd_unregister()
|
| H A D | register.c | 41 static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, in io_probe()
76 int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) in io_unregister_personality()
90 static int io_register_personality(struct io_ring_ctx *ctx) in io_register_personality()
167 static __cold int io_register_restrictions(struct io_ring_ctx *ctx, in io_register_restrictions()
277 static int io_register_enable_rings(struct io_ring_ctx *ctx) in io_register_enable_rings()
299 static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx, in __io_register_iowq_aff()
315 static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, in io_register_iowq_aff()
347 static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx) in io_unregister_iowq_aff()
352 static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx, in io_register_iowq_max_workers()
443 static int io_register_clock(struct io_ring_ctx *ctx, in io_register_clock()
[all …]
|
| H A D | fdinfo.c | 21 static __cold void common_tracking_show_fdinfo(struct io_ring_ctx *ctx, in common_tracking_show_fdinfo()
34 static __cold void napi_show_fdinfo(struct io_ring_ctx *ctx, in napi_show_fdinfo()
54 static inline void napi_show_fdinfo(struct io_ring_ctx *ctx, in napi_show_fdinfo()
60 static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) in __io_uring_show_fdinfo()
260 struct io_ring_ctx *ctx = file->private_data; in io_uring_show_fdinfo()
|
| H A D | wait.c | 32 int io_run_task_work_sig(struct io_ring_ctx *ctx) in io_run_task_work_sig()
73 struct io_ring_ctx *ctx = iowq->ctx; in io_cqring_min_timer_wakeup()
143 static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx, in __io_cqring_wait_schedule()
166 static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, in io_cqring_wait_schedule()
189 int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, in io_cqring_wait()
|
| H A D | eventfd.c | 76 void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) in io_eventfd_signal()
120 int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, in io_eventfd_register()
159 int io_eventfd_unregister(struct io_ring_ctx *ctx) in io_eventfd_unregister()
|
| H A D | notif.h | 26 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx);
45 struct io_ring_ctx *ctx = notif->ctx; in io_notif_account_mem()
|
| H A D | waitid.c | 167 int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, in io_waitid_cancel()
173 bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, in io_waitid_remove_all()
201 struct io_ring_ctx *ctx = req->ctx; in io_waitid_cb()
285 struct io_ring_ctx *ctx = req->ctx; in io_waitid()
|
| H A D | poll.c | 539 struct io_ring_ctx *ctx = req->ctx; in io_poll_add_hash()
657 struct io_ring_ctx *ctx = req->ctx; in io_req_alloc_apoll()
733 __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, in io_poll_remove_all()
758 static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, in io_poll_find()
779 static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx, in io_poll_file_find()
808 static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) in __io_poll_cancel()
825 int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, in io_poll_cancel()
919 struct io_ring_ctx *ctx = req->ctx; in io_poll_remove()
|
| H A D | zcrx.c | 371 static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx, in io_allocate_rbuf_ring()
524 static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) in io_zcrx_ifq_alloc()
669 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, in io_zcrx_get_region()
694 static int zcrx_export(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq, in zcrx_export()
728 static int import_zcrx(struct io_ring_ctx *ctx, in import_zcrx()
828 int io_register_zcrx(struct io_ring_ctx *ctx, in io_register_zcrx()
937 static inline bool is_zcrx_entry_marked(struct io_ring_ctx *ctx, unsigned long id) in is_zcrx_entry_marked()
942 static inline void set_zcrx_entry_mark(struct io_ring_ctx *ctx, unsigned long id) in set_zcrx_entry_mark()
947 void io_terminate_zcrx(struct io_ring_ctx *ctx) in io_terminate_zcrx()
967 void io_unregister_zcrx(struct io_ring_ctx *ctx) in io_unregister_zcrx()
[all …]
|
| H A D | uring_cmd.h | 16 bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
|
| /linux/include/linux/ |
| H A D | io_uring_types.h | 119 const struct io_ring_ctx *last;
293 struct io_ring_ctx { struct
369 int (*loop_step)(struct io_ring_ctx *ctx,
735 struct io_ring_ctx *ctx;
|