| /linux/io_uring/ |
| H A D | napi.h | 12 void io_napi_init(struct io_ring_ctx *ctx);
13 void io_napi_free(struct io_ring_ctx *ctx);
15 int io_register_napi(struct io_ring_ctx *ctx, void __user *arg);
16 int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg);
18 int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id,
21 void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq);
22 int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx);
24 static inline bool io_napi(struct io_ring_ctx *ctx) in io_napi()
29 static inline void io_napi_busy_loop(struct io_ring_ctx *ctx, in io_napi_busy_loop()
45 struct io_ring_ctx *ctx = req->ctx; in io_napi_add()
[all …]
|
| H A D | io_uring.h | 131 struct io_ring_ctx *ctx;
146 static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx) in io_get_rings()
155 struct io_ring_ctx *ctx = iowq->ctx; in io_should_wake()
176 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
178 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
179 void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
182 void __io_commit_cqring_flush(struct io_ring_ctx *ctx);
202 int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
203 int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
204 __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx);
[all …]
|
| H A D | zcrx.h | 89 struct io_ring_ctx *master_ctx;
97 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg);
98 int io_register_zcrx(struct io_ring_ctx *ctx,
100 void io_unregister_zcrx(struct io_ring_ctx *ctx);
101 void io_terminate_zcrx(struct io_ring_ctx *ctx);
105 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
108 static inline int io_register_zcrx(struct io_ring_ctx *ctx, in io_register_zcrx()
113 static inline void io_unregister_zcrx(struct io_ring_ctx *ctx) in io_unregister_zcrx()
116 static inline void io_terminate_zcrx(struct io_ring_ctx *ctx) in io_terminate_zcrx()
125 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, in io_zcrx_get_region()
[all …]
|
| H A D | futex.h | 13 int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
15 bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
17 bool io_futex_cache_init(struct io_ring_ctx *ctx);
18 void io_futex_cache_free(struct io_ring_ctx *ctx);
20 static inline int io_futex_cancel(struct io_ring_ctx *ctx, in io_futex_cancel()
26 static inline bool io_futex_remove_all(struct io_ring_ctx *ctx, in io_futex_remove_all()
31 static inline bool io_futex_cache_init(struct io_ring_ctx *ctx) in io_futex_cache_init()
35 static inline void io_futex_cache_free(struct io_ring_ctx *ctx) in io_futex_cache_free()
|
| H A D | tctx.h | 6 struct io_ring_ctx *ctx;
10 struct io_ring_ctx *ctx);
12 int __io_uring_add_tctx_node(struct io_ring_ctx *ctx);
13 int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx);
18 int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg,
20 int io_ringfd_unregister(struct io_ring_ctx *ctx, void __user *__arg,
26 static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx) in io_uring_add_tctx_node()
|
| H A D | napi.c | 41 int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id, in __io_napi_add_id()
90 static int __io_napi_del_id(struct io_ring_ctx *ctx, unsigned int napi_id) in __io_napi_del_id()
111 static void __io_napi_remove_stale(struct io_ring_ctx *ctx) in __io_napi_remove_stale()
131 static inline void io_napi_remove_stale(struct io_ring_ctx *ctx, bool is_stale) in io_napi_remove_stale()
169 static bool static_tracking_do_busy_loop(struct io_ring_ctx *ctx, in static_tracking_do_busy_loop()
182 dynamic_tracking_do_busy_loop(struct io_ring_ctx *ctx, in dynamic_tracking_do_busy_loop()
201 __io_napi_do_busy_loop(struct io_ring_ctx *ctx, in __io_napi_do_busy_loop()
215 static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, in io_napi_blocking_busy_loop()
248 void io_napi_init(struct io_ring_ctx *ctx) in io_napi_init()
265 void io_napi_free(struct io_ring_ctx *ctx) in io_napi_free()
[all …]
|
| H A D | msg_ring.c | 35 static void io_double_unlock_ctx(struct io_ring_ctx *octx) in io_double_unlock_ctx()
40 static int io_lock_external_ctx(struct io_ring_ctx *octx, in io_lock_external_ctx()
68 static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx) in io_msg_need_remote()
76 struct io_ring_ctx *ctx = req->ctx; in io_msg_tw_complete()
83 static void io_msg_remote_post(struct io_ring_ctx *ctx, struct io_kiocb *req, in io_msg_remote_post()
96 static int io_msg_ring_cqe_flags(struct io_ring_ctx *target_ctx, in io_msg_ring_cqe_flags()
113 static int io_msg_data_remote(struct io_ring_ctx *target_ctx, in io_msg_data_remote()
132 static int __io_msg_ring_data(struct io_ring_ctx *target_ctx, in __io_msg_ring_data()
170 struct io_ring_ctx *target_ctx = req->file->private_data; in io_msg_ring_data()
179 struct io_ring_ctx *ctx = req->ctx; in io_msg_grab_file()
[all …]
|
| H A D | tw.c | 19 static void ctx_flush_and_put(struct io_ring_ctx *ctx, io_tw_token_t tw) in ctx_flush_and_put()
66 struct io_ring_ctx *ctx = NULL; in tctx_task_work_run()
140 static void io_ctx_mark_taskrun(struct io_ring_ctx *ctx) in io_ctx_mark_taskrun()
153 struct io_ring_ctx *ctx = req->ctx; in io_req_local_work_add()
211 struct io_ring_ctx *ctx = req->ctx; in io_req_normal_work_add()
246 void __cold io_cancel_local_task_work(struct io_ring_ctx *ctx) in io_cancel_local_task_work()
276 static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events, in io_run_local_work_continue()
288 static int __io_run_local_work_loop(struct io_ring_ctx *ctx, in __io_run_local_work_loop()
310 static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw, in __io_run_local_work()
344 int io_run_local_work_locked(struct io_ring_ctx *ctx, int min_events) in io_run_local_work_locked()
[all …]
|
| H A D | loop.c | 7 static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx, in io_loop_nr_cqes()
13 static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait) in io_loop_wait_start()
19 static inline void io_loop_wait_finish(struct io_ring_ctx *ctx) in io_loop_wait_finish()
25 static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp, in io_loop_wait()
43 static int __io_run_loop(struct io_ring_ctx *ctx) in __io_run_loop()
81 int io_run_loop(struct io_ring_ctx *ctx) in io_run_loop()
|
| H A D | eventfd.h | 3 struct io_ring_ctx;
4 int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
6 int io_eventfd_unregister(struct io_ring_ctx *ctx);
8 void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer);
|
| H A D | io_uring.c | 123 static void __io_req_caches_free(struct io_ring_ctx *ctx);
180 static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx) in io_req_add_to_cache()
189 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); in io_ring_ctx_ref_free()
216 static void io_free_alloc_caches(struct io_ring_ctx *ctx) in io_free_alloc_caches()
226 static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) in io_ring_ctx_alloc()
228 struct io_ring_ctx *ctx; in io_ring_ctx_alloc()
398 struct io_ring_ctx *ctx = req->ctx; in io_prep_async_link()
448 static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) in io_queue_deferred()
471 void __io_commit_cqring_flush(struct io_ring_ctx *ctx) in __io_commit_cqring_flush()
481 static inline void __io_cq_lock(struct io_ring_ctx *ctx) in __io_cq_lock()
[all …]
|
| H A D | sqpoll.h | 24 int io_sq_offload_create(struct io_ring_ctx *ctx, struct io_uring_params *p);
25 void io_sq_thread_finish(struct io_ring_ctx *ctx);
30 void io_sqpoll_wait_sq(struct io_ring_ctx *ctx);
31 int io_sqpoll_wq_cpu_affinity(struct io_ring_ctx *ctx, cpumask_var_t mask);
|
| H A D | kbuf.h | 76 void io_destroy_buffers(struct io_ring_ctx *ctx);
82 int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
83 int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
84 int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg);
94 struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx,
|
| H A D | bpf-ops.c | 19 struct io_ring_ctx *ctx = io_loop_demangle_ctx(loop_ctx); in bpf_io_uring_submit_sqes()
28 struct io_ring_ctx *ctx = io_loop_demangle_ctx(loop_ctx); in bpf_io_uring_get_region()
162 static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops) in io_install_bpf()
185 struct io_ring_ctx *ctx; in bpf_io_reg()
203 static void io_eject_bpf(struct io_ring_ctx *ctx) in io_eject_bpf()
220 struct io_ring_ctx *ctx; in bpf_io_unreg()
233 void io_unregister_bpf_ops(struct io_ring_ctx *ctx) in io_unregister_bpf_ops()
|
| H A D | kbuf.c | 79 static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx, in io_buffer_get_list()
87 static int io_buffer_add_list(struct io_ring_ctx *ctx, in io_buffer_add_list()
111 struct io_ring_ctx *ctx = req->ctx; in io_kbuf_recycle_legacy()
234 struct io_ring_ctx *ctx = req->ctx; in io_buffer_select()
348 struct io_ring_ctx *ctx = req->ctx; in io_buffers_select()
384 struct io_ring_ctx *ctx = req->ctx; in io_buffers_peek()
438 static int io_remove_buffers_legacy(struct io_ring_ctx *ctx, in io_remove_buffers_legacy()
459 static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) in io_put_bl()
469 void io_destroy_buffers(struct io_ring_ctx *ctx) in io_destroy_buffers()
487 static void io_destroy_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) in io_destroy_bl()
[all …]
|
| H A D | futex.c | 37 bool io_futex_cache_init(struct io_ring_ctx *ctx) in io_futex_cache_init()
43 void io_futex_cache_free(struct io_ring_ctx *ctx) in io_futex_cache_free()
57 struct io_ring_ctx *ctx = req->ctx; in io_futex_complete()
115 int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, in io_futex_cancel()
121 bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, in io_futex_remove_all()
247 struct io_ring_ctx *ctx = req->ctx; in io_futexv_wait()
299 struct io_ring_ctx *ctx = req->ctx; in io_futex_wait()
|
| H A D | memmap.c | 256 static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx, in io_mmap_get_region()
281 static void *io_region_validate_mmap(struct io_ring_ctx *ctx, in io_region_validate_mmap()
296 struct io_ring_ctx *ctx = file->private_data; in io_uring_validate_mmap_request()
307 static int io_region_mmap(struct io_ring_ctx *ctx, in io_region_mmap()
320 struct io_ring_ctx *ctx = file->private_data; in io_uring_mmap()
348 struct io_ring_ctx *ctx = filp->private_data; in io_uring_get_unmapped_area()
413 struct io_ring_ctx *ctx = file->private_data; in io_uring_mmap()
450 struct io_ring_ctx *ctx = file->private_data; in io_uring_get_unmapped_area()
|
| H A D | cancel.c | 108 struct io_ring_ctx *ctx = cd->ctx; in io_try_cancel()
179 struct io_ring_ctx *ctx = cd->ctx; in __io_async_cancel()
250 struct io_ring_ctx *ctx = cd->ctx; in __io_sync_cancel()
268 int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg) in io_sync_cancel()
357 bool io_cancel_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, in io_cancel_remove_all()
378 int io_cancel_remove(struct io_ring_ctx *ctx, struct io_cancel_data *cd, in io_cancel_remove()
425 struct io_ring_ctx *ctx = head->ctx; in io_match_task_safe()
456 static __cold bool io_cancel_defer_files(struct io_ring_ctx *ctx, in io_cancel_defer_files()
489 static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) in io_uring_try_cancel_iowq()
515 __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, in io_uring_try_cancel_requests()
[all …]
|
| H A D | timeout.c | 115 struct io_ring_ctx *ctx = req->ctx; in io_timeout_complete()
165 __cold void io_flush_timeouts(struct io_ring_ctx *ctx) in io_flush_timeouts()
263 struct io_ring_ctx *ctx = req->ctx; in io_disarm_next()
306 struct io_ring_ctx *ctx = req->ctx; in io_timeout_fn()
324 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, in io_timeout_extract()
351 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) in io_timeout_cancel()
407 struct io_ring_ctx *ctx = req->ctx; in io_link_timeout_fn()
439 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, in io_linked_timeout_update()
466 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, in io_timeout_update()
535 struct io_ring_ctx *ctx = req->ctx; in io_timeout_remove()
[all …]
|
| H A D | tctx.c | 16 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx, in io_init_wq_offload()
83 struct io_ring_ctx *ctx) in io_uring_alloc_task_context()
117 static int io_tctx_install_node(struct io_ring_ctx *ctx, in io_tctx_install_node()
145 int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) in __io_uring_add_tctx_node()
198 int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx) in __io_uring_add_tctx_node_from_submit()
318 int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg, in io_ringfd_register()
377 int io_ringfd_unregister(struct io_ring_ctx *ctx, void __user *__arg, in io_ringfd_unregister()
|
| H A D | register.c | 41 static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, in io_probe()
76 int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) in io_unregister_personality()
90 static int io_register_personality(struct io_ring_ctx *ctx) in io_register_personality()
167 static __cold int io_register_restrictions(struct io_ring_ctx *ctx, in io_register_restrictions()
277 static int io_register_enable_rings(struct io_ring_ctx *ctx) in io_register_enable_rings()
299 static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx, in __io_register_iowq_aff()
315 static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, in io_register_iowq_aff()
347 static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx) in io_unregister_iowq_aff()
352 static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx, in io_register_iowq_max_workers()
443 static int io_register_clock(struct io_ring_ctx *ctx, in io_register_clock()
[all …]
|
| H A D | sqpoll.c | 95 struct io_ring_ctx *ctx; in io_sqd_update_thread_idle()
103 void io_sq_thread_finish(struct io_ring_ctx *ctx) in io_sq_thread_finish()
120 struct io_ring_ctx *ctx_attach; in io_attach_sq_data()
205 static int __io_sq_thread(struct io_ring_ctx *ctx, struct io_sq_data *sqd, in __io_sq_thread()
288 struct io_ring_ctx *ctx; in io_sq_thread()
424 void io_sqpoll_wait_sq(struct io_ring_ctx *ctx) in io_sqpoll_wait_sq()
441 __cold int io_sq_offload_create(struct io_ring_ctx *ctx, in io_sq_offload_create()
546 __cold int io_sqpoll_wq_cpu_affinity(struct io_ring_ctx *ctx, in io_sqpoll_wq_cpu_affinity()
|
| H A D | eventfd.c | 76 void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer) in io_eventfd_signal()
120 int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, in io_eventfd_register()
159 int io_eventfd_unregister(struct io_ring_ctx *ctx) in io_eventfd_unregister()
|
| H A D | notif.h | 26 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx);
45 struct io_ring_ctx *ctx = notif->ctx; in io_notif_account_mem()
|
| H A D | zcrx.c | 66 struct io_ring_ctx *ctx) in zcrx_set_ring_ctx()
415 static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx, in io_allocate_rbuf_ring()
600 static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) in io_zcrx_ifq_alloc()
749 static void zcrx_unregister_user(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) in zcrx_unregister_user()
764 static void zcrx_unregister(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) in zcrx_unregister()
770 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, in io_zcrx_get_region()
795 static int zcrx_export(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq, in zcrx_export()
832 static int import_zcrx(struct io_ring_ctx *ctx, in import_zcrx()
962 int io_register_zcrx(struct io_ring_ctx *ctx, in io_register_zcrx()
1103 static inline bool is_zcrx_entry_marked(struct io_ring_ctx *ctx, unsigned long id) in is_zcrx_entry_marked()
[all …]
|