xref: /linux/include/linux/io_uring_types.h (revision 91928e0d3cc29789f4483bffee5f36218f23942b)
1 #ifndef IO_URING_TYPES_H
2 #define IO_URING_TYPES_H
3 
4 #include <linux/blkdev.h>
5 #include <linux/hashtable.h>
6 #include <linux/task_work.h>
7 #include <linux/bitmap.h>
8 #include <linux/llist.h>
9 #include <uapi/linux/io_uring.h>
10 
11 enum {
12 	/*
13 	 * A hint to not wake right away but delay until there are enough of
14 	 * tw's queued to match the number of CQEs the task is waiting for.
15 	 *
16 	 * Must not be used with requests generating more than one CQE.
17 	 * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set.
18 	 */
19 	IOU_F_TWQ_LAZY_WAKE			= 1,
20 };
21 
22 enum io_uring_cmd_flags {
23 	IO_URING_F_COMPLETE_DEFER	= 1,
24 	IO_URING_F_UNLOCKED		= 2,
25 	/* the request is executed from poll, it should not be freed */
26 	IO_URING_F_MULTISHOT		= 4,
27 	/* executed by io-wq */
28 	IO_URING_F_IOWQ			= 8,
29 	/* int's last bit, sign checks are usually faster than a bit test */
30 	IO_URING_F_NONBLOCK		= INT_MIN,
31 
32 	/* ctx state flags, for URING_CMD */
33 	IO_URING_F_SQE128		= (1 << 8),
34 	IO_URING_F_CQE32		= (1 << 9),
35 	IO_URING_F_IOPOLL		= (1 << 10),
36 
37 	/* set when uring wants to cancel a previously issued command */
38 	IO_URING_F_CANCEL		= (1 << 11),
39 	IO_URING_F_COMPAT		= (1 << 12),
40 	IO_URING_F_TASK_DEAD		= (1 << 13),
41 };
42 
43 struct io_wq_work_node {
44 	struct io_wq_work_node *next;
45 };
46 
47 struct io_wq_work_list {
48 	struct io_wq_work_node *first;
49 	struct io_wq_work_node *last;
50 };
51 
52 struct io_wq_work {
53 	struct io_wq_work_node list;
54 	atomic_t flags;
55 	/* place it here instead of io_kiocb as it fills padding and saves 4B */
56 	int cancel_seq;
57 };
58 
59 struct io_rsrc_data {
60 	unsigned int			nr;
61 	struct io_rsrc_node		**nodes;
62 };
63 
64 struct io_file_table {
65 	struct io_rsrc_data data;
66 	unsigned long *bitmap;
67 	unsigned int alloc_hint;
68 };
69 
70 struct io_hash_bucket {
71 	struct hlist_head	list;
72 } ____cacheline_aligned_in_smp;
73 
74 struct io_hash_table {
75 	struct io_hash_bucket	*hbs;
76 	unsigned		hash_bits;
77 };
78 
79 struct io_mapped_region {
80 	struct page		**pages;
81 	void			*ptr;
82 	unsigned		nr_pages;
83 	unsigned		flags;
84 };
85 
86 /*
87  * Arbitrary limit, can be raised if need be
88  */
89 #define IO_RINGFD_REG_MAX 16
90 
91 struct io_uring_task {
92 	/* submission side */
93 	int				cached_refs;
94 	const struct io_ring_ctx 	*last;
95 	struct task_struct		*task;
96 	struct io_wq			*io_wq;
97 	struct file			*registered_rings[IO_RINGFD_REG_MAX];
98 
99 	struct xarray			xa;
100 	struct wait_queue_head		wait;
101 	atomic_t			in_cancel;
102 	atomic_t			inflight_tracked;
103 	struct percpu_counter		inflight;
104 
105 	struct { /* task_work */
106 		struct llist_head	task_list;
107 		struct callback_head	task_work;
108 	} ____cacheline_aligned_in_smp;
109 };
110 
111 struct io_uring {
112 	u32 head;
113 	u32 tail;
114 };
115 
116 /*
117  * This data is shared with the application through the mmap at offsets
118  * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
119  *
120  * The offsets to the member fields are published through struct
121  * io_sqring_offsets when calling io_uring_setup.
122  */
123 struct io_rings {
124 	/*
125 	 * Head and tail offsets into the ring; the offsets need to be
126 	 * masked to get valid indices.
127 	 *
128 	 * The kernel controls head of the sq ring and the tail of the cq ring,
129 	 * and the application controls tail of the sq ring and the head of the
130 	 * cq ring.
131 	 */
132 	struct io_uring		sq, cq;
133 	/*
134 	 * Bitmasks to apply to head and tail offsets (constant, equals
135 	 * ring_entries - 1)
136 	 */
137 	u32			sq_ring_mask, cq_ring_mask;
138 	/* Ring sizes (constant, power of 2) */
139 	u32			sq_ring_entries, cq_ring_entries;
140 	/*
141 	 * Number of invalid entries dropped by the kernel due to
142 	 * invalid index stored in array
143 	 *
144 	 * Written by the kernel, shouldn't be modified by the
145 	 * application (i.e. get number of "new events" by comparing to
146 	 * cached value).
147 	 *
148 	 * After a new SQ head value was read by the application this
149 	 * counter includes all submissions that were dropped reaching
150 	 * the new SQ head (and possibly more).
151 	 */
152 	u32			sq_dropped;
153 	/*
154 	 * Runtime SQ flags
155 	 *
156 	 * Written by the kernel, shouldn't be modified by the
157 	 * application.
158 	 *
159 	 * The application needs a full memory barrier before checking
160 	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
161 	 */
162 	atomic_t		sq_flags;
163 	/*
164 	 * Runtime CQ flags
165 	 *
166 	 * Written by the application, shouldn't be modified by the
167 	 * kernel.
168 	 */
169 	u32			cq_flags;
170 	/*
171 	 * Number of completion events lost because the queue was full;
172 	 * this should be avoided by the application by making sure
173 	 * there are not more requests pending than there is space in
174 	 * the completion queue.
175 	 *
176 	 * Written by the kernel, shouldn't be modified by the
177 	 * application (i.e. get number of "new events" by comparing to
178 	 * cached value).
179 	 *
180 	 * As completion events come in out of order this counter is not
181 	 * ordered with any other data.
182 	 */
183 	u32			cq_overflow;
184 	/*
185 	 * Ring buffer of completion events.
186 	 *
187 	 * The kernel writes completion events fresh every time they are
188 	 * produced, so the application is allowed to modify pending
189 	 * entries.
190 	 */
191 	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
192 };
193 
194 struct io_restriction {
195 	DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
196 	DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
197 	u8 sqe_flags_allowed;
198 	u8 sqe_flags_required;
199 	bool registered;
200 };
201 
202 struct io_submit_link {
203 	struct io_kiocb		*head;
204 	struct io_kiocb		*last;
205 };
206 
207 struct io_submit_state {
208 	/* inline/task_work completion list, under ->uring_lock */
209 	struct io_wq_work_node	free_list;
210 	/* batch completion logic */
211 	struct io_wq_work_list	compl_reqs;
212 	struct io_submit_link	link;
213 
214 	bool			plug_started;
215 	bool			need_plug;
216 	bool			cq_flush;
217 	unsigned short		submit_nr;
218 	struct blk_plug		plug;
219 };
220 
221 struct io_alloc_cache {
222 	void			**entries;
223 	unsigned int		nr_cached;
224 	unsigned int		max_cached;
225 	unsigned int		elem_size;
226 	unsigned int		init_clear;
227 };
228 
229 struct io_ring_ctx {
230 	/* const or read-mostly hot data */
231 	struct {
232 		unsigned int		flags;
233 		unsigned int		drain_next: 1;
234 		unsigned int		restricted: 1;
235 		unsigned int		off_timeout_used: 1;
236 		unsigned int		drain_active: 1;
237 		unsigned int		has_evfd: 1;
238 		/* all CQEs should be posted only by the submitter task */
239 		unsigned int		task_complete: 1;
240 		unsigned int		lockless_cq: 1;
241 		unsigned int		syscall_iopoll: 1;
242 		unsigned int		poll_activated: 1;
243 		unsigned int		drain_disabled: 1;
244 		unsigned int		compat: 1;
245 		unsigned int		iowq_limits_set : 1;
246 
247 		struct task_struct	*submitter_task;
248 		struct io_rings		*rings;
249 		struct percpu_ref	refs;
250 
251 		clockid_t		clockid;
252 		enum tk_offsets		clock_offset;
253 
254 		enum task_work_notify_mode	notify_method;
255 		unsigned			sq_thread_idle;
256 	} ____cacheline_aligned_in_smp;
257 
258 	/* submission data */
259 	struct {
260 		struct mutex		uring_lock;
261 
262 		/*
263 		 * Ring buffer of indices into array of io_uring_sqe, which is
264 		 * mmapped by the application using the IORING_OFF_SQES offset.
265 		 *
266 		 * This indirection could e.g. be used to assign fixed
267 		 * io_uring_sqe entries to operations and only submit them to
268 		 * the queue when needed.
269 		 *
270 		 * The kernel modifies neither the indices array nor the entries
271 		 * array.
272 		 */
273 		u32			*sq_array;
274 		struct io_uring_sqe	*sq_sqes;
275 		unsigned		cached_sq_head;
276 		unsigned		sq_entries;
277 
278 		/*
279 		 * Fixed resources fast path, should be accessed only under
280 		 * uring_lock, and updated through io_uring_register(2)
281 		 */
282 		atomic_t		cancel_seq;
283 
284 		/*
285 		 * ->iopoll_list is protected by the ctx->uring_lock for
286 		 * io_uring instances that don't use IORING_SETUP_SQPOLL.
287 		 * For SQPOLL, only the single threaded io_sq_thread() will
288 		 * manipulate the list, hence no extra locking is needed there.
289 		 */
290 		bool			poll_multi_queue;
291 		struct io_wq_work_list	iopoll_list;
292 
293 		struct io_file_table	file_table;
294 		struct io_rsrc_data	buf_table;
295 		struct io_alloc_cache	node_cache;
296 		struct io_alloc_cache	imu_cache;
297 
298 		struct io_submit_state	submit_state;
299 
300 		/*
301 		 * Modifications are protected by ->uring_lock and ->mmap_lock.
302 		 * The flags, buf_pages and buf_nr_pages fields should be stable
303 		 * once published.
304 		 */
305 		struct xarray		io_bl_xa;
306 
307 		struct io_hash_table	cancel_table;
308 		struct io_alloc_cache	apoll_cache;
309 		struct io_alloc_cache	netmsg_cache;
310 		struct io_alloc_cache	rw_cache;
311 		struct io_alloc_cache	uring_cache;
312 
313 		/*
314 		 * Any cancelable uring_cmd is added to this list in
315 		 * ->uring_cmd() by io_uring_cmd_insert_cancelable()
316 		 */
317 		struct hlist_head	cancelable_uring_cmd;
318 		/*
319 		 * For Hybrid IOPOLL, runtime in hybrid polling, without
320 		 * scheduling time
321 		 */
322 		u64					hybrid_poll_time;
323 	} ____cacheline_aligned_in_smp;
324 
325 	struct {
326 		/*
327 		 * We cache a range of free CQEs we can use, once exhausted it
328 		 * should go through a slower range setup, see __io_get_cqe()
329 		 */
330 		struct io_uring_cqe	*cqe_cached;
331 		struct io_uring_cqe	*cqe_sentinel;
332 
333 		unsigned		cached_cq_tail;
334 		unsigned		cq_entries;
335 		struct io_ev_fd	__rcu	*io_ev_fd;
336 		unsigned		cq_extra;
337 
338 		void			*cq_wait_arg;
339 		size_t			cq_wait_size;
340 	} ____cacheline_aligned_in_smp;
341 
342 	/*
343 	 * task_work and async notification delivery cacheline. Expected to
344 	 * regularly bounce b/w CPUs.
345 	 */
346 	struct {
347 		struct llist_head	work_llist;
348 		struct llist_head	retry_llist;
349 		unsigned long		check_cq;
350 		atomic_t		cq_wait_nr;
351 		atomic_t		cq_timeouts;
352 		struct wait_queue_head	cq_wait;
353 	} ____cacheline_aligned_in_smp;
354 
355 	/* timeouts */
356 	struct {
357 		raw_spinlock_t		timeout_lock;
358 		struct list_head	timeout_list;
359 		struct list_head	ltimeout_list;
360 		unsigned		cq_last_tm_flush;
361 	} ____cacheline_aligned_in_smp;
362 
363 	spinlock_t		completion_lock;
364 
365 	struct list_head	cq_overflow_list;
366 
367 	struct hlist_head	waitid_list;
368 
369 #ifdef CONFIG_FUTEX
370 	struct hlist_head	futex_list;
371 	struct io_alloc_cache	futex_cache;
372 #endif
373 
374 	const struct cred	*sq_creds;	/* cred used for __io_sq_thread() */
375 	struct io_sq_data	*sq_data;	/* if using sq thread polling */
376 
377 	struct wait_queue_head	sqo_sq_wait;
378 	struct list_head	sqd_list;
379 
380 	unsigned int		file_alloc_start;
381 	unsigned int		file_alloc_end;
382 
383 	/* Keep this last, we don't need it for the fast path */
384 	struct wait_queue_head		poll_wq;
385 	struct io_restriction		restrictions;
386 
387 	u32			pers_next;
388 	struct xarray		personalities;
389 
390 	/* hashed buffered write serialization */
391 	struct io_wq_hash		*hash_map;
392 
393 	/* Only used for accounting purposes */
394 	struct user_struct		*user;
395 	struct mm_struct		*mm_account;
396 
397 	/* ctx exit and cancelation */
398 	struct llist_head		fallback_llist;
399 	struct delayed_work		fallback_work;
400 	struct work_struct		exit_work;
401 	struct list_head		tctx_list;
402 	struct completion		ref_comp;
403 
404 	/* io-wq management, e.g. thread count */
405 	u32				iowq_limits[2];
406 
407 	struct callback_head		poll_wq_task_work;
408 	struct list_head		defer_list;
409 
410 	struct io_alloc_cache		msg_cache;
411 	spinlock_t			msg_lock;
412 
413 #ifdef CONFIG_NET_RX_BUSY_POLL
414 	struct list_head	napi_list;	/* track busy poll napi_id */
415 	spinlock_t		napi_lock;	/* napi_list lock */
416 
417 	/* napi busy poll default timeout */
418 	ktime_t			napi_busy_poll_dt;
419 	bool			napi_prefer_busy_poll;
420 	u8			napi_track_mode;
421 
422 	DECLARE_HASHTABLE(napi_ht, 4);
423 #endif
424 
425 	/* protected by ->completion_lock */
426 	unsigned			evfd_last_cq_tail;
427 
428 	/*
429 	 * Protection for resize vs mmap races - both the mmap and resize
430 	 * side will need to grab this lock, to prevent either side from
431 	 * being run concurrently with the other.
432 	 */
433 	struct mutex			mmap_lock;
434 
435 	struct io_mapped_region		sq_region;
436 	struct io_mapped_region		ring_region;
437 	/* used for optimised request parameter and wait argument passing  */
438 	struct io_mapped_region		param_region;
439 };
440 
441 /*
442  * Token indicating function is called in task work context:
443  * ctx->uring_lock is held and any completions generated will be flushed.
444  * ONLY core io_uring.c should instantiate this struct.
445  */
446 struct io_tw_state {
447 };
448 /* Alias to use in code that doesn't instantiate struct io_tw_state */
449 typedef struct io_tw_state io_tw_token_t;
450 
451 enum {
452 	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
453 	REQ_F_IO_DRAIN_BIT	= IOSQE_IO_DRAIN_BIT,
454 	REQ_F_LINK_BIT		= IOSQE_IO_LINK_BIT,
455 	REQ_F_HARDLINK_BIT	= IOSQE_IO_HARDLINK_BIT,
456 	REQ_F_FORCE_ASYNC_BIT	= IOSQE_ASYNC_BIT,
457 	REQ_F_BUFFER_SELECT_BIT	= IOSQE_BUFFER_SELECT_BIT,
458 	REQ_F_CQE_SKIP_BIT	= IOSQE_CQE_SKIP_SUCCESS_BIT,
459 
460 	/* first byte is taken by user flags, shift it to not overlap */
461 	REQ_F_FAIL_BIT		= 8,
462 	REQ_F_INFLIGHT_BIT,
463 	REQ_F_CUR_POS_BIT,
464 	REQ_F_NOWAIT_BIT,
465 	REQ_F_LINK_TIMEOUT_BIT,
466 	REQ_F_NEED_CLEANUP_BIT,
467 	REQ_F_POLLED_BIT,
468 	REQ_F_HYBRID_IOPOLL_STATE_BIT,
469 	REQ_F_BUFFER_SELECTED_BIT,
470 	REQ_F_BUFFER_RING_BIT,
471 	REQ_F_REISSUE_BIT,
472 	REQ_F_CREDS_BIT,
473 	REQ_F_REFCOUNT_BIT,
474 	REQ_F_ARM_LTIMEOUT_BIT,
475 	REQ_F_ASYNC_DATA_BIT,
476 	REQ_F_SKIP_LINK_CQES_BIT,
477 	REQ_F_SINGLE_POLL_BIT,
478 	REQ_F_DOUBLE_POLL_BIT,
479 	REQ_F_APOLL_MULTISHOT_BIT,
480 	REQ_F_CLEAR_POLLIN_BIT,
481 	/* keep async read/write and isreg together and in order */
482 	REQ_F_SUPPORT_NOWAIT_BIT,
483 	REQ_F_ISREG_BIT,
484 	REQ_F_POLL_NO_LAZY_BIT,
485 	REQ_F_CAN_POLL_BIT,
486 	REQ_F_BL_EMPTY_BIT,
487 	REQ_F_BL_NO_RECYCLE_BIT,
488 	REQ_F_BUFFERS_COMMIT_BIT,
489 	REQ_F_BUF_NODE_BIT,
490 	REQ_F_HAS_METADATA_BIT,
491 
492 	/* not a real bit, just to check we're not overflowing the space */
493 	__REQ_F_LAST_BIT,
494 };
495 
496 typedef u64 __bitwise io_req_flags_t;
497 #define IO_REQ_FLAG(bitno)	((__force io_req_flags_t) BIT_ULL((bitno)))
498 
499 enum {
500 	/* ctx owns file */
501 	REQ_F_FIXED_FILE	= IO_REQ_FLAG(REQ_F_FIXED_FILE_BIT),
502 	/* drain existing IO first */
503 	REQ_F_IO_DRAIN		= IO_REQ_FLAG(REQ_F_IO_DRAIN_BIT),
504 	/* linked sqes */
505 	REQ_F_LINK		= IO_REQ_FLAG(REQ_F_LINK_BIT),
506 	/* doesn't sever on completion < 0 */
507 	REQ_F_HARDLINK		= IO_REQ_FLAG(REQ_F_HARDLINK_BIT),
508 	/* IOSQE_ASYNC */
509 	REQ_F_FORCE_ASYNC	= IO_REQ_FLAG(REQ_F_FORCE_ASYNC_BIT),
510 	/* IOSQE_BUFFER_SELECT */
511 	REQ_F_BUFFER_SELECT	= IO_REQ_FLAG(REQ_F_BUFFER_SELECT_BIT),
512 	/* IOSQE_CQE_SKIP_SUCCESS */
513 	REQ_F_CQE_SKIP		= IO_REQ_FLAG(REQ_F_CQE_SKIP_BIT),
514 
515 	/* fail rest of links */
516 	REQ_F_FAIL		= IO_REQ_FLAG(REQ_F_FAIL_BIT),
517 	/* on inflight list, should be cancelled and waited on exit reliably */
518 	REQ_F_INFLIGHT		= IO_REQ_FLAG(REQ_F_INFLIGHT_BIT),
519 	/* read/write uses file position */
520 	REQ_F_CUR_POS		= IO_REQ_FLAG(REQ_F_CUR_POS_BIT),
521 	/* must not punt to workers */
522 	REQ_F_NOWAIT		= IO_REQ_FLAG(REQ_F_NOWAIT_BIT),
523 	/* has or had linked timeout */
524 	REQ_F_LINK_TIMEOUT	= IO_REQ_FLAG(REQ_F_LINK_TIMEOUT_BIT),
525 	/* needs cleanup */
526 	REQ_F_NEED_CLEANUP	= IO_REQ_FLAG(REQ_F_NEED_CLEANUP_BIT),
527 	/* already went through poll handler */
528 	REQ_F_POLLED		= IO_REQ_FLAG(REQ_F_POLLED_BIT),
529 	/* every req only blocks once in hybrid poll */
530 	REQ_F_IOPOLL_STATE        = IO_REQ_FLAG(REQ_F_HYBRID_IOPOLL_STATE_BIT),
531 	/* buffer already selected */
532 	REQ_F_BUFFER_SELECTED	= IO_REQ_FLAG(REQ_F_BUFFER_SELECTED_BIT),
533 	/* buffer selected from ring, needs commit */
534 	REQ_F_BUFFER_RING	= IO_REQ_FLAG(REQ_F_BUFFER_RING_BIT),
535 	/* caller should reissue async */
536 	REQ_F_REISSUE		= IO_REQ_FLAG(REQ_F_REISSUE_BIT),
537 	/* supports async reads/writes */
538 	REQ_F_SUPPORT_NOWAIT	= IO_REQ_FLAG(REQ_F_SUPPORT_NOWAIT_BIT),
539 	/* regular file */
540 	REQ_F_ISREG		= IO_REQ_FLAG(REQ_F_ISREG_BIT),
541 	/* has creds assigned */
542 	REQ_F_CREDS		= IO_REQ_FLAG(REQ_F_CREDS_BIT),
543 	/* skip refcounting if not set */
544 	REQ_F_REFCOUNT		= IO_REQ_FLAG(REQ_F_REFCOUNT_BIT),
545 	/* there is a linked timeout that has to be armed */
546 	REQ_F_ARM_LTIMEOUT	= IO_REQ_FLAG(REQ_F_ARM_LTIMEOUT_BIT),
547 	/* ->async_data allocated */
548 	REQ_F_ASYNC_DATA	= IO_REQ_FLAG(REQ_F_ASYNC_DATA_BIT),
549 	/* don't post CQEs while failing linked requests */
550 	REQ_F_SKIP_LINK_CQES	= IO_REQ_FLAG(REQ_F_SKIP_LINK_CQES_BIT),
551 	/* single poll may be active */
552 	REQ_F_SINGLE_POLL	= IO_REQ_FLAG(REQ_F_SINGLE_POLL_BIT),
553 	/* double poll may active */
554 	REQ_F_DOUBLE_POLL	= IO_REQ_FLAG(REQ_F_DOUBLE_POLL_BIT),
555 	/* fast poll multishot mode */
556 	REQ_F_APOLL_MULTISHOT	= IO_REQ_FLAG(REQ_F_APOLL_MULTISHOT_BIT),
557 	/* recvmsg special flag, clear EPOLLIN */
558 	REQ_F_CLEAR_POLLIN	= IO_REQ_FLAG(REQ_F_CLEAR_POLLIN_BIT),
559 	/* don't use lazy poll wake for this request */
560 	REQ_F_POLL_NO_LAZY	= IO_REQ_FLAG(REQ_F_POLL_NO_LAZY_BIT),
561 	/* file is pollable */
562 	REQ_F_CAN_POLL		= IO_REQ_FLAG(REQ_F_CAN_POLL_BIT),
563 	/* buffer list was empty after selection of buffer */
564 	REQ_F_BL_EMPTY		= IO_REQ_FLAG(REQ_F_BL_EMPTY_BIT),
565 	/* don't recycle provided buffers for this request */
566 	REQ_F_BL_NO_RECYCLE	= IO_REQ_FLAG(REQ_F_BL_NO_RECYCLE_BIT),
567 	/* buffer ring head needs incrementing on put */
568 	REQ_F_BUFFERS_COMMIT	= IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT),
569 	/* buf node is valid */
570 	REQ_F_BUF_NODE		= IO_REQ_FLAG(REQ_F_BUF_NODE_BIT),
571 	/* request has read/write metadata assigned */
572 	REQ_F_HAS_METADATA	= IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),
573 };
574 
575 typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);
576 
577 struct io_task_work {
578 	struct llist_node		node;
579 	io_req_tw_func_t		func;
580 };
581 
582 struct io_cqe {
583 	__u64	user_data;
584 	__s32	res;
585 	/* fd initially, then cflags for completion */
586 	union {
587 		__u32	flags;
588 		int	fd;
589 	};
590 };
591 
592 /*
593  * Each request type overlays its private data structure on top of this one.
594  * They must not exceed this one in size.
595  */
596 struct io_cmd_data {
597 	struct file		*file;
598 	/* each command gets 56 bytes of data */
599 	__u8			data[56];
600 };
601 
io_kiocb_cmd_sz_check(size_t cmd_sz)602 static inline void io_kiocb_cmd_sz_check(size_t cmd_sz)
603 {
604 	BUILD_BUG_ON(cmd_sz > sizeof(struct io_cmd_data));
605 }
606 #define io_kiocb_to_cmd(req, cmd_type) ( \
607 	io_kiocb_cmd_sz_check(sizeof(cmd_type)) , \
608 	((cmd_type *)&(req)->cmd) \
609 )
610 
cmd_to_io_kiocb(void * ptr)611 static inline struct io_kiocb *cmd_to_io_kiocb(void *ptr)
612 {
613 	return ptr;
614 }
615 
616 struct io_kiocb {
617 	union {
618 		/*
619 		 * NOTE! Each of the io_kiocb union members has the file pointer
620 		 * as the first entry in their struct definition. So you can
621 		 * access the file pointer through any of the sub-structs,
622 		 * or directly as just 'file' in this struct.
623 		 */
624 		struct file		*file;
625 		struct io_cmd_data	cmd;
626 	};
627 
628 	u8				opcode;
629 	/* polled IO has completed */
630 	u8				iopoll_completed;
631 	/*
632 	 * Can be either a fixed buffer index, or used with provided buffers.
633 	 * For the latter, before issue it points to the buffer group ID,
634 	 * and after selection it points to the buffer ID itself.
635 	 */
636 	u16				buf_index;
637 
638 	unsigned			nr_tw;
639 
640 	/* REQ_F_* flags */
641 	io_req_flags_t			flags;
642 
643 	struct io_cqe			cqe;
644 
645 	struct io_ring_ctx		*ctx;
646 	struct io_uring_task		*tctx;
647 
648 	union {
649 		/* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
650 		struct io_buffer	*kbuf;
651 
652 		/*
653 		 * stores buffer ID for ring provided buffers, valid IFF
654 		 * REQ_F_BUFFER_RING is set.
655 		 */
656 		struct io_buffer_list	*buf_list;
657 
658 		struct io_rsrc_node	*buf_node;
659 	};
660 
661 	union {
662 		/* used by request caches, completion batching and iopoll */
663 		struct io_wq_work_node	comp_list;
664 		/* cache ->apoll->events */
665 		__poll_t apoll_events;
666 	};
667 
668 	struct io_rsrc_node		*file_node;
669 
670 	atomic_t			refs;
671 	bool				cancel_seq_set;
672 	struct io_task_work		io_task_work;
673 	union {
674 		/*
675 		 * for polled requests, i.e. IORING_OP_POLL_ADD and async armed
676 		 * poll
677 		 */
678 		struct hlist_node	hash_node;
679 		/* For IOPOLL setup queues, with hybrid polling */
680 		u64                     iopoll_start;
681 	};
682 	/* internal polling, see IORING_FEAT_FAST_POLL */
683 	struct async_poll		*apoll;
684 	/* opcode allocated if it needs to store data for async defer */
685 	void				*async_data;
686 	/* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */
687 	atomic_t			poll_refs;
688 	struct io_kiocb			*link;
689 	/* custom credentials, valid IFF REQ_F_CREDS is set */
690 	const struct cred		*creds;
691 	struct io_wq_work		work;
692 
693 	struct {
694 		u64			extra1;
695 		u64			extra2;
696 	} big_cqe;
697 };
698 
699 struct io_overflow_cqe {
700 	struct list_head list;
701 	struct io_uring_cqe cqe;
702 };
703 
io_ctx_cqe32(struct io_ring_ctx * ctx)704 static inline bool io_ctx_cqe32(struct io_ring_ctx *ctx)
705 {
706 	return ctx->flags & IORING_SETUP_CQE32;
707 }
708 
709 #endif
710