xref: /linux/include/linux/iocontext.h (revision f1f8cc94651738b418ba54c039df536303b91704)
1fd0928dfSJens Axboe #ifndef IOCONTEXT_H
2fd0928dfSJens Axboe #define IOCONTEXT_H
3fd0928dfSJens Axboe 
44ac845a2SJens Axboe #include <linux/radix-tree.h>
534e6bbf2SFabio Checconi #include <linux/rcupdate.h>
6b2efa052STejun Heo #include <linux/workqueue.h>
74ac845a2SJens Axboe 
8dc86900eSTejun Heo enum {
9c5869807STejun Heo 	ICQ_IOPRIO_CHANGED,
10c5869807STejun Heo 	ICQ_CGROUP_CHANGED,
11dc86900eSTejun Heo };
12dc86900eSTejun Heo 
13*f1f8cc94STejun Heo /*
14*f1f8cc94STejun Heo  * An io_cq (icq) is association between an io_context (ioc) and a
15*f1f8cc94STejun Heo  * request_queue (q).  This is used by elevators which need to track
16*f1f8cc94STejun Heo  * information per ioc - q pair.
17*f1f8cc94STejun Heo  *
18*f1f8cc94STejun Heo  * Elevator can request use of icq by setting elevator_type->icq_size and
19*f1f8cc94STejun Heo  * ->icq_align.  Both size and align must be larger than that of struct
20*f1f8cc94STejun Heo  * io_cq and elevator can use the tail area for private information.  The
21*f1f8cc94STejun Heo  * recommended way to do this is defining a struct which contains io_cq as
22*f1f8cc94STejun Heo  * the first member followed by private members and using its size and
23*f1f8cc94STejun Heo  * align.  For example,
24*f1f8cc94STejun Heo  *
25*f1f8cc94STejun Heo  *	struct snail_io_cq {
26*f1f8cc94STejun Heo  *		struct io_cq	icq;
27*f1f8cc94STejun Heo  *		int		poke_snail;
28*f1f8cc94STejun Heo  *		int		feed_snail;
29*f1f8cc94STejun Heo  *	};
30*f1f8cc94STejun Heo  *
31*f1f8cc94STejun Heo  *	struct elevator_type snail_elv_type {
32*f1f8cc94STejun Heo  *		.ops =		{ ... },
33*f1f8cc94STejun Heo  *		.icq_size =	sizeof(struct snail_io_cq),
34*f1f8cc94STejun Heo  *		.icq_align =	__alignof__(struct snail_io_cq),
35*f1f8cc94STejun Heo  *		...
36*f1f8cc94STejun Heo  *	};
37*f1f8cc94STejun Heo  *
38*f1f8cc94STejun Heo  * If icq_size is set, block core will manage icq's.  All requests will
39*f1f8cc94STejun Heo  * have its ->elv.icq field set before elevator_ops->elevator_set_req_fn()
40*f1f8cc94STejun Heo  * is called and be holding a reference to the associated io_context.
41*f1f8cc94STejun Heo  *
42*f1f8cc94STejun Heo  * Whenever a new icq is created, elevator_ops->elevator_init_icq_fn() is
43*f1f8cc94STejun Heo  * called and, on destruction, ->elevator_exit_icq_fn().  Both functions
44*f1f8cc94STejun Heo  * are called with both the associated io_context and queue locks held.
45*f1f8cc94STejun Heo  *
46*f1f8cc94STejun Heo  * Elevator is allowed to lookup icq using ioc_lookup_icq() while holding
47*f1f8cc94STejun Heo  * queue lock but the returned icq is valid only until the queue lock is
48*f1f8cc94STejun Heo  * released.  Elevators can not and should not try to create or destroy
49*f1f8cc94STejun Heo  * icq's.
50*f1f8cc94STejun Heo  *
51*f1f8cc94STejun Heo  * As icq's are linked from both ioc and q, the locking rules are a bit
52*f1f8cc94STejun Heo  * complex.
53*f1f8cc94STejun Heo  *
54*f1f8cc94STejun Heo  * - ioc lock nests inside q lock.
55*f1f8cc94STejun Heo  *
56*f1f8cc94STejun Heo  * - ioc->icq_list and icq->ioc_node are protected by ioc lock.
57*f1f8cc94STejun Heo  *   q->icq_list and icq->q_node by q lock.
58*f1f8cc94STejun Heo  *
59*f1f8cc94STejun Heo  * - ioc->icq_tree and ioc->icq_hint are protected by ioc lock, while icq
60*f1f8cc94STejun Heo  *   itself is protected by q lock.  However, both the indexes and icq
61*f1f8cc94STejun Heo  *   itself are also RCU managed and lookup can be performed holding only
62*f1f8cc94STejun Heo  *   the q lock.
63*f1f8cc94STejun Heo  *
64*f1f8cc94STejun Heo  * - icq's are not reference counted.  They are destroyed when either the
65*f1f8cc94STejun Heo  *   ioc or q goes away.  Each request with icq set holds an extra
66*f1f8cc94STejun Heo  *   reference to ioc to ensure it stays until the request is completed.
67*f1f8cc94STejun Heo  *
68*f1f8cc94STejun Heo  * - Linking and unlinking icq's are performed while holding both ioc and q
69*f1f8cc94STejun Heo  *   locks.  Due to the lock ordering, q exit is simple but ioc exit
70*f1f8cc94STejun Heo  *   requires reverse-order double lock dance.
71*f1f8cc94STejun Heo  */
72c5869807STejun Heo struct io_cq {
73283287a5STejun Heo 	struct request_queue	*q;
74fd0928dfSJens Axboe 	struct io_context	*ioc;
75fd0928dfSJens Axboe 
767e5a8794STejun Heo 	/*
777e5a8794STejun Heo 	 * q_node and ioc_node link io_cq through icq_list of q and ioc
787e5a8794STejun Heo 	 * respectively.  Both fields are unused once ioc_exit_icq() is
797e5a8794STejun Heo 	 * called and shared with __rcu_icq_cache and __rcu_head which are
807e5a8794STejun Heo 	 * used for RCU free of io_cq.
817e5a8794STejun Heo 	 */
827e5a8794STejun Heo 	union {
83c5869807STejun Heo 		struct list_head	q_node;
847e5a8794STejun Heo 		struct kmem_cache	*__rcu_icq_cache;
857e5a8794STejun Heo 	};
867e5a8794STejun Heo 	union {
87c5869807STejun Heo 		struct hlist_node	ioc_node;
887e5a8794STejun Heo 		struct rcu_head		__rcu_head;
897e5a8794STejun Heo 	};
90fd0928dfSJens Axboe 
91dc86900eSTejun Heo 	unsigned long		changed;
92fd0928dfSJens Axboe };
93fd0928dfSJens Axboe 
94fd0928dfSJens Axboe /*
95d38ecf93SJens Axboe  * I/O subsystem state of the associated processes.  It is refcounted
96d38ecf93SJens Axboe  * and kmalloc'ed. These could be shared between processes.
97fd0928dfSJens Axboe  */
98fd0928dfSJens Axboe struct io_context {
99d9c7d394SNikanth Karthikesan 	atomic_long_t refcount;
100d38ecf93SJens Axboe 	atomic_t nr_tasks;
101d38ecf93SJens Axboe 
102d38ecf93SJens Axboe 	/* all the fields below are protected by this lock */
103d38ecf93SJens Axboe 	spinlock_t lock;
104fd0928dfSJens Axboe 
105fd0928dfSJens Axboe 	unsigned short ioprio;
10631e4c28dSVivek Goyal 
107fd0928dfSJens Axboe 	/*
108fd0928dfSJens Axboe 	 * For request batching
109fd0928dfSJens Axboe 	 */
110fd0928dfSJens Axboe 	int nr_batch_requests;     /* Number of requests left in the batch */
11158c24a61SRichard Kennedy 	unsigned long last_waited; /* Time last woken after wait for request */
112fd0928dfSJens Axboe 
113c5869807STejun Heo 	struct radix_tree_root	icq_tree;
114c5869807STejun Heo 	struct io_cq __rcu	*icq_hint;
115c5869807STejun Heo 	struct hlist_head	icq_list;
116b2efa052STejun Heo 
117b2efa052STejun Heo 	struct work_struct release_work;
118fd0928dfSJens Axboe };
119fd0928dfSJens Axboe 
120d38ecf93SJens Axboe static inline struct io_context *ioc_task_link(struct io_context *ioc)
121d38ecf93SJens Axboe {
122d38ecf93SJens Axboe 	/*
123d38ecf93SJens Axboe 	 * if ref count is zero, don't allow sharing (ioc is going away, it's
124d38ecf93SJens Axboe 	 * a race).
125d38ecf93SJens Axboe 	 */
126d9c7d394SNikanth Karthikesan 	if (ioc && atomic_long_inc_not_zero(&ioc->refcount)) {
127cbb4f264SLi Zefan 		atomic_inc(&ioc->nr_tasks);
128d38ecf93SJens Axboe 		return ioc;
129d237e5c7SJens Axboe 	}
130d38ecf93SJens Axboe 
131d38ecf93SJens Axboe 	return NULL;
132d38ecf93SJens Axboe }
133d38ecf93SJens Axboe 
134b69f2292SLouis Rilling struct task_struct;
135da9cbc87SJens Axboe #ifdef CONFIG_BLOCK
136b2efa052STejun Heo void put_io_context(struct io_context *ioc, struct request_queue *locked_q);
137b69f2292SLouis Rilling void exit_io_context(struct task_struct *task);
1386e736be7STejun Heo struct io_context *get_task_io_context(struct task_struct *task,
1396e736be7STejun Heo 				       gfp_t gfp_flags, int node);
140dc86900eSTejun Heo void ioc_ioprio_changed(struct io_context *ioc, int ioprio);
141dc86900eSTejun Heo void ioc_cgroup_changed(struct io_context *ioc);
142da9cbc87SJens Axboe #else
143da9cbc87SJens Axboe struct io_context;
144b2efa052STejun Heo static inline void put_io_context(struct io_context *ioc,
145b2efa052STejun Heo 				  struct request_queue *locked_q) { }
14642ec57a8STejun Heo static inline void exit_io_context(struct task_struct *task) { }
147da9cbc87SJens Axboe #endif
148da9cbc87SJens Axboe 
149fd0928dfSJens Axboe #endif
150