1fd0928dfSJens Axboe #ifndef IOCONTEXT_H 2fd0928dfSJens Axboe #define IOCONTEXT_H 3fd0928dfSJens Axboe 44ac845a2SJens Axboe #include <linux/radix-tree.h> 534e6bbf2SFabio Checconi #include <linux/rcupdate.h> 6b2efa052STejun Heo #include <linux/workqueue.h> 74ac845a2SJens Axboe 8dc86900eSTejun Heo enum { 9c5869807STejun Heo ICQ_IOPRIO_CHANGED, 10c5869807STejun Heo ICQ_CGROUP_CHANGED, 11dc86900eSTejun Heo }; 12dc86900eSTejun Heo 13*f1f8cc94STejun Heo /* 14*f1f8cc94STejun Heo * An io_cq (icq) is association between an io_context (ioc) and a 15*f1f8cc94STejun Heo * request_queue (q). This is used by elevators which need to track 16*f1f8cc94STejun Heo * information per ioc - q pair. 17*f1f8cc94STejun Heo * 18*f1f8cc94STejun Heo * Elevator can request use of icq by setting elevator_type->icq_size and 19*f1f8cc94STejun Heo * ->icq_align. Both size and align must be larger than that of struct 20*f1f8cc94STejun Heo * io_cq and elevator can use the tail area for private information. The 21*f1f8cc94STejun Heo * recommended way to do this is defining a struct which contains io_cq as 22*f1f8cc94STejun Heo * the first member followed by private members and using its size and 23*f1f8cc94STejun Heo * align. For example, 24*f1f8cc94STejun Heo * 25*f1f8cc94STejun Heo * struct snail_io_cq { 26*f1f8cc94STejun Heo * struct io_cq icq; 27*f1f8cc94STejun Heo * int poke_snail; 28*f1f8cc94STejun Heo * int feed_snail; 29*f1f8cc94STejun Heo * }; 30*f1f8cc94STejun Heo * 31*f1f8cc94STejun Heo * struct elevator_type snail_elv_type { 32*f1f8cc94STejun Heo * .ops = { ... }, 33*f1f8cc94STejun Heo * .icq_size = sizeof(struct snail_io_cq), 34*f1f8cc94STejun Heo * .icq_align = __alignof__(struct snail_io_cq), 35*f1f8cc94STejun Heo * ... 36*f1f8cc94STejun Heo * }; 37*f1f8cc94STejun Heo * 38*f1f8cc94STejun Heo * If icq_size is set, block core will manage icq's. All requests will 39*f1f8cc94STejun Heo * have its ->elv.icq field set before elevator_ops->elevator_set_req_fn() 40*f1f8cc94STejun Heo * is called and be holding a reference to the associated io_context. 41*f1f8cc94STejun Heo * 42*f1f8cc94STejun Heo * Whenever a new icq is created, elevator_ops->elevator_init_icq_fn() is 43*f1f8cc94STejun Heo * called and, on destruction, ->elevator_exit_icq_fn(). Both functions 44*f1f8cc94STejun Heo * are called with both the associated io_context and queue locks held. 45*f1f8cc94STejun Heo * 46*f1f8cc94STejun Heo * Elevator is allowed to lookup icq using ioc_lookup_icq() while holding 47*f1f8cc94STejun Heo * queue lock but the returned icq is valid only until the queue lock is 48*f1f8cc94STejun Heo * released. Elevators can not and should not try to create or destroy 49*f1f8cc94STejun Heo * icq's. 50*f1f8cc94STejun Heo * 51*f1f8cc94STejun Heo * As icq's are linked from both ioc and q, the locking rules are a bit 52*f1f8cc94STejun Heo * complex. 53*f1f8cc94STejun Heo * 54*f1f8cc94STejun Heo * - ioc lock nests inside q lock. 55*f1f8cc94STejun Heo * 56*f1f8cc94STejun Heo * - ioc->icq_list and icq->ioc_node are protected by ioc lock. 57*f1f8cc94STejun Heo * q->icq_list and icq->q_node by q lock. 58*f1f8cc94STejun Heo * 59*f1f8cc94STejun Heo * - ioc->icq_tree and ioc->icq_hint are protected by ioc lock, while icq 60*f1f8cc94STejun Heo * itself is protected by q lock. However, both the indexes and icq 61*f1f8cc94STejun Heo * itself are also RCU managed and lookup can be performed holding only 62*f1f8cc94STejun Heo * the q lock. 63*f1f8cc94STejun Heo * 64*f1f8cc94STejun Heo * - icq's are not reference counted. They are destroyed when either the 65*f1f8cc94STejun Heo * ioc or q goes away. Each request with icq set holds an extra 66*f1f8cc94STejun Heo * reference to ioc to ensure it stays until the request is completed. 67*f1f8cc94STejun Heo * 68*f1f8cc94STejun Heo * - Linking and unlinking icq's are performed while holding both ioc and q 69*f1f8cc94STejun Heo * locks. Due to the lock ordering, q exit is simple but ioc exit 70*f1f8cc94STejun Heo * requires reverse-order double lock dance. 71*f1f8cc94STejun Heo */ 72c5869807STejun Heo struct io_cq { 73283287a5STejun Heo struct request_queue *q; 74fd0928dfSJens Axboe struct io_context *ioc; 75fd0928dfSJens Axboe 767e5a8794STejun Heo /* 777e5a8794STejun Heo * q_node and ioc_node link io_cq through icq_list of q and ioc 787e5a8794STejun Heo * respectively. Both fields are unused once ioc_exit_icq() is 797e5a8794STejun Heo * called and shared with __rcu_icq_cache and __rcu_head which are 807e5a8794STejun Heo * used for RCU free of io_cq. 817e5a8794STejun Heo */ 827e5a8794STejun Heo union { 83c5869807STejun Heo struct list_head q_node; 847e5a8794STejun Heo struct kmem_cache *__rcu_icq_cache; 857e5a8794STejun Heo }; 867e5a8794STejun Heo union { 87c5869807STejun Heo struct hlist_node ioc_node; 887e5a8794STejun Heo struct rcu_head __rcu_head; 897e5a8794STejun Heo }; 90fd0928dfSJens Axboe 91dc86900eSTejun Heo unsigned long changed; 92fd0928dfSJens Axboe }; 93fd0928dfSJens Axboe 94fd0928dfSJens Axboe /* 95d38ecf93SJens Axboe * I/O subsystem state of the associated processes. It is refcounted 96d38ecf93SJens Axboe * and kmalloc'ed. These could be shared between processes. 97fd0928dfSJens Axboe */ 98fd0928dfSJens Axboe struct io_context { 99d9c7d394SNikanth Karthikesan atomic_long_t refcount; 100d38ecf93SJens Axboe atomic_t nr_tasks; 101d38ecf93SJens Axboe 102d38ecf93SJens Axboe /* all the fields below are protected by this lock */ 103d38ecf93SJens Axboe spinlock_t lock; 104fd0928dfSJens Axboe 105fd0928dfSJens Axboe unsigned short ioprio; 10631e4c28dSVivek Goyal 107fd0928dfSJens Axboe /* 108fd0928dfSJens Axboe * For request batching 109fd0928dfSJens Axboe */ 110fd0928dfSJens Axboe int nr_batch_requests; /* Number of requests left in the batch */ 11158c24a61SRichard Kennedy unsigned long last_waited; /* Time last woken after wait for request */ 112fd0928dfSJens Axboe 113c5869807STejun Heo struct radix_tree_root icq_tree; 114c5869807STejun Heo struct io_cq __rcu *icq_hint; 115c5869807STejun Heo struct hlist_head icq_list; 116b2efa052STejun Heo 117b2efa052STejun Heo struct work_struct release_work; 118fd0928dfSJens Axboe }; 119fd0928dfSJens Axboe 120d38ecf93SJens Axboe static inline struct io_context *ioc_task_link(struct io_context *ioc) 121d38ecf93SJens Axboe { 122d38ecf93SJens Axboe /* 123d38ecf93SJens Axboe * if ref count is zero, don't allow sharing (ioc is going away, it's 124d38ecf93SJens Axboe * a race). 125d38ecf93SJens Axboe */ 126d9c7d394SNikanth Karthikesan if (ioc && atomic_long_inc_not_zero(&ioc->refcount)) { 127cbb4f264SLi Zefan atomic_inc(&ioc->nr_tasks); 128d38ecf93SJens Axboe return ioc; 129d237e5c7SJens Axboe } 130d38ecf93SJens Axboe 131d38ecf93SJens Axboe return NULL; 132d38ecf93SJens Axboe } 133d38ecf93SJens Axboe 134b69f2292SLouis Rilling struct task_struct; 135da9cbc87SJens Axboe #ifdef CONFIG_BLOCK 136b2efa052STejun Heo void put_io_context(struct io_context *ioc, struct request_queue *locked_q); 137b69f2292SLouis Rilling void exit_io_context(struct task_struct *task); 1386e736be7STejun Heo struct io_context *get_task_io_context(struct task_struct *task, 1396e736be7STejun Heo gfp_t gfp_flags, int node); 140dc86900eSTejun Heo void ioc_ioprio_changed(struct io_context *ioc, int ioprio); 141dc86900eSTejun Heo void ioc_cgroup_changed(struct io_context *ioc); 142da9cbc87SJens Axboe #else 143da9cbc87SJens Axboe struct io_context; 144b2efa052STejun Heo static inline void put_io_context(struct io_context *ioc, 145b2efa052STejun Heo struct request_queue *locked_q) { } 14642ec57a8STejun Heo static inline void exit_io_context(struct task_struct *task) { } 147da9cbc87SJens Axboe #endif 148da9cbc87SJens Axboe 149fd0928dfSJens Axboe #endif 150