1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef BLK_THROTTLE_H 3 #define BLK_THROTTLE_H 4 5 #include "blk-cgroup-rwstat.h" 6 7 /* 8 * To implement hierarchical throttling, throtl_grps form a tree and bios 9 * are dispatched upwards level by level until they reach the top and get 10 * issued. When dispatching bios from the children and local group at each 11 * level, if the bios are dispatched into a single bio_list, there's a risk 12 * of a local or child group which can queue many bios at once filling up 13 * the list starving others. 14 * 15 * To avoid such starvation, dispatched bios are queued separately 16 * according to where they came from. When they are again dispatched to 17 * the parent, they're popped in round-robin order so that no single source 18 * hogs the dispatch window. 19 * 20 * throtl_qnode is used to keep the queued bios separated by their sources. 21 * Bios are queued to throtl_qnode which in turn is queued to 22 * throtl_service_queue and then dispatched in round-robin order. 23 * 24 * It's also used to track the reference counts on blkg's. A qnode always 25 * belongs to a throtl_grp and gets queued on itself or the parent, so 26 * incrementing the reference of the associated throtl_grp when a qnode is 27 * queued and decrementing when dequeued is enough to keep the whole blkg 28 * tree pinned while bios are in flight. 29 */ 30 struct throtl_qnode { 31 struct list_head node; /* service_queue->queued[] */ 32 struct bio_list bios_bps; /* queued bios for bps limit */ 33 struct bio_list bios_iops; /* queued bios for iops limit */ 34 struct throtl_grp *tg; /* tg this qnode belongs to */ 35 }; 36 37 struct throtl_service_queue { 38 struct throtl_service_queue *parent_sq; /* the parent service_queue */ 39 40 /* 41 * Bios queued directly to this service_queue or dispatched from 42 * children throtl_grp's. 43 */ 44 struct list_head queued[2]; /* throtl_qnode [READ/WRITE] */ 45 unsigned int nr_queued_bps[2]; /* number of queued bps bios */ 46 unsigned int nr_queued_iops[2]; /* number of queued iops bios */ 47 48 /* 49 * RB tree of active children throtl_grp's, which are sorted by 50 * their ->disptime. 51 */ 52 struct rb_root_cached pending_tree; /* RB tree of active tgs */ 53 unsigned int nr_pending; /* # queued in the tree */ 54 unsigned long first_pending_disptime; /* disptime of the first tg */ 55 struct timer_list pending_timer; /* fires on first_pending_disptime */ 56 }; 57 58 enum tg_state_flags { 59 THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */ 60 THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */ 61 /* 62 * The sq's iops queue is empty, and a bio is about to be enqueued 63 * to the first qnode's bios_iops list. 64 */ 65 THROTL_TG_IOPS_WAS_EMPTY = 1 << 2, 66 THROTL_TG_CANCELING = 1 << 3, /* starts to cancel bio */ 67 }; 68 69 struct throtl_grp { 70 /* must be the first member */ 71 struct blkg_policy_data pd; 72 73 /* active throtl group service_queue member */ 74 struct rb_node rb_node; 75 76 /* throtl_data this group belongs to */ 77 struct throtl_data *td; 78 79 /* this group's service queue */ 80 struct throtl_service_queue service_queue; 81 82 /* 83 * qnode_on_self is used when bios are directly queued to this 84 * throtl_grp so that local bios compete fairly with bios 85 * dispatched from children. qnode_on_parent is used when bios are 86 * dispatched from this throtl_grp into its parent and will compete 87 * with the sibling qnode_on_parents and the parent's 88 * qnode_on_self. 89 */ 90 struct throtl_qnode qnode_on_self[2]; 91 struct throtl_qnode qnode_on_parent[2]; 92 93 /* 94 * Dispatch time in jiffies. This is the estimated time when group 95 * will unthrottle and is ready to dispatch more bio. It is used as 96 * key to sort active groups in service tree. 97 */ 98 unsigned long disptime; 99 100 unsigned int flags; 101 102 /* are there any throtl rules between this group and td? */ 103 bool has_rules_bps[2]; 104 bool has_rules_iops[2]; 105 106 /* bytes per second rate limits */ 107 uint64_t bps[2]; 108 109 /* IOPS limits */ 110 unsigned int iops[2]; 111 112 /* 113 * Number of bytes/bio's dispatched in current slice. 114 * When new configuration is submitted while some bios are still throttled, 115 * first calculate the carryover: the amount of bytes/IOs already waited 116 * under the previous configuration. Then, [bytes/io]_disp are represented 117 * as the negative of the carryover, and they will be used to calculate the 118 * wait time under the new configuration. 119 */ 120 int64_t bytes_disp[2]; 121 int io_disp[2]; 122 123 /* When did we start a new slice */ 124 unsigned long slice_start[2]; 125 unsigned long slice_end[2]; 126 127 struct blkg_rwstat stat_bytes; 128 struct blkg_rwstat stat_ios; 129 }; 130 131 extern struct blkcg_policy blkcg_policy_throtl; 132 133 static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd) 134 { 135 return pd ? container_of(pd, struct throtl_grp, pd) : NULL; 136 } 137 138 static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg) 139 { 140 return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl)); 141 } 142 143 /* 144 * Internal throttling interface 145 */ 146 #ifndef CONFIG_BLK_DEV_THROTTLING 147 static inline void blk_throtl_exit(struct gendisk *disk) { } 148 static inline bool blk_throtl_bio(struct bio *bio) { return false; } 149 static inline void blk_throtl_cancel_bios(struct gendisk *disk) { } 150 #else /* CONFIG_BLK_DEV_THROTTLING */ 151 void blk_throtl_exit(struct gendisk *disk); 152 bool __blk_throtl_bio(struct bio *bio); 153 void blk_throtl_cancel_bios(struct gendisk *disk); 154 155 static inline bool blk_throtl_activated(struct request_queue *q) 156 { 157 /* 158 * q->td guarantees that the blk-throttle module is already loaded, 159 * and the plid of blk-throttle is assigned. 160 * blkcg_policy_enabled() guarantees that the policy is activated 161 * in the request_queue. 162 */ 163 return q->td != NULL && blkcg_policy_enabled(q, &blkcg_policy_throtl); 164 } 165 166 static inline bool blk_should_throtl(struct bio *bio) 167 { 168 struct throtl_grp *tg; 169 int rw = bio_data_dir(bio); 170 171 if (!blk_throtl_activated(bio->bi_bdev->bd_queue)) 172 return false; 173 174 tg = blkg_to_tg(bio->bi_blkg); 175 if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) { 176 if (!bio_flagged(bio, BIO_CGROUP_ACCT)) { 177 bio_set_flag(bio, BIO_CGROUP_ACCT); 178 blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf, 179 bio->bi_iter.bi_size); 180 } 181 blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1); 182 } 183 184 /* iops limit is always counted */ 185 if (tg->has_rules_iops[rw]) 186 return true; 187 188 if (tg->has_rules_bps[rw] && !bio_flagged(bio, BIO_BPS_THROTTLED)) 189 return true; 190 191 return false; 192 } 193 194 static inline bool blk_throtl_bio(struct bio *bio) 195 { 196 /* 197 * block throttling takes effect if the policy is activated 198 * in the bio's request_queue. 199 */ 200 if (!blk_should_throtl(bio)) 201 return false; 202 203 return __blk_throtl_bio(bio); 204 } 205 #endif /* CONFIG_BLK_DEV_THROTTLING */ 206 207 #endif 208