1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef BLK_THROTTLE_H
3 #define BLK_THROTTLE_H
4
5 #include "blk-cgroup-rwstat.h"
6
7 /*
8 * To implement hierarchical throttling, throtl_grps form a tree and bios
9 * are dispatched upwards level by level until they reach the top and get
10 * issued. When dispatching bios from the children and local group at each
11 * level, if the bios are dispatched into a single bio_list, there's a risk
12 * of a local or child group which can queue many bios at once filling up
13 * the list starving others.
14 *
15 * To avoid such starvation, dispatched bios are queued separately
16 * according to where they came from. When they are again dispatched to
17 * the parent, they're popped in round-robin order so that no single source
18 * hogs the dispatch window.
19 *
20 * throtl_qnode is used to keep the queued bios separated by their sources.
21 * Bios are queued to throtl_qnode which in turn is queued to
22 * throtl_service_queue and then dispatched in round-robin order.
23 *
24 * It's also used to track the reference counts on blkg's. A qnode always
25 * belongs to a throtl_grp and gets queued on itself or the parent, so
26 * incrementing the reference of the associated throtl_grp when a qnode is
27 * queued and decrementing when dequeued is enough to keep the whole blkg
28 * tree pinned while bios are in flight.
29 */
30 struct throtl_qnode {
31 struct list_head node; /* service_queue->queued[] */
32 struct bio_list bios; /* queued bios */
33 struct throtl_grp *tg; /* tg this qnode belongs to */
34 };
35
36 struct throtl_service_queue {
37 struct throtl_service_queue *parent_sq; /* the parent service_queue */
38
39 /*
40 * Bios queued directly to this service_queue or dispatched from
41 * children throtl_grp's.
42 */
43 struct list_head queued[2]; /* throtl_qnode [READ/WRITE] */
44 unsigned int nr_queued[2]; /* number of queued bios */
45
46 /*
47 * RB tree of active children throtl_grp's, which are sorted by
48 * their ->disptime.
49 */
50 struct rb_root_cached pending_tree; /* RB tree of active tgs */
51 unsigned int nr_pending; /* # queued in the tree */
52 unsigned long first_pending_disptime; /* disptime of the first tg */
53 struct timer_list pending_timer; /* fires on first_pending_disptime */
54 };
55
56 enum tg_state_flags {
57 THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */
58 THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */
59 THROTL_TG_CANCELING = 1 << 2, /* starts to cancel bio */
60 };
61
62 struct throtl_grp {
63 /* must be the first member */
64 struct blkg_policy_data pd;
65
66 /* active throtl group service_queue member */
67 struct rb_node rb_node;
68
69 /* throtl_data this group belongs to */
70 struct throtl_data *td;
71
72 /* this group's service queue */
73 struct throtl_service_queue service_queue;
74
75 /*
76 * qnode_on_self is used when bios are directly queued to this
77 * throtl_grp so that local bios compete fairly with bios
78 * dispatched from children. qnode_on_parent is used when bios are
79 * dispatched from this throtl_grp into its parent and will compete
80 * with the sibling qnode_on_parents and the parent's
81 * qnode_on_self.
82 */
83 struct throtl_qnode qnode_on_self[2];
84 struct throtl_qnode qnode_on_parent[2];
85
86 /*
87 * Dispatch time in jiffies. This is the estimated time when group
88 * will unthrottle and is ready to dispatch more bio. It is used as
89 * key to sort active groups in service tree.
90 */
91 unsigned long disptime;
92
93 unsigned int flags;
94
95 /* are there any throtl rules between this group and td? */
96 bool has_rules_bps[2];
97 bool has_rules_iops[2];
98
99 /* bytes per second rate limits */
100 uint64_t bps[2];
101
102 /* IOPS limits */
103 unsigned int iops[2];
104
105 /* Number of bytes dispatched in current slice */
106 int64_t bytes_disp[2];
107 /* Number of bio's dispatched in current slice */
108 int io_disp[2];
109
110 /*
111 * The following two fields are updated when new configuration is
112 * submitted while some bios are still throttled, they record how many
113 * bytes/ios are waited already in previous configuration, and they will
114 * be used to calculate wait time under new configuration.
115 */
116 long long carryover_bytes[2];
117 int carryover_ios[2];
118
119 unsigned long last_check_time;
120
121 /* When did we start a new slice */
122 unsigned long slice_start[2];
123 unsigned long slice_end[2];
124
125 struct blkg_rwstat stat_bytes;
126 struct blkg_rwstat stat_ios;
127 };
128
129 extern struct blkcg_policy blkcg_policy_throtl;
130
pd_to_tg(struct blkg_policy_data * pd)131 static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
132 {
133 return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
134 }
135
blkg_to_tg(struct blkcg_gq * blkg)136 static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
137 {
138 return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
139 }
140
141 /*
142 * Internal throttling interface
143 */
144 #ifndef CONFIG_BLK_DEV_THROTTLING
blk_throtl_exit(struct gendisk * disk)145 static inline void blk_throtl_exit(struct gendisk *disk) { }
blk_throtl_bio(struct bio * bio)146 static inline bool blk_throtl_bio(struct bio *bio) { return false; }
blk_throtl_cancel_bios(struct gendisk * disk)147 static inline void blk_throtl_cancel_bios(struct gendisk *disk) { }
148 #else /* CONFIG_BLK_DEV_THROTTLING */
149 void blk_throtl_exit(struct gendisk *disk);
150 bool __blk_throtl_bio(struct bio *bio);
151 void blk_throtl_cancel_bios(struct gendisk *disk);
152
blk_throtl_activated(struct request_queue * q)153 static inline bool blk_throtl_activated(struct request_queue *q)
154 {
155 return q->td != NULL;
156 }
157
blk_should_throtl(struct bio * bio)158 static inline bool blk_should_throtl(struct bio *bio)
159 {
160 struct throtl_grp *tg;
161 int rw = bio_data_dir(bio);
162
163 /*
164 * This is called under bio_queue_enter(), and it's synchronized with
165 * the activation of blk-throtl, which is protected by
166 * blk_mq_freeze_queue().
167 */
168 if (!blk_throtl_activated(bio->bi_bdev->bd_queue))
169 return false;
170
171 tg = blkg_to_tg(bio->bi_blkg);
172 if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) {
173 if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
174 bio_set_flag(bio, BIO_CGROUP_ACCT);
175 blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf,
176 bio->bi_iter.bi_size);
177 }
178 blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1);
179 }
180
181 /* iops limit is always counted */
182 if (tg->has_rules_iops[rw])
183 return true;
184
185 if (tg->has_rules_bps[rw] && !bio_flagged(bio, BIO_BPS_THROTTLED))
186 return true;
187
188 return false;
189 }
190
blk_throtl_bio(struct bio * bio)191 static inline bool blk_throtl_bio(struct bio *bio)
192 {
193
194 if (!blk_should_throtl(bio))
195 return false;
196
197 return __blk_throtl_bio(bio);
198 }
199 #endif /* CONFIG_BLK_DEV_THROTTLING */
200
201 #endif
202