Lines Matching refs:tg
47 static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg) in tg_to_blkg() argument
49 return pd_to_blkg(&tg->pd); in tg_to_blkg()
76 struct throtl_grp *tg = sq_to_tg(sq); in sq_to_td() local
78 if (tg) in sq_to_td()
79 return tg->td; in sq_to_td()
84 static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw) in tg_bps_limit() argument
86 struct blkcg_gq *blkg = tg_to_blkg(tg); in tg_bps_limit()
91 return tg->bps[rw]; in tg_bps_limit()
94 static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw) in tg_iops_limit() argument
96 struct blkcg_gq *blkg = tg_to_blkg(tg); in tg_iops_limit()
101 return tg->iops[rw]; in tg_iops_limit()
136 static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg) in throtl_qnode_init() argument
141 qn->tg = tg; in throtl_qnode_init()
174 blkg_get(tg_to_blkg(qn->tg)); in throtl_qnode_add_bio()
242 *tg_to_put = qn->tg; in throtl_pop_queued()
244 blkg_put(tg_to_blkg(qn->tg)); in throtl_pop_queued()
264 struct throtl_grp *tg; in throtl_pd_alloc() local
267 tg = kzalloc_node(sizeof(*tg), gfp, disk->node_id); in throtl_pd_alloc()
268 if (!tg) in throtl_pd_alloc()
271 if (blkg_rwstat_init(&tg->stat_bytes, gfp)) in throtl_pd_alloc()
274 if (blkg_rwstat_init(&tg->stat_ios, gfp)) in throtl_pd_alloc()
277 throtl_service_queue_init(&tg->service_queue); in throtl_pd_alloc()
280 throtl_qnode_init(&tg->qnode_on_self[rw], tg); in throtl_pd_alloc()
281 throtl_qnode_init(&tg->qnode_on_parent[rw], tg); in throtl_pd_alloc()
284 RB_CLEAR_NODE(&tg->rb_node); in throtl_pd_alloc()
285 tg->bps[READ] = U64_MAX; in throtl_pd_alloc()
286 tg->bps[WRITE] = U64_MAX; in throtl_pd_alloc()
287 tg->iops[READ] = UINT_MAX; in throtl_pd_alloc()
288 tg->iops[WRITE] = UINT_MAX; in throtl_pd_alloc()
290 return &tg->pd; in throtl_pd_alloc()
293 blkg_rwstat_exit(&tg->stat_bytes); in throtl_pd_alloc()
295 kfree(tg); in throtl_pd_alloc()
301 struct throtl_grp *tg = pd_to_tg(pd); in throtl_pd_init() local
302 struct blkcg_gq *blkg = tg_to_blkg(tg); in throtl_pd_init()
304 struct throtl_service_queue *sq = &tg->service_queue; in throtl_pd_init()
323 tg->td = td; in throtl_pd_init()
331 static void tg_update_has_rules(struct throtl_grp *tg) in tg_update_has_rules() argument
333 struct throtl_grp *parent_tg = sq_to_tg(tg->service_queue.parent_sq); in tg_update_has_rules()
337 tg->has_rules_iops[rw] = in tg_update_has_rules()
339 tg_iops_limit(tg, rw) != UINT_MAX; in tg_update_has_rules()
340 tg->has_rules_bps[rw] = in tg_update_has_rules()
342 tg_bps_limit(tg, rw) != U64_MAX; in tg_update_has_rules()
348 struct throtl_grp *tg = pd_to_tg(pd); in throtl_pd_online() local
353 tg_update_has_rules(tg); in throtl_pd_online()
358 struct throtl_grp *tg = pd_to_tg(pd); in throtl_pd_free() local
360 timer_delete_sync(&tg->service_queue.pending_timer); in throtl_pd_free()
361 blkg_rwstat_exit(&tg->stat_bytes); in throtl_pd_free()
362 blkg_rwstat_exit(&tg->stat_ios); in throtl_pd_free()
363 kfree(tg); in throtl_pd_free()
387 struct throtl_grp *tg; in update_min_dispatch_time() local
389 tg = throtl_rb_first(parent_sq); in update_min_dispatch_time()
390 if (!tg) in update_min_dispatch_time()
393 parent_sq->first_pending_disptime = tg->disptime; in update_min_dispatch_time()
396 static void tg_service_queue_add(struct throtl_grp *tg) in tg_service_queue_add() argument
398 struct throtl_service_queue *parent_sq = tg->service_queue.parent_sq; in tg_service_queue_add()
402 unsigned long key = tg->disptime; in tg_service_queue_add()
417 rb_link_node(&tg->rb_node, parent, node); in tg_service_queue_add()
418 rb_insert_color_cached(&tg->rb_node, &parent_sq->pending_tree, in tg_service_queue_add()
422 static void throtl_enqueue_tg(struct throtl_grp *tg) in throtl_enqueue_tg() argument
424 if (!(tg->flags & THROTL_TG_PENDING)) { in throtl_enqueue_tg()
425 tg_service_queue_add(tg); in throtl_enqueue_tg()
426 tg->flags |= THROTL_TG_PENDING; in throtl_enqueue_tg()
427 tg->service_queue.parent_sq->nr_pending++; in throtl_enqueue_tg()
431 static void throtl_dequeue_tg(struct throtl_grp *tg) in throtl_dequeue_tg() argument
433 if (tg->flags & THROTL_TG_PENDING) { in throtl_dequeue_tg()
435 tg->service_queue.parent_sq; in throtl_dequeue_tg()
437 throtl_rb_erase(&tg->rb_node, parent_sq); in throtl_dequeue_tg()
439 tg->flags &= ~THROTL_TG_PENDING; in throtl_dequeue_tg()
500 static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg, in throtl_start_new_slice_with_credit() argument
503 tg->bytes_disp[rw] = 0; in throtl_start_new_slice_with_credit()
504 tg->io_disp[rw] = 0; in throtl_start_new_slice_with_credit()
512 if (time_after(start, tg->slice_start[rw])) in throtl_start_new_slice_with_credit()
513 tg->slice_start[rw] = start; in throtl_start_new_slice_with_credit()
515 tg->slice_end[rw] = jiffies + DFL_THROTL_SLICE; in throtl_start_new_slice_with_credit()
516 throtl_log(&tg->service_queue, in throtl_start_new_slice_with_credit()
518 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice_with_credit()
519 tg->slice_end[rw], jiffies); in throtl_start_new_slice_with_credit()
522 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw, in throtl_start_new_slice() argument
526 tg->bytes_disp[rw] = 0; in throtl_start_new_slice()
527 tg->io_disp[rw] = 0; in throtl_start_new_slice()
529 tg->slice_start[rw] = jiffies; in throtl_start_new_slice()
530 tg->slice_end[rw] = jiffies + DFL_THROTL_SLICE; in throtl_start_new_slice()
532 throtl_log(&tg->service_queue, in throtl_start_new_slice()
534 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice()
535 tg->slice_end[rw], jiffies); in throtl_start_new_slice()
538 static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw, in throtl_set_slice_end() argument
541 tg->slice_end[rw] = roundup(jiffy_end, DFL_THROTL_SLICE); in throtl_set_slice_end()
544 static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw, in throtl_extend_slice() argument
547 if (!time_before(tg->slice_end[rw], jiffy_end)) in throtl_extend_slice()
550 throtl_set_slice_end(tg, rw, jiffy_end); in throtl_extend_slice()
551 throtl_log(&tg->service_queue, in throtl_extend_slice()
553 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_extend_slice()
554 tg->slice_end[rw], jiffies); in throtl_extend_slice()
558 static bool throtl_slice_used(struct throtl_grp *tg, bool rw) in throtl_slice_used() argument
560 if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw])) in throtl_slice_used()
606 static long long throtl_trim_bps(struct throtl_grp *tg, bool rw, in throtl_trim_bps() argument
609 u64 bps_limit = tg_bps_limit(tg, rw); in throtl_trim_bps()
617 if (bytes_trim <= 0 || tg->bytes_disp[rw] < bytes_trim) { in throtl_trim_bps()
618 bytes_trim = tg->bytes_disp[rw]; in throtl_trim_bps()
619 tg->bytes_disp[rw] = 0; in throtl_trim_bps()
621 tg->bytes_disp[rw] -= bytes_trim; in throtl_trim_bps()
627 static int throtl_trim_iops(struct throtl_grp *tg, bool rw, in throtl_trim_iops() argument
630 u32 iops_limit = tg_iops_limit(tg, rw); in throtl_trim_iops()
638 if (io_trim <= 0 || tg->io_disp[rw] < io_trim) { in throtl_trim_iops()
639 io_trim = tg->io_disp[rw]; in throtl_trim_iops()
640 tg->io_disp[rw] = 0; in throtl_trim_iops()
642 tg->io_disp[rw] -= io_trim; in throtl_trim_iops()
649 static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) in throtl_trim_slice() argument
655 BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw])); in throtl_trim_slice()
662 if (throtl_slice_used(tg, rw)) in throtl_trim_slice()
672 throtl_set_slice_end(tg, rw, jiffies + DFL_THROTL_SLICE); in throtl_trim_slice()
674 time_elapsed = rounddown(jiffies - tg->slice_start[rw], in throtl_trim_slice()
689 bytes_trim = throtl_trim_bps(tg, rw, time_elapsed); in throtl_trim_slice()
690 io_trim = throtl_trim_iops(tg, rw, time_elapsed); in throtl_trim_slice()
694 tg->slice_start[rw] += time_elapsed; in throtl_trim_slice()
696 throtl_log(&tg->service_queue, in throtl_trim_slice()
699 bytes_trim, io_trim, tg->slice_start[rw], tg->slice_end[rw], in throtl_trim_slice()
703 static void __tg_update_carryover(struct throtl_grp *tg, bool rw, in __tg_update_carryover() argument
706 unsigned long jiffy_elapsed = jiffies - tg->slice_start[rw]; in __tg_update_carryover()
707 u64 bps_limit = tg_bps_limit(tg, rw); in __tg_update_carryover()
708 u32 iops_limit = tg_iops_limit(tg, rw); in __tg_update_carryover()
718 if (sq_queued(&tg->service_queue, rw) == 0) { in __tg_update_carryover()
719 tg->bytes_disp[rw] = 0; in __tg_update_carryover()
720 tg->io_disp[rw] = 0; in __tg_update_carryover()
734 *bytes = bytes_allowed - tg->bytes_disp[rw]; in __tg_update_carryover()
739 *ios = io_allowed - tg->io_disp[rw]; in __tg_update_carryover()
742 tg->bytes_disp[rw] = -*bytes; in __tg_update_carryover()
743 tg->io_disp[rw] = -*ios; in __tg_update_carryover()
746 static void tg_update_carryover(struct throtl_grp *tg) in tg_update_carryover() argument
751 __tg_update_carryover(tg, READ, &bytes[READ], &ios[READ]); in tg_update_carryover()
752 __tg_update_carryover(tg, WRITE, &bytes[WRITE], &ios[WRITE]); in tg_update_carryover()
755 throtl_log(&tg->service_queue, "%s: %lld %lld %d %d\n", __func__, in tg_update_carryover()
759 static unsigned long tg_within_iops_limit(struct throtl_grp *tg, struct bio *bio, in tg_within_iops_limit() argument
766 jiffy_elapsed = jiffies - tg->slice_start[rw]; in tg_within_iops_limit()
771 if (io_allowed > 0 && tg->io_disp[rw] + 1 <= io_allowed) in tg_within_iops_limit()
782 static unsigned long tg_within_bps_limit(struct throtl_grp *tg, struct bio *bio, in tg_within_bps_limit() argument
791 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw]; in tg_within_bps_limit()
800 if ((bytes_allowed > 0 && tg->bytes_disp[rw] + bio_size <= bytes_allowed) in tg_within_bps_limit()
805 extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed; in tg_within_bps_limit()
819 static void throtl_charge_bps_bio(struct throtl_grp *tg, struct bio *bio) in throtl_charge_bps_bio() argument
827 tg->bytes_disp[bio_data_dir(bio)] += bio_size; in throtl_charge_bps_bio()
831 static void throtl_charge_iops_bio(struct throtl_grp *tg, struct bio *bio) in throtl_charge_iops_bio() argument
834 tg->io_disp[bio_data_dir(bio)]++; in throtl_charge_iops_bio()
843 static void tg_update_slice(struct throtl_grp *tg, bool rw) in tg_update_slice() argument
845 if (throtl_slice_used(tg, rw) && in tg_update_slice()
846 sq_queued(&tg->service_queue, rw) == 0) in tg_update_slice()
847 throtl_start_new_slice(tg, rw, true); in tg_update_slice()
849 throtl_extend_slice(tg, rw, jiffies + DFL_THROTL_SLICE); in tg_update_slice()
852 static unsigned long tg_dispatch_bps_time(struct throtl_grp *tg, struct bio *bio) in tg_dispatch_bps_time() argument
855 u64 bps_limit = tg_bps_limit(tg, rw); in tg_dispatch_bps_time()
859 if (bps_limit == U64_MAX || tg->flags & THROTL_TG_CANCELING || in tg_dispatch_bps_time()
864 tg_update_slice(tg, rw); in tg_dispatch_bps_time()
865 bps_wait = tg_within_bps_limit(tg, bio, bps_limit); in tg_dispatch_bps_time()
866 throtl_extend_slice(tg, rw, jiffies + bps_wait); in tg_dispatch_bps_time()
871 static unsigned long tg_dispatch_iops_time(struct throtl_grp *tg, struct bio *bio) in tg_dispatch_iops_time() argument
874 u32 iops_limit = tg_iops_limit(tg, rw); in tg_dispatch_iops_time()
877 if (iops_limit == UINT_MAX || tg->flags & THROTL_TG_CANCELING) in tg_dispatch_iops_time()
880 tg_update_slice(tg, rw); in tg_dispatch_iops_time()
881 iops_wait = tg_within_iops_limit(tg, bio, iops_limit); in tg_dispatch_iops_time()
882 throtl_extend_slice(tg, rw, jiffies + iops_wait); in tg_dispatch_iops_time()
891 static unsigned long tg_dispatch_time(struct throtl_grp *tg, struct bio *bio) in tg_dispatch_time() argument
902 BUG_ON(sq_queued(&tg->service_queue, rw) && in tg_dispatch_time()
903 bio != throtl_peek_queued(&tg->service_queue.queued[rw])); in tg_dispatch_time()
905 wait = tg_dispatch_bps_time(tg, bio); in tg_dispatch_time()
913 throtl_charge_bps_bio(tg, bio); in tg_dispatch_time()
915 return tg_dispatch_iops_time(tg, bio); in tg_dispatch_time()
928 struct throtl_grp *tg) in throtl_add_bio_tg() argument
930 struct throtl_service_queue *sq = &tg->service_queue; in throtl_add_bio_tg()
934 qn = &tg->qnode_on_self[rw]; in throtl_add_bio_tg()
943 tg->flags |= THROTL_TG_WAS_EMPTY; in throtl_add_bio_tg()
954 tg->flags |= THROTL_TG_IOPS_WAS_EMPTY; in throtl_add_bio_tg()
956 throtl_enqueue_tg(tg); in throtl_add_bio_tg()
959 static void tg_update_disptime(struct throtl_grp *tg) in tg_update_disptime() argument
961 struct throtl_service_queue *sq = &tg->service_queue; in tg_update_disptime()
967 read_wait = tg_dispatch_time(tg, bio); in tg_update_disptime()
971 write_wait = tg_dispatch_time(tg, bio); in tg_update_disptime()
977 throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq); in tg_update_disptime()
978 tg->disptime = disptime; in tg_update_disptime()
979 tg_service_queue_add(tg); in tg_update_disptime()
982 tg->flags &= ~THROTL_TG_WAS_EMPTY; in tg_update_disptime()
983 tg->flags &= ~THROTL_TG_IOPS_WAS_EMPTY; in tg_update_disptime()
996 static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw) in tg_dispatch_one_bio() argument
998 struct throtl_service_queue *sq = &tg->service_queue; in tg_dispatch_one_bio()
1012 throtl_charge_iops_bio(tg, bio); in tg_dispatch_one_bio()
1022 throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg); in tg_dispatch_one_bio()
1023 start_parent_slice_with_credit(tg, parent_tg, rw); in tg_dispatch_one_bio()
1026 throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw], in tg_dispatch_one_bio()
1028 BUG_ON(tg->td->nr_queued[rw] <= 0); in tg_dispatch_one_bio()
1029 tg->td->nr_queued[rw]--; in tg_dispatch_one_bio()
1032 throtl_trim_slice(tg, rw); in tg_dispatch_one_bio()
1038 static int throtl_dispatch_tg(struct throtl_grp *tg) in throtl_dispatch_tg() argument
1040 struct throtl_service_queue *sq = &tg->service_queue; in throtl_dispatch_tg()
1049 tg_dispatch_time(tg, bio) == 0) { in throtl_dispatch_tg()
1051 tg_dispatch_one_bio(tg, READ); in throtl_dispatch_tg()
1059 tg_dispatch_time(tg, bio) == 0) { in throtl_dispatch_tg()
1061 tg_dispatch_one_bio(tg, WRITE); in throtl_dispatch_tg()
1076 struct throtl_grp *tg; in throtl_select_dispatch() local
1082 tg = throtl_rb_first(parent_sq); in throtl_select_dispatch()
1083 if (!tg) in throtl_select_dispatch()
1086 if (time_before(jiffies, tg->disptime)) in throtl_select_dispatch()
1089 nr_disp += throtl_dispatch_tg(tg); in throtl_select_dispatch()
1091 sq = &tg->service_queue; in throtl_select_dispatch()
1093 tg_update_disptime(tg); in throtl_select_dispatch()
1095 throtl_dequeue_tg(tg); in throtl_select_dispatch()
1123 struct throtl_grp *tg = sq_to_tg(sq); in throtl_pending_timer_fn() local
1131 if (tg) in throtl_pending_timer_fn()
1132 q = tg->pd.blkg->q; in throtl_pending_timer_fn()
1172 if (tg->flags & THROTL_TG_WAS_EMPTY || in throtl_pending_timer_fn()
1173 tg->flags & THROTL_TG_IOPS_WAS_EMPTY) { in throtl_pending_timer_fn()
1174 tg_update_disptime(tg); in throtl_pending_timer_fn()
1178 tg = sq_to_tg(sq); in throtl_pending_timer_fn()
1228 struct throtl_grp *tg = pd_to_tg(pd); in tg_prfill_conf_u64() local
1229 u64 v = *(u64 *)((void *)tg + off); in tg_prfill_conf_u64()
1239 struct throtl_grp *tg = pd_to_tg(pd); in tg_prfill_conf_uint() local
1240 unsigned int v = *(unsigned int *)((void *)tg + off); in tg_prfill_conf_uint()
1261 static void tg_conf_updated(struct throtl_grp *tg, bool global) in tg_conf_updated() argument
1263 struct throtl_service_queue *sq = &tg->service_queue; in tg_conf_updated()
1267 throtl_log(&tg->service_queue, in tg_conf_updated()
1269 tg_bps_limit(tg, READ), tg_bps_limit(tg, WRITE), in tg_conf_updated()
1270 tg_iops_limit(tg, READ), tg_iops_limit(tg, WRITE)); in tg_conf_updated()
1281 global ? tg->td->queue->root_blkg : tg_to_blkg(tg)) { in tg_conf_updated()
1300 throtl_start_new_slice(tg, READ, false); in tg_conf_updated()
1301 throtl_start_new_slice(tg, WRITE, false); in tg_conf_updated()
1303 if (tg->flags & THROTL_TG_PENDING) { in tg_conf_updated()
1304 tg_update_disptime(tg); in tg_conf_updated()
1348 struct throtl_grp *tg; in tg_set_conf() local
1374 tg = blkg_to_tg(ctx.blkg); in tg_set_conf()
1375 tg_update_carryover(tg); in tg_set_conf()
1378 *(u64 *)((void *)tg + of_cft(of)->private) = v; in tg_set_conf()
1380 *(unsigned int *)((void *)tg + of_cft(of)->private) = v; in tg_set_conf()
1382 tg_conf_updated(tg, false); in tg_set_conf()
1478 struct throtl_grp *tg = pd_to_tg(pd); in tg_prfill_limit() local
1489 if (tg->bps[READ] == bps_dft && in tg_prfill_limit()
1490 tg->bps[WRITE] == bps_dft && in tg_prfill_limit()
1491 tg->iops[READ] == iops_dft && in tg_prfill_limit()
1492 tg->iops[WRITE] == iops_dft) in tg_prfill_limit()
1496 if (tg->bps[READ] == U64_MAX) in tg_prfill_limit()
1499 seq_printf(sf, " rbps=%llu", tg->bps[READ]); in tg_prfill_limit()
1501 if (tg->bps[WRITE] == U64_MAX) in tg_prfill_limit()
1504 seq_printf(sf, " wbps=%llu", tg->bps[WRITE]); in tg_prfill_limit()
1506 if (tg->iops[READ] == UINT_MAX) in tg_prfill_limit()
1509 seq_printf(sf, " riops=%u", tg->iops[READ]); in tg_prfill_limit()
1511 if (tg->iops[WRITE] == UINT_MAX) in tg_prfill_limit()
1514 seq_printf(sf, " wiops=%u", tg->iops[WRITE]); in tg_prfill_limit()
1532 struct throtl_grp *tg; in tg_set_limit() local
1552 tg = blkg_to_tg(ctx.blkg); in tg_set_limit()
1553 tg_update_carryover(tg); in tg_set_limit()
1555 v[0] = tg->bps[READ]; in tg_set_limit()
1556 v[1] = tg->bps[WRITE]; in tg_set_limit()
1557 v[2] = tg->iops[READ]; in tg_set_limit()
1558 v[3] = tg->iops[WRITE]; in tg_set_limit()
1595 tg->bps[READ] = v[0]; in tg_set_limit()
1596 tg->bps[WRITE] = v[1]; in tg_set_limit()
1597 tg->iops[READ] = v[2]; in tg_set_limit()
1598 tg->iops[WRITE] = v[3]; in tg_set_limit()
1600 tg_conf_updated(tg, false); in tg_set_limit()
1624 static void tg_flush_bios(struct throtl_grp *tg) in tg_flush_bios() argument
1626 struct throtl_service_queue *sq = &tg->service_queue; in tg_flush_bios()
1628 if (tg->flags & THROTL_TG_CANCELING) in tg_flush_bios()
1634 tg->flags |= THROTL_TG_CANCELING; in tg_flush_bios()
1643 if (!(tg->flags & THROTL_TG_PENDING)) in tg_flush_bios()
1650 tg_update_disptime(tg); in tg_flush_bios()
1702 static bool tg_within_limit(struct throtl_grp *tg, struct bio *bio, bool rw) in tg_within_limit() argument
1704 struct throtl_service_queue *sq = &tg->service_queue; in tg_within_limit()
1712 tg_dispatch_iops_time(tg, bio) == 0; in tg_within_limit()
1719 if (sq_queued(&tg->service_queue, rw)) { in tg_within_limit()
1721 tg_dispatch_bps_time(tg, bio) == 0) in tg_within_limit()
1722 throtl_charge_bps_bio(tg, bio); in tg_within_limit()
1727 return tg_dispatch_time(tg, bio) == 0; in tg_within_limit()
1735 struct throtl_grp *tg = blkg_to_tg(blkg); in __blk_throtl_bio() local
1739 struct throtl_data *td = tg->td; in __blk_throtl_bio()
1743 sq = &tg->service_queue; in __blk_throtl_bio()
1746 if (tg_within_limit(tg, bio, rw)) { in __blk_throtl_bio()
1748 throtl_charge_iops_bio(tg, bio); in __blk_throtl_bio()
1761 throtl_trim_slice(tg, rw); in __blk_throtl_bio()
1771 throtl_charge_bps_bio(tg, bio); in __blk_throtl_bio()
1772 throtl_charge_iops_bio(tg, bio); in __blk_throtl_bio()
1783 qn = &tg->qnode_on_parent[rw]; in __blk_throtl_bio()
1785 tg = sq_to_tg(sq); in __blk_throtl_bio()
1786 if (!tg) { in __blk_throtl_bio()
1795 tg->bytes_disp[rw], bio->bi_iter.bi_size, in __blk_throtl_bio()
1796 tg_bps_limit(tg, rw), in __blk_throtl_bio()
1797 tg->io_disp[rw], tg_iops_limit(tg, rw), in __blk_throtl_bio()
1801 throtl_add_bio_tg(bio, qn, tg); in __blk_throtl_bio()
1811 if (tg->flags & THROTL_TG_WAS_EMPTY || in __blk_throtl_bio()
1812 tg->flags & THROTL_TG_IOPS_WAS_EMPTY) { in __blk_throtl_bio()
1813 tg_update_disptime(tg); in __blk_throtl_bio()
1814 throtl_schedule_next_dispatch(tg->service_queue.parent_sq, true); in __blk_throtl_bio()