Lines Matching refs:rw
84 static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw) in tg_bps_limit() argument
91 return tg->bps[rw]; in tg_bps_limit()
94 static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw) in tg_iops_limit() argument
101 return tg->iops[rw]; in tg_iops_limit()
157 bool rw = bio_data_dir(bio); in throtl_qnode_add_bio() local
166 sq->nr_queued_iops[rw]++; in throtl_qnode_add_bio()
169 sq->nr_queued_bps[rw]++; in throtl_qnode_add_bio()
173 list_add_tail(&qn->node, &sq->queued[rw]); in throtl_qnode_add_bio()
219 struct throtl_grp **tg_to_put, bool rw) in throtl_pop_queued() argument
221 struct list_head *queued = &sq->queued[rw]; in throtl_pop_queued()
231 sq->nr_queued_iops[rw]--; in throtl_pop_queued()
235 sq->nr_queued_bps[rw]--; in throtl_pop_queued()
265 int rw; in throtl_pd_alloc() local
279 for (rw = READ; rw <= WRITE; rw++) { in throtl_pd_alloc()
280 throtl_qnode_init(&tg->qnode_on_self[rw], tg); in throtl_pd_alloc()
281 throtl_qnode_init(&tg->qnode_on_parent[rw], tg); in throtl_pd_alloc()
334 int rw; in tg_update_has_rules() local
336 for (rw = READ; rw <= WRITE; rw++) { in tg_update_has_rules()
337 tg->has_rules_iops[rw] = in tg_update_has_rules()
338 (parent_tg && parent_tg->has_rules_iops[rw]) || in tg_update_has_rules()
339 tg_iops_limit(tg, rw) != UINT_MAX; in tg_update_has_rules()
340 tg->has_rules_bps[rw] = in tg_update_has_rules()
341 (parent_tg && parent_tg->has_rules_bps[rw]) || in tg_update_has_rules()
342 tg_bps_limit(tg, rw) != U64_MAX; in tg_update_has_rules()
501 bool rw, unsigned long start) in throtl_start_new_slice_with_credit() argument
503 tg->bytes_disp[rw] = 0; in throtl_start_new_slice_with_credit()
504 tg->io_disp[rw] = 0; in throtl_start_new_slice_with_credit()
512 if (time_after(start, tg->slice_start[rw])) in throtl_start_new_slice_with_credit()
513 tg->slice_start[rw] = start; in throtl_start_new_slice_with_credit()
515 tg->slice_end[rw] = jiffies + DFL_THROTL_SLICE; in throtl_start_new_slice_with_credit()
518 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice_with_credit()
519 tg->slice_end[rw], jiffies); in throtl_start_new_slice_with_credit()
522 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw, in throtl_start_new_slice() argument
526 tg->bytes_disp[rw] = 0; in throtl_start_new_slice()
527 tg->io_disp[rw] = 0; in throtl_start_new_slice()
529 tg->slice_start[rw] = jiffies; in throtl_start_new_slice()
530 tg->slice_end[rw] = jiffies + DFL_THROTL_SLICE; in throtl_start_new_slice()
534 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice()
535 tg->slice_end[rw], jiffies); in throtl_start_new_slice()
538 static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw, in throtl_set_slice_end() argument
541 tg->slice_end[rw] = roundup(jiffy_end, DFL_THROTL_SLICE); in throtl_set_slice_end()
544 static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw, in throtl_extend_slice() argument
547 if (!time_before(tg->slice_end[rw], jiffy_end)) in throtl_extend_slice()
550 throtl_set_slice_end(tg, rw, jiffy_end); in throtl_extend_slice()
553 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_extend_slice()
554 tg->slice_end[rw], jiffies); in throtl_extend_slice()
558 static bool throtl_slice_used(struct throtl_grp *tg, bool rw) in throtl_slice_used() argument
560 if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw])) in throtl_slice_used()
606 static long long throtl_trim_bps(struct throtl_grp *tg, bool rw, in throtl_trim_bps() argument
609 u64 bps_limit = tg_bps_limit(tg, rw); in throtl_trim_bps()
617 if (bytes_trim <= 0 || tg->bytes_disp[rw] < bytes_trim) { in throtl_trim_bps()
618 bytes_trim = tg->bytes_disp[rw]; in throtl_trim_bps()
619 tg->bytes_disp[rw] = 0; in throtl_trim_bps()
621 tg->bytes_disp[rw] -= bytes_trim; in throtl_trim_bps()
627 static int throtl_trim_iops(struct throtl_grp *tg, bool rw, in throtl_trim_iops() argument
630 u32 iops_limit = tg_iops_limit(tg, rw); in throtl_trim_iops()
638 if (io_trim <= 0 || tg->io_disp[rw] < io_trim) { in throtl_trim_iops()
639 io_trim = tg->io_disp[rw]; in throtl_trim_iops()
640 tg->io_disp[rw] = 0; in throtl_trim_iops()
642 tg->io_disp[rw] -= io_trim; in throtl_trim_iops()
649 static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) in throtl_trim_slice() argument
655 BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw])); in throtl_trim_slice()
662 if (throtl_slice_used(tg, rw)) in throtl_trim_slice()
672 throtl_set_slice_end(tg, rw, jiffies + DFL_THROTL_SLICE); in throtl_trim_slice()
674 time_elapsed = rounddown(jiffies - tg->slice_start[rw], in throtl_trim_slice()
689 bytes_trim = throtl_trim_bps(tg, rw, time_elapsed); in throtl_trim_slice()
690 io_trim = throtl_trim_iops(tg, rw, time_elapsed); in throtl_trim_slice()
694 tg->slice_start[rw] += time_elapsed; in throtl_trim_slice()
698 rw == READ ? 'R' : 'W', time_elapsed / DFL_THROTL_SLICE, in throtl_trim_slice()
699 bytes_trim, io_trim, tg->slice_start[rw], tg->slice_end[rw], in throtl_trim_slice()
703 static void __tg_update_carryover(struct throtl_grp *tg, bool rw, in __tg_update_carryover() argument
706 unsigned long jiffy_elapsed = jiffies - tg->slice_start[rw]; in __tg_update_carryover()
707 u64 bps_limit = tg_bps_limit(tg, rw); in __tg_update_carryover()
708 u32 iops_limit = tg_iops_limit(tg, rw); in __tg_update_carryover()
718 if (sq_queued(&tg->service_queue, rw) == 0) { in __tg_update_carryover()
719 tg->bytes_disp[rw] = 0; in __tg_update_carryover()
720 tg->io_disp[rw] = 0; in __tg_update_carryover()
734 *bytes = bytes_allowed - tg->bytes_disp[rw]; in __tg_update_carryover()
739 *ios = io_allowed - tg->io_disp[rw]; in __tg_update_carryover()
742 tg->bytes_disp[rw] = -*bytes; in __tg_update_carryover()
743 tg->io_disp[rw] = -*ios; in __tg_update_carryover()
762 bool rw = bio_data_dir(bio); in tg_within_iops_limit() local
766 jiffy_elapsed = jiffies - tg->slice_start[rw]; in tg_within_iops_limit()
771 if (io_allowed > 0 && tg->io_disp[rw] + 1 <= io_allowed) in tg_within_iops_limit()
785 bool rw = bio_data_dir(bio); in tg_within_bps_limit() local
791 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw]; in tg_within_bps_limit()
800 if ((bytes_allowed > 0 && tg->bytes_disp[rw] + bio_size <= bytes_allowed) in tg_within_bps_limit()
805 extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed; in tg_within_bps_limit()
843 static void tg_update_slice(struct throtl_grp *tg, bool rw) in tg_update_slice() argument
845 if (throtl_slice_used(tg, rw) && in tg_update_slice()
846 sq_queued(&tg->service_queue, rw) == 0) in tg_update_slice()
847 throtl_start_new_slice(tg, rw, true); in tg_update_slice()
849 throtl_extend_slice(tg, rw, jiffies + DFL_THROTL_SLICE); in tg_update_slice()
854 bool rw = bio_data_dir(bio); in tg_dispatch_bps_time() local
855 u64 bps_limit = tg_bps_limit(tg, rw); in tg_dispatch_bps_time()
864 tg_update_slice(tg, rw); in tg_dispatch_bps_time()
866 throtl_extend_slice(tg, rw, jiffies + bps_wait); in tg_dispatch_bps_time()
873 bool rw = bio_data_dir(bio); in tg_dispatch_iops_time() local
874 u32 iops_limit = tg_iops_limit(tg, rw); in tg_dispatch_iops_time()
880 tg_update_slice(tg, rw); in tg_dispatch_iops_time()
882 throtl_extend_slice(tg, rw, jiffies + iops_wait); in tg_dispatch_iops_time()
893 bool rw = bio_data_dir(bio); in tg_dispatch_time() local
902 BUG_ON(sq_queued(&tg->service_queue, rw) && in tg_dispatch_time()
903 bio != throtl_peek_queued(&tg->service_queue.queued[rw])); in tg_dispatch_time()
931 bool rw = bio_data_dir(bio); in throtl_add_bio_tg() local
934 qn = &tg->qnode_on_self[rw]; in throtl_add_bio_tg()
942 if (sq_queued(sq, rw) == 0) in throtl_add_bio_tg()
953 bio == throtl_peek_queued(&sq->queued[rw])) in throtl_add_bio_tg()
987 struct throtl_grp *parent_tg, bool rw) in start_parent_slice_with_credit() argument
989 if (throtl_slice_used(parent_tg, rw)) { in start_parent_slice_with_credit()
990 throtl_start_new_slice_with_credit(parent_tg, rw, in start_parent_slice_with_credit()
991 child_tg->slice_start[rw]); in start_parent_slice_with_credit()
996 static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw) in tg_dispatch_one_bio() argument
1010 bio = throtl_pop_queued(sq, &tg_to_put, rw); in tg_dispatch_one_bio()
1022 throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg); in tg_dispatch_one_bio()
1023 start_parent_slice_with_credit(tg, parent_tg, rw); in tg_dispatch_one_bio()
1026 throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw], in tg_dispatch_one_bio()
1028 BUG_ON(tg->td->nr_queued[rw] <= 0); in tg_dispatch_one_bio()
1029 tg->td->nr_queued[rw]--; in tg_dispatch_one_bio()
1032 throtl_trim_slice(tg, rw); in tg_dispatch_one_bio()
1207 int rw; in blk_throtl_dispatch_work_fn() local
1212 for (rw = READ; rw <= WRITE; rw++) in blk_throtl_dispatch_work_fn()
1213 while ((bio = throtl_pop_queued(td_sq, NULL, rw))) in blk_throtl_dispatch_work_fn()
1702 static bool tg_within_limit(struct throtl_grp *tg, struct bio *bio, bool rw) in tg_within_limit() argument
1711 return sq->nr_queued_iops[rw] == 0 && in tg_within_limit()
1719 if (sq_queued(&tg->service_queue, rw)) { in tg_within_limit()
1720 if (sq->nr_queued_bps[rw] == 0 && in tg_within_limit()
1737 bool rw = bio_data_dir(bio); in __blk_throtl_bio() local
1746 if (tg_within_limit(tg, bio, rw)) { in __blk_throtl_bio()
1761 throtl_trim_slice(tg, rw); in __blk_throtl_bio()
1783 qn = &tg->qnode_on_parent[rw]; in __blk_throtl_bio()
1794 rw == READ ? 'R' : 'W', in __blk_throtl_bio()
1795 tg->bytes_disp[rw], bio->bi_iter.bi_size, in __blk_throtl_bio()
1796 tg_bps_limit(tg, rw), in __blk_throtl_bio()
1797 tg->io_disp[rw], tg_iops_limit(tg, rw), in __blk_throtl_bio()
1800 td->nr_queued[rw]++; in __blk_throtl_bio()