Lines Matching refs:bfq_queue
137 void bfq_mark_bfqq_##name(struct bfq_queue *bfqq) \
141 void bfq_clear_bfqq_##name(struct bfq_queue *bfqq) \
145 int bfq_bfqq_##name(const struct bfq_queue *bfqq) \
379 struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync, in bic_to_bfqq()
388 static void bfq_put_stable_ref(struct bfq_queue *bfqq);
391 struct bfq_queue *bfqq, in bic_set_bfqq()
395 struct bfq_queue *old_bfqq = bic->bfqq[is_sync][actuator_idx]; in bic_set_bfqq()
587 struct bfq_queue *bfqq; in bfqq_request_over_limit()
729 static struct bfq_queue *
735 struct bfq_queue *bfqq = NULL; in bfq_rq_pos_tree_lookup()
743 bfqq = rb_entry(parent, struct bfq_queue, pos_node); in bfq_rq_pos_tree_lookup()
770 static bool bfq_too_late_for_merging(struct bfq_queue *bfqq) in bfq_too_late_for_merging()
786 bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_pos_tree_add_move()
789 struct bfq_queue *__bfqq; in bfq_pos_tree_add_move()
857 struct bfq_queue *bfqq) in bfq_asymmetric_scenario()
901 void bfq_weights_tree_add(struct bfq_queue *bfqq) in bfq_weights_tree_add()
975 void bfq_weights_tree_remove(struct bfq_queue *bfqq) in bfq_weights_tree_remove()
998 static struct request *bfq_check_fifo(struct bfq_queue *bfqq, in bfq_check_fifo()
1018 struct bfq_queue *bfqq, in bfq_find_next_rq()
1046 struct bfq_queue *bfqq) in bfq_serv_to_charge()
1067 struct bfq_queue *bfqq) in bfq_updated_next_req()
1126 static void switch_back_to_interactive_wr(struct bfq_queue *bfqq, in switch_back_to_interactive_wr()
1135 bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, in bfq_bfqq_resume_state()
1201 static int bfqq_process_refs(struct bfq_queue *bfqq) in bfqq_process_refs()
1209 static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_reset_burst_list()
1211 struct bfq_queue *item; in bfq_reset_burst_list()
1232 static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_add_to_burst()
1238 struct bfq_queue *pos, *bfqq_item; in bfq_add_to_burst()
1384 static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_handle_burst()
1450 static int bfq_bfqq_budget_left(struct bfq_queue *bfqq) in bfq_bfqq_budget_left()
1587 struct bfq_queue *bfqq, in bfq_bfqq_update_budg_for_activation()
1656 struct bfq_queue *bfqq, in bfq_update_bfqq_wr_on_rq_arrival()
1753 struct bfq_queue *bfqq) in bfq_bfqq_idle_for_long_time()
1766 static bool bfq_bfqq_higher_class_or_weight(struct bfq_queue *bfqq, in bfq_bfqq_higher_class_or_weight()
1767 struct bfq_queue *in_serv_bfqq) in bfq_bfqq_higher_class_or_weight()
1818 static bool bfq_better_to_idle(struct bfq_queue *bfqq);
1821 struct bfq_queue *bfqq, in bfq_bfqq_handle_idle_busy_switch()
1988 struct bfq_queue *bfqq) in bfq_reset_inject_limit()
2053 static void bfq_update_io_intensity(struct bfq_queue *bfqq, u64 now_ns) in bfq_update_io_intensity()
2134 static void bfq_check_waker(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_check_waker()
2208 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_add_request()
2365 struct bfq_queue *bfqq = bfqd->bio_bfqq; in bfq_find_rq_fmerge()
2385 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_remove_request()
2506 struct bfq_queue *bfqq = RQ_BFQQ(req); in bfq_request_merged()
2558 struct bfq_queue *bfqq = RQ_BFQQ(rq), in bfq_requests_merged()
2596 static void bfq_bfqq_end_wr(struct bfq_queue *bfqq) in bfq_bfqq_end_wr()
2645 struct bfq_queue *bfqq; in bfq_end_wr()
2676 static struct bfq_queue *bfqq_find_close(struct bfq_data *bfqd, in bfqq_find_close()
2677 struct bfq_queue *bfqq, in bfqq_find_close()
2682 struct bfq_queue *__bfqq; in bfqq_find_close()
2700 __bfqq = rb_entry(parent, struct bfq_queue, pos_node); in bfqq_find_close()
2711 __bfqq = rb_entry(node, struct bfq_queue, pos_node); in bfqq_find_close()
2718 static struct bfq_queue *bfq_find_close_cooperator(struct bfq_data *bfqd, in bfq_find_close_cooperator()
2719 struct bfq_queue *cur_bfqq, in bfq_find_close_cooperator()
2722 struct bfq_queue *bfqq; in bfq_find_close_cooperator()
2738 static struct bfq_queue *
2739 bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) in bfq_setup_merge()
2742 struct bfq_queue *__bfqq; in bfq_setup_merge()
2814 static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, in bfq_may_be_close_cooperator()
2815 struct bfq_queue *new_bfqq) in bfq_may_be_close_cooperator()
2844 struct bfq_queue *bfqq);
2846 static struct bfq_queue *
2847 bfq_setup_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_setup_stable_merge()
2848 struct bfq_queue *stable_merge_bfqq, in bfq_setup_stable_merge()
2853 struct bfq_queue *new_bfqq = NULL; in bfq_setup_stable_merge()
2900 static struct bfq_queue *
2901 bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_setup_cooperator()
2904 struct bfq_queue *in_service_bfqq, *new_bfqq; in bfq_setup_cooperator()
2938 struct bfq_queue *stable_merge_bfqq = in bfq_setup_cooperator()
3035 static void bfq_bfqq_save_state(struct bfq_queue *bfqq) in bfq_bfqq_save_state()
3095 void bfq_reassign_last_bfqq(struct bfq_queue *cur_bfqq, in bfq_reassign_last_bfqq()
3096 struct bfq_queue *new_bfqq) in bfq_reassign_last_bfqq()
3105 void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_release_process_ref()
3127 static struct bfq_queue *bfq_merge_bfqqs(struct bfq_data *bfqd, in bfq_merge_bfqqs()
3129 struct bfq_queue *bfqq) in bfq_merge_bfqqs()
3131 struct bfq_queue *new_bfqq = bfqq->new_bfqq; in bfq_merge_bfqqs()
3235 struct bfq_queue *bfqq = bfqd->bio_bfqq, *new_bfqq; in bfq_allow_bio_merge()
3285 struct bfq_queue *bfqq) in bfq_set_budget_timeout()
3301 struct bfq_queue *bfqq) in __bfq_set_in_service_queue()
3357 static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd) in bfq_set_in_service_queue()
3359 struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); in bfq_set_in_service_queue()
3367 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_arm_slice_timer()
3652 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_dispatch_remove()
3874 struct bfq_queue *bfqq) in idling_needed_for_service_guarantees()
3889 static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, in __bfq_bfqq_expire()
3958 struct bfq_queue *bfqq, in __bfq_bfqq_recalc_budget()
4149 static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_bfqq_is_slow()
4299 struct bfq_queue *bfqq) in bfq_bfqq_softrt_next_start()
4335 struct bfq_queue *bfqq, in bfq_bfqq_expire()
4459 static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq) in bfq_bfqq_budget_timeout()
4472 static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq) in bfq_may_expire_for_budg_timeout()
4487 struct bfq_queue *bfqq) in idling_boosts_thr_without_issues()
4586 static bool bfq_better_to_idle(struct bfq_queue *bfqq) in bfq_better_to_idle()
4637 static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq) in bfq_bfqq_must_idle()
4649 static struct bfq_queue *
4652 struct bfq_queue *bfqq, *in_serv_bfqq = bfqd->in_service_queue; in bfq_choose_bfqq_for_injection()
4740 static struct bfq_queue *
4743 struct bfq_queue *bfqq; in bfq_find_active_bfqq_for_actuator()
4773 static struct bfq_queue *
4782 struct bfq_queue *bfqq = in bfq_find_bfqq_for_underused_actuator()
4798 static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) in bfq_select_queue()
4800 struct bfq_queue *bfqq, *inject_bfqq; in bfq_select_queue()
4891 struct bfq_queue *async_bfqq = NULL; in bfq_select_queue()
4892 struct bfq_queue *blocked_bfqq = in bfq_select_queue()
4895 struct bfq_queue, in bfq_select_queue()
5032 static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_update_wr_data()
5101 struct bfq_queue *bfqq) in bfq_dispatch_rq_from_bfqq()
5160 struct bfq_queue *bfqq = NULL; in __bfq_dispatch_request()
5248 struct bfq_queue *in_serv_queue, in bfq_update_dispatch_stats()
5251 struct bfq_queue *bfqq = rq ? RQ_BFQQ(rq) : NULL; in bfq_update_dispatch_stats()
5293 struct bfq_queue *in_serv_queue, in bfq_update_dispatch_stats()
5301 struct bfq_queue *in_serv_queue; in bfq_dispatch_request()
5330 void bfq_put_queue(struct bfq_queue *bfqq) in bfq_put_queue()
5332 struct bfq_queue *item; in bfq_put_queue()
5411 static void bfq_put_stable_ref(struct bfq_queue *bfqq) in bfq_put_stable_ref()
5417 void bfq_put_cooperator(struct bfq_queue *bfqq) in bfq_put_cooperator()
5419 struct bfq_queue *__bfqq, *next; in bfq_put_cooperator()
5434 static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_exit_bfqq()
5451 struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, actuator_idx); in bfq_exit_icq_bfqq()
5506 bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic) in bfq_set_next_ioprio_data()
5555 static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd,
5563 struct bfq_queue *bfqq; in bfq_check_ioprio_change()
5577 struct bfq_queue *old_bfqq = bfqq; in bfq_check_ioprio_change()
5589 static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_init_bfqq()
5659 static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd, in bfq_async_queue_prio()
5678 static struct bfq_queue *
5679 bfq_do_early_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_do_early_stable_merge()
5681 struct bfq_queue *last_bfqq_created) in bfq_do_early_stable_merge()
5684 struct bfq_queue *new_bfqq = in bfq_do_early_stable_merge()
5750 static struct bfq_queue *bfq_do_or_sched_stable_merge(struct bfq_data *bfqd, in bfq_do_or_sched_stable_merge()
5751 struct bfq_queue *bfqq, in bfq_do_or_sched_stable_merge()
5754 struct bfq_queue **source_bfqq = bfqq->entity.parent ? in bfq_do_or_sched_stable_merge()
5758 struct bfq_queue *last_bfqq_created = *source_bfqq; in bfq_do_or_sched_stable_merge()
5829 static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, in bfq_get_queue()
5836 struct bfq_queue **async_bfqq = NULL; in bfq_get_queue()
5837 struct bfq_queue *bfqq; in bfq_get_queue()
5890 struct bfq_queue *bfqq) in bfq_update_io_thinktime()
5912 bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_update_io_seektime()
5943 struct bfq_queue *bfqq, in bfq_update_has_short_ttime()
6074 static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_rq_enqueued()
6130 static void bfqq_request_allocated(struct bfq_queue *bfqq) in bfqq_request_allocated()
6138 static void bfqq_request_freed(struct bfq_queue *bfqq) in bfqq_request_freed()
6149 struct bfq_queue *bfqq = RQ_BFQQ(rq), in __bfq_insert_request()
6155 struct bfq_queue *old_bfqq = bfqq; in __bfq_insert_request()
6204 struct bfq_queue *bfqq, in bfq_update_insert_stats()
6229 struct bfq_queue *bfqq, in bfq_update_insert_stats()
6234 static struct bfq_queue *bfq_init_rq(struct request *rq);
6241 struct bfq_queue *bfqq; in bfq_insert_request()
6307 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_update_hw_tag()
6346 static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) in bfq_completed_request()
6583 struct bfq_queue *bfqq) in bfq_update_inject_limit()
6647 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_finish_requeue_request()
6716 static struct bfq_queue *
6717 bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq) in bfq_split_bfqq()
6736 static struct bfq_queue *
6742 struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, act_idx); in __bfq_get_bfqq_handle_split()
6818 static struct bfq_queue *bfq_waker_bfqq(struct bfq_queue *bfqq) in bfq_waker_bfqq()
6820 struct bfq_queue *new_bfqq = bfqq->new_bfqq; in bfq_waker_bfqq()
6821 struct bfq_queue *waker_bfqq = bfqq->waker_bfqq; in bfq_waker_bfqq()
6851 static struct bfq_queue *bfq_get_bfqq_handle_split(struct bfq_data *bfqd, in bfq_get_bfqq_handle_split()
6857 struct bfq_queue *waker_bfqq; in bfq_get_bfqq_handle_split()
6858 struct bfq_queue *bfqq; in bfq_get_bfqq_handle_split()
6927 static struct bfq_queue *bfq_init_rq(struct request *rq) in bfq_init_rq()
6934 struct bfq_queue *bfqq; in bfq_init_rq()
7003 bfq_idle_slice_timer_body(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_idle_slice_timer_body()
7057 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_idle_slice_timer()
7074 struct bfq_queue **bfqq_ptr) in __bfq_put_async_bfqq()
7076 struct bfq_queue *bfqq = *bfqq_ptr; in __bfq_put_async_bfqq()
7166 struct bfq_queue *bfqq, *n; in bfq_exit_queue()
7394 bfq_pool = KMEM_CACHE(bfq_queue, 0); in bfq_slab_setup()