91 	struct rq_qos rqos;  member
96 static inline struct rq_wb *RQWB(struct rq_qos *rqos)  in RQWB()  argument
98 	return container_of(rqos, struct rq_wb, rqos);  in RQWB()
167 	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;  in wb_recent_wait()
209 	else if (blk_queue_write_cache(rwb->rqos.disk->queue) &&  in wbt_rqw_done()
229 static void __wbt_done(struct rq_qos *rqos, enum wbt_flags wb_acct)  in __wbt_done()  argument
231 	struct rq_wb *rwb = RQWB(rqos);  in __wbt_done()
245 static void wbt_done(struct rq_qos *rqos, struct request *rq)  in wbt_done()  argument
247 	struct rq_wb *rwb = RQWB(rqos);  in wbt_done()
259 		__wbt_done(rqos, wbt_flags(rq));  in wbt_done()
305 	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;  in latency_exceeded()
358 	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;  in rwb_trace_step()
428 	if (!rwb->rqos.disk)  in wb_timer_fn()
433 	trace_wbt_timer(rwb->rqos.disk->bdi, status, rqd->scale_step, inflight);  in wb_timer_fn()
494 	struct rq_qos *rqos = wbt_rq_qos(q);  in wbt_disabled()  local
496 	return !rqos || !rwb_enabled(RQWB(rqos));  in wbt_disabled()
501 	struct rq_qos *rqos = wbt_rq_qos(q);  in wbt_get_min_lat()  local
502 	if (!rqos)  in wbt_get_min_lat()
504 	return RQWB(rqos)->min_lat_nsec;  in wbt_get_min_lat()
509 	struct rq_qos *rqos = wbt_rq_qos(q);  in wbt_set_min_lat()  local
510 	if (!rqos)  in wbt_set_min_lat()
513 	RQWB(rqos)->min_lat_nsec = val;  in wbt_set_min_lat()
515 		RQWB(rqos)->enable_state = WBT_STATE_ON_MANUAL;  in wbt_set_min_lat()
517 		RQWB(rqos)->enable_state = WBT_STATE_OFF_MANUAL;  in wbt_set_min_lat()
519 	wbt_update_limits(RQWB(rqos));  in wbt_set_min_lat()
634 static void wbt_cleanup(struct rq_qos *rqos, struct bio *bio)  in wbt_cleanup()  argument
636 	struct rq_wb *rwb = RQWB(rqos);  in wbt_cleanup()
638 	__wbt_done(rqos, flags);  in wbt_cleanup()
646 static void wbt_wait(struct rq_qos *rqos, struct bio *bio)  in wbt_wait()  argument
648 	struct rq_wb *rwb = RQWB(rqos);  in wbt_wait()
664 static void wbt_track(struct rq_qos *rqos, struct request *rq, struct bio *bio)  in wbt_track()  argument
666 	struct rq_wb *rwb = RQWB(rqos);  in wbt_track()
670 static void wbt_issue(struct rq_qos *rqos, struct request *rq)  in wbt_issue()  argument
672 	struct rq_wb *rwb = RQWB(rqos);  in wbt_issue()
690 static void wbt_requeue(struct rq_qos *rqos, struct request *rq)  in wbt_requeue()  argument
692 	struct rq_wb *rwb = RQWB(rqos);  in wbt_requeue()
707 	struct rq_qos *rqos;  in wbt_enable_default()  local
715 	rqos = wbt_rq_qos(q);  in wbt_enable_default()
716 	if (rqos) {  in wbt_enable_default()
717 		if (enable && RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT)  in wbt_enable_default()
718 			RQWB(rqos)->enable_state = WBT_STATE_ON_DEFAULT;  in wbt_enable_default()
756 static void wbt_queue_depth_changed(struct rq_qos *rqos)  in wbt_queue_depth_changed()  argument
758 	RQWB(rqos)->rq_depth.queue_depth = blk_queue_depth(rqos->disk->queue);  in wbt_queue_depth_changed()
759 	wbt_update_limits(RQWB(rqos));  in wbt_queue_depth_changed()
762 static void wbt_exit(struct rq_qos *rqos)  in wbt_exit()  argument
764 	struct rq_wb *rwb = RQWB(rqos);  in wbt_exit()
766 	blk_stat_remove_callback(rqos->disk->queue, rwb->cb);  in wbt_exit()
776 	struct rq_qos *rqos = wbt_rq_qos(disk->queue);  in wbt_disable_default()  local
778 	if (!rqos)  in wbt_disable_default()
780 	rwb = RQWB(rqos);  in wbt_disable_default()
791 	struct rq_qos *rqos = data;  in wbt_curr_win_nsec_show()  local
792 	struct rq_wb *rwb = RQWB(rqos);  in wbt_curr_win_nsec_show()
800 	struct rq_qos *rqos = data;  in wbt_enabled_show()  local
801 	struct rq_wb *rwb = RQWB(rqos);  in wbt_enabled_show()
809 	struct rq_qos *rqos = data;  in wbt_id_show()  local
811 	seq_printf(m, "%u\n", rqos->id);  in wbt_id_show()
817 	struct rq_qos *rqos = data;  in wbt_inflight_show()  local
818 	struct rq_wb *rwb = RQWB(rqos);  in wbt_inflight_show()
829 	struct rq_qos *rqos = data;  in wbt_min_lat_nsec_show()  local
830 	struct rq_wb *rwb = RQWB(rqos);  in wbt_min_lat_nsec_show()
838 	struct rq_qos *rqos = data;  in wbt_unknown_cnt_show()  local
839 	struct rq_wb *rwb = RQWB(rqos);  in wbt_unknown_cnt_show()
847 	struct rq_qos *rqos = data;  in wbt_normal_show()  local
848 	struct rq_wb *rwb = RQWB(rqos);  in wbt_normal_show()
856 	struct rq_qos *rqos = data;  in wbt_background_show()  local
857 	struct rq_wb *rwb = RQWB(rqos);  in wbt_background_show()
922 	ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops);  in wbt_init()