96 static int wbt_init(struct gendisk *disk, struct rq_wb *rwb);
170 	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;  in wb_recent_wait()
212 	else if (blk_queue_write_cache(rwb->rqos.disk->queue) &&  in wbt_rqw_done()
309 	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;  in latency_exceeded()
362 	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;  in rwb_trace_step()
432 	if (!rwb->rqos.disk)  in wb_timer_fn()
437 	trace_wbt_timer(rwb->rqos.disk->bdi, status, rqd->scale_step, inflight);  in wb_timer_fn()
739 static bool __wbt_enable_default(struct gendisk *disk)  in __wbt_enable_default()  argument
741 	struct request_queue *q = disk->queue;  in __wbt_enable_default()
745 	mutex_lock(&disk->rqos_state_mutex);  in __wbt_enable_default()
755 		mutex_unlock(&disk->rqos_state_mutex);  in __wbt_enable_default()
758 	mutex_unlock(&disk->rqos_state_mutex);  in __wbt_enable_default()
769 void wbt_enable_default(struct gendisk *disk)  in wbt_enable_default()  argument
771 	__wbt_enable_default(disk);  in wbt_enable_default()
775 void wbt_init_enable_default(struct gendisk *disk)  in wbt_init_enable_default()  argument
777 	struct request_queue *q = disk->queue;  in wbt_init_enable_default()
781 	if (!__wbt_enable_default(disk))  in wbt_init_enable_default()
788 	if (WARN_ON_ONCE(wbt_init(disk, rwb))) {  in wbt_init_enable_default()
811 	RQWB(rqos)->rq_depth.queue_depth = blk_queue_depth(rqos->disk->queue);  in wbt_queue_depth_changed()
819 	blk_stat_remove_callback(rqos->disk->queue, rwb->cb);  in wbt_exit()
826 void wbt_disable_default(struct gendisk *disk)  in wbt_disable_default()  argument
828 	struct rq_qos *rqos = wbt_rq_qos(disk->queue);  in wbt_disable_default()
832 	mutex_lock(&disk->rqos_state_mutex);  in wbt_disable_default()
838 	mutex_unlock(&disk->rqos_state_mutex);  in wbt_disable_default()
944 static int wbt_init(struct gendisk *disk, struct rq_wb *rwb)  in wbt_init()  argument
946 	struct request_queue *q = disk->queue;  in wbt_init()
965 	ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops);  in wbt_init()
974 int wbt_set_lat(struct gendisk *disk, s64 val)  in wbt_set_lat()  argument
976 	struct request_queue *q = disk->queue;  in wbt_set_lat()
995 		ret = wbt_init(disk, rwb);  in wbt_set_lat()
1012 	mutex_lock(&disk->rqos_state_mutex);  in wbt_set_lat()
1014 	mutex_unlock(&disk->rqos_state_mutex);  in wbt_set_lat()