212 				  unsigned int sched_domain, unsigned int type)  in flush_latency_buckets()  argument
214 	unsigned int *buckets = kqd->latency_buckets[sched_domain][type];  in flush_latency_buckets()
215 	atomic_t *cpu_buckets = cpu_latency->buckets[sched_domain][type];  in flush_latency_buckets()
227 				unsigned int sched_domain, unsigned int type,  in calculate_percentile()  argument
230 	unsigned int *buckets = kqd->latency_buckets[sched_domain][type];  in calculate_percentile()
243 	if (!kqd->latency_timeout[sched_domain])  in calculate_percentile()
244 		kqd->latency_timeout[sched_domain] = max(jiffies + HZ, 1UL);  in calculate_percentile()
246 	    time_is_after_jiffies(kqd->latency_timeout[sched_domain])) {  in calculate_percentile()
249 	kqd->latency_timeout[sched_domain] = 0;  in calculate_percentile()
257 	memset(buckets, 0, sizeof(kqd->latency_buckets[sched_domain][type]));  in calculate_percentile()
259 	trace_kyber_latency(kqd->dev, kyber_domain_names[sched_domain],  in calculate_percentile()
267 				unsigned int sched_domain, unsigned int depth)  in kyber_resize_domain()  argument
269 	depth = clamp(depth, 1U, kyber_depth[sched_domain]);  in kyber_resize_domain()
270 	if (depth != kqd->domain_tokens[sched_domain].sb.depth) {  in kyber_resize_domain()
271 		sbitmap_queue_resize(&kqd->domain_tokens[sched_domain], depth);  in kyber_resize_domain()
272 		trace_kyber_adjust(kqd->dev, kyber_domain_names[sched_domain],  in kyber_resize_domain()
280 	unsigned int sched_domain;  in kyber_timer_fn()  local
289 		for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) {  in kyber_timer_fn()
290 			flush_latency_buckets(kqd, cpu_latency, sched_domain,  in kyber_timer_fn()
292 			flush_latency_buckets(kqd, cpu_latency, sched_domain,  in kyber_timer_fn()
302 	for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) {  in kyber_timer_fn()
305 		p90 = calculate_percentile(kqd, sched_domain, KYBER_IO_LATENCY,  in kyber_timer_fn()
316 	for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) {  in kyber_timer_fn()
320 		p99 = calculate_percentile(kqd, sched_domain,  in kyber_timer_fn()
332 				p99 = kqd->domain_p99[sched_domain];  in kyber_timer_fn()
333 			kqd->domain_p99[sched_domain] = -1;  in kyber_timer_fn()
335 			kqd->domain_p99[sched_domain] = p99;  in kyber_timer_fn()
350 			orig_depth = kqd->domain_tokens[sched_domain].sb.depth;  in kyber_timer_fn()
352 			kyber_resize_domain(kqd, sched_domain, depth);  in kyber_timer_fn()
543 	unsigned int sched_domain;  in rq_clear_domain_token()  local
548 		sched_domain = kyber_sched_domain(rq->cmd_flags);  in rq_clear_domain_token()
549 		sbitmap_queue_clear(&kqd->domain_tokens[sched_domain], nr,  in rq_clear_domain_token()
574 	unsigned int sched_domain = kyber_sched_domain(bio->bi_opf);  in kyber_bio_merge()  local
575 	struct list_head *rq_list = &kcq->rq_list[sched_domain];  in kyber_bio_merge()
598 		unsigned int sched_domain = kyber_sched_domain(rq->cmd_flags);  in kyber_insert_requests()  local
600 		struct list_head *head = &kcq->rq_list[sched_domain];  in kyber_insert_requests()
608 		sbitmap_set_bit(&khd->kcq_map[sched_domain],  in kyber_insert_requests()
622 			       unsigned int sched_domain, unsigned int type,  in add_latency_sample()  argument
636 	atomic_inc(&cpu_latency->buckets[sched_domain][type][bucket]);  in add_latency_sample()
643 	unsigned int sched_domain;  in kyber_completed_request()  local
646 	sched_domain = kyber_sched_domain(rq->cmd_flags);  in kyber_completed_request()
647 	if (sched_domain == KYBER_OTHER)  in kyber_completed_request()
651 	target = kqd->latency_targets[sched_domain];  in kyber_completed_request()
652 	add_latency_sample(cpu_latency, sched_domain, KYBER_TOTAL_LATENCY,  in kyber_completed_request()
654 	add_latency_sample(cpu_latency, sched_domain, KYBER_IO_LATENCY, target,  in kyber_completed_request()
663 	unsigned int sched_domain;  member
673 	list_splice_tail_init(&kcq->rq_list[flush_data->sched_domain],  in flush_busy_kcq()
682 				  unsigned int sched_domain,  in kyber_flush_busy_kcqs()  argument
687 		.sched_domain = sched_domain,  in kyber_flush_busy_kcqs()
691 	sbitmap_for_each_set(&khd->kcq_map[sched_domain],  in kyber_flush_busy_kcqs()
710 	unsigned int sched_domain = khd->cur_domain;  in kyber_get_domain_token()  local
711 	struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain];  in kyber_get_domain_token()
712 	struct sbq_wait *wait = &khd->domain_wait[sched_domain];  in kyber_get_domain_token()
725 				  &khd->wait_index[sched_domain]);  in kyber_get_domain_token()
726 		khd->domain_ws[sched_domain] = ws;  in kyber_get_domain_token()
744 		ws = khd->domain_ws[sched_domain];  in kyber_get_domain_token()