1 // SPDX-License-Identifier: GPL-2.0
3  * CPUFreq governor based on scheduler-provided CPU utilization data.
30 	/* The next fields are only needed if fast switch cannot be used: */
54 	/* The field below is for single-CPU policies only: */
69 	 * Since cpufreq_update_util() is called with rq->lock held for  in sugov_should_update_freq()
70 	 * the @target_cpu, our per-CPU data is fully serialized.  in sugov_should_update_freq()
72 	 * However, drivers cannot in general deal with cross-CPU  in sugov_should_update_freq()
74 	 * sugov_update_commit() call may not for the fast switching platforms.  in sugov_should_update_freq()
83 	if (!cpufreq_this_cpu_can_update(sg_policy->policy))  in sugov_should_update_freq()
86 	if (unlikely(READ_ONCE(sg_policy->limits_changed))) {  in sugov_should_update_freq()
87 		WRITE_ONCE(sg_policy->limits_changed, false);  in sugov_should_update_freq()
88 		sg_policy->need_freq_update = true;  in sugov_should_update_freq()
101 	} else if (sg_policy->need_freq_update) {  in sugov_should_update_freq()
106 	delta_ns = time - sg_policy->last_freq_update_time;  in sugov_should_update_freq()
108 	return delta_ns >= sg_policy->freq_update_delay_ns;  in sugov_should_update_freq()
114 	if (sg_policy->need_freq_update) {  in sugov_update_next_freq()
115 		sg_policy->need_freq_update = false;  in sugov_update_next_freq()
124 		if (sg_policy->next_freq == next_freq &&  in sugov_update_next_freq()
127 	} else if (sg_policy->next_freq == next_freq) {  in sugov_update_next_freq()
131 	sg_policy->next_freq = next_freq;  in sugov_update_next_freq()
132 	sg_policy->last_freq_update_time = time;  in sugov_update_next_freq()
139 	if (!sg_policy->work_in_progress) {  in sugov_deferred_update()
140 		sg_policy->work_in_progress = true;  in sugov_deferred_update()
141 		irq_work_queue(&sg_policy->irq_work);  in sugov_deferred_update()
146  * get_capacity_ref_freq - get the reference frequency that has been used to
156 	unsigned int freq = arch_scale_freq_ref(policy->cpu);  in get_capacity_ref_freq()
162 		return policy->cpuinfo.max_freq;  in get_capacity_ref_freq()
168 	return policy->cur + (policy->cur >> 2);  in get_capacity_ref_freq()
172  * get_next_freq - Compute a new frequency for a given cpufreq policy.
177  * If the utilization is frequency-invariant, choose the new frequency to be
182  * Otherwise, approximate the would-be frequency-invariant utilization by
189  * The lowest driver-supported frequency which is equal or greater than the raw
196 	struct cpufreq_policy *policy = sg_policy->policy;  in get_next_freq()
202 	if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)  in get_next_freq()
203 		return sg_policy->next_freq;  in get_next_freq()
205 	sg_policy->cached_raw_freq = freq;  in get_next_freq()
213 	/* Add dvfs headroom to actual utilization */  in sugov_effective_cpu_perf()
228 	unsigned long min, max, util = scx_cpuperf_target(sg_cpu->cpu);  in sugov_get_util()
231 		util += cpu_util_cfs_boost(sg_cpu->cpu);  in sugov_get_util()
232 	util = effective_cpu_util(sg_cpu->cpu, util, &min, &max);  in sugov_get_util()
234 	sg_cpu->bw_min = min;  in sugov_get_util()
235 	sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max);  in sugov_get_util()
239  * sugov_iowait_reset() - Reset the IO boost status of a CPU.
252 	s64 delta_ns = time - sg_cpu->last_update;  in sugov_iowait_reset()
258 	sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0;  in sugov_iowait_reset()
259 	sg_cpu->iowait_boost_pending = set_iowait_boost;  in sugov_iowait_reset()
265  * sugov_iowait_boost() - Updates the IO boost status of a CPU.
284 	if (sg_cpu->iowait_boost &&  in sugov_iowait_boost()
293 	if (sg_cpu->iowait_boost_pending)  in sugov_iowait_boost()
295 	sg_cpu->iowait_boost_pending = true;  in sugov_iowait_boost()
298 	if (sg_cpu->iowait_boost) {  in sugov_iowait_boost()
299 		sg_cpu->iowait_boost =  in sugov_iowait_boost()
300 			min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE);  in sugov_iowait_boost()
305 	sg_cpu->iowait_boost = IOWAIT_BOOST_MIN;  in sugov_iowait_boost()
309  * sugov_iowait_apply() - Apply the IO boost to a CPU.
330 	if (!sg_cpu->iowait_boost)  in sugov_iowait_apply()
337 	if (!sg_cpu->iowait_boost_pending) {  in sugov_iowait_apply()
341 		sg_cpu->iowait_boost >>= 1;  in sugov_iowait_apply()
342 		if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) {  in sugov_iowait_apply()
343 			sg_cpu->iowait_boost = 0;  in sugov_iowait_apply()
348 	sg_cpu->iowait_boost_pending = false;  in sugov_iowait_apply()
351 	 * sg_cpu->util is already in capacity scale; convert iowait_boost  in sugov_iowait_apply()
354 	return (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT;  in sugov_iowait_apply()
372 	if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)))  in sugov_hold_freq()
379 	idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);  in sugov_hold_freq()
380 	ret = idle_calls == sg_cpu->saved_idle_calls;  in sugov_hold_freq()
382 	sg_cpu->saved_idle_calls = idle_calls;  in sugov_hold_freq()
395 	if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_min)  in ignore_dl_rate_limit()
396 		sg_cpu->sg_policy->need_freq_update = true;  in ignore_dl_rate_limit()
406 	sg_cpu->last_update = time;  in sugov_update_single_common()
410 	if (!sugov_should_update_freq(sg_cpu->sg_policy, time))  in sugov_update_single_common()
423 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_single_freq()
424 	unsigned int cached_freq = sg_policy->cached_raw_freq;  in sugov_update_single_freq()
428 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_update_single_freq()
433 	next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap);  in sugov_update_single_freq()
435 	if (sugov_hold_freq(sg_cpu) && next_f < sg_policy->next_freq &&  in sugov_update_single_freq()
436 	    !sg_policy->need_freq_update) {  in sugov_update_single_freq()
437 		next_f = sg_policy->next_freq;  in sugov_update_single_freq()
440 		sg_policy->cached_raw_freq = cached_freq;  in sugov_update_single_freq()
447 	 * This code runs under rq->lock for the target CPU, so it won't run  in sugov_update_single_freq()
449 	 * necessary to acquire the lock in the fast switch case.  in sugov_update_single_freq()
451 	if (sg_policy->policy->fast_switch_enabled) {  in sugov_update_single_freq()
452 		cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_single_freq()
454 		raw_spin_lock(&sg_policy->update_lock);  in sugov_update_single_freq()
456 		raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_single_freq()
464 	unsigned long prev_util = sg_cpu->util;  in sugov_update_single_perf()
477 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_update_single_perf()
482 	if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util)  in sugov_update_single_perf()
483 		sg_cpu->util = prev_util;  in sugov_update_single_perf()
485 	cpufreq_driver_adjust_perf(sg_cpu->cpu, sg_cpu->bw_min,  in sugov_update_single_perf()
486 				   sg_cpu->util, max_cap);  in sugov_update_single_perf()
488 	sg_cpu->sg_policy->last_freq_update_time = time;  in sugov_update_single_perf()
493 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_next_freq_shared()
494 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_next_freq_shared()
498 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_next_freq_shared()
500 	for_each_cpu(j, policy->cpus) {  in sugov_next_freq_shared()
507 		util = max(j_sg_cpu->util, util);  in sugov_next_freq_shared()
517 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_shared()
520 	raw_spin_lock(&sg_policy->update_lock);  in sugov_update_shared()
523 	sg_cpu->last_update = time;  in sugov_update_shared()
533 		if (sg_policy->policy->fast_switch_enabled)  in sugov_update_shared()
534 			cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_shared()
539 	raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_shared()
549 	 * Hold sg_policy->update_lock shortly to handle the case where:  in sugov_work()
550 	 * in case sg_policy->next_freq is read here, and then updated by  in sugov_work()
558 	raw_spin_lock_irqsave(&sg_policy->update_lock, flags);  in sugov_work()
559 	freq = sg_policy->next_freq;  in sugov_work()
560 	sg_policy->work_in_progress = false;  in sugov_work()
561 	raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);  in sugov_work()
563 	mutex_lock(&sg_policy->work_lock);  in sugov_work()
564 	__cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);  in sugov_work()
565 	mutex_unlock(&sg_policy->work_lock);  in sugov_work()
574 	kthread_queue_work(&sg_policy->worker, &sg_policy->work);  in sugov_irq_work()
591 	return sprintf(buf, "%u\n", tunables->rate_limit_us);  in rate_limit_us_show()
602 		return -EINVAL;  in rate_limit_us_store()
604 	tunables->rate_limit_us = rate_limit_us;  in rate_limit_us_store()
606 	list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)  in rate_limit_us_store()
607 		sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;  in rate_limit_us_store()
645 	sg_policy->policy = policy;  in sugov_policy_alloc()
646 	raw_spin_lock_init(&sg_policy->update_lock);  in sugov_policy_alloc()
672 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_kthread_create()
676 	if (policy->fast_switch_enabled)  in sugov_kthread_create()
679 	kthread_init_work(&sg_policy->work, sugov_work);  in sugov_kthread_create()
680 	kthread_init_worker(&sg_policy->worker);  in sugov_kthread_create()
681 	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,  in sugov_kthread_create()
683 				cpumask_first(policy->related_cpus));  in sugov_kthread_create()
696 	sg_policy->thread = thread;  in sugov_kthread_create()
697 	if (policy->dvfs_possible_from_any_cpu)  in sugov_kthread_create()
698 		set_cpus_allowed_ptr(thread, policy->related_cpus);  in sugov_kthread_create()
700 		kthread_bind_mask(thread, policy->related_cpus);  in sugov_kthread_create()
702 	init_irq_work(&sg_policy->irq_work, sugov_irq_work);  in sugov_kthread_create()
703 	mutex_init(&sg_policy->work_lock);  in sugov_kthread_create()
713 	if (sg_policy->policy->fast_switch_enabled)  in sugov_kthread_stop()
716 	kthread_flush_worker(&sg_policy->worker);  in sugov_kthread_stop()
717 	kthread_stop(sg_policy->thread);  in sugov_kthread_stop()
718 	mutex_destroy(&sg_policy->work_lock);  in sugov_kthread_stop()
727 		gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_tunables_alloc()
747 	if (policy->governor_data)  in sugov_init()
748 		return -EBUSY;  in sugov_init()
754 		ret = -ENOMEM;  in sugov_init()
766 			ret = -EINVAL;  in sugov_init()
769 		policy->governor_data = sg_policy;  in sugov_init()
770 		sg_policy->tunables = global_tunables;  in sugov_init()
772 		gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);  in sugov_init()
778 		ret = -ENOMEM;  in sugov_init()
782 	tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy);  in sugov_init()
784 	policy->governor_data = sg_policy;  in sugov_init()
785 	sg_policy->tunables = tunables;  in sugov_init()
787 	ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,  in sugov_init()
803 	kobject_put(&tunables->attr_set.kobj);  in sugov_init()
804 	policy->governor_data = NULL;  in sugov_init()
823 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_exit()
824 	struct sugov_tunables *tunables = sg_policy->tunables;  in sugov_exit()
829 	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_exit()
830 	policy->governor_data = NULL;  in sugov_exit()
845 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_start()
849 	sg_policy->freq_update_delay_ns	= sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;  in sugov_start()
850 	sg_policy->last_freq_update_time	= 0;  in sugov_start()
851 	sg_policy->next_freq			= 0;  in sugov_start()
852 	sg_policy->work_in_progress		= false;  in sugov_start()
853 	sg_policy->limits_changed		= false;  in sugov_start()
854 	sg_policy->cached_raw_freq		= 0;  in sugov_start()
856 	sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);  in sugov_start()
860 	else if (policy->fast_switch_enabled && cpufreq_driver_has_adjust_perf())  in sugov_start()
865 	for_each_cpu(cpu, policy->cpus) {  in sugov_start()
869 		sg_cpu->cpu = cpu;  in sugov_start()
870 		sg_cpu->sg_policy = sg_policy;  in sugov_start()
871 		cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu);  in sugov_start()
878 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_stop()
881 	for_each_cpu(cpu, policy->cpus)  in sugov_stop()
886 	if (!policy->fast_switch_enabled) {  in sugov_stop()
887 		irq_work_sync(&sg_policy->irq_work);  in sugov_stop()
888 		kthread_cancel_work_sync(&sg_policy->work);  in sugov_stop()
894 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_limits()
896 	if (!policy->fast_switch_enabled) {  in sugov_limits()
897 		mutex_lock(&sg_policy->work_lock);  in sugov_limits()
899 		mutex_unlock(&sg_policy->work_lock);  in sugov_limits()
911 	WRITE_ONCE(sg_policy->limits_changed, true);  in sugov_limits()
934 	return policy->governor == &schedutil_gov;  in sugov_is_governor()