Lines Matching +full:max +full:- +full:freq
1 // SPDX-License-Identifier: GPL-2.0
3 * CPUFreq governor based on scheduler-provided CPU utilization data.
52 /* The field below is for single-CPU policies only: */
67 * Since cpufreq_update_util() is called with rq->lock held for
68 * the @target_cpu, our per-CPU data is fully serialized.
70 * However, drivers cannot in general deal with cross-CPU
81 if (!cpufreq_this_cpu_can_update(sg_policy->policy))
84 if (unlikely(sg_policy->limits_changed)) {
85 sg_policy->limits_changed = false;
86 sg_policy->need_freq_update = true;
90 delta_ns = time - sg_policy->last_freq_update_time;
92 return delta_ns >= sg_policy->freq_update_delay_ns;
98 if (sg_policy->need_freq_update)
99 sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);
100 else if (sg_policy->next_freq == next_freq)
103 sg_policy->next_freq = next_freq;
104 sg_policy->last_freq_update_time = time;
111 if (!sg_policy->work_in_progress) {
112 sg_policy->work_in_progress = true;
113 irq_work_queue(&sg_policy->irq_work);
118 * get_capacity_ref_freq - get the reference frequency that has been used to
128 unsigned int freq = arch_scale_freq_ref(policy->cpu);
130 if (freq)
131 return freq;
134 return policy->cpuinfo.max_freq;
140 return policy->cur + (policy->cur >> 2);
144 * get_next_freq - Compute a new frequency for a given cpufreq policy.
147 * @max: CPU capacity.
149 * If the utilization is frequency-invariant, choose the new frequency to be
152 * next_freq = C * max_freq * util / max
154 * Otherwise, approximate the would-be frequency-invariant utilization by
157 * next_freq = C * curr_freq * util_raw / max
159 * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
161 * The lowest driver-supported frequency which is equal or greater than the raw
162 * next_freq (as calculated above) is returned, subject to policy min/max and
166 unsigned long util, unsigned long max)
168 struct cpufreq_policy *policy = sg_policy->policy;
169 unsigned int freq;
171 freq = get_capacity_ref_freq(policy);
172 freq = map_util_freq(util, freq, max);
174 if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)
175 return sg_policy->next_freq;
177 sg_policy->cached_raw_freq = freq;
178 return cpufreq_driver_resolve_freq(policy, freq);
183 unsigned long max)
187 /* Actually we don't need to target the max performance */
188 if (actual < max)
189 max = actual;
195 return max(min, max);
200 unsigned long min, max, util = scx_cpuperf_target(sg_cpu->cpu);
203 util += cpu_util_cfs_boost(sg_cpu->cpu);
204 util = effective_cpu_util(sg_cpu->cpu, util, &min, &max);
205 util = max(util, boost);
206 sg_cpu->bw_min = min;
207 sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max);
211 * sugov_iowait_reset() - Reset the IO boost status of a CPU.
224 s64 delta_ns = time - sg_cpu->last_update;
230 sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0;
231 sg_cpu->iowait_boost_pending = set_iowait_boost;
237 * sugov_iowait_boost() - Updates the IO boost status of a CPU.
256 if (sg_cpu->iowait_boost &&
265 if (sg_cpu->iowait_boost_pending)
267 sg_cpu->iowait_boost_pending = true;
270 if (sg_cpu->iowait_boost) {
271 sg_cpu->iowait_boost =
272 min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE);
277 sg_cpu->iowait_boost = IOWAIT_BOOST_MIN;
281 * sugov_iowait_apply() - Apply the IO boost to a CPU.
284 * @max_cap: the max CPU capacity
302 if (!sg_cpu->iowait_boost)
309 if (!sg_cpu->iowait_boost_pending) {
313 sg_cpu->iowait_boost >>= 1;
314 if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) {
315 sg_cpu->iowait_boost = 0;
320 sg_cpu->iowait_boost_pending = false;
323 * sg_cpu->util is already in capacity scale; convert iowait_boost
326 return (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT;
344 if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)))
351 idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);
352 ret = idle_calls == sg_cpu->saved_idle_calls;
354 sg_cpu->saved_idle_calls = idle_calls;
367 if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_min)
368 sg_cpu->sg_policy->limits_changed = true;
378 sg_cpu->last_update = time;
382 if (!sugov_should_update_freq(sg_cpu->sg_policy, time))
395 struct sugov_policy *sg_policy = sg_cpu->sg_policy;
396 unsigned int cached_freq = sg_policy->cached_raw_freq;
400 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);
405 next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap);
407 if (sugov_hold_freq(sg_cpu) && next_f < sg_policy->next_freq &&
408 !sg_policy->need_freq_update) {
409 next_f = sg_policy->next_freq;
411 /* Restore cached freq as next_freq has changed */
412 sg_policy->cached_raw_freq = cached_freq;
419 * This code runs under rq->lock for the target CPU, so it won't run
423 if (sg_policy->policy->fast_switch_enabled) {
424 cpufreq_driver_fast_switch(sg_policy->policy, next_f);
426 raw_spin_lock(&sg_policy->update_lock);
428 raw_spin_unlock(&sg_policy->update_lock);
436 unsigned long prev_util = sg_cpu->util;
449 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);
454 if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util)
455 sg_cpu->util = prev_util;
457 cpufreq_driver_adjust_perf(sg_cpu->cpu, sg_cpu->bw_min,
458 sg_cpu->util, max_cap);
460 sg_cpu->sg_policy->last_freq_update_time = time;
465 struct sugov_policy *sg_policy = sg_cpu->sg_policy;
466 struct cpufreq_policy *policy = sg_policy->policy;
470 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);
472 for_each_cpu(j, policy->cpus) {
479 util = max(j_sg_cpu->util, util);
489 struct sugov_policy *sg_policy = sg_cpu->sg_policy;
492 raw_spin_lock(&sg_policy->update_lock);
495 sg_cpu->last_update = time;
505 if (sg_policy->policy->fast_switch_enabled)
506 cpufreq_driver_fast_switch(sg_policy->policy, next_f);
511 raw_spin_unlock(&sg_policy->update_lock);
517 unsigned int freq;
521 * Hold sg_policy->update_lock shortly to handle the case where:
522 * in case sg_policy->next_freq is read here, and then updated by
530 raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
531 freq = sg_policy->next_freq;
532 sg_policy->work_in_progress = false;
533 raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);
535 mutex_lock(&sg_policy->work_lock);
536 __cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);
537 mutex_unlock(&sg_policy->work_lock);
546 kthread_queue_work(&sg_policy->worker, &sg_policy->work);
563 return sprintf(buf, "%u\n", tunables->rate_limit_us);
574 return -EINVAL;
576 tunables->rate_limit_us = rate_limit_us;
578 list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)
579 sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;
642 sg_policy->policy = policy;
643 raw_spin_lock_init(&sg_policy->update_lock);
669 struct cpufreq_policy *policy = sg_policy->policy;
673 if (policy->fast_switch_enabled)
676 kthread_init_work(&sg_policy->work, sugov_work);
677 kthread_init_worker(&sg_policy->worker);
678 thread = kthread_create(kthread_worker_fn, &sg_policy->worker,
680 cpumask_first(policy->related_cpus));
693 sg_policy->thread = thread;
694 kthread_bind_mask(thread, policy->related_cpus);
695 init_irq_work(&sg_policy->irq_work, sugov_irq_work);
696 mutex_init(&sg_policy->work_lock);
706 if (sg_policy->policy->fast_switch_enabled)
709 kthread_flush_worker(&sg_policy->worker);
710 kthread_stop(sg_policy->thread);
711 mutex_destroy(&sg_policy->work_lock);
720 gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);
740 if (policy->governor_data)
741 return -EBUSY;
747 ret = -ENOMEM;
759 ret = -EINVAL;
762 policy->governor_data = sg_policy;
763 sg_policy->tunables = global_tunables;
765 gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);
771 ret = -ENOMEM;
775 tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy);
777 policy->governor_data = sg_policy;
778 sg_policy->tunables = tunables;
780 ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,
793 kobject_put(&tunables->attr_set.kobj);
794 policy->governor_data = NULL;
813 struct sugov_policy *sg_policy = policy->governor_data;
814 struct sugov_tunables *tunables = sg_policy->tunables;
819 count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);
820 policy->governor_data = NULL;
835 struct sugov_policy *sg_policy = policy->governor_data;
839 sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
840 sg_policy->last_freq_update_time = 0;
841 sg_policy->next_freq = 0;
842 sg_policy->work_in_progress = false;
843 sg_policy->limits_changed = false;
844 sg_policy->cached_raw_freq = 0;
846 sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);
850 else if (policy->fast_switch_enabled && cpufreq_driver_has_adjust_perf())
855 for_each_cpu(cpu, policy->cpus) {
859 sg_cpu->cpu = cpu;
860 sg_cpu->sg_policy = sg_policy;
861 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu);
868 struct sugov_policy *sg_policy = policy->governor_data;
871 for_each_cpu(cpu, policy->cpus)
876 if (!policy->fast_switch_enabled) {
877 irq_work_sync(&sg_policy->irq_work);
878 kthread_cancel_work_sync(&sg_policy->work);
884 struct sugov_policy *sg_policy = policy->governor_data;
886 if (!policy->fast_switch_enabled) {
887 mutex_lock(&sg_policy->work_lock);
889 mutex_unlock(&sg_policy->work_lock);
892 sg_policy->limits_changed = true;