xref: /linux/kernel/sched/cpufreq_schedutil.c (revision 486b257357c23743d9729ea715136d105fff461e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * CPUFreq governor based on scheduler-provided CPU utilization data.
4  *
5  * Copyright (C) 2016, Intel Corporation
6  * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
7  */
8 #include <uapi/linux/sched/types.h>
9 #include "sched.h"
10 
11 #define IOWAIT_BOOST_MIN	(SCHED_CAPACITY_SCALE / 8)
12 
13 struct sugov_tunables {
14 	struct gov_attr_set	attr_set;
15 	unsigned int		rate_limit_us;
16 };
17 
18 struct sugov_policy {
19 	struct cpufreq_policy	*policy;
20 
21 	struct sugov_tunables	*tunables;
22 	struct list_head	tunables_hook;
23 
24 	raw_spinlock_t		update_lock;
25 	u64			last_freq_update_time;
26 	s64			freq_update_delay_ns;
27 	unsigned int		next_freq;
28 	unsigned int		cached_raw_freq;
29 
30 	/* The next fields are only needed if fast switch cannot be used: */
31 	struct			irq_work irq_work;
32 	struct			kthread_work work;
33 	struct			mutex work_lock;
34 	struct			kthread_worker worker;
35 	struct task_struct	*thread;
36 	bool			work_in_progress;
37 
38 	bool			limits_changed;
39 	bool			need_freq_update;
40 };
41 
42 struct sugov_cpu {
43 	struct update_util_data	update_util;
44 	struct sugov_policy	*sg_policy;
45 	unsigned int		cpu;
46 
47 	bool			iowait_boost_pending;
48 	unsigned int		iowait_boost;
49 	u64			last_update;
50 
51 	unsigned long		util;
52 	unsigned long		bw_min;
53 	unsigned long		bw_max;
54 
55 	/* The field below is for single-CPU policies only: */
56 #ifdef CONFIG_NO_HZ_COMMON
57 	unsigned long		saved_idle_calls;
58 #endif
59 };
60 
61 static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu);
62 
63 /************************ Governor internals ***********************/
64 
65 static void sugov_update_rate_limit_us(struct sugov_policy *sg_policy)
66 {
67 	/*
68 	 * Cast rate_limit_us before multiplication to force 64-bit arithmetic.
69 	 * Otherwise, on 32-bit platforms, both operands are converted to
70 	 * 32-bit unsigned long and the multiplication may overflow.
71 	 */
72 	sg_policy->freq_update_delay_ns =
73 		(s64)sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
74 }
75 
76 static void sugov_update_rate_limit_us_sync(struct sugov_policy *sg_policy)
77 {
78 	/*
79 	 * Serialize with rate_limit_us_store() to prevent this path from using
80 	 * a stale rate limit tunable value to set the limit for the given
81 	 * policy.
82 	 */
83 	guard(mutex)(&sg_policy->tunables->attr_set.update_lock);
84 
85 	sugov_update_rate_limit_us(sg_policy);
86 }
87 
88 static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
89 {
90 	s64 delta_ns;
91 
92 	/*
93 	 * Since cpufreq_update_util() is called with rq->lock held for
94 	 * the @target_cpu, our per-CPU data is fully serialized.
95 	 *
96 	 * However, drivers cannot in general deal with cross-CPU
97 	 * requests, so while get_next_freq() will work, our
98 	 * sugov_update_commit() call may not for the fast switching platforms.
99 	 *
100 	 * Hence stop here for remote requests if they aren't supported
101 	 * by the hardware, as calculating the frequency is pointless if
102 	 * we cannot in fact act on it.
103 	 *
104 	 * This is needed on the slow switching platforms too to prevent CPUs
105 	 * going offline from leaving stale IRQ work items behind.
106 	 */
107 	if (!cpufreq_this_cpu_can_update(sg_policy->policy))
108 		return false;
109 
110 	if (unlikely(READ_ONCE(sg_policy->limits_changed))) {
111 		WRITE_ONCE(sg_policy->limits_changed, false);
112 		sg_policy->need_freq_update = true;
113 
114 		/*
115 		 * The above limits_changed update must occur before the reads
116 		 * of policy limits in cpufreq_driver_resolve_freq() or a policy
117 		 * limits update might be missed, so use a memory barrier to
118 		 * ensure it.
119 		 *
120 		 * This pairs with the write memory barrier in sugov_limits().
121 		 */
122 		smp_mb();
123 
124 		return true;
125 	} else if (sg_policy->need_freq_update) {
126 		/* ignore_dl_rate_limit() wants a new frequency to be found. */
127 		return true;
128 	}
129 
130 	delta_ns = time - sg_policy->last_freq_update_time;
131 
132 	return delta_ns >= sg_policy->freq_update_delay_ns;
133 }
134 
135 static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time,
136 				   unsigned int next_freq)
137 {
138 	if (sg_policy->need_freq_update) {
139 		sg_policy->need_freq_update = false;
140 		/*
141 		 * The policy limits have changed, but if the return value of
142 		 * cpufreq_driver_resolve_freq() after applying the new limits
143 		 * is still equal to the previously selected frequency, the
144 		 * driver callback need not be invoked unless the driver
145 		 * specifically wants that to happen on every update of the
146 		 * policy limits.
147 		 */
148 		if (sg_policy->next_freq == next_freq &&
149 		    !cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS))
150 			return false;
151 	} else if (sg_policy->next_freq == next_freq) {
152 		return false;
153 	}
154 
155 	sg_policy->next_freq = next_freq;
156 	sg_policy->last_freq_update_time = time;
157 
158 	return true;
159 }
160 
161 static void sugov_deferred_update(struct sugov_policy *sg_policy)
162 {
163 	if (!sg_policy->work_in_progress) {
164 		sg_policy->work_in_progress = true;
165 		irq_work_queue(&sg_policy->irq_work);
166 	}
167 }
168 
169 /**
170  * get_capacity_ref_freq - get the reference frequency that has been used to
171  * correlate frequency and compute capacity for a given cpufreq policy. We use
172  * the CPU managing it for the arch_scale_freq_ref() call in the function.
173  * @policy: the cpufreq policy of the CPU in question.
174  *
175  * Return: the reference CPU frequency to compute a capacity.
176  */
177 static __always_inline
178 unsigned long get_capacity_ref_freq(struct cpufreq_policy *policy)
179 {
180 	unsigned int freq = arch_scale_freq_ref(policy->cpu);
181 
182 	if (freq)
183 		return freq;
184 
185 	if (arch_scale_freq_invariant())
186 		return policy->cpuinfo.max_freq;
187 
188 	/*
189 	 * Apply a 25% margin so that we select a higher frequency than
190 	 * the current one before the CPU is fully busy:
191 	 */
192 	return policy->cur + (policy->cur >> 2);
193 }
194 
195 /**
196  * get_next_freq - Compute a new frequency for a given cpufreq policy.
197  * @sg_policy: schedutil policy object to compute the new frequency for.
198  * @util: Current CPU utilization.
199  * @max: CPU capacity.
200  *
201  * If the utilization is frequency-invariant, choose the new frequency to be
202  * proportional to it, that is
203  *
204  * next_freq = C * max_freq * util / max
205  *
206  * Otherwise, approximate the would-be frequency-invariant utilization by
207  * util_raw * (curr_freq / max_freq) which leads to
208  *
209  * next_freq = C * curr_freq * util_raw / max
210  *
211  * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
212  *
213  * The lowest driver-supported frequency which is equal or greater than the raw
214  * next_freq (as calculated above) is returned, subject to policy min/max and
215  * cpufreq driver limitations.
216  */
217 static unsigned int get_next_freq(struct sugov_policy *sg_policy,
218 				  unsigned long util, unsigned long max)
219 {
220 	struct cpufreq_policy *policy = sg_policy->policy;
221 	unsigned int freq;
222 
223 	freq = get_capacity_ref_freq(policy);
224 	freq = map_util_freq(util, freq, max);
225 
226 	if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)
227 		return sg_policy->next_freq;
228 
229 	sg_policy->cached_raw_freq = freq;
230 	return cpufreq_driver_resolve_freq(policy, freq);
231 }
232 
233 unsigned long sugov_effective_cpu_perf(int cpu, unsigned long actual,
234 				 unsigned long min,
235 				 unsigned long max)
236 {
237 	/* Add dvfs headroom to actual utilization */
238 	actual = map_util_perf(actual);
239 	/* Actually we don't need to target the max performance */
240 	if (actual < max)
241 		max = actual;
242 
243 	/*
244 	 * Ensure at least minimum performance while providing more compute
245 	 * capacity when possible.
246 	 */
247 	return max(min, max);
248 }
249 
250 static void sugov_get_util(struct sugov_cpu *sg_cpu, unsigned long boost)
251 {
252 	unsigned long min, max, util = scx_cpuperf_target(sg_cpu->cpu);
253 
254 	if (!scx_switched_all())
255 		util += cpu_util_cfs_boost(sg_cpu->cpu);
256 	util = effective_cpu_util(sg_cpu->cpu, util, &min, &max);
257 	util = max(util, boost);
258 	sg_cpu->bw_min = min;
259 	sg_cpu->bw_max = max;
260 	sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max);
261 }
262 
263 /**
264  * sugov_iowait_reset() - Reset the IO boost status of a CPU.
265  * @sg_cpu: the sugov data for the CPU to boost
266  * @time: the update time from the caller
267  * @set_iowait_boost: true if an IO boost has been requested
268  *
269  * The IO wait boost of a task is disabled after a tick since the last update
270  * of a CPU. If a new IO wait boost is requested after more then a tick, then
271  * we enable the boost starting from IOWAIT_BOOST_MIN, which improves energy
272  * efficiency by ignoring sporadic wakeups from IO.
273  */
274 static bool sugov_iowait_reset(struct sugov_cpu *sg_cpu, u64 time,
275 			       bool set_iowait_boost)
276 {
277 	s64 delta_ns = time - sg_cpu->last_update;
278 
279 	/* Reset boost only if a tick has elapsed since last request */
280 	if (delta_ns <= TICK_NSEC)
281 		return false;
282 
283 	sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0;
284 	sg_cpu->iowait_boost_pending = set_iowait_boost;
285 
286 	return true;
287 }
288 
289 /**
290  * sugov_iowait_boost() - Updates the IO boost status of a CPU.
291  * @sg_cpu: the sugov data for the CPU to boost
292  * @time: the update time from the caller
293  * @flags: SCHED_CPUFREQ_IOWAIT if the task is waking up after an IO wait
294  *
295  * Each time a task wakes up after an IO operation, the CPU utilization can be
296  * boosted to a certain utilization which doubles at each "frequent and
297  * successive" wakeup from IO, ranging from IOWAIT_BOOST_MIN to the utilization
298  * of the maximum OPP.
299  *
300  * To keep doubling, an IO boost has to be requested at least once per tick,
301  * otherwise we restart from the utilization of the minimum OPP.
302  */
303 static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,
304 			       unsigned int flags)
305 {
306 	bool set_iowait_boost = flags & SCHED_CPUFREQ_IOWAIT;
307 
308 	/* Reset boost if the CPU appears to have been idle enough */
309 	if (sg_cpu->iowait_boost &&
310 	    sugov_iowait_reset(sg_cpu, time, set_iowait_boost))
311 		return;
312 
313 	/* Boost only tasks waking up after IO */
314 	if (!set_iowait_boost)
315 		return;
316 
317 	/* Ensure boost doubles only one time at each request */
318 	if (sg_cpu->iowait_boost_pending)
319 		return;
320 	sg_cpu->iowait_boost_pending = true;
321 
322 	/* Double the boost at each request */
323 	if (sg_cpu->iowait_boost) {
324 		sg_cpu->iowait_boost =
325 			min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE);
326 		return;
327 	}
328 
329 	/* First wakeup after IO: start with minimum boost */
330 	sg_cpu->iowait_boost = IOWAIT_BOOST_MIN;
331 }
332 
333 /**
334  * sugov_iowait_apply() - Apply the IO boost to a CPU.
335  * @sg_cpu: the sugov data for the cpu to boost
336  * @time: the update time from the caller
337  * @max_cap: the max CPU capacity
338  *
339  * A CPU running a task which woken up after an IO operation can have its
340  * utilization boosted to speed up the completion of those IO operations.
341  * The IO boost value is increased each time a task wakes up from IO, in
342  * sugov_iowait_boost(), and it's instead decreased by this function,
343  * each time an increase has not been requested (!iowait_boost_pending).
344  *
345  * A CPU which also appears to have been idle for at least one tick has also
346  * its IO boost utilization reset.
347  *
348  * This mechanism is designed to boost high frequently IO waiting tasks, while
349  * being more conservative on tasks which does sporadic IO operations.
350  */
351 static unsigned long sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time,
352 			       unsigned long max_cap)
353 {
354 	/* No boost currently required */
355 	if (!sg_cpu->iowait_boost)
356 		return 0;
357 
358 	/* Reset boost if the CPU appears to have been idle enough */
359 	if (sugov_iowait_reset(sg_cpu, time, false))
360 		return 0;
361 
362 	if (!sg_cpu->iowait_boost_pending) {
363 		/*
364 		 * No boost pending; reduce the boost value.
365 		 */
366 		sg_cpu->iowait_boost >>= 1;
367 		if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) {
368 			sg_cpu->iowait_boost = 0;
369 			return 0;
370 		}
371 	}
372 
373 	sg_cpu->iowait_boost_pending = false;
374 
375 	/*
376 	 * sg_cpu->util is already in capacity scale; convert iowait_boost
377 	 * into the same scale so we can compare.
378 	 */
379 	return (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT;
380 }
381 
382 #ifdef CONFIG_NO_HZ_COMMON
383 static bool sugov_hold_freq(struct sugov_cpu *sg_cpu)
384 {
385 	unsigned long idle_calls;
386 	bool ret;
387 
388 	/*
389 	 * The heuristics in this function is for the fair class. For SCX, the
390 	 * performance target comes directly from the BPF scheduler. Let's just
391 	 * follow it.
392 	 */
393 	if (scx_switched_all())
394 		return false;
395 
396 	/* if capped by uclamp_max, always update to be in compliance */
397 	if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)))
398 		return false;
399 
400 	/*
401 	 * Maintain the frequency if the CPU has not been idle recently, as
402 	 * reduction is likely to be premature.
403 	 */
404 	idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);
405 	ret = idle_calls == sg_cpu->saved_idle_calls;
406 
407 	sg_cpu->saved_idle_calls = idle_calls;
408 	return ret;
409 }
410 #else /* !CONFIG_NO_HZ_COMMON: */
411 static inline bool sugov_hold_freq(struct sugov_cpu *sg_cpu) { return false; }
412 #endif /* !CONFIG_NO_HZ_COMMON */
413 
414 /*
415  * Make sugov_should_update_freq() ignore the rate limit when DL
416  * has increased the utilization.
417  */
418 static inline void ignore_dl_rate_limit(struct sugov_cpu *sg_cpu)
419 {
420 	if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_min)
421 		sg_cpu->sg_policy->need_freq_update = true;
422 }
423 
424 static inline bool sugov_update_single_common(struct sugov_cpu *sg_cpu,
425 					      u64 time, unsigned long max_cap,
426 					      unsigned int flags)
427 {
428 	unsigned long boost;
429 
430 	sugov_iowait_boost(sg_cpu, time, flags);
431 	sg_cpu->last_update = time;
432 
433 	ignore_dl_rate_limit(sg_cpu);
434 
435 	if (!sugov_should_update_freq(sg_cpu->sg_policy, time))
436 		return false;
437 
438 	boost = sugov_iowait_apply(sg_cpu, time, max_cap);
439 	sugov_get_util(sg_cpu, boost);
440 
441 	return true;
442 }
443 
444 static void sugov_update_single_freq(struct update_util_data *hook, u64 time,
445 				     unsigned int flags)
446 {
447 	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
448 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
449 	unsigned int cached_freq = sg_policy->cached_raw_freq;
450 	unsigned long max_cap;
451 	unsigned int next_f;
452 
453 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);
454 
455 	if (!sugov_update_single_common(sg_cpu, time, max_cap, flags))
456 		return;
457 
458 	next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap);
459 
460 	if (sugov_hold_freq(sg_cpu) && next_f < sg_policy->next_freq &&
461 	    !sg_policy->need_freq_update) {
462 		next_f = sg_policy->next_freq;
463 
464 		/* Restore cached freq as next_freq has changed */
465 		sg_policy->cached_raw_freq = cached_freq;
466 	}
467 
468 	if (!sugov_update_next_freq(sg_policy, time, next_f))
469 		return;
470 
471 	/*
472 	 * This code runs under rq->lock for the target CPU, so it won't run
473 	 * concurrently on two different CPUs for the same target and it is not
474 	 * necessary to acquire the lock in the fast switch case.
475 	 */
476 	if (sg_policy->policy->fast_switch_enabled) {
477 		cpufreq_driver_fast_switch(sg_policy->policy, next_f);
478 	} else {
479 		raw_spin_lock(&sg_policy->update_lock);
480 		sugov_deferred_update(sg_policy);
481 		raw_spin_unlock(&sg_policy->update_lock);
482 	}
483 }
484 
485 static void sugov_update_single_perf(struct update_util_data *hook, u64 time,
486 				     unsigned int flags)
487 {
488 	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
489 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
490 	unsigned long prev_util = sg_cpu->util;
491 	unsigned long max_cap;
492 
493 	/*
494 	 * Fall back to the "frequency" path if frequency invariance is not
495 	 * supported, because the direct mapping between the utilization and
496 	 * the performance levels depends on the frequency invariance.
497 	 */
498 	if (!arch_scale_freq_invariant()) {
499 		sugov_update_single_freq(hook, time, flags);
500 		return;
501 	}
502 
503 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);
504 
505 	if (!sugov_update_single_common(sg_cpu, time, max_cap, flags))
506 		return;
507 
508 	if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util)
509 		sg_cpu->util = prev_util;
510 
511 	cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min,
512 				   sg_cpu->util, sg_cpu->bw_max, max_cap);
513 
514 	sg_policy->need_freq_update = false;
515 	sg_policy->last_freq_update_time = time;
516 }
517 
518 static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
519 {
520 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
521 	struct cpufreq_policy *policy = sg_policy->policy;
522 	unsigned long util = 0, max_cap;
523 	unsigned int j;
524 
525 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);
526 
527 	for_each_cpu(j, policy->cpus) {
528 		struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j);
529 		unsigned long boost;
530 
531 		boost = sugov_iowait_apply(j_sg_cpu, time, max_cap);
532 		sugov_get_util(j_sg_cpu, boost);
533 
534 		util = max(j_sg_cpu->util, util);
535 	}
536 
537 	return get_next_freq(sg_policy, util, max_cap);
538 }
539 
540 static void
541 sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags)
542 {
543 	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
544 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
545 	unsigned int next_f;
546 
547 	raw_spin_lock(&sg_policy->update_lock);
548 
549 	sugov_iowait_boost(sg_cpu, time, flags);
550 	sg_cpu->last_update = time;
551 
552 	ignore_dl_rate_limit(sg_cpu);
553 
554 	if (sugov_should_update_freq(sg_policy, time)) {
555 		next_f = sugov_next_freq_shared(sg_cpu, time);
556 
557 		if (!sugov_update_next_freq(sg_policy, time, next_f))
558 			goto unlock;
559 
560 		if (sg_policy->policy->fast_switch_enabled)
561 			cpufreq_driver_fast_switch(sg_policy->policy, next_f);
562 		else
563 			sugov_deferred_update(sg_policy);
564 	}
565 unlock:
566 	raw_spin_unlock(&sg_policy->update_lock);
567 }
568 
569 static void sugov_work(struct kthread_work *work)
570 {
571 	struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);
572 	unsigned int freq;
573 	unsigned long flags;
574 
575 	/*
576 	 * Hold sg_policy->update_lock shortly to handle the case where:
577 	 * in case sg_policy->next_freq is read here, and then updated by
578 	 * sugov_deferred_update() just before work_in_progress is set to false
579 	 * here, we may miss queueing the new update.
580 	 *
581 	 * Note: If a work was queued after the update_lock is released,
582 	 * sugov_work() will just be called again by kthread_work code; and the
583 	 * request will be proceed before the sugov thread sleeps.
584 	 */
585 	raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
586 	freq = sg_policy->next_freq;
587 	sg_policy->work_in_progress = false;
588 	raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);
589 
590 	mutex_lock(&sg_policy->work_lock);
591 	__cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);
592 	mutex_unlock(&sg_policy->work_lock);
593 }
594 
595 static void sugov_irq_work(struct irq_work *irq_work)
596 {
597 	struct sugov_policy *sg_policy;
598 
599 	sg_policy = container_of(irq_work, struct sugov_policy, irq_work);
600 
601 	kthread_queue_work(&sg_policy->worker, &sg_policy->work);
602 }
603 
604 /************************** sysfs interface ************************/
605 
606 static struct sugov_tunables *global_tunables;
607 static DEFINE_MUTEX(global_tunables_lock);
608 
609 static inline struct sugov_tunables *to_sugov_tunables(struct gov_attr_set *attr_set)
610 {
611 	return container_of(attr_set, struct sugov_tunables, attr_set);
612 }
613 
614 static ssize_t rate_limit_us_show(struct gov_attr_set *attr_set, char *buf)
615 {
616 	struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
617 
618 	return sysfs_emit(buf, "%u\n", tunables->rate_limit_us);
619 }
620 
621 static ssize_t
622 rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf, size_t count)
623 {
624 	struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
625 	struct sugov_policy *sg_policy;
626 	unsigned int rate_limit_us;
627 
628 	if (kstrtouint(buf, 10, &rate_limit_us))
629 		return -EINVAL;
630 
631 	tunables->rate_limit_us = rate_limit_us;
632 
633 	list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)
634 		sugov_update_rate_limit_us(sg_policy);
635 
636 	return count;
637 }
638 
639 static struct governor_attr rate_limit_us = __ATTR_RW(rate_limit_us);
640 
641 static struct attribute *sugov_attrs[] = {
642 	&rate_limit_us.attr,
643 	NULL
644 };
645 ATTRIBUTE_GROUPS(sugov);
646 
647 static void sugov_tunables_free(struct kobject *kobj)
648 {
649 	struct gov_attr_set *attr_set = to_gov_attr_set(kobj);
650 
651 	kfree(to_sugov_tunables(attr_set));
652 }
653 
654 static const struct kobj_type sugov_tunables_ktype = {
655 	.default_groups = sugov_groups,
656 	.sysfs_ops = &governor_sysfs_ops,
657 	.release = &sugov_tunables_free,
658 };
659 
660 /********************** cpufreq governor interface *********************/
661 
662 static struct cpufreq_governor schedutil_gov;
663 
664 static struct sugov_policy *sugov_policy_alloc(struct cpufreq_policy *policy)
665 {
666 	struct sugov_policy *sg_policy;
667 
668 	sg_policy = kzalloc_obj(*sg_policy);
669 	if (!sg_policy)
670 		return NULL;
671 
672 	sg_policy->policy = policy;
673 	raw_spin_lock_init(&sg_policy->update_lock);
674 	return sg_policy;
675 }
676 
677 static void sugov_policy_free(struct sugov_policy *sg_policy)
678 {
679 	kfree(sg_policy);
680 }
681 
682 static int sugov_kthread_create(struct sugov_policy *sg_policy)
683 {
684 	struct task_struct *thread;
685 	struct sched_attr attr = {
686 		.size		= sizeof(struct sched_attr),
687 		.sched_policy	= SCHED_DEADLINE,
688 		.sched_flags	= SCHED_FLAG_SUGOV,
689 		.sched_nice	= 0,
690 		.sched_priority	= 0,
691 		/*
692 		 * Fake (unused) bandwidth; workaround to "fix"
693 		 * priority inheritance.
694 		 */
695 		.sched_runtime	= NSEC_PER_MSEC,
696 		.sched_deadline = 10 * NSEC_PER_MSEC,
697 		.sched_period	= 10 * NSEC_PER_MSEC,
698 	};
699 	struct cpufreq_policy *policy = sg_policy->policy;
700 	int ret;
701 
702 	/* kthread only required for slow path */
703 	if (policy->fast_switch_enabled)
704 		return 0;
705 
706 	kthread_init_work(&sg_policy->work, sugov_work);
707 	kthread_init_worker(&sg_policy->worker);
708 	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,
709 				"sugov:%d",
710 				cpumask_first(policy->related_cpus));
711 	if (IS_ERR(thread)) {
712 		pr_err("failed to create sugov thread: %pe\n", thread);
713 		return PTR_ERR(thread);
714 	}
715 
716 	ret = sched_setattr_nocheck(thread, &attr);
717 	if (ret) {
718 		kthread_stop(thread);
719 		pr_warn("%s: failed to set SCHED_DEADLINE\n", __func__);
720 		return ret;
721 	}
722 
723 	sg_policy->thread = thread;
724 	if (policy->dvfs_possible_from_any_cpu)
725 		set_cpus_allowed_ptr(thread, policy->related_cpus);
726 	else
727 		kthread_bind_mask(thread, policy->related_cpus);
728 
729 	init_irq_work(&sg_policy->irq_work, sugov_irq_work);
730 	mutex_init(&sg_policy->work_lock);
731 
732 	wake_up_process(thread);
733 
734 	return 0;
735 }
736 
737 static void sugov_kthread_stop(struct sugov_policy *sg_policy)
738 {
739 	/* kthread only required for slow path */
740 	if (sg_policy->policy->fast_switch_enabled)
741 		return;
742 
743 	kthread_flush_worker(&sg_policy->worker);
744 	kthread_stop(sg_policy->thread);
745 	mutex_destroy(&sg_policy->work_lock);
746 }
747 
748 static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy)
749 {
750 	struct sugov_tunables *tunables;
751 
752 	tunables = kzalloc_obj(*tunables);
753 	if (tunables) {
754 		gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);
755 		if (!have_governor_per_policy())
756 			global_tunables = tunables;
757 	}
758 	return tunables;
759 }
760 
761 static void sugov_clear_global_tunables(void)
762 {
763 	if (!have_governor_per_policy())
764 		global_tunables = NULL;
765 }
766 
767 static int sugov_init(struct cpufreq_policy *policy)
768 {
769 	struct sugov_policy *sg_policy;
770 	struct sugov_tunables *tunables;
771 	int ret = 0;
772 
773 	/* State should be equivalent to EXIT */
774 	if (policy->governor_data)
775 		return -EBUSY;
776 
777 	cpufreq_enable_fast_switch(policy);
778 
779 	sg_policy = sugov_policy_alloc(policy);
780 	if (!sg_policy) {
781 		ret = -ENOMEM;
782 		goto disable_fast_switch;
783 	}
784 
785 	ret = sugov_kthread_create(sg_policy);
786 	if (ret)
787 		goto free_sg_policy;
788 
789 	mutex_lock(&global_tunables_lock);
790 
791 	if (global_tunables) {
792 		if (WARN_ON(have_governor_per_policy())) {
793 			ret = -EINVAL;
794 			goto stop_kthread;
795 		}
796 		policy->governor_data = sg_policy;
797 		sg_policy->tunables = global_tunables;
798 
799 		gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);
800 		goto out;
801 	}
802 
803 	tunables = sugov_tunables_alloc(sg_policy);
804 	if (!tunables) {
805 		ret = -ENOMEM;
806 		goto stop_kthread;
807 	}
808 
809 	tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy);
810 
811 	policy->governor_data = sg_policy;
812 	sg_policy->tunables = tunables;
813 
814 	ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,
815 				   get_governor_parent_kobj(policy), "%s",
816 				   schedutil_gov.name);
817 	if (ret)
818 		goto fail;
819 
820 out:
821 	/*
822 	 * Schedutil is the preferred governor for EAS, so rebuild sched domains
823 	 * on governor changes to make sure the scheduler knows about them.
824 	 */
825 	em_rebuild_sched_domains();
826 	mutex_unlock(&global_tunables_lock);
827 	return 0;
828 
829 fail:
830 	kobject_put(&tunables->attr_set.kobj);
831 	policy->governor_data = NULL;
832 	sugov_clear_global_tunables();
833 
834 stop_kthread:
835 	sugov_kthread_stop(sg_policy);
836 	mutex_unlock(&global_tunables_lock);
837 
838 free_sg_policy:
839 	sugov_policy_free(sg_policy);
840 
841 disable_fast_switch:
842 	cpufreq_disable_fast_switch(policy);
843 
844 	pr_err("initialization failed (error %d)\n", ret);
845 	return ret;
846 }
847 
848 static void sugov_exit(struct cpufreq_policy *policy)
849 {
850 	struct sugov_policy *sg_policy = policy->governor_data;
851 	struct sugov_tunables *tunables = sg_policy->tunables;
852 	unsigned int count;
853 
854 	mutex_lock(&global_tunables_lock);
855 
856 	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);
857 	policy->governor_data = NULL;
858 	if (!count)
859 		sugov_clear_global_tunables();
860 
861 	mutex_unlock(&global_tunables_lock);
862 
863 	sugov_kthread_stop(sg_policy);
864 	sugov_policy_free(sg_policy);
865 	cpufreq_disable_fast_switch(policy);
866 
867 	em_rebuild_sched_domains();
868 }
869 
870 static int sugov_start(struct cpufreq_policy *policy)
871 {
872 	struct sugov_policy *sg_policy = policy->governor_data;
873 	void (*uu)(struct update_util_data *data, u64 time, unsigned int flags);
874 	unsigned int cpu;
875 
876 	sugov_update_rate_limit_us_sync(sg_policy);
877 	sg_policy->last_freq_update_time	= 0;
878 	sg_policy->next_freq			= 0;
879 	sg_policy->work_in_progress		= false;
880 	sg_policy->limits_changed		= false;
881 	sg_policy->cached_raw_freq		= 0;
882 
883 	sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);
884 
885 	if (policy_is_shared(policy))
886 		uu = sugov_update_shared;
887 	else if (policy->fast_switch_enabled && cpufreq_driver_has_adjust_perf())
888 		uu = sugov_update_single_perf;
889 	else
890 		uu = sugov_update_single_freq;
891 
892 	for_each_cpu(cpu, policy->cpus) {
893 		struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
894 
895 		memset(sg_cpu, 0, sizeof(*sg_cpu));
896 		sg_cpu->cpu = cpu;
897 		sg_cpu->sg_policy = sg_policy;
898 	}
899 
900 	/*
901 	 * Publish the hooks only after all per-CPU data is initialized, so a
902 	 * shared policy's sugov_update_shared() never reads an uninitialized
903 	 * sibling sugov_cpu.
904 	 */
905 	for_each_cpu(cpu, policy->cpus) {
906 		struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
907 
908 		cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu);
909 	}
910 
911 	return 0;
912 }
913 
914 static void sugov_stop(struct cpufreq_policy *policy)
915 {
916 	struct sugov_policy *sg_policy = policy->governor_data;
917 	unsigned int cpu;
918 
919 	for_each_cpu(cpu, policy->cpus)
920 		cpufreq_remove_update_util_hook(cpu);
921 
922 	synchronize_rcu();
923 
924 	if (!policy->fast_switch_enabled) {
925 		irq_work_sync(&sg_policy->irq_work);
926 		kthread_cancel_work_sync(&sg_policy->work);
927 	}
928 }
929 
930 static void sugov_limits(struct cpufreq_policy *policy)
931 {
932 	struct sugov_policy *sg_policy = policy->governor_data;
933 
934 	if (!policy->fast_switch_enabled) {
935 		mutex_lock(&sg_policy->work_lock);
936 		cpufreq_policy_apply_limits(policy);
937 		mutex_unlock(&sg_policy->work_lock);
938 	}
939 
940 	/*
941 	 * The limits_changed update below must take place before the updates
942 	 * of policy limits in cpufreq_set_policy() or a policy limits update
943 	 * might be missed, so use a memory barrier to ensure it.
944 	 *
945 	 * This pairs with the memory barrier in sugov_should_update_freq().
946 	 */
947 	smp_wmb();
948 
949 	WRITE_ONCE(sg_policy->limits_changed, true);
950 }
951 
952 static struct cpufreq_governor schedutil_gov = {
953 	.name			= "schedutil",
954 	.owner			= THIS_MODULE,
955 	.flags			= CPUFREQ_GOV_DYNAMIC_SWITCHING,
956 	.init			= sugov_init,
957 	.exit			= sugov_exit,
958 	.start			= sugov_start,
959 	.stop			= sugov_stop,
960 	.limits			= sugov_limits,
961 };
962 
963 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL
964 struct cpufreq_governor *cpufreq_default_governor(void)
965 {
966 	return &schedutil_gov;
967 }
968 #endif
969 
970 bool sugov_is_governor(struct cpufreq_policy *policy)
971 {
972 	return policy->governor == &schedutil_gov;
973 }
974 
975 cpufreq_governor_init(schedutil_gov);
976