xref: /linux/drivers/cpufreq/cpufreq.c (revision c5bdb87573726d3a10c58bb3ae09daf017be2297)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/drivers/cpufreq/cpufreq.c
4  *
5  *  Copyright (C) 2001 Russell King
6  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
7  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
8  *
9  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
10  *	Added handling for CPU hotplug
11  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
12  *	Fix handling for CPU hotplug -- affected CPUs
13  */
14 
15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16 
17 #include <linux/cpu.h>
18 #include <linux/cpufreq.h>
19 #include <linux/cpu_cooling.h>
20 #include <linux/delay.h>
21 #include <linux/device.h>
22 #include <linux/init.h>
23 #include <linux/kernel_stat.h>
24 #include <linux/module.h>
25 #include <linux/mutex.h>
26 #include <linux/pm_qos.h>
27 #include <linux/slab.h>
28 #include <linux/string_choices.h>
29 #include <linux/suspend.h>
30 #include <linux/syscore_ops.h>
31 #include <linux/tick.h>
32 #include <linux/units.h>
33 #include <trace/events/power.h>
34 
35 static LIST_HEAD(cpufreq_policy_list);
36 
37 /* Macros to iterate over CPU policies */
38 #define for_each_policy(__policy)	\
39 	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
40 
41 #define for_each_suitable_policy(__policy, __active)			 \
42 	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
43 		if ((__active) == !policy_is_inactive(__policy))
44 
45 #define for_each_active_policy(__policy)		\
46 	for_each_suitable_policy(__policy, true)
47 #define for_each_inactive_policy(__policy)		\
48 	for_each_suitable_policy(__policy, false)
49 
50 /* Iterate over governors */
51 static LIST_HEAD(cpufreq_governor_list);
52 #define for_each_governor(__governor)				\
53 	list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
54 
55 static char default_governor[CPUFREQ_NAME_LEN];
56 
57 /*
58  * The "cpufreq driver" - the arch- or hardware-dependent low
59  * level driver of CPUFreq support, and its spinlock. This lock
60  * also protects the cpufreq_cpu_data array.
61  */
62 static struct cpufreq_driver *cpufreq_driver;
63 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
64 static DEFINE_RWLOCK(cpufreq_driver_lock);
65 
66 static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
67 bool cpufreq_supports_freq_invariance(void)
68 {
69 	return static_branch_likely(&cpufreq_freq_invariance);
70 }
71 
72 /* Flag to suspend/resume CPUFreq governors */
73 static bool cpufreq_suspended;
74 
75 static inline bool has_target(void)
76 {
77 	return cpufreq_driver->target_index || cpufreq_driver->target;
78 }
79 
80 bool has_target_index(void)
81 {
82 	return !!cpufreq_driver->target_index;
83 }
84 
85 /* internal prototypes */
86 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
87 static int cpufreq_init_governor(struct cpufreq_policy *policy);
88 static void cpufreq_exit_governor(struct cpufreq_policy *policy);
89 static void cpufreq_governor_limits(struct cpufreq_policy *policy);
90 static int cpufreq_set_policy(struct cpufreq_policy *policy,
91 			      struct cpufreq_governor *new_gov,
92 			      unsigned int new_pol);
93 static bool cpufreq_boost_supported(void);
94 static int cpufreq_boost_trigger_state(int state);
95 
96 /*
97  * Two notifier lists: the "policy" list is involved in the
98  * validation process for a new CPU frequency policy; the
99  * "transition" list for kernel code that needs to handle
100  * changes to devices when the CPU clock speed changes.
101  * The mutex locks both lists.
102  */
103 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
104 SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
105 
106 static int off __read_mostly;
107 static int cpufreq_disabled(void)
108 {
109 	return off;
110 }
111 void disable_cpufreq(void)
112 {
113 	off = 1;
114 }
115 EXPORT_SYMBOL_GPL(disable_cpufreq);
116 
117 static DEFINE_MUTEX(cpufreq_governor_mutex);
118 
119 bool have_governor_per_policy(void)
120 {
121 	return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
122 }
123 EXPORT_SYMBOL_GPL(have_governor_per_policy);
124 
125 static struct kobject *cpufreq_global_kobject;
126 
127 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
128 {
129 	if (have_governor_per_policy())
130 		return &policy->kobj;
131 	else
132 		return cpufreq_global_kobject;
133 }
134 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
135 
136 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
137 {
138 	u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
139 
140 	if (!io_busy)
141 		idle_time += get_cpu_iowait_time_us(cpu, wall);
142 
143 	return idle_time;
144 }
145 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
146 
147 /*
148  * This is a generic cpufreq init() routine which can be used by cpufreq
149  * drivers of SMP systems. It will do following:
150  * - validate & show freq table passed
151  * - set policies transition latency
152  * - policy->cpus with all possible CPUs
153  */
154 void cpufreq_generic_init(struct cpufreq_policy *policy,
155 		struct cpufreq_frequency_table *table,
156 		unsigned int transition_latency)
157 {
158 	policy->freq_table = table;
159 	policy->cpuinfo.transition_latency = transition_latency;
160 
161 	/*
162 	 * The driver only supports the SMP configuration where all processors
163 	 * share the clock and voltage and clock.
164 	 */
165 	cpumask_setall(policy->cpus);
166 }
167 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
168 
169 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
170 {
171 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
172 
173 	return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
174 }
175 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
176 
177 struct cpufreq_policy *cpufreq_cpu_policy(unsigned int cpu)
178 {
179 	return per_cpu(cpufreq_cpu_data, cpu);
180 }
181 EXPORT_SYMBOL_GPL(cpufreq_cpu_policy);
182 
183 unsigned int cpufreq_generic_get(unsigned int cpu)
184 {
185 	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
186 
187 	if (!policy || IS_ERR(policy->clk)) {
188 		pr_err("%s: No %s associated to cpu: %d\n",
189 		       __func__, policy ? "clk" : "policy", cpu);
190 		return 0;
191 	}
192 
193 	return clk_get_rate(policy->clk) / 1000;
194 }
195 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
196 
197 /**
198  * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
199  * @cpu: CPU to find the policy for.
200  *
201  * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
202  * the kobject reference counter of that policy.  Return a valid policy on
203  * success or NULL on failure.
204  *
205  * The policy returned by this function has to be released with the help of
206  * cpufreq_cpu_put() to balance its kobject reference counter properly.
207  */
208 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
209 {
210 	struct cpufreq_policy *policy = NULL;
211 	unsigned long flags;
212 
213 	if (WARN_ON(cpu >= nr_cpu_ids))
214 		return NULL;
215 
216 	/* get the cpufreq driver */
217 	read_lock_irqsave(&cpufreq_driver_lock, flags);
218 
219 	if (cpufreq_driver) {
220 		/* get the CPU */
221 		policy = cpufreq_cpu_get_raw(cpu);
222 		if (policy)
223 			kobject_get(&policy->kobj);
224 	}
225 
226 	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
227 
228 	return policy;
229 }
230 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
231 
232 /**
233  * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
234  * @policy: cpufreq policy returned by cpufreq_cpu_get().
235  */
236 void cpufreq_cpu_put(struct cpufreq_policy *policy)
237 {
238 	kobject_put(&policy->kobj);
239 }
240 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
241 
242 /*********************************************************************
243  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
244  *********************************************************************/
245 
246 /**
247  * adjust_jiffies - Adjust the system "loops_per_jiffy".
248  * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
249  * @ci: Frequency change information.
250  *
251  * This function alters the system "loops_per_jiffy" for the clock
252  * speed change. Note that loops_per_jiffy cannot be updated on SMP
253  * systems as each CPU might be scaled differently. So, use the arch
254  * per-CPU loops_per_jiffy value wherever possible.
255  */
256 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
257 {
258 #ifndef CONFIG_SMP
259 	static unsigned long l_p_j_ref;
260 	static unsigned int l_p_j_ref_freq;
261 
262 	if (ci->flags & CPUFREQ_CONST_LOOPS)
263 		return;
264 
265 	if (!l_p_j_ref_freq) {
266 		l_p_j_ref = loops_per_jiffy;
267 		l_p_j_ref_freq = ci->old;
268 		pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
269 			 l_p_j_ref, l_p_j_ref_freq);
270 	}
271 	if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
272 		loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
273 								ci->new);
274 		pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
275 			 loops_per_jiffy, ci->new);
276 	}
277 #endif
278 }
279 
280 /**
281  * cpufreq_notify_transition - Notify frequency transition and adjust jiffies.
282  * @policy: cpufreq policy to enable fast frequency switching for.
283  * @freqs: contain details of the frequency update.
284  * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
285  *
286  * This function calls the transition notifiers and adjust_jiffies().
287  *
288  * It is called twice on all CPU frequency changes that have external effects.
289  */
290 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
291 				      struct cpufreq_freqs *freqs,
292 				      unsigned int state)
293 {
294 	int cpu;
295 
296 	BUG_ON(irqs_disabled());
297 
298 	if (cpufreq_disabled())
299 		return;
300 
301 	freqs->policy = policy;
302 	freqs->flags = cpufreq_driver->flags;
303 	pr_debug("notification %u of frequency transition to %u kHz\n",
304 		 state, freqs->new);
305 
306 	switch (state) {
307 	case CPUFREQ_PRECHANGE:
308 		/*
309 		 * Detect if the driver reported a value as "old frequency"
310 		 * which is not equal to what the cpufreq core thinks is
311 		 * "old frequency".
312 		 */
313 		if (policy->cur && policy->cur != freqs->old) {
314 			pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
315 				 freqs->old, policy->cur);
316 			freqs->old = policy->cur;
317 		}
318 
319 		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
320 					 CPUFREQ_PRECHANGE, freqs);
321 
322 		adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
323 		break;
324 
325 	case CPUFREQ_POSTCHANGE:
326 		adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
327 		pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
328 			 cpumask_pr_args(policy->cpus));
329 
330 		for_each_cpu(cpu, policy->cpus)
331 			trace_cpu_frequency(freqs->new, cpu);
332 
333 		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
334 					 CPUFREQ_POSTCHANGE, freqs);
335 
336 		cpufreq_stats_record_transition(policy, freqs->new);
337 		policy->cur = freqs->new;
338 	}
339 }
340 
341 /* Do post notifications when there are chances that transition has failed */
342 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
343 		struct cpufreq_freqs *freqs, int transition_failed)
344 {
345 	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
346 	if (!transition_failed)
347 		return;
348 
349 	swap(freqs->old, freqs->new);
350 	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
351 	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
352 }
353 
354 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
355 		struct cpufreq_freqs *freqs)
356 {
357 
358 	/*
359 	 * Catch double invocations of _begin() which lead to self-deadlock.
360 	 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
361 	 * doesn't invoke _begin() on their behalf, and hence the chances of
362 	 * double invocations are very low. Moreover, there are scenarios
363 	 * where these checks can emit false-positive warnings in these
364 	 * drivers; so we avoid that by skipping them altogether.
365 	 */
366 	WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
367 				&& current == policy->transition_task);
368 
369 wait:
370 	wait_event(policy->transition_wait, !policy->transition_ongoing);
371 
372 	spin_lock(&policy->transition_lock);
373 
374 	if (unlikely(policy->transition_ongoing)) {
375 		spin_unlock(&policy->transition_lock);
376 		goto wait;
377 	}
378 
379 	policy->transition_ongoing = true;
380 	policy->transition_task = current;
381 
382 	spin_unlock(&policy->transition_lock);
383 
384 	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
385 }
386 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
387 
388 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
389 		struct cpufreq_freqs *freqs, int transition_failed)
390 {
391 	if (WARN_ON(!policy->transition_ongoing))
392 		return;
393 
394 	cpufreq_notify_post_transition(policy, freqs, transition_failed);
395 
396 	arch_set_freq_scale(policy->related_cpus,
397 			    policy->cur,
398 			    arch_scale_freq_ref(policy->cpu));
399 
400 	spin_lock(&policy->transition_lock);
401 	policy->transition_ongoing = false;
402 	policy->transition_task = NULL;
403 	spin_unlock(&policy->transition_lock);
404 
405 	wake_up(&policy->transition_wait);
406 }
407 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
408 
409 /*
410  * Fast frequency switching status count.  Positive means "enabled", negative
411  * means "disabled" and 0 means "not decided yet".
412  */
413 static int cpufreq_fast_switch_count;
414 static DEFINE_MUTEX(cpufreq_fast_switch_lock);
415 
416 static void cpufreq_list_transition_notifiers(void)
417 {
418 	struct notifier_block *nb;
419 
420 	pr_info("Registered transition notifiers:\n");
421 
422 	mutex_lock(&cpufreq_transition_notifier_list.mutex);
423 
424 	for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
425 		pr_info("%pS\n", nb->notifier_call);
426 
427 	mutex_unlock(&cpufreq_transition_notifier_list.mutex);
428 }
429 
430 /**
431  * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
432  * @policy: cpufreq policy to enable fast frequency switching for.
433  *
434  * Try to enable fast frequency switching for @policy.
435  *
436  * The attempt will fail if there is at least one transition notifier registered
437  * at this point, as fast frequency switching is quite fundamentally at odds
438  * with transition notifiers.  Thus if successful, it will make registration of
439  * transition notifiers fail going forward.
440  */
441 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
442 {
443 	lockdep_assert_held(&policy->rwsem);
444 
445 	if (!policy->fast_switch_possible)
446 		return;
447 
448 	mutex_lock(&cpufreq_fast_switch_lock);
449 	if (cpufreq_fast_switch_count >= 0) {
450 		cpufreq_fast_switch_count++;
451 		policy->fast_switch_enabled = true;
452 	} else {
453 		pr_warn("CPU%u: Fast frequency switching not enabled\n",
454 			policy->cpu);
455 		cpufreq_list_transition_notifiers();
456 	}
457 	mutex_unlock(&cpufreq_fast_switch_lock);
458 }
459 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
460 
461 /**
462  * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
463  * @policy: cpufreq policy to disable fast frequency switching for.
464  */
465 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
466 {
467 	mutex_lock(&cpufreq_fast_switch_lock);
468 	if (policy->fast_switch_enabled) {
469 		policy->fast_switch_enabled = false;
470 		if (!WARN_ON(cpufreq_fast_switch_count <= 0))
471 			cpufreq_fast_switch_count--;
472 	}
473 	mutex_unlock(&cpufreq_fast_switch_lock);
474 }
475 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
476 
477 static unsigned int __resolve_freq(struct cpufreq_policy *policy,
478 				   unsigned int target_freq,
479 				   unsigned int min, unsigned int max,
480 				   unsigned int relation, unsigned int *idx_p)
481 {
482 	unsigned int idx;
483 
484 	target_freq = clamp_val(target_freq, min, max);
485 
486 	if (!policy->freq_table)
487 		return target_freq;
488 
489 	idx = cpufreq_frequency_table_target(policy, target_freq, min, max, relation);
490 	if (idx_p)
491 		*idx_p = idx;
492 
493 	return policy->freq_table[idx].frequency;
494 }
495 
496 /**
497  * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
498  * one.
499  * @policy: associated policy to interrogate
500  * @target_freq: target frequency to resolve.
501  *
502  * The target to driver frequency mapping is cached in the policy.
503  *
504  * Return: Lowest driver-supported frequency greater than or equal to the
505  * given target_freq, subject to policy (min/max) and driver limitations.
506  */
507 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
508 					 unsigned int target_freq)
509 {
510 	unsigned int min = READ_ONCE(policy->min);
511 	unsigned int max = READ_ONCE(policy->max);
512 
513 	/*
514 	 * If this function runs in parallel with cpufreq_set_policy(), it may
515 	 * read policy->min before the update and policy->max after the update
516 	 * or the other way around, so there is no ordering guarantee.
517 	 *
518 	 * Resolve this by always honoring the max (in case it comes from
519 	 * thermal throttling or similar).
520 	 */
521 	if (unlikely(min > max))
522 		min = max;
523 
524 	policy->cached_target_freq = __resolve_freq(policy, target_freq, min, max,
525 						    CPUFREQ_RELATION_LE,
526 						    &policy->cached_resolved_idx);
527 	return policy->cached_target_freq;
528 }
529 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
530 
531 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
532 {
533 	unsigned int latency;
534 
535 	if (policy->transition_delay_us)
536 		return policy->transition_delay_us;
537 
538 	latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
539 	if (latency)
540 		/* Give a 50% breathing room between updates */
541 		return latency + (latency >> 1);
542 
543 	return USEC_PER_MSEC;
544 }
545 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
546 
547 /*********************************************************************
548  *                          SYSFS INTERFACE                          *
549  *********************************************************************/
550 static ssize_t show_boost(struct kobject *kobj,
551 			  struct kobj_attribute *attr, char *buf)
552 {
553 	return sysfs_emit(buf, "%d\n", cpufreq_driver->boost_enabled);
554 }
555 
556 static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
557 			   const char *buf, size_t count)
558 {
559 	bool enable;
560 
561 	if (kstrtobool(buf, &enable))
562 		return -EINVAL;
563 
564 	if (cpufreq_boost_trigger_state(enable)) {
565 		pr_err("%s: Cannot %s BOOST!\n",
566 		       __func__, str_enable_disable(enable));
567 		return -EINVAL;
568 	}
569 
570 	pr_debug("%s: cpufreq BOOST %s\n",
571 		 __func__, str_enabled_disabled(enable));
572 
573 	return count;
574 }
575 define_one_global_rw(boost);
576 
577 static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf)
578 {
579 	return sysfs_emit(buf, "%d\n", policy->boost_enabled);
580 }
581 
582 static int policy_set_boost(struct cpufreq_policy *policy, bool enable)
583 {
584 	int ret;
585 
586 	lockdep_assert_held_write(&policy->rwsem);
587 
588 	if (policy->boost_enabled == enable)
589 		return 0;
590 
591 	policy->boost_enabled = enable;
592 
593 	ret = cpufreq_driver->set_boost(policy, enable);
594 	if (ret) {
595 		policy->boost_enabled = !policy->boost_enabled;
596 		return ret;
597 	}
598 
599 	ret = freq_qos_update_request(&policy->boost_freq_req, policy->cpuinfo.max_freq);
600 	if (ret < 0) {
601 		policy->boost_enabled = !policy->boost_enabled;
602 		cpufreq_driver->set_boost(policy, policy->boost_enabled);
603 		return ret;
604 	}
605 
606 	return 0;
607 }
608 
609 static ssize_t store_local_boost(struct cpufreq_policy *policy,
610 				 const char *buf, size_t count)
611 {
612 	int ret;
613 	bool enable;
614 
615 	if (kstrtobool(buf, &enable))
616 		return -EINVAL;
617 
618 	if (!cpufreq_driver->boost_enabled)
619 		return -EINVAL;
620 
621 	if (!policy->boost_supported)
622 		return -EINVAL;
623 
624 	ret = policy_set_boost(policy, enable);
625 	if (!ret)
626 		return count;
627 
628 	return ret;
629 }
630 
631 static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost);
632 
633 static struct cpufreq_governor *find_governor(const char *str_governor)
634 {
635 	struct cpufreq_governor *t;
636 
637 	for_each_governor(t)
638 		if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
639 			return t;
640 
641 	return NULL;
642 }
643 
644 static struct cpufreq_governor *get_governor(const char *str_governor)
645 {
646 	struct cpufreq_governor *t;
647 
648 	mutex_lock(&cpufreq_governor_mutex);
649 	t = find_governor(str_governor);
650 	if (!t)
651 		goto unlock;
652 
653 	if (!try_module_get(t->owner))
654 		t = NULL;
655 
656 unlock:
657 	mutex_unlock(&cpufreq_governor_mutex);
658 
659 	return t;
660 }
661 
662 static unsigned int cpufreq_parse_policy(char *str_governor)
663 {
664 	if (!strncasecmp(str_governor, "performance", strlen("performance")))
665 		return CPUFREQ_POLICY_PERFORMANCE;
666 
667 	if (!strncasecmp(str_governor, "powersave", strlen("powersave")))
668 		return CPUFREQ_POLICY_POWERSAVE;
669 
670 	return CPUFREQ_POLICY_UNKNOWN;
671 }
672 
673 /**
674  * cpufreq_parse_governor - parse a governor string only for has_target()
675  * @str_governor: Governor name.
676  */
677 static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor)
678 {
679 	struct cpufreq_governor *t;
680 
681 	t = get_governor(str_governor);
682 	if (t)
683 		return t;
684 
685 	if (request_module("cpufreq_%s", str_governor))
686 		return NULL;
687 
688 	return get_governor(str_governor);
689 }
690 
691 /*
692  * cpufreq_per_cpu_attr_read() / show_##file_name() -
693  * print out cpufreq information
694  *
695  * Write out information from cpufreq_driver->policy[cpu]; object must be
696  * "unsigned int".
697  */
698 
699 #define show_one(file_name, object)			\
700 static ssize_t show_##file_name				\
701 (struct cpufreq_policy *policy, char *buf)		\
702 {							\
703 	return sysfs_emit(buf, "%u\n", policy->object);	\
704 }
705 
706 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
707 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
708 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
709 show_one(scaling_min_freq, min);
710 show_one(scaling_max_freq, max);
711 
712 __weak int arch_freq_get_on_cpu(int cpu)
713 {
714 	return -EOPNOTSUPP;
715 }
716 
717 static inline bool cpufreq_avg_freq_supported(struct cpufreq_policy *policy)
718 {
719 	return arch_freq_get_on_cpu(policy->cpu) != -EOPNOTSUPP;
720 }
721 
722 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
723 {
724 	ssize_t ret;
725 	int freq;
726 
727 	freq = IS_ENABLED(CONFIG_CPUFREQ_ARCH_CUR_FREQ)
728 		? arch_freq_get_on_cpu(policy->cpu)
729 		: 0;
730 
731 	if (freq > 0)
732 		ret = sysfs_emit(buf, "%u\n", freq);
733 	else if (cpufreq_driver->setpolicy && cpufreq_driver->get)
734 		ret = sysfs_emit(buf, "%u\n", cpufreq_driver->get(policy->cpu));
735 	else
736 		ret = sysfs_emit(buf, "%u\n", policy->cur);
737 	return ret;
738 }
739 
740 /*
741  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
742  */
743 #define store_one(file_name, object)			\
744 static ssize_t store_##file_name					\
745 (struct cpufreq_policy *policy, const char *buf, size_t count)		\
746 {									\
747 	unsigned long val;						\
748 	int ret;							\
749 									\
750 	ret = kstrtoul(buf, 0, &val);					\
751 	if (ret)							\
752 		return ret;						\
753 									\
754 	ret = freq_qos_update_request(&policy->object##_freq_req, val);	\
755 	return ret >= 0 ? count : ret;					\
756 }
757 
758 store_one(scaling_min_freq, min);
759 store_one(scaling_max_freq, max);
760 
761 /*
762  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
763  */
764 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
765 					char *buf)
766 {
767 	unsigned int cur_freq = __cpufreq_get(policy);
768 
769 	if (cur_freq)
770 		return sysfs_emit(buf, "%u\n", cur_freq);
771 
772 	return sysfs_emit(buf, "<unknown>\n");
773 }
774 
775 /*
776  * show_cpuinfo_avg_freq - average CPU frequency as detected by hardware
777  */
778 static ssize_t show_cpuinfo_avg_freq(struct cpufreq_policy *policy,
779 				     char *buf)
780 {
781 	int avg_freq = arch_freq_get_on_cpu(policy->cpu);
782 
783 	if (avg_freq > 0)
784 		return sysfs_emit(buf, "%u\n", avg_freq);
785 	return avg_freq != 0 ? avg_freq : -EINVAL;
786 }
787 
788 /*
789  * show_scaling_governor - show the current policy for the specified CPU
790  */
791 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
792 {
793 	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
794 		return sysfs_emit(buf, "powersave\n");
795 	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
796 		return sysfs_emit(buf, "performance\n");
797 	else if (policy->governor)
798 		return sysfs_emit(buf, "%s\n", policy->governor->name);
799 	return -EINVAL;
800 }
801 
802 /*
803  * store_scaling_governor - store policy for the specified CPU
804  */
805 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
806 					const char *buf, size_t count)
807 {
808 	char str_governor[CPUFREQ_NAME_LEN];
809 	int ret;
810 
811 	ret = sscanf(buf, "%15s", str_governor);
812 	if (ret != 1)
813 		return -EINVAL;
814 
815 	if (cpufreq_driver->setpolicy) {
816 		unsigned int new_pol;
817 
818 		new_pol = cpufreq_parse_policy(str_governor);
819 		if (!new_pol)
820 			return -EINVAL;
821 
822 		ret = cpufreq_set_policy(policy, NULL, new_pol);
823 	} else {
824 		struct cpufreq_governor *new_gov;
825 
826 		new_gov = cpufreq_parse_governor(str_governor);
827 		if (!new_gov)
828 			return -EINVAL;
829 
830 		ret = cpufreq_set_policy(policy, new_gov,
831 					 CPUFREQ_POLICY_UNKNOWN);
832 
833 		module_put(new_gov->owner);
834 	}
835 
836 	return ret ? ret : count;
837 }
838 
839 /*
840  * show_scaling_driver - show the cpufreq driver currently loaded
841  */
842 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
843 {
844 	return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
845 }
846 
847 /*
848  * show_scaling_available_governors - show the available CPUfreq governors
849  */
850 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
851 						char *buf)
852 {
853 	ssize_t i = 0;
854 	struct cpufreq_governor *t;
855 
856 	if (!has_target()) {
857 		i += sysfs_emit(buf, "performance powersave");
858 		goto out;
859 	}
860 
861 	mutex_lock(&cpufreq_governor_mutex);
862 	for_each_governor(t) {
863 		if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
864 		    - (CPUFREQ_NAME_LEN + 2)))
865 			break;
866 		i += sysfs_emit_at(buf, i, "%s ", t->name);
867 	}
868 	mutex_unlock(&cpufreq_governor_mutex);
869 out:
870 	i += sysfs_emit_at(buf, i, "\n");
871 	return i;
872 }
873 
874 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
875 {
876 	ssize_t i = 0;
877 	unsigned int cpu;
878 
879 	for_each_cpu(cpu, mask) {
880 		i += sysfs_emit_at(buf, i, "%u ", cpu);
881 		if (i >= (PAGE_SIZE - 5))
882 			break;
883 	}
884 
885 	/* Remove the extra space at the end */
886 	i--;
887 
888 	i += sysfs_emit_at(buf, i, "\n");
889 	return i;
890 }
891 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
892 
893 /*
894  * show_related_cpus - show the CPUs affected by each transition even if
895  * hw coordination is in use
896  */
897 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
898 {
899 	return cpufreq_show_cpus(policy->related_cpus, buf);
900 }
901 
902 /*
903  * show_affected_cpus - show the CPUs affected by each transition
904  */
905 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
906 {
907 	return cpufreq_show_cpus(policy->cpus, buf);
908 }
909 
910 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
911 					const char *buf, size_t count)
912 {
913 	unsigned int freq = 0;
914 	int ret;
915 
916 	if (!policy->governor || !policy->governor->store_setspeed)
917 		return -EINVAL;
918 
919 	ret = kstrtouint(buf, 0, &freq);
920 	if (ret)
921 		return ret;
922 
923 	policy->governor->store_setspeed(policy, freq);
924 
925 	return count;
926 }
927 
928 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
929 {
930 	if (!policy->governor || !policy->governor->show_setspeed)
931 		return sysfs_emit(buf, "<unsupported>\n");
932 
933 	return policy->governor->show_setspeed(policy, buf);
934 }
935 
936 /*
937  * show_bios_limit - show the current cpufreq HW/BIOS limitation
938  */
939 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
940 {
941 	unsigned int limit;
942 	int ret;
943 	ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
944 	if (!ret)
945 		return sysfs_emit(buf, "%u\n", limit);
946 	return sysfs_emit(buf, "%u\n", policy->cpuinfo.max_freq);
947 }
948 
949 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
950 cpufreq_freq_attr_ro(cpuinfo_avg_freq);
951 cpufreq_freq_attr_ro(cpuinfo_min_freq);
952 cpufreq_freq_attr_ro(cpuinfo_max_freq);
953 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
954 cpufreq_freq_attr_ro(scaling_available_governors);
955 cpufreq_freq_attr_ro(scaling_driver);
956 cpufreq_freq_attr_ro(scaling_cur_freq);
957 cpufreq_freq_attr_ro(bios_limit);
958 cpufreq_freq_attr_ro(related_cpus);
959 cpufreq_freq_attr_ro(affected_cpus);
960 cpufreq_freq_attr_rw(scaling_min_freq);
961 cpufreq_freq_attr_rw(scaling_max_freq);
962 cpufreq_freq_attr_rw(scaling_governor);
963 cpufreq_freq_attr_rw(scaling_setspeed);
964 
965 static struct attribute *cpufreq_attrs[] = {
966 	&cpuinfo_min_freq.attr,
967 	&cpuinfo_max_freq.attr,
968 	&cpuinfo_transition_latency.attr,
969 	&scaling_cur_freq.attr,
970 	&scaling_min_freq.attr,
971 	&scaling_max_freq.attr,
972 	&affected_cpus.attr,
973 	&related_cpus.attr,
974 	&scaling_governor.attr,
975 	&scaling_driver.attr,
976 	&scaling_available_governors.attr,
977 	&scaling_setspeed.attr,
978 	NULL
979 };
980 ATTRIBUTE_GROUPS(cpufreq);
981 
982 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
983 #define to_attr(a) container_of(a, struct freq_attr, attr)
984 
985 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
986 {
987 	struct cpufreq_policy *policy = to_policy(kobj);
988 	struct freq_attr *fattr = to_attr(attr);
989 
990 	if (!fattr->show)
991 		return -EIO;
992 
993 	guard(cpufreq_policy_read)(policy);
994 
995 	if (likely(!policy_is_inactive(policy)))
996 		return fattr->show(policy, buf);
997 
998 	return -EBUSY;
999 }
1000 
1001 static ssize_t store(struct kobject *kobj, struct attribute *attr,
1002 		     const char *buf, size_t count)
1003 {
1004 	struct cpufreq_policy *policy = to_policy(kobj);
1005 	struct freq_attr *fattr = to_attr(attr);
1006 
1007 	if (!fattr->store)
1008 		return -EIO;
1009 
1010 	guard(cpufreq_policy_write)(policy);
1011 
1012 	if (likely(!policy_is_inactive(policy)))
1013 		return fattr->store(policy, buf, count);
1014 
1015 	return -EBUSY;
1016 }
1017 
1018 static void cpufreq_sysfs_release(struct kobject *kobj)
1019 {
1020 	struct cpufreq_policy *policy = to_policy(kobj);
1021 	pr_debug("last reference is dropped\n");
1022 	complete(&policy->kobj_unregister);
1023 }
1024 
1025 static const struct sysfs_ops sysfs_ops = {
1026 	.show	= show,
1027 	.store	= store,
1028 };
1029 
1030 static const struct kobj_type ktype_cpufreq = {
1031 	.sysfs_ops	= &sysfs_ops,
1032 	.default_groups	= cpufreq_groups,
1033 	.release	= cpufreq_sysfs_release,
1034 };
1035 
1036 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu,
1037 				struct device *dev)
1038 {
1039 	if (unlikely(!dev))
1040 		return;
1041 
1042 	if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
1043 		return;
1044 
1045 	dev_dbg(dev, "%s: Adding symlink\n", __func__);
1046 	if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
1047 		dev_err(dev, "cpufreq symlink creation failed\n");
1048 }
1049 
1050 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu,
1051 				   struct device *dev)
1052 {
1053 	dev_dbg(dev, "%s: Removing symlink\n", __func__);
1054 	sysfs_remove_link(&dev->kobj, "cpufreq");
1055 	cpumask_clear_cpu(cpu, policy->real_cpus);
1056 }
1057 
1058 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
1059 {
1060 	struct freq_attr **drv_attr;
1061 	int ret = 0;
1062 
1063 	/* Attributes that need freq_table */
1064 	if (policy->freq_table) {
1065 		ret = sysfs_create_file(&policy->kobj,
1066 				&cpufreq_freq_attr_scaling_available_freqs.attr);
1067 		if (ret)
1068 			return ret;
1069 
1070 		if (cpufreq_boost_supported()) {
1071 			ret = sysfs_create_file(&policy->kobj,
1072 				&cpufreq_freq_attr_scaling_boost_freqs.attr);
1073 			if (ret)
1074 				return ret;
1075 		}
1076 	}
1077 
1078 	/* set up files for this cpu device */
1079 	drv_attr = cpufreq_driver->attr;
1080 	while (drv_attr && *drv_attr) {
1081 		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1082 		if (ret)
1083 			return ret;
1084 		drv_attr++;
1085 	}
1086 	if (cpufreq_driver->get) {
1087 		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1088 		if (ret)
1089 			return ret;
1090 	}
1091 
1092 	if (cpufreq_avg_freq_supported(policy)) {
1093 		ret = sysfs_create_file(&policy->kobj, &cpuinfo_avg_freq.attr);
1094 		if (ret)
1095 			return ret;
1096 	}
1097 
1098 	if (cpufreq_driver->bios_limit) {
1099 		ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1100 		if (ret)
1101 			return ret;
1102 	}
1103 
1104 	if (cpufreq_boost_supported()) {
1105 		ret = sysfs_create_file(&policy->kobj, &local_boost.attr);
1106 		if (ret)
1107 			return ret;
1108 	}
1109 
1110 	return 0;
1111 }
1112 
1113 static int cpufreq_init_policy(struct cpufreq_policy *policy)
1114 {
1115 	struct cpufreq_governor *gov = NULL;
1116 	unsigned int pol = CPUFREQ_POLICY_UNKNOWN;
1117 	int ret;
1118 
1119 	if (has_target()) {
1120 		/* Update policy governor to the one used before hotplug. */
1121 		if (policy->last_governor[0] != '\0')
1122 			gov = get_governor(policy->last_governor);
1123 		if (gov) {
1124 			pr_debug("Restoring governor %s for cpu %d\n",
1125 				 gov->name, policy->cpu);
1126 		} else {
1127 			gov = get_governor(default_governor);
1128 		}
1129 
1130 		if (!gov) {
1131 			gov = cpufreq_default_governor();
1132 			__module_get(gov->owner);
1133 		}
1134 
1135 	} else {
1136 
1137 		/* Use the default policy if there is no last_policy. */
1138 		if (policy->last_policy) {
1139 			pol = policy->last_policy;
1140 		} else {
1141 			pol = cpufreq_parse_policy(default_governor);
1142 			/*
1143 			 * In case the default governor is neither "performance"
1144 			 * nor "powersave", fall back to the initial policy
1145 			 * value set by the driver.
1146 			 */
1147 			if (pol == CPUFREQ_POLICY_UNKNOWN)
1148 				pol = policy->policy;
1149 		}
1150 		if (pol != CPUFREQ_POLICY_PERFORMANCE &&
1151 		    pol != CPUFREQ_POLICY_POWERSAVE)
1152 			return -ENODATA;
1153 	}
1154 
1155 	ret = cpufreq_set_policy(policy, gov, pol);
1156 	if (gov)
1157 		module_put(gov->owner);
1158 
1159 	return ret;
1160 }
1161 
1162 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1163 {
1164 	int ret = 0;
1165 
1166 	/* Has this CPU been taken care of already? */
1167 	if (cpumask_test_cpu(cpu, policy->cpus))
1168 		return 0;
1169 
1170 	guard(cpufreq_policy_write)(policy);
1171 
1172 	if (has_target())
1173 		cpufreq_stop_governor(policy);
1174 
1175 	cpumask_set_cpu(cpu, policy->cpus);
1176 
1177 	if (has_target()) {
1178 		ret = cpufreq_start_governor(policy);
1179 		if (ret)
1180 			pr_err("%s: Failed to start governor\n", __func__);
1181 	}
1182 
1183 	return ret;
1184 }
1185 
1186 void refresh_frequency_limits(struct cpufreq_policy *policy)
1187 {
1188 	if (!policy_is_inactive(policy)) {
1189 		pr_debug("updating policy for CPU %u\n", policy->cpu);
1190 
1191 		cpufreq_set_policy(policy, policy->governor, policy->policy);
1192 	}
1193 }
1194 EXPORT_SYMBOL(refresh_frequency_limits);
1195 
1196 static void handle_update(struct work_struct *work)
1197 {
1198 	struct cpufreq_policy *policy =
1199 		container_of(work, struct cpufreq_policy, update);
1200 
1201 	pr_debug("handle_update for cpu %u called\n", policy->cpu);
1202 
1203 	guard(cpufreq_policy_write)(policy);
1204 
1205 	refresh_frequency_limits(policy);
1206 }
1207 
1208 static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
1209 				void *data)
1210 {
1211 	struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
1212 
1213 	schedule_work(&policy->update);
1214 	return 0;
1215 }
1216 
1217 static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
1218 				void *data)
1219 {
1220 	struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
1221 
1222 	schedule_work(&policy->update);
1223 	return 0;
1224 }
1225 
1226 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1227 {
1228 	struct kobject *kobj;
1229 	struct completion *cmp;
1230 
1231 	scoped_guard(cpufreq_policy_write, policy) {
1232 		cpufreq_stats_free_table(policy);
1233 		kobj = &policy->kobj;
1234 		cmp = &policy->kobj_unregister;
1235 	}
1236 	kobject_put(kobj);
1237 
1238 	/*
1239 	 * We need to make sure that the underlying kobj is
1240 	 * actually not referenced anymore by anybody before we
1241 	 * proceed with unloading.
1242 	 */
1243 	pr_debug("waiting for dropping of refcount\n");
1244 	wait_for_completion(cmp);
1245 	pr_debug("wait complete\n");
1246 }
1247 
1248 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1249 {
1250 	struct cpufreq_policy *policy;
1251 	struct device *dev = get_cpu_device(cpu);
1252 	int ret;
1253 
1254 	if (!dev)
1255 		return NULL;
1256 
1257 	policy = kzalloc_obj(*policy);
1258 	if (!policy)
1259 		return NULL;
1260 
1261 	if (!zalloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1262 		goto err_free_policy;
1263 
1264 	if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1265 		goto err_free_cpumask;
1266 
1267 	if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1268 		goto err_free_rcpumask;
1269 
1270 	init_rwsem(&policy->rwsem);
1271 
1272 	init_completion(&policy->kobj_unregister);
1273 	ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1274 				   cpufreq_global_kobject, "policy%u", cpu);
1275 	if (ret) {
1276 		dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
1277 		/*
1278 		 * The entire policy object will be freed below, but the extra
1279 		 * memory allocated for the kobject name needs to be freed by
1280 		 * releasing the kobject.
1281 		 */
1282 		kobject_put(&policy->kobj);
1283 		goto err_free_real_cpus;
1284 	}
1285 
1286 	freq_constraints_init(&policy->constraints);
1287 
1288 	policy->nb_min.notifier_call = cpufreq_notifier_min;
1289 	policy->nb_max.notifier_call = cpufreq_notifier_max;
1290 
1291 	ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,
1292 				    &policy->nb_min);
1293 	if (ret) {
1294 		dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n",
1295 			ret, cpu);
1296 		goto err_kobj_remove;
1297 	}
1298 
1299 	ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,
1300 				    &policy->nb_max);
1301 	if (ret) {
1302 		dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n",
1303 			ret, cpu);
1304 		goto err_min_qos_notifier;
1305 	}
1306 
1307 	INIT_LIST_HEAD(&policy->policy_list);
1308 	spin_lock_init(&policy->transition_lock);
1309 	init_waitqueue_head(&policy->transition_wait);
1310 	INIT_WORK(&policy->update, handle_update);
1311 
1312 	return policy;
1313 
1314 err_min_qos_notifier:
1315 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1316 				 &policy->nb_min);
1317 err_kobj_remove:
1318 	cpufreq_policy_put_kobj(policy);
1319 err_free_real_cpus:
1320 	free_cpumask_var(policy->real_cpus);
1321 err_free_rcpumask:
1322 	free_cpumask_var(policy->related_cpus);
1323 err_free_cpumask:
1324 	free_cpumask_var(policy->cpus);
1325 err_free_policy:
1326 	kfree(policy);
1327 
1328 	return NULL;
1329 }
1330 
1331 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1332 {
1333 	unsigned long flags;
1334 	int cpu;
1335 
1336 	/*
1337 	 * The callers must ensure the policy is inactive by now, to avoid any
1338 	 * races with show()/store() callbacks.
1339 	 */
1340 	if (unlikely(!policy_is_inactive(policy)))
1341 		pr_warn("%s: Freeing active policy\n", __func__);
1342 
1343 	/* Remove policy from list */
1344 	write_lock_irqsave(&cpufreq_driver_lock, flags);
1345 	list_del(&policy->policy_list);
1346 
1347 	for_each_cpu(cpu, policy->related_cpus)
1348 		per_cpu(cpufreq_cpu_data, cpu) = NULL;
1349 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1350 
1351 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,
1352 				 &policy->nb_max);
1353 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1354 				 &policy->nb_min);
1355 
1356 	/* Cancel any pending policy->update work before freeing the policy. */
1357 	cancel_work_sync(&policy->update);
1358 
1359 	if (freq_qos_request_active(&policy->max_freq_req)) {
1360 		/*
1361 		 * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY
1362 		 * notification, since CPUFREQ_CREATE_POLICY notification was
1363 		 * sent after adding max_freq_req earlier.
1364 		 */
1365 		blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1366 					     CPUFREQ_REMOVE_POLICY, policy);
1367 		freq_qos_remove_request(&policy->max_freq_req);
1368 	}
1369 
1370 	if (freq_qos_request_active(&policy->min_freq_req))
1371 		freq_qos_remove_request(&policy->min_freq_req);
1372 	if (freq_qos_request_active(&policy->boost_freq_req))
1373 		freq_qos_remove_request(&policy->boost_freq_req);
1374 
1375 	cpufreq_policy_put_kobj(policy);
1376 	free_cpumask_var(policy->real_cpus);
1377 	free_cpumask_var(policy->related_cpus);
1378 	free_cpumask_var(policy->cpus);
1379 	kfree(policy);
1380 }
1381 
1382 static int cpufreq_policy_init_qos(struct cpufreq_policy *policy)
1383 {
1384 	unsigned int min_freq, max_freq;
1385 	int ret;
1386 
1387 	/* Use policy->min/max set by the driver as QoS requests. */
1388 	min_freq = max(FREQ_QOS_MIN_DEFAULT_VALUE, policy->min);
1389 	if (policy->max)
1390 		max_freq = min(FREQ_QOS_MAX_DEFAULT_VALUE, policy->max);
1391 	else
1392 		max_freq = FREQ_QOS_MAX_DEFAULT_VALUE;
1393 
1394 	if (policy->boost_supported) {
1395 		ret = freq_qos_add_request(&policy->constraints,
1396 						&policy->boost_freq_req,
1397 						FREQ_QOS_MAX,
1398 						policy->cpuinfo.max_freq);
1399 		if (ret < 0)
1400 			return ret;
1401 	}
1402 
1403 	ret = freq_qos_add_request(&policy->constraints, &policy->min_freq_req,
1404 				   FREQ_QOS_MIN, min_freq);
1405 	if (ret < 0)
1406 		return ret;
1407 
1408 	ret = freq_qos_add_request(&policy->constraints, &policy->max_freq_req,
1409 				   FREQ_QOS_MAX, max_freq);
1410 	if (ret < 0)
1411 		return ret;
1412 
1413 	return 0;
1414 }
1415 
1416 static int cpufreq_policy_online(struct cpufreq_policy *policy,
1417 				 unsigned int cpu, bool new_policy)
1418 {
1419 	unsigned long flags;
1420 	unsigned int j;
1421 	int ret;
1422 
1423 	guard(cpufreq_policy_write)(policy);
1424 
1425 	policy->cpu = cpu;
1426 	policy->governor = NULL;
1427 
1428 	if (!new_policy && cpufreq_driver->online) {
1429 		/* Recover policy->cpus using related_cpus */
1430 		cpumask_copy(policy->cpus, policy->related_cpus);
1431 
1432 		ret = cpufreq_driver->online(policy);
1433 		if (ret) {
1434 			pr_debug("%s: %d: initialization failed\n", __func__,
1435 				 __LINE__);
1436 			goto out_exit_policy;
1437 		}
1438 	} else {
1439 		cpumask_copy(policy->cpus, cpumask_of(cpu));
1440 
1441 		/*
1442 		 * Call driver. From then on the cpufreq must be able
1443 		 * to accept all calls to ->verify and ->setpolicy for this CPU.
1444 		 */
1445 		ret = cpufreq_driver->init(policy);
1446 		if (ret) {
1447 			pr_debug("%s: %d: initialization failed\n", __func__,
1448 				 __LINE__);
1449 			goto out_clear_policy;
1450 		}
1451 
1452 		/*
1453 		 * The initialization has succeeded and the policy is online.
1454 		 * If there is a problem with its frequency table, take it
1455 		 * offline and drop it.
1456 		 */
1457 		ret = cpufreq_table_validate_and_sort(policy);
1458 		if (ret)
1459 			goto out_offline_policy;
1460 
1461 		if (new_policy) {
1462 			ret = cpufreq_policy_init_qos(policy);
1463 			if (ret < 0)
1464 				goto out_offline_policy;
1465 		}
1466 
1467 		/*
1468 		 * If the driver hasn't set policy->min/max, set them as they
1469 		 * are used for clamping frequency requests.
1470 		 */
1471 		policy->min = policy->min ? policy->min : policy->cpuinfo.min_freq;
1472 		policy->max = policy->max ? policy->max : policy->cpuinfo.max_freq;
1473 
1474 		/* related_cpus should at least include policy->cpus. */
1475 		cpumask_copy(policy->related_cpus, policy->cpus);
1476 	}
1477 
1478 	/*
1479 	 * affected cpus must always be the one, which are online. We aren't
1480 	 * managing offline cpus here.
1481 	 */
1482 	cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1483 
1484 	if (new_policy) {
1485 		for_each_cpu(j, policy->related_cpus) {
1486 			per_cpu(cpufreq_cpu_data, j) = policy;
1487 			add_cpu_dev_symlink(policy, j, get_cpu_device(j));
1488 		}
1489 
1490 		blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1491 				CPUFREQ_CREATE_POLICY, policy);
1492 	}
1493 
1494 	if (cpufreq_driver->get && has_target()) {
1495 		policy->cur = cpufreq_driver->get(policy->cpu);
1496 		if (!policy->cur) {
1497 			ret = -EIO;
1498 			pr_err("%s: ->get() failed\n", __func__);
1499 			goto out_destroy_policy;
1500 		}
1501 	}
1502 
1503 	/*
1504 	 * Sometimes boot loaders set CPU frequency to a value outside of
1505 	 * frequency table present with cpufreq core. In such cases CPU might be
1506 	 * unstable if it has to run on that frequency for long duration of time
1507 	 * and so its better to set it to a frequency which is specified in
1508 	 * freq-table. This also makes cpufreq stats inconsistent as
1509 	 * cpufreq-stats would fail to register because current frequency of CPU
1510 	 * isn't found in freq-table.
1511 	 *
1512 	 * Because we don't want this change to effect boot process badly, we go
1513 	 * for the next freq which is >= policy->cur ('cur' must be set by now,
1514 	 * otherwise we will end up setting freq to lowest of the table as 'cur'
1515 	 * is initialized to zero).
1516 	 *
1517 	 * We are passing target-freq as "policy->cur - 1" otherwise
1518 	 * __cpufreq_driver_target() would simply fail, as policy->cur will be
1519 	 * equal to target-freq.
1520 	 */
1521 	if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1522 	    && has_target()) {
1523 		unsigned int old_freq = policy->cur;
1524 
1525 		/* Are we running at unknown frequency ? */
1526 		ret = cpufreq_frequency_table_get_index(policy, old_freq);
1527 		if (ret == -EINVAL) {
1528 			ret = __cpufreq_driver_target(policy, old_freq - 1,
1529 						      CPUFREQ_RELATION_L);
1530 
1531 			/*
1532 			 * Reaching here after boot in a few seconds may not
1533 			 * mean that system will remain stable at "unknown"
1534 			 * frequency for longer duration. Hence, a BUG_ON().
1535 			 */
1536 			BUG_ON(ret);
1537 			pr_info("%s: CPU%d: Running at unlisted initial frequency: %u kHz, changing to: %u kHz\n",
1538 				__func__, policy->cpu, old_freq, policy->cur);
1539 		}
1540 	}
1541 
1542 	if (new_policy) {
1543 		ret = cpufreq_add_dev_interface(policy);
1544 		if (ret)
1545 			goto out_destroy_policy;
1546 
1547 		cpufreq_stats_create_table(policy);
1548 
1549 		write_lock_irqsave(&cpufreq_driver_lock, flags);
1550 		list_add(&policy->policy_list, &cpufreq_policy_list);
1551 		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1552 
1553 		/*
1554 		 * Register with the energy model before
1555 		 * em_rebuild_sched_domains() is called, which will result
1556 		 * in rebuilding of the sched domains, which should only be done
1557 		 * once the energy model is properly initialized for the policy
1558 		 * first.
1559 		 *
1560 		 * Also, this should be called before the policy is registered
1561 		 * with cooling framework.
1562 		 */
1563 		if (cpufreq_driver->register_em)
1564 			cpufreq_driver->register_em(policy);
1565 	}
1566 
1567 	ret = cpufreq_init_policy(policy);
1568 	if (ret) {
1569 		pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1570 		       __func__, cpu, ret);
1571 		goto out_destroy_policy;
1572 	}
1573 
1574 	return 0;
1575 
1576 out_destroy_policy:
1577 	for_each_cpu(j, policy->real_cpus)
1578 		remove_cpu_dev_symlink(policy, j, get_cpu_device(j));
1579 
1580 out_offline_policy:
1581 	if (cpufreq_driver->offline)
1582 		cpufreq_driver->offline(policy);
1583 
1584 out_exit_policy:
1585 	if (cpufreq_driver->exit)
1586 		cpufreq_driver->exit(policy);
1587 
1588 out_clear_policy:
1589 	cpumask_clear(policy->cpus);
1590 
1591 	return ret;
1592 }
1593 
1594 static int cpufreq_online(unsigned int cpu)
1595 {
1596 	struct cpufreq_policy *policy;
1597 	bool new_policy;
1598 	int ret;
1599 
1600 	pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1601 
1602 	/* Check if this CPU already has a policy to manage it */
1603 	policy = per_cpu(cpufreq_cpu_data, cpu);
1604 	if (policy) {
1605 		WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1606 		if (!policy_is_inactive(policy))
1607 			return cpufreq_add_policy_cpu(policy, cpu);
1608 
1609 		/* This is the only online CPU for the policy.  Start over. */
1610 		new_policy = false;
1611 	} else {
1612 		new_policy = true;
1613 		policy = cpufreq_policy_alloc(cpu);
1614 		if (!policy)
1615 			return -ENOMEM;
1616 	}
1617 
1618 	ret = cpufreq_policy_online(policy, cpu, new_policy);
1619 	if (ret) {
1620 		cpufreq_policy_free(policy);
1621 		return ret;
1622 	}
1623 
1624 	kobject_uevent(&policy->kobj, KOBJ_ADD);
1625 
1626 	/* Callback for handling stuff after policy is ready */
1627 	if (cpufreq_driver->ready)
1628 		cpufreq_driver->ready(policy);
1629 
1630 	/* Register cpufreq cooling only for a new policy */
1631 	if (new_policy && cpufreq_thermal_control_enabled(cpufreq_driver))
1632 		policy->cdev = of_cpufreq_cooling_register(policy);
1633 
1634 	/*
1635 	 * Let the per-policy boost flag mirror the cpufreq_driver boost during
1636 	 * initialization for a new policy. For an existing policy, maintain the
1637 	 * previous boost value unless global boost is disabled.
1638 	 */
1639 	if (cpufreq_driver->set_boost && policy->boost_supported &&
1640 	    (new_policy || !cpufreq_boost_enabled())) {
1641 		guard(cpufreq_policy_write)(policy);
1642 
1643 		ret = policy_set_boost(policy, cpufreq_boost_enabled());
1644 		if (ret) {
1645 			/* If the set_boost fails, the online operation is not affected */
1646 			pr_info("%s: CPU%d: Cannot %s BOOST\n", __func__, policy->cpu,
1647 				str_enable_disable(cpufreq_boost_enabled()));
1648 		}
1649 	}
1650 
1651 	pr_debug("initialization complete\n");
1652 
1653 	return 0;
1654 }
1655 
1656 /**
1657  * cpufreq_add_dev - the cpufreq interface for a CPU device.
1658  * @dev: CPU device.
1659  * @sif: Subsystem interface structure pointer (not used)
1660  */
1661 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1662 {
1663 	struct cpufreq_policy *policy;
1664 	unsigned cpu = dev->id;
1665 	int ret;
1666 
1667 	dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1668 
1669 	if (cpu_online(cpu)) {
1670 		ret = cpufreq_online(cpu);
1671 		if (ret)
1672 			return ret;
1673 	}
1674 
1675 	/* Create sysfs link on CPU registration */
1676 	policy = per_cpu(cpufreq_cpu_data, cpu);
1677 	if (policy)
1678 		add_cpu_dev_symlink(policy, cpu, dev);
1679 
1680 	return 0;
1681 }
1682 
1683 static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy)
1684 {
1685 	int ret;
1686 
1687 	if (has_target())
1688 		cpufreq_stop_governor(policy);
1689 
1690 	cpumask_clear_cpu(cpu, policy->cpus);
1691 
1692 	if (!policy_is_inactive(policy)) {
1693 		/* Nominate a new CPU if necessary. */
1694 		if (cpu == policy->cpu)
1695 			policy->cpu = cpumask_any(policy->cpus);
1696 
1697 		/* Start the governor again for the active policy. */
1698 		if (has_target()) {
1699 			ret = cpufreq_start_governor(policy);
1700 			if (ret)
1701 				pr_err("%s: Failed to start governor\n", __func__);
1702 		}
1703 
1704 		return;
1705 	}
1706 
1707 	if (has_target()) {
1708 		strscpy(policy->last_governor, policy->governor->name,
1709 			CPUFREQ_NAME_LEN);
1710 		cpufreq_exit_governor(policy);
1711 	} else {
1712 		policy->last_policy = policy->policy;
1713 	}
1714 
1715 	/*
1716 	 * Perform the ->offline() during light-weight tear-down, as
1717 	 * that allows fast recovery when the CPU comes back.
1718 	 */
1719 	if (cpufreq_driver->offline) {
1720 		cpufreq_driver->offline(policy);
1721 		return;
1722 	}
1723 
1724 	if (cpufreq_driver->exit)
1725 		cpufreq_driver->exit(policy);
1726 
1727 	policy->freq_table = NULL;
1728 }
1729 
1730 static int cpufreq_offline(unsigned int cpu)
1731 {
1732 	struct cpufreq_policy *policy;
1733 
1734 	pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1735 
1736 	policy = cpufreq_cpu_get_raw(cpu);
1737 	if (!policy) {
1738 		pr_debug("%s: No cpu_data found\n", __func__);
1739 		return 0;
1740 	}
1741 
1742 	guard(cpufreq_policy_write)(policy);
1743 
1744 	__cpufreq_offline(cpu, policy);
1745 
1746 	return 0;
1747 }
1748 
1749 /*
1750  * cpufreq_remove_dev - remove a CPU device
1751  *
1752  * Removes the cpufreq interface for a CPU device.
1753  */
1754 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1755 {
1756 	unsigned int cpu = dev->id;
1757 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1758 
1759 	if (!policy)
1760 		return;
1761 
1762 	scoped_guard(cpufreq_policy_write, policy) {
1763 		if (cpu_online(cpu))
1764 			__cpufreq_offline(cpu, policy);
1765 
1766 		remove_cpu_dev_symlink(policy, cpu, dev);
1767 
1768 		if (!cpumask_empty(policy->real_cpus))
1769 			return;
1770 
1771 		/*
1772 		 * Unregister cpufreq cooling once all the CPUs of the policy
1773 		 * are removed.
1774 		 */
1775 		if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
1776 			cpufreq_cooling_unregister(policy->cdev);
1777 			policy->cdev = NULL;
1778 		}
1779 
1780 		/* We did light-weight exit earlier, do full tear down now */
1781 		if (cpufreq_driver->offline && cpufreq_driver->exit)
1782 			cpufreq_driver->exit(policy);
1783 	}
1784 
1785 	cpufreq_policy_free(policy);
1786 }
1787 
1788 /**
1789  * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference.
1790  * @policy: Policy managing CPUs.
1791  * @new_freq: New CPU frequency.
1792  *
1793  * Adjust to the current frequency first and clean up later by either calling
1794  * cpufreq_update_policy(), or scheduling handle_update().
1795  */
1796 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1797 				unsigned int new_freq)
1798 {
1799 	struct cpufreq_freqs freqs;
1800 
1801 	pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1802 		 policy->cur, new_freq);
1803 
1804 	freqs.old = policy->cur;
1805 	freqs.new = new_freq;
1806 
1807 	cpufreq_freq_transition_begin(policy, &freqs);
1808 	cpufreq_freq_transition_end(policy, &freqs, 0);
1809 }
1810 
1811 static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
1812 {
1813 	unsigned int new_freq;
1814 
1815 	if (!cpufreq_driver->get)
1816 		return 0;
1817 
1818 	new_freq = cpufreq_driver->get(policy->cpu);
1819 	if (!new_freq)
1820 		return 0;
1821 
1822 	/*
1823 	 * If fast frequency switching is used with the given policy, the check
1824 	 * against policy->cur is pointless, so skip it in that case.
1825 	 */
1826 	if (policy->fast_switch_enabled || !has_target())
1827 		return new_freq;
1828 
1829 	if (policy->cur != new_freq) {
1830 		/*
1831 		 * For some platforms, the frequency returned by hardware may be
1832 		 * slightly different from what is provided in the frequency
1833 		 * table, for example hardware may return 499 MHz instead of 500
1834 		 * MHz. In such cases it is better to avoid getting into
1835 		 * unnecessary frequency updates.
1836 		 */
1837 		if (abs(policy->cur - new_freq) < KHZ_PER_MHZ)
1838 			return policy->cur;
1839 
1840 		cpufreq_out_of_sync(policy, new_freq);
1841 		if (update)
1842 			schedule_work(&policy->update);
1843 	}
1844 
1845 	return new_freq;
1846 }
1847 
1848 /**
1849  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1850  * @cpu: CPU number
1851  *
1852  * This is the last known freq, without actually getting it from the driver.
1853  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1854  */
1855 unsigned int cpufreq_quick_get(unsigned int cpu)
1856 {
1857 	unsigned long flags;
1858 
1859 	read_lock_irqsave(&cpufreq_driver_lock, flags);
1860 
1861 	if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1862 		unsigned int ret_freq = cpufreq_driver->get(cpu);
1863 
1864 		read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1865 
1866 		return ret_freq;
1867 	}
1868 
1869 	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1870 
1871 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
1872 	if (policy)
1873 		return policy->cur;
1874 
1875 	return 0;
1876 }
1877 EXPORT_SYMBOL(cpufreq_quick_get);
1878 
1879 /**
1880  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1881  * @cpu: CPU number
1882  *
1883  * Just return the max possible frequency for a given CPU.
1884  */
1885 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1886 {
1887 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
1888 	if (policy)
1889 		return policy->max;
1890 
1891 	return 0;
1892 }
1893 EXPORT_SYMBOL(cpufreq_quick_get_max);
1894 
1895 /**
1896  * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
1897  * @cpu: CPU number
1898  *
1899  * The default return value is the max_freq field of cpuinfo.
1900  */
1901 __weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
1902 {
1903 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
1904 	if (policy)
1905 		return policy->cpuinfo.max_freq;
1906 
1907 	return 0;
1908 }
1909 EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
1910 
1911 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1912 {
1913 	if (unlikely(policy_is_inactive(policy)))
1914 		return 0;
1915 
1916 	return cpufreq_verify_current_freq(policy, true);
1917 }
1918 
1919 /**
1920  * cpufreq_get - get the current CPU frequency (in kHz)
1921  * @cpu: CPU number
1922  *
1923  * Get the CPU current (static) CPU frequency
1924  */
1925 unsigned int cpufreq_get(unsigned int cpu)
1926 {
1927 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
1928 	if (!policy)
1929 		return 0;
1930 
1931 	guard(cpufreq_policy_read)(policy);
1932 
1933 	return __cpufreq_get(policy);
1934 }
1935 EXPORT_SYMBOL(cpufreq_get);
1936 
1937 static struct subsys_interface cpufreq_interface = {
1938 	.name		= "cpufreq",
1939 	.subsys		= &cpu_subsys,
1940 	.add_dev	= cpufreq_add_dev,
1941 	.remove_dev	= cpufreq_remove_dev,
1942 };
1943 
1944 /*
1945  * In case platform wants some specific frequency to be configured
1946  * during suspend..
1947  */
1948 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1949 {
1950 	int ret;
1951 
1952 	if (!policy->suspend_freq) {
1953 		pr_debug("%s: suspend_freq not defined\n", __func__);
1954 		return 0;
1955 	}
1956 
1957 	pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1958 			policy->suspend_freq);
1959 
1960 	ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1961 			CPUFREQ_RELATION_H);
1962 	if (ret)
1963 		pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1964 				__func__, policy->suspend_freq, ret);
1965 
1966 	return ret;
1967 }
1968 EXPORT_SYMBOL(cpufreq_generic_suspend);
1969 
1970 static void cpufreq_suspend_policy(struct cpufreq_policy *policy)
1971 {
1972 	guard(cpufreq_policy_write)(policy);
1973 
1974 	cpufreq_stop_governor(policy);
1975 	policy->suspended = true;
1976 }
1977 
1978 /**
1979  * cpufreq_suspend() - Suspend CPUFreq governors.
1980  *
1981  * Called during system wide Suspend/Hibernate cycles for suspending governors
1982  * as some platforms can't change frequency after this point in suspend cycle.
1983  * Because some of the devices (like: i2c, regulators, etc) they use for
1984  * changing frequency are suspended quickly after this point.
1985  */
1986 void cpufreq_suspend(void)
1987 {
1988 	struct cpufreq_policy *policy;
1989 
1990 	if (!cpufreq_driver)
1991 		return;
1992 
1993 	cpus_read_lock();
1994 	if (!has_target() && !cpufreq_driver->suspend)
1995 		goto suspend;
1996 
1997 	pr_debug("%s: Suspending Governors\n", __func__);
1998 
1999 	for_each_active_policy(policy) {
2000 		if (has_target())
2001 			cpufreq_suspend_policy(policy);
2002 
2003 		if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
2004 			pr_err("%s: Failed to suspend driver: %s\n", __func__,
2005 				cpufreq_driver->name);
2006 	}
2007 
2008 suspend:
2009 	cpufreq_suspended = true;
2010 	cpus_read_unlock();
2011 }
2012 
2013 static void cpufreq_resume_policy(struct cpufreq_policy *policy, bool start)
2014 {
2015 	guard(cpufreq_policy_write)(policy);
2016 
2017 	if (!policy->suspended)
2018 		return;
2019 
2020 	policy->suspended = false;
2021 
2022 	if (policy_is_inactive(policy)) {
2023 		/*
2024 		 * The policy was active when it was suspended, so its governor
2025 		 * is still there.  Let the governor go.
2026 		 */
2027 		cpufreq_exit_governor(policy);
2028 		return;
2029 	}
2030 
2031 	if (!start)
2032 		return;
2033 
2034 	/*
2035 	 * Force executing the driver .target() callback, if present, in the
2036 	 * next invocation of __cpufreq_driver_target() in case policy->cur has
2037 	 * become stale while suspended, but do it only for drivers without a
2038 	 * .get() callback for which cpufreq_verify_current_freq() is a NOP.
2039 	 */
2040 	if (cpufreq_driver->target && !cpufreq_driver->get)
2041 		policy->force_target = true;
2042 
2043 	if (cpufreq_start_governor(policy))
2044 		pr_err("%s: Failed to start governor for CPU%u's policy\n",
2045 		       __func__, policy->cpu);
2046 }
2047 
2048 /**
2049  * cpufreq_resume() - Resume CPUFreq governors.
2050  *
2051  * Called during system wide Suspend/Hibernate cycle for resuming governors that
2052  * are suspended with cpufreq_suspend().
2053  */
2054 void cpufreq_resume(void)
2055 {
2056 	struct cpufreq_policy *policy;
2057 
2058 	if (!cpufreq_driver)
2059 		return;
2060 
2061 	guard(cpus_read_lock)();
2062 
2063 	if (unlikely(!cpufreq_suspended))
2064 		return;
2065 
2066 	cpufreq_suspended = false;
2067 
2068 	if (!has_target() && !cpufreq_driver->resume)
2069 		return;
2070 
2071 	pr_debug("%s: Resuming Governors\n", __func__);
2072 
2073 	for_each_policy(policy) {
2074 		int ret = 0;
2075 
2076 		if (cpufreq_driver->resume && !policy_is_inactive(policy)) {
2077 			ret = cpufreq_driver->resume(policy);
2078 			if (ret)
2079 				pr_err("%s: Failed to resume driver: %s\n", __func__,
2080 				       cpufreq_driver->name);
2081 		}
2082 
2083 		if (has_target())
2084 			cpufreq_resume_policy(policy, ret == 0);
2085 	}
2086 }
2087 
2088 /**
2089  * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones.
2090  * @flags: Flags to test against the current cpufreq driver's flags.
2091  *
2092  * Assumes that the driver is there, so callers must ensure that this is the
2093  * case.
2094  */
2095 bool cpufreq_driver_test_flags(u16 flags)
2096 {
2097 	return !!(cpufreq_driver->flags & flags);
2098 }
2099 
2100 /**
2101  * cpufreq_get_current_driver - Return the current driver's name.
2102  *
2103  * Return the name string of the currently registered cpufreq driver or NULL if
2104  * none.
2105  */
2106 const char *cpufreq_get_current_driver(void)
2107 {
2108 	if (cpufreq_driver)
2109 		return cpufreq_driver->name;
2110 
2111 	return NULL;
2112 }
2113 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
2114 
2115 /**
2116  * cpufreq_get_driver_data - Return current driver data.
2117  *
2118  * Return the private data of the currently registered cpufreq driver, or NULL
2119  * if no cpufreq driver has been registered.
2120  */
2121 void *cpufreq_get_driver_data(void)
2122 {
2123 	if (cpufreq_driver)
2124 		return cpufreq_driver->driver_data;
2125 
2126 	return NULL;
2127 }
2128 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
2129 
2130 /*********************************************************************
2131  *                     NOTIFIER LISTS INTERFACE                      *
2132  *********************************************************************/
2133 
2134 /**
2135  * cpufreq_register_notifier - Register a notifier with cpufreq.
2136  * @nb: notifier function to register.
2137  * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2138  *
2139  * Add a notifier to one of two lists: either a list of notifiers that run on
2140  * clock rate changes (once before and once after every transition), or a list
2141  * of notifiers that ron on cpufreq policy changes.
2142  *
2143  * This function may sleep and it has the same return values as
2144  * blocking_notifier_chain_register().
2145  */
2146 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
2147 {
2148 	int ret;
2149 
2150 	if (cpufreq_disabled())
2151 		return -EINVAL;
2152 
2153 	switch (list) {
2154 	case CPUFREQ_TRANSITION_NOTIFIER:
2155 		mutex_lock(&cpufreq_fast_switch_lock);
2156 
2157 		if (cpufreq_fast_switch_count > 0) {
2158 			mutex_unlock(&cpufreq_fast_switch_lock);
2159 			return -EBUSY;
2160 		}
2161 		ret = srcu_notifier_chain_register(
2162 				&cpufreq_transition_notifier_list, nb);
2163 		if (!ret)
2164 			cpufreq_fast_switch_count--;
2165 
2166 		mutex_unlock(&cpufreq_fast_switch_lock);
2167 		break;
2168 	case CPUFREQ_POLICY_NOTIFIER:
2169 		ret = blocking_notifier_chain_register(
2170 				&cpufreq_policy_notifier_list, nb);
2171 		break;
2172 	default:
2173 		ret = -EINVAL;
2174 	}
2175 
2176 	return ret;
2177 }
2178 EXPORT_SYMBOL(cpufreq_register_notifier);
2179 
2180 /**
2181  * cpufreq_unregister_notifier - Unregister a notifier from cpufreq.
2182  * @nb: notifier block to be unregistered.
2183  * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2184  *
2185  * Remove a notifier from one of the cpufreq notifier lists.
2186  *
2187  * This function may sleep and it has the same return values as
2188  * blocking_notifier_chain_unregister().
2189  */
2190 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
2191 {
2192 	int ret;
2193 
2194 	if (cpufreq_disabled())
2195 		return -EINVAL;
2196 
2197 	switch (list) {
2198 	case CPUFREQ_TRANSITION_NOTIFIER:
2199 		mutex_lock(&cpufreq_fast_switch_lock);
2200 
2201 		ret = srcu_notifier_chain_unregister(
2202 				&cpufreq_transition_notifier_list, nb);
2203 		if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
2204 			cpufreq_fast_switch_count++;
2205 
2206 		mutex_unlock(&cpufreq_fast_switch_lock);
2207 		break;
2208 	case CPUFREQ_POLICY_NOTIFIER:
2209 		ret = blocking_notifier_chain_unregister(
2210 				&cpufreq_policy_notifier_list, nb);
2211 		break;
2212 	default:
2213 		ret = -EINVAL;
2214 	}
2215 
2216 	return ret;
2217 }
2218 EXPORT_SYMBOL(cpufreq_unregister_notifier);
2219 
2220 
2221 /*********************************************************************
2222  *                              GOVERNORS                            *
2223  *********************************************************************/
2224 
2225 /**
2226  * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
2227  * @policy: cpufreq policy to switch the frequency for.
2228  * @target_freq: New frequency to set (may be approximate).
2229  *
2230  * Carry out a fast frequency switch without sleeping.
2231  *
2232  * The driver's ->fast_switch() callback invoked by this function must be
2233  * suitable for being called from within RCU-sched read-side critical sections
2234  * and it is expected to select the minimum available frequency greater than or
2235  * equal to @target_freq (CPUFREQ_RELATION_L).
2236  *
2237  * This function must not be called if policy->fast_switch_enabled is unset.
2238  *
2239  * Governors calling this function must guarantee that it will never be invoked
2240  * twice in parallel for the same policy and that it will never be called in
2241  * parallel with either ->target() or ->target_index() for the same policy.
2242  *
2243  * Returns the actual frequency set for the CPU.
2244  *
2245  * If 0 is returned by the driver's ->fast_switch() callback to indicate an
2246  * error condition, the hardware configuration must be preserved.
2247  */
2248 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
2249 					unsigned int target_freq)
2250 {
2251 	unsigned int freq;
2252 	int cpu;
2253 
2254 	target_freq = clamp_val(target_freq, policy->min, policy->max);
2255 	freq = cpufreq_driver->fast_switch(policy, target_freq);
2256 
2257 	if (!freq)
2258 		return 0;
2259 
2260 	policy->cur = freq;
2261 	arch_set_freq_scale(policy->related_cpus, freq,
2262 			    arch_scale_freq_ref(policy->cpu));
2263 	cpufreq_stats_record_transition(policy, freq);
2264 
2265 	if (trace_cpu_frequency_enabled()) {
2266 		for_each_cpu(cpu, policy->cpus)
2267 			trace_call__cpu_frequency(freq, cpu);
2268 	}
2269 
2270 	return freq;
2271 }
2272 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
2273 
2274 /**
2275  * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go.
2276  * @policy: cpufreq policy object of the target CPU.
2277  * @min_perf: Minimum (required) performance level (units of @capacity).
2278  * @target_perf: Target (desired) performance level (units of @capacity).
2279  * @max_perf: Maximum (allowed) performance level (units of @capacity).
2280  * @capacity: Capacity of the target CPU.
2281  *
2282  * Carry out a fast performance level adjustment for the CPU represented by
2283  * @policy without sleeping.
2284  *
2285  * The driver's ->adjust_perf() callback invoked by this function must be
2286  * suitable for calling from within RCU-sched read-side critical sections and
2287  * it is expected to program the processor to select suitable performance
2288  * levels between @min_perf and @max_perf inclusive and preferably close to
2289  * @target_perf going forward for the CPU represented by @policy.
2290  *
2291  * This function must not be called if policy->fast_switch_enabled is unset.
2292  *
2293  * Governors calling this function must guarantee that it will never be invoked
2294  * twice in parallel for the same CPU and that it will never be called in
2295  * parallel with either ->target() or ->target_index() or ->fast_switch() for
2296  * the same CPU.
2297  */
2298 void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy,
2299 				 unsigned long min_perf,
2300 				 unsigned long target_perf,
2301 				 unsigned long max_perf,
2302 				 unsigned long capacity)
2303 {
2304 	cpufreq_driver->adjust_perf(policy, min_perf, target_perf, max_perf, capacity);
2305 }
2306 
2307 /**
2308  * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback.
2309  *
2310  * Return 'true' if the ->adjust_perf callback is present for the
2311  * current driver or 'false' otherwise.
2312  */
2313 bool cpufreq_driver_has_adjust_perf(void)
2314 {
2315 	return !!cpufreq_driver->adjust_perf;
2316 }
2317 
2318 /* Must set freqs->new to intermediate frequency */
2319 static int __target_intermediate(struct cpufreq_policy *policy,
2320 				 struct cpufreq_freqs *freqs, int index)
2321 {
2322 	int ret;
2323 
2324 	freqs->new = cpufreq_driver->get_intermediate(policy, index);
2325 
2326 	/* We don't need to switch to intermediate freq */
2327 	if (!freqs->new)
2328 		return 0;
2329 
2330 	pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
2331 		 __func__, policy->cpu, freqs->old, freqs->new);
2332 
2333 	cpufreq_freq_transition_begin(policy, freqs);
2334 	ret = cpufreq_driver->target_intermediate(policy, index);
2335 	cpufreq_freq_transition_end(policy, freqs, ret);
2336 
2337 	if (ret)
2338 		pr_err("%s: Failed to change to intermediate frequency: %d\n",
2339 		       __func__, ret);
2340 
2341 	return ret;
2342 }
2343 
2344 static int __target_index(struct cpufreq_policy *policy, int index)
2345 {
2346 	struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
2347 	unsigned int restore_freq, intermediate_freq = 0;
2348 	unsigned int newfreq = policy->freq_table[index].frequency;
2349 	int retval = -EINVAL;
2350 	bool notify;
2351 
2352 	if (newfreq == policy->cur)
2353 		return 0;
2354 
2355 	/* Save last value to restore later on errors */
2356 	restore_freq = policy->cur;
2357 
2358 	notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
2359 	if (notify) {
2360 		/* Handle switching to intermediate frequency */
2361 		if (cpufreq_driver->get_intermediate) {
2362 			retval = __target_intermediate(policy, &freqs, index);
2363 			if (retval)
2364 				return retval;
2365 
2366 			intermediate_freq = freqs.new;
2367 			/* Set old freq to intermediate */
2368 			if (intermediate_freq)
2369 				freqs.old = freqs.new;
2370 		}
2371 
2372 		freqs.new = newfreq;
2373 		pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
2374 			 __func__, policy->cpu, freqs.old, freqs.new);
2375 
2376 		cpufreq_freq_transition_begin(policy, &freqs);
2377 	}
2378 
2379 	retval = cpufreq_driver->target_index(policy, index);
2380 	if (retval)
2381 		pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
2382 		       retval);
2383 
2384 	if (notify) {
2385 		cpufreq_freq_transition_end(policy, &freqs, retval);
2386 
2387 		/*
2388 		 * Failed after setting to intermediate freq? Driver should have
2389 		 * reverted back to initial frequency and so should we. Check
2390 		 * here for intermediate_freq instead of get_intermediate, in
2391 		 * case we haven't switched to intermediate freq at all.
2392 		 */
2393 		if (unlikely(retval && intermediate_freq)) {
2394 			freqs.old = intermediate_freq;
2395 			freqs.new = restore_freq;
2396 			cpufreq_freq_transition_begin(policy, &freqs);
2397 			cpufreq_freq_transition_end(policy, &freqs, 0);
2398 		}
2399 	}
2400 
2401 	return retval;
2402 }
2403 
2404 int __cpufreq_driver_target(struct cpufreq_policy *policy,
2405 			    unsigned int target_freq,
2406 			    unsigned int relation)
2407 {
2408 	unsigned int old_target_freq = target_freq;
2409 	unsigned int idx = 0;
2410 
2411 	if (cpufreq_disabled())
2412 		return -ENODEV;
2413 
2414 	target_freq = __resolve_freq(policy, target_freq, policy->min,
2415 				     policy->max, relation, &idx);
2416 
2417 	pr_debug("CPU %u: cur %u kHz -> target %u kHz (req %u kHz, rel %u)\n",
2418 		policy->cpu, policy->cur, target_freq, old_target_freq, relation);
2419 
2420 	/*
2421 	 * Unless invoking the target callback is forced, skip the update if the
2422 	 * frequency is not really changing.
2423 	 */
2424 	if (policy->force_target) {
2425 		/* Avoid forcing the driver callback unnecessarily next time. */
2426 		policy->force_target = false;
2427 	} else if (target_freq == policy->cur) {
2428 		return 0;
2429 	}
2430 
2431 	if (cpufreq_driver->target) {
2432 		/*
2433 		 * If the driver hasn't setup a single inefficient frequency,
2434 		 * it's unlikely it knows how to decode CPUFREQ_RELATION_E.
2435 		 */
2436 		if (!policy->efficiencies_available)
2437 			relation &= ~CPUFREQ_RELATION_E;
2438 
2439 		return cpufreq_driver->target(policy, target_freq, relation);
2440 	}
2441 
2442 	if (!cpufreq_driver->target_index)
2443 		return -EINVAL;
2444 
2445 	return __target_index(policy, idx);
2446 }
2447 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2448 
2449 int cpufreq_driver_target(struct cpufreq_policy *policy,
2450 			  unsigned int target_freq,
2451 			  unsigned int relation)
2452 {
2453 	guard(cpufreq_policy_write)(policy);
2454 
2455 	/*
2456 	 * The last CPU in the policy may have gone offline after the caller
2457 	 * obtained it, in which case the policy is inactive and must not be
2458 	 * used for changing the frequency.
2459 	 */
2460 	if (unlikely(policy_is_inactive(policy)))
2461 		return -EBUSY;
2462 
2463 	return __cpufreq_driver_target(policy, target_freq, relation);
2464 }
2465 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2466 
2467 __weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2468 {
2469 	return NULL;
2470 }
2471 
2472 static bool cpufreq_is_suspended(struct cpufreq_policy *policy)
2473 {
2474 	return cpufreq_suspended || policy->suspended;
2475 }
2476 
2477 static int cpufreq_init_governor(struct cpufreq_policy *policy)
2478 {
2479 	int ret;
2480 
2481 	/* Don't start any governor operations for suspended policies. */
2482 	if (cpufreq_is_suspended(policy))
2483 		return 0;
2484 	/*
2485 	 * Governor might not be initiated here if ACPI _PPC changed
2486 	 * notification happened, so check it.
2487 	 */
2488 	if (!policy->governor)
2489 		return -EINVAL;
2490 
2491 	/* Platform doesn't want dynamic frequency switching ? */
2492 	if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING &&
2493 	    cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
2494 		struct cpufreq_governor *gov = cpufreq_fallback_governor();
2495 
2496 		if (gov) {
2497 			pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
2498 				policy->governor->name, gov->name);
2499 			policy->governor = gov;
2500 		} else {
2501 			return -EINVAL;
2502 		}
2503 	}
2504 
2505 	if (!try_module_get(policy->governor->owner))
2506 		return -EINVAL;
2507 
2508 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2509 
2510 	if (policy->governor->init) {
2511 		ret = policy->governor->init(policy);
2512 		if (ret) {
2513 			module_put(policy->governor->owner);
2514 			return ret;
2515 		}
2516 	}
2517 
2518 	policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET);
2519 
2520 	return 0;
2521 }
2522 
2523 static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2524 {
2525 	if (cpufreq_is_suspended(policy) || !policy->governor)
2526 		return;
2527 
2528 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2529 
2530 	if (policy->governor->exit)
2531 		policy->governor->exit(policy);
2532 
2533 	module_put(policy->governor->owner);
2534 }
2535 
2536 int cpufreq_start_governor(struct cpufreq_policy *policy)
2537 {
2538 	int ret;
2539 
2540 	if (cpufreq_is_suspended(policy))
2541 		return 0;
2542 
2543 	if (!policy->governor)
2544 		return -EINVAL;
2545 
2546 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2547 
2548 	cpufreq_verify_current_freq(policy, false);
2549 
2550 	if (policy->governor->start) {
2551 		ret = policy->governor->start(policy);
2552 		if (ret)
2553 			return ret;
2554 	}
2555 
2556 	if (policy->governor->limits)
2557 		policy->governor->limits(policy);
2558 
2559 	return 0;
2560 }
2561 
2562 void cpufreq_stop_governor(struct cpufreq_policy *policy)
2563 {
2564 	if (cpufreq_is_suspended(policy) || !policy->governor)
2565 		return;
2566 
2567 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2568 
2569 	if (policy->governor->stop)
2570 		policy->governor->stop(policy);
2571 }
2572 
2573 static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2574 {
2575 	if (cpufreq_is_suspended(policy) || !policy->governor)
2576 		return;
2577 
2578 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2579 
2580 	if (policy->governor->limits)
2581 		policy->governor->limits(policy);
2582 }
2583 
2584 int cpufreq_register_governor(struct cpufreq_governor *governor)
2585 {
2586 	int err;
2587 
2588 	if (!governor)
2589 		return -EINVAL;
2590 
2591 	if (cpufreq_disabled())
2592 		return -ENODEV;
2593 
2594 	mutex_lock(&cpufreq_governor_mutex);
2595 
2596 	err = -EBUSY;
2597 	if (!find_governor(governor->name)) {
2598 		err = 0;
2599 		list_add(&governor->governor_list, &cpufreq_governor_list);
2600 	}
2601 
2602 	mutex_unlock(&cpufreq_governor_mutex);
2603 	return err;
2604 }
2605 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2606 
2607 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2608 {
2609 	struct cpufreq_policy *policy;
2610 	unsigned long flags;
2611 
2612 	if (!governor)
2613 		return;
2614 
2615 	if (cpufreq_disabled())
2616 		return;
2617 
2618 	/* clear last_governor for all inactive policies */
2619 	read_lock_irqsave(&cpufreq_driver_lock, flags);
2620 	for_each_inactive_policy(policy) {
2621 		if (!strcmp(policy->last_governor, governor->name)) {
2622 			policy->governor = NULL;
2623 			policy->last_governor[0] = '\0';
2624 		}
2625 	}
2626 	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2627 
2628 	mutex_lock(&cpufreq_governor_mutex);
2629 	list_del(&governor->governor_list);
2630 	mutex_unlock(&cpufreq_governor_mutex);
2631 }
2632 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2633 
2634 
2635 /*********************************************************************
2636  *                          POLICY INTERFACE                         *
2637  *********************************************************************/
2638 
2639 DEFINE_PER_CPU(unsigned long, cpufreq_pressure);
2640 
2641 /**
2642  * cpufreq_update_pressure() - Update cpufreq pressure for CPUs
2643  * @policy: cpufreq policy of the CPUs.
2644  *
2645  * Update the value of cpufreq pressure for all @cpus in the policy.
2646  */
2647 static void cpufreq_update_pressure(struct cpufreq_policy *policy)
2648 {
2649 	unsigned long max_capacity, capped_freq, pressure;
2650 	u32 max_freq;
2651 	int cpu;
2652 
2653 	cpu = cpumask_first(policy->related_cpus);
2654 	max_freq = arch_scale_freq_ref(cpu);
2655 	if (!max_freq && cpufreq_driver->scale_freq_ref)
2656 		max_freq = cpufreq_driver->scale_freq_ref(policy);
2657 
2658 	capped_freq = policy->max;
2659 
2660 	/*
2661 	 * Handle properly the boost frequencies, which should simply clean
2662 	 * the cpufreq pressure value.
2663 	 */
2664 	if (max_freq <= capped_freq) {
2665 		pressure = 0;
2666 	} else {
2667 		max_capacity = arch_scale_cpu_capacity(cpu);
2668 		pressure = max_capacity -
2669 			   mult_frac(max_capacity, capped_freq, max_freq);
2670 	}
2671 
2672 	for_each_cpu(cpu, policy->related_cpus)
2673 		WRITE_ONCE(per_cpu(cpufreq_pressure, cpu), pressure);
2674 }
2675 
2676 /**
2677  * cpufreq_set_policy - Modify cpufreq policy parameters.
2678  * @policy: Policy object to modify.
2679  * @new_gov: Policy governor pointer.
2680  * @new_pol: Policy value (for drivers with built-in governors).
2681  *
2682  * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency
2683  * limits to be set for the policy, update @policy with the verified limits
2684  * values and either invoke the driver's ->setpolicy() callback (if present) or
2685  * carry out a governor update for @policy.  That is, run the current governor's
2686  * ->limits() callback (if @new_gov points to the same object as the one in
2687  * @policy) or replace the governor for @policy with @new_gov.
2688  *
2689  * The cpuinfo part of @policy is not updated by this function.
2690  */
2691 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2692 			      struct cpufreq_governor *new_gov,
2693 			      unsigned int new_pol)
2694 {
2695 	bool need_update_limits = cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS;
2696 	struct cpufreq_policy_data new_data;
2697 	struct cpufreq_governor *old_gov;
2698 	unsigned int freq;
2699 	int ret;
2700 
2701 	memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2702 	new_data.freq_table = policy->freq_table;
2703 	new_data.cpu = policy->cpu;
2704 	/*
2705 	 * PM QoS framework collects all the requests from users and provide us
2706 	 * the final aggregated value here.
2707 	 */
2708 	new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);
2709 	new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);
2710 
2711 	pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2712 		 new_data.cpu, new_data.min, new_data.max);
2713 
2714 	/*
2715 	 * Verify that the CPU speed can be set within these limits and make sure
2716 	 * that min <= max.
2717 	 */
2718 	ret = cpufreq_driver->verify(&new_data);
2719 	if (ret)
2720 		return ret;
2721 
2722 	/*
2723 	 * Resolve policy min/max to available frequencies. It ensures
2724 	 * no frequency resolution will neither overshoot the requested maximum
2725 	 * nor undershoot the requested minimum.
2726 	 *
2727 	 * Avoid storing intermediate values in policy->max or policy->min and
2728 	 * compiler optimizations around them because they may be accessed
2729 	 * concurrently by cpufreq_driver_resolve_freq() during the update.
2730 	 */
2731 	freq = __resolve_freq(policy, new_data.max, new_data.min, new_data.max,
2732 			      CPUFREQ_RELATION_H, NULL);
2733 	if (freq != policy->max) {
2734 		WRITE_ONCE(policy->max, freq);
2735 		if (need_update_limits)
2736 			policy->force_target = true;
2737 	}
2738 
2739 	freq = __resolve_freq(policy, new_data.min, new_data.min, new_data.max,
2740 			      CPUFREQ_RELATION_L, NULL);
2741 	freq = min(freq, policy->max);
2742 	if (freq != policy->min) {
2743 		WRITE_ONCE(policy->min, freq);
2744 		if (need_update_limits)
2745 			policy->force_target = true;
2746 	}
2747 
2748 	trace_cpu_frequency_limits(policy);
2749 
2750 	cpufreq_update_pressure(policy);
2751 
2752 	pr_debug("new min and max freqs are %u - %u kHz\n",
2753 		 policy->min, policy->max);
2754 
2755 	if (cpufreq_driver->setpolicy) {
2756 		policy->policy = new_pol;
2757 		pr_debug("setting range\n");
2758 		return cpufreq_driver->setpolicy(policy);
2759 	}
2760 
2761 	if (new_gov == policy->governor) {
2762 		pr_debug("governor limits update\n");
2763 		cpufreq_governor_limits(policy);
2764 		return 0;
2765 	}
2766 
2767 	pr_debug("governor switch\n");
2768 
2769 	/* save old, working values */
2770 	old_gov = policy->governor;
2771 	/* end old governor */
2772 	if (old_gov) {
2773 		cpufreq_stop_governor(policy);
2774 		cpufreq_exit_governor(policy);
2775 	}
2776 
2777 	/*
2778 	 * Initialize cached_target_freq while no governor is running to avoid
2779 	 * racing with cpufreq_driver_resolve_freq(), which updates it.
2780 	 */
2781 	policy->cached_target_freq = UINT_MAX;
2782 
2783 	/* start new governor */
2784 	policy->governor = new_gov;
2785 	ret = cpufreq_init_governor(policy);
2786 	if (!ret) {
2787 		ret = cpufreq_start_governor(policy);
2788 		if (!ret) {
2789 			pr_debug("governor change\n");
2790 			return 0;
2791 		}
2792 		cpufreq_exit_governor(policy);
2793 	}
2794 
2795 	/* new governor failed, so re-start old one */
2796 	pr_debug("starting governor %s failed\n", policy->governor->name);
2797 	if (old_gov) {
2798 		policy->governor = old_gov;
2799 		if (cpufreq_init_governor(policy)) {
2800 			policy->governor = NULL;
2801 		} else if (cpufreq_start_governor(policy)) {
2802 			cpufreq_exit_governor(policy);
2803 			policy->governor = NULL;
2804 		}
2805 	}
2806 
2807 	return ret;
2808 }
2809 
2810 static void cpufreq_policy_refresh(struct cpufreq_policy *policy)
2811 {
2812 	guard(cpufreq_policy_write)(policy);
2813 
2814 	/*
2815 	 * BIOS might change freq behind our back
2816 	 * -> ask driver for current freq and notify governors about a change
2817 	 */
2818 	if (cpufreq_driver->get && has_target() &&
2819 	    (cpufreq_is_suspended(policy) ||
2820 	     WARN_ON(!cpufreq_verify_current_freq(policy, false))))
2821 		return;
2822 
2823 	refresh_frequency_limits(policy);
2824 }
2825 
2826 /**
2827  * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
2828  * @cpu: CPU to re-evaluate the policy for.
2829  *
2830  * Update the current frequency for the cpufreq policy of @cpu and use
2831  * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
2832  * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
2833  * for the policy in question, among other things.
2834  */
2835 void cpufreq_update_policy(unsigned int cpu)
2836 {
2837 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
2838 	if (!policy)
2839 		return;
2840 
2841 	cpufreq_policy_refresh(policy);
2842 }
2843 EXPORT_SYMBOL(cpufreq_update_policy);
2844 
2845 /**
2846  * cpufreq_update_limits - Update policy limits for a given CPU.
2847  * @cpu: CPU to update the policy limits for.
2848  *
2849  * Invoke the driver's ->update_limits callback if present or call
2850  * cpufreq_policy_refresh() for @cpu.
2851  */
2852 void cpufreq_update_limits(unsigned int cpu)
2853 {
2854 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
2855 	if (!policy)
2856 		return;
2857 
2858 	if (cpufreq_driver->update_limits)
2859 		cpufreq_driver->update_limits(policy);
2860 	else
2861 		cpufreq_policy_refresh(policy);
2862 }
2863 EXPORT_SYMBOL_GPL(cpufreq_update_limits);
2864 
2865 /*********************************************************************
2866  *               BOOST						     *
2867  *********************************************************************/
2868 int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
2869 {
2870 	int ret;
2871 
2872 	if (!policy->freq_table)
2873 		return -ENXIO;
2874 
2875 	ret = cpufreq_frequency_table_cpuinfo(policy);
2876 	if (ret)
2877 		pr_err("%s: Policy frequency update failed\n", __func__);
2878 
2879 	return ret;
2880 }
2881 EXPORT_SYMBOL_GPL(cpufreq_boost_set_sw);
2882 
2883 static DEFINE_MUTEX(cpufreq_boost_mutex);
2884 
2885 static int cpufreq_boost_trigger_state(int state)
2886 {
2887 	struct cpufreq_policy *policy;
2888 	unsigned long flags;
2889 	bool boost_enabled;
2890 	int ret = -EOPNOTSUPP;
2891 
2892 	guard(mutex)(&cpufreq_boost_mutex);
2893 
2894 	/*
2895 	 * Don't compare 'cpufreq_driver->boost_enabled' with 'state' here to
2896 	 * make sure all policies are in sync with global boost flag.
2897 	 */
2898 
2899 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2900 	boost_enabled = cpufreq_driver->boost_enabled;
2901 	cpufreq_driver->boost_enabled = state;
2902 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2903 
2904 	cpus_read_lock();
2905 	for_each_active_policy(policy) {
2906 		if (!policy->boost_supported)
2907 			continue;
2908 
2909 		guard(cpufreq_policy_write)(policy);
2910 
2911 		ret = policy_set_boost(policy, state);
2912 		if (unlikely(ret))
2913 			break;
2914 	}
2915 
2916 	cpus_read_unlock();
2917 
2918 	if (likely(!ret))
2919 		return 0;
2920 
2921 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2922 	cpufreq_driver->boost_enabled = boost_enabled;
2923 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2924 
2925 	pr_err("%s: Cannot %s BOOST\n",
2926 	       __func__, str_enable_disable(state));
2927 
2928 	return ret;
2929 }
2930 
2931 static bool cpufreq_boost_supported(void)
2932 {
2933 	return cpufreq_driver->set_boost;
2934 }
2935 
2936 static int create_boost_sysfs_file(void)
2937 {
2938 	int ret;
2939 
2940 	ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2941 	if (ret)
2942 		pr_err("%s: cannot register global BOOST sysfs file\n",
2943 		       __func__);
2944 
2945 	return ret;
2946 }
2947 
2948 static void remove_boost_sysfs_file(void)
2949 {
2950 	if (cpufreq_boost_supported())
2951 		sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2952 }
2953 
2954 bool cpufreq_boost_enabled(void)
2955 {
2956 	return cpufreq_driver->boost_enabled;
2957 }
2958 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2959 
2960 /*********************************************************************
2961  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2962  *********************************************************************/
2963 static enum cpuhp_state hp_online;
2964 
2965 static int cpuhp_cpufreq_online(unsigned int cpu)
2966 {
2967 	cpufreq_online(cpu);
2968 
2969 	return 0;
2970 }
2971 
2972 static int cpuhp_cpufreq_offline(unsigned int cpu)
2973 {
2974 	cpufreq_offline(cpu);
2975 
2976 	return 0;
2977 }
2978 
2979 /**
2980  * cpufreq_register_driver - register a CPU Frequency driver
2981  * @driver_data: A struct cpufreq_driver containing the values#
2982  * submitted by the CPU Frequency driver.
2983  *
2984  * Registers a CPU Frequency driver to this core code. This code
2985  * returns zero on success, -EEXIST when another driver got here first
2986  * (and isn't unregistered in the meantime).
2987  *
2988  */
2989 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2990 {
2991 	unsigned long flags;
2992 	int ret;
2993 
2994 	if (cpufreq_disabled())
2995 		return -ENODEV;
2996 
2997 	/*
2998 	 * The cpufreq core depends heavily on the availability of device
2999 	 * structure, make sure they are available before proceeding further.
3000 	 */
3001 	if (!get_cpu_device(0))
3002 		return -EPROBE_DEFER;
3003 
3004 	if (!driver_data || !driver_data->verify || !driver_data->init ||
3005 	     (driver_data->target_index && driver_data->target) ||
3006 	     (!!driver_data->setpolicy == (driver_data->target_index || driver_data->target)) ||
3007 	     (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
3008 	     (!driver_data->online != !driver_data->offline) ||
3009 		 (driver_data->adjust_perf && !driver_data->fast_switch))
3010 		return -EINVAL;
3011 
3012 	pr_debug("trying to register driver %s\n", driver_data->name);
3013 
3014 	/* Protect against concurrent CPU online/offline. */
3015 	cpus_read_lock();
3016 
3017 	write_lock_irqsave(&cpufreq_driver_lock, flags);
3018 	if (cpufreq_driver) {
3019 		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
3020 		ret = -EEXIST;
3021 		goto out;
3022 	}
3023 	cpufreq_driver = driver_data;
3024 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
3025 
3026 	if (driver_data->setpolicy)
3027 		driver_data->flags |= CPUFREQ_CONST_LOOPS;
3028 
3029 	if (cpufreq_boost_supported()) {
3030 		ret = create_boost_sysfs_file();
3031 		if (ret)
3032 			goto err_null_driver;
3033 	}
3034 
3035 	/*
3036 	 * Mark support for the scheduler's frequency invariance engine for
3037 	 * drivers that implement target(), target_index() or fast_switch().
3038 	 */
3039 	if (!cpufreq_driver->setpolicy) {
3040 		static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
3041 		pr_debug("cpufreq: supports frequency invariance\n");
3042 	}
3043 
3044 	ret = subsys_interface_register(&cpufreq_interface);
3045 	if (ret)
3046 		goto err_boost_unreg;
3047 
3048 	if (unlikely(list_empty(&cpufreq_policy_list))) {
3049 		/* if all ->init() calls failed, unregister */
3050 		ret = -ENODEV;
3051 		pr_debug("%s: No CPU initialized for driver %s\n", __func__,
3052 			 driver_data->name);
3053 		goto err_if_unreg;
3054 	}
3055 
3056 	ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
3057 						   "cpufreq:online",
3058 						   cpuhp_cpufreq_online,
3059 						   cpuhp_cpufreq_offline);
3060 	if (ret < 0)
3061 		goto err_if_unreg;
3062 	hp_online = ret;
3063 	ret = 0;
3064 
3065 	pr_debug("driver %s up and running\n", driver_data->name);
3066 	goto out;
3067 
3068 err_if_unreg:
3069 	subsys_interface_unregister(&cpufreq_interface);
3070 err_boost_unreg:
3071 	if (!cpufreq_driver->setpolicy)
3072 		static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
3073 	remove_boost_sysfs_file();
3074 err_null_driver:
3075 	write_lock_irqsave(&cpufreq_driver_lock, flags);
3076 	cpufreq_driver = NULL;
3077 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
3078 out:
3079 	cpus_read_unlock();
3080 	return ret;
3081 }
3082 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
3083 
3084 /*
3085  * cpufreq_unregister_driver - unregister the current CPUFreq driver
3086  *
3087  * Unregister the current CPUFreq driver. Only call this if you have
3088  * the right to do so, i.e. if you have succeeded in initialising before!
3089  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
3090  * currently not initialised.
3091  */
3092 void cpufreq_unregister_driver(struct cpufreq_driver *driver)
3093 {
3094 	unsigned long flags;
3095 
3096 	if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver)))
3097 		return;
3098 
3099 	pr_debug("unregistering driver %s\n", driver->name);
3100 
3101 	/* Protect against concurrent cpu hotplug */
3102 	cpus_read_lock();
3103 	subsys_interface_unregister(&cpufreq_interface);
3104 	remove_boost_sysfs_file();
3105 	static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
3106 	cpuhp_remove_state_nocalls_cpuslocked(hp_online);
3107 
3108 	write_lock_irqsave(&cpufreq_driver_lock, flags);
3109 
3110 	cpufreq_driver = NULL;
3111 
3112 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
3113 	cpus_read_unlock();
3114 }
3115 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
3116 
3117 static int __init cpufreq_core_init(void)
3118 {
3119 	struct cpufreq_governor *gov = cpufreq_default_governor();
3120 	struct device *dev_root;
3121 
3122 	if (cpufreq_disabled())
3123 		return -ENODEV;
3124 
3125 	dev_root = bus_get_dev_root(&cpu_subsys);
3126 	if (dev_root) {
3127 		cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj);
3128 		put_device(dev_root);
3129 	}
3130 	BUG_ON(!cpufreq_global_kobject);
3131 
3132 	if (!strlen(default_governor))
3133 		strscpy(default_governor, gov->name, CPUFREQ_NAME_LEN);
3134 
3135 	return 0;
3136 }
3137 
3138 static bool cpufreq_policy_is_good_for_eas(unsigned int cpu)
3139 {
3140 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
3141 	if (!policy) {
3142 		pr_debug("cpufreq policy not set for CPU: %d\n", cpu);
3143 		return false;
3144 	}
3145 
3146 	return sugov_is_governor(policy);
3147 }
3148 
3149 bool cpufreq_ready_for_eas(const struct cpumask *cpu_mask)
3150 {
3151 	unsigned int cpu;
3152 
3153 	/* Do not attempt EAS if schedutil is not being used. */
3154 	for_each_cpu(cpu, cpu_mask) {
3155 		if (!cpufreq_policy_is_good_for_eas(cpu)) {
3156 			pr_debug("rd %*pbl: schedutil is mandatory for EAS\n",
3157 				 cpumask_pr_args(cpu_mask));
3158 			return false;
3159 		}
3160 	}
3161 
3162 	return true;
3163 }
3164 
3165 module_param(off, int, 0444);
3166 module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444);
3167 core_initcall(cpufreq_core_init);
3168