xref: /linux/drivers/cpufreq/cpufreq.c (revision 1fa9970a4e61e9383a8ee738b44e2194d4ce7ff5)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/drivers/cpufreq/cpufreq.c
4  *
5  *  Copyright (C) 2001 Russell King
6  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
7  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
8  *
9  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
10  *	Added handling for CPU hotplug
11  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
12  *	Fix handling for CPU hotplug -- affected CPUs
13  */
14 
15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16 
17 #include <linux/cpu.h>
18 #include <linux/cpufreq.h>
19 #include <linux/cpu_cooling.h>
20 #include <linux/delay.h>
21 #include <linux/device.h>
22 #include <linux/init.h>
23 #include <linux/kernel_stat.h>
24 #include <linux/module.h>
25 #include <linux/mutex.h>
26 #include <linux/pm_qos.h>
27 #include <linux/slab.h>
28 #include <linux/string_choices.h>
29 #include <linux/suspend.h>
30 #include <linux/syscore_ops.h>
31 #include <linux/tick.h>
32 #include <linux/units.h>
33 #include <trace/events/power.h>
34 
35 static LIST_HEAD(cpufreq_policy_list);
36 
37 /* Macros to iterate over CPU policies */
38 #define for_each_suitable_policy(__policy, __active)			 \
39 	list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
40 		if ((__active) == !policy_is_inactive(__policy))
41 
42 #define for_each_active_policy(__policy)		\
43 	for_each_suitable_policy(__policy, true)
44 #define for_each_inactive_policy(__policy)		\
45 	for_each_suitable_policy(__policy, false)
46 
47 /* Iterate over governors */
48 static LIST_HEAD(cpufreq_governor_list);
49 #define for_each_governor(__governor)				\
50 	list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
51 
52 static char default_governor[CPUFREQ_NAME_LEN];
53 
54 /*
55  * The "cpufreq driver" - the arch- or hardware-dependent low
56  * level driver of CPUFreq support, and its spinlock. This lock
57  * also protects the cpufreq_cpu_data array.
58  */
59 static struct cpufreq_driver *cpufreq_driver;
60 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
61 static DEFINE_RWLOCK(cpufreq_driver_lock);
62 
63 static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
cpufreq_supports_freq_invariance(void)64 bool cpufreq_supports_freq_invariance(void)
65 {
66 	return static_branch_likely(&cpufreq_freq_invariance);
67 }
68 
69 /* Flag to suspend/resume CPUFreq governors */
70 static bool cpufreq_suspended;
71 
has_target(void)72 static inline bool has_target(void)
73 {
74 	return cpufreq_driver->target_index || cpufreq_driver->target;
75 }
76 
has_target_index(void)77 bool has_target_index(void)
78 {
79 	return !!cpufreq_driver->target_index;
80 }
81 
82 /* internal prototypes */
83 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
84 static int cpufreq_init_governor(struct cpufreq_policy *policy);
85 static void cpufreq_exit_governor(struct cpufreq_policy *policy);
86 static void cpufreq_governor_limits(struct cpufreq_policy *policy);
87 static int cpufreq_set_policy(struct cpufreq_policy *policy,
88 			      struct cpufreq_governor *new_gov,
89 			      unsigned int new_pol);
90 static bool cpufreq_boost_supported(void);
91 
92 /*
93  * Two notifier lists: the "policy" list is involved in the
94  * validation process for a new CPU frequency policy; the
95  * "transition" list for kernel code that needs to handle
96  * changes to devices when the CPU clock speed changes.
97  * The mutex locks both lists.
98  */
99 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
100 SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
101 
102 static int off __read_mostly;
cpufreq_disabled(void)103 static int cpufreq_disabled(void)
104 {
105 	return off;
106 }
disable_cpufreq(void)107 void disable_cpufreq(void)
108 {
109 	off = 1;
110 }
111 static DEFINE_MUTEX(cpufreq_governor_mutex);
112 
have_governor_per_policy(void)113 bool have_governor_per_policy(void)
114 {
115 	return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
116 }
117 EXPORT_SYMBOL_GPL(have_governor_per_policy);
118 
119 static struct kobject *cpufreq_global_kobject;
120 
get_governor_parent_kobj(struct cpufreq_policy * policy)121 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
122 {
123 	if (have_governor_per_policy())
124 		return &policy->kobj;
125 	else
126 		return cpufreq_global_kobject;
127 }
128 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
129 
get_cpu_idle_time_jiffy(unsigned int cpu,u64 * wall)130 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
131 {
132 	struct kernel_cpustat kcpustat;
133 	u64 cur_wall_time;
134 	u64 idle_time;
135 	u64 busy_time;
136 
137 	cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
138 
139 	kcpustat_cpu_fetch(&kcpustat, cpu);
140 
141 	busy_time = kcpustat.cpustat[CPUTIME_USER];
142 	busy_time += kcpustat.cpustat[CPUTIME_SYSTEM];
143 	busy_time += kcpustat.cpustat[CPUTIME_IRQ];
144 	busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ];
145 	busy_time += kcpustat.cpustat[CPUTIME_STEAL];
146 	busy_time += kcpustat.cpustat[CPUTIME_NICE];
147 
148 	idle_time = cur_wall_time - busy_time;
149 	if (wall)
150 		*wall = div_u64(cur_wall_time, NSEC_PER_USEC);
151 
152 	return div_u64(idle_time, NSEC_PER_USEC);
153 }
154 
get_cpu_idle_time(unsigned int cpu,u64 * wall,int io_busy)155 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
156 {
157 	u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
158 
159 	if (idle_time == -1ULL)
160 		return get_cpu_idle_time_jiffy(cpu, wall);
161 	else if (!io_busy)
162 		idle_time += get_cpu_iowait_time_us(cpu, wall);
163 
164 	return idle_time;
165 }
166 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
167 
168 /*
169  * This is a generic cpufreq init() routine which can be used by cpufreq
170  * drivers of SMP systems. It will do following:
171  * - validate & show freq table passed
172  * - set policies transition latency
173  * - policy->cpus with all possible CPUs
174  */
cpufreq_generic_init(struct cpufreq_policy * policy,struct cpufreq_frequency_table * table,unsigned int transition_latency)175 void cpufreq_generic_init(struct cpufreq_policy *policy,
176 		struct cpufreq_frequency_table *table,
177 		unsigned int transition_latency)
178 {
179 	policy->freq_table = table;
180 	policy->cpuinfo.transition_latency = transition_latency;
181 
182 	/*
183 	 * The driver only supports the SMP configuration where all processors
184 	 * share the clock and voltage and clock.
185 	 */
186 	cpumask_setall(policy->cpus);
187 }
188 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
189 
cpufreq_cpu_get_raw(unsigned int cpu)190 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
191 {
192 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
193 
194 	return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
195 }
196 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
197 
cpufreq_generic_get(unsigned int cpu)198 unsigned int cpufreq_generic_get(unsigned int cpu)
199 {
200 	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
201 
202 	if (!policy || IS_ERR(policy->clk)) {
203 		pr_err("%s: No %s associated to cpu: %d\n",
204 		       __func__, policy ? "clk" : "policy", cpu);
205 		return 0;
206 	}
207 
208 	return clk_get_rate(policy->clk) / 1000;
209 }
210 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
211 
212 /**
213  * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
214  * @cpu: CPU to find the policy for.
215  *
216  * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
217  * the kobject reference counter of that policy.  Return a valid policy on
218  * success or NULL on failure.
219  *
220  * The policy returned by this function has to be released with the help of
221  * cpufreq_cpu_put() to balance its kobject reference counter properly.
222  */
cpufreq_cpu_get(unsigned int cpu)223 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
224 {
225 	struct cpufreq_policy *policy = NULL;
226 	unsigned long flags;
227 
228 	if (WARN_ON(cpu >= nr_cpu_ids))
229 		return NULL;
230 
231 	/* get the cpufreq driver */
232 	read_lock_irqsave(&cpufreq_driver_lock, flags);
233 
234 	if (cpufreq_driver) {
235 		/* get the CPU */
236 		policy = cpufreq_cpu_get_raw(cpu);
237 		if (policy)
238 			kobject_get(&policy->kobj);
239 	}
240 
241 	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
242 
243 	return policy;
244 }
245 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
246 
247 /**
248  * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
249  * @policy: cpufreq policy returned by cpufreq_cpu_get().
250  */
cpufreq_cpu_put(struct cpufreq_policy * policy)251 void cpufreq_cpu_put(struct cpufreq_policy *policy)
252 {
253 	kobject_put(&policy->kobj);
254 }
255 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
256 
257 /**
258  * cpufreq_cpu_release - Unlock a policy and decrement its usage counter.
259  * @policy: cpufreq policy returned by cpufreq_cpu_acquire().
260  */
cpufreq_cpu_release(struct cpufreq_policy * policy)261 void cpufreq_cpu_release(struct cpufreq_policy *policy)
262 {
263 	if (WARN_ON(!policy))
264 		return;
265 
266 	lockdep_assert_held(&policy->rwsem);
267 
268 	up_write(&policy->rwsem);
269 
270 	cpufreq_cpu_put(policy);
271 }
272 
273 /**
274  * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it.
275  * @cpu: CPU to find the policy for.
276  *
277  * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and
278  * if the policy returned by it is not NULL, acquire its rwsem for writing.
279  * Return the policy if it is active or release it and return NULL otherwise.
280  *
281  * The policy returned by this function has to be released with the help of
282  * cpufreq_cpu_release() in order to release its rwsem and balance its usage
283  * counter properly.
284  */
cpufreq_cpu_acquire(unsigned int cpu)285 struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu)
286 {
287 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
288 
289 	if (!policy)
290 		return NULL;
291 
292 	down_write(&policy->rwsem);
293 
294 	if (policy_is_inactive(policy)) {
295 		cpufreq_cpu_release(policy);
296 		return NULL;
297 	}
298 
299 	return policy;
300 }
301 
302 /*********************************************************************
303  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
304  *********************************************************************/
305 
306 /**
307  * adjust_jiffies - Adjust the system "loops_per_jiffy".
308  * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
309  * @ci: Frequency change information.
310  *
311  * This function alters the system "loops_per_jiffy" for the clock
312  * speed change. Note that loops_per_jiffy cannot be updated on SMP
313  * systems as each CPU might be scaled differently. So, use the arch
314  * per-CPU loops_per_jiffy value wherever possible.
315  */
adjust_jiffies(unsigned long val,struct cpufreq_freqs * ci)316 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
317 {
318 #ifndef CONFIG_SMP
319 	static unsigned long l_p_j_ref;
320 	static unsigned int l_p_j_ref_freq;
321 
322 	if (ci->flags & CPUFREQ_CONST_LOOPS)
323 		return;
324 
325 	if (!l_p_j_ref_freq) {
326 		l_p_j_ref = loops_per_jiffy;
327 		l_p_j_ref_freq = ci->old;
328 		pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
329 			 l_p_j_ref, l_p_j_ref_freq);
330 	}
331 	if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
332 		loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
333 								ci->new);
334 		pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
335 			 loops_per_jiffy, ci->new);
336 	}
337 #endif
338 }
339 
340 /**
341  * cpufreq_notify_transition - Notify frequency transition and adjust jiffies.
342  * @policy: cpufreq policy to enable fast frequency switching for.
343  * @freqs: contain details of the frequency update.
344  * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
345  *
346  * This function calls the transition notifiers and adjust_jiffies().
347  *
348  * It is called twice on all CPU frequency changes that have external effects.
349  */
cpufreq_notify_transition(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,unsigned int state)350 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
351 				      struct cpufreq_freqs *freqs,
352 				      unsigned int state)
353 {
354 	int cpu;
355 
356 	BUG_ON(irqs_disabled());
357 
358 	if (cpufreq_disabled())
359 		return;
360 
361 	freqs->policy = policy;
362 	freqs->flags = cpufreq_driver->flags;
363 	pr_debug("notification %u of frequency transition to %u kHz\n",
364 		 state, freqs->new);
365 
366 	switch (state) {
367 	case CPUFREQ_PRECHANGE:
368 		/*
369 		 * Detect if the driver reported a value as "old frequency"
370 		 * which is not equal to what the cpufreq core thinks is
371 		 * "old frequency".
372 		 */
373 		if (policy->cur && policy->cur != freqs->old) {
374 			pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
375 				 freqs->old, policy->cur);
376 			freqs->old = policy->cur;
377 		}
378 
379 		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
380 					 CPUFREQ_PRECHANGE, freqs);
381 
382 		adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
383 		break;
384 
385 	case CPUFREQ_POSTCHANGE:
386 		adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
387 		pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
388 			 cpumask_pr_args(policy->cpus));
389 
390 		for_each_cpu(cpu, policy->cpus)
391 			trace_cpu_frequency(freqs->new, cpu);
392 
393 		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
394 					 CPUFREQ_POSTCHANGE, freqs);
395 
396 		cpufreq_stats_record_transition(policy, freqs->new);
397 		policy->cur = freqs->new;
398 	}
399 }
400 
401 /* Do post notifications when there are chances that transition has failed */
cpufreq_notify_post_transition(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,int transition_failed)402 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
403 		struct cpufreq_freqs *freqs, int transition_failed)
404 {
405 	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
406 	if (!transition_failed)
407 		return;
408 
409 	swap(freqs->old, freqs->new);
410 	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
411 	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
412 }
413 
cpufreq_freq_transition_begin(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs)414 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
415 		struct cpufreq_freqs *freqs)
416 {
417 
418 	/*
419 	 * Catch double invocations of _begin() which lead to self-deadlock.
420 	 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
421 	 * doesn't invoke _begin() on their behalf, and hence the chances of
422 	 * double invocations are very low. Moreover, there are scenarios
423 	 * where these checks can emit false-positive warnings in these
424 	 * drivers; so we avoid that by skipping them altogether.
425 	 */
426 	WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
427 				&& current == policy->transition_task);
428 
429 wait:
430 	wait_event(policy->transition_wait, !policy->transition_ongoing);
431 
432 	spin_lock(&policy->transition_lock);
433 
434 	if (unlikely(policy->transition_ongoing)) {
435 		spin_unlock(&policy->transition_lock);
436 		goto wait;
437 	}
438 
439 	policy->transition_ongoing = true;
440 	policy->transition_task = current;
441 
442 	spin_unlock(&policy->transition_lock);
443 
444 	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
445 }
446 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
447 
cpufreq_freq_transition_end(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,int transition_failed)448 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
449 		struct cpufreq_freqs *freqs, int transition_failed)
450 {
451 	if (WARN_ON(!policy->transition_ongoing))
452 		return;
453 
454 	cpufreq_notify_post_transition(policy, freqs, transition_failed);
455 
456 	arch_set_freq_scale(policy->related_cpus,
457 			    policy->cur,
458 			    arch_scale_freq_ref(policy->cpu));
459 
460 	spin_lock(&policy->transition_lock);
461 	policy->transition_ongoing = false;
462 	policy->transition_task = NULL;
463 	spin_unlock(&policy->transition_lock);
464 
465 	wake_up(&policy->transition_wait);
466 }
467 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
468 
469 /*
470  * Fast frequency switching status count.  Positive means "enabled", negative
471  * means "disabled" and 0 means "not decided yet".
472  */
473 static int cpufreq_fast_switch_count;
474 static DEFINE_MUTEX(cpufreq_fast_switch_lock);
475 
cpufreq_list_transition_notifiers(void)476 static void cpufreq_list_transition_notifiers(void)
477 {
478 	struct notifier_block *nb;
479 
480 	pr_info("Registered transition notifiers:\n");
481 
482 	mutex_lock(&cpufreq_transition_notifier_list.mutex);
483 
484 	for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
485 		pr_info("%pS\n", nb->notifier_call);
486 
487 	mutex_unlock(&cpufreq_transition_notifier_list.mutex);
488 }
489 
490 /**
491  * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
492  * @policy: cpufreq policy to enable fast frequency switching for.
493  *
494  * Try to enable fast frequency switching for @policy.
495  *
496  * The attempt will fail if there is at least one transition notifier registered
497  * at this point, as fast frequency switching is quite fundamentally at odds
498  * with transition notifiers.  Thus if successful, it will make registration of
499  * transition notifiers fail going forward.
500  */
cpufreq_enable_fast_switch(struct cpufreq_policy * policy)501 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
502 {
503 	lockdep_assert_held(&policy->rwsem);
504 
505 	if (!policy->fast_switch_possible)
506 		return;
507 
508 	mutex_lock(&cpufreq_fast_switch_lock);
509 	if (cpufreq_fast_switch_count >= 0) {
510 		cpufreq_fast_switch_count++;
511 		policy->fast_switch_enabled = true;
512 	} else {
513 		pr_warn("CPU%u: Fast frequency switching not enabled\n",
514 			policy->cpu);
515 		cpufreq_list_transition_notifiers();
516 	}
517 	mutex_unlock(&cpufreq_fast_switch_lock);
518 }
519 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
520 
521 /**
522  * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
523  * @policy: cpufreq policy to disable fast frequency switching for.
524  */
cpufreq_disable_fast_switch(struct cpufreq_policy * policy)525 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
526 {
527 	mutex_lock(&cpufreq_fast_switch_lock);
528 	if (policy->fast_switch_enabled) {
529 		policy->fast_switch_enabled = false;
530 		if (!WARN_ON(cpufreq_fast_switch_count <= 0))
531 			cpufreq_fast_switch_count--;
532 	}
533 	mutex_unlock(&cpufreq_fast_switch_lock);
534 }
535 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
536 
__resolve_freq(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)537 static unsigned int __resolve_freq(struct cpufreq_policy *policy,
538 		unsigned int target_freq, unsigned int relation)
539 {
540 	unsigned int idx;
541 
542 	target_freq = clamp_val(target_freq, policy->min, policy->max);
543 
544 	if (!policy->freq_table)
545 		return target_freq;
546 
547 	idx = cpufreq_frequency_table_target(policy, target_freq, relation);
548 	policy->cached_resolved_idx = idx;
549 	policy->cached_target_freq = target_freq;
550 	return policy->freq_table[idx].frequency;
551 }
552 
553 /**
554  * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
555  * one.
556  * @policy: associated policy to interrogate
557  * @target_freq: target frequency to resolve.
558  *
559  * The target to driver frequency mapping is cached in the policy.
560  *
561  * Return: Lowest driver-supported frequency greater than or equal to the
562  * given target_freq, subject to policy (min/max) and driver limitations.
563  */
cpufreq_driver_resolve_freq(struct cpufreq_policy * policy,unsigned int target_freq)564 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
565 					 unsigned int target_freq)
566 {
567 	return __resolve_freq(policy, target_freq, CPUFREQ_RELATION_LE);
568 }
569 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
570 
cpufreq_policy_transition_delay_us(struct cpufreq_policy * policy)571 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
572 {
573 	unsigned int latency;
574 
575 	if (policy->transition_delay_us)
576 		return policy->transition_delay_us;
577 
578 	latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
579 	if (latency)
580 		/* Give a 50% breathing room between updates */
581 		return latency + (latency >> 1);
582 
583 	return USEC_PER_MSEC;
584 }
585 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
586 
587 /*********************************************************************
588  *                          SYSFS INTERFACE                          *
589  *********************************************************************/
show_boost(struct kobject * kobj,struct kobj_attribute * attr,char * buf)590 static ssize_t show_boost(struct kobject *kobj,
591 			  struct kobj_attribute *attr, char *buf)
592 {
593 	return sysfs_emit(buf, "%d\n", cpufreq_driver->boost_enabled);
594 }
595 
store_boost(struct kobject * kobj,struct kobj_attribute * attr,const char * buf,size_t count)596 static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
597 			   const char *buf, size_t count)
598 {
599 	bool enable;
600 
601 	if (kstrtobool(buf, &enable))
602 		return -EINVAL;
603 
604 	if (cpufreq_boost_trigger_state(enable)) {
605 		pr_err("%s: Cannot %s BOOST!\n",
606 		       __func__, str_enable_disable(enable));
607 		return -EINVAL;
608 	}
609 
610 	pr_debug("%s: cpufreq BOOST %s\n",
611 		 __func__, str_enabled_disabled(enable));
612 
613 	return count;
614 }
615 define_one_global_rw(boost);
616 
show_local_boost(struct cpufreq_policy * policy,char * buf)617 static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf)
618 {
619 	return sysfs_emit(buf, "%d\n", policy->boost_enabled);
620 }
621 
store_local_boost(struct cpufreq_policy * policy,const char * buf,size_t count)622 static ssize_t store_local_boost(struct cpufreq_policy *policy,
623 				 const char *buf, size_t count)
624 {
625 	int ret;
626 	bool enable;
627 
628 	if (kstrtobool(buf, &enable))
629 		return -EINVAL;
630 
631 	if (!cpufreq_driver->boost_enabled)
632 		return -EINVAL;
633 
634 	if (policy->boost_enabled == enable)
635 		return count;
636 
637 	policy->boost_enabled = enable;
638 
639 	cpus_read_lock();
640 	ret = cpufreq_driver->set_boost(policy, enable);
641 	cpus_read_unlock();
642 
643 	if (ret) {
644 		policy->boost_enabled = !policy->boost_enabled;
645 		return ret;
646 	}
647 
648 	return count;
649 }
650 
651 static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost);
652 
find_governor(const char * str_governor)653 static struct cpufreq_governor *find_governor(const char *str_governor)
654 {
655 	struct cpufreq_governor *t;
656 
657 	for_each_governor(t)
658 		if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
659 			return t;
660 
661 	return NULL;
662 }
663 
get_governor(const char * str_governor)664 static struct cpufreq_governor *get_governor(const char *str_governor)
665 {
666 	struct cpufreq_governor *t;
667 
668 	mutex_lock(&cpufreq_governor_mutex);
669 	t = find_governor(str_governor);
670 	if (!t)
671 		goto unlock;
672 
673 	if (!try_module_get(t->owner))
674 		t = NULL;
675 
676 unlock:
677 	mutex_unlock(&cpufreq_governor_mutex);
678 
679 	return t;
680 }
681 
cpufreq_parse_policy(char * str_governor)682 static unsigned int cpufreq_parse_policy(char *str_governor)
683 {
684 	if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN))
685 		return CPUFREQ_POLICY_PERFORMANCE;
686 
687 	if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN))
688 		return CPUFREQ_POLICY_POWERSAVE;
689 
690 	return CPUFREQ_POLICY_UNKNOWN;
691 }
692 
693 /**
694  * cpufreq_parse_governor - parse a governor string only for has_target()
695  * @str_governor: Governor name.
696  */
cpufreq_parse_governor(char * str_governor)697 static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor)
698 {
699 	struct cpufreq_governor *t;
700 
701 	t = get_governor(str_governor);
702 	if (t)
703 		return t;
704 
705 	if (request_module("cpufreq_%s", str_governor))
706 		return NULL;
707 
708 	return get_governor(str_governor);
709 }
710 
711 /*
712  * cpufreq_per_cpu_attr_read() / show_##file_name() -
713  * print out cpufreq information
714  *
715  * Write out information from cpufreq_driver->policy[cpu]; object must be
716  * "unsigned int".
717  */
718 
719 #define show_one(file_name, object)			\
720 static ssize_t show_##file_name				\
721 (struct cpufreq_policy *policy, char *buf)		\
722 {							\
723 	return sysfs_emit(buf, "%u\n", policy->object);	\
724 }
725 
726 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
727 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
728 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
729 show_one(scaling_min_freq, min);
730 show_one(scaling_max_freq, max);
731 
arch_freq_get_on_cpu(int cpu)732 __weak unsigned int arch_freq_get_on_cpu(int cpu)
733 {
734 	return 0;
735 }
736 
show_scaling_cur_freq(struct cpufreq_policy * policy,char * buf)737 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
738 {
739 	ssize_t ret;
740 	unsigned int freq;
741 
742 	freq = arch_freq_get_on_cpu(policy->cpu);
743 	if (freq)
744 		ret = sysfs_emit(buf, "%u\n", freq);
745 	else if (cpufreq_driver->setpolicy && cpufreq_driver->get)
746 		ret = sysfs_emit(buf, "%u\n", cpufreq_driver->get(policy->cpu));
747 	else
748 		ret = sysfs_emit(buf, "%u\n", policy->cur);
749 	return ret;
750 }
751 
752 /*
753  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
754  */
755 #define store_one(file_name, object)			\
756 static ssize_t store_##file_name					\
757 (struct cpufreq_policy *policy, const char *buf, size_t count)		\
758 {									\
759 	unsigned long val;						\
760 	int ret;							\
761 									\
762 	ret = kstrtoul(buf, 0, &val);					\
763 	if (ret)							\
764 		return ret;						\
765 									\
766 	ret = freq_qos_update_request(policy->object##_freq_req, val);\
767 	return ret >= 0 ? count : ret;					\
768 }
769 
770 store_one(scaling_min_freq, min);
771 store_one(scaling_max_freq, max);
772 
773 /*
774  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
775  */
show_cpuinfo_cur_freq(struct cpufreq_policy * policy,char * buf)776 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
777 					char *buf)
778 {
779 	unsigned int cur_freq = __cpufreq_get(policy);
780 
781 	if (cur_freq)
782 		return sysfs_emit(buf, "%u\n", cur_freq);
783 
784 	return sysfs_emit(buf, "<unknown>\n");
785 }
786 
787 /*
788  * show_scaling_governor - show the current policy for the specified CPU
789  */
show_scaling_governor(struct cpufreq_policy * policy,char * buf)790 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
791 {
792 	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
793 		return sysfs_emit(buf, "powersave\n");
794 	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
795 		return sysfs_emit(buf, "performance\n");
796 	else if (policy->governor)
797 		return sysfs_emit(buf, "%s\n", policy->governor->name);
798 	return -EINVAL;
799 }
800 
801 /*
802  * store_scaling_governor - store policy for the specified CPU
803  */
store_scaling_governor(struct cpufreq_policy * policy,const char * buf,size_t count)804 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
805 					const char *buf, size_t count)
806 {
807 	char str_governor[16];
808 	int ret;
809 
810 	ret = sscanf(buf, "%15s", str_governor);
811 	if (ret != 1)
812 		return -EINVAL;
813 
814 	if (cpufreq_driver->setpolicy) {
815 		unsigned int new_pol;
816 
817 		new_pol = cpufreq_parse_policy(str_governor);
818 		if (!new_pol)
819 			return -EINVAL;
820 
821 		ret = cpufreq_set_policy(policy, NULL, new_pol);
822 	} else {
823 		struct cpufreq_governor *new_gov;
824 
825 		new_gov = cpufreq_parse_governor(str_governor);
826 		if (!new_gov)
827 			return -EINVAL;
828 
829 		ret = cpufreq_set_policy(policy, new_gov,
830 					 CPUFREQ_POLICY_UNKNOWN);
831 
832 		module_put(new_gov->owner);
833 	}
834 
835 	return ret ? ret : count;
836 }
837 
838 /*
839  * show_scaling_driver - show the cpufreq driver currently loaded
840  */
show_scaling_driver(struct cpufreq_policy * policy,char * buf)841 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
842 {
843 	return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
844 }
845 
846 /*
847  * show_scaling_available_governors - show the available CPUfreq governors
848  */
show_scaling_available_governors(struct cpufreq_policy * policy,char * buf)849 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
850 						char *buf)
851 {
852 	ssize_t i = 0;
853 	struct cpufreq_governor *t;
854 
855 	if (!has_target()) {
856 		i += sysfs_emit(buf, "performance powersave");
857 		goto out;
858 	}
859 
860 	mutex_lock(&cpufreq_governor_mutex);
861 	for_each_governor(t) {
862 		if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
863 		    - (CPUFREQ_NAME_LEN + 2)))
864 			break;
865 		i += sysfs_emit_at(buf, i, "%s ", t->name);
866 	}
867 	mutex_unlock(&cpufreq_governor_mutex);
868 out:
869 	i += sysfs_emit_at(buf, i, "\n");
870 	return i;
871 }
872 
cpufreq_show_cpus(const struct cpumask * mask,char * buf)873 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
874 {
875 	ssize_t i = 0;
876 	unsigned int cpu;
877 
878 	for_each_cpu(cpu, mask) {
879 		i += sysfs_emit_at(buf, i, "%u ", cpu);
880 		if (i >= (PAGE_SIZE - 5))
881 			break;
882 	}
883 
884 	/* Remove the extra space at the end */
885 	i--;
886 
887 	i += sysfs_emit_at(buf, i, "\n");
888 	return i;
889 }
890 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
891 
892 /*
893  * show_related_cpus - show the CPUs affected by each transition even if
894  * hw coordination is in use
895  */
show_related_cpus(struct cpufreq_policy * policy,char * buf)896 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
897 {
898 	return cpufreq_show_cpus(policy->related_cpus, buf);
899 }
900 
901 /*
902  * show_affected_cpus - show the CPUs affected by each transition
903  */
show_affected_cpus(struct cpufreq_policy * policy,char * buf)904 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
905 {
906 	return cpufreq_show_cpus(policy->cpus, buf);
907 }
908 
store_scaling_setspeed(struct cpufreq_policy * policy,const char * buf,size_t count)909 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
910 					const char *buf, size_t count)
911 {
912 	unsigned int freq = 0;
913 	unsigned int ret;
914 
915 	if (!policy->governor || !policy->governor->store_setspeed)
916 		return -EINVAL;
917 
918 	ret = sscanf(buf, "%u", &freq);
919 	if (ret != 1)
920 		return -EINVAL;
921 
922 	policy->governor->store_setspeed(policy, freq);
923 
924 	return count;
925 }
926 
show_scaling_setspeed(struct cpufreq_policy * policy,char * buf)927 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
928 {
929 	if (!policy->governor || !policy->governor->show_setspeed)
930 		return sysfs_emit(buf, "<unsupported>\n");
931 
932 	return policy->governor->show_setspeed(policy, buf);
933 }
934 
935 /*
936  * show_bios_limit - show the current cpufreq HW/BIOS limitation
937  */
show_bios_limit(struct cpufreq_policy * policy,char * buf)938 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
939 {
940 	unsigned int limit;
941 	int ret;
942 	ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
943 	if (!ret)
944 		return sysfs_emit(buf, "%u\n", limit);
945 	return sysfs_emit(buf, "%u\n", policy->cpuinfo.max_freq);
946 }
947 
948 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
949 cpufreq_freq_attr_ro(cpuinfo_min_freq);
950 cpufreq_freq_attr_ro(cpuinfo_max_freq);
951 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
952 cpufreq_freq_attr_ro(scaling_available_governors);
953 cpufreq_freq_attr_ro(scaling_driver);
954 cpufreq_freq_attr_ro(scaling_cur_freq);
955 cpufreq_freq_attr_ro(bios_limit);
956 cpufreq_freq_attr_ro(related_cpus);
957 cpufreq_freq_attr_ro(affected_cpus);
958 cpufreq_freq_attr_rw(scaling_min_freq);
959 cpufreq_freq_attr_rw(scaling_max_freq);
960 cpufreq_freq_attr_rw(scaling_governor);
961 cpufreq_freq_attr_rw(scaling_setspeed);
962 
963 static struct attribute *cpufreq_attrs[] = {
964 	&cpuinfo_min_freq.attr,
965 	&cpuinfo_max_freq.attr,
966 	&cpuinfo_transition_latency.attr,
967 	&scaling_min_freq.attr,
968 	&scaling_max_freq.attr,
969 	&affected_cpus.attr,
970 	&related_cpus.attr,
971 	&scaling_governor.attr,
972 	&scaling_driver.attr,
973 	&scaling_available_governors.attr,
974 	&scaling_setspeed.attr,
975 	NULL
976 };
977 ATTRIBUTE_GROUPS(cpufreq);
978 
979 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
980 #define to_attr(a) container_of(a, struct freq_attr, attr)
981 
show(struct kobject * kobj,struct attribute * attr,char * buf)982 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
983 {
984 	struct cpufreq_policy *policy = to_policy(kobj);
985 	struct freq_attr *fattr = to_attr(attr);
986 	ssize_t ret = -EBUSY;
987 
988 	if (!fattr->show)
989 		return -EIO;
990 
991 	down_read(&policy->rwsem);
992 	if (likely(!policy_is_inactive(policy)))
993 		ret = fattr->show(policy, buf);
994 	up_read(&policy->rwsem);
995 
996 	return ret;
997 }
998 
store(struct kobject * kobj,struct attribute * attr,const char * buf,size_t count)999 static ssize_t store(struct kobject *kobj, struct attribute *attr,
1000 		     const char *buf, size_t count)
1001 {
1002 	struct cpufreq_policy *policy = to_policy(kobj);
1003 	struct freq_attr *fattr = to_attr(attr);
1004 	ssize_t ret = -EBUSY;
1005 
1006 	if (!fattr->store)
1007 		return -EIO;
1008 
1009 	down_write(&policy->rwsem);
1010 	if (likely(!policy_is_inactive(policy)))
1011 		ret = fattr->store(policy, buf, count);
1012 	up_write(&policy->rwsem);
1013 
1014 	return ret;
1015 }
1016 
cpufreq_sysfs_release(struct kobject * kobj)1017 static void cpufreq_sysfs_release(struct kobject *kobj)
1018 {
1019 	struct cpufreq_policy *policy = to_policy(kobj);
1020 	pr_debug("last reference is dropped\n");
1021 	complete(&policy->kobj_unregister);
1022 }
1023 
1024 static const struct sysfs_ops sysfs_ops = {
1025 	.show	= show,
1026 	.store	= store,
1027 };
1028 
1029 static const struct kobj_type ktype_cpufreq = {
1030 	.sysfs_ops	= &sysfs_ops,
1031 	.default_groups	= cpufreq_groups,
1032 	.release	= cpufreq_sysfs_release,
1033 };
1034 
add_cpu_dev_symlink(struct cpufreq_policy * policy,unsigned int cpu,struct device * dev)1035 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu,
1036 				struct device *dev)
1037 {
1038 	if (unlikely(!dev))
1039 		return;
1040 
1041 	if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
1042 		return;
1043 
1044 	dev_dbg(dev, "%s: Adding symlink\n", __func__);
1045 	if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
1046 		dev_err(dev, "cpufreq symlink creation failed\n");
1047 }
1048 
remove_cpu_dev_symlink(struct cpufreq_policy * policy,int cpu,struct device * dev)1049 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu,
1050 				   struct device *dev)
1051 {
1052 	dev_dbg(dev, "%s: Removing symlink\n", __func__);
1053 	sysfs_remove_link(&dev->kobj, "cpufreq");
1054 	cpumask_clear_cpu(cpu, policy->real_cpus);
1055 }
1056 
cpufreq_add_dev_interface(struct cpufreq_policy * policy)1057 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
1058 {
1059 	struct freq_attr **drv_attr;
1060 	int ret = 0;
1061 
1062 	/* set up files for this cpu device */
1063 	drv_attr = cpufreq_driver->attr;
1064 	while (drv_attr && *drv_attr) {
1065 		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1066 		if (ret)
1067 			return ret;
1068 		drv_attr++;
1069 	}
1070 	if (cpufreq_driver->get) {
1071 		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1072 		if (ret)
1073 			return ret;
1074 	}
1075 
1076 	ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1077 	if (ret)
1078 		return ret;
1079 
1080 	if (cpufreq_driver->bios_limit) {
1081 		ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1082 		if (ret)
1083 			return ret;
1084 	}
1085 
1086 	if (cpufreq_boost_supported()) {
1087 		ret = sysfs_create_file(&policy->kobj, &local_boost.attr);
1088 		if (ret)
1089 			return ret;
1090 	}
1091 
1092 	return 0;
1093 }
1094 
cpufreq_init_policy(struct cpufreq_policy * policy)1095 static int cpufreq_init_policy(struct cpufreq_policy *policy)
1096 {
1097 	struct cpufreq_governor *gov = NULL;
1098 	unsigned int pol = CPUFREQ_POLICY_UNKNOWN;
1099 	int ret;
1100 
1101 	if (has_target()) {
1102 		/* Update policy governor to the one used before hotplug. */
1103 		gov = get_governor(policy->last_governor);
1104 		if (gov) {
1105 			pr_debug("Restoring governor %s for cpu %d\n",
1106 				 gov->name, policy->cpu);
1107 		} else {
1108 			gov = get_governor(default_governor);
1109 		}
1110 
1111 		if (!gov) {
1112 			gov = cpufreq_default_governor();
1113 			__module_get(gov->owner);
1114 		}
1115 
1116 	} else {
1117 
1118 		/* Use the default policy if there is no last_policy. */
1119 		if (policy->last_policy) {
1120 			pol = policy->last_policy;
1121 		} else {
1122 			pol = cpufreq_parse_policy(default_governor);
1123 			/*
1124 			 * In case the default governor is neither "performance"
1125 			 * nor "powersave", fall back to the initial policy
1126 			 * value set by the driver.
1127 			 */
1128 			if (pol == CPUFREQ_POLICY_UNKNOWN)
1129 				pol = policy->policy;
1130 		}
1131 		if (pol != CPUFREQ_POLICY_PERFORMANCE &&
1132 		    pol != CPUFREQ_POLICY_POWERSAVE)
1133 			return -ENODATA;
1134 	}
1135 
1136 	ret = cpufreq_set_policy(policy, gov, pol);
1137 	if (gov)
1138 		module_put(gov->owner);
1139 
1140 	return ret;
1141 }
1142 
cpufreq_add_policy_cpu(struct cpufreq_policy * policy,unsigned int cpu)1143 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1144 {
1145 	int ret = 0;
1146 
1147 	/* Has this CPU been taken care of already? */
1148 	if (cpumask_test_cpu(cpu, policy->cpus))
1149 		return 0;
1150 
1151 	down_write(&policy->rwsem);
1152 	if (has_target())
1153 		cpufreq_stop_governor(policy);
1154 
1155 	cpumask_set_cpu(cpu, policy->cpus);
1156 
1157 	if (has_target()) {
1158 		ret = cpufreq_start_governor(policy);
1159 		if (ret)
1160 			pr_err("%s: Failed to start governor\n", __func__);
1161 	}
1162 	up_write(&policy->rwsem);
1163 	return ret;
1164 }
1165 
refresh_frequency_limits(struct cpufreq_policy * policy)1166 void refresh_frequency_limits(struct cpufreq_policy *policy)
1167 {
1168 	if (!policy_is_inactive(policy)) {
1169 		pr_debug("updating policy for CPU %u\n", policy->cpu);
1170 
1171 		cpufreq_set_policy(policy, policy->governor, policy->policy);
1172 	}
1173 }
1174 EXPORT_SYMBOL(refresh_frequency_limits);
1175 
handle_update(struct work_struct * work)1176 static void handle_update(struct work_struct *work)
1177 {
1178 	struct cpufreq_policy *policy =
1179 		container_of(work, struct cpufreq_policy, update);
1180 
1181 	pr_debug("handle_update for cpu %u called\n", policy->cpu);
1182 	down_write(&policy->rwsem);
1183 	refresh_frequency_limits(policy);
1184 	up_write(&policy->rwsem);
1185 }
1186 
cpufreq_notifier_min(struct notifier_block * nb,unsigned long freq,void * data)1187 static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
1188 				void *data)
1189 {
1190 	struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
1191 
1192 	schedule_work(&policy->update);
1193 	return 0;
1194 }
1195 
cpufreq_notifier_max(struct notifier_block * nb,unsigned long freq,void * data)1196 static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
1197 				void *data)
1198 {
1199 	struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
1200 
1201 	schedule_work(&policy->update);
1202 	return 0;
1203 }
1204 
cpufreq_policy_put_kobj(struct cpufreq_policy * policy)1205 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1206 {
1207 	struct kobject *kobj;
1208 	struct completion *cmp;
1209 
1210 	down_write(&policy->rwsem);
1211 	cpufreq_stats_free_table(policy);
1212 	kobj = &policy->kobj;
1213 	cmp = &policy->kobj_unregister;
1214 	up_write(&policy->rwsem);
1215 	kobject_put(kobj);
1216 
1217 	/*
1218 	 * We need to make sure that the underlying kobj is
1219 	 * actually not referenced anymore by anybody before we
1220 	 * proceed with unloading.
1221 	 */
1222 	pr_debug("waiting for dropping of refcount\n");
1223 	wait_for_completion(cmp);
1224 	pr_debug("wait complete\n");
1225 }
1226 
cpufreq_policy_alloc(unsigned int cpu)1227 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1228 {
1229 	struct cpufreq_policy *policy;
1230 	struct device *dev = get_cpu_device(cpu);
1231 	int ret;
1232 
1233 	if (!dev)
1234 		return NULL;
1235 
1236 	policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1237 	if (!policy)
1238 		return NULL;
1239 
1240 	if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1241 		goto err_free_policy;
1242 
1243 	if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1244 		goto err_free_cpumask;
1245 
1246 	if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1247 		goto err_free_rcpumask;
1248 
1249 	init_completion(&policy->kobj_unregister);
1250 	ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1251 				   cpufreq_global_kobject, "policy%u", cpu);
1252 	if (ret) {
1253 		dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
1254 		/*
1255 		 * The entire policy object will be freed below, but the extra
1256 		 * memory allocated for the kobject name needs to be freed by
1257 		 * releasing the kobject.
1258 		 */
1259 		kobject_put(&policy->kobj);
1260 		goto err_free_real_cpus;
1261 	}
1262 
1263 	freq_constraints_init(&policy->constraints);
1264 
1265 	policy->nb_min.notifier_call = cpufreq_notifier_min;
1266 	policy->nb_max.notifier_call = cpufreq_notifier_max;
1267 
1268 	ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,
1269 				    &policy->nb_min);
1270 	if (ret) {
1271 		dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n",
1272 			ret, cpu);
1273 		goto err_kobj_remove;
1274 	}
1275 
1276 	ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,
1277 				    &policy->nb_max);
1278 	if (ret) {
1279 		dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n",
1280 			ret, cpu);
1281 		goto err_min_qos_notifier;
1282 	}
1283 
1284 	INIT_LIST_HEAD(&policy->policy_list);
1285 	init_rwsem(&policy->rwsem);
1286 	spin_lock_init(&policy->transition_lock);
1287 	init_waitqueue_head(&policy->transition_wait);
1288 	INIT_WORK(&policy->update, handle_update);
1289 
1290 	policy->cpu = cpu;
1291 	return policy;
1292 
1293 err_min_qos_notifier:
1294 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1295 				 &policy->nb_min);
1296 err_kobj_remove:
1297 	cpufreq_policy_put_kobj(policy);
1298 err_free_real_cpus:
1299 	free_cpumask_var(policy->real_cpus);
1300 err_free_rcpumask:
1301 	free_cpumask_var(policy->related_cpus);
1302 err_free_cpumask:
1303 	free_cpumask_var(policy->cpus);
1304 err_free_policy:
1305 	kfree(policy);
1306 
1307 	return NULL;
1308 }
1309 
cpufreq_policy_free(struct cpufreq_policy * policy)1310 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1311 {
1312 	unsigned long flags;
1313 	int cpu;
1314 
1315 	/*
1316 	 * The callers must ensure the policy is inactive by now, to avoid any
1317 	 * races with show()/store() callbacks.
1318 	 */
1319 	if (unlikely(!policy_is_inactive(policy)))
1320 		pr_warn("%s: Freeing active policy\n", __func__);
1321 
1322 	/* Remove policy from list */
1323 	write_lock_irqsave(&cpufreq_driver_lock, flags);
1324 	list_del(&policy->policy_list);
1325 
1326 	for_each_cpu(cpu, policy->related_cpus)
1327 		per_cpu(cpufreq_cpu_data, cpu) = NULL;
1328 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1329 
1330 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,
1331 				 &policy->nb_max);
1332 	freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1333 				 &policy->nb_min);
1334 
1335 	/* Cancel any pending policy->update work before freeing the policy. */
1336 	cancel_work_sync(&policy->update);
1337 
1338 	if (policy->max_freq_req) {
1339 		/*
1340 		 * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY
1341 		 * notification, since CPUFREQ_CREATE_POLICY notification was
1342 		 * sent after adding max_freq_req earlier.
1343 		 */
1344 		blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1345 					     CPUFREQ_REMOVE_POLICY, policy);
1346 		freq_qos_remove_request(policy->max_freq_req);
1347 	}
1348 
1349 	freq_qos_remove_request(policy->min_freq_req);
1350 	kfree(policy->min_freq_req);
1351 
1352 	cpufreq_policy_put_kobj(policy);
1353 	free_cpumask_var(policy->real_cpus);
1354 	free_cpumask_var(policy->related_cpus);
1355 	free_cpumask_var(policy->cpus);
1356 	kfree(policy);
1357 }
1358 
cpufreq_online(unsigned int cpu)1359 static int cpufreq_online(unsigned int cpu)
1360 {
1361 	struct cpufreq_policy *policy;
1362 	bool new_policy;
1363 	unsigned long flags;
1364 	unsigned int j;
1365 	int ret;
1366 
1367 	pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1368 
1369 	/* Check if this CPU already has a policy to manage it */
1370 	policy = per_cpu(cpufreq_cpu_data, cpu);
1371 	if (policy) {
1372 		WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1373 		if (!policy_is_inactive(policy))
1374 			return cpufreq_add_policy_cpu(policy, cpu);
1375 
1376 		/* This is the only online CPU for the policy.  Start over. */
1377 		new_policy = false;
1378 		down_write(&policy->rwsem);
1379 		policy->cpu = cpu;
1380 		policy->governor = NULL;
1381 	} else {
1382 		new_policy = true;
1383 		policy = cpufreq_policy_alloc(cpu);
1384 		if (!policy)
1385 			return -ENOMEM;
1386 		down_write(&policy->rwsem);
1387 	}
1388 
1389 	if (!new_policy && cpufreq_driver->online) {
1390 		/* Recover policy->cpus using related_cpus */
1391 		cpumask_copy(policy->cpus, policy->related_cpus);
1392 
1393 		ret = cpufreq_driver->online(policy);
1394 		if (ret) {
1395 			pr_debug("%s: %d: initialization failed\n", __func__,
1396 				 __LINE__);
1397 			goto out_exit_policy;
1398 		}
1399 	} else {
1400 		cpumask_copy(policy->cpus, cpumask_of(cpu));
1401 
1402 		/*
1403 		 * Call driver. From then on the cpufreq must be able
1404 		 * to accept all calls to ->verify and ->setpolicy for this CPU.
1405 		 */
1406 		ret = cpufreq_driver->init(policy);
1407 		if (ret) {
1408 			pr_debug("%s: %d: initialization failed\n", __func__,
1409 				 __LINE__);
1410 			goto out_free_policy;
1411 		}
1412 
1413 		/*
1414 		 * The initialization has succeeded and the policy is online.
1415 		 * If there is a problem with its frequency table, take it
1416 		 * offline and drop it.
1417 		 */
1418 		ret = cpufreq_table_validate_and_sort(policy);
1419 		if (ret)
1420 			goto out_offline_policy;
1421 
1422 		/* related_cpus should at least include policy->cpus. */
1423 		cpumask_copy(policy->related_cpus, policy->cpus);
1424 	}
1425 
1426 	/*
1427 	 * affected cpus must always be the one, which are online. We aren't
1428 	 * managing offline cpus here.
1429 	 */
1430 	cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1431 
1432 	if (new_policy) {
1433 		for_each_cpu(j, policy->related_cpus) {
1434 			per_cpu(cpufreq_cpu_data, j) = policy;
1435 			add_cpu_dev_symlink(policy, j, get_cpu_device(j));
1436 		}
1437 
1438 		policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),
1439 					       GFP_KERNEL);
1440 		if (!policy->min_freq_req) {
1441 			ret = -ENOMEM;
1442 			goto out_destroy_policy;
1443 		}
1444 
1445 		ret = freq_qos_add_request(&policy->constraints,
1446 					   policy->min_freq_req, FREQ_QOS_MIN,
1447 					   FREQ_QOS_MIN_DEFAULT_VALUE);
1448 		if (ret < 0) {
1449 			/*
1450 			 * So we don't call freq_qos_remove_request() for an
1451 			 * uninitialized request.
1452 			 */
1453 			kfree(policy->min_freq_req);
1454 			policy->min_freq_req = NULL;
1455 			goto out_destroy_policy;
1456 		}
1457 
1458 		/*
1459 		 * This must be initialized right here to avoid calling
1460 		 * freq_qos_remove_request() on uninitialized request in case
1461 		 * of errors.
1462 		 */
1463 		policy->max_freq_req = policy->min_freq_req + 1;
1464 
1465 		ret = freq_qos_add_request(&policy->constraints,
1466 					   policy->max_freq_req, FREQ_QOS_MAX,
1467 					   FREQ_QOS_MAX_DEFAULT_VALUE);
1468 		if (ret < 0) {
1469 			policy->max_freq_req = NULL;
1470 			goto out_destroy_policy;
1471 		}
1472 
1473 		blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1474 				CPUFREQ_CREATE_POLICY, policy);
1475 	} else {
1476 		ret = freq_qos_update_request(policy->max_freq_req, policy->max);
1477 		if (ret < 0)
1478 			goto out_destroy_policy;
1479 	}
1480 
1481 	if (cpufreq_driver->get && has_target()) {
1482 		policy->cur = cpufreq_driver->get(policy->cpu);
1483 		if (!policy->cur) {
1484 			ret = -EIO;
1485 			pr_err("%s: ->get() failed\n", __func__);
1486 			goto out_destroy_policy;
1487 		}
1488 	}
1489 
1490 	/*
1491 	 * Sometimes boot loaders set CPU frequency to a value outside of
1492 	 * frequency table present with cpufreq core. In such cases CPU might be
1493 	 * unstable if it has to run on that frequency for long duration of time
1494 	 * and so its better to set it to a frequency which is specified in
1495 	 * freq-table. This also makes cpufreq stats inconsistent as
1496 	 * cpufreq-stats would fail to register because current frequency of CPU
1497 	 * isn't found in freq-table.
1498 	 *
1499 	 * Because we don't want this change to effect boot process badly, we go
1500 	 * for the next freq which is >= policy->cur ('cur' must be set by now,
1501 	 * otherwise we will end up setting freq to lowest of the table as 'cur'
1502 	 * is initialized to zero).
1503 	 *
1504 	 * We are passing target-freq as "policy->cur - 1" otherwise
1505 	 * __cpufreq_driver_target() would simply fail, as policy->cur will be
1506 	 * equal to target-freq.
1507 	 */
1508 	if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1509 	    && has_target()) {
1510 		unsigned int old_freq = policy->cur;
1511 
1512 		/* Are we running at unknown frequency ? */
1513 		ret = cpufreq_frequency_table_get_index(policy, old_freq);
1514 		if (ret == -EINVAL) {
1515 			ret = __cpufreq_driver_target(policy, old_freq - 1,
1516 						      CPUFREQ_RELATION_L);
1517 
1518 			/*
1519 			 * Reaching here after boot in a few seconds may not
1520 			 * mean that system will remain stable at "unknown"
1521 			 * frequency for longer duration. Hence, a BUG_ON().
1522 			 */
1523 			BUG_ON(ret);
1524 			pr_info("%s: CPU%d: Running at unlisted initial frequency: %u kHz, changing to: %u kHz\n",
1525 				__func__, policy->cpu, old_freq, policy->cur);
1526 		}
1527 	}
1528 
1529 	if (new_policy) {
1530 		ret = cpufreq_add_dev_interface(policy);
1531 		if (ret)
1532 			goto out_destroy_policy;
1533 
1534 		cpufreq_stats_create_table(policy);
1535 
1536 		write_lock_irqsave(&cpufreq_driver_lock, flags);
1537 		list_add(&policy->policy_list, &cpufreq_policy_list);
1538 		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1539 
1540 		/*
1541 		 * Register with the energy model before
1542 		 * em_rebuild_sched_domains() is called, which will result
1543 		 * in rebuilding of the sched domains, which should only be done
1544 		 * once the energy model is properly initialized for the policy
1545 		 * first.
1546 		 *
1547 		 * Also, this should be called before the policy is registered
1548 		 * with cooling framework.
1549 		 */
1550 		if (cpufreq_driver->register_em)
1551 			cpufreq_driver->register_em(policy);
1552 	}
1553 
1554 	ret = cpufreq_init_policy(policy);
1555 	if (ret) {
1556 		pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1557 		       __func__, cpu, ret);
1558 		goto out_destroy_policy;
1559 	}
1560 
1561 	up_write(&policy->rwsem);
1562 
1563 	kobject_uevent(&policy->kobj, KOBJ_ADD);
1564 
1565 	/* Callback for handling stuff after policy is ready */
1566 	if (cpufreq_driver->ready)
1567 		cpufreq_driver->ready(policy);
1568 
1569 	/* Register cpufreq cooling only for a new policy */
1570 	if (new_policy && cpufreq_thermal_control_enabled(cpufreq_driver))
1571 		policy->cdev = of_cpufreq_cooling_register(policy);
1572 
1573 	/* Let the per-policy boost flag mirror the cpufreq_driver boost during init */
1574 	if (cpufreq_driver->set_boost &&
1575 	    policy->boost_enabled != cpufreq_boost_enabled()) {
1576 		policy->boost_enabled = cpufreq_boost_enabled();
1577 		ret = cpufreq_driver->set_boost(policy, policy->boost_enabled);
1578 		if (ret) {
1579 			/* If the set_boost fails, the online operation is not affected */
1580 			pr_info("%s: CPU%d: Cannot %s BOOST\n", __func__, policy->cpu,
1581 				policy->boost_enabled ? "enable" : "disable");
1582 			policy->boost_enabled = !policy->boost_enabled;
1583 		}
1584 	}
1585 
1586 	pr_debug("initialization complete\n");
1587 
1588 	return 0;
1589 
1590 out_destroy_policy:
1591 	for_each_cpu(j, policy->real_cpus)
1592 		remove_cpu_dev_symlink(policy, j, get_cpu_device(j));
1593 
1594 out_offline_policy:
1595 	if (cpufreq_driver->offline)
1596 		cpufreq_driver->offline(policy);
1597 
1598 out_exit_policy:
1599 	if (cpufreq_driver->exit)
1600 		cpufreq_driver->exit(policy);
1601 
1602 out_free_policy:
1603 	cpumask_clear(policy->cpus);
1604 	up_write(&policy->rwsem);
1605 
1606 	cpufreq_policy_free(policy);
1607 	return ret;
1608 }
1609 
1610 /**
1611  * cpufreq_add_dev - the cpufreq interface for a CPU device.
1612  * @dev: CPU device.
1613  * @sif: Subsystem interface structure pointer (not used)
1614  */
cpufreq_add_dev(struct device * dev,struct subsys_interface * sif)1615 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1616 {
1617 	struct cpufreq_policy *policy;
1618 	unsigned cpu = dev->id;
1619 	int ret;
1620 
1621 	dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1622 
1623 	if (cpu_online(cpu)) {
1624 		ret = cpufreq_online(cpu);
1625 		if (ret)
1626 			return ret;
1627 	}
1628 
1629 	/* Create sysfs link on CPU registration */
1630 	policy = per_cpu(cpufreq_cpu_data, cpu);
1631 	if (policy)
1632 		add_cpu_dev_symlink(policy, cpu, dev);
1633 
1634 	return 0;
1635 }
1636 
__cpufreq_offline(unsigned int cpu,struct cpufreq_policy * policy)1637 static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy)
1638 {
1639 	int ret;
1640 
1641 	if (has_target())
1642 		cpufreq_stop_governor(policy);
1643 
1644 	cpumask_clear_cpu(cpu, policy->cpus);
1645 
1646 	if (!policy_is_inactive(policy)) {
1647 		/* Nominate a new CPU if necessary. */
1648 		if (cpu == policy->cpu)
1649 			policy->cpu = cpumask_any(policy->cpus);
1650 
1651 		/* Start the governor again for the active policy. */
1652 		if (has_target()) {
1653 			ret = cpufreq_start_governor(policy);
1654 			if (ret)
1655 				pr_err("%s: Failed to start governor\n", __func__);
1656 		}
1657 
1658 		return;
1659 	}
1660 
1661 	if (has_target())
1662 		strscpy(policy->last_governor, policy->governor->name,
1663 			CPUFREQ_NAME_LEN);
1664 	else
1665 		policy->last_policy = policy->policy;
1666 
1667 	if (has_target())
1668 		cpufreq_exit_governor(policy);
1669 
1670 	/*
1671 	 * Perform the ->offline() during light-weight tear-down, as
1672 	 * that allows fast recovery when the CPU comes back.
1673 	 */
1674 	if (cpufreq_driver->offline) {
1675 		cpufreq_driver->offline(policy);
1676 		return;
1677 	}
1678 
1679 	if (cpufreq_driver->exit)
1680 		cpufreq_driver->exit(policy);
1681 
1682 	policy->freq_table = NULL;
1683 }
1684 
cpufreq_offline(unsigned int cpu)1685 static int cpufreq_offline(unsigned int cpu)
1686 {
1687 	struct cpufreq_policy *policy;
1688 
1689 	pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1690 
1691 	policy = cpufreq_cpu_get_raw(cpu);
1692 	if (!policy) {
1693 		pr_debug("%s: No cpu_data found\n", __func__);
1694 		return 0;
1695 	}
1696 
1697 	down_write(&policy->rwsem);
1698 
1699 	__cpufreq_offline(cpu, policy);
1700 
1701 	up_write(&policy->rwsem);
1702 	return 0;
1703 }
1704 
1705 /*
1706  * cpufreq_remove_dev - remove a CPU device
1707  *
1708  * Removes the cpufreq interface for a CPU device.
1709  */
cpufreq_remove_dev(struct device * dev,struct subsys_interface * sif)1710 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1711 {
1712 	unsigned int cpu = dev->id;
1713 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1714 
1715 	if (!policy)
1716 		return;
1717 
1718 	down_write(&policy->rwsem);
1719 
1720 	if (cpu_online(cpu))
1721 		__cpufreq_offline(cpu, policy);
1722 
1723 	remove_cpu_dev_symlink(policy, cpu, dev);
1724 
1725 	if (!cpumask_empty(policy->real_cpus)) {
1726 		up_write(&policy->rwsem);
1727 		return;
1728 	}
1729 
1730 	/*
1731 	 * Unregister cpufreq cooling once all the CPUs of the policy are
1732 	 * removed.
1733 	 */
1734 	if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
1735 		cpufreq_cooling_unregister(policy->cdev);
1736 		policy->cdev = NULL;
1737 	}
1738 
1739 	/* We did light-weight exit earlier, do full tear down now */
1740 	if (cpufreq_driver->offline && cpufreq_driver->exit)
1741 		cpufreq_driver->exit(policy);
1742 
1743 	up_write(&policy->rwsem);
1744 
1745 	cpufreq_policy_free(policy);
1746 }
1747 
1748 /**
1749  * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference.
1750  * @policy: Policy managing CPUs.
1751  * @new_freq: New CPU frequency.
1752  *
1753  * Adjust to the current frequency first and clean up later by either calling
1754  * cpufreq_update_policy(), or scheduling handle_update().
1755  */
cpufreq_out_of_sync(struct cpufreq_policy * policy,unsigned int new_freq)1756 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1757 				unsigned int new_freq)
1758 {
1759 	struct cpufreq_freqs freqs;
1760 
1761 	pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1762 		 policy->cur, new_freq);
1763 
1764 	freqs.old = policy->cur;
1765 	freqs.new = new_freq;
1766 
1767 	cpufreq_freq_transition_begin(policy, &freqs);
1768 	cpufreq_freq_transition_end(policy, &freqs, 0);
1769 }
1770 
cpufreq_verify_current_freq(struct cpufreq_policy * policy,bool update)1771 static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
1772 {
1773 	unsigned int new_freq;
1774 
1775 	new_freq = cpufreq_driver->get(policy->cpu);
1776 	if (!new_freq)
1777 		return 0;
1778 
1779 	/*
1780 	 * If fast frequency switching is used with the given policy, the check
1781 	 * against policy->cur is pointless, so skip it in that case.
1782 	 */
1783 	if (policy->fast_switch_enabled || !has_target())
1784 		return new_freq;
1785 
1786 	if (policy->cur != new_freq) {
1787 		/*
1788 		 * For some platforms, the frequency returned by hardware may be
1789 		 * slightly different from what is provided in the frequency
1790 		 * table, for example hardware may return 499 MHz instead of 500
1791 		 * MHz. In such cases it is better to avoid getting into
1792 		 * unnecessary frequency updates.
1793 		 */
1794 		if (abs(policy->cur - new_freq) < KHZ_PER_MHZ)
1795 			return policy->cur;
1796 
1797 		cpufreq_out_of_sync(policy, new_freq);
1798 		if (update)
1799 			schedule_work(&policy->update);
1800 	}
1801 
1802 	return new_freq;
1803 }
1804 
1805 /**
1806  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1807  * @cpu: CPU number
1808  *
1809  * This is the last known freq, without actually getting it from the driver.
1810  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1811  */
cpufreq_quick_get(unsigned int cpu)1812 unsigned int cpufreq_quick_get(unsigned int cpu)
1813 {
1814 	struct cpufreq_policy *policy;
1815 	unsigned int ret_freq = 0;
1816 	unsigned long flags;
1817 
1818 	read_lock_irqsave(&cpufreq_driver_lock, flags);
1819 
1820 	if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1821 		ret_freq = cpufreq_driver->get(cpu);
1822 		read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1823 		return ret_freq;
1824 	}
1825 
1826 	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1827 
1828 	policy = cpufreq_cpu_get(cpu);
1829 	if (policy) {
1830 		ret_freq = policy->cur;
1831 		cpufreq_cpu_put(policy);
1832 	}
1833 
1834 	return ret_freq;
1835 }
1836 EXPORT_SYMBOL(cpufreq_quick_get);
1837 
1838 /**
1839  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1840  * @cpu: CPU number
1841  *
1842  * Just return the max possible frequency for a given CPU.
1843  */
cpufreq_quick_get_max(unsigned int cpu)1844 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1845 {
1846 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1847 	unsigned int ret_freq = 0;
1848 
1849 	if (policy) {
1850 		ret_freq = policy->max;
1851 		cpufreq_cpu_put(policy);
1852 	}
1853 
1854 	return ret_freq;
1855 }
1856 EXPORT_SYMBOL(cpufreq_quick_get_max);
1857 
1858 /**
1859  * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
1860  * @cpu: CPU number
1861  *
1862  * The default return value is the max_freq field of cpuinfo.
1863  */
cpufreq_get_hw_max_freq(unsigned int cpu)1864 __weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
1865 {
1866 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1867 	unsigned int ret_freq = 0;
1868 
1869 	if (policy) {
1870 		ret_freq = policy->cpuinfo.max_freq;
1871 		cpufreq_cpu_put(policy);
1872 	}
1873 
1874 	return ret_freq;
1875 }
1876 EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
1877 
__cpufreq_get(struct cpufreq_policy * policy)1878 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1879 {
1880 	if (unlikely(policy_is_inactive(policy)))
1881 		return 0;
1882 
1883 	return cpufreq_verify_current_freq(policy, true);
1884 }
1885 
1886 /**
1887  * cpufreq_get - get the current CPU frequency (in kHz)
1888  * @cpu: CPU number
1889  *
1890  * Get the CPU current (static) CPU frequency
1891  */
cpufreq_get(unsigned int cpu)1892 unsigned int cpufreq_get(unsigned int cpu)
1893 {
1894 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1895 	unsigned int ret_freq = 0;
1896 
1897 	if (policy) {
1898 		down_read(&policy->rwsem);
1899 		if (cpufreq_driver->get)
1900 			ret_freq = __cpufreq_get(policy);
1901 		up_read(&policy->rwsem);
1902 
1903 		cpufreq_cpu_put(policy);
1904 	}
1905 
1906 	return ret_freq;
1907 }
1908 EXPORT_SYMBOL(cpufreq_get);
1909 
1910 static struct subsys_interface cpufreq_interface = {
1911 	.name		= "cpufreq",
1912 	.subsys		= &cpu_subsys,
1913 	.add_dev	= cpufreq_add_dev,
1914 	.remove_dev	= cpufreq_remove_dev,
1915 };
1916 
1917 /*
1918  * In case platform wants some specific frequency to be configured
1919  * during suspend..
1920  */
cpufreq_generic_suspend(struct cpufreq_policy * policy)1921 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1922 {
1923 	int ret;
1924 
1925 	if (!policy->suspend_freq) {
1926 		pr_debug("%s: suspend_freq not defined\n", __func__);
1927 		return 0;
1928 	}
1929 
1930 	pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1931 			policy->suspend_freq);
1932 
1933 	ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1934 			CPUFREQ_RELATION_H);
1935 	if (ret)
1936 		pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1937 				__func__, policy->suspend_freq, ret);
1938 
1939 	return ret;
1940 }
1941 EXPORT_SYMBOL(cpufreq_generic_suspend);
1942 
1943 /**
1944  * cpufreq_suspend() - Suspend CPUFreq governors.
1945  *
1946  * Called during system wide Suspend/Hibernate cycles for suspending governors
1947  * as some platforms can't change frequency after this point in suspend cycle.
1948  * Because some of the devices (like: i2c, regulators, etc) they use for
1949  * changing frequency are suspended quickly after this point.
1950  */
cpufreq_suspend(void)1951 void cpufreq_suspend(void)
1952 {
1953 	struct cpufreq_policy *policy;
1954 
1955 	if (!cpufreq_driver)
1956 		return;
1957 
1958 	if (!has_target() && !cpufreq_driver->suspend)
1959 		goto suspend;
1960 
1961 	pr_debug("%s: Suspending Governors\n", __func__);
1962 
1963 	for_each_active_policy(policy) {
1964 		if (has_target()) {
1965 			down_write(&policy->rwsem);
1966 			cpufreq_stop_governor(policy);
1967 			up_write(&policy->rwsem);
1968 		}
1969 
1970 		if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
1971 			pr_err("%s: Failed to suspend driver: %s\n", __func__,
1972 				cpufreq_driver->name);
1973 	}
1974 
1975 suspend:
1976 	cpufreq_suspended = true;
1977 }
1978 
1979 /**
1980  * cpufreq_resume() - Resume CPUFreq governors.
1981  *
1982  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1983  * are suspended with cpufreq_suspend().
1984  */
cpufreq_resume(void)1985 void cpufreq_resume(void)
1986 {
1987 	struct cpufreq_policy *policy;
1988 	int ret;
1989 
1990 	if (!cpufreq_driver)
1991 		return;
1992 
1993 	if (unlikely(!cpufreq_suspended))
1994 		return;
1995 
1996 	cpufreq_suspended = false;
1997 
1998 	if (!has_target() && !cpufreq_driver->resume)
1999 		return;
2000 
2001 	pr_debug("%s: Resuming Governors\n", __func__);
2002 
2003 	for_each_active_policy(policy) {
2004 		if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
2005 			pr_err("%s: Failed to resume driver: %s\n", __func__,
2006 				cpufreq_driver->name);
2007 		} else if (has_target()) {
2008 			down_write(&policy->rwsem);
2009 			ret = cpufreq_start_governor(policy);
2010 			up_write(&policy->rwsem);
2011 
2012 			if (ret)
2013 				pr_err("%s: Failed to start governor for CPU%u's policy\n",
2014 				       __func__, policy->cpu);
2015 		}
2016 	}
2017 }
2018 
2019 /**
2020  * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones.
2021  * @flags: Flags to test against the current cpufreq driver's flags.
2022  *
2023  * Assumes that the driver is there, so callers must ensure that this is the
2024  * case.
2025  */
cpufreq_driver_test_flags(u16 flags)2026 bool cpufreq_driver_test_flags(u16 flags)
2027 {
2028 	return !!(cpufreq_driver->flags & flags);
2029 }
2030 
2031 /**
2032  * cpufreq_get_current_driver - Return the current driver's name.
2033  *
2034  * Return the name string of the currently registered cpufreq driver or NULL if
2035  * none.
2036  */
cpufreq_get_current_driver(void)2037 const char *cpufreq_get_current_driver(void)
2038 {
2039 	if (cpufreq_driver)
2040 		return cpufreq_driver->name;
2041 
2042 	return NULL;
2043 }
2044 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
2045 
2046 /**
2047  * cpufreq_get_driver_data - Return current driver data.
2048  *
2049  * Return the private data of the currently registered cpufreq driver, or NULL
2050  * if no cpufreq driver has been registered.
2051  */
cpufreq_get_driver_data(void)2052 void *cpufreq_get_driver_data(void)
2053 {
2054 	if (cpufreq_driver)
2055 		return cpufreq_driver->driver_data;
2056 
2057 	return NULL;
2058 }
2059 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
2060 
2061 /*********************************************************************
2062  *                     NOTIFIER LISTS INTERFACE                      *
2063  *********************************************************************/
2064 
2065 /**
2066  * cpufreq_register_notifier - Register a notifier with cpufreq.
2067  * @nb: notifier function to register.
2068  * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2069  *
2070  * Add a notifier to one of two lists: either a list of notifiers that run on
2071  * clock rate changes (once before and once after every transition), or a list
2072  * of notifiers that ron on cpufreq policy changes.
2073  *
2074  * This function may sleep and it has the same return values as
2075  * blocking_notifier_chain_register().
2076  */
cpufreq_register_notifier(struct notifier_block * nb,unsigned int list)2077 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
2078 {
2079 	int ret;
2080 
2081 	if (cpufreq_disabled())
2082 		return -EINVAL;
2083 
2084 	switch (list) {
2085 	case CPUFREQ_TRANSITION_NOTIFIER:
2086 		mutex_lock(&cpufreq_fast_switch_lock);
2087 
2088 		if (cpufreq_fast_switch_count > 0) {
2089 			mutex_unlock(&cpufreq_fast_switch_lock);
2090 			return -EBUSY;
2091 		}
2092 		ret = srcu_notifier_chain_register(
2093 				&cpufreq_transition_notifier_list, nb);
2094 		if (!ret)
2095 			cpufreq_fast_switch_count--;
2096 
2097 		mutex_unlock(&cpufreq_fast_switch_lock);
2098 		break;
2099 	case CPUFREQ_POLICY_NOTIFIER:
2100 		ret = blocking_notifier_chain_register(
2101 				&cpufreq_policy_notifier_list, nb);
2102 		break;
2103 	default:
2104 		ret = -EINVAL;
2105 	}
2106 
2107 	return ret;
2108 }
2109 EXPORT_SYMBOL(cpufreq_register_notifier);
2110 
2111 /**
2112  * cpufreq_unregister_notifier - Unregister a notifier from cpufreq.
2113  * @nb: notifier block to be unregistered.
2114  * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2115  *
2116  * Remove a notifier from one of the cpufreq notifier lists.
2117  *
2118  * This function may sleep and it has the same return values as
2119  * blocking_notifier_chain_unregister().
2120  */
cpufreq_unregister_notifier(struct notifier_block * nb,unsigned int list)2121 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
2122 {
2123 	int ret;
2124 
2125 	if (cpufreq_disabled())
2126 		return -EINVAL;
2127 
2128 	switch (list) {
2129 	case CPUFREQ_TRANSITION_NOTIFIER:
2130 		mutex_lock(&cpufreq_fast_switch_lock);
2131 
2132 		ret = srcu_notifier_chain_unregister(
2133 				&cpufreq_transition_notifier_list, nb);
2134 		if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
2135 			cpufreq_fast_switch_count++;
2136 
2137 		mutex_unlock(&cpufreq_fast_switch_lock);
2138 		break;
2139 	case CPUFREQ_POLICY_NOTIFIER:
2140 		ret = blocking_notifier_chain_unregister(
2141 				&cpufreq_policy_notifier_list, nb);
2142 		break;
2143 	default:
2144 		ret = -EINVAL;
2145 	}
2146 
2147 	return ret;
2148 }
2149 EXPORT_SYMBOL(cpufreq_unregister_notifier);
2150 
2151 
2152 /*********************************************************************
2153  *                              GOVERNORS                            *
2154  *********************************************************************/
2155 
2156 /**
2157  * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
2158  * @policy: cpufreq policy to switch the frequency for.
2159  * @target_freq: New frequency to set (may be approximate).
2160  *
2161  * Carry out a fast frequency switch without sleeping.
2162  *
2163  * The driver's ->fast_switch() callback invoked by this function must be
2164  * suitable for being called from within RCU-sched read-side critical sections
2165  * and it is expected to select the minimum available frequency greater than or
2166  * equal to @target_freq (CPUFREQ_RELATION_L).
2167  *
2168  * This function must not be called if policy->fast_switch_enabled is unset.
2169  *
2170  * Governors calling this function must guarantee that it will never be invoked
2171  * twice in parallel for the same policy and that it will never be called in
2172  * parallel with either ->target() or ->target_index() for the same policy.
2173  *
2174  * Returns the actual frequency set for the CPU.
2175  *
2176  * If 0 is returned by the driver's ->fast_switch() callback to indicate an
2177  * error condition, the hardware configuration must be preserved.
2178  */
cpufreq_driver_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)2179 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
2180 					unsigned int target_freq)
2181 {
2182 	unsigned int freq;
2183 	int cpu;
2184 
2185 	target_freq = clamp_val(target_freq, policy->min, policy->max);
2186 	freq = cpufreq_driver->fast_switch(policy, target_freq);
2187 
2188 	if (!freq)
2189 		return 0;
2190 
2191 	policy->cur = freq;
2192 	arch_set_freq_scale(policy->related_cpus, freq,
2193 			    arch_scale_freq_ref(policy->cpu));
2194 	cpufreq_stats_record_transition(policy, freq);
2195 
2196 	if (trace_cpu_frequency_enabled()) {
2197 		for_each_cpu(cpu, policy->cpus)
2198 			trace_cpu_frequency(freq, cpu);
2199 	}
2200 
2201 	return freq;
2202 }
2203 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
2204 
2205 /**
2206  * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go.
2207  * @cpu: Target CPU.
2208  * @min_perf: Minimum (required) performance level (units of @capacity).
2209  * @target_perf: Target (desired) performance level (units of @capacity).
2210  * @capacity: Capacity of the target CPU.
2211  *
2212  * Carry out a fast performance level switch of @cpu without sleeping.
2213  *
2214  * The driver's ->adjust_perf() callback invoked by this function must be
2215  * suitable for being called from within RCU-sched read-side critical sections
2216  * and it is expected to select a suitable performance level equal to or above
2217  * @min_perf and preferably equal to or below @target_perf.
2218  *
2219  * This function must not be called if policy->fast_switch_enabled is unset.
2220  *
2221  * Governors calling this function must guarantee that it will never be invoked
2222  * twice in parallel for the same CPU and that it will never be called in
2223  * parallel with either ->target() or ->target_index() or ->fast_switch() for
2224  * the same CPU.
2225  */
cpufreq_driver_adjust_perf(unsigned int cpu,unsigned long min_perf,unsigned long target_perf,unsigned long capacity)2226 void cpufreq_driver_adjust_perf(unsigned int cpu,
2227 				 unsigned long min_perf,
2228 				 unsigned long target_perf,
2229 				 unsigned long capacity)
2230 {
2231 	cpufreq_driver->adjust_perf(cpu, min_perf, target_perf, capacity);
2232 }
2233 
2234 /**
2235  * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback.
2236  *
2237  * Return 'true' if the ->adjust_perf callback is present for the
2238  * current driver or 'false' otherwise.
2239  */
cpufreq_driver_has_adjust_perf(void)2240 bool cpufreq_driver_has_adjust_perf(void)
2241 {
2242 	return !!cpufreq_driver->adjust_perf;
2243 }
2244 
2245 /* Must set freqs->new to intermediate frequency */
__target_intermediate(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,int index)2246 static int __target_intermediate(struct cpufreq_policy *policy,
2247 				 struct cpufreq_freqs *freqs, int index)
2248 {
2249 	int ret;
2250 
2251 	freqs->new = cpufreq_driver->get_intermediate(policy, index);
2252 
2253 	/* We don't need to switch to intermediate freq */
2254 	if (!freqs->new)
2255 		return 0;
2256 
2257 	pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
2258 		 __func__, policy->cpu, freqs->old, freqs->new);
2259 
2260 	cpufreq_freq_transition_begin(policy, freqs);
2261 	ret = cpufreq_driver->target_intermediate(policy, index);
2262 	cpufreq_freq_transition_end(policy, freqs, ret);
2263 
2264 	if (ret)
2265 		pr_err("%s: Failed to change to intermediate frequency: %d\n",
2266 		       __func__, ret);
2267 
2268 	return ret;
2269 }
2270 
__target_index(struct cpufreq_policy * policy,int index)2271 static int __target_index(struct cpufreq_policy *policy, int index)
2272 {
2273 	struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
2274 	unsigned int restore_freq, intermediate_freq = 0;
2275 	unsigned int newfreq = policy->freq_table[index].frequency;
2276 	int retval = -EINVAL;
2277 	bool notify;
2278 
2279 	if (newfreq == policy->cur)
2280 		return 0;
2281 
2282 	/* Save last value to restore later on errors */
2283 	restore_freq = policy->cur;
2284 
2285 	notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
2286 	if (notify) {
2287 		/* Handle switching to intermediate frequency */
2288 		if (cpufreq_driver->get_intermediate) {
2289 			retval = __target_intermediate(policy, &freqs, index);
2290 			if (retval)
2291 				return retval;
2292 
2293 			intermediate_freq = freqs.new;
2294 			/* Set old freq to intermediate */
2295 			if (intermediate_freq)
2296 				freqs.old = freqs.new;
2297 		}
2298 
2299 		freqs.new = newfreq;
2300 		pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
2301 			 __func__, policy->cpu, freqs.old, freqs.new);
2302 
2303 		cpufreq_freq_transition_begin(policy, &freqs);
2304 	}
2305 
2306 	retval = cpufreq_driver->target_index(policy, index);
2307 	if (retval)
2308 		pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
2309 		       retval);
2310 
2311 	if (notify) {
2312 		cpufreq_freq_transition_end(policy, &freqs, retval);
2313 
2314 		/*
2315 		 * Failed after setting to intermediate freq? Driver should have
2316 		 * reverted back to initial frequency and so should we. Check
2317 		 * here for intermediate_freq instead of get_intermediate, in
2318 		 * case we haven't switched to intermediate freq at all.
2319 		 */
2320 		if (unlikely(retval && intermediate_freq)) {
2321 			freqs.old = intermediate_freq;
2322 			freqs.new = restore_freq;
2323 			cpufreq_freq_transition_begin(policy, &freqs);
2324 			cpufreq_freq_transition_end(policy, &freqs, 0);
2325 		}
2326 	}
2327 
2328 	return retval;
2329 }
2330 
__cpufreq_driver_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)2331 int __cpufreq_driver_target(struct cpufreq_policy *policy,
2332 			    unsigned int target_freq,
2333 			    unsigned int relation)
2334 {
2335 	unsigned int old_target_freq = target_freq;
2336 
2337 	if (cpufreq_disabled())
2338 		return -ENODEV;
2339 
2340 	target_freq = __resolve_freq(policy, target_freq, relation);
2341 
2342 	pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
2343 		 policy->cpu, target_freq, relation, old_target_freq);
2344 
2345 	/*
2346 	 * This might look like a redundant call as we are checking it again
2347 	 * after finding index. But it is left intentionally for cases where
2348 	 * exactly same freq is called again and so we can save on few function
2349 	 * calls.
2350 	 */
2351 	if (target_freq == policy->cur &&
2352 	    !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS))
2353 		return 0;
2354 
2355 	if (cpufreq_driver->target) {
2356 		/*
2357 		 * If the driver hasn't setup a single inefficient frequency,
2358 		 * it's unlikely it knows how to decode CPUFREQ_RELATION_E.
2359 		 */
2360 		if (!policy->efficiencies_available)
2361 			relation &= ~CPUFREQ_RELATION_E;
2362 
2363 		return cpufreq_driver->target(policy, target_freq, relation);
2364 	}
2365 
2366 	if (!cpufreq_driver->target_index)
2367 		return -EINVAL;
2368 
2369 	return __target_index(policy, policy->cached_resolved_idx);
2370 }
2371 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2372 
cpufreq_driver_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)2373 int cpufreq_driver_target(struct cpufreq_policy *policy,
2374 			  unsigned int target_freq,
2375 			  unsigned int relation)
2376 {
2377 	int ret;
2378 
2379 	down_write(&policy->rwsem);
2380 
2381 	ret = __cpufreq_driver_target(policy, target_freq, relation);
2382 
2383 	up_write(&policy->rwsem);
2384 
2385 	return ret;
2386 }
2387 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2388 
cpufreq_fallback_governor(void)2389 __weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2390 {
2391 	return NULL;
2392 }
2393 
cpufreq_init_governor(struct cpufreq_policy * policy)2394 static int cpufreq_init_governor(struct cpufreq_policy *policy)
2395 {
2396 	int ret;
2397 
2398 	/* Don't start any governor operations if we are entering suspend */
2399 	if (cpufreq_suspended)
2400 		return 0;
2401 	/*
2402 	 * Governor might not be initiated here if ACPI _PPC changed
2403 	 * notification happened, so check it.
2404 	 */
2405 	if (!policy->governor)
2406 		return -EINVAL;
2407 
2408 	/* Platform doesn't want dynamic frequency switching ? */
2409 	if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING &&
2410 	    cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
2411 		struct cpufreq_governor *gov = cpufreq_fallback_governor();
2412 
2413 		if (gov) {
2414 			pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
2415 				policy->governor->name, gov->name);
2416 			policy->governor = gov;
2417 		} else {
2418 			return -EINVAL;
2419 		}
2420 	}
2421 
2422 	if (!try_module_get(policy->governor->owner))
2423 		return -EINVAL;
2424 
2425 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2426 
2427 	if (policy->governor->init) {
2428 		ret = policy->governor->init(policy);
2429 		if (ret) {
2430 			module_put(policy->governor->owner);
2431 			return ret;
2432 		}
2433 	}
2434 
2435 	policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET);
2436 
2437 	return 0;
2438 }
2439 
cpufreq_exit_governor(struct cpufreq_policy * policy)2440 static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2441 {
2442 	if (cpufreq_suspended || !policy->governor)
2443 		return;
2444 
2445 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2446 
2447 	if (policy->governor->exit)
2448 		policy->governor->exit(policy);
2449 
2450 	module_put(policy->governor->owner);
2451 }
2452 
cpufreq_start_governor(struct cpufreq_policy * policy)2453 int cpufreq_start_governor(struct cpufreq_policy *policy)
2454 {
2455 	int ret;
2456 
2457 	if (cpufreq_suspended)
2458 		return 0;
2459 
2460 	if (!policy->governor)
2461 		return -EINVAL;
2462 
2463 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2464 
2465 	if (cpufreq_driver->get)
2466 		cpufreq_verify_current_freq(policy, false);
2467 
2468 	if (policy->governor->start) {
2469 		ret = policy->governor->start(policy);
2470 		if (ret)
2471 			return ret;
2472 	}
2473 
2474 	if (policy->governor->limits)
2475 		policy->governor->limits(policy);
2476 
2477 	return 0;
2478 }
2479 
cpufreq_stop_governor(struct cpufreq_policy * policy)2480 void cpufreq_stop_governor(struct cpufreq_policy *policy)
2481 {
2482 	if (cpufreq_suspended || !policy->governor)
2483 		return;
2484 
2485 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2486 
2487 	if (policy->governor->stop)
2488 		policy->governor->stop(policy);
2489 }
2490 
cpufreq_governor_limits(struct cpufreq_policy * policy)2491 static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2492 {
2493 	if (cpufreq_suspended || !policy->governor)
2494 		return;
2495 
2496 	pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2497 
2498 	if (policy->governor->limits)
2499 		policy->governor->limits(policy);
2500 }
2501 
cpufreq_register_governor(struct cpufreq_governor * governor)2502 int cpufreq_register_governor(struct cpufreq_governor *governor)
2503 {
2504 	int err;
2505 
2506 	if (!governor)
2507 		return -EINVAL;
2508 
2509 	if (cpufreq_disabled())
2510 		return -ENODEV;
2511 
2512 	mutex_lock(&cpufreq_governor_mutex);
2513 
2514 	err = -EBUSY;
2515 	if (!find_governor(governor->name)) {
2516 		err = 0;
2517 		list_add(&governor->governor_list, &cpufreq_governor_list);
2518 	}
2519 
2520 	mutex_unlock(&cpufreq_governor_mutex);
2521 	return err;
2522 }
2523 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2524 
cpufreq_unregister_governor(struct cpufreq_governor * governor)2525 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2526 {
2527 	struct cpufreq_policy *policy;
2528 	unsigned long flags;
2529 
2530 	if (!governor)
2531 		return;
2532 
2533 	if (cpufreq_disabled())
2534 		return;
2535 
2536 	/* clear last_governor for all inactive policies */
2537 	read_lock_irqsave(&cpufreq_driver_lock, flags);
2538 	for_each_inactive_policy(policy) {
2539 		if (!strcmp(policy->last_governor, governor->name)) {
2540 			policy->governor = NULL;
2541 			strcpy(policy->last_governor, "\0");
2542 		}
2543 	}
2544 	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2545 
2546 	mutex_lock(&cpufreq_governor_mutex);
2547 	list_del(&governor->governor_list);
2548 	mutex_unlock(&cpufreq_governor_mutex);
2549 }
2550 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2551 
2552 
2553 /*********************************************************************
2554  *                          POLICY INTERFACE                         *
2555  *********************************************************************/
2556 
2557 /**
2558  * cpufreq_get_policy - get the current cpufreq_policy
2559  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2560  *	is written
2561  * @cpu: CPU to find the policy for
2562  *
2563  * Reads the current cpufreq policy.
2564  */
cpufreq_get_policy(struct cpufreq_policy * policy,unsigned int cpu)2565 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2566 {
2567 	struct cpufreq_policy *cpu_policy;
2568 	if (!policy)
2569 		return -EINVAL;
2570 
2571 	cpu_policy = cpufreq_cpu_get(cpu);
2572 	if (!cpu_policy)
2573 		return -EINVAL;
2574 
2575 	memcpy(policy, cpu_policy, sizeof(*policy));
2576 
2577 	cpufreq_cpu_put(cpu_policy);
2578 	return 0;
2579 }
2580 EXPORT_SYMBOL(cpufreq_get_policy);
2581 
2582 DEFINE_PER_CPU(unsigned long, cpufreq_pressure);
2583 
2584 /**
2585  * cpufreq_update_pressure() - Update cpufreq pressure for CPUs
2586  * @policy: cpufreq policy of the CPUs.
2587  *
2588  * Update the value of cpufreq pressure for all @cpus in the policy.
2589  */
cpufreq_update_pressure(struct cpufreq_policy * policy)2590 static void cpufreq_update_pressure(struct cpufreq_policy *policy)
2591 {
2592 	unsigned long max_capacity, capped_freq, pressure;
2593 	u32 max_freq;
2594 	int cpu;
2595 
2596 	cpu = cpumask_first(policy->related_cpus);
2597 	max_freq = arch_scale_freq_ref(cpu);
2598 	capped_freq = policy->max;
2599 
2600 	/*
2601 	 * Handle properly the boost frequencies, which should simply clean
2602 	 * the cpufreq pressure value.
2603 	 */
2604 	if (max_freq <= capped_freq) {
2605 		pressure = 0;
2606 	} else {
2607 		max_capacity = arch_scale_cpu_capacity(cpu);
2608 		pressure = max_capacity -
2609 			   mult_frac(max_capacity, capped_freq, max_freq);
2610 	}
2611 
2612 	for_each_cpu(cpu, policy->related_cpus)
2613 		WRITE_ONCE(per_cpu(cpufreq_pressure, cpu), pressure);
2614 }
2615 
2616 /**
2617  * cpufreq_set_policy - Modify cpufreq policy parameters.
2618  * @policy: Policy object to modify.
2619  * @new_gov: Policy governor pointer.
2620  * @new_pol: Policy value (for drivers with built-in governors).
2621  *
2622  * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency
2623  * limits to be set for the policy, update @policy with the verified limits
2624  * values and either invoke the driver's ->setpolicy() callback (if present) or
2625  * carry out a governor update for @policy.  That is, run the current governor's
2626  * ->limits() callback (if @new_gov points to the same object as the one in
2627  * @policy) or replace the governor for @policy with @new_gov.
2628  *
2629  * The cpuinfo part of @policy is not updated by this function.
2630  */
cpufreq_set_policy(struct cpufreq_policy * policy,struct cpufreq_governor * new_gov,unsigned int new_pol)2631 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2632 			      struct cpufreq_governor *new_gov,
2633 			      unsigned int new_pol)
2634 {
2635 	struct cpufreq_policy_data new_data;
2636 	struct cpufreq_governor *old_gov;
2637 	int ret;
2638 
2639 	memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2640 	new_data.freq_table = policy->freq_table;
2641 	new_data.cpu = policy->cpu;
2642 	/*
2643 	 * PM QoS framework collects all the requests from users and provide us
2644 	 * the final aggregated value here.
2645 	 */
2646 	new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);
2647 	new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);
2648 
2649 	pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2650 		 new_data.cpu, new_data.min, new_data.max);
2651 
2652 	/*
2653 	 * Verify that the CPU speed can be set within these limits and make sure
2654 	 * that min <= max.
2655 	 */
2656 	ret = cpufreq_driver->verify(&new_data);
2657 	if (ret)
2658 		return ret;
2659 
2660 	/*
2661 	 * Resolve policy min/max to available frequencies. It ensures
2662 	 * no frequency resolution will neither overshoot the requested maximum
2663 	 * nor undershoot the requested minimum.
2664 	 */
2665 	policy->min = new_data.min;
2666 	policy->max = new_data.max;
2667 	policy->min = __resolve_freq(policy, policy->min, CPUFREQ_RELATION_L);
2668 	policy->max = __resolve_freq(policy, policy->max, CPUFREQ_RELATION_H);
2669 	trace_cpu_frequency_limits(policy);
2670 
2671 	cpufreq_update_pressure(policy);
2672 
2673 	policy->cached_target_freq = UINT_MAX;
2674 
2675 	pr_debug("new min and max freqs are %u - %u kHz\n",
2676 		 policy->min, policy->max);
2677 
2678 	if (cpufreq_driver->setpolicy) {
2679 		policy->policy = new_pol;
2680 		pr_debug("setting range\n");
2681 		return cpufreq_driver->setpolicy(policy);
2682 	}
2683 
2684 	if (new_gov == policy->governor) {
2685 		pr_debug("governor limits update\n");
2686 		cpufreq_governor_limits(policy);
2687 		return 0;
2688 	}
2689 
2690 	pr_debug("governor switch\n");
2691 
2692 	/* save old, working values */
2693 	old_gov = policy->governor;
2694 	/* end old governor */
2695 	if (old_gov) {
2696 		cpufreq_stop_governor(policy);
2697 		cpufreq_exit_governor(policy);
2698 	}
2699 
2700 	/* start new governor */
2701 	policy->governor = new_gov;
2702 	ret = cpufreq_init_governor(policy);
2703 	if (!ret) {
2704 		ret = cpufreq_start_governor(policy);
2705 		if (!ret) {
2706 			pr_debug("governor change\n");
2707 			return 0;
2708 		}
2709 		cpufreq_exit_governor(policy);
2710 	}
2711 
2712 	/* new governor failed, so re-start old one */
2713 	pr_debug("starting governor %s failed\n", policy->governor->name);
2714 	if (old_gov) {
2715 		policy->governor = old_gov;
2716 		if (cpufreq_init_governor(policy))
2717 			policy->governor = NULL;
2718 		else
2719 			cpufreq_start_governor(policy);
2720 	}
2721 
2722 	return ret;
2723 }
2724 
2725 /**
2726  * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
2727  * @cpu: CPU to re-evaluate the policy for.
2728  *
2729  * Update the current frequency for the cpufreq policy of @cpu and use
2730  * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
2731  * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
2732  * for the policy in question, among other things.
2733  */
cpufreq_update_policy(unsigned int cpu)2734 void cpufreq_update_policy(unsigned int cpu)
2735 {
2736 	struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
2737 
2738 	if (!policy)
2739 		return;
2740 
2741 	/*
2742 	 * BIOS might change freq behind our back
2743 	 * -> ask driver for current freq and notify governors about a change
2744 	 */
2745 	if (cpufreq_driver->get && has_target() &&
2746 	    (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
2747 		goto unlock;
2748 
2749 	refresh_frequency_limits(policy);
2750 
2751 unlock:
2752 	cpufreq_cpu_release(policy);
2753 }
2754 EXPORT_SYMBOL(cpufreq_update_policy);
2755 
2756 /**
2757  * cpufreq_update_limits - Update policy limits for a given CPU.
2758  * @cpu: CPU to update the policy limits for.
2759  *
2760  * Invoke the driver's ->update_limits callback if present or call
2761  * cpufreq_update_policy() for @cpu.
2762  */
cpufreq_update_limits(unsigned int cpu)2763 void cpufreq_update_limits(unsigned int cpu)
2764 {
2765 	if (cpufreq_driver->update_limits)
2766 		cpufreq_driver->update_limits(cpu);
2767 	else
2768 		cpufreq_update_policy(cpu);
2769 }
2770 EXPORT_SYMBOL_GPL(cpufreq_update_limits);
2771 
2772 /*********************************************************************
2773  *               BOOST						     *
2774  *********************************************************************/
cpufreq_boost_set_sw(struct cpufreq_policy * policy,int state)2775 static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
2776 {
2777 	int ret;
2778 
2779 	if (!policy->freq_table)
2780 		return -ENXIO;
2781 
2782 	ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table);
2783 	if (ret) {
2784 		pr_err("%s: Policy frequency update failed\n", __func__);
2785 		return ret;
2786 	}
2787 
2788 	ret = freq_qos_update_request(policy->max_freq_req, policy->max);
2789 	if (ret < 0)
2790 		return ret;
2791 
2792 	return 0;
2793 }
2794 
cpufreq_boost_trigger_state(int state)2795 int cpufreq_boost_trigger_state(int state)
2796 {
2797 	struct cpufreq_policy *policy;
2798 	unsigned long flags;
2799 	int ret = 0;
2800 
2801 	if (cpufreq_driver->boost_enabled == state)
2802 		return 0;
2803 
2804 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2805 	cpufreq_driver->boost_enabled = state;
2806 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2807 
2808 	cpus_read_lock();
2809 	for_each_active_policy(policy) {
2810 		policy->boost_enabled = state;
2811 		ret = cpufreq_driver->set_boost(policy, state);
2812 		if (ret) {
2813 			policy->boost_enabled = !policy->boost_enabled;
2814 			goto err_reset_state;
2815 		}
2816 	}
2817 	cpus_read_unlock();
2818 
2819 	return 0;
2820 
2821 err_reset_state:
2822 	cpus_read_unlock();
2823 
2824 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2825 	cpufreq_driver->boost_enabled = !state;
2826 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2827 
2828 	pr_err("%s: Cannot %s BOOST\n",
2829 	       __func__, str_enable_disable(state));
2830 
2831 	return ret;
2832 }
2833 
cpufreq_boost_supported(void)2834 static bool cpufreq_boost_supported(void)
2835 {
2836 	return cpufreq_driver->set_boost;
2837 }
2838 
create_boost_sysfs_file(void)2839 static int create_boost_sysfs_file(void)
2840 {
2841 	int ret;
2842 
2843 	ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2844 	if (ret)
2845 		pr_err("%s: cannot register global BOOST sysfs file\n",
2846 		       __func__);
2847 
2848 	return ret;
2849 }
2850 
remove_boost_sysfs_file(void)2851 static void remove_boost_sysfs_file(void)
2852 {
2853 	if (cpufreq_boost_supported())
2854 		sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2855 }
2856 
cpufreq_enable_boost_support(void)2857 int cpufreq_enable_boost_support(void)
2858 {
2859 	if (!cpufreq_driver)
2860 		return -EINVAL;
2861 
2862 	if (cpufreq_boost_supported())
2863 		return 0;
2864 
2865 	cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2866 
2867 	/* This will get removed on driver unregister */
2868 	return create_boost_sysfs_file();
2869 }
2870 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2871 
cpufreq_boost_enabled(void)2872 bool cpufreq_boost_enabled(void)
2873 {
2874 	return cpufreq_driver->boost_enabled;
2875 }
2876 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2877 
2878 /*********************************************************************
2879  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2880  *********************************************************************/
2881 static enum cpuhp_state hp_online;
2882 
cpuhp_cpufreq_online(unsigned int cpu)2883 static int cpuhp_cpufreq_online(unsigned int cpu)
2884 {
2885 	cpufreq_online(cpu);
2886 
2887 	return 0;
2888 }
2889 
cpuhp_cpufreq_offline(unsigned int cpu)2890 static int cpuhp_cpufreq_offline(unsigned int cpu)
2891 {
2892 	cpufreq_offline(cpu);
2893 
2894 	return 0;
2895 }
2896 
2897 /**
2898  * cpufreq_register_driver - register a CPU Frequency driver
2899  * @driver_data: A struct cpufreq_driver containing the values#
2900  * submitted by the CPU Frequency driver.
2901  *
2902  * Registers a CPU Frequency driver to this core code. This code
2903  * returns zero on success, -EEXIST when another driver got here first
2904  * (and isn't unregistered in the meantime).
2905  *
2906  */
cpufreq_register_driver(struct cpufreq_driver * driver_data)2907 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2908 {
2909 	unsigned long flags;
2910 	int ret;
2911 
2912 	if (cpufreq_disabled())
2913 		return -ENODEV;
2914 
2915 	/*
2916 	 * The cpufreq core depends heavily on the availability of device
2917 	 * structure, make sure they are available before proceeding further.
2918 	 */
2919 	if (!get_cpu_device(0))
2920 		return -EPROBE_DEFER;
2921 
2922 	if (!driver_data || !driver_data->verify || !driver_data->init ||
2923 	    !(driver_data->setpolicy || driver_data->target_index ||
2924 		    driver_data->target) ||
2925 	     (driver_data->setpolicy && (driver_data->target_index ||
2926 		    driver_data->target)) ||
2927 	     (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
2928 	     (!driver_data->online != !driver_data->offline) ||
2929 		 (driver_data->adjust_perf && !driver_data->fast_switch))
2930 		return -EINVAL;
2931 
2932 	pr_debug("trying to register driver %s\n", driver_data->name);
2933 
2934 	/* Protect against concurrent CPU online/offline. */
2935 	cpus_read_lock();
2936 
2937 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2938 	if (cpufreq_driver) {
2939 		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2940 		ret = -EEXIST;
2941 		goto out;
2942 	}
2943 	cpufreq_driver = driver_data;
2944 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2945 
2946 	/*
2947 	 * Mark support for the scheduler's frequency invariance engine for
2948 	 * drivers that implement target(), target_index() or fast_switch().
2949 	 */
2950 	if (!cpufreq_driver->setpolicy) {
2951 		static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
2952 		pr_debug("supports frequency invariance");
2953 	}
2954 
2955 	if (driver_data->setpolicy)
2956 		driver_data->flags |= CPUFREQ_CONST_LOOPS;
2957 
2958 	if (cpufreq_boost_supported()) {
2959 		ret = create_boost_sysfs_file();
2960 		if (ret)
2961 			goto err_null_driver;
2962 	}
2963 
2964 	ret = subsys_interface_register(&cpufreq_interface);
2965 	if (ret)
2966 		goto err_boost_unreg;
2967 
2968 	if (unlikely(list_empty(&cpufreq_policy_list))) {
2969 		/* if all ->init() calls failed, unregister */
2970 		ret = -ENODEV;
2971 		pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2972 			 driver_data->name);
2973 		goto err_if_unreg;
2974 	}
2975 
2976 	ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
2977 						   "cpufreq:online",
2978 						   cpuhp_cpufreq_online,
2979 						   cpuhp_cpufreq_offline);
2980 	if (ret < 0)
2981 		goto err_if_unreg;
2982 	hp_online = ret;
2983 	ret = 0;
2984 
2985 	pr_debug("driver %s up and running\n", driver_data->name);
2986 	goto out;
2987 
2988 err_if_unreg:
2989 	subsys_interface_unregister(&cpufreq_interface);
2990 err_boost_unreg:
2991 	remove_boost_sysfs_file();
2992 err_null_driver:
2993 	write_lock_irqsave(&cpufreq_driver_lock, flags);
2994 	cpufreq_driver = NULL;
2995 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2996 out:
2997 	cpus_read_unlock();
2998 	return ret;
2999 }
3000 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
3001 
3002 /*
3003  * cpufreq_unregister_driver - unregister the current CPUFreq driver
3004  *
3005  * Unregister the current CPUFreq driver. Only call this if you have
3006  * the right to do so, i.e. if you have succeeded in initialising before!
3007  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
3008  * currently not initialised.
3009  */
cpufreq_unregister_driver(struct cpufreq_driver * driver)3010 void cpufreq_unregister_driver(struct cpufreq_driver *driver)
3011 {
3012 	unsigned long flags;
3013 
3014 	if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver)))
3015 		return;
3016 
3017 	pr_debug("unregistering driver %s\n", driver->name);
3018 
3019 	/* Protect against concurrent cpu hotplug */
3020 	cpus_read_lock();
3021 	subsys_interface_unregister(&cpufreq_interface);
3022 	remove_boost_sysfs_file();
3023 	static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
3024 	cpuhp_remove_state_nocalls_cpuslocked(hp_online);
3025 
3026 	write_lock_irqsave(&cpufreq_driver_lock, flags);
3027 
3028 	cpufreq_driver = NULL;
3029 
3030 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
3031 	cpus_read_unlock();
3032 }
3033 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
3034 
cpufreq_core_init(void)3035 static int __init cpufreq_core_init(void)
3036 {
3037 	struct cpufreq_governor *gov = cpufreq_default_governor();
3038 	struct device *dev_root;
3039 
3040 	if (cpufreq_disabled())
3041 		return -ENODEV;
3042 
3043 	dev_root = bus_get_dev_root(&cpu_subsys);
3044 	if (dev_root) {
3045 		cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj);
3046 		put_device(dev_root);
3047 	}
3048 	BUG_ON(!cpufreq_global_kobject);
3049 
3050 	if (!strlen(default_governor))
3051 		strscpy(default_governor, gov->name, CPUFREQ_NAME_LEN);
3052 
3053 	return 0;
3054 }
3055 module_param(off, int, 0444);
3056 module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444);
3057 core_initcall(cpufreq_core_init);
3058