xref: /linux/include/linux/cpufreq.h (revision 342d3a73c5dbc2f1cd0c5142e2d11bda4dc54cd7)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * linux/include/linux/cpufreq.h
4  *
5  * Copyright (C) 2001 Russell King
6  *           (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
7  */
8 #ifndef _LINUX_CPUFREQ_H
9 #define _LINUX_CPUFREQ_H
10 
11 #include <linux/clk.h>
12 #include <linux/cpu.h>
13 #include <linux/cpumask.h>
14 #include <linux/completion.h>
15 #include <linux/kobject.h>
16 #include <linux/notifier.h>
17 #include <linux/of.h>
18 #include <linux/pm_opp.h>
19 #include <linux/pm_qos.h>
20 #include <linux/spinlock.h>
21 #include <linux/sysfs.h>
22 #include <linux/minmax.h>
23 
24 /*********************************************************************
25  *                        CPUFREQ INTERFACE                          *
26  *********************************************************************/
27 /*
28  * Frequency values here are CPU kHz
29  */
30 
31 #define CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS	NSEC_PER_MSEC
32 
33 #define CPUFREQ_NAME_LEN		16
34 /* Print length for names. Extra 1 space for accommodating '\n' in prints */
35 #define CPUFREQ_NAME_PLEN		(CPUFREQ_NAME_LEN + 1)
36 
37 struct cpufreq_governor;
38 
39 enum cpufreq_table_sorting {
40 	CPUFREQ_TABLE_UNSORTED,
41 	CPUFREQ_TABLE_SORTED_ASCENDING,
42 	CPUFREQ_TABLE_SORTED_DESCENDING
43 };
44 
45 struct cpufreq_cpuinfo {
46 	unsigned int		max_freq;
47 	unsigned int		min_freq;
48 
49 	/* in 10^(-9) s = nanoseconds */
50 	unsigned int		transition_latency;
51 };
52 
53 struct cpufreq_policy {
54 	/* CPUs sharing clock, require sw coordination */
55 	cpumask_var_t		cpus;	/* Online CPUs only */
56 	cpumask_var_t		related_cpus; /* Online + Offline CPUs */
57 	cpumask_var_t		real_cpus; /* Related and present */
58 
59 	unsigned int		shared_type; /* ACPI: ANY or ALL affected CPUs
60 						should set cpufreq */
61 	unsigned int		cpu;    /* cpu managing this policy, must be online */
62 
63 	struct clk		*clk;
64 	struct cpufreq_cpuinfo	cpuinfo;/* see above */
65 
66 	unsigned int		min;    /* in kHz */
67 	unsigned int		max;    /* in kHz */
68 	unsigned int		cur;    /* in kHz, only needed if cpufreq
69 					 * governors are used */
70 	unsigned int		suspend_freq; /* freq to set during suspend */
71 
72 	unsigned int		policy; /* see above */
73 	unsigned int		last_policy; /* policy before unplug */
74 	struct cpufreq_governor	*governor; /* see below */
75 	void			*governor_data;
76 	char			last_governor[CPUFREQ_NAME_LEN]; /* last governor used */
77 
78 	struct work_struct	update; /* if update_policy() needs to be
79 					 * called, but you're in IRQ context */
80 
81 	struct freq_constraints	constraints;
82 	struct freq_qos_request	min_freq_req;
83 	struct freq_qos_request	max_freq_req;
84 	struct freq_qos_request boost_freq_req;
85 
86 	struct cpufreq_frequency_table	*freq_table;
87 	enum cpufreq_table_sorting freq_table_sorted;
88 
89 	struct list_head        policy_list;
90 	struct kobject		kobj;
91 	struct completion	kobj_unregister;
92 
93 	/*
94 	 * The rules for this semaphore:
95 	 * - Any routine that wants to read from the policy structure will
96 	 *   do a down_read on this semaphore.
97 	 * - Any routine that will write to the policy structure and/or may take away
98 	 *   the policy altogether (eg. CPU hotplug), will hold this lock in write
99 	 *   mode before doing so.
100 	 */
101 	struct rw_semaphore	rwsem;
102 
103 	/*
104 	 * Fast switch flags:
105 	 * - fast_switch_possible should be set by the driver if it can
106 	 *   guarantee that frequency can be changed on any CPU sharing the
107 	 *   policy and that the change will affect all of the policy CPUs then.
108 	 * - fast_switch_enabled is to be set by governors that support fast
109 	 *   frequency switching with the help of cpufreq_enable_fast_switch().
110 	 */
111 	bool			fast_switch_possible;
112 	bool			fast_switch_enabled;
113 
114 	/*
115 	 * Set if the CPUFREQ_GOV_STRICT_TARGET flag is set for the current
116 	 * governor.
117 	 */
118 	bool			strict_target;
119 
120 	/*
121 	 * Set if inefficient frequencies were found in the frequency table.
122 	 * This indicates if the relation flag CPUFREQ_RELATION_E can be
123 	 * honored.
124 	 */
125 	bool			efficiencies_available;
126 
127 	/*
128 	 * Preferred average time interval between consecutive invocations of
129 	 * the driver to set the frequency for this policy.  To be set by the
130 	 * scaling driver (0, which is the default, means no preference).
131 	 */
132 	unsigned int		transition_delay_us;
133 
134 	/*
135 	 * Remote DVFS flag (Not added to the driver structure as we don't want
136 	 * to access another structure from scheduler hotpath).
137 	 *
138 	 * Should be set if CPUs can do DVFS on behalf of other CPUs from
139 	 * different cpufreq policies.
140 	 */
141 	bool			dvfs_possible_from_any_cpu;
142 
143 	/* Per policy boost enabled flag. */
144 	bool			boost_enabled;
145 
146 	/* Per policy boost supported flag. */
147 	bool			boost_supported;
148 
149 	/* Force passing target frequency to the driver. */
150 	bool			force_target;
151 
152 	/* The policy has been suspended. */
153 	bool			suspended;
154 
155 	 /* Cached frequency lookup from cpufreq_driver_resolve_freq. */
156 	unsigned int cached_target_freq;
157 	unsigned int cached_resolved_idx;
158 
159 	/* Synchronization for frequency transitions */
160 	bool			transition_ongoing; /* Tracks transition status */
161 	spinlock_t		transition_lock;
162 	wait_queue_head_t	transition_wait;
163 	struct task_struct	*transition_task; /* Task which is doing the transition */
164 
165 	/* cpufreq-stats */
166 	struct cpufreq_stats	*stats;
167 
168 	/* For cpufreq driver's internal use */
169 	void			*driver_data;
170 
171 	/* Pointer to the cooling device if used for thermal mitigation */
172 	struct thermal_cooling_device *cdev;
173 
174 	struct notifier_block nb_min;
175 	struct notifier_block nb_max;
176 };
177 
178 DEFINE_GUARD(cpufreq_policy_write, struct cpufreq_policy *,
179 	     down_write(&_T->rwsem), up_write(&_T->rwsem))
180 
181 DEFINE_GUARD(cpufreq_policy_read, struct cpufreq_policy *,
182 	     down_read(&_T->rwsem), up_read(&_T->rwsem))
183 
184 /*
185  * Used for passing new cpufreq policy data to the cpufreq driver's ->verify()
186  * callback for sanitization.  That callback is only expected to modify the min
187  * and max values, if necessary, and specifically it must not update the
188  * frequency table.
189  */
190 struct cpufreq_policy_data {
191 	struct cpufreq_cpuinfo		cpuinfo;
192 	struct cpufreq_frequency_table	*freq_table;
193 	unsigned int			cpu;
194 	unsigned int			min;    /* in kHz */
195 	unsigned int			max;    /* in kHz */
196 };
197 
198 struct cpufreq_freqs {
199 	struct cpufreq_policy *policy;
200 	unsigned int old;
201 	unsigned int new;
202 	u8 flags;		/* flags of cpufreq_driver, see below. */
203 };
204 
205 /* Only for ACPI */
206 #define CPUFREQ_SHARED_TYPE_NONE (0) /* None */
207 #define CPUFREQ_SHARED_TYPE_HW	 (1) /* HW does needed coordination */
208 #define CPUFREQ_SHARED_TYPE_ALL	 (2) /* All dependent CPUs should set freq */
209 #define CPUFREQ_SHARED_TYPE_ANY	 (3) /* Freq can be set from any dependent CPU*/
210 
211 #ifdef CONFIG_CPU_FREQ
212 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu);
213 struct cpufreq_policy *cpufreq_cpu_policy(unsigned int cpu);
214 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu);
215 void cpufreq_cpu_put(struct cpufreq_policy *policy);
216 #else
217 static inline struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
218 {
219 	return NULL;
220 }
221 static inline struct cpufreq_policy *cpufreq_cpu_policy(unsigned int cpu)
222 {
223 	return NULL;
224 }
225 static inline struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
226 {
227 	return NULL;
228 }
229 static inline void cpufreq_cpu_put(struct cpufreq_policy *policy) { }
230 #endif
231 
232 /* Scope based cleanup macro for cpufreq_policy kobject reference counting */
233 DEFINE_FREE(put_cpufreq_policy, struct cpufreq_policy *, if (_T) cpufreq_cpu_put(_T))
234 
235 static inline bool policy_is_inactive(struct cpufreq_policy *policy)
236 {
237 	return cpumask_empty(policy->cpus);
238 }
239 
240 static inline bool policy_is_shared(struct cpufreq_policy *policy)
241 {
242 	return cpumask_nth(1, policy->cpus) < nr_cpumask_bits;
243 }
244 
245 #ifdef CONFIG_CPU_FREQ
246 unsigned int cpufreq_get(unsigned int cpu);
247 unsigned int cpufreq_quick_get(unsigned int cpu);
248 unsigned int cpufreq_quick_get_max(unsigned int cpu);
249 unsigned int cpufreq_get_hw_max_freq(unsigned int cpu);
250 void disable_cpufreq(void);
251 
252 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy);
253 
254 void refresh_frequency_limits(struct cpufreq_policy *policy);
255 void cpufreq_update_policy(unsigned int cpu);
256 void cpufreq_update_limits(unsigned int cpu);
257 bool have_governor_per_policy(void);
258 bool cpufreq_supports_freq_invariance(void);
259 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy);
260 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy);
261 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy);
262 bool has_target_index(void);
263 
264 DECLARE_PER_CPU(unsigned long, cpufreq_pressure);
265 static inline unsigned long cpufreq_get_pressure(int cpu)
266 {
267 	return READ_ONCE(per_cpu(cpufreq_pressure, cpu));
268 }
269 #else
270 static inline unsigned int cpufreq_get(unsigned int cpu)
271 {
272 	return 0;
273 }
274 static inline unsigned int cpufreq_quick_get(unsigned int cpu)
275 {
276 	return 0;
277 }
278 static inline unsigned int cpufreq_quick_get_max(unsigned int cpu)
279 {
280 	return 0;
281 }
282 static inline unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
283 {
284 	return 0;
285 }
286 static inline bool cpufreq_supports_freq_invariance(void)
287 {
288 	return false;
289 }
290 static inline void disable_cpufreq(void) { }
291 static inline void cpufreq_update_limits(unsigned int cpu) { }
292 static inline unsigned long cpufreq_get_pressure(int cpu)
293 {
294 	return 0;
295 }
296 #endif
297 
298 #ifdef CONFIG_CPU_FREQ_STAT
299 void cpufreq_stats_create_table(struct cpufreq_policy *policy);
300 void cpufreq_stats_free_table(struct cpufreq_policy *policy);
301 void cpufreq_stats_record_transition(struct cpufreq_policy *policy,
302 				     unsigned int new_freq);
303 #else
304 static inline void cpufreq_stats_create_table(struct cpufreq_policy *policy) { }
305 static inline void cpufreq_stats_free_table(struct cpufreq_policy *policy) { }
306 static inline void cpufreq_stats_record_transition(struct cpufreq_policy *policy,
307 						   unsigned int new_freq) { }
308 #endif /* CONFIG_CPU_FREQ_STAT */
309 
310 /*********************************************************************
311  *                      CPUFREQ DRIVER INTERFACE                     *
312  *********************************************************************/
313 
314 #define CPUFREQ_RELATION_L 0  /* lowest frequency at or above target */
315 #define CPUFREQ_RELATION_H 1  /* highest frequency below or at target */
316 #define CPUFREQ_RELATION_C 2  /* closest frequency to target */
317 /* relation flags */
318 #define CPUFREQ_RELATION_E BIT(2) /* Get if possible an efficient frequency */
319 
320 #define CPUFREQ_RELATION_LE (CPUFREQ_RELATION_L | CPUFREQ_RELATION_E)
321 #define CPUFREQ_RELATION_HE (CPUFREQ_RELATION_H | CPUFREQ_RELATION_E)
322 #define CPUFREQ_RELATION_CE (CPUFREQ_RELATION_C | CPUFREQ_RELATION_E)
323 
324 struct freq_attr {
325 	struct attribute attr;
326 	ssize_t (*show)(struct cpufreq_policy *, char *);
327 	ssize_t (*store)(struct cpufreq_policy *, const char *, size_t count);
328 };
329 
330 #define cpufreq_freq_attr_ro(_name)		\
331 static struct freq_attr _name =			\
332 __ATTR(_name, 0444, show_##_name, NULL)
333 
334 #define cpufreq_freq_attr_ro_perm(_name, _perm)	\
335 static struct freq_attr _name =			\
336 __ATTR(_name, _perm, show_##_name, NULL)
337 
338 #define cpufreq_freq_attr_rw(_name)		\
339 static struct freq_attr _name =			\
340 __ATTR(_name, 0644, show_##_name, store_##_name)
341 
342 #define cpufreq_freq_attr_wo(_name)		\
343 static struct freq_attr _name =			\
344 __ATTR(_name, 0200, NULL, store_##_name)
345 
346 #define define_one_global_ro(_name)		\
347 static struct kobj_attribute _name =		\
348 __ATTR(_name, 0444, show_##_name, NULL)
349 
350 #define define_one_global_rw(_name)		\
351 static struct kobj_attribute _name =		\
352 __ATTR(_name, 0644, show_##_name, store_##_name)
353 
354 
355 struct cpufreq_driver {
356 	char		name[CPUFREQ_NAME_LEN];
357 	u16		flags;
358 	void		*driver_data;
359 
360 	/* needed by all drivers */
361 	int		(*init)(struct cpufreq_policy *policy);
362 	int		(*verify)(struct cpufreq_policy_data *policy);
363 
364 	/* define one out of two */
365 	int		(*setpolicy)(struct cpufreq_policy *policy);
366 
367 	int		(*target)(struct cpufreq_policy *policy,
368 				  unsigned int target_freq,
369 				  unsigned int relation);	/* Deprecated */
370 	int		(*target_index)(struct cpufreq_policy *policy,
371 					unsigned int index);
372 	unsigned int	(*fast_switch)(struct cpufreq_policy *policy,
373 				       unsigned int target_freq);
374 	/*
375 	 * ->fast_switch() replacement for drivers that use an internal
376 	 * representation of performance levels and can pass hints other than
377 	 * the target performance level to the hardware. This can only be set
378 	 * if ->fast_switch is set too, because in those cases (under specific
379 	 * conditions) scale invariance can be disabled, which causes the
380 	 * schedutil governor to fall back to the latter.
381 	 */
382 	void		(*adjust_perf)(struct cpufreq_policy *policy,
383 				       unsigned long min_perf,
384 				       unsigned long target_perf,
385 				       unsigned long max_perf,
386 				       unsigned long capacity);
387 
388 	/*
389 	 * Only for drivers with target_index() and CPUFREQ_ASYNC_NOTIFICATION
390 	 * unset.
391 	 *
392 	 * get_intermediate should return a stable intermediate frequency
393 	 * platform wants to switch to and target_intermediate() should set CPU
394 	 * to that frequency, before jumping to the frequency corresponding
395 	 * to 'index'. Core will take care of sending notifications and driver
396 	 * doesn't have to handle them in target_intermediate() or
397 	 * target_index().
398 	 *
399 	 * Drivers can return '0' from get_intermediate() in case they don't
400 	 * wish to switch to intermediate frequency for some target frequency.
401 	 * In that case core will directly call ->target_index().
402 	 */
403 	unsigned int	(*get_intermediate)(struct cpufreq_policy *policy,
404 					    unsigned int index);
405 	int		(*target_intermediate)(struct cpufreq_policy *policy,
406 					       unsigned int index);
407 
408 	/* should be defined, if possible, return 0 on error */
409 	unsigned int	(*get)(unsigned int cpu);
410 
411 	/* Called to update policy limits on firmware notifications. */
412 	void		(*update_limits)(struct cpufreq_policy *policy);
413 
414 	/* optional */
415 	int		(*bios_limit)(int cpu, unsigned int *limit);
416 
417 	int		(*online)(struct cpufreq_policy *policy);
418 	int		(*offline)(struct cpufreq_policy *policy);
419 	void		(*exit)(struct cpufreq_policy *policy);
420 	int		(*suspend)(struct cpufreq_policy *policy);
421 	int		(*resume)(struct cpufreq_policy *policy);
422 
423 	/* Will be called after the driver is fully initialized */
424 	void		(*ready)(struct cpufreq_policy *policy);
425 
426 	/* Return the capacity reference frequency for policy. */
427 	unsigned int	(*scale_freq_ref)(struct cpufreq_policy *policy);
428 
429 	struct freq_attr **attr;
430 
431 	/* platform specific boost support code */
432 	bool		boost_enabled;
433 	int		(*set_boost)(struct cpufreq_policy *policy, int state);
434 
435 	/*
436 	 * Set by drivers that want to register with the energy model after the
437 	 * policy is properly initialized, but before the governor is started.
438 	 */
439 	void		(*register_em)(struct cpufreq_policy *policy);
440 };
441 
442 /* flags */
443 
444 /*
445  * Set by drivers that need to update internal upper and lower boundaries along
446  * with the target frequency and so the core and governors should also invoke
447  * the driver if the target frequency does not change, but the policy min or max
448  * may have changed.
449  */
450 #define CPUFREQ_NEED_UPDATE_LIMITS		BIT(0)
451 
452 /* loops_per_jiffy or other kernel "constants" aren't affected by frequency transitions */
453 #define CPUFREQ_CONST_LOOPS			BIT(1)
454 
455 /*
456  * Set by drivers that want the core to automatically register the cpufreq
457  * driver as a thermal cooling device.
458  */
459 #define CPUFREQ_IS_COOLING_DEV			BIT(2)
460 
461 /*
462  * This should be set by platforms having multiple clock-domains, i.e.
463  * supporting multiple policies. With this sysfs directories of governor would
464  * be created in cpu/cpu<num>/cpufreq/ directory and so they can use the same
465  * governor with different tunables for different clusters.
466  */
467 #define CPUFREQ_HAVE_GOVERNOR_PER_POLICY	BIT(3)
468 
469 /*
470  * Driver will do POSTCHANGE notifications from outside of their ->target()
471  * routine and so must set cpufreq_driver->flags with this flag, so that core
472  * can handle them specially.
473  */
474 #define CPUFREQ_ASYNC_NOTIFICATION		BIT(4)
475 
476 /*
477  * Set by drivers which want cpufreq core to check if CPU is running at a
478  * frequency present in freq-table exposed by the driver. For these drivers if
479  * CPU is found running at an out of table freq, we will try to set it to a freq
480  * from the table. And if that fails, we will stop further boot process by
481  * issuing a BUG_ON().
482  */
483 #define CPUFREQ_NEED_INITIAL_FREQ_CHECK	BIT(5)
484 
485 /*
486  * Set by drivers to disallow use of governors with "dynamic_switching" flag
487  * set.
488  */
489 #define CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING	BIT(6)
490 
491 int cpufreq_register_driver(struct cpufreq_driver *driver_data);
492 void cpufreq_unregister_driver(struct cpufreq_driver *driver_data);
493 
494 bool cpufreq_driver_test_flags(u16 flags);
495 const char *cpufreq_get_current_driver(void);
496 void *cpufreq_get_driver_data(void);
497 
498 static inline int cpufreq_thermal_control_enabled(struct cpufreq_driver *drv)
499 {
500 	return IS_ENABLED(CONFIG_CPU_THERMAL) &&
501 		(drv->flags & CPUFREQ_IS_COOLING_DEV);
502 }
503 
504 static inline void cpufreq_verify_within_limits(struct cpufreq_policy_data *policy,
505 						unsigned int min,
506 						unsigned int max)
507 {
508 	policy->max = clamp(policy->max, min, max);
509 	policy->min = clamp(policy->min, min, policy->max);
510 }
511 
512 static inline void
513 cpufreq_verify_within_cpu_limits(struct cpufreq_policy_data *policy)
514 {
515 	cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
516 				     policy->cpuinfo.max_freq);
517 }
518 
519 #ifdef CONFIG_CPU_FREQ
520 void cpufreq_suspend(void);
521 void cpufreq_resume(void);
522 int cpufreq_generic_suspend(struct cpufreq_policy *policy);
523 #else
524 static inline void cpufreq_suspend(void) {}
525 static inline void cpufreq_resume(void) {}
526 #endif
527 
528 /*********************************************************************
529  *                     CPUFREQ NOTIFIER INTERFACE                    *
530  *********************************************************************/
531 
532 #define CPUFREQ_TRANSITION_NOTIFIER	(0)
533 #define CPUFREQ_POLICY_NOTIFIER		(1)
534 
535 /* Transition notifiers */
536 #define CPUFREQ_PRECHANGE		(0)
537 #define CPUFREQ_POSTCHANGE		(1)
538 
539 /* Policy Notifiers  */
540 #define CPUFREQ_CREATE_POLICY		(0)
541 #define CPUFREQ_REMOVE_POLICY		(1)
542 
543 #ifdef CONFIG_CPU_FREQ
544 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list);
545 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list);
546 
547 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
548 		struct cpufreq_freqs *freqs);
549 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
550 		struct cpufreq_freqs *freqs, int transition_failed);
551 
552 #else /* CONFIG_CPU_FREQ */
553 static inline int cpufreq_register_notifier(struct notifier_block *nb,
554 						unsigned int list)
555 {
556 	return 0;
557 }
558 static inline int cpufreq_unregister_notifier(struct notifier_block *nb,
559 						unsigned int list)
560 {
561 	return 0;
562 }
563 #endif /* !CONFIG_CPU_FREQ */
564 
565 /**
566  * cpufreq_scale - "old * mult / div" calculation for large values (32-bit-arch
567  * safe)
568  * @old:   old value
569  * @div:   divisor
570  * @mult:  multiplier
571  *
572  *
573  * new = old * mult / div
574  */
575 static inline unsigned long cpufreq_scale(unsigned long old, u_int div,
576 		u_int mult)
577 {
578 #if BITS_PER_LONG == 32
579 	u64 result = ((u64) old) * ((u64) mult);
580 	do_div(result, div);
581 	return (unsigned long) result;
582 
583 #elif BITS_PER_LONG == 64
584 	unsigned long result = old * ((u64) mult);
585 	result /= div;
586 	return result;
587 #endif
588 }
589 
590 /*********************************************************************
591  *                          CPUFREQ GOVERNORS                        *
592  *********************************************************************/
593 
594 #define CPUFREQ_POLICY_UNKNOWN		(0)
595 /*
596  * If (cpufreq_driver->target) exists, the ->governor decides what frequency
597  * within the limits is used. If (cpufreq_driver->setpolicy> exists, these
598  * two generic policies are available:
599  */
600 #define CPUFREQ_POLICY_POWERSAVE	(1)
601 #define CPUFREQ_POLICY_PERFORMANCE	(2)
602 
603 struct cpufreq_governor {
604 	char	name[CPUFREQ_NAME_LEN];
605 	int	(*init)(struct cpufreq_policy *policy);
606 	void	(*exit)(struct cpufreq_policy *policy);
607 	int	(*start)(struct cpufreq_policy *policy);
608 	void	(*stop)(struct cpufreq_policy *policy);
609 	void	(*limits)(struct cpufreq_policy *policy);
610 	ssize_t	(*show_setspeed)	(struct cpufreq_policy *policy,
611 					 char *buf);
612 	int	(*store_setspeed)	(struct cpufreq_policy *policy,
613 					 unsigned int freq);
614 	struct list_head	governor_list;
615 	struct module		*owner;
616 	u8			flags;
617 };
618 
619 /* Governor flags */
620 
621 /* For governors which change frequency dynamically by themselves */
622 #define CPUFREQ_GOV_DYNAMIC_SWITCHING	BIT(0)
623 
624 /* For governors wanting the target frequency to be set exactly */
625 #define CPUFREQ_GOV_STRICT_TARGET	BIT(1)
626 
627 
628 /* Pass a target to the cpufreq driver */
629 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
630 					unsigned int target_freq);
631 void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy,
632 				unsigned long min_perf,
633 				unsigned long target_perf,
634 				unsigned long max_perf,
635 				unsigned long capacity);
636 bool cpufreq_driver_has_adjust_perf(void);
637 int cpufreq_driver_target(struct cpufreq_policy *policy,
638 				 unsigned int target_freq,
639 				 unsigned int relation);
640 int __cpufreq_driver_target(struct cpufreq_policy *policy,
641 				   unsigned int target_freq,
642 				   unsigned int relation);
643 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
644 					 unsigned int target_freq);
645 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy);
646 int cpufreq_register_governor(struct cpufreq_governor *governor);
647 void cpufreq_unregister_governor(struct cpufreq_governor *governor);
648 int cpufreq_start_governor(struct cpufreq_policy *policy);
649 void cpufreq_stop_governor(struct cpufreq_policy *policy);
650 
651 #define cpufreq_governor_init(__governor)			\
652 static int __init __governor##_init(void)			\
653 {								\
654 	return cpufreq_register_governor(&__governor);	\
655 }								\
656 core_initcall(__governor##_init)
657 
658 #define cpufreq_governor_exit(__governor)			\
659 static void __exit __governor##_exit(void)			\
660 {								\
661 	return cpufreq_unregister_governor(&__governor);	\
662 }								\
663 module_exit(__governor##_exit)
664 
665 struct cpufreq_governor *cpufreq_default_governor(void);
666 struct cpufreq_governor *cpufreq_fallback_governor(void);
667 
668 #ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
669 bool sugov_is_governor(struct cpufreq_policy *policy);
670 #else
671 static inline bool sugov_is_governor(struct cpufreq_policy *policy)
672 {
673 	return false;
674 }
675 #endif
676 
677 static inline void cpufreq_policy_apply_limits(struct cpufreq_policy *policy)
678 {
679 	if (policy->max < policy->cur)
680 		__cpufreq_driver_target(policy, policy->max,
681 					CPUFREQ_RELATION_HE);
682 	else if (policy->min > policy->cur)
683 		__cpufreq_driver_target(policy, policy->min,
684 					CPUFREQ_RELATION_LE);
685 }
686 
687 /* Governor attribute set */
688 struct gov_attr_set {
689 	struct kobject kobj;
690 	struct list_head policy_list;
691 	struct mutex update_lock;
692 	int usage_count;
693 };
694 
695 /* sysfs ops for cpufreq governors */
696 extern const struct sysfs_ops governor_sysfs_ops;
697 
698 static inline struct gov_attr_set *to_gov_attr_set(struct kobject *kobj)
699 {
700 	return container_of(kobj, struct gov_attr_set, kobj);
701 }
702 
703 void gov_attr_set_init(struct gov_attr_set *attr_set, struct list_head *list_node);
704 void gov_attr_set_get(struct gov_attr_set *attr_set, struct list_head *list_node);
705 unsigned int gov_attr_set_put(struct gov_attr_set *attr_set, struct list_head *list_node);
706 
707 /* Governor sysfs attribute */
708 struct governor_attr {
709 	struct attribute attr;
710 	ssize_t (*show)(struct gov_attr_set *attr_set, char *buf);
711 	ssize_t (*store)(struct gov_attr_set *attr_set, const char *buf,
712 			 size_t count);
713 };
714 
715 /*********************************************************************
716  *                     FREQUENCY TABLE HELPERS                       *
717  *********************************************************************/
718 
719 /* Special Values of .frequency field */
720 #define CPUFREQ_ENTRY_INVALID		~0u
721 #define CPUFREQ_TABLE_END		~1u
722 /* Special Values of .flags field */
723 #define CPUFREQ_BOOST_FREQ		(1 << 0)
724 #define CPUFREQ_INEFFICIENT_FREQ	(1 << 1)
725 
726 struct cpufreq_frequency_table {
727 	unsigned int	flags;
728 	unsigned int	driver_data; /* driver specific data, not used by core */
729 	unsigned int	frequency; /* kHz - doesn't need to be in ascending
730 				    * order */
731 };
732 
733 /*
734  * cpufreq_for_each_entry -	iterate over a cpufreq_frequency_table
735  * @pos:	the cpufreq_frequency_table * to use as a loop cursor.
736  * @table:	the cpufreq_frequency_table * to iterate over.
737  */
738 
739 #define cpufreq_for_each_entry(pos, table)	\
740 	for (pos = table; pos->frequency != CPUFREQ_TABLE_END; pos++)
741 
742 /*
743  * cpufreq_for_each_entry_idx -	iterate over a cpufreq_frequency_table
744  *	with index
745  * @pos:	the cpufreq_frequency_table * to use as a loop cursor.
746  * @table:	the cpufreq_frequency_table * to iterate over.
747  * @idx:	the table entry currently being processed
748  */
749 
750 #define cpufreq_for_each_entry_idx(pos, table, idx)	\
751 	for (pos = table, idx = 0; pos->frequency != CPUFREQ_TABLE_END; \
752 		pos++, idx++)
753 
754 /*
755  * cpufreq_for_each_valid_entry -     iterate over a cpufreq_frequency_table
756  *	excluding CPUFREQ_ENTRY_INVALID frequencies.
757  * @pos:        the cpufreq_frequency_table * to use as a loop cursor.
758  * @table:      the cpufreq_frequency_table * to iterate over.
759  */
760 
761 #define cpufreq_for_each_valid_entry(pos, table)			\
762 	for (pos = table; pos->frequency != CPUFREQ_TABLE_END; pos++)	\
763 		if (pos->frequency == CPUFREQ_ENTRY_INVALID)		\
764 			continue;					\
765 		else
766 
767 /*
768  * cpufreq_for_each_valid_entry_idx -     iterate with index over a cpufreq
769  *	frequency_table excluding CPUFREQ_ENTRY_INVALID frequencies.
770  * @pos:	the cpufreq_frequency_table * to use as a loop cursor.
771  * @table:	the cpufreq_frequency_table * to iterate over.
772  * @idx:	the table entry currently being processed
773  */
774 
775 #define cpufreq_for_each_valid_entry_idx(pos, table, idx)		\
776 	cpufreq_for_each_entry_idx(pos, table, idx)			\
777 		if (pos->frequency == CPUFREQ_ENTRY_INVALID)		\
778 			continue;					\
779 		else
780 
781 /**
782  * cpufreq_for_each_efficient_entry_idx - iterate with index over a cpufreq
783  *	frequency_table excluding CPUFREQ_ENTRY_INVALID and
784  *	CPUFREQ_INEFFICIENT_FREQ frequencies.
785  * @pos: the &struct cpufreq_frequency_table to use as a loop cursor.
786  * @table: the &struct cpufreq_frequency_table to iterate over.
787  * @idx: the table entry currently being processed.
788  * @efficiencies: set to true to only iterate over efficient frequencies.
789  */
790 
791 #define cpufreq_for_each_efficient_entry_idx(pos, table, idx, efficiencies)	\
792 	cpufreq_for_each_valid_entry_idx(pos, table, idx)			\
793 		if (efficiencies && (pos->flags & CPUFREQ_INEFFICIENT_FREQ))	\
794 			continue;						\
795 		else
796 
797 
798 int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy);
799 
800 int cpufreq_frequency_table_verify(struct cpufreq_policy_data *policy);
801 
802 int cpufreq_generic_frequency_table_verify(struct cpufreq_policy_data *policy);
803 
804 int cpufreq_table_index_unsorted(struct cpufreq_policy *policy,
805 				 unsigned int target_freq, unsigned int min,
806 				 unsigned int max, unsigned int relation);
807 int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy,
808 		unsigned int freq);
809 
810 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf);
811 
812 #ifdef CONFIG_CPU_FREQ
813 bool cpufreq_boost_enabled(void);
814 int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state);
815 
816 /* Find lowest freq at or above target in a table in ascending order */
817 static inline int cpufreq_table_find_index_al(struct cpufreq_policy *policy,
818 					      unsigned int target_freq,
819 					      bool efficiencies)
820 {
821 	struct cpufreq_frequency_table *table = policy->freq_table;
822 	struct cpufreq_frequency_table *pos;
823 	unsigned int freq;
824 	int idx, best = -1;
825 
826 	cpufreq_for_each_efficient_entry_idx(pos, table, idx, efficiencies) {
827 		freq = pos->frequency;
828 
829 		if (freq >= target_freq)
830 			return idx;
831 
832 		best = idx;
833 	}
834 
835 	return best;
836 }
837 
838 /* Find lowest freq at or above target in a table in descending order */
839 static inline int cpufreq_table_find_index_dl(struct cpufreq_policy *policy,
840 					      unsigned int target_freq,
841 					      bool efficiencies)
842 {
843 	struct cpufreq_frequency_table *table = policy->freq_table;
844 	struct cpufreq_frequency_table *pos;
845 	unsigned int freq;
846 	int idx, best = -1;
847 
848 	cpufreq_for_each_efficient_entry_idx(pos, table, idx, efficiencies) {
849 		freq = pos->frequency;
850 
851 		if (freq == target_freq)
852 			return idx;
853 
854 		if (freq > target_freq) {
855 			best = idx;
856 			continue;
857 		}
858 
859 		/* No freq found above target_freq */
860 		if (best == -1)
861 			return idx;
862 
863 		return best;
864 	}
865 
866 	return best;
867 }
868 
869 static inline int find_index_l(struct cpufreq_policy *policy,
870 			       unsigned int target_freq,
871 			       unsigned int min, unsigned int max,
872 			       bool efficiencies)
873 {
874 	target_freq = clamp_val(target_freq, min, max);
875 
876 	if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING)
877 		return cpufreq_table_find_index_al(policy, target_freq,
878 						   efficiencies);
879 	else
880 		return cpufreq_table_find_index_dl(policy, target_freq,
881 						   efficiencies);
882 }
883 
884 /* Works only on sorted freq-tables */
885 static inline int cpufreq_table_find_index_l(struct cpufreq_policy *policy,
886 					     unsigned int target_freq,
887 					     bool efficiencies)
888 {
889 	return find_index_l(policy, target_freq, policy->min, policy->max, efficiencies);
890 }
891 
892 /* Find highest freq at or below target in a table in ascending order */
893 static inline int cpufreq_table_find_index_ah(struct cpufreq_policy *policy,
894 					      unsigned int target_freq,
895 					      bool efficiencies)
896 {
897 	struct cpufreq_frequency_table *table = policy->freq_table;
898 	struct cpufreq_frequency_table *pos;
899 	unsigned int freq;
900 	int idx, best = -1;
901 
902 	cpufreq_for_each_efficient_entry_idx(pos, table, idx, efficiencies) {
903 		freq = pos->frequency;
904 
905 		if (freq == target_freq)
906 			return idx;
907 
908 		if (freq < target_freq) {
909 			best = idx;
910 			continue;
911 		}
912 
913 		/* No freq found below target_freq */
914 		if (best == -1)
915 			return idx;
916 
917 		return best;
918 	}
919 
920 	return best;
921 }
922 
923 /* Find highest freq at or below target in a table in descending order */
924 static inline int cpufreq_table_find_index_dh(struct cpufreq_policy *policy,
925 					      unsigned int target_freq,
926 					      bool efficiencies)
927 {
928 	struct cpufreq_frequency_table *table = policy->freq_table;
929 	struct cpufreq_frequency_table *pos;
930 	unsigned int freq;
931 	int idx, best = -1;
932 
933 	cpufreq_for_each_efficient_entry_idx(pos, table, idx, efficiencies) {
934 		freq = pos->frequency;
935 
936 		if (freq <= target_freq)
937 			return idx;
938 
939 		best = idx;
940 	}
941 
942 	return best;
943 }
944 
945 static inline int find_index_h(struct cpufreq_policy *policy,
946 			       unsigned int target_freq,
947 			       unsigned int min, unsigned int max,
948 			       bool efficiencies)
949 {
950 	target_freq = clamp_val(target_freq, min, max);
951 
952 	if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING)
953 		return cpufreq_table_find_index_ah(policy, target_freq,
954 						   efficiencies);
955 	else
956 		return cpufreq_table_find_index_dh(policy, target_freq,
957 						   efficiencies);
958 }
959 
960 /* Works only on sorted freq-tables */
961 static inline int cpufreq_table_find_index_h(struct cpufreq_policy *policy,
962 					     unsigned int target_freq,
963 					     bool efficiencies)
964 {
965 	return find_index_h(policy, target_freq, policy->min, policy->max, efficiencies);
966 }
967 
968 /* Find closest freq to target in a table in ascending order */
969 static inline int cpufreq_table_find_index_ac(struct cpufreq_policy *policy,
970 					      unsigned int target_freq,
971 					      bool efficiencies)
972 {
973 	struct cpufreq_frequency_table *table = policy->freq_table;
974 	struct cpufreq_frequency_table *pos;
975 	unsigned int freq;
976 	int idx, best = -1;
977 
978 	cpufreq_for_each_efficient_entry_idx(pos, table, idx, efficiencies) {
979 		freq = pos->frequency;
980 
981 		if (freq == target_freq)
982 			return idx;
983 
984 		if (freq < target_freq) {
985 			best = idx;
986 			continue;
987 		}
988 
989 		/* No freq found below target_freq */
990 		if (best == -1)
991 			return idx;
992 
993 		/* Choose the closest freq */
994 		if (target_freq - table[best].frequency > freq - target_freq)
995 			return idx;
996 
997 		return best;
998 	}
999 
1000 	return best;
1001 }
1002 
1003 /* Find closest freq to target in a table in descending order */
1004 static inline int cpufreq_table_find_index_dc(struct cpufreq_policy *policy,
1005 					      unsigned int target_freq,
1006 					      bool efficiencies)
1007 {
1008 	struct cpufreq_frequency_table *table = policy->freq_table;
1009 	struct cpufreq_frequency_table *pos;
1010 	unsigned int freq;
1011 	int idx, best = -1;
1012 
1013 	cpufreq_for_each_efficient_entry_idx(pos, table, idx, efficiencies) {
1014 		freq = pos->frequency;
1015 
1016 		if (freq == target_freq)
1017 			return idx;
1018 
1019 		if (freq > target_freq) {
1020 			best = idx;
1021 			continue;
1022 		}
1023 
1024 		/* No freq found above target_freq */
1025 		if (best == -1)
1026 			return idx;
1027 
1028 		/* Choose the closest freq */
1029 		if (table[best].frequency - target_freq > target_freq - freq)
1030 			return idx;
1031 
1032 		return best;
1033 	}
1034 
1035 	return best;
1036 }
1037 
1038 static inline int find_index_c(struct cpufreq_policy *policy,
1039 			       unsigned int target_freq,
1040 			       unsigned int min, unsigned int max,
1041 			       bool efficiencies)
1042 {
1043 	target_freq = clamp_val(target_freq, min, max);
1044 
1045 	if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING)
1046 		return cpufreq_table_find_index_ac(policy, target_freq,
1047 						   efficiencies);
1048 	else
1049 		return cpufreq_table_find_index_dc(policy, target_freq,
1050 						   efficiencies);
1051 }
1052 
1053 /* Works only on sorted freq-tables */
1054 static inline int cpufreq_table_find_index_c(struct cpufreq_policy *policy,
1055 					     unsigned int target_freq,
1056 					     bool efficiencies)
1057 {
1058 	return find_index_c(policy, target_freq, policy->min, policy->max, efficiencies);
1059 }
1060 
1061 static inline bool cpufreq_is_in_limits(struct cpufreq_policy *policy,
1062 					unsigned int min, unsigned int max,
1063 					int idx)
1064 {
1065 	unsigned int freq;
1066 
1067 	if (idx < 0)
1068 		return false;
1069 
1070 	freq = policy->freq_table[idx].frequency;
1071 
1072 	return freq == clamp_val(freq, min, max);
1073 }
1074 
1075 static inline int cpufreq_frequency_table_target(struct cpufreq_policy *policy,
1076 						 unsigned int target_freq,
1077 						 unsigned int min,
1078 						 unsigned int max,
1079 						 unsigned int relation)
1080 {
1081 	bool efficiencies = policy->efficiencies_available &&
1082 			    (relation & CPUFREQ_RELATION_E);
1083 	int idx;
1084 
1085 	/* cpufreq_table_index_unsorted() has no use for this flag anyway */
1086 	relation &= ~CPUFREQ_RELATION_E;
1087 
1088 	if (unlikely(policy->freq_table_sorted == CPUFREQ_TABLE_UNSORTED))
1089 		return cpufreq_table_index_unsorted(policy, target_freq, min,
1090 						    max, relation);
1091 retry:
1092 	switch (relation) {
1093 	case CPUFREQ_RELATION_L:
1094 		idx = find_index_l(policy, target_freq, min, max, efficiencies);
1095 		break;
1096 	case CPUFREQ_RELATION_H:
1097 		idx = find_index_h(policy, target_freq, min, max, efficiencies);
1098 		break;
1099 	case CPUFREQ_RELATION_C:
1100 		idx = find_index_c(policy, target_freq, min, max, efficiencies);
1101 		break;
1102 	default:
1103 		WARN_ON_ONCE(1);
1104 		return 0;
1105 	}
1106 
1107 	/* Limit frequency index to honor min and max */
1108 	if (!cpufreq_is_in_limits(policy, min, max, idx) && efficiencies) {
1109 		efficiencies = false;
1110 		goto retry;
1111 	}
1112 
1113 	return idx;
1114 }
1115 
1116 static inline int cpufreq_table_count_valid_entries(const struct cpufreq_policy *policy)
1117 {
1118 	struct cpufreq_frequency_table *pos;
1119 	int count = 0;
1120 
1121 	if (unlikely(!policy->freq_table))
1122 		return 0;
1123 
1124 	cpufreq_for_each_valid_entry(pos, policy->freq_table)
1125 		count++;
1126 
1127 	return count;
1128 }
1129 
1130 /**
1131  * cpufreq_table_set_inefficient() - Mark a frequency as inefficient
1132  * @policy:	the &struct cpufreq_policy containing the inefficient frequency
1133  * @frequency:	the inefficient frequency
1134  *
1135  * The &struct cpufreq_policy must use a sorted frequency table
1136  *
1137  * Return:	%0 on success or a negative errno code
1138  */
1139 
1140 static inline int
1141 cpufreq_table_set_inefficient(struct cpufreq_policy *policy,
1142 			      unsigned int frequency)
1143 {
1144 	struct cpufreq_frequency_table *pos;
1145 
1146 	/* Not supported */
1147 	if (policy->freq_table_sorted == CPUFREQ_TABLE_UNSORTED)
1148 		return -EINVAL;
1149 
1150 	cpufreq_for_each_valid_entry(pos, policy->freq_table) {
1151 		if (pos->frequency == frequency) {
1152 			pos->flags |= CPUFREQ_INEFFICIENT_FREQ;
1153 			policy->efficiencies_available = true;
1154 			return 0;
1155 		}
1156 	}
1157 
1158 	return -EINVAL;
1159 }
1160 
1161 static inline int parse_perf_domain(int cpu, const char *list_name,
1162 				    const char *cell_name,
1163 				    struct of_phandle_args *args)
1164 {
1165 	int ret;
1166 
1167 	struct device_node *cpu_np __free(device_node) = of_cpu_device_node_get(cpu);
1168 	if (!cpu_np)
1169 		return -ENODEV;
1170 
1171 	ret = of_parse_phandle_with_args(cpu_np, list_name, cell_name, 0,
1172 					 args);
1173 	if (ret < 0)
1174 		return ret;
1175 	return 0;
1176 }
1177 
1178 static inline int of_perf_domain_get_sharing_cpumask(int pcpu, const char *list_name,
1179 						     const char *cell_name, struct cpumask *cpumask,
1180 						     struct of_phandle_args *pargs)
1181 {
1182 	int cpu, ret;
1183 	struct of_phandle_args args;
1184 
1185 	ret = parse_perf_domain(pcpu, list_name, cell_name, pargs);
1186 	if (ret < 0)
1187 		return ret;
1188 
1189 	cpumask_set_cpu(pcpu, cpumask);
1190 
1191 	for_each_possible_cpu(cpu) {
1192 		if (cpu == pcpu)
1193 			continue;
1194 
1195 		ret = parse_perf_domain(cpu, list_name, cell_name, &args);
1196 		if (ret < 0)
1197 			continue;
1198 
1199 		if (of_phandle_args_equal(pargs, &args))
1200 			cpumask_set_cpu(cpu, cpumask);
1201 
1202 		of_node_put(args.np);
1203 	}
1204 
1205 	return 0;
1206 }
1207 #else
1208 static inline bool cpufreq_boost_enabled(void)
1209 {
1210 	return false;
1211 }
1212 
1213 static inline int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
1214 {
1215 	return -EOPNOTSUPP;
1216 }
1217 
1218 static inline int
1219 cpufreq_table_set_inefficient(struct cpufreq_policy *policy,
1220 			      unsigned int frequency)
1221 {
1222 	return -EINVAL;
1223 }
1224 
1225 static inline int of_perf_domain_get_sharing_cpumask(int pcpu, const char *list_name,
1226 						     const char *cell_name, struct cpumask *cpumask,
1227 						     struct of_phandle_args *pargs)
1228 {
1229 	return -EOPNOTSUPP;
1230 }
1231 #endif
1232 
1233 extern int arch_freq_get_on_cpu(int cpu);
1234 
1235 #ifndef arch_set_freq_scale
1236 static __always_inline
1237 void arch_set_freq_scale(const struct cpumask *cpus,
1238 			 unsigned long cur_freq,
1239 			 unsigned long max_freq)
1240 {
1241 }
1242 #endif
1243 
1244 /* the following are really really optional */
1245 extern struct freq_attr cpufreq_freq_attr_scaling_available_freqs;
1246 extern struct freq_attr cpufreq_freq_attr_scaling_boost_freqs;
1247 int cpufreq_table_validate_and_sort(struct cpufreq_policy *policy);
1248 
1249 unsigned int cpufreq_generic_get(unsigned int cpu);
1250 void cpufreq_generic_init(struct cpufreq_policy *policy,
1251 		struct cpufreq_frequency_table *table,
1252 		unsigned int transition_latency);
1253 
1254 bool cpufreq_ready_for_eas(const struct cpumask *cpu_mask);
1255 
1256 static inline void cpufreq_register_em_with_opp(struct cpufreq_policy *policy)
1257 {
1258 	dev_pm_opp_of_register_em(get_cpu_device(policy->cpu),
1259 				  policy->related_cpus);
1260 }
1261 #endif /* _LINUX_CPUFREQ_H */
1262