1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/drivers/cpufreq/cpufreq.c 4 * 5 * Copyright (C) 2001 Russell King 6 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> 7 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org> 8 * 9 * Oct 2005 - Ashok Raj <ashok.raj@intel.com> 10 * Added handling for CPU hotplug 11 * Feb 2006 - Jacob Shin <jacob.shin@amd.com> 12 * Fix handling for CPU hotplug -- affected CPUs 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/cpu.h> 18 #include <linux/cpufreq.h> 19 #include <linux/cpu_cooling.h> 20 #include <linux/delay.h> 21 #include <linux/device.h> 22 #include <linux/init.h> 23 #include <linux/kernel_stat.h> 24 #include <linux/module.h> 25 #include <linux/mutex.h> 26 #include <linux/pm_qos.h> 27 #include <linux/slab.h> 28 #include <linux/string_choices.h> 29 #include <linux/suspend.h> 30 #include <linux/syscore_ops.h> 31 #include <linux/tick.h> 32 #include <linux/units.h> 33 #include <trace/events/power.h> 34 35 static LIST_HEAD(cpufreq_policy_list); 36 37 /* Macros to iterate over CPU policies */ 38 #define for_each_suitable_policy(__policy, __active) \ 39 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \ 40 if ((__active) == !policy_is_inactive(__policy)) 41 42 #define for_each_active_policy(__policy) \ 43 for_each_suitable_policy(__policy, true) 44 #define for_each_inactive_policy(__policy) \ 45 for_each_suitable_policy(__policy, false) 46 47 /* Iterate over governors */ 48 static LIST_HEAD(cpufreq_governor_list); 49 #define for_each_governor(__governor) \ 50 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list) 51 52 static char default_governor[CPUFREQ_NAME_LEN]; 53 54 /* 55 * The "cpufreq driver" - the arch- or hardware-dependent low 56 * level driver of CPUFreq support, and its spinlock. This lock 57 * also protects the cpufreq_cpu_data array. 58 */ 59 static struct cpufreq_driver *cpufreq_driver; 60 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); 61 static DEFINE_RWLOCK(cpufreq_driver_lock); 62 63 static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance); 64 bool cpufreq_supports_freq_invariance(void) 65 { 66 return static_branch_likely(&cpufreq_freq_invariance); 67 } 68 69 /* Flag to suspend/resume CPUFreq governors */ 70 static bool cpufreq_suspended; 71 72 static inline bool has_target(void) 73 { 74 return cpufreq_driver->target_index || cpufreq_driver->target; 75 } 76 77 bool has_target_index(void) 78 { 79 return !!cpufreq_driver->target_index; 80 } 81 82 /* internal prototypes */ 83 static unsigned int __cpufreq_get(struct cpufreq_policy *policy); 84 static int cpufreq_init_governor(struct cpufreq_policy *policy); 85 static void cpufreq_exit_governor(struct cpufreq_policy *policy); 86 static void cpufreq_governor_limits(struct cpufreq_policy *policy); 87 static int cpufreq_set_policy(struct cpufreq_policy *policy, 88 struct cpufreq_governor *new_gov, 89 unsigned int new_pol); 90 static bool cpufreq_boost_supported(void); 91 static int cpufreq_boost_trigger_state(int state); 92 93 /* 94 * Two notifier lists: the "policy" list is involved in the 95 * validation process for a new CPU frequency policy; the 96 * "transition" list for kernel code that needs to handle 97 * changes to devices when the CPU clock speed changes. 98 * The mutex locks both lists. 99 */ 100 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list); 101 SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list); 102 103 static int off __read_mostly; 104 static int cpufreq_disabled(void) 105 { 106 return off; 107 } 108 void disable_cpufreq(void) 109 { 110 off = 1; 111 } 112 static DEFINE_MUTEX(cpufreq_governor_mutex); 113 114 bool have_governor_per_policy(void) 115 { 116 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY); 117 } 118 EXPORT_SYMBOL_GPL(have_governor_per_policy); 119 120 static struct kobject *cpufreq_global_kobject; 121 122 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) 123 { 124 if (have_governor_per_policy()) 125 return &policy->kobj; 126 else 127 return cpufreq_global_kobject; 128 } 129 EXPORT_SYMBOL_GPL(get_governor_parent_kobj); 130 131 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) 132 { 133 struct kernel_cpustat kcpustat; 134 u64 cur_wall_time; 135 u64 idle_time; 136 u64 busy_time; 137 138 cur_wall_time = jiffies64_to_nsecs(get_jiffies_64()); 139 140 kcpustat_cpu_fetch(&kcpustat, cpu); 141 142 busy_time = kcpustat.cpustat[CPUTIME_USER]; 143 busy_time += kcpustat.cpustat[CPUTIME_SYSTEM]; 144 busy_time += kcpustat.cpustat[CPUTIME_IRQ]; 145 busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ]; 146 busy_time += kcpustat.cpustat[CPUTIME_STEAL]; 147 busy_time += kcpustat.cpustat[CPUTIME_NICE]; 148 149 idle_time = cur_wall_time - busy_time; 150 if (wall) 151 *wall = div_u64(cur_wall_time, NSEC_PER_USEC); 152 153 return div_u64(idle_time, NSEC_PER_USEC); 154 } 155 156 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) 157 { 158 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL); 159 160 if (idle_time == -1ULL) 161 return get_cpu_idle_time_jiffy(cpu, wall); 162 else if (!io_busy) 163 idle_time += get_cpu_iowait_time_us(cpu, wall); 164 165 return idle_time; 166 } 167 EXPORT_SYMBOL_GPL(get_cpu_idle_time); 168 169 /* 170 * This is a generic cpufreq init() routine which can be used by cpufreq 171 * drivers of SMP systems. It will do following: 172 * - validate & show freq table passed 173 * - set policies transition latency 174 * - policy->cpus with all possible CPUs 175 */ 176 void cpufreq_generic_init(struct cpufreq_policy *policy, 177 struct cpufreq_frequency_table *table, 178 unsigned int transition_latency) 179 { 180 policy->freq_table = table; 181 policy->cpuinfo.transition_latency = transition_latency; 182 183 /* 184 * The driver only supports the SMP configuration where all processors 185 * share the clock and voltage and clock. 186 */ 187 cpumask_setall(policy->cpus); 188 } 189 EXPORT_SYMBOL_GPL(cpufreq_generic_init); 190 191 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu) 192 { 193 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 194 195 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL; 196 } 197 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw); 198 199 unsigned int cpufreq_generic_get(unsigned int cpu) 200 { 201 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); 202 203 if (!policy || IS_ERR(policy->clk)) { 204 pr_err("%s: No %s associated to cpu: %d\n", 205 __func__, policy ? "clk" : "policy", cpu); 206 return 0; 207 } 208 209 return clk_get_rate(policy->clk) / 1000; 210 } 211 EXPORT_SYMBOL_GPL(cpufreq_generic_get); 212 213 /** 214 * cpufreq_cpu_get - Return policy for a CPU and mark it as busy. 215 * @cpu: CPU to find the policy for. 216 * 217 * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment 218 * the kobject reference counter of that policy. Return a valid policy on 219 * success or NULL on failure. 220 * 221 * The policy returned by this function has to be released with the help of 222 * cpufreq_cpu_put() to balance its kobject reference counter properly. 223 */ 224 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) 225 { 226 struct cpufreq_policy *policy = NULL; 227 unsigned long flags; 228 229 if (WARN_ON(cpu >= nr_cpu_ids)) 230 return NULL; 231 232 /* get the cpufreq driver */ 233 read_lock_irqsave(&cpufreq_driver_lock, flags); 234 235 if (cpufreq_driver) { 236 /* get the CPU */ 237 policy = cpufreq_cpu_get_raw(cpu); 238 if (policy) 239 kobject_get(&policy->kobj); 240 } 241 242 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 243 244 return policy; 245 } 246 EXPORT_SYMBOL_GPL(cpufreq_cpu_get); 247 248 /** 249 * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy. 250 * @policy: cpufreq policy returned by cpufreq_cpu_get(). 251 */ 252 void cpufreq_cpu_put(struct cpufreq_policy *policy) 253 { 254 kobject_put(&policy->kobj); 255 } 256 EXPORT_SYMBOL_GPL(cpufreq_cpu_put); 257 258 /** 259 * cpufreq_cpu_release - Unlock a policy and decrement its usage counter. 260 * @policy: cpufreq policy returned by cpufreq_cpu_acquire(). 261 */ 262 void cpufreq_cpu_release(struct cpufreq_policy *policy) 263 { 264 if (WARN_ON(!policy)) 265 return; 266 267 lockdep_assert_held(&policy->rwsem); 268 269 up_write(&policy->rwsem); 270 271 cpufreq_cpu_put(policy); 272 } 273 274 /** 275 * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it. 276 * @cpu: CPU to find the policy for. 277 * 278 * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and 279 * if the policy returned by it is not NULL, acquire its rwsem for writing. 280 * Return the policy if it is active or release it and return NULL otherwise. 281 * 282 * The policy returned by this function has to be released with the help of 283 * cpufreq_cpu_release() in order to release its rwsem and balance its usage 284 * counter properly. 285 */ 286 struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu) 287 { 288 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 289 290 if (!policy) 291 return NULL; 292 293 down_write(&policy->rwsem); 294 295 if (policy_is_inactive(policy)) { 296 cpufreq_cpu_release(policy); 297 return NULL; 298 } 299 300 return policy; 301 } 302 303 /********************************************************************* 304 * EXTERNALLY AFFECTING FREQUENCY CHANGES * 305 *********************************************************************/ 306 307 /** 308 * adjust_jiffies - Adjust the system "loops_per_jiffy". 309 * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE. 310 * @ci: Frequency change information. 311 * 312 * This function alters the system "loops_per_jiffy" for the clock 313 * speed change. Note that loops_per_jiffy cannot be updated on SMP 314 * systems as each CPU might be scaled differently. So, use the arch 315 * per-CPU loops_per_jiffy value wherever possible. 316 */ 317 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) 318 { 319 #ifndef CONFIG_SMP 320 static unsigned long l_p_j_ref; 321 static unsigned int l_p_j_ref_freq; 322 323 if (ci->flags & CPUFREQ_CONST_LOOPS) 324 return; 325 326 if (!l_p_j_ref_freq) { 327 l_p_j_ref = loops_per_jiffy; 328 l_p_j_ref_freq = ci->old; 329 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n", 330 l_p_j_ref, l_p_j_ref_freq); 331 } 332 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) { 333 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq, 334 ci->new); 335 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n", 336 loops_per_jiffy, ci->new); 337 } 338 #endif 339 } 340 341 /** 342 * cpufreq_notify_transition - Notify frequency transition and adjust jiffies. 343 * @policy: cpufreq policy to enable fast frequency switching for. 344 * @freqs: contain details of the frequency update. 345 * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE. 346 * 347 * This function calls the transition notifiers and adjust_jiffies(). 348 * 349 * It is called twice on all CPU frequency changes that have external effects. 350 */ 351 static void cpufreq_notify_transition(struct cpufreq_policy *policy, 352 struct cpufreq_freqs *freqs, 353 unsigned int state) 354 { 355 int cpu; 356 357 BUG_ON(irqs_disabled()); 358 359 if (cpufreq_disabled()) 360 return; 361 362 freqs->policy = policy; 363 freqs->flags = cpufreq_driver->flags; 364 pr_debug("notification %u of frequency transition to %u kHz\n", 365 state, freqs->new); 366 367 switch (state) { 368 case CPUFREQ_PRECHANGE: 369 /* 370 * Detect if the driver reported a value as "old frequency" 371 * which is not equal to what the cpufreq core thinks is 372 * "old frequency". 373 */ 374 if (policy->cur && policy->cur != freqs->old) { 375 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n", 376 freqs->old, policy->cur); 377 freqs->old = policy->cur; 378 } 379 380 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 381 CPUFREQ_PRECHANGE, freqs); 382 383 adjust_jiffies(CPUFREQ_PRECHANGE, freqs); 384 break; 385 386 case CPUFREQ_POSTCHANGE: 387 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs); 388 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new, 389 cpumask_pr_args(policy->cpus)); 390 391 for_each_cpu(cpu, policy->cpus) 392 trace_cpu_frequency(freqs->new, cpu); 393 394 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 395 CPUFREQ_POSTCHANGE, freqs); 396 397 cpufreq_stats_record_transition(policy, freqs->new); 398 policy->cur = freqs->new; 399 } 400 } 401 402 /* Do post notifications when there are chances that transition has failed */ 403 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy, 404 struct cpufreq_freqs *freqs, int transition_failed) 405 { 406 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 407 if (!transition_failed) 408 return; 409 410 swap(freqs->old, freqs->new); 411 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 412 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 413 } 414 415 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, 416 struct cpufreq_freqs *freqs) 417 { 418 419 /* 420 * Catch double invocations of _begin() which lead to self-deadlock. 421 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core 422 * doesn't invoke _begin() on their behalf, and hence the chances of 423 * double invocations are very low. Moreover, there are scenarios 424 * where these checks can emit false-positive warnings in these 425 * drivers; so we avoid that by skipping them altogether. 426 */ 427 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION) 428 && current == policy->transition_task); 429 430 wait: 431 wait_event(policy->transition_wait, !policy->transition_ongoing); 432 433 spin_lock(&policy->transition_lock); 434 435 if (unlikely(policy->transition_ongoing)) { 436 spin_unlock(&policy->transition_lock); 437 goto wait; 438 } 439 440 policy->transition_ongoing = true; 441 policy->transition_task = current; 442 443 spin_unlock(&policy->transition_lock); 444 445 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 446 } 447 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin); 448 449 void cpufreq_freq_transition_end(struct cpufreq_policy *policy, 450 struct cpufreq_freqs *freqs, int transition_failed) 451 { 452 if (WARN_ON(!policy->transition_ongoing)) 453 return; 454 455 cpufreq_notify_post_transition(policy, freqs, transition_failed); 456 457 arch_set_freq_scale(policy->related_cpus, 458 policy->cur, 459 arch_scale_freq_ref(policy->cpu)); 460 461 spin_lock(&policy->transition_lock); 462 policy->transition_ongoing = false; 463 policy->transition_task = NULL; 464 spin_unlock(&policy->transition_lock); 465 466 wake_up(&policy->transition_wait); 467 } 468 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end); 469 470 /* 471 * Fast frequency switching status count. Positive means "enabled", negative 472 * means "disabled" and 0 means "not decided yet". 473 */ 474 static int cpufreq_fast_switch_count; 475 static DEFINE_MUTEX(cpufreq_fast_switch_lock); 476 477 static void cpufreq_list_transition_notifiers(void) 478 { 479 struct notifier_block *nb; 480 481 pr_info("Registered transition notifiers:\n"); 482 483 mutex_lock(&cpufreq_transition_notifier_list.mutex); 484 485 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next) 486 pr_info("%pS\n", nb->notifier_call); 487 488 mutex_unlock(&cpufreq_transition_notifier_list.mutex); 489 } 490 491 /** 492 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy. 493 * @policy: cpufreq policy to enable fast frequency switching for. 494 * 495 * Try to enable fast frequency switching for @policy. 496 * 497 * The attempt will fail if there is at least one transition notifier registered 498 * at this point, as fast frequency switching is quite fundamentally at odds 499 * with transition notifiers. Thus if successful, it will make registration of 500 * transition notifiers fail going forward. 501 */ 502 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy) 503 { 504 lockdep_assert_held(&policy->rwsem); 505 506 if (!policy->fast_switch_possible) 507 return; 508 509 mutex_lock(&cpufreq_fast_switch_lock); 510 if (cpufreq_fast_switch_count >= 0) { 511 cpufreq_fast_switch_count++; 512 policy->fast_switch_enabled = true; 513 } else { 514 pr_warn("CPU%u: Fast frequency switching not enabled\n", 515 policy->cpu); 516 cpufreq_list_transition_notifiers(); 517 } 518 mutex_unlock(&cpufreq_fast_switch_lock); 519 } 520 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch); 521 522 /** 523 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy. 524 * @policy: cpufreq policy to disable fast frequency switching for. 525 */ 526 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) 527 { 528 mutex_lock(&cpufreq_fast_switch_lock); 529 if (policy->fast_switch_enabled) { 530 policy->fast_switch_enabled = false; 531 if (!WARN_ON(cpufreq_fast_switch_count <= 0)) 532 cpufreq_fast_switch_count--; 533 } 534 mutex_unlock(&cpufreq_fast_switch_lock); 535 } 536 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); 537 538 static unsigned int __resolve_freq(struct cpufreq_policy *policy, 539 unsigned int target_freq, unsigned int relation) 540 { 541 unsigned int idx; 542 543 if (!policy->freq_table) 544 return target_freq; 545 546 idx = cpufreq_frequency_table_target(policy, target_freq, relation); 547 policy->cached_resolved_idx = idx; 548 policy->cached_target_freq = target_freq; 549 return policy->freq_table[idx].frequency; 550 } 551 552 /** 553 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported 554 * one. 555 * @policy: associated policy to interrogate 556 * @target_freq: target frequency to resolve. 557 * 558 * The target to driver frequency mapping is cached in the policy. 559 * 560 * Return: Lowest driver-supported frequency greater than or equal to the 561 * given target_freq, subject to policy (min/max) and driver limitations. 562 */ 563 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy, 564 unsigned int target_freq) 565 { 566 unsigned int min = READ_ONCE(policy->min); 567 unsigned int max = READ_ONCE(policy->max); 568 569 /* 570 * If this function runs in parallel with cpufreq_set_policy(), it may 571 * read policy->min before the update and policy->max after the update 572 * or the other way around, so there is no ordering guarantee. 573 * 574 * Resolve this by always honoring the max (in case it comes from 575 * thermal throttling or similar). 576 */ 577 if (unlikely(min > max)) 578 min = max; 579 580 return __resolve_freq(policy, clamp_val(target_freq, min, max), 581 CPUFREQ_RELATION_LE); 582 } 583 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq); 584 585 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy) 586 { 587 unsigned int latency; 588 589 if (policy->transition_delay_us) 590 return policy->transition_delay_us; 591 592 latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC; 593 if (latency) 594 /* Give a 50% breathing room between updates */ 595 return latency + (latency >> 1); 596 597 return USEC_PER_MSEC; 598 } 599 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us); 600 601 /********************************************************************* 602 * SYSFS INTERFACE * 603 *********************************************************************/ 604 static ssize_t show_boost(struct kobject *kobj, 605 struct kobj_attribute *attr, char *buf) 606 { 607 return sysfs_emit(buf, "%d\n", cpufreq_driver->boost_enabled); 608 } 609 610 static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr, 611 const char *buf, size_t count) 612 { 613 bool enable; 614 615 if (kstrtobool(buf, &enable)) 616 return -EINVAL; 617 618 if (cpufreq_boost_trigger_state(enable)) { 619 pr_err("%s: Cannot %s BOOST!\n", 620 __func__, str_enable_disable(enable)); 621 return -EINVAL; 622 } 623 624 pr_debug("%s: cpufreq BOOST %s\n", 625 __func__, str_enabled_disabled(enable)); 626 627 return count; 628 } 629 define_one_global_rw(boost); 630 631 static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf) 632 { 633 return sysfs_emit(buf, "%d\n", policy->boost_enabled); 634 } 635 636 static ssize_t store_local_boost(struct cpufreq_policy *policy, 637 const char *buf, size_t count) 638 { 639 int ret; 640 bool enable; 641 642 if (kstrtobool(buf, &enable)) 643 return -EINVAL; 644 645 if (!cpufreq_driver->boost_enabled) 646 return -EINVAL; 647 648 if (!policy->boost_supported) 649 return -EINVAL; 650 651 if (policy->boost_enabled == enable) 652 return count; 653 654 policy->boost_enabled = enable; 655 656 cpus_read_lock(); 657 ret = cpufreq_driver->set_boost(policy, enable); 658 cpus_read_unlock(); 659 660 if (ret) { 661 policy->boost_enabled = !policy->boost_enabled; 662 return ret; 663 } 664 665 return count; 666 } 667 668 static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost); 669 670 static struct cpufreq_governor *find_governor(const char *str_governor) 671 { 672 struct cpufreq_governor *t; 673 674 for_each_governor(t) 675 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN)) 676 return t; 677 678 return NULL; 679 } 680 681 static struct cpufreq_governor *get_governor(const char *str_governor) 682 { 683 struct cpufreq_governor *t; 684 685 mutex_lock(&cpufreq_governor_mutex); 686 t = find_governor(str_governor); 687 if (!t) 688 goto unlock; 689 690 if (!try_module_get(t->owner)) 691 t = NULL; 692 693 unlock: 694 mutex_unlock(&cpufreq_governor_mutex); 695 696 return t; 697 } 698 699 static unsigned int cpufreq_parse_policy(char *str_governor) 700 { 701 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) 702 return CPUFREQ_POLICY_PERFORMANCE; 703 704 if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN)) 705 return CPUFREQ_POLICY_POWERSAVE; 706 707 return CPUFREQ_POLICY_UNKNOWN; 708 } 709 710 /** 711 * cpufreq_parse_governor - parse a governor string only for has_target() 712 * @str_governor: Governor name. 713 */ 714 static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor) 715 { 716 struct cpufreq_governor *t; 717 718 t = get_governor(str_governor); 719 if (t) 720 return t; 721 722 if (request_module("cpufreq_%s", str_governor)) 723 return NULL; 724 725 return get_governor(str_governor); 726 } 727 728 /* 729 * cpufreq_per_cpu_attr_read() / show_##file_name() - 730 * print out cpufreq information 731 * 732 * Write out information from cpufreq_driver->policy[cpu]; object must be 733 * "unsigned int". 734 */ 735 736 #define show_one(file_name, object) \ 737 static ssize_t show_##file_name \ 738 (struct cpufreq_policy *policy, char *buf) \ 739 { \ 740 return sysfs_emit(buf, "%u\n", policy->object); \ 741 } 742 743 show_one(cpuinfo_min_freq, cpuinfo.min_freq); 744 show_one(cpuinfo_max_freq, cpuinfo.max_freq); 745 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency); 746 show_one(scaling_min_freq, min); 747 show_one(scaling_max_freq, max); 748 749 __weak int arch_freq_get_on_cpu(int cpu) 750 { 751 return -EOPNOTSUPP; 752 } 753 754 static inline bool cpufreq_avg_freq_supported(struct cpufreq_policy *policy) 755 { 756 return arch_freq_get_on_cpu(policy->cpu) != -EOPNOTSUPP; 757 } 758 759 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf) 760 { 761 ssize_t ret; 762 int freq; 763 764 freq = IS_ENABLED(CONFIG_CPUFREQ_ARCH_CUR_FREQ) 765 ? arch_freq_get_on_cpu(policy->cpu) 766 : 0; 767 768 if (freq > 0) 769 ret = sysfs_emit(buf, "%u\n", freq); 770 else if (cpufreq_driver->setpolicy && cpufreq_driver->get) 771 ret = sysfs_emit(buf, "%u\n", cpufreq_driver->get(policy->cpu)); 772 else 773 ret = sysfs_emit(buf, "%u\n", policy->cur); 774 return ret; 775 } 776 777 /* 778 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access 779 */ 780 #define store_one(file_name, object) \ 781 static ssize_t store_##file_name \ 782 (struct cpufreq_policy *policy, const char *buf, size_t count) \ 783 { \ 784 unsigned long val; \ 785 int ret; \ 786 \ 787 ret = kstrtoul(buf, 0, &val); \ 788 if (ret) \ 789 return ret; \ 790 \ 791 ret = freq_qos_update_request(policy->object##_freq_req, val);\ 792 return ret >= 0 ? count : ret; \ 793 } 794 795 store_one(scaling_min_freq, min); 796 store_one(scaling_max_freq, max); 797 798 /* 799 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware 800 */ 801 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, 802 char *buf) 803 { 804 unsigned int cur_freq = __cpufreq_get(policy); 805 806 if (cur_freq) 807 return sysfs_emit(buf, "%u\n", cur_freq); 808 809 return sysfs_emit(buf, "<unknown>\n"); 810 } 811 812 /* 813 * show_cpuinfo_avg_freq - average CPU frequency as detected by hardware 814 */ 815 static ssize_t show_cpuinfo_avg_freq(struct cpufreq_policy *policy, 816 char *buf) 817 { 818 int avg_freq = arch_freq_get_on_cpu(policy->cpu); 819 820 if (avg_freq > 0) 821 return sysfs_emit(buf, "%u\n", avg_freq); 822 return avg_freq != 0 ? avg_freq : -EINVAL; 823 } 824 825 /* 826 * show_scaling_governor - show the current policy for the specified CPU 827 */ 828 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) 829 { 830 if (policy->policy == CPUFREQ_POLICY_POWERSAVE) 831 return sysfs_emit(buf, "powersave\n"); 832 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) 833 return sysfs_emit(buf, "performance\n"); 834 else if (policy->governor) 835 return sysfs_emit(buf, "%s\n", policy->governor->name); 836 return -EINVAL; 837 } 838 839 /* 840 * store_scaling_governor - store policy for the specified CPU 841 */ 842 static ssize_t store_scaling_governor(struct cpufreq_policy *policy, 843 const char *buf, size_t count) 844 { 845 char str_governor[16]; 846 int ret; 847 848 ret = sscanf(buf, "%15s", str_governor); 849 if (ret != 1) 850 return -EINVAL; 851 852 if (cpufreq_driver->setpolicy) { 853 unsigned int new_pol; 854 855 new_pol = cpufreq_parse_policy(str_governor); 856 if (!new_pol) 857 return -EINVAL; 858 859 ret = cpufreq_set_policy(policy, NULL, new_pol); 860 } else { 861 struct cpufreq_governor *new_gov; 862 863 new_gov = cpufreq_parse_governor(str_governor); 864 if (!new_gov) 865 return -EINVAL; 866 867 ret = cpufreq_set_policy(policy, new_gov, 868 CPUFREQ_POLICY_UNKNOWN); 869 870 module_put(new_gov->owner); 871 } 872 873 return ret ? ret : count; 874 } 875 876 /* 877 * show_scaling_driver - show the cpufreq driver currently loaded 878 */ 879 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf) 880 { 881 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name); 882 } 883 884 /* 885 * show_scaling_available_governors - show the available CPUfreq governors 886 */ 887 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy, 888 char *buf) 889 { 890 ssize_t i = 0; 891 struct cpufreq_governor *t; 892 893 if (!has_target()) { 894 i += sysfs_emit(buf, "performance powersave"); 895 goto out; 896 } 897 898 mutex_lock(&cpufreq_governor_mutex); 899 for_each_governor(t) { 900 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) 901 - (CPUFREQ_NAME_LEN + 2))) 902 break; 903 i += sysfs_emit_at(buf, i, "%s ", t->name); 904 } 905 mutex_unlock(&cpufreq_governor_mutex); 906 out: 907 i += sysfs_emit_at(buf, i, "\n"); 908 return i; 909 } 910 911 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf) 912 { 913 ssize_t i = 0; 914 unsigned int cpu; 915 916 for_each_cpu(cpu, mask) { 917 i += sysfs_emit_at(buf, i, "%u ", cpu); 918 if (i >= (PAGE_SIZE - 5)) 919 break; 920 } 921 922 /* Remove the extra space at the end */ 923 i--; 924 925 i += sysfs_emit_at(buf, i, "\n"); 926 return i; 927 } 928 EXPORT_SYMBOL_GPL(cpufreq_show_cpus); 929 930 /* 931 * show_related_cpus - show the CPUs affected by each transition even if 932 * hw coordination is in use 933 */ 934 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf) 935 { 936 return cpufreq_show_cpus(policy->related_cpus, buf); 937 } 938 939 /* 940 * show_affected_cpus - show the CPUs affected by each transition 941 */ 942 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf) 943 { 944 return cpufreq_show_cpus(policy->cpus, buf); 945 } 946 947 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy, 948 const char *buf, size_t count) 949 { 950 unsigned int freq = 0; 951 unsigned int ret; 952 953 if (!policy->governor || !policy->governor->store_setspeed) 954 return -EINVAL; 955 956 ret = sscanf(buf, "%u", &freq); 957 if (ret != 1) 958 return -EINVAL; 959 960 policy->governor->store_setspeed(policy, freq); 961 962 return count; 963 } 964 965 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf) 966 { 967 if (!policy->governor || !policy->governor->show_setspeed) 968 return sysfs_emit(buf, "<unsupported>\n"); 969 970 return policy->governor->show_setspeed(policy, buf); 971 } 972 973 /* 974 * show_bios_limit - show the current cpufreq HW/BIOS limitation 975 */ 976 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf) 977 { 978 unsigned int limit; 979 int ret; 980 ret = cpufreq_driver->bios_limit(policy->cpu, &limit); 981 if (!ret) 982 return sysfs_emit(buf, "%u\n", limit); 983 return sysfs_emit(buf, "%u\n", policy->cpuinfo.max_freq); 984 } 985 986 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400); 987 cpufreq_freq_attr_ro(cpuinfo_avg_freq); 988 cpufreq_freq_attr_ro(cpuinfo_min_freq); 989 cpufreq_freq_attr_ro(cpuinfo_max_freq); 990 cpufreq_freq_attr_ro(cpuinfo_transition_latency); 991 cpufreq_freq_attr_ro(scaling_available_governors); 992 cpufreq_freq_attr_ro(scaling_driver); 993 cpufreq_freq_attr_ro(scaling_cur_freq); 994 cpufreq_freq_attr_ro(bios_limit); 995 cpufreq_freq_attr_ro(related_cpus); 996 cpufreq_freq_attr_ro(affected_cpus); 997 cpufreq_freq_attr_rw(scaling_min_freq); 998 cpufreq_freq_attr_rw(scaling_max_freq); 999 cpufreq_freq_attr_rw(scaling_governor); 1000 cpufreq_freq_attr_rw(scaling_setspeed); 1001 1002 static struct attribute *cpufreq_attrs[] = { 1003 &cpuinfo_min_freq.attr, 1004 &cpuinfo_max_freq.attr, 1005 &cpuinfo_transition_latency.attr, 1006 &scaling_min_freq.attr, 1007 &scaling_max_freq.attr, 1008 &affected_cpus.attr, 1009 &related_cpus.attr, 1010 &scaling_governor.attr, 1011 &scaling_driver.attr, 1012 &scaling_available_governors.attr, 1013 &scaling_setspeed.attr, 1014 NULL 1015 }; 1016 ATTRIBUTE_GROUPS(cpufreq); 1017 1018 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj) 1019 #define to_attr(a) container_of(a, struct freq_attr, attr) 1020 1021 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) 1022 { 1023 struct cpufreq_policy *policy = to_policy(kobj); 1024 struct freq_attr *fattr = to_attr(attr); 1025 ssize_t ret = -EBUSY; 1026 1027 if (!fattr->show) 1028 return -EIO; 1029 1030 down_read(&policy->rwsem); 1031 if (likely(!policy_is_inactive(policy))) 1032 ret = fattr->show(policy, buf); 1033 up_read(&policy->rwsem); 1034 1035 return ret; 1036 } 1037 1038 static ssize_t store(struct kobject *kobj, struct attribute *attr, 1039 const char *buf, size_t count) 1040 { 1041 struct cpufreq_policy *policy = to_policy(kobj); 1042 struct freq_attr *fattr = to_attr(attr); 1043 ssize_t ret = -EBUSY; 1044 1045 if (!fattr->store) 1046 return -EIO; 1047 1048 down_write(&policy->rwsem); 1049 if (likely(!policy_is_inactive(policy))) 1050 ret = fattr->store(policy, buf, count); 1051 up_write(&policy->rwsem); 1052 1053 return ret; 1054 } 1055 1056 static void cpufreq_sysfs_release(struct kobject *kobj) 1057 { 1058 struct cpufreq_policy *policy = to_policy(kobj); 1059 pr_debug("last reference is dropped\n"); 1060 complete(&policy->kobj_unregister); 1061 } 1062 1063 static const struct sysfs_ops sysfs_ops = { 1064 .show = show, 1065 .store = store, 1066 }; 1067 1068 static const struct kobj_type ktype_cpufreq = { 1069 .sysfs_ops = &sysfs_ops, 1070 .default_groups = cpufreq_groups, 1071 .release = cpufreq_sysfs_release, 1072 }; 1073 1074 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu, 1075 struct device *dev) 1076 { 1077 if (unlikely(!dev)) 1078 return; 1079 1080 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus)) 1081 return; 1082 1083 dev_dbg(dev, "%s: Adding symlink\n", __func__); 1084 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq")) 1085 dev_err(dev, "cpufreq symlink creation failed\n"); 1086 } 1087 1088 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu, 1089 struct device *dev) 1090 { 1091 dev_dbg(dev, "%s: Removing symlink\n", __func__); 1092 sysfs_remove_link(&dev->kobj, "cpufreq"); 1093 cpumask_clear_cpu(cpu, policy->real_cpus); 1094 } 1095 1096 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy) 1097 { 1098 struct freq_attr **drv_attr; 1099 int ret = 0; 1100 1101 /* Attributes that need freq_table */ 1102 if (policy->freq_table) { 1103 ret = sysfs_create_file(&policy->kobj, 1104 &cpufreq_freq_attr_scaling_available_freqs.attr); 1105 if (ret) 1106 return ret; 1107 1108 if (cpufreq_boost_supported()) { 1109 ret = sysfs_create_file(&policy->kobj, 1110 &cpufreq_freq_attr_scaling_boost_freqs.attr); 1111 if (ret) 1112 return ret; 1113 } 1114 } 1115 1116 /* set up files for this cpu device */ 1117 drv_attr = cpufreq_driver->attr; 1118 while (drv_attr && *drv_attr) { 1119 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr)); 1120 if (ret) 1121 return ret; 1122 drv_attr++; 1123 } 1124 if (cpufreq_driver->get) { 1125 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr); 1126 if (ret) 1127 return ret; 1128 } 1129 1130 if (cpufreq_avg_freq_supported(policy)) { 1131 ret = sysfs_create_file(&policy->kobj, &cpuinfo_avg_freq.attr); 1132 if (ret) 1133 return ret; 1134 } 1135 1136 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr); 1137 if (ret) 1138 return ret; 1139 1140 if (cpufreq_driver->bios_limit) { 1141 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr); 1142 if (ret) 1143 return ret; 1144 } 1145 1146 if (cpufreq_boost_supported()) { 1147 ret = sysfs_create_file(&policy->kobj, &local_boost.attr); 1148 if (ret) 1149 return ret; 1150 } 1151 1152 return 0; 1153 } 1154 1155 static int cpufreq_init_policy(struct cpufreq_policy *policy) 1156 { 1157 struct cpufreq_governor *gov = NULL; 1158 unsigned int pol = CPUFREQ_POLICY_UNKNOWN; 1159 int ret; 1160 1161 if (has_target()) { 1162 /* Update policy governor to the one used before hotplug. */ 1163 gov = get_governor(policy->last_governor); 1164 if (gov) { 1165 pr_debug("Restoring governor %s for cpu %d\n", 1166 gov->name, policy->cpu); 1167 } else { 1168 gov = get_governor(default_governor); 1169 } 1170 1171 if (!gov) { 1172 gov = cpufreq_default_governor(); 1173 __module_get(gov->owner); 1174 } 1175 1176 } else { 1177 1178 /* Use the default policy if there is no last_policy. */ 1179 if (policy->last_policy) { 1180 pol = policy->last_policy; 1181 } else { 1182 pol = cpufreq_parse_policy(default_governor); 1183 /* 1184 * In case the default governor is neither "performance" 1185 * nor "powersave", fall back to the initial policy 1186 * value set by the driver. 1187 */ 1188 if (pol == CPUFREQ_POLICY_UNKNOWN) 1189 pol = policy->policy; 1190 } 1191 if (pol != CPUFREQ_POLICY_PERFORMANCE && 1192 pol != CPUFREQ_POLICY_POWERSAVE) 1193 return -ENODATA; 1194 } 1195 1196 ret = cpufreq_set_policy(policy, gov, pol); 1197 if (gov) 1198 module_put(gov->owner); 1199 1200 return ret; 1201 } 1202 1203 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu) 1204 { 1205 int ret = 0; 1206 1207 /* Has this CPU been taken care of already? */ 1208 if (cpumask_test_cpu(cpu, policy->cpus)) 1209 return 0; 1210 1211 down_write(&policy->rwsem); 1212 if (has_target()) 1213 cpufreq_stop_governor(policy); 1214 1215 cpumask_set_cpu(cpu, policy->cpus); 1216 1217 if (has_target()) { 1218 ret = cpufreq_start_governor(policy); 1219 if (ret) 1220 pr_err("%s: Failed to start governor\n", __func__); 1221 } 1222 up_write(&policy->rwsem); 1223 return ret; 1224 } 1225 1226 void refresh_frequency_limits(struct cpufreq_policy *policy) 1227 { 1228 if (!policy_is_inactive(policy)) { 1229 pr_debug("updating policy for CPU %u\n", policy->cpu); 1230 1231 cpufreq_set_policy(policy, policy->governor, policy->policy); 1232 } 1233 } 1234 EXPORT_SYMBOL(refresh_frequency_limits); 1235 1236 static void handle_update(struct work_struct *work) 1237 { 1238 struct cpufreq_policy *policy = 1239 container_of(work, struct cpufreq_policy, update); 1240 1241 pr_debug("handle_update for cpu %u called\n", policy->cpu); 1242 down_write(&policy->rwsem); 1243 refresh_frequency_limits(policy); 1244 up_write(&policy->rwsem); 1245 } 1246 1247 static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq, 1248 void *data) 1249 { 1250 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min); 1251 1252 schedule_work(&policy->update); 1253 return 0; 1254 } 1255 1256 static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq, 1257 void *data) 1258 { 1259 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max); 1260 1261 schedule_work(&policy->update); 1262 return 0; 1263 } 1264 1265 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) 1266 { 1267 struct kobject *kobj; 1268 struct completion *cmp; 1269 1270 down_write(&policy->rwsem); 1271 cpufreq_stats_free_table(policy); 1272 kobj = &policy->kobj; 1273 cmp = &policy->kobj_unregister; 1274 up_write(&policy->rwsem); 1275 kobject_put(kobj); 1276 1277 /* 1278 * We need to make sure that the underlying kobj is 1279 * actually not referenced anymore by anybody before we 1280 * proceed with unloading. 1281 */ 1282 pr_debug("waiting for dropping of refcount\n"); 1283 wait_for_completion(cmp); 1284 pr_debug("wait complete\n"); 1285 } 1286 1287 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) 1288 { 1289 struct cpufreq_policy *policy; 1290 struct device *dev = get_cpu_device(cpu); 1291 int ret; 1292 1293 if (!dev) 1294 return NULL; 1295 1296 policy = kzalloc(sizeof(*policy), GFP_KERNEL); 1297 if (!policy) 1298 return NULL; 1299 1300 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) 1301 goto err_free_policy; 1302 1303 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) 1304 goto err_free_cpumask; 1305 1306 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL)) 1307 goto err_free_rcpumask; 1308 1309 init_completion(&policy->kobj_unregister); 1310 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, 1311 cpufreq_global_kobject, "policy%u", cpu); 1312 if (ret) { 1313 dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret); 1314 /* 1315 * The entire policy object will be freed below, but the extra 1316 * memory allocated for the kobject name needs to be freed by 1317 * releasing the kobject. 1318 */ 1319 kobject_put(&policy->kobj); 1320 goto err_free_real_cpus; 1321 } 1322 1323 freq_constraints_init(&policy->constraints); 1324 1325 policy->nb_min.notifier_call = cpufreq_notifier_min; 1326 policy->nb_max.notifier_call = cpufreq_notifier_max; 1327 1328 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN, 1329 &policy->nb_min); 1330 if (ret) { 1331 dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n", 1332 ret, cpu); 1333 goto err_kobj_remove; 1334 } 1335 1336 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX, 1337 &policy->nb_max); 1338 if (ret) { 1339 dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n", 1340 ret, cpu); 1341 goto err_min_qos_notifier; 1342 } 1343 1344 INIT_LIST_HEAD(&policy->policy_list); 1345 init_rwsem(&policy->rwsem); 1346 spin_lock_init(&policy->transition_lock); 1347 init_waitqueue_head(&policy->transition_wait); 1348 INIT_WORK(&policy->update, handle_update); 1349 1350 policy->cpu = cpu; 1351 return policy; 1352 1353 err_min_qos_notifier: 1354 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN, 1355 &policy->nb_min); 1356 err_kobj_remove: 1357 cpufreq_policy_put_kobj(policy); 1358 err_free_real_cpus: 1359 free_cpumask_var(policy->real_cpus); 1360 err_free_rcpumask: 1361 free_cpumask_var(policy->related_cpus); 1362 err_free_cpumask: 1363 free_cpumask_var(policy->cpus); 1364 err_free_policy: 1365 kfree(policy); 1366 1367 return NULL; 1368 } 1369 1370 static void cpufreq_policy_free(struct cpufreq_policy *policy) 1371 { 1372 unsigned long flags; 1373 int cpu; 1374 1375 /* 1376 * The callers must ensure the policy is inactive by now, to avoid any 1377 * races with show()/store() callbacks. 1378 */ 1379 if (unlikely(!policy_is_inactive(policy))) 1380 pr_warn("%s: Freeing active policy\n", __func__); 1381 1382 /* Remove policy from list */ 1383 write_lock_irqsave(&cpufreq_driver_lock, flags); 1384 list_del(&policy->policy_list); 1385 1386 for_each_cpu(cpu, policy->related_cpus) 1387 per_cpu(cpufreq_cpu_data, cpu) = NULL; 1388 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1389 1390 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX, 1391 &policy->nb_max); 1392 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN, 1393 &policy->nb_min); 1394 1395 /* Cancel any pending policy->update work before freeing the policy. */ 1396 cancel_work_sync(&policy->update); 1397 1398 if (policy->max_freq_req) { 1399 /* 1400 * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY 1401 * notification, since CPUFREQ_CREATE_POLICY notification was 1402 * sent after adding max_freq_req earlier. 1403 */ 1404 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1405 CPUFREQ_REMOVE_POLICY, policy); 1406 freq_qos_remove_request(policy->max_freq_req); 1407 } 1408 1409 freq_qos_remove_request(policy->min_freq_req); 1410 kfree(policy->min_freq_req); 1411 1412 cpufreq_policy_put_kobj(policy); 1413 free_cpumask_var(policy->real_cpus); 1414 free_cpumask_var(policy->related_cpus); 1415 free_cpumask_var(policy->cpus); 1416 kfree(policy); 1417 } 1418 1419 static int cpufreq_online(unsigned int cpu) 1420 { 1421 struct cpufreq_policy *policy; 1422 bool new_policy; 1423 unsigned long flags; 1424 unsigned int j; 1425 int ret; 1426 1427 pr_debug("%s: bringing CPU%u online\n", __func__, cpu); 1428 1429 /* Check if this CPU already has a policy to manage it */ 1430 policy = per_cpu(cpufreq_cpu_data, cpu); 1431 if (policy) { 1432 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); 1433 if (!policy_is_inactive(policy)) 1434 return cpufreq_add_policy_cpu(policy, cpu); 1435 1436 /* This is the only online CPU for the policy. Start over. */ 1437 new_policy = false; 1438 down_write(&policy->rwsem); 1439 policy->cpu = cpu; 1440 policy->governor = NULL; 1441 } else { 1442 new_policy = true; 1443 policy = cpufreq_policy_alloc(cpu); 1444 if (!policy) 1445 return -ENOMEM; 1446 down_write(&policy->rwsem); 1447 } 1448 1449 if (!new_policy && cpufreq_driver->online) { 1450 /* Recover policy->cpus using related_cpus */ 1451 cpumask_copy(policy->cpus, policy->related_cpus); 1452 1453 ret = cpufreq_driver->online(policy); 1454 if (ret) { 1455 pr_debug("%s: %d: initialization failed\n", __func__, 1456 __LINE__); 1457 goto out_exit_policy; 1458 } 1459 } else { 1460 cpumask_copy(policy->cpus, cpumask_of(cpu)); 1461 1462 /* 1463 * Call driver. From then on the cpufreq must be able 1464 * to accept all calls to ->verify and ->setpolicy for this CPU. 1465 */ 1466 ret = cpufreq_driver->init(policy); 1467 if (ret) { 1468 pr_debug("%s: %d: initialization failed\n", __func__, 1469 __LINE__); 1470 goto out_free_policy; 1471 } 1472 1473 /* 1474 * The initialization has succeeded and the policy is online. 1475 * If there is a problem with its frequency table, take it 1476 * offline and drop it. 1477 */ 1478 ret = cpufreq_table_validate_and_sort(policy); 1479 if (ret) 1480 goto out_offline_policy; 1481 1482 /* related_cpus should at least include policy->cpus. */ 1483 cpumask_copy(policy->related_cpus, policy->cpus); 1484 } 1485 1486 /* 1487 * affected cpus must always be the one, which are online. We aren't 1488 * managing offline cpus here. 1489 */ 1490 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); 1491 1492 if (new_policy) { 1493 for_each_cpu(j, policy->related_cpus) { 1494 per_cpu(cpufreq_cpu_data, j) = policy; 1495 add_cpu_dev_symlink(policy, j, get_cpu_device(j)); 1496 } 1497 1498 policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req), 1499 GFP_KERNEL); 1500 if (!policy->min_freq_req) { 1501 ret = -ENOMEM; 1502 goto out_destroy_policy; 1503 } 1504 1505 ret = freq_qos_add_request(&policy->constraints, 1506 policy->min_freq_req, FREQ_QOS_MIN, 1507 FREQ_QOS_MIN_DEFAULT_VALUE); 1508 if (ret < 0) { 1509 /* 1510 * So we don't call freq_qos_remove_request() for an 1511 * uninitialized request. 1512 */ 1513 kfree(policy->min_freq_req); 1514 policy->min_freq_req = NULL; 1515 goto out_destroy_policy; 1516 } 1517 1518 /* 1519 * This must be initialized right here to avoid calling 1520 * freq_qos_remove_request() on uninitialized request in case 1521 * of errors. 1522 */ 1523 policy->max_freq_req = policy->min_freq_req + 1; 1524 1525 ret = freq_qos_add_request(&policy->constraints, 1526 policy->max_freq_req, FREQ_QOS_MAX, 1527 FREQ_QOS_MAX_DEFAULT_VALUE); 1528 if (ret < 0) { 1529 policy->max_freq_req = NULL; 1530 goto out_destroy_policy; 1531 } 1532 1533 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1534 CPUFREQ_CREATE_POLICY, policy); 1535 } else { 1536 ret = freq_qos_update_request(policy->max_freq_req, policy->max); 1537 if (ret < 0) 1538 goto out_destroy_policy; 1539 } 1540 1541 if (cpufreq_driver->get && has_target()) { 1542 policy->cur = cpufreq_driver->get(policy->cpu); 1543 if (!policy->cur) { 1544 ret = -EIO; 1545 pr_err("%s: ->get() failed\n", __func__); 1546 goto out_destroy_policy; 1547 } 1548 } 1549 1550 /* 1551 * Sometimes boot loaders set CPU frequency to a value outside of 1552 * frequency table present with cpufreq core. In such cases CPU might be 1553 * unstable if it has to run on that frequency for long duration of time 1554 * and so its better to set it to a frequency which is specified in 1555 * freq-table. This also makes cpufreq stats inconsistent as 1556 * cpufreq-stats would fail to register because current frequency of CPU 1557 * isn't found in freq-table. 1558 * 1559 * Because we don't want this change to effect boot process badly, we go 1560 * for the next freq which is >= policy->cur ('cur' must be set by now, 1561 * otherwise we will end up setting freq to lowest of the table as 'cur' 1562 * is initialized to zero). 1563 * 1564 * We are passing target-freq as "policy->cur - 1" otherwise 1565 * __cpufreq_driver_target() would simply fail, as policy->cur will be 1566 * equal to target-freq. 1567 */ 1568 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK) 1569 && has_target()) { 1570 unsigned int old_freq = policy->cur; 1571 1572 /* Are we running at unknown frequency ? */ 1573 ret = cpufreq_frequency_table_get_index(policy, old_freq); 1574 if (ret == -EINVAL) { 1575 ret = __cpufreq_driver_target(policy, old_freq - 1, 1576 CPUFREQ_RELATION_L); 1577 1578 /* 1579 * Reaching here after boot in a few seconds may not 1580 * mean that system will remain stable at "unknown" 1581 * frequency for longer duration. Hence, a BUG_ON(). 1582 */ 1583 BUG_ON(ret); 1584 pr_info("%s: CPU%d: Running at unlisted initial frequency: %u kHz, changing to: %u kHz\n", 1585 __func__, policy->cpu, old_freq, policy->cur); 1586 } 1587 } 1588 1589 if (new_policy) { 1590 ret = cpufreq_add_dev_interface(policy); 1591 if (ret) 1592 goto out_destroy_policy; 1593 1594 cpufreq_stats_create_table(policy); 1595 1596 write_lock_irqsave(&cpufreq_driver_lock, flags); 1597 list_add(&policy->policy_list, &cpufreq_policy_list); 1598 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1599 1600 /* 1601 * Register with the energy model before 1602 * em_rebuild_sched_domains() is called, which will result 1603 * in rebuilding of the sched domains, which should only be done 1604 * once the energy model is properly initialized for the policy 1605 * first. 1606 * 1607 * Also, this should be called before the policy is registered 1608 * with cooling framework. 1609 */ 1610 if (cpufreq_driver->register_em) 1611 cpufreq_driver->register_em(policy); 1612 } 1613 1614 ret = cpufreq_init_policy(policy); 1615 if (ret) { 1616 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", 1617 __func__, cpu, ret); 1618 goto out_destroy_policy; 1619 } 1620 1621 up_write(&policy->rwsem); 1622 1623 kobject_uevent(&policy->kobj, KOBJ_ADD); 1624 1625 /* Callback for handling stuff after policy is ready */ 1626 if (cpufreq_driver->ready) 1627 cpufreq_driver->ready(policy); 1628 1629 /* Register cpufreq cooling only for a new policy */ 1630 if (new_policy && cpufreq_thermal_control_enabled(cpufreq_driver)) 1631 policy->cdev = of_cpufreq_cooling_register(policy); 1632 1633 /* Let the per-policy boost flag mirror the cpufreq_driver boost during init */ 1634 if (cpufreq_driver->set_boost && policy->boost_supported && 1635 policy->boost_enabled != cpufreq_boost_enabled()) { 1636 policy->boost_enabled = cpufreq_boost_enabled(); 1637 ret = cpufreq_driver->set_boost(policy, policy->boost_enabled); 1638 if (ret) { 1639 /* If the set_boost fails, the online operation is not affected */ 1640 pr_info("%s: CPU%d: Cannot %s BOOST\n", __func__, policy->cpu, 1641 str_enable_disable(policy->boost_enabled)); 1642 policy->boost_enabled = !policy->boost_enabled; 1643 } 1644 } 1645 1646 pr_debug("initialization complete\n"); 1647 1648 return 0; 1649 1650 out_destroy_policy: 1651 for_each_cpu(j, policy->real_cpus) 1652 remove_cpu_dev_symlink(policy, j, get_cpu_device(j)); 1653 1654 out_offline_policy: 1655 if (cpufreq_driver->offline) 1656 cpufreq_driver->offline(policy); 1657 1658 out_exit_policy: 1659 if (cpufreq_driver->exit) 1660 cpufreq_driver->exit(policy); 1661 1662 out_free_policy: 1663 cpumask_clear(policy->cpus); 1664 up_write(&policy->rwsem); 1665 1666 cpufreq_policy_free(policy); 1667 return ret; 1668 } 1669 1670 /** 1671 * cpufreq_add_dev - the cpufreq interface for a CPU device. 1672 * @dev: CPU device. 1673 * @sif: Subsystem interface structure pointer (not used) 1674 */ 1675 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) 1676 { 1677 struct cpufreq_policy *policy; 1678 unsigned cpu = dev->id; 1679 int ret; 1680 1681 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu); 1682 1683 if (cpu_online(cpu)) { 1684 ret = cpufreq_online(cpu); 1685 if (ret) 1686 return ret; 1687 } 1688 1689 /* Create sysfs link on CPU registration */ 1690 policy = per_cpu(cpufreq_cpu_data, cpu); 1691 if (policy) 1692 add_cpu_dev_symlink(policy, cpu, dev); 1693 1694 return 0; 1695 } 1696 1697 static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy) 1698 { 1699 int ret; 1700 1701 if (has_target()) 1702 cpufreq_stop_governor(policy); 1703 1704 cpumask_clear_cpu(cpu, policy->cpus); 1705 1706 if (!policy_is_inactive(policy)) { 1707 /* Nominate a new CPU if necessary. */ 1708 if (cpu == policy->cpu) 1709 policy->cpu = cpumask_any(policy->cpus); 1710 1711 /* Start the governor again for the active policy. */ 1712 if (has_target()) { 1713 ret = cpufreq_start_governor(policy); 1714 if (ret) 1715 pr_err("%s: Failed to start governor\n", __func__); 1716 } 1717 1718 return; 1719 } 1720 1721 if (has_target()) 1722 strscpy(policy->last_governor, policy->governor->name, 1723 CPUFREQ_NAME_LEN); 1724 else 1725 policy->last_policy = policy->policy; 1726 1727 if (has_target()) 1728 cpufreq_exit_governor(policy); 1729 1730 /* 1731 * Perform the ->offline() during light-weight tear-down, as 1732 * that allows fast recovery when the CPU comes back. 1733 */ 1734 if (cpufreq_driver->offline) { 1735 cpufreq_driver->offline(policy); 1736 return; 1737 } 1738 1739 if (cpufreq_driver->exit) 1740 cpufreq_driver->exit(policy); 1741 1742 policy->freq_table = NULL; 1743 } 1744 1745 static int cpufreq_offline(unsigned int cpu) 1746 { 1747 struct cpufreq_policy *policy; 1748 1749 pr_debug("%s: unregistering CPU %u\n", __func__, cpu); 1750 1751 policy = cpufreq_cpu_get_raw(cpu); 1752 if (!policy) { 1753 pr_debug("%s: No cpu_data found\n", __func__); 1754 return 0; 1755 } 1756 1757 down_write(&policy->rwsem); 1758 1759 __cpufreq_offline(cpu, policy); 1760 1761 up_write(&policy->rwsem); 1762 return 0; 1763 } 1764 1765 /* 1766 * cpufreq_remove_dev - remove a CPU device 1767 * 1768 * Removes the cpufreq interface for a CPU device. 1769 */ 1770 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) 1771 { 1772 unsigned int cpu = dev->id; 1773 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 1774 1775 if (!policy) 1776 return; 1777 1778 down_write(&policy->rwsem); 1779 1780 if (cpu_online(cpu)) 1781 __cpufreq_offline(cpu, policy); 1782 1783 remove_cpu_dev_symlink(policy, cpu, dev); 1784 1785 if (!cpumask_empty(policy->real_cpus)) { 1786 up_write(&policy->rwsem); 1787 return; 1788 } 1789 1790 /* 1791 * Unregister cpufreq cooling once all the CPUs of the policy are 1792 * removed. 1793 */ 1794 if (cpufreq_thermal_control_enabled(cpufreq_driver)) { 1795 cpufreq_cooling_unregister(policy->cdev); 1796 policy->cdev = NULL; 1797 } 1798 1799 /* We did light-weight exit earlier, do full tear down now */ 1800 if (cpufreq_driver->offline && cpufreq_driver->exit) 1801 cpufreq_driver->exit(policy); 1802 1803 up_write(&policy->rwsem); 1804 1805 cpufreq_policy_free(policy); 1806 } 1807 1808 /** 1809 * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference. 1810 * @policy: Policy managing CPUs. 1811 * @new_freq: New CPU frequency. 1812 * 1813 * Adjust to the current frequency first and clean up later by either calling 1814 * cpufreq_update_policy(), or scheduling handle_update(). 1815 */ 1816 static void cpufreq_out_of_sync(struct cpufreq_policy *policy, 1817 unsigned int new_freq) 1818 { 1819 struct cpufreq_freqs freqs; 1820 1821 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n", 1822 policy->cur, new_freq); 1823 1824 freqs.old = policy->cur; 1825 freqs.new = new_freq; 1826 1827 cpufreq_freq_transition_begin(policy, &freqs); 1828 cpufreq_freq_transition_end(policy, &freqs, 0); 1829 } 1830 1831 static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update) 1832 { 1833 unsigned int new_freq; 1834 1835 new_freq = cpufreq_driver->get(policy->cpu); 1836 if (!new_freq) 1837 return 0; 1838 1839 /* 1840 * If fast frequency switching is used with the given policy, the check 1841 * against policy->cur is pointless, so skip it in that case. 1842 */ 1843 if (policy->fast_switch_enabled || !has_target()) 1844 return new_freq; 1845 1846 if (policy->cur != new_freq) { 1847 /* 1848 * For some platforms, the frequency returned by hardware may be 1849 * slightly different from what is provided in the frequency 1850 * table, for example hardware may return 499 MHz instead of 500 1851 * MHz. In such cases it is better to avoid getting into 1852 * unnecessary frequency updates. 1853 */ 1854 if (abs(policy->cur - new_freq) < KHZ_PER_MHZ) 1855 return policy->cur; 1856 1857 cpufreq_out_of_sync(policy, new_freq); 1858 if (update) 1859 schedule_work(&policy->update); 1860 } 1861 1862 return new_freq; 1863 } 1864 1865 /** 1866 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur 1867 * @cpu: CPU number 1868 * 1869 * This is the last known freq, without actually getting it from the driver. 1870 * Return value will be same as what is shown in scaling_cur_freq in sysfs. 1871 */ 1872 unsigned int cpufreq_quick_get(unsigned int cpu) 1873 { 1874 struct cpufreq_policy *policy; 1875 unsigned int ret_freq = 0; 1876 unsigned long flags; 1877 1878 read_lock_irqsave(&cpufreq_driver_lock, flags); 1879 1880 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) { 1881 ret_freq = cpufreq_driver->get(cpu); 1882 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1883 return ret_freq; 1884 } 1885 1886 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1887 1888 policy = cpufreq_cpu_get(cpu); 1889 if (policy) { 1890 ret_freq = policy->cur; 1891 cpufreq_cpu_put(policy); 1892 } 1893 1894 return ret_freq; 1895 } 1896 EXPORT_SYMBOL(cpufreq_quick_get); 1897 1898 /** 1899 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU 1900 * @cpu: CPU number 1901 * 1902 * Just return the max possible frequency for a given CPU. 1903 */ 1904 unsigned int cpufreq_quick_get_max(unsigned int cpu) 1905 { 1906 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1907 unsigned int ret_freq = 0; 1908 1909 if (policy) { 1910 ret_freq = policy->max; 1911 cpufreq_cpu_put(policy); 1912 } 1913 1914 return ret_freq; 1915 } 1916 EXPORT_SYMBOL(cpufreq_quick_get_max); 1917 1918 /** 1919 * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU 1920 * @cpu: CPU number 1921 * 1922 * The default return value is the max_freq field of cpuinfo. 1923 */ 1924 __weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu) 1925 { 1926 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1927 unsigned int ret_freq = 0; 1928 1929 if (policy) { 1930 ret_freq = policy->cpuinfo.max_freq; 1931 cpufreq_cpu_put(policy); 1932 } 1933 1934 return ret_freq; 1935 } 1936 EXPORT_SYMBOL(cpufreq_get_hw_max_freq); 1937 1938 static unsigned int __cpufreq_get(struct cpufreq_policy *policy) 1939 { 1940 if (unlikely(policy_is_inactive(policy))) 1941 return 0; 1942 1943 return cpufreq_verify_current_freq(policy, true); 1944 } 1945 1946 /** 1947 * cpufreq_get - get the current CPU frequency (in kHz) 1948 * @cpu: CPU number 1949 * 1950 * Get the CPU current (static) CPU frequency 1951 */ 1952 unsigned int cpufreq_get(unsigned int cpu) 1953 { 1954 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1955 unsigned int ret_freq = 0; 1956 1957 if (policy) { 1958 down_read(&policy->rwsem); 1959 if (cpufreq_driver->get) 1960 ret_freq = __cpufreq_get(policy); 1961 up_read(&policy->rwsem); 1962 1963 cpufreq_cpu_put(policy); 1964 } 1965 1966 return ret_freq; 1967 } 1968 EXPORT_SYMBOL(cpufreq_get); 1969 1970 static struct subsys_interface cpufreq_interface = { 1971 .name = "cpufreq", 1972 .subsys = &cpu_subsys, 1973 .add_dev = cpufreq_add_dev, 1974 .remove_dev = cpufreq_remove_dev, 1975 }; 1976 1977 /* 1978 * In case platform wants some specific frequency to be configured 1979 * during suspend.. 1980 */ 1981 int cpufreq_generic_suspend(struct cpufreq_policy *policy) 1982 { 1983 int ret; 1984 1985 if (!policy->suspend_freq) { 1986 pr_debug("%s: suspend_freq not defined\n", __func__); 1987 return 0; 1988 } 1989 1990 pr_debug("%s: Setting suspend-freq: %u\n", __func__, 1991 policy->suspend_freq); 1992 1993 ret = __cpufreq_driver_target(policy, policy->suspend_freq, 1994 CPUFREQ_RELATION_H); 1995 if (ret) 1996 pr_err("%s: unable to set suspend-freq: %u. err: %d\n", 1997 __func__, policy->suspend_freq, ret); 1998 1999 return ret; 2000 } 2001 EXPORT_SYMBOL(cpufreq_generic_suspend); 2002 2003 /** 2004 * cpufreq_suspend() - Suspend CPUFreq governors. 2005 * 2006 * Called during system wide Suspend/Hibernate cycles for suspending governors 2007 * as some platforms can't change frequency after this point in suspend cycle. 2008 * Because some of the devices (like: i2c, regulators, etc) they use for 2009 * changing frequency are suspended quickly after this point. 2010 */ 2011 void cpufreq_suspend(void) 2012 { 2013 struct cpufreq_policy *policy; 2014 2015 if (!cpufreq_driver) 2016 return; 2017 2018 if (!has_target() && !cpufreq_driver->suspend) 2019 goto suspend; 2020 2021 pr_debug("%s: Suspending Governors\n", __func__); 2022 2023 for_each_active_policy(policy) { 2024 if (has_target()) { 2025 down_write(&policy->rwsem); 2026 cpufreq_stop_governor(policy); 2027 up_write(&policy->rwsem); 2028 } 2029 2030 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy)) 2031 pr_err("%s: Failed to suspend driver: %s\n", __func__, 2032 cpufreq_driver->name); 2033 } 2034 2035 suspend: 2036 cpufreq_suspended = true; 2037 } 2038 2039 /** 2040 * cpufreq_resume() - Resume CPUFreq governors. 2041 * 2042 * Called during system wide Suspend/Hibernate cycle for resuming governors that 2043 * are suspended with cpufreq_suspend(). 2044 */ 2045 void cpufreq_resume(void) 2046 { 2047 struct cpufreq_policy *policy; 2048 int ret; 2049 2050 if (!cpufreq_driver) 2051 return; 2052 2053 if (unlikely(!cpufreq_suspended)) 2054 return; 2055 2056 cpufreq_suspended = false; 2057 2058 if (!has_target() && !cpufreq_driver->resume) 2059 return; 2060 2061 pr_debug("%s: Resuming Governors\n", __func__); 2062 2063 for_each_active_policy(policy) { 2064 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) { 2065 pr_err("%s: Failed to resume driver: %s\n", __func__, 2066 cpufreq_driver->name); 2067 } else if (has_target()) { 2068 down_write(&policy->rwsem); 2069 ret = cpufreq_start_governor(policy); 2070 up_write(&policy->rwsem); 2071 2072 if (ret) 2073 pr_err("%s: Failed to start governor for CPU%u's policy\n", 2074 __func__, policy->cpu); 2075 } 2076 } 2077 } 2078 2079 /** 2080 * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones. 2081 * @flags: Flags to test against the current cpufreq driver's flags. 2082 * 2083 * Assumes that the driver is there, so callers must ensure that this is the 2084 * case. 2085 */ 2086 bool cpufreq_driver_test_flags(u16 flags) 2087 { 2088 return !!(cpufreq_driver->flags & flags); 2089 } 2090 2091 /** 2092 * cpufreq_get_current_driver - Return the current driver's name. 2093 * 2094 * Return the name string of the currently registered cpufreq driver or NULL if 2095 * none. 2096 */ 2097 const char *cpufreq_get_current_driver(void) 2098 { 2099 if (cpufreq_driver) 2100 return cpufreq_driver->name; 2101 2102 return NULL; 2103 } 2104 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver); 2105 2106 /** 2107 * cpufreq_get_driver_data - Return current driver data. 2108 * 2109 * Return the private data of the currently registered cpufreq driver, or NULL 2110 * if no cpufreq driver has been registered. 2111 */ 2112 void *cpufreq_get_driver_data(void) 2113 { 2114 if (cpufreq_driver) 2115 return cpufreq_driver->driver_data; 2116 2117 return NULL; 2118 } 2119 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data); 2120 2121 /********************************************************************* 2122 * NOTIFIER LISTS INTERFACE * 2123 *********************************************************************/ 2124 2125 /** 2126 * cpufreq_register_notifier - Register a notifier with cpufreq. 2127 * @nb: notifier function to register. 2128 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER. 2129 * 2130 * Add a notifier to one of two lists: either a list of notifiers that run on 2131 * clock rate changes (once before and once after every transition), or a list 2132 * of notifiers that ron on cpufreq policy changes. 2133 * 2134 * This function may sleep and it has the same return values as 2135 * blocking_notifier_chain_register(). 2136 */ 2137 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) 2138 { 2139 int ret; 2140 2141 if (cpufreq_disabled()) 2142 return -EINVAL; 2143 2144 switch (list) { 2145 case CPUFREQ_TRANSITION_NOTIFIER: 2146 mutex_lock(&cpufreq_fast_switch_lock); 2147 2148 if (cpufreq_fast_switch_count > 0) { 2149 mutex_unlock(&cpufreq_fast_switch_lock); 2150 return -EBUSY; 2151 } 2152 ret = srcu_notifier_chain_register( 2153 &cpufreq_transition_notifier_list, nb); 2154 if (!ret) 2155 cpufreq_fast_switch_count--; 2156 2157 mutex_unlock(&cpufreq_fast_switch_lock); 2158 break; 2159 case CPUFREQ_POLICY_NOTIFIER: 2160 ret = blocking_notifier_chain_register( 2161 &cpufreq_policy_notifier_list, nb); 2162 break; 2163 default: 2164 ret = -EINVAL; 2165 } 2166 2167 return ret; 2168 } 2169 EXPORT_SYMBOL(cpufreq_register_notifier); 2170 2171 /** 2172 * cpufreq_unregister_notifier - Unregister a notifier from cpufreq. 2173 * @nb: notifier block to be unregistered. 2174 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER. 2175 * 2176 * Remove a notifier from one of the cpufreq notifier lists. 2177 * 2178 * This function may sleep and it has the same return values as 2179 * blocking_notifier_chain_unregister(). 2180 */ 2181 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list) 2182 { 2183 int ret; 2184 2185 if (cpufreq_disabled()) 2186 return -EINVAL; 2187 2188 switch (list) { 2189 case CPUFREQ_TRANSITION_NOTIFIER: 2190 mutex_lock(&cpufreq_fast_switch_lock); 2191 2192 ret = srcu_notifier_chain_unregister( 2193 &cpufreq_transition_notifier_list, nb); 2194 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0)) 2195 cpufreq_fast_switch_count++; 2196 2197 mutex_unlock(&cpufreq_fast_switch_lock); 2198 break; 2199 case CPUFREQ_POLICY_NOTIFIER: 2200 ret = blocking_notifier_chain_unregister( 2201 &cpufreq_policy_notifier_list, nb); 2202 break; 2203 default: 2204 ret = -EINVAL; 2205 } 2206 2207 return ret; 2208 } 2209 EXPORT_SYMBOL(cpufreq_unregister_notifier); 2210 2211 2212 /********************************************************************* 2213 * GOVERNORS * 2214 *********************************************************************/ 2215 2216 /** 2217 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch. 2218 * @policy: cpufreq policy to switch the frequency for. 2219 * @target_freq: New frequency to set (may be approximate). 2220 * 2221 * Carry out a fast frequency switch without sleeping. 2222 * 2223 * The driver's ->fast_switch() callback invoked by this function must be 2224 * suitable for being called from within RCU-sched read-side critical sections 2225 * and it is expected to select the minimum available frequency greater than or 2226 * equal to @target_freq (CPUFREQ_RELATION_L). 2227 * 2228 * This function must not be called if policy->fast_switch_enabled is unset. 2229 * 2230 * Governors calling this function must guarantee that it will never be invoked 2231 * twice in parallel for the same policy and that it will never be called in 2232 * parallel with either ->target() or ->target_index() for the same policy. 2233 * 2234 * Returns the actual frequency set for the CPU. 2235 * 2236 * If 0 is returned by the driver's ->fast_switch() callback to indicate an 2237 * error condition, the hardware configuration must be preserved. 2238 */ 2239 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, 2240 unsigned int target_freq) 2241 { 2242 unsigned int freq; 2243 int cpu; 2244 2245 target_freq = clamp_val(target_freq, policy->min, policy->max); 2246 freq = cpufreq_driver->fast_switch(policy, target_freq); 2247 2248 if (!freq) 2249 return 0; 2250 2251 policy->cur = freq; 2252 arch_set_freq_scale(policy->related_cpus, freq, 2253 arch_scale_freq_ref(policy->cpu)); 2254 cpufreq_stats_record_transition(policy, freq); 2255 2256 if (trace_cpu_frequency_enabled()) { 2257 for_each_cpu(cpu, policy->cpus) 2258 trace_cpu_frequency(freq, cpu); 2259 } 2260 2261 return freq; 2262 } 2263 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); 2264 2265 /** 2266 * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go. 2267 * @cpu: Target CPU. 2268 * @min_perf: Minimum (required) performance level (units of @capacity). 2269 * @target_perf: Target (desired) performance level (units of @capacity). 2270 * @capacity: Capacity of the target CPU. 2271 * 2272 * Carry out a fast performance level switch of @cpu without sleeping. 2273 * 2274 * The driver's ->adjust_perf() callback invoked by this function must be 2275 * suitable for being called from within RCU-sched read-side critical sections 2276 * and it is expected to select a suitable performance level equal to or above 2277 * @min_perf and preferably equal to or below @target_perf. 2278 * 2279 * This function must not be called if policy->fast_switch_enabled is unset. 2280 * 2281 * Governors calling this function must guarantee that it will never be invoked 2282 * twice in parallel for the same CPU and that it will never be called in 2283 * parallel with either ->target() or ->target_index() or ->fast_switch() for 2284 * the same CPU. 2285 */ 2286 void cpufreq_driver_adjust_perf(unsigned int cpu, 2287 unsigned long min_perf, 2288 unsigned long target_perf, 2289 unsigned long capacity) 2290 { 2291 cpufreq_driver->adjust_perf(cpu, min_perf, target_perf, capacity); 2292 } 2293 2294 /** 2295 * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback. 2296 * 2297 * Return 'true' if the ->adjust_perf callback is present for the 2298 * current driver or 'false' otherwise. 2299 */ 2300 bool cpufreq_driver_has_adjust_perf(void) 2301 { 2302 return !!cpufreq_driver->adjust_perf; 2303 } 2304 2305 /* Must set freqs->new to intermediate frequency */ 2306 static int __target_intermediate(struct cpufreq_policy *policy, 2307 struct cpufreq_freqs *freqs, int index) 2308 { 2309 int ret; 2310 2311 freqs->new = cpufreq_driver->get_intermediate(policy, index); 2312 2313 /* We don't need to switch to intermediate freq */ 2314 if (!freqs->new) 2315 return 0; 2316 2317 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n", 2318 __func__, policy->cpu, freqs->old, freqs->new); 2319 2320 cpufreq_freq_transition_begin(policy, freqs); 2321 ret = cpufreq_driver->target_intermediate(policy, index); 2322 cpufreq_freq_transition_end(policy, freqs, ret); 2323 2324 if (ret) 2325 pr_err("%s: Failed to change to intermediate frequency: %d\n", 2326 __func__, ret); 2327 2328 return ret; 2329 } 2330 2331 static int __target_index(struct cpufreq_policy *policy, int index) 2332 { 2333 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; 2334 unsigned int restore_freq, intermediate_freq = 0; 2335 unsigned int newfreq = policy->freq_table[index].frequency; 2336 int retval = -EINVAL; 2337 bool notify; 2338 2339 if (newfreq == policy->cur) 2340 return 0; 2341 2342 /* Save last value to restore later on errors */ 2343 restore_freq = policy->cur; 2344 2345 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); 2346 if (notify) { 2347 /* Handle switching to intermediate frequency */ 2348 if (cpufreq_driver->get_intermediate) { 2349 retval = __target_intermediate(policy, &freqs, index); 2350 if (retval) 2351 return retval; 2352 2353 intermediate_freq = freqs.new; 2354 /* Set old freq to intermediate */ 2355 if (intermediate_freq) 2356 freqs.old = freqs.new; 2357 } 2358 2359 freqs.new = newfreq; 2360 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", 2361 __func__, policy->cpu, freqs.old, freqs.new); 2362 2363 cpufreq_freq_transition_begin(policy, &freqs); 2364 } 2365 2366 retval = cpufreq_driver->target_index(policy, index); 2367 if (retval) 2368 pr_err("%s: Failed to change cpu frequency: %d\n", __func__, 2369 retval); 2370 2371 if (notify) { 2372 cpufreq_freq_transition_end(policy, &freqs, retval); 2373 2374 /* 2375 * Failed after setting to intermediate freq? Driver should have 2376 * reverted back to initial frequency and so should we. Check 2377 * here for intermediate_freq instead of get_intermediate, in 2378 * case we haven't switched to intermediate freq at all. 2379 */ 2380 if (unlikely(retval && intermediate_freq)) { 2381 freqs.old = intermediate_freq; 2382 freqs.new = restore_freq; 2383 cpufreq_freq_transition_begin(policy, &freqs); 2384 cpufreq_freq_transition_end(policy, &freqs, 0); 2385 } 2386 } 2387 2388 return retval; 2389 } 2390 2391 int __cpufreq_driver_target(struct cpufreq_policy *policy, 2392 unsigned int target_freq, 2393 unsigned int relation) 2394 { 2395 unsigned int old_target_freq = target_freq; 2396 2397 if (cpufreq_disabled()) 2398 return -ENODEV; 2399 2400 target_freq = clamp_val(target_freq, policy->min, policy->max); 2401 target_freq = __resolve_freq(policy, target_freq, relation); 2402 2403 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", 2404 policy->cpu, target_freq, relation, old_target_freq); 2405 2406 /* 2407 * This might look like a redundant call as we are checking it again 2408 * after finding index. But it is left intentionally for cases where 2409 * exactly same freq is called again and so we can save on few function 2410 * calls. 2411 */ 2412 if (target_freq == policy->cur && 2413 !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS)) 2414 return 0; 2415 2416 if (cpufreq_driver->target) { 2417 /* 2418 * If the driver hasn't setup a single inefficient frequency, 2419 * it's unlikely it knows how to decode CPUFREQ_RELATION_E. 2420 */ 2421 if (!policy->efficiencies_available) 2422 relation &= ~CPUFREQ_RELATION_E; 2423 2424 return cpufreq_driver->target(policy, target_freq, relation); 2425 } 2426 2427 if (!cpufreq_driver->target_index) 2428 return -EINVAL; 2429 2430 return __target_index(policy, policy->cached_resolved_idx); 2431 } 2432 EXPORT_SYMBOL_GPL(__cpufreq_driver_target); 2433 2434 int cpufreq_driver_target(struct cpufreq_policy *policy, 2435 unsigned int target_freq, 2436 unsigned int relation) 2437 { 2438 int ret; 2439 2440 down_write(&policy->rwsem); 2441 2442 ret = __cpufreq_driver_target(policy, target_freq, relation); 2443 2444 up_write(&policy->rwsem); 2445 2446 return ret; 2447 } 2448 EXPORT_SYMBOL_GPL(cpufreq_driver_target); 2449 2450 __weak struct cpufreq_governor *cpufreq_fallback_governor(void) 2451 { 2452 return NULL; 2453 } 2454 2455 static int cpufreq_init_governor(struct cpufreq_policy *policy) 2456 { 2457 int ret; 2458 2459 /* Don't start any governor operations if we are entering suspend */ 2460 if (cpufreq_suspended) 2461 return 0; 2462 /* 2463 * Governor might not be initiated here if ACPI _PPC changed 2464 * notification happened, so check it. 2465 */ 2466 if (!policy->governor) 2467 return -EINVAL; 2468 2469 /* Platform doesn't want dynamic frequency switching ? */ 2470 if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING && 2471 cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) { 2472 struct cpufreq_governor *gov = cpufreq_fallback_governor(); 2473 2474 if (gov) { 2475 pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n", 2476 policy->governor->name, gov->name); 2477 policy->governor = gov; 2478 } else { 2479 return -EINVAL; 2480 } 2481 } 2482 2483 if (!try_module_get(policy->governor->owner)) 2484 return -EINVAL; 2485 2486 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2487 2488 if (policy->governor->init) { 2489 ret = policy->governor->init(policy); 2490 if (ret) { 2491 module_put(policy->governor->owner); 2492 return ret; 2493 } 2494 } 2495 2496 policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET); 2497 2498 return 0; 2499 } 2500 2501 static void cpufreq_exit_governor(struct cpufreq_policy *policy) 2502 { 2503 if (cpufreq_suspended || !policy->governor) 2504 return; 2505 2506 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2507 2508 if (policy->governor->exit) 2509 policy->governor->exit(policy); 2510 2511 module_put(policy->governor->owner); 2512 } 2513 2514 int cpufreq_start_governor(struct cpufreq_policy *policy) 2515 { 2516 int ret; 2517 2518 if (cpufreq_suspended) 2519 return 0; 2520 2521 if (!policy->governor) 2522 return -EINVAL; 2523 2524 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2525 2526 if (cpufreq_driver->get) 2527 cpufreq_verify_current_freq(policy, false); 2528 2529 if (policy->governor->start) { 2530 ret = policy->governor->start(policy); 2531 if (ret) 2532 return ret; 2533 } 2534 2535 if (policy->governor->limits) 2536 policy->governor->limits(policy); 2537 2538 return 0; 2539 } 2540 2541 void cpufreq_stop_governor(struct cpufreq_policy *policy) 2542 { 2543 if (cpufreq_suspended || !policy->governor) 2544 return; 2545 2546 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2547 2548 if (policy->governor->stop) 2549 policy->governor->stop(policy); 2550 } 2551 2552 static void cpufreq_governor_limits(struct cpufreq_policy *policy) 2553 { 2554 if (cpufreq_suspended || !policy->governor) 2555 return; 2556 2557 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2558 2559 if (policy->governor->limits) 2560 policy->governor->limits(policy); 2561 } 2562 2563 int cpufreq_register_governor(struct cpufreq_governor *governor) 2564 { 2565 int err; 2566 2567 if (!governor) 2568 return -EINVAL; 2569 2570 if (cpufreq_disabled()) 2571 return -ENODEV; 2572 2573 mutex_lock(&cpufreq_governor_mutex); 2574 2575 err = -EBUSY; 2576 if (!find_governor(governor->name)) { 2577 err = 0; 2578 list_add(&governor->governor_list, &cpufreq_governor_list); 2579 } 2580 2581 mutex_unlock(&cpufreq_governor_mutex); 2582 return err; 2583 } 2584 EXPORT_SYMBOL_GPL(cpufreq_register_governor); 2585 2586 void cpufreq_unregister_governor(struct cpufreq_governor *governor) 2587 { 2588 struct cpufreq_policy *policy; 2589 unsigned long flags; 2590 2591 if (!governor) 2592 return; 2593 2594 if (cpufreq_disabled()) 2595 return; 2596 2597 /* clear last_governor for all inactive policies */ 2598 read_lock_irqsave(&cpufreq_driver_lock, flags); 2599 for_each_inactive_policy(policy) { 2600 if (!strcmp(policy->last_governor, governor->name)) { 2601 policy->governor = NULL; 2602 strcpy(policy->last_governor, "\0"); 2603 } 2604 } 2605 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 2606 2607 mutex_lock(&cpufreq_governor_mutex); 2608 list_del(&governor->governor_list); 2609 mutex_unlock(&cpufreq_governor_mutex); 2610 } 2611 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor); 2612 2613 2614 /********************************************************************* 2615 * POLICY INTERFACE * 2616 *********************************************************************/ 2617 2618 /** 2619 * cpufreq_get_policy - get the current cpufreq_policy 2620 * @policy: struct cpufreq_policy into which the current cpufreq_policy 2621 * is written 2622 * @cpu: CPU to find the policy for 2623 * 2624 * Reads the current cpufreq policy. 2625 */ 2626 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu) 2627 { 2628 struct cpufreq_policy *cpu_policy; 2629 if (!policy) 2630 return -EINVAL; 2631 2632 cpu_policy = cpufreq_cpu_get(cpu); 2633 if (!cpu_policy) 2634 return -EINVAL; 2635 2636 memcpy(policy, cpu_policy, sizeof(*policy)); 2637 2638 cpufreq_cpu_put(cpu_policy); 2639 return 0; 2640 } 2641 EXPORT_SYMBOL(cpufreq_get_policy); 2642 2643 DEFINE_PER_CPU(unsigned long, cpufreq_pressure); 2644 2645 /** 2646 * cpufreq_update_pressure() - Update cpufreq pressure for CPUs 2647 * @policy: cpufreq policy of the CPUs. 2648 * 2649 * Update the value of cpufreq pressure for all @cpus in the policy. 2650 */ 2651 static void cpufreq_update_pressure(struct cpufreq_policy *policy) 2652 { 2653 unsigned long max_capacity, capped_freq, pressure; 2654 u32 max_freq; 2655 int cpu; 2656 2657 cpu = cpumask_first(policy->related_cpus); 2658 max_freq = arch_scale_freq_ref(cpu); 2659 capped_freq = policy->max; 2660 2661 /* 2662 * Handle properly the boost frequencies, which should simply clean 2663 * the cpufreq pressure value. 2664 */ 2665 if (max_freq <= capped_freq) { 2666 pressure = 0; 2667 } else { 2668 max_capacity = arch_scale_cpu_capacity(cpu); 2669 pressure = max_capacity - 2670 mult_frac(max_capacity, capped_freq, max_freq); 2671 } 2672 2673 for_each_cpu(cpu, policy->related_cpus) 2674 WRITE_ONCE(per_cpu(cpufreq_pressure, cpu), pressure); 2675 } 2676 2677 /** 2678 * cpufreq_set_policy - Modify cpufreq policy parameters. 2679 * @policy: Policy object to modify. 2680 * @new_gov: Policy governor pointer. 2681 * @new_pol: Policy value (for drivers with built-in governors). 2682 * 2683 * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency 2684 * limits to be set for the policy, update @policy with the verified limits 2685 * values and either invoke the driver's ->setpolicy() callback (if present) or 2686 * carry out a governor update for @policy. That is, run the current governor's 2687 * ->limits() callback (if @new_gov points to the same object as the one in 2688 * @policy) or replace the governor for @policy with @new_gov. 2689 * 2690 * The cpuinfo part of @policy is not updated by this function. 2691 */ 2692 static int cpufreq_set_policy(struct cpufreq_policy *policy, 2693 struct cpufreq_governor *new_gov, 2694 unsigned int new_pol) 2695 { 2696 struct cpufreq_policy_data new_data; 2697 struct cpufreq_governor *old_gov; 2698 int ret; 2699 2700 memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); 2701 new_data.freq_table = policy->freq_table; 2702 new_data.cpu = policy->cpu; 2703 /* 2704 * PM QoS framework collects all the requests from users and provide us 2705 * the final aggregated value here. 2706 */ 2707 new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN); 2708 new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX); 2709 2710 pr_debug("setting new policy for CPU %u: %u - %u kHz\n", 2711 new_data.cpu, new_data.min, new_data.max); 2712 2713 /* 2714 * Verify that the CPU speed can be set within these limits and make sure 2715 * that min <= max. 2716 */ 2717 ret = cpufreq_driver->verify(&new_data); 2718 if (ret) 2719 return ret; 2720 2721 /* 2722 * Resolve policy min/max to available frequencies. It ensures 2723 * no frequency resolution will neither overshoot the requested maximum 2724 * nor undershoot the requested minimum. 2725 * 2726 * Avoid storing intermediate values in policy->max or policy->min and 2727 * compiler optimizations around them because they may be accessed 2728 * concurrently by cpufreq_driver_resolve_freq() during the update. 2729 */ 2730 WRITE_ONCE(policy->max, __resolve_freq(policy, new_data.max, CPUFREQ_RELATION_H)); 2731 new_data.min = __resolve_freq(policy, new_data.min, CPUFREQ_RELATION_L); 2732 WRITE_ONCE(policy->min, new_data.min > policy->max ? policy->max : new_data.min); 2733 2734 trace_cpu_frequency_limits(policy); 2735 2736 cpufreq_update_pressure(policy); 2737 2738 policy->cached_target_freq = UINT_MAX; 2739 2740 pr_debug("new min and max freqs are %u - %u kHz\n", 2741 policy->min, policy->max); 2742 2743 if (cpufreq_driver->setpolicy) { 2744 policy->policy = new_pol; 2745 pr_debug("setting range\n"); 2746 return cpufreq_driver->setpolicy(policy); 2747 } 2748 2749 if (new_gov == policy->governor) { 2750 pr_debug("governor limits update\n"); 2751 cpufreq_governor_limits(policy); 2752 return 0; 2753 } 2754 2755 pr_debug("governor switch\n"); 2756 2757 /* save old, working values */ 2758 old_gov = policy->governor; 2759 /* end old governor */ 2760 if (old_gov) { 2761 cpufreq_stop_governor(policy); 2762 cpufreq_exit_governor(policy); 2763 } 2764 2765 /* start new governor */ 2766 policy->governor = new_gov; 2767 ret = cpufreq_init_governor(policy); 2768 if (!ret) { 2769 ret = cpufreq_start_governor(policy); 2770 if (!ret) { 2771 pr_debug("governor change\n"); 2772 return 0; 2773 } 2774 cpufreq_exit_governor(policy); 2775 } 2776 2777 /* new governor failed, so re-start old one */ 2778 pr_debug("starting governor %s failed\n", policy->governor->name); 2779 if (old_gov) { 2780 policy->governor = old_gov; 2781 if (cpufreq_init_governor(policy)) 2782 policy->governor = NULL; 2783 else 2784 cpufreq_start_governor(policy); 2785 } 2786 2787 return ret; 2788 } 2789 2790 /** 2791 * cpufreq_update_policy - Re-evaluate an existing cpufreq policy. 2792 * @cpu: CPU to re-evaluate the policy for. 2793 * 2794 * Update the current frequency for the cpufreq policy of @cpu and use 2795 * cpufreq_set_policy() to re-apply the min and max limits, which triggers the 2796 * evaluation of policy notifiers and the cpufreq driver's ->verify() callback 2797 * for the policy in question, among other things. 2798 */ 2799 void cpufreq_update_policy(unsigned int cpu) 2800 { 2801 struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); 2802 2803 if (!policy) 2804 return; 2805 2806 /* 2807 * BIOS might change freq behind our back 2808 * -> ask driver for current freq and notify governors about a change 2809 */ 2810 if (cpufreq_driver->get && has_target() && 2811 (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false)))) 2812 goto unlock; 2813 2814 refresh_frequency_limits(policy); 2815 2816 unlock: 2817 cpufreq_cpu_release(policy); 2818 } 2819 EXPORT_SYMBOL(cpufreq_update_policy); 2820 2821 /** 2822 * cpufreq_update_limits - Update policy limits for a given CPU. 2823 * @cpu: CPU to update the policy limits for. 2824 * 2825 * Invoke the driver's ->update_limits callback if present or call 2826 * cpufreq_update_policy() for @cpu. 2827 */ 2828 void cpufreq_update_limits(unsigned int cpu) 2829 { 2830 struct cpufreq_policy *policy __free(put_cpufreq_policy); 2831 2832 policy = cpufreq_cpu_get(cpu); 2833 if (!policy) 2834 return; 2835 2836 if (cpufreq_driver->update_limits) 2837 cpufreq_driver->update_limits(cpu); 2838 else 2839 cpufreq_update_policy(cpu); 2840 } 2841 EXPORT_SYMBOL_GPL(cpufreq_update_limits); 2842 2843 /********************************************************************* 2844 * BOOST * 2845 *********************************************************************/ 2846 int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state) 2847 { 2848 int ret; 2849 2850 if (!policy->freq_table) 2851 return -ENXIO; 2852 2853 ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table); 2854 if (ret) { 2855 pr_err("%s: Policy frequency update failed\n", __func__); 2856 return ret; 2857 } 2858 2859 ret = freq_qos_update_request(policy->max_freq_req, policy->max); 2860 if (ret < 0) 2861 return ret; 2862 2863 return 0; 2864 } 2865 EXPORT_SYMBOL_GPL(cpufreq_boost_set_sw); 2866 2867 static int cpufreq_boost_trigger_state(int state) 2868 { 2869 struct cpufreq_policy *policy; 2870 unsigned long flags; 2871 int ret = 0; 2872 2873 if (cpufreq_driver->boost_enabled == state) 2874 return 0; 2875 2876 write_lock_irqsave(&cpufreq_driver_lock, flags); 2877 cpufreq_driver->boost_enabled = state; 2878 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2879 2880 cpus_read_lock(); 2881 for_each_active_policy(policy) { 2882 if (!policy->boost_supported) 2883 continue; 2884 2885 policy->boost_enabled = state; 2886 ret = cpufreq_driver->set_boost(policy, state); 2887 if (ret) { 2888 policy->boost_enabled = !policy->boost_enabled; 2889 goto err_reset_state; 2890 } 2891 } 2892 cpus_read_unlock(); 2893 2894 return 0; 2895 2896 err_reset_state: 2897 cpus_read_unlock(); 2898 2899 write_lock_irqsave(&cpufreq_driver_lock, flags); 2900 cpufreq_driver->boost_enabled = !state; 2901 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2902 2903 pr_err("%s: Cannot %s BOOST\n", 2904 __func__, str_enable_disable(state)); 2905 2906 return ret; 2907 } 2908 2909 static bool cpufreq_boost_supported(void) 2910 { 2911 return cpufreq_driver->set_boost; 2912 } 2913 2914 static int create_boost_sysfs_file(void) 2915 { 2916 int ret; 2917 2918 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr); 2919 if (ret) 2920 pr_err("%s: cannot register global BOOST sysfs file\n", 2921 __func__); 2922 2923 return ret; 2924 } 2925 2926 static void remove_boost_sysfs_file(void) 2927 { 2928 if (cpufreq_boost_supported()) 2929 sysfs_remove_file(cpufreq_global_kobject, &boost.attr); 2930 } 2931 2932 bool cpufreq_boost_enabled(void) 2933 { 2934 return cpufreq_driver->boost_enabled; 2935 } 2936 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled); 2937 2938 /********************************************************************* 2939 * REGISTER / UNREGISTER CPUFREQ DRIVER * 2940 *********************************************************************/ 2941 static enum cpuhp_state hp_online; 2942 2943 static int cpuhp_cpufreq_online(unsigned int cpu) 2944 { 2945 cpufreq_online(cpu); 2946 2947 return 0; 2948 } 2949 2950 static int cpuhp_cpufreq_offline(unsigned int cpu) 2951 { 2952 cpufreq_offline(cpu); 2953 2954 return 0; 2955 } 2956 2957 /** 2958 * cpufreq_register_driver - register a CPU Frequency driver 2959 * @driver_data: A struct cpufreq_driver containing the values# 2960 * submitted by the CPU Frequency driver. 2961 * 2962 * Registers a CPU Frequency driver to this core code. This code 2963 * returns zero on success, -EEXIST when another driver got here first 2964 * (and isn't unregistered in the meantime). 2965 * 2966 */ 2967 int cpufreq_register_driver(struct cpufreq_driver *driver_data) 2968 { 2969 unsigned long flags; 2970 int ret; 2971 2972 if (cpufreq_disabled()) 2973 return -ENODEV; 2974 2975 /* 2976 * The cpufreq core depends heavily on the availability of device 2977 * structure, make sure they are available before proceeding further. 2978 */ 2979 if (!get_cpu_device(0)) 2980 return -EPROBE_DEFER; 2981 2982 if (!driver_data || !driver_data->verify || !driver_data->init || 2983 !(driver_data->setpolicy || driver_data->target_index || 2984 driver_data->target) || 2985 (driver_data->setpolicy && (driver_data->target_index || 2986 driver_data->target)) || 2987 (!driver_data->get_intermediate != !driver_data->target_intermediate) || 2988 (!driver_data->online != !driver_data->offline) || 2989 (driver_data->adjust_perf && !driver_data->fast_switch)) 2990 return -EINVAL; 2991 2992 pr_debug("trying to register driver %s\n", driver_data->name); 2993 2994 /* Protect against concurrent CPU online/offline. */ 2995 cpus_read_lock(); 2996 2997 write_lock_irqsave(&cpufreq_driver_lock, flags); 2998 if (cpufreq_driver) { 2999 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 3000 ret = -EEXIST; 3001 goto out; 3002 } 3003 cpufreq_driver = driver_data; 3004 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 3005 3006 /* 3007 * Mark support for the scheduler's frequency invariance engine for 3008 * drivers that implement target(), target_index() or fast_switch(). 3009 */ 3010 if (!cpufreq_driver->setpolicy) { 3011 static_branch_enable_cpuslocked(&cpufreq_freq_invariance); 3012 pr_debug("supports frequency invariance"); 3013 } 3014 3015 if (driver_data->setpolicy) 3016 driver_data->flags |= CPUFREQ_CONST_LOOPS; 3017 3018 if (cpufreq_boost_supported()) { 3019 ret = create_boost_sysfs_file(); 3020 if (ret) 3021 goto err_null_driver; 3022 } 3023 3024 ret = subsys_interface_register(&cpufreq_interface); 3025 if (ret) 3026 goto err_boost_unreg; 3027 3028 if (unlikely(list_empty(&cpufreq_policy_list))) { 3029 /* if all ->init() calls failed, unregister */ 3030 ret = -ENODEV; 3031 pr_debug("%s: No CPU initialized for driver %s\n", __func__, 3032 driver_data->name); 3033 goto err_if_unreg; 3034 } 3035 3036 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, 3037 "cpufreq:online", 3038 cpuhp_cpufreq_online, 3039 cpuhp_cpufreq_offline); 3040 if (ret < 0) 3041 goto err_if_unreg; 3042 hp_online = ret; 3043 ret = 0; 3044 3045 pr_debug("driver %s up and running\n", driver_data->name); 3046 goto out; 3047 3048 err_if_unreg: 3049 subsys_interface_unregister(&cpufreq_interface); 3050 err_boost_unreg: 3051 remove_boost_sysfs_file(); 3052 err_null_driver: 3053 write_lock_irqsave(&cpufreq_driver_lock, flags); 3054 cpufreq_driver = NULL; 3055 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 3056 out: 3057 cpus_read_unlock(); 3058 return ret; 3059 } 3060 EXPORT_SYMBOL_GPL(cpufreq_register_driver); 3061 3062 /* 3063 * cpufreq_unregister_driver - unregister the current CPUFreq driver 3064 * 3065 * Unregister the current CPUFreq driver. Only call this if you have 3066 * the right to do so, i.e. if you have succeeded in initialising before! 3067 * Returns zero if successful, and -EINVAL if the cpufreq_driver is 3068 * currently not initialised. 3069 */ 3070 void cpufreq_unregister_driver(struct cpufreq_driver *driver) 3071 { 3072 unsigned long flags; 3073 3074 if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver))) 3075 return; 3076 3077 pr_debug("unregistering driver %s\n", driver->name); 3078 3079 /* Protect against concurrent cpu hotplug */ 3080 cpus_read_lock(); 3081 subsys_interface_unregister(&cpufreq_interface); 3082 remove_boost_sysfs_file(); 3083 static_branch_disable_cpuslocked(&cpufreq_freq_invariance); 3084 cpuhp_remove_state_nocalls_cpuslocked(hp_online); 3085 3086 write_lock_irqsave(&cpufreq_driver_lock, flags); 3087 3088 cpufreq_driver = NULL; 3089 3090 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 3091 cpus_read_unlock(); 3092 } 3093 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver); 3094 3095 static int __init cpufreq_core_init(void) 3096 { 3097 struct cpufreq_governor *gov = cpufreq_default_governor(); 3098 struct device *dev_root; 3099 3100 if (cpufreq_disabled()) 3101 return -ENODEV; 3102 3103 dev_root = bus_get_dev_root(&cpu_subsys); 3104 if (dev_root) { 3105 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj); 3106 put_device(dev_root); 3107 } 3108 BUG_ON(!cpufreq_global_kobject); 3109 3110 if (!strlen(default_governor)) 3111 strscpy(default_governor, gov->name, CPUFREQ_NAME_LEN); 3112 3113 return 0; 3114 } 3115 module_param(off, int, 0444); 3116 module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444); 3117 core_initcall(cpufreq_core_init); 3118