1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/drivers/cpufreq/cpufreq.c 4 * 5 * Copyright (C) 2001 Russell King 6 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> 7 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org> 8 * 9 * Oct 2005 - Ashok Raj <ashok.raj@intel.com> 10 * Added handling for CPU hotplug 11 * Feb 2006 - Jacob Shin <jacob.shin@amd.com> 12 * Fix handling for CPU hotplug -- affected CPUs 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/cpu.h> 18 #include <linux/cpufreq.h> 19 #include <linux/cpu_cooling.h> 20 #include <linux/delay.h> 21 #include <linux/device.h> 22 #include <linux/init.h> 23 #include <linux/kernel_stat.h> 24 #include <linux/module.h> 25 #include <linux/mutex.h> 26 #include <linux/pm_qos.h> 27 #include <linux/slab.h> 28 #include <linux/string_choices.h> 29 #include <linux/suspend.h> 30 #include <linux/syscore_ops.h> 31 #include <linux/tick.h> 32 #include <linux/units.h> 33 #include <trace/events/power.h> 34 35 static LIST_HEAD(cpufreq_policy_list); 36 37 /* Macros to iterate over CPU policies */ 38 #define for_each_suitable_policy(__policy, __active) \ 39 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \ 40 if ((__active) == !policy_is_inactive(__policy)) 41 42 #define for_each_active_policy(__policy) \ 43 for_each_suitable_policy(__policy, true) 44 #define for_each_inactive_policy(__policy) \ 45 for_each_suitable_policy(__policy, false) 46 47 /* Iterate over governors */ 48 static LIST_HEAD(cpufreq_governor_list); 49 #define for_each_governor(__governor) \ 50 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list) 51 52 static char default_governor[CPUFREQ_NAME_LEN]; 53 54 /* 55 * The "cpufreq driver" - the arch- or hardware-dependent low 56 * level driver of CPUFreq support, and its spinlock. This lock 57 * also protects the cpufreq_cpu_data array. 58 */ 59 static struct cpufreq_driver *cpufreq_driver; 60 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); 61 static DEFINE_RWLOCK(cpufreq_driver_lock); 62 63 static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance); 64 bool cpufreq_supports_freq_invariance(void) 65 { 66 return static_branch_likely(&cpufreq_freq_invariance); 67 } 68 69 /* Flag to suspend/resume CPUFreq governors */ 70 static bool cpufreq_suspended; 71 72 static inline bool has_target(void) 73 { 74 return cpufreq_driver->target_index || cpufreq_driver->target; 75 } 76 77 bool has_target_index(void) 78 { 79 return !!cpufreq_driver->target_index; 80 } 81 82 /* internal prototypes */ 83 static unsigned int __cpufreq_get(struct cpufreq_policy *policy); 84 static int cpufreq_init_governor(struct cpufreq_policy *policy); 85 static void cpufreq_exit_governor(struct cpufreq_policy *policy); 86 static void cpufreq_governor_limits(struct cpufreq_policy *policy); 87 static int cpufreq_set_policy(struct cpufreq_policy *policy, 88 struct cpufreq_governor *new_gov, 89 unsigned int new_pol); 90 static bool cpufreq_boost_supported(void); 91 static int cpufreq_boost_trigger_state(int state); 92 93 /* 94 * Two notifier lists: the "policy" list is involved in the 95 * validation process for a new CPU frequency policy; the 96 * "transition" list for kernel code that needs to handle 97 * changes to devices when the CPU clock speed changes. 98 * The mutex locks both lists. 99 */ 100 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list); 101 SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list); 102 103 static int off __read_mostly; 104 static int cpufreq_disabled(void) 105 { 106 return off; 107 } 108 void disable_cpufreq(void) 109 { 110 off = 1; 111 } 112 EXPORT_SYMBOL_GPL(disable_cpufreq); 113 114 static DEFINE_MUTEX(cpufreq_governor_mutex); 115 116 bool have_governor_per_policy(void) 117 { 118 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY); 119 } 120 EXPORT_SYMBOL_GPL(have_governor_per_policy); 121 122 static struct kobject *cpufreq_global_kobject; 123 124 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) 125 { 126 if (have_governor_per_policy()) 127 return &policy->kobj; 128 else 129 return cpufreq_global_kobject; 130 } 131 EXPORT_SYMBOL_GPL(get_governor_parent_kobj); 132 133 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) 134 { 135 struct kernel_cpustat kcpustat; 136 u64 cur_wall_time; 137 u64 idle_time; 138 u64 busy_time; 139 140 cur_wall_time = jiffies64_to_nsecs(get_jiffies_64()); 141 142 kcpustat_cpu_fetch(&kcpustat, cpu); 143 144 busy_time = kcpustat.cpustat[CPUTIME_USER]; 145 busy_time += kcpustat.cpustat[CPUTIME_SYSTEM]; 146 busy_time += kcpustat.cpustat[CPUTIME_IRQ]; 147 busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ]; 148 busy_time += kcpustat.cpustat[CPUTIME_STEAL]; 149 busy_time += kcpustat.cpustat[CPUTIME_NICE]; 150 151 idle_time = cur_wall_time - busy_time; 152 if (wall) 153 *wall = div_u64(cur_wall_time, NSEC_PER_USEC); 154 155 return div_u64(idle_time, NSEC_PER_USEC); 156 } 157 158 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) 159 { 160 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL); 161 162 if (idle_time == -1ULL) 163 return get_cpu_idle_time_jiffy(cpu, wall); 164 else if (!io_busy) 165 idle_time += get_cpu_iowait_time_us(cpu, wall); 166 167 return idle_time; 168 } 169 EXPORT_SYMBOL_GPL(get_cpu_idle_time); 170 171 /* 172 * This is a generic cpufreq init() routine which can be used by cpufreq 173 * drivers of SMP systems. It will do following: 174 * - validate & show freq table passed 175 * - set policies transition latency 176 * - policy->cpus with all possible CPUs 177 */ 178 void cpufreq_generic_init(struct cpufreq_policy *policy, 179 struct cpufreq_frequency_table *table, 180 unsigned int transition_latency) 181 { 182 policy->freq_table = table; 183 policy->cpuinfo.transition_latency = transition_latency; 184 185 /* 186 * The driver only supports the SMP configuration where all processors 187 * share the clock and voltage and clock. 188 */ 189 cpumask_setall(policy->cpus); 190 } 191 EXPORT_SYMBOL_GPL(cpufreq_generic_init); 192 193 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu) 194 { 195 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 196 197 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL; 198 } 199 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw); 200 201 struct cpufreq_policy *cpufreq_cpu_policy(unsigned int cpu) 202 { 203 return per_cpu(cpufreq_cpu_data, cpu); 204 } 205 EXPORT_SYMBOL_GPL(cpufreq_cpu_policy); 206 207 unsigned int cpufreq_generic_get(unsigned int cpu) 208 { 209 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); 210 211 if (!policy || IS_ERR(policy->clk)) { 212 pr_err("%s: No %s associated to cpu: %d\n", 213 __func__, policy ? "clk" : "policy", cpu); 214 return 0; 215 } 216 217 return clk_get_rate(policy->clk) / 1000; 218 } 219 EXPORT_SYMBOL_GPL(cpufreq_generic_get); 220 221 /** 222 * cpufreq_cpu_get - Return policy for a CPU and mark it as busy. 223 * @cpu: CPU to find the policy for. 224 * 225 * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment 226 * the kobject reference counter of that policy. Return a valid policy on 227 * success or NULL on failure. 228 * 229 * The policy returned by this function has to be released with the help of 230 * cpufreq_cpu_put() to balance its kobject reference counter properly. 231 */ 232 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) 233 { 234 struct cpufreq_policy *policy = NULL; 235 unsigned long flags; 236 237 if (WARN_ON(cpu >= nr_cpu_ids)) 238 return NULL; 239 240 /* get the cpufreq driver */ 241 read_lock_irqsave(&cpufreq_driver_lock, flags); 242 243 if (cpufreq_driver) { 244 /* get the CPU */ 245 policy = cpufreq_cpu_get_raw(cpu); 246 if (policy) 247 kobject_get(&policy->kobj); 248 } 249 250 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 251 252 return policy; 253 } 254 EXPORT_SYMBOL_GPL(cpufreq_cpu_get); 255 256 /** 257 * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy. 258 * @policy: cpufreq policy returned by cpufreq_cpu_get(). 259 */ 260 void cpufreq_cpu_put(struct cpufreq_policy *policy) 261 { 262 kobject_put(&policy->kobj); 263 } 264 EXPORT_SYMBOL_GPL(cpufreq_cpu_put); 265 266 /********************************************************************* 267 * EXTERNALLY AFFECTING FREQUENCY CHANGES * 268 *********************************************************************/ 269 270 /** 271 * adjust_jiffies - Adjust the system "loops_per_jiffy". 272 * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE. 273 * @ci: Frequency change information. 274 * 275 * This function alters the system "loops_per_jiffy" for the clock 276 * speed change. Note that loops_per_jiffy cannot be updated on SMP 277 * systems as each CPU might be scaled differently. So, use the arch 278 * per-CPU loops_per_jiffy value wherever possible. 279 */ 280 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) 281 { 282 #ifndef CONFIG_SMP 283 static unsigned long l_p_j_ref; 284 static unsigned int l_p_j_ref_freq; 285 286 if (ci->flags & CPUFREQ_CONST_LOOPS) 287 return; 288 289 if (!l_p_j_ref_freq) { 290 l_p_j_ref = loops_per_jiffy; 291 l_p_j_ref_freq = ci->old; 292 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n", 293 l_p_j_ref, l_p_j_ref_freq); 294 } 295 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) { 296 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq, 297 ci->new); 298 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n", 299 loops_per_jiffy, ci->new); 300 } 301 #endif 302 } 303 304 /** 305 * cpufreq_notify_transition - Notify frequency transition and adjust jiffies. 306 * @policy: cpufreq policy to enable fast frequency switching for. 307 * @freqs: contain details of the frequency update. 308 * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE. 309 * 310 * This function calls the transition notifiers and adjust_jiffies(). 311 * 312 * It is called twice on all CPU frequency changes that have external effects. 313 */ 314 static void cpufreq_notify_transition(struct cpufreq_policy *policy, 315 struct cpufreq_freqs *freqs, 316 unsigned int state) 317 { 318 int cpu; 319 320 BUG_ON(irqs_disabled()); 321 322 if (cpufreq_disabled()) 323 return; 324 325 freqs->policy = policy; 326 freqs->flags = cpufreq_driver->flags; 327 pr_debug("notification %u of frequency transition to %u kHz\n", 328 state, freqs->new); 329 330 switch (state) { 331 case CPUFREQ_PRECHANGE: 332 /* 333 * Detect if the driver reported a value as "old frequency" 334 * which is not equal to what the cpufreq core thinks is 335 * "old frequency". 336 */ 337 if (policy->cur && policy->cur != freqs->old) { 338 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n", 339 freqs->old, policy->cur); 340 freqs->old = policy->cur; 341 } 342 343 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 344 CPUFREQ_PRECHANGE, freqs); 345 346 adjust_jiffies(CPUFREQ_PRECHANGE, freqs); 347 break; 348 349 case CPUFREQ_POSTCHANGE: 350 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs); 351 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new, 352 cpumask_pr_args(policy->cpus)); 353 354 for_each_cpu(cpu, policy->cpus) 355 trace_cpu_frequency(freqs->new, cpu); 356 357 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 358 CPUFREQ_POSTCHANGE, freqs); 359 360 cpufreq_stats_record_transition(policy, freqs->new); 361 policy->cur = freqs->new; 362 } 363 } 364 365 /* Do post notifications when there are chances that transition has failed */ 366 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy, 367 struct cpufreq_freqs *freqs, int transition_failed) 368 { 369 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 370 if (!transition_failed) 371 return; 372 373 swap(freqs->old, freqs->new); 374 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 375 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 376 } 377 378 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, 379 struct cpufreq_freqs *freqs) 380 { 381 382 /* 383 * Catch double invocations of _begin() which lead to self-deadlock. 384 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core 385 * doesn't invoke _begin() on their behalf, and hence the chances of 386 * double invocations are very low. Moreover, there are scenarios 387 * where these checks can emit false-positive warnings in these 388 * drivers; so we avoid that by skipping them altogether. 389 */ 390 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION) 391 && current == policy->transition_task); 392 393 wait: 394 wait_event(policy->transition_wait, !policy->transition_ongoing); 395 396 spin_lock(&policy->transition_lock); 397 398 if (unlikely(policy->transition_ongoing)) { 399 spin_unlock(&policy->transition_lock); 400 goto wait; 401 } 402 403 policy->transition_ongoing = true; 404 policy->transition_task = current; 405 406 spin_unlock(&policy->transition_lock); 407 408 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 409 } 410 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin); 411 412 void cpufreq_freq_transition_end(struct cpufreq_policy *policy, 413 struct cpufreq_freqs *freqs, int transition_failed) 414 { 415 if (WARN_ON(!policy->transition_ongoing)) 416 return; 417 418 cpufreq_notify_post_transition(policy, freqs, transition_failed); 419 420 arch_set_freq_scale(policy->related_cpus, 421 policy->cur, 422 arch_scale_freq_ref(policy->cpu)); 423 424 spin_lock(&policy->transition_lock); 425 policy->transition_ongoing = false; 426 policy->transition_task = NULL; 427 spin_unlock(&policy->transition_lock); 428 429 wake_up(&policy->transition_wait); 430 } 431 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end); 432 433 /* 434 * Fast frequency switching status count. Positive means "enabled", negative 435 * means "disabled" and 0 means "not decided yet". 436 */ 437 static int cpufreq_fast_switch_count; 438 static DEFINE_MUTEX(cpufreq_fast_switch_lock); 439 440 static void cpufreq_list_transition_notifiers(void) 441 { 442 struct notifier_block *nb; 443 444 pr_info("Registered transition notifiers:\n"); 445 446 mutex_lock(&cpufreq_transition_notifier_list.mutex); 447 448 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next) 449 pr_info("%pS\n", nb->notifier_call); 450 451 mutex_unlock(&cpufreq_transition_notifier_list.mutex); 452 } 453 454 /** 455 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy. 456 * @policy: cpufreq policy to enable fast frequency switching for. 457 * 458 * Try to enable fast frequency switching for @policy. 459 * 460 * The attempt will fail if there is at least one transition notifier registered 461 * at this point, as fast frequency switching is quite fundamentally at odds 462 * with transition notifiers. Thus if successful, it will make registration of 463 * transition notifiers fail going forward. 464 */ 465 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy) 466 { 467 lockdep_assert_held(&policy->rwsem); 468 469 if (!policy->fast_switch_possible) 470 return; 471 472 mutex_lock(&cpufreq_fast_switch_lock); 473 if (cpufreq_fast_switch_count >= 0) { 474 cpufreq_fast_switch_count++; 475 policy->fast_switch_enabled = true; 476 } else { 477 pr_warn("CPU%u: Fast frequency switching not enabled\n", 478 policy->cpu); 479 cpufreq_list_transition_notifiers(); 480 } 481 mutex_unlock(&cpufreq_fast_switch_lock); 482 } 483 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch); 484 485 /** 486 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy. 487 * @policy: cpufreq policy to disable fast frequency switching for. 488 */ 489 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) 490 { 491 mutex_lock(&cpufreq_fast_switch_lock); 492 if (policy->fast_switch_enabled) { 493 policy->fast_switch_enabled = false; 494 if (!WARN_ON(cpufreq_fast_switch_count <= 0)) 495 cpufreq_fast_switch_count--; 496 } 497 mutex_unlock(&cpufreq_fast_switch_lock); 498 } 499 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); 500 501 static unsigned int __resolve_freq(struct cpufreq_policy *policy, 502 unsigned int target_freq, 503 unsigned int min, unsigned int max, 504 unsigned int relation) 505 { 506 unsigned int idx; 507 508 target_freq = clamp_val(target_freq, min, max); 509 510 if (!policy->freq_table) 511 return target_freq; 512 513 idx = cpufreq_frequency_table_target(policy, target_freq, min, max, relation); 514 policy->cached_resolved_idx = idx; 515 policy->cached_target_freq = target_freq; 516 return policy->freq_table[idx].frequency; 517 } 518 519 /** 520 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported 521 * one. 522 * @policy: associated policy to interrogate 523 * @target_freq: target frequency to resolve. 524 * 525 * The target to driver frequency mapping is cached in the policy. 526 * 527 * Return: Lowest driver-supported frequency greater than or equal to the 528 * given target_freq, subject to policy (min/max) and driver limitations. 529 */ 530 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy, 531 unsigned int target_freq) 532 { 533 unsigned int min = READ_ONCE(policy->min); 534 unsigned int max = READ_ONCE(policy->max); 535 536 /* 537 * If this function runs in parallel with cpufreq_set_policy(), it may 538 * read policy->min before the update and policy->max after the update 539 * or the other way around, so there is no ordering guarantee. 540 * 541 * Resolve this by always honoring the max (in case it comes from 542 * thermal throttling or similar). 543 */ 544 if (unlikely(min > max)) 545 min = max; 546 547 return __resolve_freq(policy, target_freq, min, max, CPUFREQ_RELATION_LE); 548 } 549 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq); 550 551 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy) 552 { 553 unsigned int latency; 554 555 if (policy->transition_delay_us) 556 return policy->transition_delay_us; 557 558 latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC; 559 if (latency) 560 /* Give a 50% breathing room between updates */ 561 return latency + (latency >> 1); 562 563 return USEC_PER_MSEC; 564 } 565 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us); 566 567 /********************************************************************* 568 * SYSFS INTERFACE * 569 *********************************************************************/ 570 static ssize_t show_boost(struct kobject *kobj, 571 struct kobj_attribute *attr, char *buf) 572 { 573 return sysfs_emit(buf, "%d\n", cpufreq_driver->boost_enabled); 574 } 575 576 static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr, 577 const char *buf, size_t count) 578 { 579 bool enable; 580 581 if (kstrtobool(buf, &enable)) 582 return -EINVAL; 583 584 if (cpufreq_boost_trigger_state(enable)) { 585 pr_err("%s: Cannot %s BOOST!\n", 586 __func__, str_enable_disable(enable)); 587 return -EINVAL; 588 } 589 590 pr_debug("%s: cpufreq BOOST %s\n", 591 __func__, str_enabled_disabled(enable)); 592 593 return count; 594 } 595 define_one_global_rw(boost); 596 597 static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf) 598 { 599 return sysfs_emit(buf, "%d\n", policy->boost_enabled); 600 } 601 602 static int policy_set_boost(struct cpufreq_policy *policy, bool enable) 603 { 604 int ret; 605 606 if (policy->boost_enabled == enable) 607 return 0; 608 609 policy->boost_enabled = enable; 610 611 ret = cpufreq_driver->set_boost(policy, enable); 612 if (ret) { 613 policy->boost_enabled = !policy->boost_enabled; 614 return ret; 615 } 616 617 ret = freq_qos_update_request(&policy->boost_freq_req, policy->cpuinfo.max_freq); 618 if (ret < 0) { 619 policy->boost_enabled = !policy->boost_enabled; 620 cpufreq_driver->set_boost(policy, policy->boost_enabled); 621 return ret; 622 } 623 624 return 0; 625 } 626 627 static ssize_t store_local_boost(struct cpufreq_policy *policy, 628 const char *buf, size_t count) 629 { 630 int ret; 631 bool enable; 632 633 if (kstrtobool(buf, &enable)) 634 return -EINVAL; 635 636 if (!cpufreq_driver->boost_enabled) 637 return -EINVAL; 638 639 if (!policy->boost_supported) 640 return -EINVAL; 641 642 ret = policy_set_boost(policy, enable); 643 if (!ret) 644 return count; 645 646 return ret; 647 } 648 649 static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost); 650 651 static struct cpufreq_governor *find_governor(const char *str_governor) 652 { 653 struct cpufreq_governor *t; 654 655 for_each_governor(t) 656 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN)) 657 return t; 658 659 return NULL; 660 } 661 662 static struct cpufreq_governor *get_governor(const char *str_governor) 663 { 664 struct cpufreq_governor *t; 665 666 mutex_lock(&cpufreq_governor_mutex); 667 t = find_governor(str_governor); 668 if (!t) 669 goto unlock; 670 671 if (!try_module_get(t->owner)) 672 t = NULL; 673 674 unlock: 675 mutex_unlock(&cpufreq_governor_mutex); 676 677 return t; 678 } 679 680 static unsigned int cpufreq_parse_policy(char *str_governor) 681 { 682 if (!strncasecmp(str_governor, "performance", strlen("performance"))) 683 return CPUFREQ_POLICY_PERFORMANCE; 684 685 if (!strncasecmp(str_governor, "powersave", strlen("powersave"))) 686 return CPUFREQ_POLICY_POWERSAVE; 687 688 return CPUFREQ_POLICY_UNKNOWN; 689 } 690 691 /** 692 * cpufreq_parse_governor - parse a governor string only for has_target() 693 * @str_governor: Governor name. 694 */ 695 static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor) 696 { 697 struct cpufreq_governor *t; 698 699 t = get_governor(str_governor); 700 if (t) 701 return t; 702 703 if (request_module("cpufreq_%s", str_governor)) 704 return NULL; 705 706 return get_governor(str_governor); 707 } 708 709 /* 710 * cpufreq_per_cpu_attr_read() / show_##file_name() - 711 * print out cpufreq information 712 * 713 * Write out information from cpufreq_driver->policy[cpu]; object must be 714 * "unsigned int". 715 */ 716 717 #define show_one(file_name, object) \ 718 static ssize_t show_##file_name \ 719 (struct cpufreq_policy *policy, char *buf) \ 720 { \ 721 return sysfs_emit(buf, "%u\n", policy->object); \ 722 } 723 724 show_one(cpuinfo_min_freq, cpuinfo.min_freq); 725 show_one(cpuinfo_max_freq, cpuinfo.max_freq); 726 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency); 727 show_one(scaling_min_freq, min); 728 show_one(scaling_max_freq, max); 729 730 __weak int arch_freq_get_on_cpu(int cpu) 731 { 732 return -EOPNOTSUPP; 733 } 734 735 static inline bool cpufreq_avg_freq_supported(struct cpufreq_policy *policy) 736 { 737 return arch_freq_get_on_cpu(policy->cpu) != -EOPNOTSUPP; 738 } 739 740 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf) 741 { 742 ssize_t ret; 743 int freq; 744 745 freq = IS_ENABLED(CONFIG_CPUFREQ_ARCH_CUR_FREQ) 746 ? arch_freq_get_on_cpu(policy->cpu) 747 : 0; 748 749 if (freq > 0) 750 ret = sysfs_emit(buf, "%u\n", freq); 751 else if (cpufreq_driver->setpolicy && cpufreq_driver->get) 752 ret = sysfs_emit(buf, "%u\n", cpufreq_driver->get(policy->cpu)); 753 else 754 ret = sysfs_emit(buf, "%u\n", policy->cur); 755 return ret; 756 } 757 758 /* 759 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access 760 */ 761 #define store_one(file_name, object) \ 762 static ssize_t store_##file_name \ 763 (struct cpufreq_policy *policy, const char *buf, size_t count) \ 764 { \ 765 unsigned long val; \ 766 int ret; \ 767 \ 768 ret = kstrtoul(buf, 0, &val); \ 769 if (ret) \ 770 return ret; \ 771 \ 772 ret = freq_qos_update_request(&policy->object##_freq_req, val); \ 773 return ret >= 0 ? count : ret; \ 774 } 775 776 store_one(scaling_min_freq, min); 777 store_one(scaling_max_freq, max); 778 779 /* 780 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware 781 */ 782 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, 783 char *buf) 784 { 785 unsigned int cur_freq = __cpufreq_get(policy); 786 787 if (cur_freq) 788 return sysfs_emit(buf, "%u\n", cur_freq); 789 790 return sysfs_emit(buf, "<unknown>\n"); 791 } 792 793 /* 794 * show_cpuinfo_avg_freq - average CPU frequency as detected by hardware 795 */ 796 static ssize_t show_cpuinfo_avg_freq(struct cpufreq_policy *policy, 797 char *buf) 798 { 799 int avg_freq = arch_freq_get_on_cpu(policy->cpu); 800 801 if (avg_freq > 0) 802 return sysfs_emit(buf, "%u\n", avg_freq); 803 return avg_freq != 0 ? avg_freq : -EINVAL; 804 } 805 806 /* 807 * show_scaling_governor - show the current policy for the specified CPU 808 */ 809 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) 810 { 811 if (policy->policy == CPUFREQ_POLICY_POWERSAVE) 812 return sysfs_emit(buf, "powersave\n"); 813 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) 814 return sysfs_emit(buf, "performance\n"); 815 else if (policy->governor) 816 return sysfs_emit(buf, "%s\n", policy->governor->name); 817 return -EINVAL; 818 } 819 820 /* 821 * store_scaling_governor - store policy for the specified CPU 822 */ 823 static ssize_t store_scaling_governor(struct cpufreq_policy *policy, 824 const char *buf, size_t count) 825 { 826 char str_governor[CPUFREQ_NAME_LEN]; 827 int ret; 828 829 ret = sscanf(buf, "%15s", str_governor); 830 if (ret != 1) 831 return -EINVAL; 832 833 if (cpufreq_driver->setpolicy) { 834 unsigned int new_pol; 835 836 new_pol = cpufreq_parse_policy(str_governor); 837 if (!new_pol) 838 return -EINVAL; 839 840 ret = cpufreq_set_policy(policy, NULL, new_pol); 841 } else { 842 struct cpufreq_governor *new_gov; 843 844 new_gov = cpufreq_parse_governor(str_governor); 845 if (!new_gov) 846 return -EINVAL; 847 848 ret = cpufreq_set_policy(policy, new_gov, 849 CPUFREQ_POLICY_UNKNOWN); 850 851 module_put(new_gov->owner); 852 } 853 854 return ret ? ret : count; 855 } 856 857 /* 858 * show_scaling_driver - show the cpufreq driver currently loaded 859 */ 860 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf) 861 { 862 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name); 863 } 864 865 /* 866 * show_scaling_available_governors - show the available CPUfreq governors 867 */ 868 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy, 869 char *buf) 870 { 871 ssize_t i = 0; 872 struct cpufreq_governor *t; 873 874 if (!has_target()) { 875 i += sysfs_emit(buf, "performance powersave"); 876 goto out; 877 } 878 879 mutex_lock(&cpufreq_governor_mutex); 880 for_each_governor(t) { 881 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) 882 - (CPUFREQ_NAME_LEN + 2))) 883 break; 884 i += sysfs_emit_at(buf, i, "%s ", t->name); 885 } 886 mutex_unlock(&cpufreq_governor_mutex); 887 out: 888 i += sysfs_emit_at(buf, i, "\n"); 889 return i; 890 } 891 892 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf) 893 { 894 ssize_t i = 0; 895 unsigned int cpu; 896 897 for_each_cpu(cpu, mask) { 898 i += sysfs_emit_at(buf, i, "%u ", cpu); 899 if (i >= (PAGE_SIZE - 5)) 900 break; 901 } 902 903 /* Remove the extra space at the end */ 904 i--; 905 906 i += sysfs_emit_at(buf, i, "\n"); 907 return i; 908 } 909 EXPORT_SYMBOL_GPL(cpufreq_show_cpus); 910 911 /* 912 * show_related_cpus - show the CPUs affected by each transition even if 913 * hw coordination is in use 914 */ 915 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf) 916 { 917 return cpufreq_show_cpus(policy->related_cpus, buf); 918 } 919 920 /* 921 * show_affected_cpus - show the CPUs affected by each transition 922 */ 923 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf) 924 { 925 return cpufreq_show_cpus(policy->cpus, buf); 926 } 927 928 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy, 929 const char *buf, size_t count) 930 { 931 unsigned int freq = 0; 932 int ret; 933 934 if (!policy->governor || !policy->governor->store_setspeed) 935 return -EINVAL; 936 937 ret = kstrtouint(buf, 0, &freq); 938 if (ret) 939 return ret; 940 941 policy->governor->store_setspeed(policy, freq); 942 943 return count; 944 } 945 946 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf) 947 { 948 if (!policy->governor || !policy->governor->show_setspeed) 949 return sysfs_emit(buf, "<unsupported>\n"); 950 951 return policy->governor->show_setspeed(policy, buf); 952 } 953 954 /* 955 * show_bios_limit - show the current cpufreq HW/BIOS limitation 956 */ 957 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf) 958 { 959 unsigned int limit; 960 int ret; 961 ret = cpufreq_driver->bios_limit(policy->cpu, &limit); 962 if (!ret) 963 return sysfs_emit(buf, "%u\n", limit); 964 return sysfs_emit(buf, "%u\n", policy->cpuinfo.max_freq); 965 } 966 967 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400); 968 cpufreq_freq_attr_ro(cpuinfo_avg_freq); 969 cpufreq_freq_attr_ro(cpuinfo_min_freq); 970 cpufreq_freq_attr_ro(cpuinfo_max_freq); 971 cpufreq_freq_attr_ro(cpuinfo_transition_latency); 972 cpufreq_freq_attr_ro(scaling_available_governors); 973 cpufreq_freq_attr_ro(scaling_driver); 974 cpufreq_freq_attr_ro(scaling_cur_freq); 975 cpufreq_freq_attr_ro(bios_limit); 976 cpufreq_freq_attr_ro(related_cpus); 977 cpufreq_freq_attr_ro(affected_cpus); 978 cpufreq_freq_attr_rw(scaling_min_freq); 979 cpufreq_freq_attr_rw(scaling_max_freq); 980 cpufreq_freq_attr_rw(scaling_governor); 981 cpufreq_freq_attr_rw(scaling_setspeed); 982 983 static struct attribute *cpufreq_attrs[] = { 984 &cpuinfo_min_freq.attr, 985 &cpuinfo_max_freq.attr, 986 &cpuinfo_transition_latency.attr, 987 &scaling_cur_freq.attr, 988 &scaling_min_freq.attr, 989 &scaling_max_freq.attr, 990 &affected_cpus.attr, 991 &related_cpus.attr, 992 &scaling_governor.attr, 993 &scaling_driver.attr, 994 &scaling_available_governors.attr, 995 &scaling_setspeed.attr, 996 NULL 997 }; 998 ATTRIBUTE_GROUPS(cpufreq); 999 1000 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj) 1001 #define to_attr(a) container_of(a, struct freq_attr, attr) 1002 1003 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) 1004 { 1005 struct cpufreq_policy *policy = to_policy(kobj); 1006 struct freq_attr *fattr = to_attr(attr); 1007 1008 if (!fattr->show) 1009 return -EIO; 1010 1011 guard(cpufreq_policy_read)(policy); 1012 1013 if (likely(!policy_is_inactive(policy))) 1014 return fattr->show(policy, buf); 1015 1016 return -EBUSY; 1017 } 1018 1019 static ssize_t store(struct kobject *kobj, struct attribute *attr, 1020 const char *buf, size_t count) 1021 { 1022 struct cpufreq_policy *policy = to_policy(kobj); 1023 struct freq_attr *fattr = to_attr(attr); 1024 1025 if (!fattr->store) 1026 return -EIO; 1027 1028 guard(cpufreq_policy_write)(policy); 1029 1030 if (likely(!policy_is_inactive(policy))) 1031 return fattr->store(policy, buf, count); 1032 1033 return -EBUSY; 1034 } 1035 1036 static void cpufreq_sysfs_release(struct kobject *kobj) 1037 { 1038 struct cpufreq_policy *policy = to_policy(kobj); 1039 pr_debug("last reference is dropped\n"); 1040 complete(&policy->kobj_unregister); 1041 } 1042 1043 static const struct sysfs_ops sysfs_ops = { 1044 .show = show, 1045 .store = store, 1046 }; 1047 1048 static const struct kobj_type ktype_cpufreq = { 1049 .sysfs_ops = &sysfs_ops, 1050 .default_groups = cpufreq_groups, 1051 .release = cpufreq_sysfs_release, 1052 }; 1053 1054 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu, 1055 struct device *dev) 1056 { 1057 if (unlikely(!dev)) 1058 return; 1059 1060 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus)) 1061 return; 1062 1063 dev_dbg(dev, "%s: Adding symlink\n", __func__); 1064 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq")) 1065 dev_err(dev, "cpufreq symlink creation failed\n"); 1066 } 1067 1068 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu, 1069 struct device *dev) 1070 { 1071 dev_dbg(dev, "%s: Removing symlink\n", __func__); 1072 sysfs_remove_link(&dev->kobj, "cpufreq"); 1073 cpumask_clear_cpu(cpu, policy->real_cpus); 1074 } 1075 1076 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy) 1077 { 1078 struct freq_attr **drv_attr; 1079 int ret = 0; 1080 1081 /* Attributes that need freq_table */ 1082 if (policy->freq_table) { 1083 ret = sysfs_create_file(&policy->kobj, 1084 &cpufreq_freq_attr_scaling_available_freqs.attr); 1085 if (ret) 1086 return ret; 1087 1088 if (cpufreq_boost_supported()) { 1089 ret = sysfs_create_file(&policy->kobj, 1090 &cpufreq_freq_attr_scaling_boost_freqs.attr); 1091 if (ret) 1092 return ret; 1093 } 1094 } 1095 1096 /* set up files for this cpu device */ 1097 drv_attr = cpufreq_driver->attr; 1098 while (drv_attr && *drv_attr) { 1099 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr)); 1100 if (ret) 1101 return ret; 1102 drv_attr++; 1103 } 1104 if (cpufreq_driver->get) { 1105 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr); 1106 if (ret) 1107 return ret; 1108 } 1109 1110 if (cpufreq_avg_freq_supported(policy)) { 1111 ret = sysfs_create_file(&policy->kobj, &cpuinfo_avg_freq.attr); 1112 if (ret) 1113 return ret; 1114 } 1115 1116 if (cpufreq_driver->bios_limit) { 1117 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr); 1118 if (ret) 1119 return ret; 1120 } 1121 1122 if (cpufreq_boost_supported()) { 1123 ret = sysfs_create_file(&policy->kobj, &local_boost.attr); 1124 if (ret) 1125 return ret; 1126 } 1127 1128 return 0; 1129 } 1130 1131 static int cpufreq_init_policy(struct cpufreq_policy *policy) 1132 { 1133 struct cpufreq_governor *gov = NULL; 1134 unsigned int pol = CPUFREQ_POLICY_UNKNOWN; 1135 int ret; 1136 1137 if (has_target()) { 1138 /* Update policy governor to the one used before hotplug. */ 1139 if (policy->last_governor[0] != '\0') 1140 gov = get_governor(policy->last_governor); 1141 if (gov) { 1142 pr_debug("Restoring governor %s for cpu %d\n", 1143 gov->name, policy->cpu); 1144 } else { 1145 gov = get_governor(default_governor); 1146 } 1147 1148 if (!gov) { 1149 gov = cpufreq_default_governor(); 1150 __module_get(gov->owner); 1151 } 1152 1153 } else { 1154 1155 /* Use the default policy if there is no last_policy. */ 1156 if (policy->last_policy) { 1157 pol = policy->last_policy; 1158 } else { 1159 pol = cpufreq_parse_policy(default_governor); 1160 /* 1161 * In case the default governor is neither "performance" 1162 * nor "powersave", fall back to the initial policy 1163 * value set by the driver. 1164 */ 1165 if (pol == CPUFREQ_POLICY_UNKNOWN) 1166 pol = policy->policy; 1167 } 1168 if (pol != CPUFREQ_POLICY_PERFORMANCE && 1169 pol != CPUFREQ_POLICY_POWERSAVE) 1170 return -ENODATA; 1171 } 1172 1173 ret = cpufreq_set_policy(policy, gov, pol); 1174 if (gov) 1175 module_put(gov->owner); 1176 1177 return ret; 1178 } 1179 1180 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu) 1181 { 1182 int ret = 0; 1183 1184 /* Has this CPU been taken care of already? */ 1185 if (cpumask_test_cpu(cpu, policy->cpus)) 1186 return 0; 1187 1188 guard(cpufreq_policy_write)(policy); 1189 1190 if (has_target()) 1191 cpufreq_stop_governor(policy); 1192 1193 cpumask_set_cpu(cpu, policy->cpus); 1194 1195 if (has_target()) { 1196 ret = cpufreq_start_governor(policy); 1197 if (ret) 1198 pr_err("%s: Failed to start governor\n", __func__); 1199 } 1200 1201 return ret; 1202 } 1203 1204 void refresh_frequency_limits(struct cpufreq_policy *policy) 1205 { 1206 if (!policy_is_inactive(policy)) { 1207 pr_debug("updating policy for CPU %u\n", policy->cpu); 1208 1209 cpufreq_set_policy(policy, policy->governor, policy->policy); 1210 } 1211 } 1212 EXPORT_SYMBOL(refresh_frequency_limits); 1213 1214 static void handle_update(struct work_struct *work) 1215 { 1216 struct cpufreq_policy *policy = 1217 container_of(work, struct cpufreq_policy, update); 1218 1219 pr_debug("handle_update for cpu %u called\n", policy->cpu); 1220 1221 guard(cpufreq_policy_write)(policy); 1222 1223 refresh_frequency_limits(policy); 1224 } 1225 1226 static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq, 1227 void *data) 1228 { 1229 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min); 1230 1231 schedule_work(&policy->update); 1232 return 0; 1233 } 1234 1235 static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq, 1236 void *data) 1237 { 1238 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max); 1239 1240 schedule_work(&policy->update); 1241 return 0; 1242 } 1243 1244 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) 1245 { 1246 struct kobject *kobj; 1247 struct completion *cmp; 1248 1249 scoped_guard(cpufreq_policy_write, policy) { 1250 cpufreq_stats_free_table(policy); 1251 kobj = &policy->kobj; 1252 cmp = &policy->kobj_unregister; 1253 } 1254 kobject_put(kobj); 1255 1256 /* 1257 * We need to make sure that the underlying kobj is 1258 * actually not referenced anymore by anybody before we 1259 * proceed with unloading. 1260 */ 1261 pr_debug("waiting for dropping of refcount\n"); 1262 wait_for_completion(cmp); 1263 pr_debug("wait complete\n"); 1264 } 1265 1266 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) 1267 { 1268 struct cpufreq_policy *policy; 1269 struct device *dev = get_cpu_device(cpu); 1270 int ret; 1271 1272 if (!dev) 1273 return NULL; 1274 1275 policy = kzalloc_obj(*policy); 1276 if (!policy) 1277 return NULL; 1278 1279 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) 1280 goto err_free_policy; 1281 1282 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) 1283 goto err_free_cpumask; 1284 1285 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL)) 1286 goto err_free_rcpumask; 1287 1288 init_completion(&policy->kobj_unregister); 1289 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, 1290 cpufreq_global_kobject, "policy%u", cpu); 1291 if (ret) { 1292 dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret); 1293 /* 1294 * The entire policy object will be freed below, but the extra 1295 * memory allocated for the kobject name needs to be freed by 1296 * releasing the kobject. 1297 */ 1298 kobject_put(&policy->kobj); 1299 goto err_free_real_cpus; 1300 } 1301 1302 init_rwsem(&policy->rwsem); 1303 1304 freq_constraints_init(&policy->constraints); 1305 1306 policy->nb_min.notifier_call = cpufreq_notifier_min; 1307 policy->nb_max.notifier_call = cpufreq_notifier_max; 1308 1309 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN, 1310 &policy->nb_min); 1311 if (ret) { 1312 dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n", 1313 ret, cpu); 1314 goto err_kobj_remove; 1315 } 1316 1317 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX, 1318 &policy->nb_max); 1319 if (ret) { 1320 dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n", 1321 ret, cpu); 1322 goto err_min_qos_notifier; 1323 } 1324 1325 INIT_LIST_HEAD(&policy->policy_list); 1326 spin_lock_init(&policy->transition_lock); 1327 init_waitqueue_head(&policy->transition_wait); 1328 INIT_WORK(&policy->update, handle_update); 1329 1330 return policy; 1331 1332 err_min_qos_notifier: 1333 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN, 1334 &policy->nb_min); 1335 err_kobj_remove: 1336 cpufreq_policy_put_kobj(policy); 1337 err_free_real_cpus: 1338 free_cpumask_var(policy->real_cpus); 1339 err_free_rcpumask: 1340 free_cpumask_var(policy->related_cpus); 1341 err_free_cpumask: 1342 free_cpumask_var(policy->cpus); 1343 err_free_policy: 1344 kfree(policy); 1345 1346 return NULL; 1347 } 1348 1349 static void cpufreq_policy_free(struct cpufreq_policy *policy) 1350 { 1351 unsigned long flags; 1352 int cpu; 1353 1354 /* 1355 * The callers must ensure the policy is inactive by now, to avoid any 1356 * races with show()/store() callbacks. 1357 */ 1358 if (unlikely(!policy_is_inactive(policy))) 1359 pr_warn("%s: Freeing active policy\n", __func__); 1360 1361 /* Remove policy from list */ 1362 write_lock_irqsave(&cpufreq_driver_lock, flags); 1363 list_del(&policy->policy_list); 1364 1365 for_each_cpu(cpu, policy->related_cpus) 1366 per_cpu(cpufreq_cpu_data, cpu) = NULL; 1367 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1368 1369 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX, 1370 &policy->nb_max); 1371 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN, 1372 &policy->nb_min); 1373 1374 /* Cancel any pending policy->update work before freeing the policy. */ 1375 cancel_work_sync(&policy->update); 1376 1377 if (freq_qos_request_active(&policy->max_freq_req)) { 1378 /* 1379 * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY 1380 * notification, since CPUFREQ_CREATE_POLICY notification was 1381 * sent after adding max_freq_req earlier. 1382 */ 1383 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1384 CPUFREQ_REMOVE_POLICY, policy); 1385 freq_qos_remove_request(&policy->max_freq_req); 1386 } 1387 1388 if (freq_qos_request_active(&policy->min_freq_req)) 1389 freq_qos_remove_request(&policy->min_freq_req); 1390 if (freq_qos_request_active(&policy->boost_freq_req)) 1391 freq_qos_remove_request(&policy->boost_freq_req); 1392 1393 cpufreq_policy_put_kobj(policy); 1394 free_cpumask_var(policy->real_cpus); 1395 free_cpumask_var(policy->related_cpus); 1396 free_cpumask_var(policy->cpus); 1397 kfree(policy); 1398 } 1399 1400 static int cpufreq_policy_online(struct cpufreq_policy *policy, 1401 unsigned int cpu, bool new_policy) 1402 { 1403 unsigned long flags; 1404 unsigned int j; 1405 int ret; 1406 1407 guard(cpufreq_policy_write)(policy); 1408 1409 policy->cpu = cpu; 1410 policy->governor = NULL; 1411 1412 if (!new_policy && cpufreq_driver->online) { 1413 /* Recover policy->cpus using related_cpus */ 1414 cpumask_copy(policy->cpus, policy->related_cpus); 1415 1416 ret = cpufreq_driver->online(policy); 1417 if (ret) { 1418 pr_debug("%s: %d: initialization failed\n", __func__, 1419 __LINE__); 1420 goto out_exit_policy; 1421 } 1422 } else { 1423 cpumask_copy(policy->cpus, cpumask_of(cpu)); 1424 1425 /* 1426 * Call driver. From then on the cpufreq must be able 1427 * to accept all calls to ->verify and ->setpolicy for this CPU. 1428 */ 1429 ret = cpufreq_driver->init(policy); 1430 if (ret) { 1431 pr_debug("%s: %d: initialization failed\n", __func__, 1432 __LINE__); 1433 goto out_clear_policy; 1434 } 1435 1436 /* 1437 * The initialization has succeeded and the policy is online. 1438 * If there is a problem with its frequency table, take it 1439 * offline and drop it. 1440 */ 1441 ret = cpufreq_table_validate_and_sort(policy); 1442 if (ret) 1443 goto out_offline_policy; 1444 1445 /* related_cpus should at least include policy->cpus. */ 1446 cpumask_copy(policy->related_cpus, policy->cpus); 1447 } 1448 1449 /* 1450 * affected cpus must always be the one, which are online. We aren't 1451 * managing offline cpus here. 1452 */ 1453 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); 1454 1455 if (new_policy) { 1456 for_each_cpu(j, policy->related_cpus) { 1457 per_cpu(cpufreq_cpu_data, j) = policy; 1458 add_cpu_dev_symlink(policy, j, get_cpu_device(j)); 1459 } 1460 1461 if (policy->boost_supported) { 1462 ret = freq_qos_add_request(&policy->constraints, 1463 &policy->boost_freq_req, 1464 FREQ_QOS_MAX, 1465 policy->cpuinfo.max_freq); 1466 if (ret < 0) 1467 goto out_destroy_policy; 1468 } 1469 1470 ret = freq_qos_add_request(&policy->constraints, 1471 &policy->min_freq_req, FREQ_QOS_MIN, 1472 FREQ_QOS_MIN_DEFAULT_VALUE); 1473 if (ret < 0) 1474 goto out_destroy_policy; 1475 1476 ret = freq_qos_add_request(&policy->constraints, 1477 &policy->max_freq_req, FREQ_QOS_MAX, 1478 FREQ_QOS_MAX_DEFAULT_VALUE); 1479 if (ret < 0) 1480 goto out_destroy_policy; 1481 1482 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1483 CPUFREQ_CREATE_POLICY, policy); 1484 } 1485 1486 if (cpufreq_driver->get && has_target()) { 1487 policy->cur = cpufreq_driver->get(policy->cpu); 1488 if (!policy->cur) { 1489 ret = -EIO; 1490 pr_err("%s: ->get() failed\n", __func__); 1491 goto out_destroy_policy; 1492 } 1493 } 1494 1495 /* 1496 * Sometimes boot loaders set CPU frequency to a value outside of 1497 * frequency table present with cpufreq core. In such cases CPU might be 1498 * unstable if it has to run on that frequency for long duration of time 1499 * and so its better to set it to a frequency which is specified in 1500 * freq-table. This also makes cpufreq stats inconsistent as 1501 * cpufreq-stats would fail to register because current frequency of CPU 1502 * isn't found in freq-table. 1503 * 1504 * Because we don't want this change to effect boot process badly, we go 1505 * for the next freq which is >= policy->cur ('cur' must be set by now, 1506 * otherwise we will end up setting freq to lowest of the table as 'cur' 1507 * is initialized to zero). 1508 * 1509 * We are passing target-freq as "policy->cur - 1" otherwise 1510 * __cpufreq_driver_target() would simply fail, as policy->cur will be 1511 * equal to target-freq. 1512 */ 1513 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK) 1514 && has_target()) { 1515 unsigned int old_freq = policy->cur; 1516 1517 /* Are we running at unknown frequency ? */ 1518 ret = cpufreq_frequency_table_get_index(policy, old_freq); 1519 if (ret == -EINVAL) { 1520 ret = __cpufreq_driver_target(policy, old_freq - 1, 1521 CPUFREQ_RELATION_L); 1522 1523 /* 1524 * Reaching here after boot in a few seconds may not 1525 * mean that system will remain stable at "unknown" 1526 * frequency for longer duration. Hence, a BUG_ON(). 1527 */ 1528 BUG_ON(ret); 1529 pr_info("%s: CPU%d: Running at unlisted initial frequency: %u kHz, changing to: %u kHz\n", 1530 __func__, policy->cpu, old_freq, policy->cur); 1531 } 1532 } 1533 1534 if (new_policy) { 1535 ret = cpufreq_add_dev_interface(policy); 1536 if (ret) 1537 goto out_destroy_policy; 1538 1539 cpufreq_stats_create_table(policy); 1540 1541 write_lock_irqsave(&cpufreq_driver_lock, flags); 1542 list_add(&policy->policy_list, &cpufreq_policy_list); 1543 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1544 1545 /* 1546 * Register with the energy model before 1547 * em_rebuild_sched_domains() is called, which will result 1548 * in rebuilding of the sched domains, which should only be done 1549 * once the energy model is properly initialized for the policy 1550 * first. 1551 * 1552 * Also, this should be called before the policy is registered 1553 * with cooling framework. 1554 */ 1555 if (cpufreq_driver->register_em) 1556 cpufreq_driver->register_em(policy); 1557 } 1558 1559 ret = cpufreq_init_policy(policy); 1560 if (ret) { 1561 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", 1562 __func__, cpu, ret); 1563 goto out_destroy_policy; 1564 } 1565 1566 return 0; 1567 1568 out_destroy_policy: 1569 for_each_cpu(j, policy->real_cpus) 1570 remove_cpu_dev_symlink(policy, j, get_cpu_device(j)); 1571 1572 out_offline_policy: 1573 if (cpufreq_driver->offline) 1574 cpufreq_driver->offline(policy); 1575 1576 out_exit_policy: 1577 if (cpufreq_driver->exit) 1578 cpufreq_driver->exit(policy); 1579 1580 out_clear_policy: 1581 cpumask_clear(policy->cpus); 1582 1583 return ret; 1584 } 1585 1586 static int cpufreq_online(unsigned int cpu) 1587 { 1588 struct cpufreq_policy *policy; 1589 bool new_policy; 1590 int ret; 1591 1592 pr_debug("%s: bringing CPU%u online\n", __func__, cpu); 1593 1594 /* Check if this CPU already has a policy to manage it */ 1595 policy = per_cpu(cpufreq_cpu_data, cpu); 1596 if (policy) { 1597 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); 1598 if (!policy_is_inactive(policy)) 1599 return cpufreq_add_policy_cpu(policy, cpu); 1600 1601 /* This is the only online CPU for the policy. Start over. */ 1602 new_policy = false; 1603 } else { 1604 new_policy = true; 1605 policy = cpufreq_policy_alloc(cpu); 1606 if (!policy) 1607 return -ENOMEM; 1608 } 1609 1610 ret = cpufreq_policy_online(policy, cpu, new_policy); 1611 if (ret) { 1612 cpufreq_policy_free(policy); 1613 return ret; 1614 } 1615 1616 kobject_uevent(&policy->kobj, KOBJ_ADD); 1617 1618 /* Callback for handling stuff after policy is ready */ 1619 if (cpufreq_driver->ready) 1620 cpufreq_driver->ready(policy); 1621 1622 /* Register cpufreq cooling only for a new policy */ 1623 if (new_policy && cpufreq_thermal_control_enabled(cpufreq_driver)) 1624 policy->cdev = of_cpufreq_cooling_register(policy); 1625 1626 /* 1627 * Let the per-policy boost flag mirror the cpufreq_driver boost during 1628 * initialization for a new policy. For an existing policy, maintain the 1629 * previous boost value unless global boost is disabled. 1630 */ 1631 if (cpufreq_driver->set_boost && policy->boost_supported && 1632 (new_policy || !cpufreq_boost_enabled())) { 1633 ret = policy_set_boost(policy, cpufreq_boost_enabled()); 1634 if (ret) { 1635 /* If the set_boost fails, the online operation is not affected */ 1636 pr_info("%s: CPU%d: Cannot %s BOOST\n", __func__, policy->cpu, 1637 str_enable_disable(cpufreq_boost_enabled())); 1638 } 1639 } 1640 1641 pr_debug("initialization complete\n"); 1642 1643 return 0; 1644 } 1645 1646 /** 1647 * cpufreq_add_dev - the cpufreq interface for a CPU device. 1648 * @dev: CPU device. 1649 * @sif: Subsystem interface structure pointer (not used) 1650 */ 1651 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) 1652 { 1653 struct cpufreq_policy *policy; 1654 unsigned cpu = dev->id; 1655 int ret; 1656 1657 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu); 1658 1659 if (cpu_online(cpu)) { 1660 ret = cpufreq_online(cpu); 1661 if (ret) 1662 return ret; 1663 } 1664 1665 /* Create sysfs link on CPU registration */ 1666 policy = per_cpu(cpufreq_cpu_data, cpu); 1667 if (policy) 1668 add_cpu_dev_symlink(policy, cpu, dev); 1669 1670 return 0; 1671 } 1672 1673 static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy) 1674 { 1675 int ret; 1676 1677 if (has_target()) 1678 cpufreq_stop_governor(policy); 1679 1680 cpumask_clear_cpu(cpu, policy->cpus); 1681 1682 if (!policy_is_inactive(policy)) { 1683 /* Nominate a new CPU if necessary. */ 1684 if (cpu == policy->cpu) 1685 policy->cpu = cpumask_any(policy->cpus); 1686 1687 /* Start the governor again for the active policy. */ 1688 if (has_target()) { 1689 ret = cpufreq_start_governor(policy); 1690 if (ret) 1691 pr_err("%s: Failed to start governor\n", __func__); 1692 } 1693 1694 return; 1695 } 1696 1697 if (has_target()) { 1698 strscpy(policy->last_governor, policy->governor->name, 1699 CPUFREQ_NAME_LEN); 1700 cpufreq_exit_governor(policy); 1701 } else { 1702 policy->last_policy = policy->policy; 1703 } 1704 1705 /* 1706 * Perform the ->offline() during light-weight tear-down, as 1707 * that allows fast recovery when the CPU comes back. 1708 */ 1709 if (cpufreq_driver->offline) { 1710 cpufreq_driver->offline(policy); 1711 return; 1712 } 1713 1714 if (cpufreq_driver->exit) 1715 cpufreq_driver->exit(policy); 1716 1717 policy->freq_table = NULL; 1718 } 1719 1720 static int cpufreq_offline(unsigned int cpu) 1721 { 1722 struct cpufreq_policy *policy; 1723 1724 pr_debug("%s: unregistering CPU %u\n", __func__, cpu); 1725 1726 policy = cpufreq_cpu_get_raw(cpu); 1727 if (!policy) { 1728 pr_debug("%s: No cpu_data found\n", __func__); 1729 return 0; 1730 } 1731 1732 guard(cpufreq_policy_write)(policy); 1733 1734 __cpufreq_offline(cpu, policy); 1735 1736 return 0; 1737 } 1738 1739 /* 1740 * cpufreq_remove_dev - remove a CPU device 1741 * 1742 * Removes the cpufreq interface for a CPU device. 1743 */ 1744 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) 1745 { 1746 unsigned int cpu = dev->id; 1747 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 1748 1749 if (!policy) 1750 return; 1751 1752 scoped_guard(cpufreq_policy_write, policy) { 1753 if (cpu_online(cpu)) 1754 __cpufreq_offline(cpu, policy); 1755 1756 remove_cpu_dev_symlink(policy, cpu, dev); 1757 1758 if (!cpumask_empty(policy->real_cpus)) 1759 return; 1760 1761 /* 1762 * Unregister cpufreq cooling once all the CPUs of the policy 1763 * are removed. 1764 */ 1765 if (cpufreq_thermal_control_enabled(cpufreq_driver)) { 1766 cpufreq_cooling_unregister(policy->cdev); 1767 policy->cdev = NULL; 1768 } 1769 1770 /* We did light-weight exit earlier, do full tear down now */ 1771 if (cpufreq_driver->offline && cpufreq_driver->exit) 1772 cpufreq_driver->exit(policy); 1773 } 1774 1775 cpufreq_policy_free(policy); 1776 } 1777 1778 /** 1779 * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference. 1780 * @policy: Policy managing CPUs. 1781 * @new_freq: New CPU frequency. 1782 * 1783 * Adjust to the current frequency first and clean up later by either calling 1784 * cpufreq_update_policy(), or scheduling handle_update(). 1785 */ 1786 static void cpufreq_out_of_sync(struct cpufreq_policy *policy, 1787 unsigned int new_freq) 1788 { 1789 struct cpufreq_freqs freqs; 1790 1791 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n", 1792 policy->cur, new_freq); 1793 1794 freqs.old = policy->cur; 1795 freqs.new = new_freq; 1796 1797 cpufreq_freq_transition_begin(policy, &freqs); 1798 cpufreq_freq_transition_end(policy, &freqs, 0); 1799 } 1800 1801 static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update) 1802 { 1803 unsigned int new_freq; 1804 1805 if (!cpufreq_driver->get) 1806 return 0; 1807 1808 new_freq = cpufreq_driver->get(policy->cpu); 1809 if (!new_freq) 1810 return 0; 1811 1812 /* 1813 * If fast frequency switching is used with the given policy, the check 1814 * against policy->cur is pointless, so skip it in that case. 1815 */ 1816 if (policy->fast_switch_enabled || !has_target()) 1817 return new_freq; 1818 1819 if (policy->cur != new_freq) { 1820 /* 1821 * For some platforms, the frequency returned by hardware may be 1822 * slightly different from what is provided in the frequency 1823 * table, for example hardware may return 499 MHz instead of 500 1824 * MHz. In such cases it is better to avoid getting into 1825 * unnecessary frequency updates. 1826 */ 1827 if (abs(policy->cur - new_freq) < KHZ_PER_MHZ) 1828 return policy->cur; 1829 1830 cpufreq_out_of_sync(policy, new_freq); 1831 if (update) 1832 schedule_work(&policy->update); 1833 } 1834 1835 return new_freq; 1836 } 1837 1838 /** 1839 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur 1840 * @cpu: CPU number 1841 * 1842 * This is the last known freq, without actually getting it from the driver. 1843 * Return value will be same as what is shown in scaling_cur_freq in sysfs. 1844 */ 1845 unsigned int cpufreq_quick_get(unsigned int cpu) 1846 { 1847 unsigned long flags; 1848 1849 read_lock_irqsave(&cpufreq_driver_lock, flags); 1850 1851 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) { 1852 unsigned int ret_freq = cpufreq_driver->get(cpu); 1853 1854 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1855 1856 return ret_freq; 1857 } 1858 1859 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1860 1861 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 1862 if (policy) 1863 return policy->cur; 1864 1865 return 0; 1866 } 1867 EXPORT_SYMBOL(cpufreq_quick_get); 1868 1869 /** 1870 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU 1871 * @cpu: CPU number 1872 * 1873 * Just return the max possible frequency for a given CPU. 1874 */ 1875 unsigned int cpufreq_quick_get_max(unsigned int cpu) 1876 { 1877 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 1878 if (policy) 1879 return policy->max; 1880 1881 return 0; 1882 } 1883 EXPORT_SYMBOL(cpufreq_quick_get_max); 1884 1885 /** 1886 * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU 1887 * @cpu: CPU number 1888 * 1889 * The default return value is the max_freq field of cpuinfo. 1890 */ 1891 __weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu) 1892 { 1893 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 1894 if (policy) 1895 return policy->cpuinfo.max_freq; 1896 1897 return 0; 1898 } 1899 EXPORT_SYMBOL(cpufreq_get_hw_max_freq); 1900 1901 static unsigned int __cpufreq_get(struct cpufreq_policy *policy) 1902 { 1903 if (unlikely(policy_is_inactive(policy))) 1904 return 0; 1905 1906 return cpufreq_verify_current_freq(policy, true); 1907 } 1908 1909 /** 1910 * cpufreq_get - get the current CPU frequency (in kHz) 1911 * @cpu: CPU number 1912 * 1913 * Get the CPU current (static) CPU frequency 1914 */ 1915 unsigned int cpufreq_get(unsigned int cpu) 1916 { 1917 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 1918 if (!policy) 1919 return 0; 1920 1921 guard(cpufreq_policy_read)(policy); 1922 1923 return __cpufreq_get(policy); 1924 } 1925 EXPORT_SYMBOL(cpufreq_get); 1926 1927 static struct subsys_interface cpufreq_interface = { 1928 .name = "cpufreq", 1929 .subsys = &cpu_subsys, 1930 .add_dev = cpufreq_add_dev, 1931 .remove_dev = cpufreq_remove_dev, 1932 }; 1933 1934 /* 1935 * In case platform wants some specific frequency to be configured 1936 * during suspend.. 1937 */ 1938 int cpufreq_generic_suspend(struct cpufreq_policy *policy) 1939 { 1940 int ret; 1941 1942 if (!policy->suspend_freq) { 1943 pr_debug("%s: suspend_freq not defined\n", __func__); 1944 return 0; 1945 } 1946 1947 pr_debug("%s: Setting suspend-freq: %u\n", __func__, 1948 policy->suspend_freq); 1949 1950 ret = __cpufreq_driver_target(policy, policy->suspend_freq, 1951 CPUFREQ_RELATION_H); 1952 if (ret) 1953 pr_err("%s: unable to set suspend-freq: %u. err: %d\n", 1954 __func__, policy->suspend_freq, ret); 1955 1956 return ret; 1957 } 1958 EXPORT_SYMBOL(cpufreq_generic_suspend); 1959 1960 /** 1961 * cpufreq_suspend() - Suspend CPUFreq governors. 1962 * 1963 * Called during system wide Suspend/Hibernate cycles for suspending governors 1964 * as some platforms can't change frequency after this point in suspend cycle. 1965 * Because some of the devices (like: i2c, regulators, etc) they use for 1966 * changing frequency are suspended quickly after this point. 1967 */ 1968 void cpufreq_suspend(void) 1969 { 1970 struct cpufreq_policy *policy; 1971 1972 if (!cpufreq_driver) 1973 return; 1974 1975 if (!has_target() && !cpufreq_driver->suspend) 1976 goto suspend; 1977 1978 pr_debug("%s: Suspending Governors\n", __func__); 1979 1980 for_each_active_policy(policy) { 1981 if (has_target()) { 1982 scoped_guard(cpufreq_policy_write, policy) { 1983 cpufreq_stop_governor(policy); 1984 } 1985 } 1986 1987 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy)) 1988 pr_err("%s: Failed to suspend driver: %s\n", __func__, 1989 cpufreq_driver->name); 1990 } 1991 1992 suspend: 1993 cpufreq_suspended = true; 1994 } 1995 1996 /** 1997 * cpufreq_resume() - Resume CPUFreq governors. 1998 * 1999 * Called during system wide Suspend/Hibernate cycle for resuming governors that 2000 * are suspended with cpufreq_suspend(). 2001 */ 2002 void cpufreq_resume(void) 2003 { 2004 struct cpufreq_policy *policy; 2005 int ret; 2006 2007 if (!cpufreq_driver) 2008 return; 2009 2010 if (unlikely(!cpufreq_suspended)) 2011 return; 2012 2013 cpufreq_suspended = false; 2014 2015 if (!has_target() && !cpufreq_driver->resume) 2016 return; 2017 2018 pr_debug("%s: Resuming Governors\n", __func__); 2019 2020 for_each_active_policy(policy) { 2021 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) { 2022 pr_err("%s: Failed to resume driver: %s\n", __func__, 2023 cpufreq_driver->name); 2024 } else if (has_target()) { 2025 scoped_guard(cpufreq_policy_write, policy) { 2026 ret = cpufreq_start_governor(policy); 2027 } 2028 2029 if (ret) 2030 pr_err("%s: Failed to start governor for CPU%u's policy\n", 2031 __func__, policy->cpu); 2032 } 2033 } 2034 } 2035 2036 /** 2037 * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones. 2038 * @flags: Flags to test against the current cpufreq driver's flags. 2039 * 2040 * Assumes that the driver is there, so callers must ensure that this is the 2041 * case. 2042 */ 2043 bool cpufreq_driver_test_flags(u16 flags) 2044 { 2045 return !!(cpufreq_driver->flags & flags); 2046 } 2047 2048 /** 2049 * cpufreq_get_current_driver - Return the current driver's name. 2050 * 2051 * Return the name string of the currently registered cpufreq driver or NULL if 2052 * none. 2053 */ 2054 const char *cpufreq_get_current_driver(void) 2055 { 2056 if (cpufreq_driver) 2057 return cpufreq_driver->name; 2058 2059 return NULL; 2060 } 2061 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver); 2062 2063 /** 2064 * cpufreq_get_driver_data - Return current driver data. 2065 * 2066 * Return the private data of the currently registered cpufreq driver, or NULL 2067 * if no cpufreq driver has been registered. 2068 */ 2069 void *cpufreq_get_driver_data(void) 2070 { 2071 if (cpufreq_driver) 2072 return cpufreq_driver->driver_data; 2073 2074 return NULL; 2075 } 2076 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data); 2077 2078 /********************************************************************* 2079 * NOTIFIER LISTS INTERFACE * 2080 *********************************************************************/ 2081 2082 /** 2083 * cpufreq_register_notifier - Register a notifier with cpufreq. 2084 * @nb: notifier function to register. 2085 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER. 2086 * 2087 * Add a notifier to one of two lists: either a list of notifiers that run on 2088 * clock rate changes (once before and once after every transition), or a list 2089 * of notifiers that ron on cpufreq policy changes. 2090 * 2091 * This function may sleep and it has the same return values as 2092 * blocking_notifier_chain_register(). 2093 */ 2094 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) 2095 { 2096 int ret; 2097 2098 if (cpufreq_disabled()) 2099 return -EINVAL; 2100 2101 switch (list) { 2102 case CPUFREQ_TRANSITION_NOTIFIER: 2103 mutex_lock(&cpufreq_fast_switch_lock); 2104 2105 if (cpufreq_fast_switch_count > 0) { 2106 mutex_unlock(&cpufreq_fast_switch_lock); 2107 return -EBUSY; 2108 } 2109 ret = srcu_notifier_chain_register( 2110 &cpufreq_transition_notifier_list, nb); 2111 if (!ret) 2112 cpufreq_fast_switch_count--; 2113 2114 mutex_unlock(&cpufreq_fast_switch_lock); 2115 break; 2116 case CPUFREQ_POLICY_NOTIFIER: 2117 ret = blocking_notifier_chain_register( 2118 &cpufreq_policy_notifier_list, nb); 2119 break; 2120 default: 2121 ret = -EINVAL; 2122 } 2123 2124 return ret; 2125 } 2126 EXPORT_SYMBOL(cpufreq_register_notifier); 2127 2128 /** 2129 * cpufreq_unregister_notifier - Unregister a notifier from cpufreq. 2130 * @nb: notifier block to be unregistered. 2131 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER. 2132 * 2133 * Remove a notifier from one of the cpufreq notifier lists. 2134 * 2135 * This function may sleep and it has the same return values as 2136 * blocking_notifier_chain_unregister(). 2137 */ 2138 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list) 2139 { 2140 int ret; 2141 2142 if (cpufreq_disabled()) 2143 return -EINVAL; 2144 2145 switch (list) { 2146 case CPUFREQ_TRANSITION_NOTIFIER: 2147 mutex_lock(&cpufreq_fast_switch_lock); 2148 2149 ret = srcu_notifier_chain_unregister( 2150 &cpufreq_transition_notifier_list, nb); 2151 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0)) 2152 cpufreq_fast_switch_count++; 2153 2154 mutex_unlock(&cpufreq_fast_switch_lock); 2155 break; 2156 case CPUFREQ_POLICY_NOTIFIER: 2157 ret = blocking_notifier_chain_unregister( 2158 &cpufreq_policy_notifier_list, nb); 2159 break; 2160 default: 2161 ret = -EINVAL; 2162 } 2163 2164 return ret; 2165 } 2166 EXPORT_SYMBOL(cpufreq_unregister_notifier); 2167 2168 2169 /********************************************************************* 2170 * GOVERNORS * 2171 *********************************************************************/ 2172 2173 /** 2174 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch. 2175 * @policy: cpufreq policy to switch the frequency for. 2176 * @target_freq: New frequency to set (may be approximate). 2177 * 2178 * Carry out a fast frequency switch without sleeping. 2179 * 2180 * The driver's ->fast_switch() callback invoked by this function must be 2181 * suitable for being called from within RCU-sched read-side critical sections 2182 * and it is expected to select the minimum available frequency greater than or 2183 * equal to @target_freq (CPUFREQ_RELATION_L). 2184 * 2185 * This function must not be called if policy->fast_switch_enabled is unset. 2186 * 2187 * Governors calling this function must guarantee that it will never be invoked 2188 * twice in parallel for the same policy and that it will never be called in 2189 * parallel with either ->target() or ->target_index() for the same policy. 2190 * 2191 * Returns the actual frequency set for the CPU. 2192 * 2193 * If 0 is returned by the driver's ->fast_switch() callback to indicate an 2194 * error condition, the hardware configuration must be preserved. 2195 */ 2196 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, 2197 unsigned int target_freq) 2198 { 2199 unsigned int freq; 2200 int cpu; 2201 2202 target_freq = clamp_val(target_freq, policy->min, policy->max); 2203 freq = cpufreq_driver->fast_switch(policy, target_freq); 2204 2205 if (!freq) 2206 return 0; 2207 2208 policy->cur = freq; 2209 arch_set_freq_scale(policy->related_cpus, freq, 2210 arch_scale_freq_ref(policy->cpu)); 2211 cpufreq_stats_record_transition(policy, freq); 2212 2213 if (trace_cpu_frequency_enabled()) { 2214 for_each_cpu(cpu, policy->cpus) 2215 trace_cpu_frequency(freq, cpu); 2216 } 2217 2218 return freq; 2219 } 2220 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); 2221 2222 /** 2223 * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go. 2224 * @policy: cpufreq policy object of the target CPU. 2225 * @min_perf: Minimum (required) performance level (units of @capacity). 2226 * @target_perf: Target (desired) performance level (units of @capacity). 2227 * @capacity: Capacity of the target CPU. 2228 * 2229 * Carry out a fast performance level switch of @cpu without sleeping. 2230 * 2231 * The driver's ->adjust_perf() callback invoked by this function must be 2232 * suitable for being called from within RCU-sched read-side critical sections 2233 * and it is expected to select a suitable performance level equal to or above 2234 * @min_perf and preferably equal to or below @target_perf. 2235 * 2236 * This function must not be called if policy->fast_switch_enabled is unset. 2237 * 2238 * Governors calling this function must guarantee that it will never be invoked 2239 * twice in parallel for the same CPU and that it will never be called in 2240 * parallel with either ->target() or ->target_index() or ->fast_switch() for 2241 * the same CPU. 2242 */ 2243 void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy, 2244 unsigned long min_perf, 2245 unsigned long target_perf, 2246 unsigned long capacity) 2247 { 2248 cpufreq_driver->adjust_perf(policy, min_perf, target_perf, capacity); 2249 } 2250 2251 /** 2252 * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback. 2253 * 2254 * Return 'true' if the ->adjust_perf callback is present for the 2255 * current driver or 'false' otherwise. 2256 */ 2257 bool cpufreq_driver_has_adjust_perf(void) 2258 { 2259 return !!cpufreq_driver->adjust_perf; 2260 } 2261 2262 /* Must set freqs->new to intermediate frequency */ 2263 static int __target_intermediate(struct cpufreq_policy *policy, 2264 struct cpufreq_freqs *freqs, int index) 2265 { 2266 int ret; 2267 2268 freqs->new = cpufreq_driver->get_intermediate(policy, index); 2269 2270 /* We don't need to switch to intermediate freq */ 2271 if (!freqs->new) 2272 return 0; 2273 2274 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n", 2275 __func__, policy->cpu, freqs->old, freqs->new); 2276 2277 cpufreq_freq_transition_begin(policy, freqs); 2278 ret = cpufreq_driver->target_intermediate(policy, index); 2279 cpufreq_freq_transition_end(policy, freqs, ret); 2280 2281 if (ret) 2282 pr_err("%s: Failed to change to intermediate frequency: %d\n", 2283 __func__, ret); 2284 2285 return ret; 2286 } 2287 2288 static int __target_index(struct cpufreq_policy *policy, int index) 2289 { 2290 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; 2291 unsigned int restore_freq, intermediate_freq = 0; 2292 unsigned int newfreq = policy->freq_table[index].frequency; 2293 int retval = -EINVAL; 2294 bool notify; 2295 2296 if (newfreq == policy->cur) 2297 return 0; 2298 2299 /* Save last value to restore later on errors */ 2300 restore_freq = policy->cur; 2301 2302 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); 2303 if (notify) { 2304 /* Handle switching to intermediate frequency */ 2305 if (cpufreq_driver->get_intermediate) { 2306 retval = __target_intermediate(policy, &freqs, index); 2307 if (retval) 2308 return retval; 2309 2310 intermediate_freq = freqs.new; 2311 /* Set old freq to intermediate */ 2312 if (intermediate_freq) 2313 freqs.old = freqs.new; 2314 } 2315 2316 freqs.new = newfreq; 2317 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", 2318 __func__, policy->cpu, freqs.old, freqs.new); 2319 2320 cpufreq_freq_transition_begin(policy, &freqs); 2321 } 2322 2323 retval = cpufreq_driver->target_index(policy, index); 2324 if (retval) 2325 pr_err("%s: Failed to change cpu frequency: %d\n", __func__, 2326 retval); 2327 2328 if (notify) { 2329 cpufreq_freq_transition_end(policy, &freqs, retval); 2330 2331 /* 2332 * Failed after setting to intermediate freq? Driver should have 2333 * reverted back to initial frequency and so should we. Check 2334 * here for intermediate_freq instead of get_intermediate, in 2335 * case we haven't switched to intermediate freq at all. 2336 */ 2337 if (unlikely(retval && intermediate_freq)) { 2338 freqs.old = intermediate_freq; 2339 freqs.new = restore_freq; 2340 cpufreq_freq_transition_begin(policy, &freqs); 2341 cpufreq_freq_transition_end(policy, &freqs, 0); 2342 } 2343 } 2344 2345 return retval; 2346 } 2347 2348 int __cpufreq_driver_target(struct cpufreq_policy *policy, 2349 unsigned int target_freq, 2350 unsigned int relation) 2351 { 2352 unsigned int old_target_freq = target_freq; 2353 2354 if (cpufreq_disabled()) 2355 return -ENODEV; 2356 2357 target_freq = __resolve_freq(policy, target_freq, policy->min, 2358 policy->max, relation); 2359 2360 pr_debug("CPU %u: cur %u kHz -> target %u kHz (req %u kHz, rel %u)\n", 2361 policy->cpu, policy->cur, target_freq, old_target_freq, relation); 2362 2363 /* 2364 * This might look like a redundant call as we are checking it again 2365 * after finding index. But it is left intentionally for cases where 2366 * exactly same freq is called again and so we can save on few function 2367 * calls. 2368 */ 2369 if (target_freq == policy->cur && 2370 !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS)) 2371 return 0; 2372 2373 if (cpufreq_driver->target) { 2374 /* 2375 * If the driver hasn't setup a single inefficient frequency, 2376 * it's unlikely it knows how to decode CPUFREQ_RELATION_E. 2377 */ 2378 if (!policy->efficiencies_available) 2379 relation &= ~CPUFREQ_RELATION_E; 2380 2381 return cpufreq_driver->target(policy, target_freq, relation); 2382 } 2383 2384 if (!cpufreq_driver->target_index) 2385 return -EINVAL; 2386 2387 return __target_index(policy, policy->cached_resolved_idx); 2388 } 2389 EXPORT_SYMBOL_GPL(__cpufreq_driver_target); 2390 2391 int cpufreq_driver_target(struct cpufreq_policy *policy, 2392 unsigned int target_freq, 2393 unsigned int relation) 2394 { 2395 guard(cpufreq_policy_write)(policy); 2396 2397 return __cpufreq_driver_target(policy, target_freq, relation); 2398 } 2399 EXPORT_SYMBOL_GPL(cpufreq_driver_target); 2400 2401 __weak struct cpufreq_governor *cpufreq_fallback_governor(void) 2402 { 2403 return NULL; 2404 } 2405 2406 static int cpufreq_init_governor(struct cpufreq_policy *policy) 2407 { 2408 int ret; 2409 2410 /* Don't start any governor operations if we are entering suspend */ 2411 if (cpufreq_suspended) 2412 return 0; 2413 /* 2414 * Governor might not be initiated here if ACPI _PPC changed 2415 * notification happened, so check it. 2416 */ 2417 if (!policy->governor) 2418 return -EINVAL; 2419 2420 /* Platform doesn't want dynamic frequency switching ? */ 2421 if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING && 2422 cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) { 2423 struct cpufreq_governor *gov = cpufreq_fallback_governor(); 2424 2425 if (gov) { 2426 pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n", 2427 policy->governor->name, gov->name); 2428 policy->governor = gov; 2429 } else { 2430 return -EINVAL; 2431 } 2432 } 2433 2434 if (!try_module_get(policy->governor->owner)) 2435 return -EINVAL; 2436 2437 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2438 2439 if (policy->governor->init) { 2440 ret = policy->governor->init(policy); 2441 if (ret) { 2442 module_put(policy->governor->owner); 2443 return ret; 2444 } 2445 } 2446 2447 policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET); 2448 2449 return 0; 2450 } 2451 2452 static void cpufreq_exit_governor(struct cpufreq_policy *policy) 2453 { 2454 if (cpufreq_suspended || !policy->governor) 2455 return; 2456 2457 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2458 2459 if (policy->governor->exit) 2460 policy->governor->exit(policy); 2461 2462 module_put(policy->governor->owner); 2463 } 2464 2465 int cpufreq_start_governor(struct cpufreq_policy *policy) 2466 { 2467 int ret; 2468 2469 if (cpufreq_suspended) 2470 return 0; 2471 2472 if (!policy->governor) 2473 return -EINVAL; 2474 2475 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2476 2477 cpufreq_verify_current_freq(policy, false); 2478 2479 if (policy->governor->start) { 2480 ret = policy->governor->start(policy); 2481 if (ret) 2482 return ret; 2483 } 2484 2485 if (policy->governor->limits) 2486 policy->governor->limits(policy); 2487 2488 return 0; 2489 } 2490 2491 void cpufreq_stop_governor(struct cpufreq_policy *policy) 2492 { 2493 if (cpufreq_suspended || !policy->governor) 2494 return; 2495 2496 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2497 2498 if (policy->governor->stop) 2499 policy->governor->stop(policy); 2500 } 2501 2502 static void cpufreq_governor_limits(struct cpufreq_policy *policy) 2503 { 2504 if (cpufreq_suspended || !policy->governor) 2505 return; 2506 2507 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2508 2509 if (policy->governor->limits) 2510 policy->governor->limits(policy); 2511 } 2512 2513 int cpufreq_register_governor(struct cpufreq_governor *governor) 2514 { 2515 int err; 2516 2517 if (!governor) 2518 return -EINVAL; 2519 2520 if (cpufreq_disabled()) 2521 return -ENODEV; 2522 2523 mutex_lock(&cpufreq_governor_mutex); 2524 2525 err = -EBUSY; 2526 if (!find_governor(governor->name)) { 2527 err = 0; 2528 list_add(&governor->governor_list, &cpufreq_governor_list); 2529 } 2530 2531 mutex_unlock(&cpufreq_governor_mutex); 2532 return err; 2533 } 2534 EXPORT_SYMBOL_GPL(cpufreq_register_governor); 2535 2536 void cpufreq_unregister_governor(struct cpufreq_governor *governor) 2537 { 2538 struct cpufreq_policy *policy; 2539 unsigned long flags; 2540 2541 if (!governor) 2542 return; 2543 2544 if (cpufreq_disabled()) 2545 return; 2546 2547 /* clear last_governor for all inactive policies */ 2548 read_lock_irqsave(&cpufreq_driver_lock, flags); 2549 for_each_inactive_policy(policy) { 2550 if (!strcmp(policy->last_governor, governor->name)) { 2551 policy->governor = NULL; 2552 policy->last_governor[0] = '\0'; 2553 } 2554 } 2555 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 2556 2557 mutex_lock(&cpufreq_governor_mutex); 2558 list_del(&governor->governor_list); 2559 mutex_unlock(&cpufreq_governor_mutex); 2560 } 2561 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor); 2562 2563 2564 /********************************************************************* 2565 * POLICY INTERFACE * 2566 *********************************************************************/ 2567 2568 DEFINE_PER_CPU(unsigned long, cpufreq_pressure); 2569 2570 /** 2571 * cpufreq_update_pressure() - Update cpufreq pressure for CPUs 2572 * @policy: cpufreq policy of the CPUs. 2573 * 2574 * Update the value of cpufreq pressure for all @cpus in the policy. 2575 */ 2576 static void cpufreq_update_pressure(struct cpufreq_policy *policy) 2577 { 2578 unsigned long max_capacity, capped_freq, pressure; 2579 u32 max_freq; 2580 int cpu; 2581 2582 cpu = cpumask_first(policy->related_cpus); 2583 max_freq = arch_scale_freq_ref(cpu); 2584 capped_freq = policy->max; 2585 2586 /* 2587 * Handle properly the boost frequencies, which should simply clean 2588 * the cpufreq pressure value. 2589 */ 2590 if (max_freq <= capped_freq) { 2591 pressure = 0; 2592 } else { 2593 max_capacity = arch_scale_cpu_capacity(cpu); 2594 pressure = max_capacity - 2595 mult_frac(max_capacity, capped_freq, max_freq); 2596 } 2597 2598 for_each_cpu(cpu, policy->related_cpus) 2599 WRITE_ONCE(per_cpu(cpufreq_pressure, cpu), pressure); 2600 } 2601 2602 /** 2603 * cpufreq_set_policy - Modify cpufreq policy parameters. 2604 * @policy: Policy object to modify. 2605 * @new_gov: Policy governor pointer. 2606 * @new_pol: Policy value (for drivers with built-in governors). 2607 * 2608 * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency 2609 * limits to be set for the policy, update @policy with the verified limits 2610 * values and either invoke the driver's ->setpolicy() callback (if present) or 2611 * carry out a governor update for @policy. That is, run the current governor's 2612 * ->limits() callback (if @new_gov points to the same object as the one in 2613 * @policy) or replace the governor for @policy with @new_gov. 2614 * 2615 * The cpuinfo part of @policy is not updated by this function. 2616 */ 2617 static int cpufreq_set_policy(struct cpufreq_policy *policy, 2618 struct cpufreq_governor *new_gov, 2619 unsigned int new_pol) 2620 { 2621 struct cpufreq_policy_data new_data; 2622 struct cpufreq_governor *old_gov; 2623 int ret; 2624 2625 memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); 2626 new_data.freq_table = policy->freq_table; 2627 new_data.cpu = policy->cpu; 2628 /* 2629 * PM QoS framework collects all the requests from users and provide us 2630 * the final aggregated value here. 2631 */ 2632 new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN); 2633 new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX); 2634 2635 pr_debug("setting new policy for CPU %u: %u - %u kHz\n", 2636 new_data.cpu, new_data.min, new_data.max); 2637 2638 /* 2639 * Verify that the CPU speed can be set within these limits and make sure 2640 * that min <= max. 2641 */ 2642 ret = cpufreq_driver->verify(&new_data); 2643 if (ret) 2644 return ret; 2645 2646 /* 2647 * Resolve policy min/max to available frequencies. It ensures 2648 * no frequency resolution will neither overshoot the requested maximum 2649 * nor undershoot the requested minimum. 2650 * 2651 * Avoid storing intermediate values in policy->max or policy->min and 2652 * compiler optimizations around them because they may be accessed 2653 * concurrently by cpufreq_driver_resolve_freq() during the update. 2654 */ 2655 WRITE_ONCE(policy->max, __resolve_freq(policy, new_data.max, 2656 new_data.min, new_data.max, 2657 CPUFREQ_RELATION_H)); 2658 new_data.min = __resolve_freq(policy, new_data.min, new_data.min, 2659 new_data.max, CPUFREQ_RELATION_L); 2660 WRITE_ONCE(policy->min, new_data.min > policy->max ? policy->max : new_data.min); 2661 2662 trace_cpu_frequency_limits(policy); 2663 2664 cpufreq_update_pressure(policy); 2665 2666 policy->cached_target_freq = UINT_MAX; 2667 2668 pr_debug("new min and max freqs are %u - %u kHz\n", 2669 policy->min, policy->max); 2670 2671 if (cpufreq_driver->setpolicy) { 2672 policy->policy = new_pol; 2673 pr_debug("setting range\n"); 2674 return cpufreq_driver->setpolicy(policy); 2675 } 2676 2677 if (new_gov == policy->governor) { 2678 pr_debug("governor limits update\n"); 2679 cpufreq_governor_limits(policy); 2680 return 0; 2681 } 2682 2683 pr_debug("governor switch\n"); 2684 2685 /* save old, working values */ 2686 old_gov = policy->governor; 2687 /* end old governor */ 2688 if (old_gov) { 2689 cpufreq_stop_governor(policy); 2690 cpufreq_exit_governor(policy); 2691 } 2692 2693 /* start new governor */ 2694 policy->governor = new_gov; 2695 ret = cpufreq_init_governor(policy); 2696 if (!ret) { 2697 ret = cpufreq_start_governor(policy); 2698 if (!ret) { 2699 pr_debug("governor change\n"); 2700 return 0; 2701 } 2702 cpufreq_exit_governor(policy); 2703 } 2704 2705 /* new governor failed, so re-start old one */ 2706 pr_debug("starting governor %s failed\n", policy->governor->name); 2707 if (old_gov) { 2708 policy->governor = old_gov; 2709 if (cpufreq_init_governor(policy)) { 2710 policy->governor = NULL; 2711 } else if (cpufreq_start_governor(policy)) { 2712 cpufreq_exit_governor(policy); 2713 policy->governor = NULL; 2714 } 2715 } 2716 2717 return ret; 2718 } 2719 2720 static void cpufreq_policy_refresh(struct cpufreq_policy *policy) 2721 { 2722 guard(cpufreq_policy_write)(policy); 2723 2724 /* 2725 * BIOS might change freq behind our back 2726 * -> ask driver for current freq and notify governors about a change 2727 */ 2728 if (cpufreq_driver->get && has_target() && 2729 (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false)))) 2730 return; 2731 2732 refresh_frequency_limits(policy); 2733 } 2734 2735 /** 2736 * cpufreq_update_policy - Re-evaluate an existing cpufreq policy. 2737 * @cpu: CPU to re-evaluate the policy for. 2738 * 2739 * Update the current frequency for the cpufreq policy of @cpu and use 2740 * cpufreq_set_policy() to re-apply the min and max limits, which triggers the 2741 * evaluation of policy notifiers and the cpufreq driver's ->verify() callback 2742 * for the policy in question, among other things. 2743 */ 2744 void cpufreq_update_policy(unsigned int cpu) 2745 { 2746 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 2747 if (!policy) 2748 return; 2749 2750 cpufreq_policy_refresh(policy); 2751 } 2752 EXPORT_SYMBOL(cpufreq_update_policy); 2753 2754 /** 2755 * cpufreq_update_limits - Update policy limits for a given CPU. 2756 * @cpu: CPU to update the policy limits for. 2757 * 2758 * Invoke the driver's ->update_limits callback if present or call 2759 * cpufreq_policy_refresh() for @cpu. 2760 */ 2761 void cpufreq_update_limits(unsigned int cpu) 2762 { 2763 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 2764 if (!policy) 2765 return; 2766 2767 if (cpufreq_driver->update_limits) 2768 cpufreq_driver->update_limits(policy); 2769 else 2770 cpufreq_policy_refresh(policy); 2771 } 2772 EXPORT_SYMBOL_GPL(cpufreq_update_limits); 2773 2774 /********************************************************************* 2775 * BOOST * 2776 *********************************************************************/ 2777 int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state) 2778 { 2779 int ret; 2780 2781 if (!policy->freq_table) 2782 return -ENXIO; 2783 2784 ret = cpufreq_frequency_table_cpuinfo(policy); 2785 if (ret) 2786 pr_err("%s: Policy frequency update failed\n", __func__); 2787 2788 return ret; 2789 } 2790 EXPORT_SYMBOL_GPL(cpufreq_boost_set_sw); 2791 2792 static int cpufreq_boost_trigger_state(int state) 2793 { 2794 struct cpufreq_policy *policy; 2795 unsigned long flags; 2796 int ret = -EOPNOTSUPP; 2797 2798 /* 2799 * Don't compare 'cpufreq_driver->boost_enabled' with 'state' here to 2800 * make sure all policies are in sync with global boost flag. 2801 */ 2802 2803 write_lock_irqsave(&cpufreq_driver_lock, flags); 2804 cpufreq_driver->boost_enabled = state; 2805 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2806 2807 cpus_read_lock(); 2808 for_each_active_policy(policy) { 2809 if (!policy->boost_supported) 2810 continue; 2811 2812 ret = policy_set_boost(policy, state); 2813 if (unlikely(ret)) 2814 break; 2815 } 2816 2817 cpus_read_unlock(); 2818 2819 if (likely(!ret)) 2820 return 0; 2821 2822 write_lock_irqsave(&cpufreq_driver_lock, flags); 2823 cpufreq_driver->boost_enabled = !state; 2824 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2825 2826 pr_err("%s: Cannot %s BOOST\n", 2827 __func__, str_enable_disable(state)); 2828 2829 return ret; 2830 } 2831 2832 static bool cpufreq_boost_supported(void) 2833 { 2834 return cpufreq_driver->set_boost; 2835 } 2836 2837 static int create_boost_sysfs_file(void) 2838 { 2839 int ret; 2840 2841 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr); 2842 if (ret) 2843 pr_err("%s: cannot register global BOOST sysfs file\n", 2844 __func__); 2845 2846 return ret; 2847 } 2848 2849 static void remove_boost_sysfs_file(void) 2850 { 2851 if (cpufreq_boost_supported()) 2852 sysfs_remove_file(cpufreq_global_kobject, &boost.attr); 2853 } 2854 2855 bool cpufreq_boost_enabled(void) 2856 { 2857 return cpufreq_driver->boost_enabled; 2858 } 2859 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled); 2860 2861 /********************************************************************* 2862 * REGISTER / UNREGISTER CPUFREQ DRIVER * 2863 *********************************************************************/ 2864 static enum cpuhp_state hp_online; 2865 2866 static int cpuhp_cpufreq_online(unsigned int cpu) 2867 { 2868 cpufreq_online(cpu); 2869 2870 return 0; 2871 } 2872 2873 static int cpuhp_cpufreq_offline(unsigned int cpu) 2874 { 2875 cpufreq_offline(cpu); 2876 2877 return 0; 2878 } 2879 2880 /** 2881 * cpufreq_register_driver - register a CPU Frequency driver 2882 * @driver_data: A struct cpufreq_driver containing the values# 2883 * submitted by the CPU Frequency driver. 2884 * 2885 * Registers a CPU Frequency driver to this core code. This code 2886 * returns zero on success, -EEXIST when another driver got here first 2887 * (and isn't unregistered in the meantime). 2888 * 2889 */ 2890 int cpufreq_register_driver(struct cpufreq_driver *driver_data) 2891 { 2892 unsigned long flags; 2893 int ret; 2894 2895 if (cpufreq_disabled()) 2896 return -ENODEV; 2897 2898 /* 2899 * The cpufreq core depends heavily on the availability of device 2900 * structure, make sure they are available before proceeding further. 2901 */ 2902 if (!get_cpu_device(0)) 2903 return -EPROBE_DEFER; 2904 2905 if (!driver_data || !driver_data->verify || !driver_data->init || 2906 (driver_data->target_index && driver_data->target) || 2907 (!!driver_data->setpolicy == (driver_data->target_index || driver_data->target)) || 2908 (!driver_data->get_intermediate != !driver_data->target_intermediate) || 2909 (!driver_data->online != !driver_data->offline) || 2910 (driver_data->adjust_perf && !driver_data->fast_switch)) 2911 return -EINVAL; 2912 2913 pr_debug("trying to register driver %s\n", driver_data->name); 2914 2915 /* Protect against concurrent CPU online/offline. */ 2916 cpus_read_lock(); 2917 2918 write_lock_irqsave(&cpufreq_driver_lock, flags); 2919 if (cpufreq_driver) { 2920 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2921 ret = -EEXIST; 2922 goto out; 2923 } 2924 cpufreq_driver = driver_data; 2925 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2926 2927 if (driver_data->setpolicy) 2928 driver_data->flags |= CPUFREQ_CONST_LOOPS; 2929 2930 if (cpufreq_boost_supported()) { 2931 ret = create_boost_sysfs_file(); 2932 if (ret) 2933 goto err_null_driver; 2934 } 2935 2936 /* 2937 * Mark support for the scheduler's frequency invariance engine for 2938 * drivers that implement target(), target_index() or fast_switch(). 2939 */ 2940 if (!cpufreq_driver->setpolicy) { 2941 static_branch_enable_cpuslocked(&cpufreq_freq_invariance); 2942 pr_debug("cpufreq: supports frequency invariance\n"); 2943 } 2944 2945 ret = subsys_interface_register(&cpufreq_interface); 2946 if (ret) 2947 goto err_boost_unreg; 2948 2949 if (unlikely(list_empty(&cpufreq_policy_list))) { 2950 /* if all ->init() calls failed, unregister */ 2951 ret = -ENODEV; 2952 pr_debug("%s: No CPU initialized for driver %s\n", __func__, 2953 driver_data->name); 2954 goto err_if_unreg; 2955 } 2956 2957 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, 2958 "cpufreq:online", 2959 cpuhp_cpufreq_online, 2960 cpuhp_cpufreq_offline); 2961 if (ret < 0) 2962 goto err_if_unreg; 2963 hp_online = ret; 2964 ret = 0; 2965 2966 pr_debug("driver %s up and running\n", driver_data->name); 2967 goto out; 2968 2969 err_if_unreg: 2970 subsys_interface_unregister(&cpufreq_interface); 2971 err_boost_unreg: 2972 if (!cpufreq_driver->setpolicy) 2973 static_branch_disable_cpuslocked(&cpufreq_freq_invariance); 2974 remove_boost_sysfs_file(); 2975 err_null_driver: 2976 write_lock_irqsave(&cpufreq_driver_lock, flags); 2977 cpufreq_driver = NULL; 2978 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2979 out: 2980 cpus_read_unlock(); 2981 return ret; 2982 } 2983 EXPORT_SYMBOL_GPL(cpufreq_register_driver); 2984 2985 /* 2986 * cpufreq_unregister_driver - unregister the current CPUFreq driver 2987 * 2988 * Unregister the current CPUFreq driver. Only call this if you have 2989 * the right to do so, i.e. if you have succeeded in initialising before! 2990 * Returns zero if successful, and -EINVAL if the cpufreq_driver is 2991 * currently not initialised. 2992 */ 2993 void cpufreq_unregister_driver(struct cpufreq_driver *driver) 2994 { 2995 unsigned long flags; 2996 2997 if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver))) 2998 return; 2999 3000 pr_debug("unregistering driver %s\n", driver->name); 3001 3002 /* Protect against concurrent cpu hotplug */ 3003 cpus_read_lock(); 3004 subsys_interface_unregister(&cpufreq_interface); 3005 remove_boost_sysfs_file(); 3006 static_branch_disable_cpuslocked(&cpufreq_freq_invariance); 3007 cpuhp_remove_state_nocalls_cpuslocked(hp_online); 3008 3009 write_lock_irqsave(&cpufreq_driver_lock, flags); 3010 3011 cpufreq_driver = NULL; 3012 3013 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 3014 cpus_read_unlock(); 3015 } 3016 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver); 3017 3018 static int __init cpufreq_core_init(void) 3019 { 3020 struct cpufreq_governor *gov = cpufreq_default_governor(); 3021 struct device *dev_root; 3022 3023 if (cpufreq_disabled()) 3024 return -ENODEV; 3025 3026 dev_root = bus_get_dev_root(&cpu_subsys); 3027 if (dev_root) { 3028 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj); 3029 put_device(dev_root); 3030 } 3031 BUG_ON(!cpufreq_global_kobject); 3032 3033 if (!strlen(default_governor)) 3034 strscpy(default_governor, gov->name, CPUFREQ_NAME_LEN); 3035 3036 return 0; 3037 } 3038 3039 static bool cpufreq_policy_is_good_for_eas(unsigned int cpu) 3040 { 3041 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 3042 if (!policy) { 3043 pr_debug("cpufreq policy not set for CPU: %d\n", cpu); 3044 return false; 3045 } 3046 3047 return sugov_is_governor(policy); 3048 } 3049 3050 bool cpufreq_ready_for_eas(const struct cpumask *cpu_mask) 3051 { 3052 unsigned int cpu; 3053 3054 /* Do not attempt EAS if schedutil is not being used. */ 3055 for_each_cpu(cpu, cpu_mask) { 3056 if (!cpufreq_policy_is_good_for_eas(cpu)) { 3057 pr_debug("rd %*pbl: schedutil is mandatory for EAS\n", 3058 cpumask_pr_args(cpu_mask)); 3059 return false; 3060 } 3061 } 3062 3063 return true; 3064 } 3065 3066 module_param(off, int, 0444); 3067 module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444); 3068 core_initcall(cpufreq_core_init); 3069