1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/drivers/cpufreq/cpufreq.c 4 * 5 * Copyright (C) 2001 Russell King 6 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> 7 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org> 8 * 9 * Oct 2005 - Ashok Raj <ashok.raj@intel.com> 10 * Added handling for CPU hotplug 11 * Feb 2006 - Jacob Shin <jacob.shin@amd.com> 12 * Fix handling for CPU hotplug -- affected CPUs 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/cpu.h> 18 #include <linux/cpufreq.h> 19 #include <linux/cpu_cooling.h> 20 #include <linux/delay.h> 21 #include <linux/device.h> 22 #include <linux/init.h> 23 #include <linux/kernel_stat.h> 24 #include <linux/module.h> 25 #include <linux/mutex.h> 26 #include <linux/pm_qos.h> 27 #include <linux/slab.h> 28 #include <linux/string_choices.h> 29 #include <linux/suspend.h> 30 #include <linux/syscore_ops.h> 31 #include <linux/tick.h> 32 #include <linux/units.h> 33 #include <trace/events/power.h> 34 35 static LIST_HEAD(cpufreq_policy_list); 36 37 /* Macros to iterate over CPU policies */ 38 #define for_each_policy(__policy) \ 39 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) 40 41 #define for_each_suitable_policy(__policy, __active) \ 42 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \ 43 if ((__active) == !policy_is_inactive(__policy)) 44 45 #define for_each_active_policy(__policy) \ 46 for_each_suitable_policy(__policy, true) 47 #define for_each_inactive_policy(__policy) \ 48 for_each_suitable_policy(__policy, false) 49 50 /* Iterate over governors */ 51 static LIST_HEAD(cpufreq_governor_list); 52 #define for_each_governor(__governor) \ 53 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list) 54 55 static char default_governor[CPUFREQ_NAME_LEN]; 56 57 /* 58 * The "cpufreq driver" - the arch- or hardware-dependent low 59 * level driver of CPUFreq support, and its spinlock. This lock 60 * also protects the cpufreq_cpu_data array. 61 */ 62 static struct cpufreq_driver *cpufreq_driver; 63 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); 64 static DEFINE_RWLOCK(cpufreq_driver_lock); 65 66 static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance); 67 bool cpufreq_supports_freq_invariance(void) 68 { 69 return static_branch_likely(&cpufreq_freq_invariance); 70 } 71 72 /* Flag to suspend/resume CPUFreq governors */ 73 static bool cpufreq_suspended; 74 75 static inline bool has_target(void) 76 { 77 return cpufreq_driver->target_index || cpufreq_driver->target; 78 } 79 80 bool has_target_index(void) 81 { 82 return !!cpufreq_driver->target_index; 83 } 84 85 /* internal prototypes */ 86 static unsigned int __cpufreq_get(struct cpufreq_policy *policy); 87 static int cpufreq_init_governor(struct cpufreq_policy *policy); 88 static void cpufreq_exit_governor(struct cpufreq_policy *policy); 89 static void cpufreq_governor_limits(struct cpufreq_policy *policy); 90 static int cpufreq_set_policy(struct cpufreq_policy *policy, 91 struct cpufreq_governor *new_gov, 92 unsigned int new_pol); 93 static bool cpufreq_boost_supported(void); 94 static int cpufreq_boost_trigger_state(int state); 95 96 /* 97 * Two notifier lists: the "policy" list is involved in the 98 * validation process for a new CPU frequency policy; the 99 * "transition" list for kernel code that needs to handle 100 * changes to devices when the CPU clock speed changes. 101 * The mutex locks both lists. 102 */ 103 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list); 104 SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list); 105 106 static int off __read_mostly; 107 static int cpufreq_disabled(void) 108 { 109 return off; 110 } 111 void disable_cpufreq(void) 112 { 113 off = 1; 114 } 115 EXPORT_SYMBOL_GPL(disable_cpufreq); 116 117 static DEFINE_MUTEX(cpufreq_governor_mutex); 118 119 bool have_governor_per_policy(void) 120 { 121 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY); 122 } 123 EXPORT_SYMBOL_GPL(have_governor_per_policy); 124 125 static struct kobject *cpufreq_global_kobject; 126 127 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) 128 { 129 if (have_governor_per_policy()) 130 return &policy->kobj; 131 else 132 return cpufreq_global_kobject; 133 } 134 EXPORT_SYMBOL_GPL(get_governor_parent_kobj); 135 136 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) 137 { 138 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL); 139 140 if (!io_busy) 141 idle_time += get_cpu_iowait_time_us(cpu, wall); 142 143 return idle_time; 144 } 145 EXPORT_SYMBOL_GPL(get_cpu_idle_time); 146 147 /* 148 * This is a generic cpufreq init() routine which can be used by cpufreq 149 * drivers of SMP systems. It will do following: 150 * - validate & show freq table passed 151 * - set policies transition latency 152 * - policy->cpus with all possible CPUs 153 */ 154 void cpufreq_generic_init(struct cpufreq_policy *policy, 155 struct cpufreq_frequency_table *table, 156 unsigned int transition_latency) 157 { 158 policy->freq_table = table; 159 policy->cpuinfo.transition_latency = transition_latency; 160 161 /* 162 * The driver only supports the SMP configuration where all processors 163 * share the clock and voltage and clock. 164 */ 165 cpumask_setall(policy->cpus); 166 } 167 EXPORT_SYMBOL_GPL(cpufreq_generic_init); 168 169 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu) 170 { 171 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 172 173 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL; 174 } 175 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw); 176 177 struct cpufreq_policy *cpufreq_cpu_policy(unsigned int cpu) 178 { 179 return per_cpu(cpufreq_cpu_data, cpu); 180 } 181 EXPORT_SYMBOL_GPL(cpufreq_cpu_policy); 182 183 unsigned int cpufreq_generic_get(unsigned int cpu) 184 { 185 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); 186 187 if (!policy || IS_ERR(policy->clk)) { 188 pr_err("%s: No %s associated to cpu: %d\n", 189 __func__, policy ? "clk" : "policy", cpu); 190 return 0; 191 } 192 193 return clk_get_rate(policy->clk) / 1000; 194 } 195 EXPORT_SYMBOL_GPL(cpufreq_generic_get); 196 197 /** 198 * cpufreq_cpu_get - Return policy for a CPU and mark it as busy. 199 * @cpu: CPU to find the policy for. 200 * 201 * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment 202 * the kobject reference counter of that policy. Return a valid policy on 203 * success or NULL on failure. 204 * 205 * The policy returned by this function has to be released with the help of 206 * cpufreq_cpu_put() to balance its kobject reference counter properly. 207 */ 208 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) 209 { 210 struct cpufreq_policy *policy = NULL; 211 unsigned long flags; 212 213 if (WARN_ON(cpu >= nr_cpu_ids)) 214 return NULL; 215 216 /* get the cpufreq driver */ 217 read_lock_irqsave(&cpufreq_driver_lock, flags); 218 219 if (cpufreq_driver) { 220 /* get the CPU */ 221 policy = cpufreq_cpu_get_raw(cpu); 222 if (policy) 223 kobject_get(&policy->kobj); 224 } 225 226 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 227 228 return policy; 229 } 230 EXPORT_SYMBOL_GPL(cpufreq_cpu_get); 231 232 /** 233 * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy. 234 * @policy: cpufreq policy returned by cpufreq_cpu_get(). 235 */ 236 void cpufreq_cpu_put(struct cpufreq_policy *policy) 237 { 238 kobject_put(&policy->kobj); 239 } 240 EXPORT_SYMBOL_GPL(cpufreq_cpu_put); 241 242 /********************************************************************* 243 * EXTERNALLY AFFECTING FREQUENCY CHANGES * 244 *********************************************************************/ 245 246 /** 247 * adjust_jiffies - Adjust the system "loops_per_jiffy". 248 * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE. 249 * @ci: Frequency change information. 250 * 251 * This function alters the system "loops_per_jiffy" for the clock 252 * speed change. Note that loops_per_jiffy cannot be updated on SMP 253 * systems as each CPU might be scaled differently. So, use the arch 254 * per-CPU loops_per_jiffy value wherever possible. 255 */ 256 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) 257 { 258 #ifndef CONFIG_SMP 259 static unsigned long l_p_j_ref; 260 static unsigned int l_p_j_ref_freq; 261 262 if (ci->flags & CPUFREQ_CONST_LOOPS) 263 return; 264 265 if (!l_p_j_ref_freq) { 266 l_p_j_ref = loops_per_jiffy; 267 l_p_j_ref_freq = ci->old; 268 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n", 269 l_p_j_ref, l_p_j_ref_freq); 270 } 271 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) { 272 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq, 273 ci->new); 274 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n", 275 loops_per_jiffy, ci->new); 276 } 277 #endif 278 } 279 280 /** 281 * cpufreq_notify_transition - Notify frequency transition and adjust jiffies. 282 * @policy: cpufreq policy to enable fast frequency switching for. 283 * @freqs: contain details of the frequency update. 284 * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE. 285 * 286 * This function calls the transition notifiers and adjust_jiffies(). 287 * 288 * It is called twice on all CPU frequency changes that have external effects. 289 */ 290 static void cpufreq_notify_transition(struct cpufreq_policy *policy, 291 struct cpufreq_freqs *freqs, 292 unsigned int state) 293 { 294 int cpu; 295 296 BUG_ON(irqs_disabled()); 297 298 if (cpufreq_disabled()) 299 return; 300 301 freqs->policy = policy; 302 freqs->flags = cpufreq_driver->flags; 303 pr_debug("notification %u of frequency transition to %u kHz\n", 304 state, freqs->new); 305 306 switch (state) { 307 case CPUFREQ_PRECHANGE: 308 /* 309 * Detect if the driver reported a value as "old frequency" 310 * which is not equal to what the cpufreq core thinks is 311 * "old frequency". 312 */ 313 if (policy->cur && policy->cur != freqs->old) { 314 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n", 315 freqs->old, policy->cur); 316 freqs->old = policy->cur; 317 } 318 319 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 320 CPUFREQ_PRECHANGE, freqs); 321 322 adjust_jiffies(CPUFREQ_PRECHANGE, freqs); 323 break; 324 325 case CPUFREQ_POSTCHANGE: 326 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs); 327 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new, 328 cpumask_pr_args(policy->cpus)); 329 330 for_each_cpu(cpu, policy->cpus) 331 trace_cpu_frequency(freqs->new, cpu); 332 333 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 334 CPUFREQ_POSTCHANGE, freqs); 335 336 cpufreq_stats_record_transition(policy, freqs->new); 337 policy->cur = freqs->new; 338 } 339 } 340 341 /* Do post notifications when there are chances that transition has failed */ 342 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy, 343 struct cpufreq_freqs *freqs, int transition_failed) 344 { 345 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 346 if (!transition_failed) 347 return; 348 349 swap(freqs->old, freqs->new); 350 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 351 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 352 } 353 354 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, 355 struct cpufreq_freqs *freqs) 356 { 357 358 /* 359 * Catch double invocations of _begin() which lead to self-deadlock. 360 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core 361 * doesn't invoke _begin() on their behalf, and hence the chances of 362 * double invocations are very low. Moreover, there are scenarios 363 * where these checks can emit false-positive warnings in these 364 * drivers; so we avoid that by skipping them altogether. 365 */ 366 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION) 367 && current == policy->transition_task); 368 369 wait: 370 wait_event(policy->transition_wait, !policy->transition_ongoing); 371 372 spin_lock(&policy->transition_lock); 373 374 if (unlikely(policy->transition_ongoing)) { 375 spin_unlock(&policy->transition_lock); 376 goto wait; 377 } 378 379 policy->transition_ongoing = true; 380 policy->transition_task = current; 381 382 spin_unlock(&policy->transition_lock); 383 384 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 385 } 386 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin); 387 388 void cpufreq_freq_transition_end(struct cpufreq_policy *policy, 389 struct cpufreq_freqs *freqs, int transition_failed) 390 { 391 if (WARN_ON(!policy->transition_ongoing)) 392 return; 393 394 cpufreq_notify_post_transition(policy, freqs, transition_failed); 395 396 arch_set_freq_scale(policy->related_cpus, 397 policy->cur, 398 arch_scale_freq_ref(policy->cpu)); 399 400 spin_lock(&policy->transition_lock); 401 policy->transition_ongoing = false; 402 policy->transition_task = NULL; 403 spin_unlock(&policy->transition_lock); 404 405 wake_up(&policy->transition_wait); 406 } 407 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end); 408 409 /* 410 * Fast frequency switching status count. Positive means "enabled", negative 411 * means "disabled" and 0 means "not decided yet". 412 */ 413 static int cpufreq_fast_switch_count; 414 static DEFINE_MUTEX(cpufreq_fast_switch_lock); 415 416 static void cpufreq_list_transition_notifiers(void) 417 { 418 struct notifier_block *nb; 419 420 pr_info("Registered transition notifiers:\n"); 421 422 mutex_lock(&cpufreq_transition_notifier_list.mutex); 423 424 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next) 425 pr_info("%pS\n", nb->notifier_call); 426 427 mutex_unlock(&cpufreq_transition_notifier_list.mutex); 428 } 429 430 /** 431 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy. 432 * @policy: cpufreq policy to enable fast frequency switching for. 433 * 434 * Try to enable fast frequency switching for @policy. 435 * 436 * The attempt will fail if there is at least one transition notifier registered 437 * at this point, as fast frequency switching is quite fundamentally at odds 438 * with transition notifiers. Thus if successful, it will make registration of 439 * transition notifiers fail going forward. 440 */ 441 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy) 442 { 443 lockdep_assert_held(&policy->rwsem); 444 445 if (!policy->fast_switch_possible) 446 return; 447 448 mutex_lock(&cpufreq_fast_switch_lock); 449 if (cpufreq_fast_switch_count >= 0) { 450 cpufreq_fast_switch_count++; 451 policy->fast_switch_enabled = true; 452 } else { 453 pr_warn("CPU%u: Fast frequency switching not enabled\n", 454 policy->cpu); 455 cpufreq_list_transition_notifiers(); 456 } 457 mutex_unlock(&cpufreq_fast_switch_lock); 458 } 459 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch); 460 461 /** 462 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy. 463 * @policy: cpufreq policy to disable fast frequency switching for. 464 */ 465 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) 466 { 467 mutex_lock(&cpufreq_fast_switch_lock); 468 if (policy->fast_switch_enabled) { 469 policy->fast_switch_enabled = false; 470 if (!WARN_ON(cpufreq_fast_switch_count <= 0)) 471 cpufreq_fast_switch_count--; 472 } 473 mutex_unlock(&cpufreq_fast_switch_lock); 474 } 475 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); 476 477 static unsigned int __resolve_freq(struct cpufreq_policy *policy, 478 unsigned int target_freq, 479 unsigned int min, unsigned int max, 480 unsigned int relation, unsigned int *idx_p) 481 { 482 unsigned int idx; 483 484 target_freq = clamp_val(target_freq, min, max); 485 486 if (!policy->freq_table) 487 return target_freq; 488 489 idx = cpufreq_frequency_table_target(policy, target_freq, min, max, relation); 490 if (idx_p) 491 *idx_p = idx; 492 493 return policy->freq_table[idx].frequency; 494 } 495 496 /** 497 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported 498 * one. 499 * @policy: associated policy to interrogate 500 * @target_freq: target frequency to resolve. 501 * 502 * The target to driver frequency mapping is cached in the policy. 503 * 504 * Return: Lowest driver-supported frequency greater than or equal to the 505 * given target_freq, subject to policy (min/max) and driver limitations. 506 */ 507 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy, 508 unsigned int target_freq) 509 { 510 unsigned int min = READ_ONCE(policy->min); 511 unsigned int max = READ_ONCE(policy->max); 512 513 /* 514 * If this function runs in parallel with cpufreq_set_policy(), it may 515 * read policy->min before the update and policy->max after the update 516 * or the other way around, so there is no ordering guarantee. 517 * 518 * Resolve this by always honoring the max (in case it comes from 519 * thermal throttling or similar). 520 */ 521 if (unlikely(min > max)) 522 min = max; 523 524 policy->cached_target_freq = __resolve_freq(policy, target_freq, min, max, 525 CPUFREQ_RELATION_LE, 526 &policy->cached_resolved_idx); 527 return policy->cached_target_freq; 528 } 529 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq); 530 531 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy) 532 { 533 unsigned int latency; 534 535 if (policy->transition_delay_us) 536 return policy->transition_delay_us; 537 538 latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC; 539 if (latency) 540 /* Give a 50% breathing room between updates */ 541 return latency + (latency >> 1); 542 543 return USEC_PER_MSEC; 544 } 545 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us); 546 547 /********************************************************************* 548 * SYSFS INTERFACE * 549 *********************************************************************/ 550 static ssize_t show_boost(struct kobject *kobj, 551 struct kobj_attribute *attr, char *buf) 552 { 553 return sysfs_emit(buf, "%d\n", cpufreq_driver->boost_enabled); 554 } 555 556 static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr, 557 const char *buf, size_t count) 558 { 559 bool enable; 560 561 if (kstrtobool(buf, &enable)) 562 return -EINVAL; 563 564 if (cpufreq_boost_trigger_state(enable)) { 565 pr_err("%s: Cannot %s BOOST!\n", 566 __func__, str_enable_disable(enable)); 567 return -EINVAL; 568 } 569 570 pr_debug("%s: cpufreq BOOST %s\n", 571 __func__, str_enabled_disabled(enable)); 572 573 return count; 574 } 575 define_one_global_rw(boost); 576 577 static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf) 578 { 579 return sysfs_emit(buf, "%d\n", policy->boost_enabled); 580 } 581 582 static int policy_set_boost(struct cpufreq_policy *policy, bool enable) 583 { 584 int ret; 585 586 lockdep_assert_held_write(&policy->rwsem); 587 588 if (policy->boost_enabled == enable) 589 return 0; 590 591 policy->boost_enabled = enable; 592 593 ret = cpufreq_driver->set_boost(policy, enable); 594 if (ret) { 595 policy->boost_enabled = !policy->boost_enabled; 596 return ret; 597 } 598 599 ret = freq_qos_update_request(&policy->boost_freq_req, policy->cpuinfo.max_freq); 600 if (ret < 0) { 601 policy->boost_enabled = !policy->boost_enabled; 602 cpufreq_driver->set_boost(policy, policy->boost_enabled); 603 return ret; 604 } 605 606 return 0; 607 } 608 609 static ssize_t store_local_boost(struct cpufreq_policy *policy, 610 const char *buf, size_t count) 611 { 612 int ret; 613 bool enable; 614 615 if (kstrtobool(buf, &enable)) 616 return -EINVAL; 617 618 if (!cpufreq_driver->boost_enabled) 619 return -EINVAL; 620 621 if (!policy->boost_supported) 622 return -EINVAL; 623 624 ret = policy_set_boost(policy, enable); 625 if (!ret) 626 return count; 627 628 return ret; 629 } 630 631 static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost); 632 633 static struct cpufreq_governor *find_governor(const char *str_governor) 634 { 635 struct cpufreq_governor *t; 636 637 for_each_governor(t) 638 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN)) 639 return t; 640 641 return NULL; 642 } 643 644 static struct cpufreq_governor *get_governor(const char *str_governor) 645 { 646 struct cpufreq_governor *t; 647 648 mutex_lock(&cpufreq_governor_mutex); 649 t = find_governor(str_governor); 650 if (!t) 651 goto unlock; 652 653 if (!try_module_get(t->owner)) 654 t = NULL; 655 656 unlock: 657 mutex_unlock(&cpufreq_governor_mutex); 658 659 return t; 660 } 661 662 static unsigned int cpufreq_parse_policy(char *str_governor) 663 { 664 if (!strncasecmp(str_governor, "performance", strlen("performance"))) 665 return CPUFREQ_POLICY_PERFORMANCE; 666 667 if (!strncasecmp(str_governor, "powersave", strlen("powersave"))) 668 return CPUFREQ_POLICY_POWERSAVE; 669 670 return CPUFREQ_POLICY_UNKNOWN; 671 } 672 673 /** 674 * cpufreq_parse_governor - parse a governor string only for has_target() 675 * @str_governor: Governor name. 676 */ 677 static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor) 678 { 679 struct cpufreq_governor *t; 680 681 t = get_governor(str_governor); 682 if (t) 683 return t; 684 685 if (request_module("cpufreq_%s", str_governor)) 686 return NULL; 687 688 return get_governor(str_governor); 689 } 690 691 /* 692 * cpufreq_per_cpu_attr_read() / show_##file_name() - 693 * print out cpufreq information 694 * 695 * Write out information from cpufreq_driver->policy[cpu]; object must be 696 * "unsigned int". 697 */ 698 699 #define show_one(file_name, object) \ 700 static ssize_t show_##file_name \ 701 (struct cpufreq_policy *policy, char *buf) \ 702 { \ 703 return sysfs_emit(buf, "%u\n", policy->object); \ 704 } 705 706 show_one(cpuinfo_min_freq, cpuinfo.min_freq); 707 show_one(cpuinfo_max_freq, cpuinfo.max_freq); 708 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency); 709 show_one(scaling_min_freq, min); 710 show_one(scaling_max_freq, max); 711 712 __weak int arch_freq_get_on_cpu(int cpu) 713 { 714 return -EOPNOTSUPP; 715 } 716 717 static inline bool cpufreq_avg_freq_supported(struct cpufreq_policy *policy) 718 { 719 return arch_freq_get_on_cpu(policy->cpu) != -EOPNOTSUPP; 720 } 721 722 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf) 723 { 724 ssize_t ret; 725 int freq; 726 727 freq = IS_ENABLED(CONFIG_CPUFREQ_ARCH_CUR_FREQ) 728 ? arch_freq_get_on_cpu(policy->cpu) 729 : 0; 730 731 if (freq > 0) 732 ret = sysfs_emit(buf, "%u\n", freq); 733 else if (cpufreq_driver->setpolicy && cpufreq_driver->get) 734 ret = sysfs_emit(buf, "%u\n", cpufreq_driver->get(policy->cpu)); 735 else 736 ret = sysfs_emit(buf, "%u\n", policy->cur); 737 return ret; 738 } 739 740 /* 741 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access 742 */ 743 #define store_one(file_name, object) \ 744 static ssize_t store_##file_name \ 745 (struct cpufreq_policy *policy, const char *buf, size_t count) \ 746 { \ 747 unsigned long val; \ 748 int ret; \ 749 \ 750 ret = kstrtoul(buf, 0, &val); \ 751 if (ret) \ 752 return ret; \ 753 \ 754 ret = freq_qos_update_request(&policy->object##_freq_req, val); \ 755 return ret >= 0 ? count : ret; \ 756 } 757 758 store_one(scaling_min_freq, min); 759 store_one(scaling_max_freq, max); 760 761 /* 762 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware 763 */ 764 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, 765 char *buf) 766 { 767 unsigned int cur_freq = __cpufreq_get(policy); 768 769 if (cur_freq) 770 return sysfs_emit(buf, "%u\n", cur_freq); 771 772 return sysfs_emit(buf, "<unknown>\n"); 773 } 774 775 /* 776 * show_cpuinfo_avg_freq - average CPU frequency as detected by hardware 777 */ 778 static ssize_t show_cpuinfo_avg_freq(struct cpufreq_policy *policy, 779 char *buf) 780 { 781 int avg_freq = arch_freq_get_on_cpu(policy->cpu); 782 783 if (avg_freq > 0) 784 return sysfs_emit(buf, "%u\n", avg_freq); 785 return avg_freq != 0 ? avg_freq : -EINVAL; 786 } 787 788 /* 789 * show_scaling_governor - show the current policy for the specified CPU 790 */ 791 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) 792 { 793 if (policy->policy == CPUFREQ_POLICY_POWERSAVE) 794 return sysfs_emit(buf, "powersave\n"); 795 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) 796 return sysfs_emit(buf, "performance\n"); 797 else if (policy->governor) 798 return sysfs_emit(buf, "%s\n", policy->governor->name); 799 return -EINVAL; 800 } 801 802 /* 803 * store_scaling_governor - store policy for the specified CPU 804 */ 805 static ssize_t store_scaling_governor(struct cpufreq_policy *policy, 806 const char *buf, size_t count) 807 { 808 char str_governor[CPUFREQ_NAME_LEN]; 809 int ret; 810 811 ret = sscanf(buf, "%15s", str_governor); 812 if (ret != 1) 813 return -EINVAL; 814 815 if (cpufreq_driver->setpolicy) { 816 unsigned int new_pol; 817 818 new_pol = cpufreq_parse_policy(str_governor); 819 if (!new_pol) 820 return -EINVAL; 821 822 ret = cpufreq_set_policy(policy, NULL, new_pol); 823 } else { 824 struct cpufreq_governor *new_gov; 825 826 new_gov = cpufreq_parse_governor(str_governor); 827 if (!new_gov) 828 return -EINVAL; 829 830 ret = cpufreq_set_policy(policy, new_gov, 831 CPUFREQ_POLICY_UNKNOWN); 832 833 module_put(new_gov->owner); 834 } 835 836 return ret ? ret : count; 837 } 838 839 /* 840 * show_scaling_driver - show the cpufreq driver currently loaded 841 */ 842 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf) 843 { 844 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name); 845 } 846 847 /* 848 * show_scaling_available_governors - show the available CPUfreq governors 849 */ 850 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy, 851 char *buf) 852 { 853 ssize_t i = 0; 854 struct cpufreq_governor *t; 855 856 if (!has_target()) { 857 i += sysfs_emit(buf, "performance powersave"); 858 goto out; 859 } 860 861 mutex_lock(&cpufreq_governor_mutex); 862 for_each_governor(t) { 863 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) 864 - (CPUFREQ_NAME_LEN + 2))) 865 break; 866 i += sysfs_emit_at(buf, i, "%s ", t->name); 867 } 868 mutex_unlock(&cpufreq_governor_mutex); 869 out: 870 i += sysfs_emit_at(buf, i, "\n"); 871 return i; 872 } 873 874 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf) 875 { 876 ssize_t i = 0; 877 unsigned int cpu; 878 879 for_each_cpu(cpu, mask) { 880 i += sysfs_emit_at(buf, i, "%u ", cpu); 881 if (i >= (PAGE_SIZE - 5)) 882 break; 883 } 884 885 /* Remove the extra space at the end */ 886 i--; 887 888 i += sysfs_emit_at(buf, i, "\n"); 889 return i; 890 } 891 EXPORT_SYMBOL_GPL(cpufreq_show_cpus); 892 893 /* 894 * show_related_cpus - show the CPUs affected by each transition even if 895 * hw coordination is in use 896 */ 897 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf) 898 { 899 return cpufreq_show_cpus(policy->related_cpus, buf); 900 } 901 902 /* 903 * show_affected_cpus - show the CPUs affected by each transition 904 */ 905 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf) 906 { 907 return cpufreq_show_cpus(policy->cpus, buf); 908 } 909 910 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy, 911 const char *buf, size_t count) 912 { 913 unsigned int freq = 0; 914 int ret; 915 916 if (!policy->governor || !policy->governor->store_setspeed) 917 return -EINVAL; 918 919 ret = kstrtouint(buf, 0, &freq); 920 if (ret) 921 return ret; 922 923 policy->governor->store_setspeed(policy, freq); 924 925 return count; 926 } 927 928 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf) 929 { 930 if (!policy->governor || !policy->governor->show_setspeed) 931 return sysfs_emit(buf, "<unsupported>\n"); 932 933 return policy->governor->show_setspeed(policy, buf); 934 } 935 936 /* 937 * show_bios_limit - show the current cpufreq HW/BIOS limitation 938 */ 939 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf) 940 { 941 unsigned int limit; 942 int ret; 943 ret = cpufreq_driver->bios_limit(policy->cpu, &limit); 944 if (!ret) 945 return sysfs_emit(buf, "%u\n", limit); 946 return sysfs_emit(buf, "%u\n", policy->cpuinfo.max_freq); 947 } 948 949 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400); 950 cpufreq_freq_attr_ro(cpuinfo_avg_freq); 951 cpufreq_freq_attr_ro(cpuinfo_min_freq); 952 cpufreq_freq_attr_ro(cpuinfo_max_freq); 953 cpufreq_freq_attr_ro(cpuinfo_transition_latency); 954 cpufreq_freq_attr_ro(scaling_available_governors); 955 cpufreq_freq_attr_ro(scaling_driver); 956 cpufreq_freq_attr_ro(scaling_cur_freq); 957 cpufreq_freq_attr_ro(bios_limit); 958 cpufreq_freq_attr_ro(related_cpus); 959 cpufreq_freq_attr_ro(affected_cpus); 960 cpufreq_freq_attr_rw(scaling_min_freq); 961 cpufreq_freq_attr_rw(scaling_max_freq); 962 cpufreq_freq_attr_rw(scaling_governor); 963 cpufreq_freq_attr_rw(scaling_setspeed); 964 965 static struct attribute *cpufreq_attrs[] = { 966 &cpuinfo_min_freq.attr, 967 &cpuinfo_max_freq.attr, 968 &cpuinfo_transition_latency.attr, 969 &scaling_cur_freq.attr, 970 &scaling_min_freq.attr, 971 &scaling_max_freq.attr, 972 &affected_cpus.attr, 973 &related_cpus.attr, 974 &scaling_governor.attr, 975 &scaling_driver.attr, 976 &scaling_available_governors.attr, 977 &scaling_setspeed.attr, 978 NULL 979 }; 980 ATTRIBUTE_GROUPS(cpufreq); 981 982 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj) 983 #define to_attr(a) container_of(a, struct freq_attr, attr) 984 985 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) 986 { 987 struct cpufreq_policy *policy = to_policy(kobj); 988 struct freq_attr *fattr = to_attr(attr); 989 990 if (!fattr->show) 991 return -EIO; 992 993 guard(cpufreq_policy_read)(policy); 994 995 if (likely(!policy_is_inactive(policy))) 996 return fattr->show(policy, buf); 997 998 return -EBUSY; 999 } 1000 1001 static ssize_t store(struct kobject *kobj, struct attribute *attr, 1002 const char *buf, size_t count) 1003 { 1004 struct cpufreq_policy *policy = to_policy(kobj); 1005 struct freq_attr *fattr = to_attr(attr); 1006 1007 if (!fattr->store) 1008 return -EIO; 1009 1010 guard(cpufreq_policy_write)(policy); 1011 1012 if (likely(!policy_is_inactive(policy))) 1013 return fattr->store(policy, buf, count); 1014 1015 return -EBUSY; 1016 } 1017 1018 static void cpufreq_sysfs_release(struct kobject *kobj) 1019 { 1020 struct cpufreq_policy *policy = to_policy(kobj); 1021 pr_debug("last reference is dropped\n"); 1022 complete(&policy->kobj_unregister); 1023 } 1024 1025 static const struct sysfs_ops sysfs_ops = { 1026 .show = show, 1027 .store = store, 1028 }; 1029 1030 static const struct kobj_type ktype_cpufreq = { 1031 .sysfs_ops = &sysfs_ops, 1032 .default_groups = cpufreq_groups, 1033 .release = cpufreq_sysfs_release, 1034 }; 1035 1036 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu, 1037 struct device *dev) 1038 { 1039 if (unlikely(!dev)) 1040 return; 1041 1042 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus)) 1043 return; 1044 1045 dev_dbg(dev, "%s: Adding symlink\n", __func__); 1046 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq")) 1047 dev_err(dev, "cpufreq symlink creation failed\n"); 1048 } 1049 1050 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu, 1051 struct device *dev) 1052 { 1053 dev_dbg(dev, "%s: Removing symlink\n", __func__); 1054 sysfs_remove_link(&dev->kobj, "cpufreq"); 1055 cpumask_clear_cpu(cpu, policy->real_cpus); 1056 } 1057 1058 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy) 1059 { 1060 struct freq_attr **drv_attr; 1061 int ret = 0; 1062 1063 /* Attributes that need freq_table */ 1064 if (policy->freq_table) { 1065 ret = sysfs_create_file(&policy->kobj, 1066 &cpufreq_freq_attr_scaling_available_freqs.attr); 1067 if (ret) 1068 return ret; 1069 1070 if (cpufreq_boost_supported()) { 1071 ret = sysfs_create_file(&policy->kobj, 1072 &cpufreq_freq_attr_scaling_boost_freqs.attr); 1073 if (ret) 1074 return ret; 1075 } 1076 } 1077 1078 /* set up files for this cpu device */ 1079 drv_attr = cpufreq_driver->attr; 1080 while (drv_attr && *drv_attr) { 1081 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr)); 1082 if (ret) 1083 return ret; 1084 drv_attr++; 1085 } 1086 if (cpufreq_driver->get) { 1087 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr); 1088 if (ret) 1089 return ret; 1090 } 1091 1092 if (cpufreq_avg_freq_supported(policy)) { 1093 ret = sysfs_create_file(&policy->kobj, &cpuinfo_avg_freq.attr); 1094 if (ret) 1095 return ret; 1096 } 1097 1098 if (cpufreq_driver->bios_limit) { 1099 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr); 1100 if (ret) 1101 return ret; 1102 } 1103 1104 if (cpufreq_boost_supported()) { 1105 ret = sysfs_create_file(&policy->kobj, &local_boost.attr); 1106 if (ret) 1107 return ret; 1108 } 1109 1110 return 0; 1111 } 1112 1113 static int cpufreq_init_policy(struct cpufreq_policy *policy) 1114 { 1115 struct cpufreq_governor *gov = NULL; 1116 unsigned int pol = CPUFREQ_POLICY_UNKNOWN; 1117 int ret; 1118 1119 if (has_target()) { 1120 /* Update policy governor to the one used before hotplug. */ 1121 if (policy->last_governor[0] != '\0') 1122 gov = get_governor(policy->last_governor); 1123 if (gov) { 1124 pr_debug("Restoring governor %s for cpu %d\n", 1125 gov->name, policy->cpu); 1126 } else { 1127 gov = get_governor(default_governor); 1128 } 1129 1130 if (!gov) { 1131 gov = cpufreq_default_governor(); 1132 __module_get(gov->owner); 1133 } 1134 1135 } else { 1136 1137 /* Use the default policy if there is no last_policy. */ 1138 if (policy->last_policy) { 1139 pol = policy->last_policy; 1140 } else { 1141 pol = cpufreq_parse_policy(default_governor); 1142 /* 1143 * In case the default governor is neither "performance" 1144 * nor "powersave", fall back to the initial policy 1145 * value set by the driver. 1146 */ 1147 if (pol == CPUFREQ_POLICY_UNKNOWN) 1148 pol = policy->policy; 1149 } 1150 if (pol != CPUFREQ_POLICY_PERFORMANCE && 1151 pol != CPUFREQ_POLICY_POWERSAVE) 1152 return -ENODATA; 1153 } 1154 1155 ret = cpufreq_set_policy(policy, gov, pol); 1156 if (gov) 1157 module_put(gov->owner); 1158 1159 return ret; 1160 } 1161 1162 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu) 1163 { 1164 int ret = 0; 1165 1166 /* Has this CPU been taken care of already? */ 1167 if (cpumask_test_cpu(cpu, policy->cpus)) 1168 return 0; 1169 1170 guard(cpufreq_policy_write)(policy); 1171 1172 if (has_target()) 1173 cpufreq_stop_governor(policy); 1174 1175 cpumask_set_cpu(cpu, policy->cpus); 1176 1177 if (has_target()) { 1178 ret = cpufreq_start_governor(policy); 1179 if (ret) 1180 pr_err("%s: Failed to start governor\n", __func__); 1181 } 1182 1183 return ret; 1184 } 1185 1186 void refresh_frequency_limits(struct cpufreq_policy *policy) 1187 { 1188 if (!policy_is_inactive(policy)) { 1189 pr_debug("updating policy for CPU %u\n", policy->cpu); 1190 1191 cpufreq_set_policy(policy, policy->governor, policy->policy); 1192 } 1193 } 1194 EXPORT_SYMBOL(refresh_frequency_limits); 1195 1196 static void handle_update(struct work_struct *work) 1197 { 1198 struct cpufreq_policy *policy = 1199 container_of(work, struct cpufreq_policy, update); 1200 1201 pr_debug("handle_update for cpu %u called\n", policy->cpu); 1202 1203 guard(cpufreq_policy_write)(policy); 1204 1205 refresh_frequency_limits(policy); 1206 } 1207 1208 static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq, 1209 void *data) 1210 { 1211 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min); 1212 1213 schedule_work(&policy->update); 1214 return 0; 1215 } 1216 1217 static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq, 1218 void *data) 1219 { 1220 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max); 1221 1222 schedule_work(&policy->update); 1223 return 0; 1224 } 1225 1226 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) 1227 { 1228 struct kobject *kobj; 1229 struct completion *cmp; 1230 1231 scoped_guard(cpufreq_policy_write, policy) { 1232 cpufreq_stats_free_table(policy); 1233 kobj = &policy->kobj; 1234 cmp = &policy->kobj_unregister; 1235 } 1236 kobject_put(kobj); 1237 1238 /* 1239 * We need to make sure that the underlying kobj is 1240 * actually not referenced anymore by anybody before we 1241 * proceed with unloading. 1242 */ 1243 pr_debug("waiting for dropping of refcount\n"); 1244 wait_for_completion(cmp); 1245 pr_debug("wait complete\n"); 1246 } 1247 1248 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) 1249 { 1250 struct cpufreq_policy *policy; 1251 struct device *dev = get_cpu_device(cpu); 1252 int ret; 1253 1254 if (!dev) 1255 return NULL; 1256 1257 policy = kzalloc_obj(*policy); 1258 if (!policy) 1259 return NULL; 1260 1261 if (!zalloc_cpumask_var(&policy->cpus, GFP_KERNEL)) 1262 goto err_free_policy; 1263 1264 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) 1265 goto err_free_cpumask; 1266 1267 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL)) 1268 goto err_free_rcpumask; 1269 1270 init_rwsem(&policy->rwsem); 1271 1272 init_completion(&policy->kobj_unregister); 1273 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, 1274 cpufreq_global_kobject, "policy%u", cpu); 1275 if (ret) { 1276 dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret); 1277 /* 1278 * The entire policy object will be freed below, but the extra 1279 * memory allocated for the kobject name needs to be freed by 1280 * releasing the kobject. 1281 */ 1282 kobject_put(&policy->kobj); 1283 goto err_free_real_cpus; 1284 } 1285 1286 freq_constraints_init(&policy->constraints); 1287 1288 policy->nb_min.notifier_call = cpufreq_notifier_min; 1289 policy->nb_max.notifier_call = cpufreq_notifier_max; 1290 1291 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN, 1292 &policy->nb_min); 1293 if (ret) { 1294 dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n", 1295 ret, cpu); 1296 goto err_kobj_remove; 1297 } 1298 1299 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX, 1300 &policy->nb_max); 1301 if (ret) { 1302 dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n", 1303 ret, cpu); 1304 goto err_min_qos_notifier; 1305 } 1306 1307 INIT_LIST_HEAD(&policy->policy_list); 1308 spin_lock_init(&policy->transition_lock); 1309 init_waitqueue_head(&policy->transition_wait); 1310 INIT_WORK(&policy->update, handle_update); 1311 1312 return policy; 1313 1314 err_min_qos_notifier: 1315 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN, 1316 &policy->nb_min); 1317 err_kobj_remove: 1318 cpufreq_policy_put_kobj(policy); 1319 err_free_real_cpus: 1320 free_cpumask_var(policy->real_cpus); 1321 err_free_rcpumask: 1322 free_cpumask_var(policy->related_cpus); 1323 err_free_cpumask: 1324 free_cpumask_var(policy->cpus); 1325 err_free_policy: 1326 kfree(policy); 1327 1328 return NULL; 1329 } 1330 1331 static void cpufreq_policy_free(struct cpufreq_policy *policy) 1332 { 1333 unsigned long flags; 1334 int cpu; 1335 1336 /* 1337 * The callers must ensure the policy is inactive by now, to avoid any 1338 * races with show()/store() callbacks. 1339 */ 1340 if (unlikely(!policy_is_inactive(policy))) 1341 pr_warn("%s: Freeing active policy\n", __func__); 1342 1343 /* Remove policy from list */ 1344 write_lock_irqsave(&cpufreq_driver_lock, flags); 1345 list_del(&policy->policy_list); 1346 1347 for_each_cpu(cpu, policy->related_cpus) 1348 per_cpu(cpufreq_cpu_data, cpu) = NULL; 1349 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1350 1351 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX, 1352 &policy->nb_max); 1353 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN, 1354 &policy->nb_min); 1355 1356 /* Cancel any pending policy->update work before freeing the policy. */ 1357 cancel_work_sync(&policy->update); 1358 1359 if (freq_qos_request_active(&policy->max_freq_req)) { 1360 /* 1361 * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY 1362 * notification, since CPUFREQ_CREATE_POLICY notification was 1363 * sent after adding max_freq_req earlier. 1364 */ 1365 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1366 CPUFREQ_REMOVE_POLICY, policy); 1367 freq_qos_remove_request(&policy->max_freq_req); 1368 } 1369 1370 if (freq_qos_request_active(&policy->min_freq_req)) 1371 freq_qos_remove_request(&policy->min_freq_req); 1372 if (freq_qos_request_active(&policy->boost_freq_req)) 1373 freq_qos_remove_request(&policy->boost_freq_req); 1374 1375 cpufreq_policy_put_kobj(policy); 1376 free_cpumask_var(policy->real_cpus); 1377 free_cpumask_var(policy->related_cpus); 1378 free_cpumask_var(policy->cpus); 1379 kfree(policy); 1380 } 1381 1382 static int cpufreq_policy_init_qos(struct cpufreq_policy *policy) 1383 { 1384 unsigned int min_freq, max_freq; 1385 int ret; 1386 1387 /* Use policy->min/max set by the driver as QoS requests. */ 1388 min_freq = max(FREQ_QOS_MIN_DEFAULT_VALUE, policy->min); 1389 if (policy->max) 1390 max_freq = min(FREQ_QOS_MAX_DEFAULT_VALUE, policy->max); 1391 else 1392 max_freq = FREQ_QOS_MAX_DEFAULT_VALUE; 1393 1394 if (policy->boost_supported) { 1395 ret = freq_qos_add_request(&policy->constraints, 1396 &policy->boost_freq_req, 1397 FREQ_QOS_MAX, 1398 policy->cpuinfo.max_freq); 1399 if (ret < 0) 1400 return ret; 1401 } 1402 1403 ret = freq_qos_add_request(&policy->constraints, &policy->min_freq_req, 1404 FREQ_QOS_MIN, min_freq); 1405 if (ret < 0) 1406 return ret; 1407 1408 ret = freq_qos_add_request(&policy->constraints, &policy->max_freq_req, 1409 FREQ_QOS_MAX, max_freq); 1410 if (ret < 0) 1411 return ret; 1412 1413 return 0; 1414 } 1415 1416 static int cpufreq_policy_online(struct cpufreq_policy *policy, 1417 unsigned int cpu, bool new_policy) 1418 { 1419 unsigned long flags; 1420 unsigned int j; 1421 int ret; 1422 1423 guard(cpufreq_policy_write)(policy); 1424 1425 policy->cpu = cpu; 1426 policy->governor = NULL; 1427 1428 if (!new_policy && cpufreq_driver->online) { 1429 /* Recover policy->cpus using related_cpus */ 1430 cpumask_copy(policy->cpus, policy->related_cpus); 1431 1432 ret = cpufreq_driver->online(policy); 1433 if (ret) { 1434 pr_debug("%s: %d: initialization failed\n", __func__, 1435 __LINE__); 1436 goto out_exit_policy; 1437 } 1438 } else { 1439 cpumask_copy(policy->cpus, cpumask_of(cpu)); 1440 1441 /* 1442 * Call driver. From then on the cpufreq must be able 1443 * to accept all calls to ->verify and ->setpolicy for this CPU. 1444 */ 1445 ret = cpufreq_driver->init(policy); 1446 if (ret) { 1447 pr_debug("%s: %d: initialization failed\n", __func__, 1448 __LINE__); 1449 goto out_clear_policy; 1450 } 1451 1452 /* 1453 * The initialization has succeeded and the policy is online. 1454 * If there is a problem with its frequency table, take it 1455 * offline and drop it. 1456 */ 1457 ret = cpufreq_table_validate_and_sort(policy); 1458 if (ret) 1459 goto out_offline_policy; 1460 1461 if (new_policy) { 1462 ret = cpufreq_policy_init_qos(policy); 1463 if (ret < 0) 1464 goto out_offline_policy; 1465 } 1466 1467 /* 1468 * If the driver hasn't set policy->min/max, set them as they 1469 * are used for clamping frequency requests. 1470 */ 1471 policy->min = policy->min ? policy->min : policy->cpuinfo.min_freq; 1472 policy->max = policy->max ? policy->max : policy->cpuinfo.max_freq; 1473 1474 /* related_cpus should at least include policy->cpus. */ 1475 cpumask_copy(policy->related_cpus, policy->cpus); 1476 } 1477 1478 /* 1479 * affected cpus must always be the one, which are online. We aren't 1480 * managing offline cpus here. 1481 */ 1482 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); 1483 1484 if (new_policy) { 1485 for_each_cpu(j, policy->related_cpus) { 1486 per_cpu(cpufreq_cpu_data, j) = policy; 1487 add_cpu_dev_symlink(policy, j, get_cpu_device(j)); 1488 } 1489 1490 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1491 CPUFREQ_CREATE_POLICY, policy); 1492 } 1493 1494 if (cpufreq_driver->get && has_target()) { 1495 policy->cur = cpufreq_driver->get(policy->cpu); 1496 if (!policy->cur) { 1497 ret = -EIO; 1498 pr_err("%s: ->get() failed\n", __func__); 1499 goto out_destroy_policy; 1500 } 1501 } 1502 1503 /* 1504 * Sometimes boot loaders set CPU frequency to a value outside of 1505 * frequency table present with cpufreq core. In such cases CPU might be 1506 * unstable if it has to run on that frequency for long duration of time 1507 * and so its better to set it to a frequency which is specified in 1508 * freq-table. This also makes cpufreq stats inconsistent as 1509 * cpufreq-stats would fail to register because current frequency of CPU 1510 * isn't found in freq-table. 1511 * 1512 * Because we don't want this change to effect boot process badly, we go 1513 * for the next freq which is >= policy->cur ('cur' must be set by now, 1514 * otherwise we will end up setting freq to lowest of the table as 'cur' 1515 * is initialized to zero). 1516 * 1517 * We are passing target-freq as "policy->cur - 1" otherwise 1518 * __cpufreq_driver_target() would simply fail, as policy->cur will be 1519 * equal to target-freq. 1520 */ 1521 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK) 1522 && has_target()) { 1523 unsigned int old_freq = policy->cur; 1524 1525 /* Are we running at unknown frequency ? */ 1526 ret = cpufreq_frequency_table_get_index(policy, old_freq); 1527 if (ret == -EINVAL) { 1528 ret = __cpufreq_driver_target(policy, old_freq - 1, 1529 CPUFREQ_RELATION_L); 1530 1531 /* 1532 * Reaching here after boot in a few seconds may not 1533 * mean that system will remain stable at "unknown" 1534 * frequency for longer duration. Hence, a BUG_ON(). 1535 */ 1536 BUG_ON(ret); 1537 pr_info("%s: CPU%d: Running at unlisted initial frequency: %u kHz, changing to: %u kHz\n", 1538 __func__, policy->cpu, old_freq, policy->cur); 1539 } 1540 } 1541 1542 if (new_policy) { 1543 ret = cpufreq_add_dev_interface(policy); 1544 if (ret) 1545 goto out_destroy_policy; 1546 1547 cpufreq_stats_create_table(policy); 1548 1549 write_lock_irqsave(&cpufreq_driver_lock, flags); 1550 list_add(&policy->policy_list, &cpufreq_policy_list); 1551 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1552 1553 /* 1554 * Register with the energy model before 1555 * em_rebuild_sched_domains() is called, which will result 1556 * in rebuilding of the sched domains, which should only be done 1557 * once the energy model is properly initialized for the policy 1558 * first. 1559 * 1560 * Also, this should be called before the policy is registered 1561 * with cooling framework. 1562 */ 1563 if (cpufreq_driver->register_em) 1564 cpufreq_driver->register_em(policy); 1565 } 1566 1567 ret = cpufreq_init_policy(policy); 1568 if (ret) { 1569 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", 1570 __func__, cpu, ret); 1571 goto out_destroy_policy; 1572 } 1573 1574 return 0; 1575 1576 out_destroy_policy: 1577 for_each_cpu(j, policy->real_cpus) 1578 remove_cpu_dev_symlink(policy, j, get_cpu_device(j)); 1579 1580 out_offline_policy: 1581 if (cpufreq_driver->offline) 1582 cpufreq_driver->offline(policy); 1583 1584 out_exit_policy: 1585 if (cpufreq_driver->exit) 1586 cpufreq_driver->exit(policy); 1587 1588 out_clear_policy: 1589 cpumask_clear(policy->cpus); 1590 1591 return ret; 1592 } 1593 1594 static int cpufreq_online(unsigned int cpu) 1595 { 1596 struct cpufreq_policy *policy; 1597 bool new_policy; 1598 int ret; 1599 1600 pr_debug("%s: bringing CPU%u online\n", __func__, cpu); 1601 1602 /* Check if this CPU already has a policy to manage it */ 1603 policy = per_cpu(cpufreq_cpu_data, cpu); 1604 if (policy) { 1605 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); 1606 if (!policy_is_inactive(policy)) 1607 return cpufreq_add_policy_cpu(policy, cpu); 1608 1609 /* This is the only online CPU for the policy. Start over. */ 1610 new_policy = false; 1611 } else { 1612 new_policy = true; 1613 policy = cpufreq_policy_alloc(cpu); 1614 if (!policy) 1615 return -ENOMEM; 1616 } 1617 1618 ret = cpufreq_policy_online(policy, cpu, new_policy); 1619 if (ret) { 1620 cpufreq_policy_free(policy); 1621 return ret; 1622 } 1623 1624 kobject_uevent(&policy->kobj, KOBJ_ADD); 1625 1626 /* Callback for handling stuff after policy is ready */ 1627 if (cpufreq_driver->ready) 1628 cpufreq_driver->ready(policy); 1629 1630 /* Register cpufreq cooling only for a new policy */ 1631 if (new_policy && cpufreq_thermal_control_enabled(cpufreq_driver)) 1632 policy->cdev = of_cpufreq_cooling_register(policy); 1633 1634 /* 1635 * Let the per-policy boost flag mirror the cpufreq_driver boost during 1636 * initialization for a new policy. For an existing policy, maintain the 1637 * previous boost value unless global boost is disabled. 1638 */ 1639 if (cpufreq_driver->set_boost && policy->boost_supported && 1640 (new_policy || !cpufreq_boost_enabled())) { 1641 guard(cpufreq_policy_write)(policy); 1642 1643 ret = policy_set_boost(policy, cpufreq_boost_enabled()); 1644 if (ret) { 1645 /* If the set_boost fails, the online operation is not affected */ 1646 pr_info("%s: CPU%d: Cannot %s BOOST\n", __func__, policy->cpu, 1647 str_enable_disable(cpufreq_boost_enabled())); 1648 } 1649 } 1650 1651 pr_debug("initialization complete\n"); 1652 1653 return 0; 1654 } 1655 1656 /** 1657 * cpufreq_add_dev - the cpufreq interface for a CPU device. 1658 * @dev: CPU device. 1659 * @sif: Subsystem interface structure pointer (not used) 1660 */ 1661 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) 1662 { 1663 struct cpufreq_policy *policy; 1664 unsigned cpu = dev->id; 1665 int ret; 1666 1667 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu); 1668 1669 if (cpu_online(cpu)) { 1670 ret = cpufreq_online(cpu); 1671 if (ret) 1672 return ret; 1673 } 1674 1675 /* Create sysfs link on CPU registration */ 1676 policy = per_cpu(cpufreq_cpu_data, cpu); 1677 if (policy) 1678 add_cpu_dev_symlink(policy, cpu, dev); 1679 1680 return 0; 1681 } 1682 1683 static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy) 1684 { 1685 int ret; 1686 1687 if (has_target()) 1688 cpufreq_stop_governor(policy); 1689 1690 cpumask_clear_cpu(cpu, policy->cpus); 1691 1692 if (!policy_is_inactive(policy)) { 1693 /* Nominate a new CPU if necessary. */ 1694 if (cpu == policy->cpu) 1695 policy->cpu = cpumask_any(policy->cpus); 1696 1697 /* Start the governor again for the active policy. */ 1698 if (has_target()) { 1699 ret = cpufreq_start_governor(policy); 1700 if (ret) 1701 pr_err("%s: Failed to start governor\n", __func__); 1702 } 1703 1704 return; 1705 } 1706 1707 if (has_target()) { 1708 strscpy(policy->last_governor, policy->governor->name, 1709 CPUFREQ_NAME_LEN); 1710 cpufreq_exit_governor(policy); 1711 } else { 1712 policy->last_policy = policy->policy; 1713 } 1714 1715 /* 1716 * Perform the ->offline() during light-weight tear-down, as 1717 * that allows fast recovery when the CPU comes back. 1718 */ 1719 if (cpufreq_driver->offline) { 1720 cpufreq_driver->offline(policy); 1721 return; 1722 } 1723 1724 if (cpufreq_driver->exit) 1725 cpufreq_driver->exit(policy); 1726 1727 policy->freq_table = NULL; 1728 } 1729 1730 static int cpufreq_offline(unsigned int cpu) 1731 { 1732 struct cpufreq_policy *policy; 1733 1734 pr_debug("%s: unregistering CPU %u\n", __func__, cpu); 1735 1736 policy = cpufreq_cpu_get_raw(cpu); 1737 if (!policy) { 1738 pr_debug("%s: No cpu_data found\n", __func__); 1739 return 0; 1740 } 1741 1742 guard(cpufreq_policy_write)(policy); 1743 1744 __cpufreq_offline(cpu, policy); 1745 1746 return 0; 1747 } 1748 1749 /* 1750 * cpufreq_remove_dev - remove a CPU device 1751 * 1752 * Removes the cpufreq interface for a CPU device. 1753 */ 1754 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) 1755 { 1756 unsigned int cpu = dev->id; 1757 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 1758 1759 if (!policy) 1760 return; 1761 1762 scoped_guard(cpufreq_policy_write, policy) { 1763 if (cpu_online(cpu)) 1764 __cpufreq_offline(cpu, policy); 1765 1766 remove_cpu_dev_symlink(policy, cpu, dev); 1767 1768 if (!cpumask_empty(policy->real_cpus)) 1769 return; 1770 1771 /* 1772 * Unregister cpufreq cooling once all the CPUs of the policy 1773 * are removed. 1774 */ 1775 if (cpufreq_thermal_control_enabled(cpufreq_driver)) { 1776 cpufreq_cooling_unregister(policy->cdev); 1777 policy->cdev = NULL; 1778 } 1779 1780 /* We did light-weight exit earlier, do full tear down now */ 1781 if (cpufreq_driver->offline && cpufreq_driver->exit) 1782 cpufreq_driver->exit(policy); 1783 } 1784 1785 cpufreq_policy_free(policy); 1786 } 1787 1788 /** 1789 * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference. 1790 * @policy: Policy managing CPUs. 1791 * @new_freq: New CPU frequency. 1792 * 1793 * Adjust to the current frequency first and clean up later by either calling 1794 * cpufreq_update_policy(), or scheduling handle_update(). 1795 */ 1796 static void cpufreq_out_of_sync(struct cpufreq_policy *policy, 1797 unsigned int new_freq) 1798 { 1799 struct cpufreq_freqs freqs; 1800 1801 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n", 1802 policy->cur, new_freq); 1803 1804 freqs.old = policy->cur; 1805 freqs.new = new_freq; 1806 1807 cpufreq_freq_transition_begin(policy, &freqs); 1808 cpufreq_freq_transition_end(policy, &freqs, 0); 1809 } 1810 1811 static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update) 1812 { 1813 unsigned int new_freq; 1814 1815 if (!cpufreq_driver->get) 1816 return 0; 1817 1818 new_freq = cpufreq_driver->get(policy->cpu); 1819 if (!new_freq) 1820 return 0; 1821 1822 /* 1823 * If fast frequency switching is used with the given policy, the check 1824 * against policy->cur is pointless, so skip it in that case. 1825 */ 1826 if (policy->fast_switch_enabled || !has_target()) 1827 return new_freq; 1828 1829 if (policy->cur != new_freq) { 1830 /* 1831 * For some platforms, the frequency returned by hardware may be 1832 * slightly different from what is provided in the frequency 1833 * table, for example hardware may return 499 MHz instead of 500 1834 * MHz. In such cases it is better to avoid getting into 1835 * unnecessary frequency updates. 1836 */ 1837 if (abs(policy->cur - new_freq) < KHZ_PER_MHZ) 1838 return policy->cur; 1839 1840 cpufreq_out_of_sync(policy, new_freq); 1841 if (update) 1842 schedule_work(&policy->update); 1843 } 1844 1845 return new_freq; 1846 } 1847 1848 /** 1849 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur 1850 * @cpu: CPU number 1851 * 1852 * This is the last known freq, without actually getting it from the driver. 1853 * Return value will be same as what is shown in scaling_cur_freq in sysfs. 1854 */ 1855 unsigned int cpufreq_quick_get(unsigned int cpu) 1856 { 1857 unsigned long flags; 1858 1859 read_lock_irqsave(&cpufreq_driver_lock, flags); 1860 1861 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) { 1862 unsigned int ret_freq = cpufreq_driver->get(cpu); 1863 1864 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1865 1866 return ret_freq; 1867 } 1868 1869 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1870 1871 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 1872 if (policy) 1873 return policy->cur; 1874 1875 return 0; 1876 } 1877 EXPORT_SYMBOL(cpufreq_quick_get); 1878 1879 /** 1880 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU 1881 * @cpu: CPU number 1882 * 1883 * Just return the max possible frequency for a given CPU. 1884 */ 1885 unsigned int cpufreq_quick_get_max(unsigned int cpu) 1886 { 1887 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 1888 if (policy) 1889 return policy->max; 1890 1891 return 0; 1892 } 1893 EXPORT_SYMBOL(cpufreq_quick_get_max); 1894 1895 /** 1896 * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU 1897 * @cpu: CPU number 1898 * 1899 * The default return value is the max_freq field of cpuinfo. 1900 */ 1901 __weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu) 1902 { 1903 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 1904 if (policy) 1905 return policy->cpuinfo.max_freq; 1906 1907 return 0; 1908 } 1909 EXPORT_SYMBOL(cpufreq_get_hw_max_freq); 1910 1911 static unsigned int __cpufreq_get(struct cpufreq_policy *policy) 1912 { 1913 if (unlikely(policy_is_inactive(policy))) 1914 return 0; 1915 1916 return cpufreq_verify_current_freq(policy, true); 1917 } 1918 1919 /** 1920 * cpufreq_get - get the current CPU frequency (in kHz) 1921 * @cpu: CPU number 1922 * 1923 * Get the CPU current (static) CPU frequency 1924 */ 1925 unsigned int cpufreq_get(unsigned int cpu) 1926 { 1927 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 1928 if (!policy) 1929 return 0; 1930 1931 guard(cpufreq_policy_read)(policy); 1932 1933 return __cpufreq_get(policy); 1934 } 1935 EXPORT_SYMBOL(cpufreq_get); 1936 1937 static struct subsys_interface cpufreq_interface = { 1938 .name = "cpufreq", 1939 .subsys = &cpu_subsys, 1940 .add_dev = cpufreq_add_dev, 1941 .remove_dev = cpufreq_remove_dev, 1942 }; 1943 1944 /* 1945 * In case platform wants some specific frequency to be configured 1946 * during suspend.. 1947 */ 1948 int cpufreq_generic_suspend(struct cpufreq_policy *policy) 1949 { 1950 int ret; 1951 1952 if (!policy->suspend_freq) { 1953 pr_debug("%s: suspend_freq not defined\n", __func__); 1954 return 0; 1955 } 1956 1957 pr_debug("%s: Setting suspend-freq: %u\n", __func__, 1958 policy->suspend_freq); 1959 1960 ret = __cpufreq_driver_target(policy, policy->suspend_freq, 1961 CPUFREQ_RELATION_H); 1962 if (ret) 1963 pr_err("%s: unable to set suspend-freq: %u. err: %d\n", 1964 __func__, policy->suspend_freq, ret); 1965 1966 return ret; 1967 } 1968 EXPORT_SYMBOL(cpufreq_generic_suspend); 1969 1970 static void cpufreq_suspend_policy(struct cpufreq_policy *policy) 1971 { 1972 guard(cpufreq_policy_write)(policy); 1973 1974 cpufreq_stop_governor(policy); 1975 policy->suspended = true; 1976 } 1977 1978 /** 1979 * cpufreq_suspend() - Suspend CPUFreq governors. 1980 * 1981 * Called during system wide Suspend/Hibernate cycles for suspending governors 1982 * as some platforms can't change frequency after this point in suspend cycle. 1983 * Because some of the devices (like: i2c, regulators, etc) they use for 1984 * changing frequency are suspended quickly after this point. 1985 */ 1986 void cpufreq_suspend(void) 1987 { 1988 struct cpufreq_policy *policy; 1989 1990 if (!cpufreq_driver) 1991 return; 1992 1993 cpus_read_lock(); 1994 if (!has_target() && !cpufreq_driver->suspend) 1995 goto suspend; 1996 1997 pr_debug("%s: Suspending Governors\n", __func__); 1998 1999 for_each_active_policy(policy) { 2000 if (has_target()) 2001 cpufreq_suspend_policy(policy); 2002 2003 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy)) 2004 pr_err("%s: Failed to suspend driver: %s\n", __func__, 2005 cpufreq_driver->name); 2006 } 2007 2008 suspend: 2009 cpufreq_suspended = true; 2010 cpus_read_unlock(); 2011 } 2012 2013 static void cpufreq_resume_policy(struct cpufreq_policy *policy, bool start) 2014 { 2015 guard(cpufreq_policy_write)(policy); 2016 2017 if (!policy->suspended) 2018 return; 2019 2020 policy->suspended = false; 2021 2022 if (policy_is_inactive(policy)) { 2023 /* 2024 * The policy was active when it was suspended, so its governor 2025 * is still there. Let the governor go. 2026 */ 2027 cpufreq_exit_governor(policy); 2028 return; 2029 } 2030 2031 if (!start) 2032 return; 2033 2034 /* 2035 * Force executing the driver .target() callback, if present, in the 2036 * next invocation of __cpufreq_driver_target() in case policy->cur has 2037 * become stale while suspended, but do it only for drivers without a 2038 * .get() callback for which cpufreq_verify_current_freq() is a NOP. 2039 */ 2040 if (cpufreq_driver->target && !cpufreq_driver->get) 2041 policy->force_target = true; 2042 2043 if (cpufreq_start_governor(policy)) 2044 pr_err("%s: Failed to start governor for CPU%u's policy\n", 2045 __func__, policy->cpu); 2046 } 2047 2048 /** 2049 * cpufreq_resume() - Resume CPUFreq governors. 2050 * 2051 * Called during system wide Suspend/Hibernate cycle for resuming governors that 2052 * are suspended with cpufreq_suspend(). 2053 */ 2054 void cpufreq_resume(void) 2055 { 2056 struct cpufreq_policy *policy; 2057 2058 if (!cpufreq_driver) 2059 return; 2060 2061 guard(cpus_read_lock)(); 2062 2063 if (unlikely(!cpufreq_suspended)) 2064 return; 2065 2066 cpufreq_suspended = false; 2067 2068 if (!has_target() && !cpufreq_driver->resume) 2069 return; 2070 2071 pr_debug("%s: Resuming Governors\n", __func__); 2072 2073 for_each_policy(policy) { 2074 int ret = 0; 2075 2076 if (cpufreq_driver->resume && !policy_is_inactive(policy)) { 2077 ret = cpufreq_driver->resume(policy); 2078 if (ret) 2079 pr_err("%s: Failed to resume driver: %s\n", __func__, 2080 cpufreq_driver->name); 2081 } 2082 2083 if (has_target()) 2084 cpufreq_resume_policy(policy, ret == 0); 2085 } 2086 } 2087 2088 /** 2089 * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones. 2090 * @flags: Flags to test against the current cpufreq driver's flags. 2091 * 2092 * Assumes that the driver is there, so callers must ensure that this is the 2093 * case. 2094 */ 2095 bool cpufreq_driver_test_flags(u16 flags) 2096 { 2097 return !!(cpufreq_driver->flags & flags); 2098 } 2099 2100 /** 2101 * cpufreq_get_current_driver - Return the current driver's name. 2102 * 2103 * Return the name string of the currently registered cpufreq driver or NULL if 2104 * none. 2105 */ 2106 const char *cpufreq_get_current_driver(void) 2107 { 2108 if (cpufreq_driver) 2109 return cpufreq_driver->name; 2110 2111 return NULL; 2112 } 2113 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver); 2114 2115 /** 2116 * cpufreq_get_driver_data - Return current driver data. 2117 * 2118 * Return the private data of the currently registered cpufreq driver, or NULL 2119 * if no cpufreq driver has been registered. 2120 */ 2121 void *cpufreq_get_driver_data(void) 2122 { 2123 if (cpufreq_driver) 2124 return cpufreq_driver->driver_data; 2125 2126 return NULL; 2127 } 2128 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data); 2129 2130 /********************************************************************* 2131 * NOTIFIER LISTS INTERFACE * 2132 *********************************************************************/ 2133 2134 /** 2135 * cpufreq_register_notifier - Register a notifier with cpufreq. 2136 * @nb: notifier function to register. 2137 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER. 2138 * 2139 * Add a notifier to one of two lists: either a list of notifiers that run on 2140 * clock rate changes (once before and once after every transition), or a list 2141 * of notifiers that ron on cpufreq policy changes. 2142 * 2143 * This function may sleep and it has the same return values as 2144 * blocking_notifier_chain_register(). 2145 */ 2146 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) 2147 { 2148 int ret; 2149 2150 if (cpufreq_disabled()) 2151 return -EINVAL; 2152 2153 switch (list) { 2154 case CPUFREQ_TRANSITION_NOTIFIER: 2155 mutex_lock(&cpufreq_fast_switch_lock); 2156 2157 if (cpufreq_fast_switch_count > 0) { 2158 mutex_unlock(&cpufreq_fast_switch_lock); 2159 return -EBUSY; 2160 } 2161 ret = srcu_notifier_chain_register( 2162 &cpufreq_transition_notifier_list, nb); 2163 if (!ret) 2164 cpufreq_fast_switch_count--; 2165 2166 mutex_unlock(&cpufreq_fast_switch_lock); 2167 break; 2168 case CPUFREQ_POLICY_NOTIFIER: 2169 ret = blocking_notifier_chain_register( 2170 &cpufreq_policy_notifier_list, nb); 2171 break; 2172 default: 2173 ret = -EINVAL; 2174 } 2175 2176 return ret; 2177 } 2178 EXPORT_SYMBOL(cpufreq_register_notifier); 2179 2180 /** 2181 * cpufreq_unregister_notifier - Unregister a notifier from cpufreq. 2182 * @nb: notifier block to be unregistered. 2183 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER. 2184 * 2185 * Remove a notifier from one of the cpufreq notifier lists. 2186 * 2187 * This function may sleep and it has the same return values as 2188 * blocking_notifier_chain_unregister(). 2189 */ 2190 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list) 2191 { 2192 int ret; 2193 2194 if (cpufreq_disabled()) 2195 return -EINVAL; 2196 2197 switch (list) { 2198 case CPUFREQ_TRANSITION_NOTIFIER: 2199 mutex_lock(&cpufreq_fast_switch_lock); 2200 2201 ret = srcu_notifier_chain_unregister( 2202 &cpufreq_transition_notifier_list, nb); 2203 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0)) 2204 cpufreq_fast_switch_count++; 2205 2206 mutex_unlock(&cpufreq_fast_switch_lock); 2207 break; 2208 case CPUFREQ_POLICY_NOTIFIER: 2209 ret = blocking_notifier_chain_unregister( 2210 &cpufreq_policy_notifier_list, nb); 2211 break; 2212 default: 2213 ret = -EINVAL; 2214 } 2215 2216 return ret; 2217 } 2218 EXPORT_SYMBOL(cpufreq_unregister_notifier); 2219 2220 2221 /********************************************************************* 2222 * GOVERNORS * 2223 *********************************************************************/ 2224 2225 /** 2226 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch. 2227 * @policy: cpufreq policy to switch the frequency for. 2228 * @target_freq: New frequency to set (may be approximate). 2229 * 2230 * Carry out a fast frequency switch without sleeping. 2231 * 2232 * The driver's ->fast_switch() callback invoked by this function must be 2233 * suitable for being called from within RCU-sched read-side critical sections 2234 * and it is expected to select the minimum available frequency greater than or 2235 * equal to @target_freq (CPUFREQ_RELATION_L). 2236 * 2237 * This function must not be called if policy->fast_switch_enabled is unset. 2238 * 2239 * Governors calling this function must guarantee that it will never be invoked 2240 * twice in parallel for the same policy and that it will never be called in 2241 * parallel with either ->target() or ->target_index() for the same policy. 2242 * 2243 * Returns the actual frequency set for the CPU. 2244 * 2245 * If 0 is returned by the driver's ->fast_switch() callback to indicate an 2246 * error condition, the hardware configuration must be preserved. 2247 */ 2248 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, 2249 unsigned int target_freq) 2250 { 2251 unsigned int freq; 2252 int cpu; 2253 2254 target_freq = clamp_val(target_freq, policy->min, policy->max); 2255 freq = cpufreq_driver->fast_switch(policy, target_freq); 2256 2257 if (!freq) 2258 return 0; 2259 2260 policy->cur = freq; 2261 arch_set_freq_scale(policy->related_cpus, freq, 2262 arch_scale_freq_ref(policy->cpu)); 2263 cpufreq_stats_record_transition(policy, freq); 2264 2265 if (trace_cpu_frequency_enabled()) { 2266 for_each_cpu(cpu, policy->cpus) 2267 trace_call__cpu_frequency(freq, cpu); 2268 } 2269 2270 return freq; 2271 } 2272 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); 2273 2274 /** 2275 * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go. 2276 * @policy: cpufreq policy object of the target CPU. 2277 * @min_perf: Minimum (required) performance level (units of @capacity). 2278 * @target_perf: Target (desired) performance level (units of @capacity). 2279 * @max_perf: Maximum (allowed) performance level (units of @capacity). 2280 * @capacity: Capacity of the target CPU. 2281 * 2282 * Carry out a fast performance level adjustment for the CPU represented by 2283 * @policy without sleeping. 2284 * 2285 * The driver's ->adjust_perf() callback invoked by this function must be 2286 * suitable for calling from within RCU-sched read-side critical sections and 2287 * it is expected to program the processor to select suitable performance 2288 * levels between @min_perf and @max_perf inclusive and preferably close to 2289 * @target_perf going forward for the CPU represented by @policy. 2290 * 2291 * This function must not be called if policy->fast_switch_enabled is unset. 2292 * 2293 * Governors calling this function must guarantee that it will never be invoked 2294 * twice in parallel for the same CPU and that it will never be called in 2295 * parallel with either ->target() or ->target_index() or ->fast_switch() for 2296 * the same CPU. 2297 */ 2298 void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy, 2299 unsigned long min_perf, 2300 unsigned long target_perf, 2301 unsigned long max_perf, 2302 unsigned long capacity) 2303 { 2304 cpufreq_driver->adjust_perf(policy, min_perf, target_perf, max_perf, capacity); 2305 } 2306 2307 /** 2308 * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback. 2309 * 2310 * Return 'true' if the ->adjust_perf callback is present for the 2311 * current driver or 'false' otherwise. 2312 */ 2313 bool cpufreq_driver_has_adjust_perf(void) 2314 { 2315 return !!cpufreq_driver->adjust_perf; 2316 } 2317 2318 /* Must set freqs->new to intermediate frequency */ 2319 static int __target_intermediate(struct cpufreq_policy *policy, 2320 struct cpufreq_freqs *freqs, int index) 2321 { 2322 int ret; 2323 2324 freqs->new = cpufreq_driver->get_intermediate(policy, index); 2325 2326 /* We don't need to switch to intermediate freq */ 2327 if (!freqs->new) 2328 return 0; 2329 2330 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n", 2331 __func__, policy->cpu, freqs->old, freqs->new); 2332 2333 cpufreq_freq_transition_begin(policy, freqs); 2334 ret = cpufreq_driver->target_intermediate(policy, index); 2335 cpufreq_freq_transition_end(policy, freqs, ret); 2336 2337 if (ret) 2338 pr_err("%s: Failed to change to intermediate frequency: %d\n", 2339 __func__, ret); 2340 2341 return ret; 2342 } 2343 2344 static int __target_index(struct cpufreq_policy *policy, int index) 2345 { 2346 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; 2347 unsigned int restore_freq, intermediate_freq = 0; 2348 unsigned int newfreq = policy->freq_table[index].frequency; 2349 int retval = -EINVAL; 2350 bool notify; 2351 2352 if (newfreq == policy->cur) 2353 return 0; 2354 2355 /* Save last value to restore later on errors */ 2356 restore_freq = policy->cur; 2357 2358 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); 2359 if (notify) { 2360 /* Handle switching to intermediate frequency */ 2361 if (cpufreq_driver->get_intermediate) { 2362 retval = __target_intermediate(policy, &freqs, index); 2363 if (retval) 2364 return retval; 2365 2366 intermediate_freq = freqs.new; 2367 /* Set old freq to intermediate */ 2368 if (intermediate_freq) 2369 freqs.old = freqs.new; 2370 } 2371 2372 freqs.new = newfreq; 2373 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", 2374 __func__, policy->cpu, freqs.old, freqs.new); 2375 2376 cpufreq_freq_transition_begin(policy, &freqs); 2377 } 2378 2379 retval = cpufreq_driver->target_index(policy, index); 2380 if (retval) 2381 pr_err("%s: Failed to change cpu frequency: %d\n", __func__, 2382 retval); 2383 2384 if (notify) { 2385 cpufreq_freq_transition_end(policy, &freqs, retval); 2386 2387 /* 2388 * Failed after setting to intermediate freq? Driver should have 2389 * reverted back to initial frequency and so should we. Check 2390 * here for intermediate_freq instead of get_intermediate, in 2391 * case we haven't switched to intermediate freq at all. 2392 */ 2393 if (unlikely(retval && intermediate_freq)) { 2394 freqs.old = intermediate_freq; 2395 freqs.new = restore_freq; 2396 cpufreq_freq_transition_begin(policy, &freqs); 2397 cpufreq_freq_transition_end(policy, &freqs, 0); 2398 } 2399 } 2400 2401 return retval; 2402 } 2403 2404 int __cpufreq_driver_target(struct cpufreq_policy *policy, 2405 unsigned int target_freq, 2406 unsigned int relation) 2407 { 2408 unsigned int old_target_freq = target_freq; 2409 unsigned int idx = 0; 2410 2411 if (cpufreq_disabled()) 2412 return -ENODEV; 2413 2414 target_freq = __resolve_freq(policy, target_freq, policy->min, 2415 policy->max, relation, &idx); 2416 2417 pr_debug("CPU %u: cur %u kHz -> target %u kHz (req %u kHz, rel %u)\n", 2418 policy->cpu, policy->cur, target_freq, old_target_freq, relation); 2419 2420 /* 2421 * Unless invoking the target callback is forced, skip the update if the 2422 * frequency is not really changing. 2423 */ 2424 if (policy->force_target) { 2425 /* Avoid forcing the driver callback unnecessarily next time. */ 2426 policy->force_target = false; 2427 } else if (target_freq == policy->cur) { 2428 return 0; 2429 } 2430 2431 if (cpufreq_driver->target) { 2432 /* 2433 * If the driver hasn't setup a single inefficient frequency, 2434 * it's unlikely it knows how to decode CPUFREQ_RELATION_E. 2435 */ 2436 if (!policy->efficiencies_available) 2437 relation &= ~CPUFREQ_RELATION_E; 2438 2439 return cpufreq_driver->target(policy, target_freq, relation); 2440 } 2441 2442 if (!cpufreq_driver->target_index) 2443 return -EINVAL; 2444 2445 return __target_index(policy, idx); 2446 } 2447 EXPORT_SYMBOL_GPL(__cpufreq_driver_target); 2448 2449 int cpufreq_driver_target(struct cpufreq_policy *policy, 2450 unsigned int target_freq, 2451 unsigned int relation) 2452 { 2453 guard(cpufreq_policy_write)(policy); 2454 2455 /* 2456 * The last CPU in the policy may have gone offline after the caller 2457 * obtained it, in which case the policy is inactive and must not be 2458 * used for changing the frequency. 2459 */ 2460 if (unlikely(policy_is_inactive(policy))) 2461 return -EBUSY; 2462 2463 return __cpufreq_driver_target(policy, target_freq, relation); 2464 } 2465 EXPORT_SYMBOL_GPL(cpufreq_driver_target); 2466 2467 __weak struct cpufreq_governor *cpufreq_fallback_governor(void) 2468 { 2469 return NULL; 2470 } 2471 2472 static bool cpufreq_is_suspended(struct cpufreq_policy *policy) 2473 { 2474 return cpufreq_suspended || policy->suspended; 2475 } 2476 2477 static int cpufreq_init_governor(struct cpufreq_policy *policy) 2478 { 2479 int ret; 2480 2481 /* Don't start any governor operations for suspended policies. */ 2482 if (cpufreq_is_suspended(policy)) 2483 return 0; 2484 /* 2485 * Governor might not be initiated here if ACPI _PPC changed 2486 * notification happened, so check it. 2487 */ 2488 if (!policy->governor) 2489 return -EINVAL; 2490 2491 /* Platform doesn't want dynamic frequency switching ? */ 2492 if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING && 2493 cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) { 2494 struct cpufreq_governor *gov = cpufreq_fallback_governor(); 2495 2496 if (gov) { 2497 pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n", 2498 policy->governor->name, gov->name); 2499 policy->governor = gov; 2500 } else { 2501 return -EINVAL; 2502 } 2503 } 2504 2505 if (!try_module_get(policy->governor->owner)) 2506 return -EINVAL; 2507 2508 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2509 2510 if (policy->governor->init) { 2511 ret = policy->governor->init(policy); 2512 if (ret) { 2513 module_put(policy->governor->owner); 2514 return ret; 2515 } 2516 } 2517 2518 policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET); 2519 2520 return 0; 2521 } 2522 2523 static void cpufreq_exit_governor(struct cpufreq_policy *policy) 2524 { 2525 if (cpufreq_is_suspended(policy) || !policy->governor) 2526 return; 2527 2528 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2529 2530 if (policy->governor->exit) 2531 policy->governor->exit(policy); 2532 2533 module_put(policy->governor->owner); 2534 } 2535 2536 int cpufreq_start_governor(struct cpufreq_policy *policy) 2537 { 2538 int ret; 2539 2540 if (cpufreq_is_suspended(policy)) 2541 return 0; 2542 2543 if (!policy->governor) 2544 return -EINVAL; 2545 2546 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2547 2548 cpufreq_verify_current_freq(policy, false); 2549 2550 if (policy->governor->start) { 2551 ret = policy->governor->start(policy); 2552 if (ret) 2553 return ret; 2554 } 2555 2556 if (policy->governor->limits) 2557 policy->governor->limits(policy); 2558 2559 return 0; 2560 } 2561 2562 void cpufreq_stop_governor(struct cpufreq_policy *policy) 2563 { 2564 if (cpufreq_is_suspended(policy) || !policy->governor) 2565 return; 2566 2567 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2568 2569 if (policy->governor->stop) 2570 policy->governor->stop(policy); 2571 } 2572 2573 static void cpufreq_governor_limits(struct cpufreq_policy *policy) 2574 { 2575 if (cpufreq_is_suspended(policy) || !policy->governor) 2576 return; 2577 2578 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2579 2580 if (policy->governor->limits) 2581 policy->governor->limits(policy); 2582 } 2583 2584 int cpufreq_register_governor(struct cpufreq_governor *governor) 2585 { 2586 int err; 2587 2588 if (!governor) 2589 return -EINVAL; 2590 2591 if (cpufreq_disabled()) 2592 return -ENODEV; 2593 2594 mutex_lock(&cpufreq_governor_mutex); 2595 2596 err = -EBUSY; 2597 if (!find_governor(governor->name)) { 2598 err = 0; 2599 list_add(&governor->governor_list, &cpufreq_governor_list); 2600 } 2601 2602 mutex_unlock(&cpufreq_governor_mutex); 2603 return err; 2604 } 2605 EXPORT_SYMBOL_GPL(cpufreq_register_governor); 2606 2607 void cpufreq_unregister_governor(struct cpufreq_governor *governor) 2608 { 2609 struct cpufreq_policy *policy; 2610 unsigned long flags; 2611 2612 if (!governor) 2613 return; 2614 2615 if (cpufreq_disabled()) 2616 return; 2617 2618 /* clear last_governor for all inactive policies */ 2619 read_lock_irqsave(&cpufreq_driver_lock, flags); 2620 for_each_inactive_policy(policy) { 2621 if (!strcmp(policy->last_governor, governor->name)) { 2622 policy->governor = NULL; 2623 policy->last_governor[0] = '\0'; 2624 } 2625 } 2626 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 2627 2628 mutex_lock(&cpufreq_governor_mutex); 2629 list_del(&governor->governor_list); 2630 mutex_unlock(&cpufreq_governor_mutex); 2631 } 2632 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor); 2633 2634 2635 /********************************************************************* 2636 * POLICY INTERFACE * 2637 *********************************************************************/ 2638 2639 DEFINE_PER_CPU(unsigned long, cpufreq_pressure); 2640 2641 /** 2642 * cpufreq_update_pressure() - Update cpufreq pressure for CPUs 2643 * @policy: cpufreq policy of the CPUs. 2644 * 2645 * Update the value of cpufreq pressure for all @cpus in the policy. 2646 */ 2647 static void cpufreq_update_pressure(struct cpufreq_policy *policy) 2648 { 2649 unsigned long max_capacity, capped_freq, pressure; 2650 u32 max_freq; 2651 int cpu; 2652 2653 cpu = cpumask_first(policy->related_cpus); 2654 max_freq = arch_scale_freq_ref(cpu); 2655 if (!max_freq && cpufreq_driver->scale_freq_ref) 2656 max_freq = cpufreq_driver->scale_freq_ref(policy); 2657 2658 capped_freq = policy->max; 2659 2660 /* 2661 * Handle properly the boost frequencies, which should simply clean 2662 * the cpufreq pressure value. 2663 */ 2664 if (max_freq <= capped_freq) { 2665 pressure = 0; 2666 } else { 2667 max_capacity = arch_scale_cpu_capacity(cpu); 2668 pressure = max_capacity - 2669 mult_frac(max_capacity, capped_freq, max_freq); 2670 } 2671 2672 for_each_cpu(cpu, policy->related_cpus) 2673 WRITE_ONCE(per_cpu(cpufreq_pressure, cpu), pressure); 2674 } 2675 2676 /** 2677 * cpufreq_set_policy - Modify cpufreq policy parameters. 2678 * @policy: Policy object to modify. 2679 * @new_gov: Policy governor pointer. 2680 * @new_pol: Policy value (for drivers with built-in governors). 2681 * 2682 * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency 2683 * limits to be set for the policy, update @policy with the verified limits 2684 * values and either invoke the driver's ->setpolicy() callback (if present) or 2685 * carry out a governor update for @policy. That is, run the current governor's 2686 * ->limits() callback (if @new_gov points to the same object as the one in 2687 * @policy) or replace the governor for @policy with @new_gov. 2688 * 2689 * The cpuinfo part of @policy is not updated by this function. 2690 */ 2691 static int cpufreq_set_policy(struct cpufreq_policy *policy, 2692 struct cpufreq_governor *new_gov, 2693 unsigned int new_pol) 2694 { 2695 bool need_update_limits = cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS; 2696 struct cpufreq_policy_data new_data; 2697 struct cpufreq_governor *old_gov; 2698 unsigned int freq; 2699 int ret; 2700 2701 memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); 2702 new_data.freq_table = policy->freq_table; 2703 new_data.cpu = policy->cpu; 2704 /* 2705 * PM QoS framework collects all the requests from users and provide us 2706 * the final aggregated value here. 2707 */ 2708 new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN); 2709 new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX); 2710 2711 pr_debug("setting new policy for CPU %u: %u - %u kHz\n", 2712 new_data.cpu, new_data.min, new_data.max); 2713 2714 /* 2715 * Verify that the CPU speed can be set within these limits and make sure 2716 * that min <= max. 2717 */ 2718 ret = cpufreq_driver->verify(&new_data); 2719 if (ret) 2720 return ret; 2721 2722 /* 2723 * Resolve policy min/max to available frequencies. It ensures 2724 * no frequency resolution will neither overshoot the requested maximum 2725 * nor undershoot the requested minimum. 2726 * 2727 * Avoid storing intermediate values in policy->max or policy->min and 2728 * compiler optimizations around them because they may be accessed 2729 * concurrently by cpufreq_driver_resolve_freq() during the update. 2730 */ 2731 freq = __resolve_freq(policy, new_data.max, new_data.min, new_data.max, 2732 CPUFREQ_RELATION_H, NULL); 2733 if (freq != policy->max) { 2734 WRITE_ONCE(policy->max, freq); 2735 if (need_update_limits) 2736 policy->force_target = true; 2737 } 2738 2739 freq = __resolve_freq(policy, new_data.min, new_data.min, new_data.max, 2740 CPUFREQ_RELATION_L, NULL); 2741 freq = min(freq, policy->max); 2742 if (freq != policy->min) { 2743 WRITE_ONCE(policy->min, freq); 2744 if (need_update_limits) 2745 policy->force_target = true; 2746 } 2747 2748 trace_cpu_frequency_limits(policy); 2749 2750 cpufreq_update_pressure(policy); 2751 2752 pr_debug("new min and max freqs are %u - %u kHz\n", 2753 policy->min, policy->max); 2754 2755 if (cpufreq_driver->setpolicy) { 2756 policy->policy = new_pol; 2757 pr_debug("setting range\n"); 2758 return cpufreq_driver->setpolicy(policy); 2759 } 2760 2761 if (new_gov == policy->governor) { 2762 pr_debug("governor limits update\n"); 2763 cpufreq_governor_limits(policy); 2764 return 0; 2765 } 2766 2767 pr_debug("governor switch\n"); 2768 2769 /* save old, working values */ 2770 old_gov = policy->governor; 2771 /* end old governor */ 2772 if (old_gov) { 2773 cpufreq_stop_governor(policy); 2774 cpufreq_exit_governor(policy); 2775 } 2776 2777 /* 2778 * Initialize cached_target_freq while no governor is running to avoid 2779 * racing with cpufreq_driver_resolve_freq(), which updates it. 2780 */ 2781 policy->cached_target_freq = UINT_MAX; 2782 2783 /* start new governor */ 2784 policy->governor = new_gov; 2785 ret = cpufreq_init_governor(policy); 2786 if (!ret) { 2787 ret = cpufreq_start_governor(policy); 2788 if (!ret) { 2789 pr_debug("governor change\n"); 2790 return 0; 2791 } 2792 cpufreq_exit_governor(policy); 2793 } 2794 2795 /* new governor failed, so re-start old one */ 2796 pr_debug("starting governor %s failed\n", policy->governor->name); 2797 if (old_gov) { 2798 policy->governor = old_gov; 2799 if (cpufreq_init_governor(policy)) { 2800 policy->governor = NULL; 2801 } else if (cpufreq_start_governor(policy)) { 2802 cpufreq_exit_governor(policy); 2803 policy->governor = NULL; 2804 } 2805 } 2806 2807 return ret; 2808 } 2809 2810 static void cpufreq_policy_refresh(struct cpufreq_policy *policy) 2811 { 2812 guard(cpufreq_policy_write)(policy); 2813 2814 /* 2815 * BIOS might change freq behind our back 2816 * -> ask driver for current freq and notify governors about a change 2817 */ 2818 if (cpufreq_driver->get && has_target() && 2819 (cpufreq_is_suspended(policy) || 2820 WARN_ON(!cpufreq_verify_current_freq(policy, false)))) 2821 return; 2822 2823 refresh_frequency_limits(policy); 2824 } 2825 2826 /** 2827 * cpufreq_update_policy - Re-evaluate an existing cpufreq policy. 2828 * @cpu: CPU to re-evaluate the policy for. 2829 * 2830 * Update the current frequency for the cpufreq policy of @cpu and use 2831 * cpufreq_set_policy() to re-apply the min and max limits, which triggers the 2832 * evaluation of policy notifiers and the cpufreq driver's ->verify() callback 2833 * for the policy in question, among other things. 2834 */ 2835 void cpufreq_update_policy(unsigned int cpu) 2836 { 2837 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 2838 if (!policy) 2839 return; 2840 2841 cpufreq_policy_refresh(policy); 2842 } 2843 EXPORT_SYMBOL(cpufreq_update_policy); 2844 2845 /** 2846 * cpufreq_update_limits - Update policy limits for a given CPU. 2847 * @cpu: CPU to update the policy limits for. 2848 * 2849 * Invoke the driver's ->update_limits callback if present or call 2850 * cpufreq_policy_refresh() for @cpu. 2851 */ 2852 void cpufreq_update_limits(unsigned int cpu) 2853 { 2854 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 2855 if (!policy) 2856 return; 2857 2858 if (cpufreq_driver->update_limits) 2859 cpufreq_driver->update_limits(policy); 2860 else 2861 cpufreq_policy_refresh(policy); 2862 } 2863 EXPORT_SYMBOL_GPL(cpufreq_update_limits); 2864 2865 /********************************************************************* 2866 * BOOST * 2867 *********************************************************************/ 2868 int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state) 2869 { 2870 int ret; 2871 2872 if (!policy->freq_table) 2873 return -ENXIO; 2874 2875 ret = cpufreq_frequency_table_cpuinfo(policy); 2876 if (ret) 2877 pr_err("%s: Policy frequency update failed\n", __func__); 2878 2879 return ret; 2880 } 2881 EXPORT_SYMBOL_GPL(cpufreq_boost_set_sw); 2882 2883 static DEFINE_MUTEX(cpufreq_boost_mutex); 2884 2885 static int cpufreq_boost_trigger_state(int state) 2886 { 2887 struct cpufreq_policy *policy; 2888 unsigned long flags; 2889 bool boost_enabled; 2890 int ret = -EOPNOTSUPP; 2891 2892 guard(mutex)(&cpufreq_boost_mutex); 2893 2894 /* 2895 * Don't compare 'cpufreq_driver->boost_enabled' with 'state' here to 2896 * make sure all policies are in sync with global boost flag. 2897 */ 2898 2899 write_lock_irqsave(&cpufreq_driver_lock, flags); 2900 boost_enabled = cpufreq_driver->boost_enabled; 2901 cpufreq_driver->boost_enabled = state; 2902 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2903 2904 cpus_read_lock(); 2905 for_each_active_policy(policy) { 2906 if (!policy->boost_supported) 2907 continue; 2908 2909 guard(cpufreq_policy_write)(policy); 2910 2911 ret = policy_set_boost(policy, state); 2912 if (unlikely(ret)) 2913 break; 2914 } 2915 2916 cpus_read_unlock(); 2917 2918 if (likely(!ret)) 2919 return 0; 2920 2921 write_lock_irqsave(&cpufreq_driver_lock, flags); 2922 cpufreq_driver->boost_enabled = boost_enabled; 2923 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2924 2925 pr_err("%s: Cannot %s BOOST\n", 2926 __func__, str_enable_disable(state)); 2927 2928 return ret; 2929 } 2930 2931 static bool cpufreq_boost_supported(void) 2932 { 2933 return cpufreq_driver->set_boost; 2934 } 2935 2936 static int create_boost_sysfs_file(void) 2937 { 2938 int ret; 2939 2940 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr); 2941 if (ret) 2942 pr_err("%s: cannot register global BOOST sysfs file\n", 2943 __func__); 2944 2945 return ret; 2946 } 2947 2948 static void remove_boost_sysfs_file(void) 2949 { 2950 if (cpufreq_boost_supported()) 2951 sysfs_remove_file(cpufreq_global_kobject, &boost.attr); 2952 } 2953 2954 bool cpufreq_boost_enabled(void) 2955 { 2956 return cpufreq_driver->boost_enabled; 2957 } 2958 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled); 2959 2960 /********************************************************************* 2961 * REGISTER / UNREGISTER CPUFREQ DRIVER * 2962 *********************************************************************/ 2963 static enum cpuhp_state hp_online; 2964 2965 static int cpuhp_cpufreq_online(unsigned int cpu) 2966 { 2967 cpufreq_online(cpu); 2968 2969 return 0; 2970 } 2971 2972 static int cpuhp_cpufreq_offline(unsigned int cpu) 2973 { 2974 cpufreq_offline(cpu); 2975 2976 return 0; 2977 } 2978 2979 /** 2980 * cpufreq_register_driver - register a CPU Frequency driver 2981 * @driver_data: A struct cpufreq_driver containing the values# 2982 * submitted by the CPU Frequency driver. 2983 * 2984 * Registers a CPU Frequency driver to this core code. This code 2985 * returns zero on success, -EEXIST when another driver got here first 2986 * (and isn't unregistered in the meantime). 2987 * 2988 */ 2989 int cpufreq_register_driver(struct cpufreq_driver *driver_data) 2990 { 2991 unsigned long flags; 2992 int ret; 2993 2994 if (cpufreq_disabled()) 2995 return -ENODEV; 2996 2997 /* 2998 * The cpufreq core depends heavily on the availability of device 2999 * structure, make sure they are available before proceeding further. 3000 */ 3001 if (!get_cpu_device(0)) 3002 return -EPROBE_DEFER; 3003 3004 if (!driver_data || !driver_data->verify || !driver_data->init || 3005 (driver_data->target_index && driver_data->target) || 3006 (!!driver_data->setpolicy == (driver_data->target_index || driver_data->target)) || 3007 (!driver_data->get_intermediate != !driver_data->target_intermediate) || 3008 (!driver_data->online != !driver_data->offline) || 3009 (driver_data->adjust_perf && !driver_data->fast_switch)) 3010 return -EINVAL; 3011 3012 pr_debug("trying to register driver %s\n", driver_data->name); 3013 3014 /* Protect against concurrent CPU online/offline. */ 3015 cpus_read_lock(); 3016 3017 write_lock_irqsave(&cpufreq_driver_lock, flags); 3018 if (cpufreq_driver) { 3019 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 3020 ret = -EEXIST; 3021 goto out; 3022 } 3023 cpufreq_driver = driver_data; 3024 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 3025 3026 if (driver_data->setpolicy) 3027 driver_data->flags |= CPUFREQ_CONST_LOOPS; 3028 3029 if (cpufreq_boost_supported()) { 3030 ret = create_boost_sysfs_file(); 3031 if (ret) 3032 goto err_null_driver; 3033 } 3034 3035 /* 3036 * Mark support for the scheduler's frequency invariance engine for 3037 * drivers that implement target(), target_index() or fast_switch(). 3038 */ 3039 if (!cpufreq_driver->setpolicy) { 3040 static_branch_enable_cpuslocked(&cpufreq_freq_invariance); 3041 pr_debug("cpufreq: supports frequency invariance\n"); 3042 } 3043 3044 ret = subsys_interface_register(&cpufreq_interface); 3045 if (ret) 3046 goto err_boost_unreg; 3047 3048 if (unlikely(list_empty(&cpufreq_policy_list))) { 3049 /* if all ->init() calls failed, unregister */ 3050 ret = -ENODEV; 3051 pr_debug("%s: No CPU initialized for driver %s\n", __func__, 3052 driver_data->name); 3053 goto err_if_unreg; 3054 } 3055 3056 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, 3057 "cpufreq:online", 3058 cpuhp_cpufreq_online, 3059 cpuhp_cpufreq_offline); 3060 if (ret < 0) 3061 goto err_if_unreg; 3062 hp_online = ret; 3063 ret = 0; 3064 3065 pr_debug("driver %s up and running\n", driver_data->name); 3066 goto out; 3067 3068 err_if_unreg: 3069 subsys_interface_unregister(&cpufreq_interface); 3070 err_boost_unreg: 3071 if (!cpufreq_driver->setpolicy) 3072 static_branch_disable_cpuslocked(&cpufreq_freq_invariance); 3073 remove_boost_sysfs_file(); 3074 err_null_driver: 3075 write_lock_irqsave(&cpufreq_driver_lock, flags); 3076 cpufreq_driver = NULL; 3077 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 3078 out: 3079 cpus_read_unlock(); 3080 return ret; 3081 } 3082 EXPORT_SYMBOL_GPL(cpufreq_register_driver); 3083 3084 /* 3085 * cpufreq_unregister_driver - unregister the current CPUFreq driver 3086 * 3087 * Unregister the current CPUFreq driver. Only call this if you have 3088 * the right to do so, i.e. if you have succeeded in initialising before! 3089 * Returns zero if successful, and -EINVAL if the cpufreq_driver is 3090 * currently not initialised. 3091 */ 3092 void cpufreq_unregister_driver(struct cpufreq_driver *driver) 3093 { 3094 unsigned long flags; 3095 3096 if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver))) 3097 return; 3098 3099 pr_debug("unregistering driver %s\n", driver->name); 3100 3101 /* Protect against concurrent cpu hotplug */ 3102 cpus_read_lock(); 3103 subsys_interface_unregister(&cpufreq_interface); 3104 remove_boost_sysfs_file(); 3105 static_branch_disable_cpuslocked(&cpufreq_freq_invariance); 3106 cpuhp_remove_state_nocalls_cpuslocked(hp_online); 3107 3108 write_lock_irqsave(&cpufreq_driver_lock, flags); 3109 3110 cpufreq_driver = NULL; 3111 3112 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 3113 cpus_read_unlock(); 3114 } 3115 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver); 3116 3117 static int __init cpufreq_core_init(void) 3118 { 3119 struct cpufreq_governor *gov = cpufreq_default_governor(); 3120 struct device *dev_root; 3121 3122 if (cpufreq_disabled()) 3123 return -ENODEV; 3124 3125 dev_root = bus_get_dev_root(&cpu_subsys); 3126 if (dev_root) { 3127 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj); 3128 put_device(dev_root); 3129 } 3130 BUG_ON(!cpufreq_global_kobject); 3131 3132 if (!strlen(default_governor)) 3133 strscpy(default_governor, gov->name, CPUFREQ_NAME_LEN); 3134 3135 return 0; 3136 } 3137 3138 static bool cpufreq_policy_is_good_for_eas(unsigned int cpu) 3139 { 3140 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 3141 if (!policy) { 3142 pr_debug("cpufreq policy not set for CPU: %d\n", cpu); 3143 return false; 3144 } 3145 3146 return sugov_is_governor(policy); 3147 } 3148 3149 bool cpufreq_ready_for_eas(const struct cpumask *cpu_mask) 3150 { 3151 unsigned int cpu; 3152 3153 /* Do not attempt EAS if schedutil is not being used. */ 3154 for_each_cpu(cpu, cpu_mask) { 3155 if (!cpufreq_policy_is_good_for_eas(cpu)) { 3156 pr_debug("rd %*pbl: schedutil is mandatory for EAS\n", 3157 cpumask_pr_args(cpu_mask)); 3158 return false; 3159 } 3160 } 3161 3162 return true; 3163 } 3164 3165 module_param(off, int, 0444); 3166 module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444); 3167 core_initcall(cpufreq_core_init); 3168