1 /* 2 * linux/drivers/cpufreq/cpufreq.c 3 * 4 * Copyright (C) 2001 Russell King 5 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> 6 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org> 7 * 8 * Oct 2005 - Ashok Raj <ashok.raj@intel.com> 9 * Added handling for CPU hotplug 10 * Feb 2006 - Jacob Shin <jacob.shin@amd.com> 11 * Fix handling for CPU hotplug -- affected CPUs 12 * 13 * This program is free software; you can redistribute it and/or modify 14 * it under the terms of the GNU General Public License version 2 as 15 * published by the Free Software Foundation. 16 */ 17 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <linux/cpu.h> 21 #include <linux/cpufreq.h> 22 #include <linux/delay.h> 23 #include <linux/device.h> 24 #include <linux/init.h> 25 #include <linux/kernel_stat.h> 26 #include <linux/module.h> 27 #include <linux/mutex.h> 28 #include <linux/slab.h> 29 #include <linux/suspend.h> 30 #include <linux/syscore_ops.h> 31 #include <linux/tick.h> 32 #include <trace/events/power.h> 33 34 static LIST_HEAD(cpufreq_policy_list); 35 36 static inline bool policy_is_inactive(struct cpufreq_policy *policy) 37 { 38 return cpumask_empty(policy->cpus); 39 } 40 41 /* Macros to iterate over CPU policies */ 42 #define for_each_suitable_policy(__policy, __active) \ 43 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \ 44 if ((__active) == !policy_is_inactive(__policy)) 45 46 #define for_each_active_policy(__policy) \ 47 for_each_suitable_policy(__policy, true) 48 #define for_each_inactive_policy(__policy) \ 49 for_each_suitable_policy(__policy, false) 50 51 #define for_each_policy(__policy) \ 52 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) 53 54 /* Iterate over governors */ 55 static LIST_HEAD(cpufreq_governor_list); 56 #define for_each_governor(__governor) \ 57 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list) 58 59 /** 60 * The "cpufreq driver" - the arch- or hardware-dependent low 61 * level driver of CPUFreq support, and its spinlock. This lock 62 * also protects the cpufreq_cpu_data array. 63 */ 64 static struct cpufreq_driver *cpufreq_driver; 65 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); 66 static DEFINE_RWLOCK(cpufreq_driver_lock); 67 68 /* Flag to suspend/resume CPUFreq governors */ 69 static bool cpufreq_suspended; 70 71 static inline bool has_target(void) 72 { 73 return cpufreq_driver->target_index || cpufreq_driver->target; 74 } 75 76 /* internal prototypes */ 77 static unsigned int __cpufreq_get(struct cpufreq_policy *policy); 78 static int cpufreq_init_governor(struct cpufreq_policy *policy); 79 static void cpufreq_exit_governor(struct cpufreq_policy *policy); 80 static int cpufreq_start_governor(struct cpufreq_policy *policy); 81 static void cpufreq_stop_governor(struct cpufreq_policy *policy); 82 static void cpufreq_governor_limits(struct cpufreq_policy *policy); 83 84 /** 85 * Two notifier lists: the "policy" list is involved in the 86 * validation process for a new CPU frequency policy; the 87 * "transition" list for kernel code that needs to handle 88 * changes to devices when the CPU clock speed changes. 89 * The mutex locks both lists. 90 */ 91 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list); 92 static struct srcu_notifier_head cpufreq_transition_notifier_list; 93 94 static bool init_cpufreq_transition_notifier_list_called; 95 static int __init init_cpufreq_transition_notifier_list(void) 96 { 97 srcu_init_notifier_head(&cpufreq_transition_notifier_list); 98 init_cpufreq_transition_notifier_list_called = true; 99 return 0; 100 } 101 pure_initcall(init_cpufreq_transition_notifier_list); 102 103 static int off __read_mostly; 104 static int cpufreq_disabled(void) 105 { 106 return off; 107 } 108 void disable_cpufreq(void) 109 { 110 off = 1; 111 } 112 static DEFINE_MUTEX(cpufreq_governor_mutex); 113 114 bool have_governor_per_policy(void) 115 { 116 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY); 117 } 118 EXPORT_SYMBOL_GPL(have_governor_per_policy); 119 120 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) 121 { 122 if (have_governor_per_policy()) 123 return &policy->kobj; 124 else 125 return cpufreq_global_kobject; 126 } 127 EXPORT_SYMBOL_GPL(get_governor_parent_kobj); 128 129 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) 130 { 131 u64 idle_time; 132 u64 cur_wall_time; 133 u64 busy_time; 134 135 cur_wall_time = jiffies64_to_nsecs(get_jiffies_64()); 136 137 busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; 138 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; 139 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; 140 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; 141 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; 142 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; 143 144 idle_time = cur_wall_time - busy_time; 145 if (wall) 146 *wall = div_u64(cur_wall_time, NSEC_PER_USEC); 147 148 return div_u64(idle_time, NSEC_PER_USEC); 149 } 150 151 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) 152 { 153 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL); 154 155 if (idle_time == -1ULL) 156 return get_cpu_idle_time_jiffy(cpu, wall); 157 else if (!io_busy) 158 idle_time += get_cpu_iowait_time_us(cpu, wall); 159 160 return idle_time; 161 } 162 EXPORT_SYMBOL_GPL(get_cpu_idle_time); 163 164 /* 165 * This is a generic cpufreq init() routine which can be used by cpufreq 166 * drivers of SMP systems. It will do following: 167 * - validate & show freq table passed 168 * - set policies transition latency 169 * - policy->cpus with all possible CPUs 170 */ 171 int cpufreq_generic_init(struct cpufreq_policy *policy, 172 struct cpufreq_frequency_table *table, 173 unsigned int transition_latency) 174 { 175 int ret; 176 177 ret = cpufreq_table_validate_and_show(policy, table); 178 if (ret) { 179 pr_err("%s: invalid frequency table: %d\n", __func__, ret); 180 return ret; 181 } 182 183 policy->cpuinfo.transition_latency = transition_latency; 184 185 /* 186 * The driver only supports the SMP configuration where all processors 187 * share the clock and voltage and clock. 188 */ 189 cpumask_setall(policy->cpus); 190 191 return 0; 192 } 193 EXPORT_SYMBOL_GPL(cpufreq_generic_init); 194 195 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu) 196 { 197 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 198 199 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL; 200 } 201 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw); 202 203 unsigned int cpufreq_generic_get(unsigned int cpu) 204 { 205 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); 206 207 if (!policy || IS_ERR(policy->clk)) { 208 pr_err("%s: No %s associated to cpu: %d\n", 209 __func__, policy ? "clk" : "policy", cpu); 210 return 0; 211 } 212 213 return clk_get_rate(policy->clk) / 1000; 214 } 215 EXPORT_SYMBOL_GPL(cpufreq_generic_get); 216 217 /** 218 * cpufreq_cpu_get: returns policy for a cpu and marks it busy. 219 * 220 * @cpu: cpu to find policy for. 221 * 222 * This returns policy for 'cpu', returns NULL if it doesn't exist. 223 * It also increments the kobject reference count to mark it busy and so would 224 * require a corresponding call to cpufreq_cpu_put() to decrement it back. 225 * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be 226 * freed as that depends on the kobj count. 227 * 228 * Return: A valid policy on success, otherwise NULL on failure. 229 */ 230 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) 231 { 232 struct cpufreq_policy *policy = NULL; 233 unsigned long flags; 234 235 if (WARN_ON(cpu >= nr_cpu_ids)) 236 return NULL; 237 238 /* get the cpufreq driver */ 239 read_lock_irqsave(&cpufreq_driver_lock, flags); 240 241 if (cpufreq_driver) { 242 /* get the CPU */ 243 policy = cpufreq_cpu_get_raw(cpu); 244 if (policy) 245 kobject_get(&policy->kobj); 246 } 247 248 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 249 250 return policy; 251 } 252 EXPORT_SYMBOL_GPL(cpufreq_cpu_get); 253 254 /** 255 * cpufreq_cpu_put: Decrements the usage count of a policy 256 * 257 * @policy: policy earlier returned by cpufreq_cpu_get(). 258 * 259 * This decrements the kobject reference count incremented earlier by calling 260 * cpufreq_cpu_get(). 261 */ 262 void cpufreq_cpu_put(struct cpufreq_policy *policy) 263 { 264 kobject_put(&policy->kobj); 265 } 266 EXPORT_SYMBOL_GPL(cpufreq_cpu_put); 267 268 /********************************************************************* 269 * EXTERNALLY AFFECTING FREQUENCY CHANGES * 270 *********************************************************************/ 271 272 /** 273 * adjust_jiffies - adjust the system "loops_per_jiffy" 274 * 275 * This function alters the system "loops_per_jiffy" for the clock 276 * speed change. Note that loops_per_jiffy cannot be updated on SMP 277 * systems as each CPU might be scaled differently. So, use the arch 278 * per-CPU loops_per_jiffy value wherever possible. 279 */ 280 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) 281 { 282 #ifndef CONFIG_SMP 283 static unsigned long l_p_j_ref; 284 static unsigned int l_p_j_ref_freq; 285 286 if (ci->flags & CPUFREQ_CONST_LOOPS) 287 return; 288 289 if (!l_p_j_ref_freq) { 290 l_p_j_ref = loops_per_jiffy; 291 l_p_j_ref_freq = ci->old; 292 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n", 293 l_p_j_ref, l_p_j_ref_freq); 294 } 295 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) { 296 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq, 297 ci->new); 298 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n", 299 loops_per_jiffy, ci->new); 300 } 301 #endif 302 } 303 304 static void __cpufreq_notify_transition(struct cpufreq_policy *policy, 305 struct cpufreq_freqs *freqs, unsigned int state) 306 { 307 BUG_ON(irqs_disabled()); 308 309 if (cpufreq_disabled()) 310 return; 311 312 freqs->flags = cpufreq_driver->flags; 313 pr_debug("notification %u of frequency transition to %u kHz\n", 314 state, freqs->new); 315 316 switch (state) { 317 318 case CPUFREQ_PRECHANGE: 319 /* detect if the driver reported a value as "old frequency" 320 * which is not equal to what the cpufreq core thinks is 321 * "old frequency". 322 */ 323 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) { 324 if ((policy) && (policy->cpu == freqs->cpu) && 325 (policy->cur) && (policy->cur != freqs->old)) { 326 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n", 327 freqs->old, policy->cur); 328 freqs->old = policy->cur; 329 } 330 } 331 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 332 CPUFREQ_PRECHANGE, freqs); 333 adjust_jiffies(CPUFREQ_PRECHANGE, freqs); 334 break; 335 336 case CPUFREQ_POSTCHANGE: 337 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs); 338 pr_debug("FREQ: %lu - CPU: %lu\n", 339 (unsigned long)freqs->new, (unsigned long)freqs->cpu); 340 trace_cpu_frequency(freqs->new, freqs->cpu); 341 cpufreq_stats_record_transition(policy, freqs->new); 342 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 343 CPUFREQ_POSTCHANGE, freqs); 344 if (likely(policy) && likely(policy->cpu == freqs->cpu)) 345 policy->cur = freqs->new; 346 break; 347 } 348 } 349 350 /** 351 * cpufreq_notify_transition - call notifier chain and adjust_jiffies 352 * on frequency transition. 353 * 354 * This function calls the transition notifiers and the "adjust_jiffies" 355 * function. It is called twice on all CPU frequency changes that have 356 * external effects. 357 */ 358 static void cpufreq_notify_transition(struct cpufreq_policy *policy, 359 struct cpufreq_freqs *freqs, unsigned int state) 360 { 361 for_each_cpu(freqs->cpu, policy->cpus) 362 __cpufreq_notify_transition(policy, freqs, state); 363 } 364 365 /* Do post notifications when there are chances that transition has failed */ 366 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy, 367 struct cpufreq_freqs *freqs, int transition_failed) 368 { 369 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 370 if (!transition_failed) 371 return; 372 373 swap(freqs->old, freqs->new); 374 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 375 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 376 } 377 378 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, 379 struct cpufreq_freqs *freqs) 380 { 381 382 /* 383 * Catch double invocations of _begin() which lead to self-deadlock. 384 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core 385 * doesn't invoke _begin() on their behalf, and hence the chances of 386 * double invocations are very low. Moreover, there are scenarios 387 * where these checks can emit false-positive warnings in these 388 * drivers; so we avoid that by skipping them altogether. 389 */ 390 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION) 391 && current == policy->transition_task); 392 393 wait: 394 wait_event(policy->transition_wait, !policy->transition_ongoing); 395 396 spin_lock(&policy->transition_lock); 397 398 if (unlikely(policy->transition_ongoing)) { 399 spin_unlock(&policy->transition_lock); 400 goto wait; 401 } 402 403 policy->transition_ongoing = true; 404 policy->transition_task = current; 405 406 spin_unlock(&policy->transition_lock); 407 408 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 409 } 410 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin); 411 412 void cpufreq_freq_transition_end(struct cpufreq_policy *policy, 413 struct cpufreq_freqs *freqs, int transition_failed) 414 { 415 if (unlikely(WARN_ON(!policy->transition_ongoing))) 416 return; 417 418 cpufreq_notify_post_transition(policy, freqs, transition_failed); 419 420 policy->transition_ongoing = false; 421 policy->transition_task = NULL; 422 423 wake_up(&policy->transition_wait); 424 } 425 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end); 426 427 /* 428 * Fast frequency switching status count. Positive means "enabled", negative 429 * means "disabled" and 0 means "not decided yet". 430 */ 431 static int cpufreq_fast_switch_count; 432 static DEFINE_MUTEX(cpufreq_fast_switch_lock); 433 434 static void cpufreq_list_transition_notifiers(void) 435 { 436 struct notifier_block *nb; 437 438 pr_info("Registered transition notifiers:\n"); 439 440 mutex_lock(&cpufreq_transition_notifier_list.mutex); 441 442 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next) 443 pr_info("%pF\n", nb->notifier_call); 444 445 mutex_unlock(&cpufreq_transition_notifier_list.mutex); 446 } 447 448 /** 449 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy. 450 * @policy: cpufreq policy to enable fast frequency switching for. 451 * 452 * Try to enable fast frequency switching for @policy. 453 * 454 * The attempt will fail if there is at least one transition notifier registered 455 * at this point, as fast frequency switching is quite fundamentally at odds 456 * with transition notifiers. Thus if successful, it will make registration of 457 * transition notifiers fail going forward. 458 */ 459 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy) 460 { 461 lockdep_assert_held(&policy->rwsem); 462 463 if (!policy->fast_switch_possible) 464 return; 465 466 mutex_lock(&cpufreq_fast_switch_lock); 467 if (cpufreq_fast_switch_count >= 0) { 468 cpufreq_fast_switch_count++; 469 policy->fast_switch_enabled = true; 470 } else { 471 pr_warn("CPU%u: Fast frequency switching not enabled\n", 472 policy->cpu); 473 cpufreq_list_transition_notifiers(); 474 } 475 mutex_unlock(&cpufreq_fast_switch_lock); 476 } 477 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch); 478 479 /** 480 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy. 481 * @policy: cpufreq policy to disable fast frequency switching for. 482 */ 483 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) 484 { 485 mutex_lock(&cpufreq_fast_switch_lock); 486 if (policy->fast_switch_enabled) { 487 policy->fast_switch_enabled = false; 488 if (!WARN_ON(cpufreq_fast_switch_count <= 0)) 489 cpufreq_fast_switch_count--; 490 } 491 mutex_unlock(&cpufreq_fast_switch_lock); 492 } 493 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); 494 495 /** 496 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported 497 * one. 498 * @target_freq: target frequency to resolve. 499 * 500 * The target to driver frequency mapping is cached in the policy. 501 * 502 * Return: Lowest driver-supported frequency greater than or equal to the 503 * given target_freq, subject to policy (min/max) and driver limitations. 504 */ 505 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy, 506 unsigned int target_freq) 507 { 508 target_freq = clamp_val(target_freq, policy->min, policy->max); 509 policy->cached_target_freq = target_freq; 510 511 if (cpufreq_driver->target_index) { 512 int idx; 513 514 idx = cpufreq_frequency_table_target(policy, target_freq, 515 CPUFREQ_RELATION_L); 516 policy->cached_resolved_idx = idx; 517 return policy->freq_table[idx].frequency; 518 } 519 520 if (cpufreq_driver->resolve_freq) 521 return cpufreq_driver->resolve_freq(policy, target_freq); 522 523 return target_freq; 524 } 525 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq); 526 527 /********************************************************************* 528 * SYSFS INTERFACE * 529 *********************************************************************/ 530 static ssize_t show_boost(struct kobject *kobj, 531 struct attribute *attr, char *buf) 532 { 533 return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled); 534 } 535 536 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr, 537 const char *buf, size_t count) 538 { 539 int ret, enable; 540 541 ret = sscanf(buf, "%d", &enable); 542 if (ret != 1 || enable < 0 || enable > 1) 543 return -EINVAL; 544 545 if (cpufreq_boost_trigger_state(enable)) { 546 pr_err("%s: Cannot %s BOOST!\n", 547 __func__, enable ? "enable" : "disable"); 548 return -EINVAL; 549 } 550 551 pr_debug("%s: cpufreq BOOST %s\n", 552 __func__, enable ? "enabled" : "disabled"); 553 554 return count; 555 } 556 define_one_global_rw(boost); 557 558 static struct cpufreq_governor *find_governor(const char *str_governor) 559 { 560 struct cpufreq_governor *t; 561 562 for_each_governor(t) 563 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN)) 564 return t; 565 566 return NULL; 567 } 568 569 /** 570 * cpufreq_parse_governor - parse a governor string 571 */ 572 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy, 573 struct cpufreq_governor **governor) 574 { 575 int err = -EINVAL; 576 577 if (cpufreq_driver->setpolicy) { 578 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) { 579 *policy = CPUFREQ_POLICY_PERFORMANCE; 580 err = 0; 581 } else if (!strncasecmp(str_governor, "powersave", 582 CPUFREQ_NAME_LEN)) { 583 *policy = CPUFREQ_POLICY_POWERSAVE; 584 err = 0; 585 } 586 } else { 587 struct cpufreq_governor *t; 588 589 mutex_lock(&cpufreq_governor_mutex); 590 591 t = find_governor(str_governor); 592 593 if (t == NULL) { 594 int ret; 595 596 mutex_unlock(&cpufreq_governor_mutex); 597 ret = request_module("cpufreq_%s", str_governor); 598 mutex_lock(&cpufreq_governor_mutex); 599 600 if (ret == 0) 601 t = find_governor(str_governor); 602 } 603 604 if (t != NULL) { 605 *governor = t; 606 err = 0; 607 } 608 609 mutex_unlock(&cpufreq_governor_mutex); 610 } 611 return err; 612 } 613 614 /** 615 * cpufreq_per_cpu_attr_read() / show_##file_name() - 616 * print out cpufreq information 617 * 618 * Write out information from cpufreq_driver->policy[cpu]; object must be 619 * "unsigned int". 620 */ 621 622 #define show_one(file_name, object) \ 623 static ssize_t show_##file_name \ 624 (struct cpufreq_policy *policy, char *buf) \ 625 { \ 626 return sprintf(buf, "%u\n", policy->object); \ 627 } 628 629 show_one(cpuinfo_min_freq, cpuinfo.min_freq); 630 show_one(cpuinfo_max_freq, cpuinfo.max_freq); 631 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency); 632 show_one(scaling_min_freq, min); 633 show_one(scaling_max_freq, max); 634 635 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf) 636 { 637 ssize_t ret; 638 639 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) 640 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu)); 641 else 642 ret = sprintf(buf, "%u\n", policy->cur); 643 return ret; 644 } 645 646 static int cpufreq_set_policy(struct cpufreq_policy *policy, 647 struct cpufreq_policy *new_policy); 648 649 /** 650 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access 651 */ 652 #define store_one(file_name, object) \ 653 static ssize_t store_##file_name \ 654 (struct cpufreq_policy *policy, const char *buf, size_t count) \ 655 { \ 656 int ret, temp; \ 657 struct cpufreq_policy new_policy; \ 658 \ 659 memcpy(&new_policy, policy, sizeof(*policy)); \ 660 \ 661 ret = sscanf(buf, "%u", &new_policy.object); \ 662 if (ret != 1) \ 663 return -EINVAL; \ 664 \ 665 temp = new_policy.object; \ 666 ret = cpufreq_set_policy(policy, &new_policy); \ 667 if (!ret) \ 668 policy->user_policy.object = temp; \ 669 \ 670 return ret ? ret : count; \ 671 } 672 673 store_one(scaling_min_freq, min); 674 store_one(scaling_max_freq, max); 675 676 /** 677 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware 678 */ 679 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, 680 char *buf) 681 { 682 unsigned int cur_freq = __cpufreq_get(policy); 683 684 if (cur_freq) 685 return sprintf(buf, "%u\n", cur_freq); 686 687 return sprintf(buf, "<unknown>\n"); 688 } 689 690 /** 691 * show_scaling_governor - show the current policy for the specified CPU 692 */ 693 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) 694 { 695 if (policy->policy == CPUFREQ_POLICY_POWERSAVE) 696 return sprintf(buf, "powersave\n"); 697 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) 698 return sprintf(buf, "performance\n"); 699 else if (policy->governor) 700 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", 701 policy->governor->name); 702 return -EINVAL; 703 } 704 705 /** 706 * store_scaling_governor - store policy for the specified CPU 707 */ 708 static ssize_t store_scaling_governor(struct cpufreq_policy *policy, 709 const char *buf, size_t count) 710 { 711 int ret; 712 char str_governor[16]; 713 struct cpufreq_policy new_policy; 714 715 memcpy(&new_policy, policy, sizeof(*policy)); 716 717 ret = sscanf(buf, "%15s", str_governor); 718 if (ret != 1) 719 return -EINVAL; 720 721 if (cpufreq_parse_governor(str_governor, &new_policy.policy, 722 &new_policy.governor)) 723 return -EINVAL; 724 725 ret = cpufreq_set_policy(policy, &new_policy); 726 return ret ? ret : count; 727 } 728 729 /** 730 * show_scaling_driver - show the cpufreq driver currently loaded 731 */ 732 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf) 733 { 734 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name); 735 } 736 737 /** 738 * show_scaling_available_governors - show the available CPUfreq governors 739 */ 740 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy, 741 char *buf) 742 { 743 ssize_t i = 0; 744 struct cpufreq_governor *t; 745 746 if (!has_target()) { 747 i += sprintf(buf, "performance powersave"); 748 goto out; 749 } 750 751 for_each_governor(t) { 752 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) 753 - (CPUFREQ_NAME_LEN + 2))) 754 goto out; 755 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name); 756 } 757 out: 758 i += sprintf(&buf[i], "\n"); 759 return i; 760 } 761 762 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf) 763 { 764 ssize_t i = 0; 765 unsigned int cpu; 766 767 for_each_cpu(cpu, mask) { 768 if (i) 769 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " "); 770 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu); 771 if (i >= (PAGE_SIZE - 5)) 772 break; 773 } 774 i += sprintf(&buf[i], "\n"); 775 return i; 776 } 777 EXPORT_SYMBOL_GPL(cpufreq_show_cpus); 778 779 /** 780 * show_related_cpus - show the CPUs affected by each transition even if 781 * hw coordination is in use 782 */ 783 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf) 784 { 785 return cpufreq_show_cpus(policy->related_cpus, buf); 786 } 787 788 /** 789 * show_affected_cpus - show the CPUs affected by each transition 790 */ 791 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf) 792 { 793 return cpufreq_show_cpus(policy->cpus, buf); 794 } 795 796 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy, 797 const char *buf, size_t count) 798 { 799 unsigned int freq = 0; 800 unsigned int ret; 801 802 if (!policy->governor || !policy->governor->store_setspeed) 803 return -EINVAL; 804 805 ret = sscanf(buf, "%u", &freq); 806 if (ret != 1) 807 return -EINVAL; 808 809 policy->governor->store_setspeed(policy, freq); 810 811 return count; 812 } 813 814 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf) 815 { 816 if (!policy->governor || !policy->governor->show_setspeed) 817 return sprintf(buf, "<unsupported>\n"); 818 819 return policy->governor->show_setspeed(policy, buf); 820 } 821 822 /** 823 * show_bios_limit - show the current cpufreq HW/BIOS limitation 824 */ 825 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf) 826 { 827 unsigned int limit; 828 int ret; 829 if (cpufreq_driver->bios_limit) { 830 ret = cpufreq_driver->bios_limit(policy->cpu, &limit); 831 if (!ret) 832 return sprintf(buf, "%u\n", limit); 833 } 834 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq); 835 } 836 837 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400); 838 cpufreq_freq_attr_ro(cpuinfo_min_freq); 839 cpufreq_freq_attr_ro(cpuinfo_max_freq); 840 cpufreq_freq_attr_ro(cpuinfo_transition_latency); 841 cpufreq_freq_attr_ro(scaling_available_governors); 842 cpufreq_freq_attr_ro(scaling_driver); 843 cpufreq_freq_attr_ro(scaling_cur_freq); 844 cpufreq_freq_attr_ro(bios_limit); 845 cpufreq_freq_attr_ro(related_cpus); 846 cpufreq_freq_attr_ro(affected_cpus); 847 cpufreq_freq_attr_rw(scaling_min_freq); 848 cpufreq_freq_attr_rw(scaling_max_freq); 849 cpufreq_freq_attr_rw(scaling_governor); 850 cpufreq_freq_attr_rw(scaling_setspeed); 851 852 static struct attribute *default_attrs[] = { 853 &cpuinfo_min_freq.attr, 854 &cpuinfo_max_freq.attr, 855 &cpuinfo_transition_latency.attr, 856 &scaling_min_freq.attr, 857 &scaling_max_freq.attr, 858 &affected_cpus.attr, 859 &related_cpus.attr, 860 &scaling_governor.attr, 861 &scaling_driver.attr, 862 &scaling_available_governors.attr, 863 &scaling_setspeed.attr, 864 NULL 865 }; 866 867 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj) 868 #define to_attr(a) container_of(a, struct freq_attr, attr) 869 870 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) 871 { 872 struct cpufreq_policy *policy = to_policy(kobj); 873 struct freq_attr *fattr = to_attr(attr); 874 ssize_t ret; 875 876 down_read(&policy->rwsem); 877 ret = fattr->show(policy, buf); 878 up_read(&policy->rwsem); 879 880 return ret; 881 } 882 883 static ssize_t store(struct kobject *kobj, struct attribute *attr, 884 const char *buf, size_t count) 885 { 886 struct cpufreq_policy *policy = to_policy(kobj); 887 struct freq_attr *fattr = to_attr(attr); 888 ssize_t ret = -EINVAL; 889 890 get_online_cpus(); 891 892 if (cpu_online(policy->cpu)) { 893 down_write(&policy->rwsem); 894 ret = fattr->store(policy, buf, count); 895 up_write(&policy->rwsem); 896 } 897 898 put_online_cpus(); 899 900 return ret; 901 } 902 903 static void cpufreq_sysfs_release(struct kobject *kobj) 904 { 905 struct cpufreq_policy *policy = to_policy(kobj); 906 pr_debug("last reference is dropped\n"); 907 complete(&policy->kobj_unregister); 908 } 909 910 static const struct sysfs_ops sysfs_ops = { 911 .show = show, 912 .store = store, 913 }; 914 915 static struct kobj_type ktype_cpufreq = { 916 .sysfs_ops = &sysfs_ops, 917 .default_attrs = default_attrs, 918 .release = cpufreq_sysfs_release, 919 }; 920 921 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu) 922 { 923 struct device *dev = get_cpu_device(cpu); 924 925 if (!dev) 926 return; 927 928 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus)) 929 return; 930 931 dev_dbg(dev, "%s: Adding symlink\n", __func__); 932 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq")) 933 dev_err(dev, "cpufreq symlink creation failed\n"); 934 } 935 936 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, 937 struct device *dev) 938 { 939 dev_dbg(dev, "%s: Removing symlink\n", __func__); 940 sysfs_remove_link(&dev->kobj, "cpufreq"); 941 } 942 943 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy) 944 { 945 struct freq_attr **drv_attr; 946 int ret = 0; 947 948 /* set up files for this cpu device */ 949 drv_attr = cpufreq_driver->attr; 950 while (drv_attr && *drv_attr) { 951 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr)); 952 if (ret) 953 return ret; 954 drv_attr++; 955 } 956 if (cpufreq_driver->get) { 957 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr); 958 if (ret) 959 return ret; 960 } 961 962 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr); 963 if (ret) 964 return ret; 965 966 if (cpufreq_driver->bios_limit) { 967 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr); 968 if (ret) 969 return ret; 970 } 971 972 return 0; 973 } 974 975 __weak struct cpufreq_governor *cpufreq_default_governor(void) 976 { 977 return NULL; 978 } 979 980 static int cpufreq_init_policy(struct cpufreq_policy *policy) 981 { 982 struct cpufreq_governor *gov = NULL; 983 struct cpufreq_policy new_policy; 984 985 memcpy(&new_policy, policy, sizeof(*policy)); 986 987 /* Update governor of new_policy to the governor used before hotplug */ 988 gov = find_governor(policy->last_governor); 989 if (gov) { 990 pr_debug("Restoring governor %s for cpu %d\n", 991 policy->governor->name, policy->cpu); 992 } else { 993 gov = cpufreq_default_governor(); 994 if (!gov) 995 return -ENODATA; 996 } 997 998 new_policy.governor = gov; 999 1000 /* Use the default policy if there is no last_policy. */ 1001 if (cpufreq_driver->setpolicy) { 1002 if (policy->last_policy) 1003 new_policy.policy = policy->last_policy; 1004 else 1005 cpufreq_parse_governor(gov->name, &new_policy.policy, 1006 NULL); 1007 } 1008 /* set default policy */ 1009 return cpufreq_set_policy(policy, &new_policy); 1010 } 1011 1012 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu) 1013 { 1014 int ret = 0; 1015 1016 /* Has this CPU been taken care of already? */ 1017 if (cpumask_test_cpu(cpu, policy->cpus)) 1018 return 0; 1019 1020 down_write(&policy->rwsem); 1021 if (has_target()) 1022 cpufreq_stop_governor(policy); 1023 1024 cpumask_set_cpu(cpu, policy->cpus); 1025 1026 if (has_target()) { 1027 ret = cpufreq_start_governor(policy); 1028 if (ret) 1029 pr_err("%s: Failed to start governor\n", __func__); 1030 } 1031 up_write(&policy->rwsem); 1032 return ret; 1033 } 1034 1035 static void handle_update(struct work_struct *work) 1036 { 1037 struct cpufreq_policy *policy = 1038 container_of(work, struct cpufreq_policy, update); 1039 unsigned int cpu = policy->cpu; 1040 pr_debug("handle_update for cpu %u called\n", cpu); 1041 cpufreq_update_policy(cpu); 1042 } 1043 1044 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) 1045 { 1046 struct cpufreq_policy *policy; 1047 int ret; 1048 1049 policy = kzalloc(sizeof(*policy), GFP_KERNEL); 1050 if (!policy) 1051 return NULL; 1052 1053 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) 1054 goto err_free_policy; 1055 1056 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) 1057 goto err_free_cpumask; 1058 1059 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL)) 1060 goto err_free_rcpumask; 1061 1062 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, 1063 cpufreq_global_kobject, "policy%u", cpu); 1064 if (ret) { 1065 pr_err("%s: failed to init policy->kobj: %d\n", __func__, ret); 1066 goto err_free_real_cpus; 1067 } 1068 1069 INIT_LIST_HEAD(&policy->policy_list); 1070 init_rwsem(&policy->rwsem); 1071 spin_lock_init(&policy->transition_lock); 1072 init_waitqueue_head(&policy->transition_wait); 1073 init_completion(&policy->kobj_unregister); 1074 INIT_WORK(&policy->update, handle_update); 1075 1076 policy->cpu = cpu; 1077 return policy; 1078 1079 err_free_real_cpus: 1080 free_cpumask_var(policy->real_cpus); 1081 err_free_rcpumask: 1082 free_cpumask_var(policy->related_cpus); 1083 err_free_cpumask: 1084 free_cpumask_var(policy->cpus); 1085 err_free_policy: 1086 kfree(policy); 1087 1088 return NULL; 1089 } 1090 1091 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) 1092 { 1093 struct kobject *kobj; 1094 struct completion *cmp; 1095 1096 down_write(&policy->rwsem); 1097 cpufreq_stats_free_table(policy); 1098 kobj = &policy->kobj; 1099 cmp = &policy->kobj_unregister; 1100 up_write(&policy->rwsem); 1101 kobject_put(kobj); 1102 1103 /* 1104 * We need to make sure that the underlying kobj is 1105 * actually not referenced anymore by anybody before we 1106 * proceed with unloading. 1107 */ 1108 pr_debug("waiting for dropping of refcount\n"); 1109 wait_for_completion(cmp); 1110 pr_debug("wait complete\n"); 1111 } 1112 1113 static void cpufreq_policy_free(struct cpufreq_policy *policy) 1114 { 1115 unsigned long flags; 1116 int cpu; 1117 1118 /* Remove policy from list */ 1119 write_lock_irqsave(&cpufreq_driver_lock, flags); 1120 list_del(&policy->policy_list); 1121 1122 for_each_cpu(cpu, policy->related_cpus) 1123 per_cpu(cpufreq_cpu_data, cpu) = NULL; 1124 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1125 1126 cpufreq_policy_put_kobj(policy); 1127 free_cpumask_var(policy->real_cpus); 1128 free_cpumask_var(policy->related_cpus); 1129 free_cpumask_var(policy->cpus); 1130 kfree(policy); 1131 } 1132 1133 static int cpufreq_online(unsigned int cpu) 1134 { 1135 struct cpufreq_policy *policy; 1136 bool new_policy; 1137 unsigned long flags; 1138 unsigned int j; 1139 int ret; 1140 1141 pr_debug("%s: bringing CPU%u online\n", __func__, cpu); 1142 1143 /* Check if this CPU already has a policy to manage it */ 1144 policy = per_cpu(cpufreq_cpu_data, cpu); 1145 if (policy) { 1146 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); 1147 if (!policy_is_inactive(policy)) 1148 return cpufreq_add_policy_cpu(policy, cpu); 1149 1150 /* This is the only online CPU for the policy. Start over. */ 1151 new_policy = false; 1152 down_write(&policy->rwsem); 1153 policy->cpu = cpu; 1154 policy->governor = NULL; 1155 up_write(&policy->rwsem); 1156 } else { 1157 new_policy = true; 1158 policy = cpufreq_policy_alloc(cpu); 1159 if (!policy) 1160 return -ENOMEM; 1161 } 1162 1163 cpumask_copy(policy->cpus, cpumask_of(cpu)); 1164 1165 /* call driver. From then on the cpufreq must be able 1166 * to accept all calls to ->verify and ->setpolicy for this CPU 1167 */ 1168 ret = cpufreq_driver->init(policy); 1169 if (ret) { 1170 pr_debug("initialization failed\n"); 1171 goto out_free_policy; 1172 } 1173 1174 down_write(&policy->rwsem); 1175 1176 if (new_policy) { 1177 /* related_cpus should at least include policy->cpus. */ 1178 cpumask_copy(policy->related_cpus, policy->cpus); 1179 } 1180 1181 /* 1182 * affected cpus must always be the one, which are online. We aren't 1183 * managing offline cpus here. 1184 */ 1185 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); 1186 1187 if (new_policy) { 1188 policy->user_policy.min = policy->min; 1189 policy->user_policy.max = policy->max; 1190 1191 for_each_cpu(j, policy->related_cpus) { 1192 per_cpu(cpufreq_cpu_data, j) = policy; 1193 add_cpu_dev_symlink(policy, j); 1194 } 1195 } else { 1196 policy->min = policy->user_policy.min; 1197 policy->max = policy->user_policy.max; 1198 } 1199 1200 if (cpufreq_driver->get && !cpufreq_driver->setpolicy) { 1201 policy->cur = cpufreq_driver->get(policy->cpu); 1202 if (!policy->cur) { 1203 pr_err("%s: ->get() failed\n", __func__); 1204 goto out_exit_policy; 1205 } 1206 } 1207 1208 /* 1209 * Sometimes boot loaders set CPU frequency to a value outside of 1210 * frequency table present with cpufreq core. In such cases CPU might be 1211 * unstable if it has to run on that frequency for long duration of time 1212 * and so its better to set it to a frequency which is specified in 1213 * freq-table. This also makes cpufreq stats inconsistent as 1214 * cpufreq-stats would fail to register because current frequency of CPU 1215 * isn't found in freq-table. 1216 * 1217 * Because we don't want this change to effect boot process badly, we go 1218 * for the next freq which is >= policy->cur ('cur' must be set by now, 1219 * otherwise we will end up setting freq to lowest of the table as 'cur' 1220 * is initialized to zero). 1221 * 1222 * We are passing target-freq as "policy->cur - 1" otherwise 1223 * __cpufreq_driver_target() would simply fail, as policy->cur will be 1224 * equal to target-freq. 1225 */ 1226 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK) 1227 && has_target()) { 1228 /* Are we running at unknown frequency ? */ 1229 ret = cpufreq_frequency_table_get_index(policy, policy->cur); 1230 if (ret == -EINVAL) { 1231 /* Warn user and fix it */ 1232 pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n", 1233 __func__, policy->cpu, policy->cur); 1234 ret = __cpufreq_driver_target(policy, policy->cur - 1, 1235 CPUFREQ_RELATION_L); 1236 1237 /* 1238 * Reaching here after boot in a few seconds may not 1239 * mean that system will remain stable at "unknown" 1240 * frequency for longer duration. Hence, a BUG_ON(). 1241 */ 1242 BUG_ON(ret); 1243 pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n", 1244 __func__, policy->cpu, policy->cur); 1245 } 1246 } 1247 1248 if (new_policy) { 1249 ret = cpufreq_add_dev_interface(policy); 1250 if (ret) 1251 goto out_exit_policy; 1252 1253 cpufreq_stats_create_table(policy); 1254 1255 write_lock_irqsave(&cpufreq_driver_lock, flags); 1256 list_add(&policy->policy_list, &cpufreq_policy_list); 1257 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1258 } 1259 1260 ret = cpufreq_init_policy(policy); 1261 if (ret) { 1262 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", 1263 __func__, cpu, ret); 1264 /* cpufreq_policy_free() will notify based on this */ 1265 new_policy = false; 1266 goto out_exit_policy; 1267 } 1268 1269 up_write(&policy->rwsem); 1270 1271 kobject_uevent(&policy->kobj, KOBJ_ADD); 1272 1273 /* Callback for handling stuff after policy is ready */ 1274 if (cpufreq_driver->ready) 1275 cpufreq_driver->ready(policy); 1276 1277 pr_debug("initialization complete\n"); 1278 1279 return 0; 1280 1281 out_exit_policy: 1282 up_write(&policy->rwsem); 1283 1284 if (cpufreq_driver->exit) 1285 cpufreq_driver->exit(policy); 1286 1287 for_each_cpu(j, policy->real_cpus) 1288 remove_cpu_dev_symlink(policy, get_cpu_device(j)); 1289 1290 out_free_policy: 1291 cpufreq_policy_free(policy); 1292 return ret; 1293 } 1294 1295 /** 1296 * cpufreq_add_dev - the cpufreq interface for a CPU device. 1297 * @dev: CPU device. 1298 * @sif: Subsystem interface structure pointer (not used) 1299 */ 1300 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) 1301 { 1302 struct cpufreq_policy *policy; 1303 unsigned cpu = dev->id; 1304 int ret; 1305 1306 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu); 1307 1308 if (cpu_online(cpu)) { 1309 ret = cpufreq_online(cpu); 1310 if (ret) 1311 return ret; 1312 } 1313 1314 /* Create sysfs link on CPU registration */ 1315 policy = per_cpu(cpufreq_cpu_data, cpu); 1316 if (policy) 1317 add_cpu_dev_symlink(policy, cpu); 1318 1319 return 0; 1320 } 1321 1322 static int cpufreq_offline(unsigned int cpu) 1323 { 1324 struct cpufreq_policy *policy; 1325 int ret; 1326 1327 pr_debug("%s: unregistering CPU %u\n", __func__, cpu); 1328 1329 policy = cpufreq_cpu_get_raw(cpu); 1330 if (!policy) { 1331 pr_debug("%s: No cpu_data found\n", __func__); 1332 return 0; 1333 } 1334 1335 down_write(&policy->rwsem); 1336 if (has_target()) 1337 cpufreq_stop_governor(policy); 1338 1339 cpumask_clear_cpu(cpu, policy->cpus); 1340 1341 if (policy_is_inactive(policy)) { 1342 if (has_target()) 1343 strncpy(policy->last_governor, policy->governor->name, 1344 CPUFREQ_NAME_LEN); 1345 else 1346 policy->last_policy = policy->policy; 1347 } else if (cpu == policy->cpu) { 1348 /* Nominate new CPU */ 1349 policy->cpu = cpumask_any(policy->cpus); 1350 } 1351 1352 /* Start governor again for active policy */ 1353 if (!policy_is_inactive(policy)) { 1354 if (has_target()) { 1355 ret = cpufreq_start_governor(policy); 1356 if (ret) 1357 pr_err("%s: Failed to start governor\n", __func__); 1358 } 1359 1360 goto unlock; 1361 } 1362 1363 if (cpufreq_driver->stop_cpu) 1364 cpufreq_driver->stop_cpu(policy); 1365 1366 if (has_target()) 1367 cpufreq_exit_governor(policy); 1368 1369 /* 1370 * Perform the ->exit() even during light-weight tear-down, 1371 * since this is a core component, and is essential for the 1372 * subsequent light-weight ->init() to succeed. 1373 */ 1374 if (cpufreq_driver->exit) { 1375 cpufreq_driver->exit(policy); 1376 policy->freq_table = NULL; 1377 } 1378 1379 unlock: 1380 up_write(&policy->rwsem); 1381 return 0; 1382 } 1383 1384 /** 1385 * cpufreq_remove_dev - remove a CPU device 1386 * 1387 * Removes the cpufreq interface for a CPU device. 1388 */ 1389 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) 1390 { 1391 unsigned int cpu = dev->id; 1392 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 1393 1394 if (!policy) 1395 return; 1396 1397 if (cpu_online(cpu)) 1398 cpufreq_offline(cpu); 1399 1400 cpumask_clear_cpu(cpu, policy->real_cpus); 1401 remove_cpu_dev_symlink(policy, dev); 1402 1403 if (cpumask_empty(policy->real_cpus)) 1404 cpufreq_policy_free(policy); 1405 } 1406 1407 /** 1408 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're 1409 * in deep trouble. 1410 * @policy: policy managing CPUs 1411 * @new_freq: CPU frequency the CPU actually runs at 1412 * 1413 * We adjust to current frequency first, and need to clean up later. 1414 * So either call to cpufreq_update_policy() or schedule handle_update()). 1415 */ 1416 static void cpufreq_out_of_sync(struct cpufreq_policy *policy, 1417 unsigned int new_freq) 1418 { 1419 struct cpufreq_freqs freqs; 1420 1421 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n", 1422 policy->cur, new_freq); 1423 1424 freqs.old = policy->cur; 1425 freqs.new = new_freq; 1426 1427 cpufreq_freq_transition_begin(policy, &freqs); 1428 cpufreq_freq_transition_end(policy, &freqs, 0); 1429 } 1430 1431 /** 1432 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur 1433 * @cpu: CPU number 1434 * 1435 * This is the last known freq, without actually getting it from the driver. 1436 * Return value will be same as what is shown in scaling_cur_freq in sysfs. 1437 */ 1438 unsigned int cpufreq_quick_get(unsigned int cpu) 1439 { 1440 struct cpufreq_policy *policy; 1441 unsigned int ret_freq = 0; 1442 unsigned long flags; 1443 1444 read_lock_irqsave(&cpufreq_driver_lock, flags); 1445 1446 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) { 1447 ret_freq = cpufreq_driver->get(cpu); 1448 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1449 return ret_freq; 1450 } 1451 1452 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1453 1454 policy = cpufreq_cpu_get(cpu); 1455 if (policy) { 1456 ret_freq = policy->cur; 1457 cpufreq_cpu_put(policy); 1458 } 1459 1460 return ret_freq; 1461 } 1462 EXPORT_SYMBOL(cpufreq_quick_get); 1463 1464 /** 1465 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU 1466 * @cpu: CPU number 1467 * 1468 * Just return the max possible frequency for a given CPU. 1469 */ 1470 unsigned int cpufreq_quick_get_max(unsigned int cpu) 1471 { 1472 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1473 unsigned int ret_freq = 0; 1474 1475 if (policy) { 1476 ret_freq = policy->max; 1477 cpufreq_cpu_put(policy); 1478 } 1479 1480 return ret_freq; 1481 } 1482 EXPORT_SYMBOL(cpufreq_quick_get_max); 1483 1484 static unsigned int __cpufreq_get(struct cpufreq_policy *policy) 1485 { 1486 unsigned int ret_freq = 0; 1487 1488 if (!cpufreq_driver->get) 1489 return ret_freq; 1490 1491 ret_freq = cpufreq_driver->get(policy->cpu); 1492 1493 /* 1494 * Updating inactive policies is invalid, so avoid doing that. Also 1495 * if fast frequency switching is used with the given policy, the check 1496 * against policy->cur is pointless, so skip it in that case too. 1497 */ 1498 if (unlikely(policy_is_inactive(policy)) || policy->fast_switch_enabled) 1499 return ret_freq; 1500 1501 if (ret_freq && policy->cur && 1502 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) { 1503 /* verify no discrepancy between actual and 1504 saved value exists */ 1505 if (unlikely(ret_freq != policy->cur)) { 1506 cpufreq_out_of_sync(policy, ret_freq); 1507 schedule_work(&policy->update); 1508 } 1509 } 1510 1511 return ret_freq; 1512 } 1513 1514 /** 1515 * cpufreq_get - get the current CPU frequency (in kHz) 1516 * @cpu: CPU number 1517 * 1518 * Get the CPU current (static) CPU frequency 1519 */ 1520 unsigned int cpufreq_get(unsigned int cpu) 1521 { 1522 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1523 unsigned int ret_freq = 0; 1524 1525 if (policy) { 1526 down_read(&policy->rwsem); 1527 1528 if (!policy_is_inactive(policy)) 1529 ret_freq = __cpufreq_get(policy); 1530 1531 up_read(&policy->rwsem); 1532 1533 cpufreq_cpu_put(policy); 1534 } 1535 1536 return ret_freq; 1537 } 1538 EXPORT_SYMBOL(cpufreq_get); 1539 1540 static unsigned int cpufreq_update_current_freq(struct cpufreq_policy *policy) 1541 { 1542 unsigned int new_freq; 1543 1544 new_freq = cpufreq_driver->get(policy->cpu); 1545 if (!new_freq) 1546 return 0; 1547 1548 if (!policy->cur) { 1549 pr_debug("cpufreq: Driver did not initialize current freq\n"); 1550 policy->cur = new_freq; 1551 } else if (policy->cur != new_freq && has_target()) { 1552 cpufreq_out_of_sync(policy, new_freq); 1553 } 1554 1555 return new_freq; 1556 } 1557 1558 static struct subsys_interface cpufreq_interface = { 1559 .name = "cpufreq", 1560 .subsys = &cpu_subsys, 1561 .add_dev = cpufreq_add_dev, 1562 .remove_dev = cpufreq_remove_dev, 1563 }; 1564 1565 /* 1566 * In case platform wants some specific frequency to be configured 1567 * during suspend.. 1568 */ 1569 int cpufreq_generic_suspend(struct cpufreq_policy *policy) 1570 { 1571 int ret; 1572 1573 if (!policy->suspend_freq) { 1574 pr_debug("%s: suspend_freq not defined\n", __func__); 1575 return 0; 1576 } 1577 1578 pr_debug("%s: Setting suspend-freq: %u\n", __func__, 1579 policy->suspend_freq); 1580 1581 ret = __cpufreq_driver_target(policy, policy->suspend_freq, 1582 CPUFREQ_RELATION_H); 1583 if (ret) 1584 pr_err("%s: unable to set suspend-freq: %u. err: %d\n", 1585 __func__, policy->suspend_freq, ret); 1586 1587 return ret; 1588 } 1589 EXPORT_SYMBOL(cpufreq_generic_suspend); 1590 1591 /** 1592 * cpufreq_suspend() - Suspend CPUFreq governors 1593 * 1594 * Called during system wide Suspend/Hibernate cycles for suspending governors 1595 * as some platforms can't change frequency after this point in suspend cycle. 1596 * Because some of the devices (like: i2c, regulators, etc) they use for 1597 * changing frequency are suspended quickly after this point. 1598 */ 1599 void cpufreq_suspend(void) 1600 { 1601 struct cpufreq_policy *policy; 1602 1603 if (!cpufreq_driver) 1604 return; 1605 1606 if (!has_target() && !cpufreq_driver->suspend) 1607 goto suspend; 1608 1609 pr_debug("%s: Suspending Governors\n", __func__); 1610 1611 for_each_active_policy(policy) { 1612 if (has_target()) { 1613 down_write(&policy->rwsem); 1614 cpufreq_stop_governor(policy); 1615 up_write(&policy->rwsem); 1616 } 1617 1618 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy)) 1619 pr_err("%s: Failed to suspend driver: %p\n", __func__, 1620 policy); 1621 } 1622 1623 suspend: 1624 cpufreq_suspended = true; 1625 } 1626 1627 /** 1628 * cpufreq_resume() - Resume CPUFreq governors 1629 * 1630 * Called during system wide Suspend/Hibernate cycle for resuming governors that 1631 * are suspended with cpufreq_suspend(). 1632 */ 1633 void cpufreq_resume(void) 1634 { 1635 struct cpufreq_policy *policy; 1636 int ret; 1637 1638 if (!cpufreq_driver) 1639 return; 1640 1641 cpufreq_suspended = false; 1642 1643 if (!has_target() && !cpufreq_driver->resume) 1644 return; 1645 1646 pr_debug("%s: Resuming Governors\n", __func__); 1647 1648 for_each_active_policy(policy) { 1649 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) { 1650 pr_err("%s: Failed to resume driver: %p\n", __func__, 1651 policy); 1652 } else if (has_target()) { 1653 down_write(&policy->rwsem); 1654 ret = cpufreq_start_governor(policy); 1655 up_write(&policy->rwsem); 1656 1657 if (ret) 1658 pr_err("%s: Failed to start governor for policy: %p\n", 1659 __func__, policy); 1660 } 1661 } 1662 } 1663 1664 /** 1665 * cpufreq_get_current_driver - return current driver's name 1666 * 1667 * Return the name string of the currently loaded cpufreq driver 1668 * or NULL, if none. 1669 */ 1670 const char *cpufreq_get_current_driver(void) 1671 { 1672 if (cpufreq_driver) 1673 return cpufreq_driver->name; 1674 1675 return NULL; 1676 } 1677 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver); 1678 1679 /** 1680 * cpufreq_get_driver_data - return current driver data 1681 * 1682 * Return the private data of the currently loaded cpufreq 1683 * driver, or NULL if no cpufreq driver is loaded. 1684 */ 1685 void *cpufreq_get_driver_data(void) 1686 { 1687 if (cpufreq_driver) 1688 return cpufreq_driver->driver_data; 1689 1690 return NULL; 1691 } 1692 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data); 1693 1694 /********************************************************************* 1695 * NOTIFIER LISTS INTERFACE * 1696 *********************************************************************/ 1697 1698 /** 1699 * cpufreq_register_notifier - register a driver with cpufreq 1700 * @nb: notifier function to register 1701 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER 1702 * 1703 * Add a driver to one of two lists: either a list of drivers that 1704 * are notified about clock rate changes (once before and once after 1705 * the transition), or a list of drivers that are notified about 1706 * changes in cpufreq policy. 1707 * 1708 * This function may sleep, and has the same return conditions as 1709 * blocking_notifier_chain_register. 1710 */ 1711 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) 1712 { 1713 int ret; 1714 1715 if (cpufreq_disabled()) 1716 return -EINVAL; 1717 1718 WARN_ON(!init_cpufreq_transition_notifier_list_called); 1719 1720 switch (list) { 1721 case CPUFREQ_TRANSITION_NOTIFIER: 1722 mutex_lock(&cpufreq_fast_switch_lock); 1723 1724 if (cpufreq_fast_switch_count > 0) { 1725 mutex_unlock(&cpufreq_fast_switch_lock); 1726 return -EBUSY; 1727 } 1728 ret = srcu_notifier_chain_register( 1729 &cpufreq_transition_notifier_list, nb); 1730 if (!ret) 1731 cpufreq_fast_switch_count--; 1732 1733 mutex_unlock(&cpufreq_fast_switch_lock); 1734 break; 1735 case CPUFREQ_POLICY_NOTIFIER: 1736 ret = blocking_notifier_chain_register( 1737 &cpufreq_policy_notifier_list, nb); 1738 break; 1739 default: 1740 ret = -EINVAL; 1741 } 1742 1743 return ret; 1744 } 1745 EXPORT_SYMBOL(cpufreq_register_notifier); 1746 1747 /** 1748 * cpufreq_unregister_notifier - unregister a driver with cpufreq 1749 * @nb: notifier block to be unregistered 1750 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER 1751 * 1752 * Remove a driver from the CPU frequency notifier list. 1753 * 1754 * This function may sleep, and has the same return conditions as 1755 * blocking_notifier_chain_unregister. 1756 */ 1757 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list) 1758 { 1759 int ret; 1760 1761 if (cpufreq_disabled()) 1762 return -EINVAL; 1763 1764 switch (list) { 1765 case CPUFREQ_TRANSITION_NOTIFIER: 1766 mutex_lock(&cpufreq_fast_switch_lock); 1767 1768 ret = srcu_notifier_chain_unregister( 1769 &cpufreq_transition_notifier_list, nb); 1770 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0)) 1771 cpufreq_fast_switch_count++; 1772 1773 mutex_unlock(&cpufreq_fast_switch_lock); 1774 break; 1775 case CPUFREQ_POLICY_NOTIFIER: 1776 ret = blocking_notifier_chain_unregister( 1777 &cpufreq_policy_notifier_list, nb); 1778 break; 1779 default: 1780 ret = -EINVAL; 1781 } 1782 1783 return ret; 1784 } 1785 EXPORT_SYMBOL(cpufreq_unregister_notifier); 1786 1787 1788 /********************************************************************* 1789 * GOVERNORS * 1790 *********************************************************************/ 1791 1792 /** 1793 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch. 1794 * @policy: cpufreq policy to switch the frequency for. 1795 * @target_freq: New frequency to set (may be approximate). 1796 * 1797 * Carry out a fast frequency switch without sleeping. 1798 * 1799 * The driver's ->fast_switch() callback invoked by this function must be 1800 * suitable for being called from within RCU-sched read-side critical sections 1801 * and it is expected to select the minimum available frequency greater than or 1802 * equal to @target_freq (CPUFREQ_RELATION_L). 1803 * 1804 * This function must not be called if policy->fast_switch_enabled is unset. 1805 * 1806 * Governors calling this function must guarantee that it will never be invoked 1807 * twice in parallel for the same policy and that it will never be called in 1808 * parallel with either ->target() or ->target_index() for the same policy. 1809 * 1810 * If CPUFREQ_ENTRY_INVALID is returned by the driver's ->fast_switch() 1811 * callback to indicate an error condition, the hardware configuration must be 1812 * preserved. 1813 */ 1814 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, 1815 unsigned int target_freq) 1816 { 1817 target_freq = clamp_val(target_freq, policy->min, policy->max); 1818 1819 return cpufreq_driver->fast_switch(policy, target_freq); 1820 } 1821 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); 1822 1823 /* Must set freqs->new to intermediate frequency */ 1824 static int __target_intermediate(struct cpufreq_policy *policy, 1825 struct cpufreq_freqs *freqs, int index) 1826 { 1827 int ret; 1828 1829 freqs->new = cpufreq_driver->get_intermediate(policy, index); 1830 1831 /* We don't need to switch to intermediate freq */ 1832 if (!freqs->new) 1833 return 0; 1834 1835 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n", 1836 __func__, policy->cpu, freqs->old, freqs->new); 1837 1838 cpufreq_freq_transition_begin(policy, freqs); 1839 ret = cpufreq_driver->target_intermediate(policy, index); 1840 cpufreq_freq_transition_end(policy, freqs, ret); 1841 1842 if (ret) 1843 pr_err("%s: Failed to change to intermediate frequency: %d\n", 1844 __func__, ret); 1845 1846 return ret; 1847 } 1848 1849 static int __target_index(struct cpufreq_policy *policy, int index) 1850 { 1851 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; 1852 unsigned int intermediate_freq = 0; 1853 unsigned int newfreq = policy->freq_table[index].frequency; 1854 int retval = -EINVAL; 1855 bool notify; 1856 1857 if (newfreq == policy->cur) 1858 return 0; 1859 1860 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); 1861 if (notify) { 1862 /* Handle switching to intermediate frequency */ 1863 if (cpufreq_driver->get_intermediate) { 1864 retval = __target_intermediate(policy, &freqs, index); 1865 if (retval) 1866 return retval; 1867 1868 intermediate_freq = freqs.new; 1869 /* Set old freq to intermediate */ 1870 if (intermediate_freq) 1871 freqs.old = freqs.new; 1872 } 1873 1874 freqs.new = newfreq; 1875 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", 1876 __func__, policy->cpu, freqs.old, freqs.new); 1877 1878 cpufreq_freq_transition_begin(policy, &freqs); 1879 } 1880 1881 retval = cpufreq_driver->target_index(policy, index); 1882 if (retval) 1883 pr_err("%s: Failed to change cpu frequency: %d\n", __func__, 1884 retval); 1885 1886 if (notify) { 1887 cpufreq_freq_transition_end(policy, &freqs, retval); 1888 1889 /* 1890 * Failed after setting to intermediate freq? Driver should have 1891 * reverted back to initial frequency and so should we. Check 1892 * here for intermediate_freq instead of get_intermediate, in 1893 * case we haven't switched to intermediate freq at all. 1894 */ 1895 if (unlikely(retval && intermediate_freq)) { 1896 freqs.old = intermediate_freq; 1897 freqs.new = policy->restore_freq; 1898 cpufreq_freq_transition_begin(policy, &freqs); 1899 cpufreq_freq_transition_end(policy, &freqs, 0); 1900 } 1901 } 1902 1903 return retval; 1904 } 1905 1906 int __cpufreq_driver_target(struct cpufreq_policy *policy, 1907 unsigned int target_freq, 1908 unsigned int relation) 1909 { 1910 unsigned int old_target_freq = target_freq; 1911 int index; 1912 1913 if (cpufreq_disabled()) 1914 return -ENODEV; 1915 1916 /* Make sure that target_freq is within supported range */ 1917 target_freq = clamp_val(target_freq, policy->min, policy->max); 1918 1919 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", 1920 policy->cpu, target_freq, relation, old_target_freq); 1921 1922 /* 1923 * This might look like a redundant call as we are checking it again 1924 * after finding index. But it is left intentionally for cases where 1925 * exactly same freq is called again and so we can save on few function 1926 * calls. 1927 */ 1928 if (target_freq == policy->cur) 1929 return 0; 1930 1931 /* Save last value to restore later on errors */ 1932 policy->restore_freq = policy->cur; 1933 1934 if (cpufreq_driver->target) 1935 return cpufreq_driver->target(policy, target_freq, relation); 1936 1937 if (!cpufreq_driver->target_index) 1938 return -EINVAL; 1939 1940 index = cpufreq_frequency_table_target(policy, target_freq, relation); 1941 1942 return __target_index(policy, index); 1943 } 1944 EXPORT_SYMBOL_GPL(__cpufreq_driver_target); 1945 1946 int cpufreq_driver_target(struct cpufreq_policy *policy, 1947 unsigned int target_freq, 1948 unsigned int relation) 1949 { 1950 int ret = -EINVAL; 1951 1952 down_write(&policy->rwsem); 1953 1954 ret = __cpufreq_driver_target(policy, target_freq, relation); 1955 1956 up_write(&policy->rwsem); 1957 1958 return ret; 1959 } 1960 EXPORT_SYMBOL_GPL(cpufreq_driver_target); 1961 1962 __weak struct cpufreq_governor *cpufreq_fallback_governor(void) 1963 { 1964 return NULL; 1965 } 1966 1967 static int cpufreq_init_governor(struct cpufreq_policy *policy) 1968 { 1969 int ret; 1970 1971 /* Don't start any governor operations if we are entering suspend */ 1972 if (cpufreq_suspended) 1973 return 0; 1974 /* 1975 * Governor might not be initiated here if ACPI _PPC changed 1976 * notification happened, so check it. 1977 */ 1978 if (!policy->governor) 1979 return -EINVAL; 1980 1981 if (policy->governor->max_transition_latency && 1982 policy->cpuinfo.transition_latency > 1983 policy->governor->max_transition_latency) { 1984 struct cpufreq_governor *gov = cpufreq_fallback_governor(); 1985 1986 if (gov) { 1987 pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n", 1988 policy->governor->name, gov->name); 1989 policy->governor = gov; 1990 } else { 1991 return -EINVAL; 1992 } 1993 } 1994 1995 if (!try_module_get(policy->governor->owner)) 1996 return -EINVAL; 1997 1998 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 1999 2000 if (policy->governor->init) { 2001 ret = policy->governor->init(policy); 2002 if (ret) { 2003 module_put(policy->governor->owner); 2004 return ret; 2005 } 2006 } 2007 2008 return 0; 2009 } 2010 2011 static void cpufreq_exit_governor(struct cpufreq_policy *policy) 2012 { 2013 if (cpufreq_suspended || !policy->governor) 2014 return; 2015 2016 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2017 2018 if (policy->governor->exit) 2019 policy->governor->exit(policy); 2020 2021 module_put(policy->governor->owner); 2022 } 2023 2024 static int cpufreq_start_governor(struct cpufreq_policy *policy) 2025 { 2026 int ret; 2027 2028 if (cpufreq_suspended) 2029 return 0; 2030 2031 if (!policy->governor) 2032 return -EINVAL; 2033 2034 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2035 2036 if (cpufreq_driver->get && !cpufreq_driver->setpolicy) 2037 cpufreq_update_current_freq(policy); 2038 2039 if (policy->governor->start) { 2040 ret = policy->governor->start(policy); 2041 if (ret) 2042 return ret; 2043 } 2044 2045 if (policy->governor->limits) 2046 policy->governor->limits(policy); 2047 2048 return 0; 2049 } 2050 2051 static void cpufreq_stop_governor(struct cpufreq_policy *policy) 2052 { 2053 if (cpufreq_suspended || !policy->governor) 2054 return; 2055 2056 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2057 2058 if (policy->governor->stop) 2059 policy->governor->stop(policy); 2060 } 2061 2062 static void cpufreq_governor_limits(struct cpufreq_policy *policy) 2063 { 2064 if (cpufreq_suspended || !policy->governor) 2065 return; 2066 2067 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2068 2069 if (policy->governor->limits) 2070 policy->governor->limits(policy); 2071 } 2072 2073 int cpufreq_register_governor(struct cpufreq_governor *governor) 2074 { 2075 int err; 2076 2077 if (!governor) 2078 return -EINVAL; 2079 2080 if (cpufreq_disabled()) 2081 return -ENODEV; 2082 2083 mutex_lock(&cpufreq_governor_mutex); 2084 2085 err = -EBUSY; 2086 if (!find_governor(governor->name)) { 2087 err = 0; 2088 list_add(&governor->governor_list, &cpufreq_governor_list); 2089 } 2090 2091 mutex_unlock(&cpufreq_governor_mutex); 2092 return err; 2093 } 2094 EXPORT_SYMBOL_GPL(cpufreq_register_governor); 2095 2096 void cpufreq_unregister_governor(struct cpufreq_governor *governor) 2097 { 2098 struct cpufreq_policy *policy; 2099 unsigned long flags; 2100 2101 if (!governor) 2102 return; 2103 2104 if (cpufreq_disabled()) 2105 return; 2106 2107 /* clear last_governor for all inactive policies */ 2108 read_lock_irqsave(&cpufreq_driver_lock, flags); 2109 for_each_inactive_policy(policy) { 2110 if (!strcmp(policy->last_governor, governor->name)) { 2111 policy->governor = NULL; 2112 strcpy(policy->last_governor, "\0"); 2113 } 2114 } 2115 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 2116 2117 mutex_lock(&cpufreq_governor_mutex); 2118 list_del(&governor->governor_list); 2119 mutex_unlock(&cpufreq_governor_mutex); 2120 return; 2121 } 2122 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor); 2123 2124 2125 /********************************************************************* 2126 * POLICY INTERFACE * 2127 *********************************************************************/ 2128 2129 /** 2130 * cpufreq_get_policy - get the current cpufreq_policy 2131 * @policy: struct cpufreq_policy into which the current cpufreq_policy 2132 * is written 2133 * 2134 * Reads the current cpufreq policy. 2135 */ 2136 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu) 2137 { 2138 struct cpufreq_policy *cpu_policy; 2139 if (!policy) 2140 return -EINVAL; 2141 2142 cpu_policy = cpufreq_cpu_get(cpu); 2143 if (!cpu_policy) 2144 return -EINVAL; 2145 2146 memcpy(policy, cpu_policy, sizeof(*policy)); 2147 2148 cpufreq_cpu_put(cpu_policy); 2149 return 0; 2150 } 2151 EXPORT_SYMBOL(cpufreq_get_policy); 2152 2153 /* 2154 * policy : current policy. 2155 * new_policy: policy to be set. 2156 */ 2157 static int cpufreq_set_policy(struct cpufreq_policy *policy, 2158 struct cpufreq_policy *new_policy) 2159 { 2160 struct cpufreq_governor *old_gov; 2161 int ret; 2162 2163 pr_debug("setting new policy for CPU %u: %u - %u kHz\n", 2164 new_policy->cpu, new_policy->min, new_policy->max); 2165 2166 memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); 2167 2168 /* 2169 * This check works well when we store new min/max freq attributes, 2170 * because new_policy is a copy of policy with one field updated. 2171 */ 2172 if (new_policy->min > new_policy->max) 2173 return -EINVAL; 2174 2175 /* verify the cpu speed can be set within this limit */ 2176 ret = cpufreq_driver->verify(new_policy); 2177 if (ret) 2178 return ret; 2179 2180 /* adjust if necessary - all reasons */ 2181 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 2182 CPUFREQ_ADJUST, new_policy); 2183 2184 /* 2185 * verify the cpu speed can be set within this limit, which might be 2186 * different to the first one 2187 */ 2188 ret = cpufreq_driver->verify(new_policy); 2189 if (ret) 2190 return ret; 2191 2192 /* notification of the new policy */ 2193 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 2194 CPUFREQ_NOTIFY, new_policy); 2195 2196 policy->min = new_policy->min; 2197 policy->max = new_policy->max; 2198 2199 policy->cached_target_freq = UINT_MAX; 2200 2201 pr_debug("new min and max freqs are %u - %u kHz\n", 2202 policy->min, policy->max); 2203 2204 if (cpufreq_driver->setpolicy) { 2205 policy->policy = new_policy->policy; 2206 pr_debug("setting range\n"); 2207 return cpufreq_driver->setpolicy(new_policy); 2208 } 2209 2210 if (new_policy->governor == policy->governor) { 2211 pr_debug("cpufreq: governor limits update\n"); 2212 cpufreq_governor_limits(policy); 2213 return 0; 2214 } 2215 2216 pr_debug("governor switch\n"); 2217 2218 /* save old, working values */ 2219 old_gov = policy->governor; 2220 /* end old governor */ 2221 if (old_gov) { 2222 cpufreq_stop_governor(policy); 2223 cpufreq_exit_governor(policy); 2224 } 2225 2226 /* start new governor */ 2227 policy->governor = new_policy->governor; 2228 ret = cpufreq_init_governor(policy); 2229 if (!ret) { 2230 ret = cpufreq_start_governor(policy); 2231 if (!ret) { 2232 pr_debug("cpufreq: governor change\n"); 2233 return 0; 2234 } 2235 cpufreq_exit_governor(policy); 2236 } 2237 2238 /* new governor failed, so re-start old one */ 2239 pr_debug("starting governor %s failed\n", policy->governor->name); 2240 if (old_gov) { 2241 policy->governor = old_gov; 2242 if (cpufreq_init_governor(policy)) 2243 policy->governor = NULL; 2244 else 2245 cpufreq_start_governor(policy); 2246 } 2247 2248 return ret; 2249 } 2250 2251 /** 2252 * cpufreq_update_policy - re-evaluate an existing cpufreq policy 2253 * @cpu: CPU which shall be re-evaluated 2254 * 2255 * Useful for policy notifiers which have different necessities 2256 * at different times. 2257 */ 2258 void cpufreq_update_policy(unsigned int cpu) 2259 { 2260 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 2261 struct cpufreq_policy new_policy; 2262 2263 if (!policy) 2264 return; 2265 2266 down_write(&policy->rwsem); 2267 2268 if (policy_is_inactive(policy)) 2269 goto unlock; 2270 2271 pr_debug("updating policy for CPU %u\n", cpu); 2272 memcpy(&new_policy, policy, sizeof(*policy)); 2273 new_policy.min = policy->user_policy.min; 2274 new_policy.max = policy->user_policy.max; 2275 2276 /* 2277 * BIOS might change freq behind our back 2278 * -> ask driver for current freq and notify governors about a change 2279 */ 2280 if (cpufreq_driver->get && !cpufreq_driver->setpolicy) { 2281 if (cpufreq_suspended) 2282 goto unlock; 2283 2284 new_policy.cur = cpufreq_update_current_freq(policy); 2285 if (WARN_ON(!new_policy.cur)) 2286 goto unlock; 2287 } 2288 2289 cpufreq_set_policy(policy, &new_policy); 2290 2291 unlock: 2292 up_write(&policy->rwsem); 2293 2294 cpufreq_cpu_put(policy); 2295 } 2296 EXPORT_SYMBOL(cpufreq_update_policy); 2297 2298 /********************************************************************* 2299 * BOOST * 2300 *********************************************************************/ 2301 static int cpufreq_boost_set_sw(int state) 2302 { 2303 struct cpufreq_policy *policy; 2304 int ret = -EINVAL; 2305 2306 for_each_active_policy(policy) { 2307 if (!policy->freq_table) 2308 continue; 2309 2310 ret = cpufreq_frequency_table_cpuinfo(policy, 2311 policy->freq_table); 2312 if (ret) { 2313 pr_err("%s: Policy frequency update failed\n", 2314 __func__); 2315 break; 2316 } 2317 2318 down_write(&policy->rwsem); 2319 policy->user_policy.max = policy->max; 2320 cpufreq_governor_limits(policy); 2321 up_write(&policy->rwsem); 2322 } 2323 2324 return ret; 2325 } 2326 2327 int cpufreq_boost_trigger_state(int state) 2328 { 2329 unsigned long flags; 2330 int ret = 0; 2331 2332 if (cpufreq_driver->boost_enabled == state) 2333 return 0; 2334 2335 write_lock_irqsave(&cpufreq_driver_lock, flags); 2336 cpufreq_driver->boost_enabled = state; 2337 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2338 2339 ret = cpufreq_driver->set_boost(state); 2340 if (ret) { 2341 write_lock_irqsave(&cpufreq_driver_lock, flags); 2342 cpufreq_driver->boost_enabled = !state; 2343 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2344 2345 pr_err("%s: Cannot %s BOOST\n", 2346 __func__, state ? "enable" : "disable"); 2347 } 2348 2349 return ret; 2350 } 2351 2352 static bool cpufreq_boost_supported(void) 2353 { 2354 return likely(cpufreq_driver) && cpufreq_driver->set_boost; 2355 } 2356 2357 static int create_boost_sysfs_file(void) 2358 { 2359 int ret; 2360 2361 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr); 2362 if (ret) 2363 pr_err("%s: cannot register global BOOST sysfs file\n", 2364 __func__); 2365 2366 return ret; 2367 } 2368 2369 static void remove_boost_sysfs_file(void) 2370 { 2371 if (cpufreq_boost_supported()) 2372 sysfs_remove_file(cpufreq_global_kobject, &boost.attr); 2373 } 2374 2375 int cpufreq_enable_boost_support(void) 2376 { 2377 if (!cpufreq_driver) 2378 return -EINVAL; 2379 2380 if (cpufreq_boost_supported()) 2381 return 0; 2382 2383 cpufreq_driver->set_boost = cpufreq_boost_set_sw; 2384 2385 /* This will get removed on driver unregister */ 2386 return create_boost_sysfs_file(); 2387 } 2388 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support); 2389 2390 int cpufreq_boost_enabled(void) 2391 { 2392 return cpufreq_driver->boost_enabled; 2393 } 2394 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled); 2395 2396 /********************************************************************* 2397 * REGISTER / UNREGISTER CPUFREQ DRIVER * 2398 *********************************************************************/ 2399 static enum cpuhp_state hp_online; 2400 2401 static int cpuhp_cpufreq_online(unsigned int cpu) 2402 { 2403 cpufreq_online(cpu); 2404 2405 return 0; 2406 } 2407 2408 static int cpuhp_cpufreq_offline(unsigned int cpu) 2409 { 2410 cpufreq_offline(cpu); 2411 2412 return 0; 2413 } 2414 2415 /** 2416 * cpufreq_register_driver - register a CPU Frequency driver 2417 * @driver_data: A struct cpufreq_driver containing the values# 2418 * submitted by the CPU Frequency driver. 2419 * 2420 * Registers a CPU Frequency driver to this core code. This code 2421 * returns zero on success, -EEXIST when another driver got here first 2422 * (and isn't unregistered in the meantime). 2423 * 2424 */ 2425 int cpufreq_register_driver(struct cpufreq_driver *driver_data) 2426 { 2427 unsigned long flags; 2428 int ret; 2429 2430 if (cpufreq_disabled()) 2431 return -ENODEV; 2432 2433 if (!driver_data || !driver_data->verify || !driver_data->init || 2434 !(driver_data->setpolicy || driver_data->target_index || 2435 driver_data->target) || 2436 (driver_data->setpolicy && (driver_data->target_index || 2437 driver_data->target)) || 2438 (!!driver_data->get_intermediate != !!driver_data->target_intermediate)) 2439 return -EINVAL; 2440 2441 pr_debug("trying to register driver %s\n", driver_data->name); 2442 2443 /* Protect against concurrent CPU online/offline. */ 2444 get_online_cpus(); 2445 2446 write_lock_irqsave(&cpufreq_driver_lock, flags); 2447 if (cpufreq_driver) { 2448 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2449 ret = -EEXIST; 2450 goto out; 2451 } 2452 cpufreq_driver = driver_data; 2453 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2454 2455 if (driver_data->setpolicy) 2456 driver_data->flags |= CPUFREQ_CONST_LOOPS; 2457 2458 if (cpufreq_boost_supported()) { 2459 ret = create_boost_sysfs_file(); 2460 if (ret) 2461 goto err_null_driver; 2462 } 2463 2464 ret = subsys_interface_register(&cpufreq_interface); 2465 if (ret) 2466 goto err_boost_unreg; 2467 2468 if (!(cpufreq_driver->flags & CPUFREQ_STICKY) && 2469 list_empty(&cpufreq_policy_list)) { 2470 /* if all ->init() calls failed, unregister */ 2471 pr_debug("%s: No CPU initialized for driver %s\n", __func__, 2472 driver_data->name); 2473 goto err_if_unreg; 2474 } 2475 2476 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "cpufreq:online", 2477 cpuhp_cpufreq_online, 2478 cpuhp_cpufreq_offline); 2479 if (ret < 0) 2480 goto err_if_unreg; 2481 hp_online = ret; 2482 ret = 0; 2483 2484 pr_debug("driver %s up and running\n", driver_data->name); 2485 goto out; 2486 2487 err_if_unreg: 2488 subsys_interface_unregister(&cpufreq_interface); 2489 err_boost_unreg: 2490 remove_boost_sysfs_file(); 2491 err_null_driver: 2492 write_lock_irqsave(&cpufreq_driver_lock, flags); 2493 cpufreq_driver = NULL; 2494 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2495 out: 2496 put_online_cpus(); 2497 return ret; 2498 } 2499 EXPORT_SYMBOL_GPL(cpufreq_register_driver); 2500 2501 /** 2502 * cpufreq_unregister_driver - unregister the current CPUFreq driver 2503 * 2504 * Unregister the current CPUFreq driver. Only call this if you have 2505 * the right to do so, i.e. if you have succeeded in initialising before! 2506 * Returns zero if successful, and -EINVAL if the cpufreq_driver is 2507 * currently not initialised. 2508 */ 2509 int cpufreq_unregister_driver(struct cpufreq_driver *driver) 2510 { 2511 unsigned long flags; 2512 2513 if (!cpufreq_driver || (driver != cpufreq_driver)) 2514 return -EINVAL; 2515 2516 pr_debug("unregistering driver %s\n", driver->name); 2517 2518 /* Protect against concurrent cpu hotplug */ 2519 get_online_cpus(); 2520 subsys_interface_unregister(&cpufreq_interface); 2521 remove_boost_sysfs_file(); 2522 cpuhp_remove_state_nocalls(hp_online); 2523 2524 write_lock_irqsave(&cpufreq_driver_lock, flags); 2525 2526 cpufreq_driver = NULL; 2527 2528 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2529 put_online_cpus(); 2530 2531 return 0; 2532 } 2533 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver); 2534 2535 /* 2536 * Stop cpufreq at shutdown to make sure it isn't holding any locks 2537 * or mutexes when secondary CPUs are halted. 2538 */ 2539 static struct syscore_ops cpufreq_syscore_ops = { 2540 .shutdown = cpufreq_suspend, 2541 }; 2542 2543 struct kobject *cpufreq_global_kobject; 2544 EXPORT_SYMBOL(cpufreq_global_kobject); 2545 2546 static int __init cpufreq_core_init(void) 2547 { 2548 if (cpufreq_disabled()) 2549 return -ENODEV; 2550 2551 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj); 2552 BUG_ON(!cpufreq_global_kobject); 2553 2554 register_syscore_ops(&cpufreq_syscore_ops); 2555 2556 return 0; 2557 } 2558 module_param(off, int, 0444); 2559 core_initcall(cpufreq_core_init); 2560