1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Energy Model of devices 4 * 5 * Copyright (c) 2018-2021, Arm ltd. 6 * Written by: Quentin Perret, Arm ltd. 7 * Improvements provided by: Lukasz Luba, Arm ltd. 8 */ 9 10 #define pr_fmt(fmt) "energy_model: " fmt 11 12 #include <linux/cpu.h> 13 #include <linux/cpufreq.h> 14 #include <linux/cpumask.h> 15 #include <linux/debugfs.h> 16 #include <linux/energy_model.h> 17 #include <linux/sched/topology.h> 18 #include <linux/slab.h> 19 20 #include "em_netlink.h" 21 22 /* 23 * Mutex serializing the registrations of performance domains and letting 24 * callbacks defined by drivers sleep. 25 */ 26 static DEFINE_MUTEX(em_pd_mutex); 27 28 /* 29 * Manage performance domains with IDs. One can iterate the performance domains 30 * through the list and pick one with their associated ID. The mutex serializes 31 * the list access. When holding em_pd_list_mutex, em_pd_mutex should not be 32 * taken to avoid potential deadlock. 33 */ 34 static DEFINE_IDA(em_pd_ida); 35 static LIST_HEAD(em_pd_list); 36 static DEFINE_MUTEX(em_pd_list_mutex); 37 38 static void em_cpufreq_update_efficiencies(struct device *dev, 39 struct em_perf_state *table); 40 static void em_check_capacity_update(void); 41 static void em_update_workfn(struct work_struct *work); 42 static DECLARE_DELAYED_WORK(em_update_work, em_update_workfn); 43 44 static bool _is_cpu_device(struct device *dev) 45 { 46 return (dev->bus == &cpu_subsys); 47 } 48 49 #ifdef CONFIG_DEBUG_FS 50 static struct dentry *rootdir; 51 52 struct em_dbg_info { 53 struct em_perf_domain *pd; 54 int ps_id; 55 }; 56 57 #define DEFINE_EM_DBG_SHOW(name, fname) \ 58 static int em_debug_##fname##_show(struct seq_file *s, void *unused) \ 59 { \ 60 struct em_dbg_info *em_dbg = s->private; \ 61 struct em_perf_state *table; \ 62 unsigned long val; \ 63 \ 64 rcu_read_lock(); \ 65 table = em_perf_state_from_pd(em_dbg->pd); \ 66 val = table[em_dbg->ps_id].name; \ 67 rcu_read_unlock(); \ 68 \ 69 seq_printf(s, "%lu\n", val); \ 70 return 0; \ 71 } \ 72 DEFINE_SHOW_ATTRIBUTE(em_debug_##fname) 73 74 DEFINE_EM_DBG_SHOW(frequency, frequency); 75 DEFINE_EM_DBG_SHOW(power, power); 76 DEFINE_EM_DBG_SHOW(cost, cost); 77 DEFINE_EM_DBG_SHOW(performance, performance); 78 DEFINE_EM_DBG_SHOW(flags, inefficiency); 79 80 static void em_debug_create_ps(struct em_perf_domain *em_pd, 81 struct em_dbg_info *em_dbg, int i, 82 struct dentry *pd) 83 { 84 struct em_perf_state *table; 85 unsigned long freq; 86 struct dentry *d; 87 char name[24]; 88 89 em_dbg[i].pd = em_pd; 90 em_dbg[i].ps_id = i; 91 92 rcu_read_lock(); 93 table = em_perf_state_from_pd(em_pd); 94 freq = table[i].frequency; 95 rcu_read_unlock(); 96 97 snprintf(name, sizeof(name), "ps:%lu", freq); 98 99 /* Create per-ps directory */ 100 d = debugfs_create_dir(name, pd); 101 debugfs_create_file("frequency", 0444, d, &em_dbg[i], 102 &em_debug_frequency_fops); 103 debugfs_create_file("power", 0444, d, &em_dbg[i], 104 &em_debug_power_fops); 105 debugfs_create_file("cost", 0444, d, &em_dbg[i], 106 &em_debug_cost_fops); 107 debugfs_create_file("performance", 0444, d, &em_dbg[i], 108 &em_debug_performance_fops); 109 debugfs_create_file("inefficient", 0444, d, &em_dbg[i], 110 &em_debug_inefficiency_fops); 111 } 112 113 static int em_debug_cpus_show(struct seq_file *s, void *unused) 114 { 115 seq_printf(s, "%*pbl\n", cpumask_pr_args(to_cpumask(s->private))); 116 117 return 0; 118 } 119 DEFINE_SHOW_ATTRIBUTE(em_debug_cpus); 120 121 static int em_debug_flags_show(struct seq_file *s, void *unused) 122 { 123 struct em_perf_domain *pd = s->private; 124 125 seq_printf(s, "%#lx\n", pd->flags); 126 127 return 0; 128 } 129 DEFINE_SHOW_ATTRIBUTE(em_debug_flags); 130 131 static int em_debug_id_show(struct seq_file *s, void *unused) 132 { 133 struct em_perf_domain *pd = s->private; 134 135 seq_printf(s, "%d\n", pd->id); 136 137 return 0; 138 } 139 DEFINE_SHOW_ATTRIBUTE(em_debug_id); 140 141 static void em_debug_create_pd(struct device *dev) 142 { 143 struct em_dbg_info *em_dbg; 144 struct dentry *d; 145 int i; 146 147 /* Create the directory of the performance domain */ 148 d = debugfs_create_dir(dev_name(dev), rootdir); 149 150 if (_is_cpu_device(dev)) 151 debugfs_create_file("cpus", 0444, d, dev->em_pd->cpus, 152 &em_debug_cpus_fops); 153 154 debugfs_create_file("flags", 0444, d, dev->em_pd, 155 &em_debug_flags_fops); 156 157 debugfs_create_file("id", 0444, d, dev->em_pd, &em_debug_id_fops); 158 159 em_dbg = devm_kzalloc_objs(dev, *em_dbg, dev->em_pd->nr_perf_states); 160 if (!em_dbg) 161 return; 162 163 /* Create a sub-directory for each performance state */ 164 for (i = 0; i < dev->em_pd->nr_perf_states; i++) 165 em_debug_create_ps(dev->em_pd, em_dbg, i, d); 166 167 } 168 169 static void em_debug_remove_pd(struct device *dev) 170 { 171 debugfs_lookup_and_remove(dev_name(dev), rootdir); 172 } 173 174 static int __init em_debug_init(void) 175 { 176 /* Create /sys/kernel/debug/energy_model directory */ 177 rootdir = debugfs_create_dir("energy_model", NULL); 178 179 return 0; 180 } 181 fs_initcall(em_debug_init); 182 #else /* CONFIG_DEBUG_FS */ 183 static void em_debug_create_pd(struct device *dev) {} 184 static void em_debug_remove_pd(struct device *dev) {} 185 #endif 186 187 static void em_release_table_kref(struct kref *kref) 188 { 189 /* It was the last owner of this table so we can free */ 190 kfree_rcu(container_of(kref, struct em_perf_table, kref), rcu); 191 } 192 193 /** 194 * em_table_free() - Handles safe free of the EM table when needed 195 * @table : EM table which is going to be freed 196 * 197 * No return values. 198 */ 199 void em_table_free(struct em_perf_table *table) 200 { 201 kref_put(&table->kref, em_release_table_kref); 202 } 203 204 /** 205 * em_table_alloc() - Allocate a new EM table 206 * @pd : EM performance domain for which this must be done 207 * 208 * Allocate a new EM table and initialize its kref to indicate that it 209 * has a user. 210 * Returns allocated table or NULL. 211 */ 212 struct em_perf_table *em_table_alloc(struct em_perf_domain *pd) 213 { 214 struct em_perf_table *table; 215 int table_size; 216 217 table_size = sizeof(struct em_perf_state) * pd->nr_perf_states; 218 219 table = kzalloc(sizeof(*table) + table_size, GFP_KERNEL); 220 if (!table) 221 return NULL; 222 223 kref_init(&table->kref); 224 225 return table; 226 } 227 228 static void em_init_performance(struct device *dev, struct em_perf_domain *pd, 229 struct em_perf_state *table, int nr_states) 230 { 231 u64 fmax, max_cap; 232 int i, cpu; 233 234 /* This is needed only for CPUs and EAS skip other devices */ 235 if (!_is_cpu_device(dev)) 236 return; 237 238 cpu = cpumask_first(em_span_cpus(pd)); 239 240 /* 241 * Calculate the performance value for each frequency with 242 * linear relationship. The final CPU capacity might not be ready at 243 * boot time, but the EM will be updated a bit later with correct one. 244 */ 245 fmax = (u64) table[nr_states - 1].frequency; 246 max_cap = (u64) arch_scale_cpu_capacity(cpu); 247 for (i = 0; i < nr_states; i++) 248 table[i].performance = div64_u64(max_cap * table[i].frequency, 249 fmax); 250 } 251 252 static int em_compute_costs(struct device *dev, struct em_perf_state *table, 253 const struct em_data_callback *cb, int nr_states, 254 unsigned long flags) 255 { 256 unsigned long prev_cost = ULONG_MAX; 257 int i, ret; 258 259 /* This is needed only for CPUs and EAS skip other devices */ 260 if (!_is_cpu_device(dev)) 261 return 0; 262 263 /* Compute the cost of each performance state. */ 264 for (i = nr_states - 1; i >= 0; i--) { 265 unsigned long power_res, cost; 266 267 if ((flags & EM_PERF_DOMAIN_ARTIFICIAL) && cb->get_cost) { 268 ret = cb->get_cost(dev, table[i].frequency, &cost); 269 if (ret || !cost || cost > EM_MAX_POWER) { 270 dev_err(dev, "EM: invalid cost %lu %d\n", 271 cost, ret); 272 return -EINVAL; 273 } 274 } else { 275 /* increase resolution of 'cost' precision */ 276 power_res = table[i].power * 10; 277 cost = power_res / table[i].performance; 278 } 279 280 table[i].cost = cost; 281 282 if (table[i].cost >= prev_cost) { 283 table[i].flags = EM_PERF_STATE_INEFFICIENT; 284 dev_dbg(dev, "EM: OPP:%lu is inefficient\n", 285 table[i].frequency); 286 } else { 287 prev_cost = table[i].cost; 288 } 289 } 290 291 return 0; 292 } 293 294 /** 295 * em_dev_compute_costs() - Calculate cost values for new runtime EM table 296 * @dev : Device for which the EM table is to be updated 297 * @table : The new EM table that is going to get the costs calculated 298 * @nr_states : Number of performance states 299 * 300 * Calculate the em_perf_state::cost values for new runtime EM table. The 301 * values are used for EAS during task placement. It also calculates and sets 302 * the efficiency flag for each performance state. When the function finish 303 * successfully the EM table is ready to be updated and used by EAS. 304 * 305 * Return 0 on success or a proper error in case of failure. 306 */ 307 int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, 308 int nr_states) 309 { 310 return em_compute_costs(dev, table, NULL, nr_states, 0); 311 } 312 313 /** 314 * em_dev_update_perf_domain() - Update runtime EM table for a device 315 * @dev : Device for which the EM is to be updated 316 * @new_table : The new EM table that is going to be used from now 317 * 318 * Update EM runtime modifiable table for the @dev using the provided @table. 319 * 320 * This function uses a mutex to serialize writers, so it must not be called 321 * from a non-sleeping context. 322 * 323 * Return 0 on success or an error code on failure. 324 */ 325 int em_dev_update_perf_domain(struct device *dev, 326 struct em_perf_table *new_table) 327 { 328 struct em_perf_table *old_table; 329 struct em_perf_domain *pd; 330 331 if (!dev) 332 return -EINVAL; 333 334 /* Serialize update/unregister or concurrent updates */ 335 mutex_lock(&em_pd_mutex); 336 337 if (!dev->em_pd) { 338 mutex_unlock(&em_pd_mutex); 339 return -EINVAL; 340 } 341 pd = dev->em_pd; 342 343 kref_get(&new_table->kref); 344 345 old_table = rcu_dereference_protected(pd->em_table, 346 lockdep_is_held(&em_pd_mutex)); 347 rcu_assign_pointer(pd->em_table, new_table); 348 349 em_cpufreq_update_efficiencies(dev, new_table->state); 350 351 em_table_free(old_table); 352 353 mutex_unlock(&em_pd_mutex); 354 355 em_notify_pd_updated(pd); 356 return 0; 357 } 358 EXPORT_SYMBOL_GPL(em_dev_update_perf_domain); 359 360 static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, 361 struct em_perf_state *table, 362 const struct em_data_callback *cb, 363 unsigned long flags) 364 { 365 unsigned long power, freq, prev_freq = 0; 366 int nr_states = pd->nr_perf_states; 367 int i, ret; 368 369 /* Build the list of performance states for this performance domain */ 370 for (i = 0, freq = 0; i < nr_states; i++, freq++) { 371 /* 372 * active_power() is a driver callback which ceils 'freq' to 373 * lowest performance state of 'dev' above 'freq' and updates 374 * 'power' and 'freq' accordingly. 375 */ 376 ret = cb->active_power(dev, &power, &freq); 377 if (ret) { 378 dev_err(dev, "EM: invalid perf. state: %d\n", 379 ret); 380 return -EINVAL; 381 } 382 383 /* 384 * We expect the driver callback to increase the frequency for 385 * higher performance states. 386 */ 387 if (freq <= prev_freq) { 388 dev_err(dev, "EM: non-increasing freq: %lu\n", 389 freq); 390 return -EINVAL; 391 } 392 393 /* 394 * The power returned by active_state() is expected to be 395 * positive and be in range. 396 */ 397 if (!power || power > EM_MAX_POWER) { 398 dev_err(dev, "EM: invalid power: %lu\n", 399 power); 400 return -EINVAL; 401 } 402 403 table[i].power = power; 404 table[i].frequency = prev_freq = freq; 405 } 406 407 em_init_performance(dev, pd, table, nr_states); 408 409 ret = em_compute_costs(dev, table, cb, nr_states, flags); 410 if (ret) 411 return -EINVAL; 412 413 return 0; 414 } 415 416 static int em_create_pd(struct device *dev, int nr_states, 417 const struct em_data_callback *cb, 418 const cpumask_t *cpus, 419 unsigned long flags) 420 { 421 struct em_perf_table *em_table; 422 struct em_perf_domain *pd; 423 struct device *cpu_dev; 424 int cpu, ret, num_cpus, id; 425 426 if (_is_cpu_device(dev)) { 427 num_cpus = cpumask_weight(cpus); 428 429 /* Prevent max possible energy calculation to not overflow */ 430 if (num_cpus > EM_MAX_NUM_CPUS) { 431 dev_err(dev, "EM: too many CPUs, overflow possible\n"); 432 return -EINVAL; 433 } 434 435 pd = kzalloc(sizeof(*pd) + cpumask_size(), GFP_KERNEL); 436 if (!pd) 437 return -ENOMEM; 438 439 cpumask_copy(em_span_cpus(pd), cpus); 440 } else { 441 /* Readers expect pd->cpus to exist and remain empty. */ 442 pd = kzalloc_flex(*pd, cpus, 443 BITS_TO_LONGS(large_cpumask_bits)); 444 if (!pd) 445 return -ENOMEM; 446 } 447 448 pd->nr_perf_states = nr_states; 449 450 INIT_LIST_HEAD(&pd->node); 451 452 id = ida_alloc(&em_pd_ida, GFP_KERNEL); 453 if (id < 0) { 454 kfree(pd); 455 return id; 456 } 457 pd->id = id; 458 459 em_table = em_table_alloc(pd); 460 if (!em_table) 461 goto free_pd; 462 463 ret = em_create_perf_table(dev, pd, em_table->state, cb, flags); 464 if (ret) 465 goto free_pd_table; 466 467 rcu_assign_pointer(pd->em_table, em_table); 468 469 if (_is_cpu_device(dev)) 470 for_each_cpu(cpu, cpus) { 471 cpu_dev = get_cpu_device(cpu); 472 cpu_dev->em_pd = pd; 473 } 474 475 dev->em_pd = pd; 476 477 return 0; 478 479 free_pd_table: 480 kfree(em_table); 481 free_pd: 482 kfree(pd); 483 ida_free(&em_pd_ida, id); 484 return -EINVAL; 485 } 486 487 static void 488 em_cpufreq_update_efficiencies(struct device *dev, struct em_perf_state *table) 489 { 490 struct em_perf_domain *pd = dev->em_pd; 491 struct cpufreq_policy *policy; 492 int found = 0; 493 int i, cpu; 494 495 if (!_is_cpu_device(dev)) 496 return; 497 498 /* Try to get a CPU which is active and in this PD */ 499 cpu = cpumask_first_and(em_span_cpus(pd), cpu_active_mask); 500 if (cpu >= nr_cpu_ids) { 501 dev_warn(dev, "EM: No online CPU for CPUFreq policy\n"); 502 return; 503 } 504 505 policy = cpufreq_cpu_get(cpu); 506 if (!policy) { 507 dev_warn(dev, "EM: Access to CPUFreq policy failed\n"); 508 return; 509 } 510 511 for (i = 0; i < pd->nr_perf_states; i++) { 512 if (!(table[i].flags & EM_PERF_STATE_INEFFICIENT)) 513 continue; 514 515 if (!cpufreq_table_set_inefficient(policy, table[i].frequency)) 516 found++; 517 } 518 519 cpufreq_cpu_put(policy); 520 521 if (!found) 522 return; 523 524 /* 525 * Efficiencies have been installed in CPUFreq, inefficient frequencies 526 * will be skipped. The EM can do the same. 527 */ 528 pd->flags |= EM_PERF_DOMAIN_SKIP_INEFFICIENCIES; 529 } 530 531 /** 532 * em_pd_get() - Return the performance domain for a device 533 * @dev : Device to find the performance domain for 534 * 535 * Returns the performance domain to which @dev belongs, or NULL if it doesn't 536 * exist. 537 */ 538 struct em_perf_domain *em_pd_get(struct device *dev) 539 { 540 if (IS_ERR_OR_NULL(dev)) 541 return NULL; 542 543 return dev->em_pd; 544 } 545 EXPORT_SYMBOL_GPL(em_pd_get); 546 547 /** 548 * em_cpu_get() - Return the performance domain for a CPU 549 * @cpu : CPU to find the performance domain for 550 * 551 * Returns the performance domain to which @cpu belongs, or NULL if it doesn't 552 * exist. 553 */ 554 struct em_perf_domain *em_cpu_get(int cpu) 555 { 556 struct device *cpu_dev; 557 558 cpu_dev = get_cpu_device(cpu); 559 if (!cpu_dev) 560 return NULL; 561 562 return em_pd_get(cpu_dev); 563 } 564 EXPORT_SYMBOL_GPL(em_cpu_get); 565 566 /** 567 * em_dev_register_perf_domain() - Register the Energy Model (EM) for a device 568 * @dev : Device for which the EM is to register 569 * @nr_states : Number of performance states to register 570 * @cb : Callback functions providing the data of the Energy Model 571 * @cpus : Pointer to cpumask_t, which in case of a CPU device is 572 * obligatory. It can be taken from i.e. 'policy->cpus'. For other 573 * type of devices this should be set to NULL. 574 * @microwatts : Flag indicating that the power values are in micro-Watts or 575 * in some other scale. It must be set properly. 576 * 577 * Create Energy Model tables for a performance domain using the callbacks 578 * defined in cb. 579 * 580 * The @microwatts is important to set with correct value. Some kernel 581 * sub-systems might rely on this flag and check if all devices in the EM are 582 * using the same scale. 583 * 584 * If multiple clients register the same performance domain, all but the first 585 * registration will be ignored. 586 * 587 * Return 0 on success 588 */ 589 int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, 590 const struct em_data_callback *cb, 591 const cpumask_t *cpus, bool microwatts) 592 { 593 int ret = em_dev_register_pd_no_update(dev, nr_states, cb, cpus, microwatts); 594 595 if (_is_cpu_device(dev)) 596 em_check_capacity_update(); 597 598 return ret; 599 } 600 EXPORT_SYMBOL_GPL(em_dev_register_perf_domain); 601 602 /** 603 * em_dev_register_pd_no_update() - Register a perf domain for a device 604 * @dev : Device to register the PD for 605 * @nr_states : Number of performance states in the new PD 606 * @cb : Callback functions for populating the energy model 607 * @cpus : CPUs to include in the new PD (mandatory if @dev is a CPU device) 608 * @microwatts : Whether or not the power values in the EM will be in uW 609 * 610 * Like em_dev_register_perf_domain(), but does not trigger a CPU capacity 611 * update after registering the PD, even if @dev is a CPU device. 612 */ 613 int em_dev_register_pd_no_update(struct device *dev, unsigned int nr_states, 614 const struct em_data_callback *cb, 615 const cpumask_t *cpus, bool microwatts) 616 { 617 struct em_perf_table *em_table; 618 unsigned long cap, prev_cap = 0; 619 unsigned long flags = 0; 620 int cpu, ret; 621 622 if (!dev || !nr_states || !cb) 623 return -EINVAL; 624 625 /* 626 * Use a mutex to serialize the registration of performance domains and 627 * let the driver-defined callback functions sleep. 628 */ 629 mutex_lock(&em_pd_mutex); 630 631 if (dev->em_pd) { 632 ret = -EEXIST; 633 goto unlock; 634 } 635 636 if (_is_cpu_device(dev)) { 637 if (!cpus) { 638 dev_err(dev, "EM: invalid CPU mask\n"); 639 ret = -EINVAL; 640 goto unlock; 641 } 642 643 for_each_cpu(cpu, cpus) { 644 if (em_cpu_get(cpu)) { 645 dev_err(dev, "EM: exists for CPU%d\n", cpu); 646 ret = -EEXIST; 647 goto unlock; 648 } 649 /* 650 * All CPUs of a domain must have the same 651 * micro-architecture since they all share the same 652 * table. 653 */ 654 cap = arch_scale_cpu_capacity(cpu); 655 if (prev_cap && prev_cap != cap) { 656 dev_err(dev, "EM: CPUs of %*pbl must have the same capacity\n", 657 cpumask_pr_args(cpus)); 658 659 ret = -EINVAL; 660 goto unlock; 661 } 662 prev_cap = cap; 663 } 664 } 665 666 if (microwatts) 667 flags |= EM_PERF_DOMAIN_MICROWATTS; 668 else if (cb->get_cost) 669 flags |= EM_PERF_DOMAIN_ARTIFICIAL; 670 671 /* 672 * EM only supports uW (exception is artificial EM). 673 * Therefore, check and force the drivers to provide 674 * power in uW. 675 */ 676 if (!microwatts && !(flags & EM_PERF_DOMAIN_ARTIFICIAL)) { 677 dev_err(dev, "EM: only supports uW power values\n"); 678 ret = -EINVAL; 679 goto unlock; 680 } 681 682 ret = em_create_pd(dev, nr_states, cb, cpus, flags); 683 if (ret) 684 goto unlock; 685 686 dev->em_pd->flags |= flags; 687 dev->em_pd->min_perf_state = 0; 688 dev->em_pd->max_perf_state = nr_states - 1; 689 690 em_table = rcu_dereference_protected(dev->em_pd->em_table, 691 lockdep_is_held(&em_pd_mutex)); 692 em_cpufreq_update_efficiencies(dev, em_table->state); 693 694 em_debug_create_pd(dev); 695 dev_info(dev, "EM: created perf domain\n"); 696 697 unlock: 698 mutex_unlock(&em_pd_mutex); 699 if (ret) 700 return ret; 701 702 mutex_lock(&em_pd_list_mutex); 703 list_add_tail(&dev->em_pd->node, &em_pd_list); 704 mutex_unlock(&em_pd_list_mutex); 705 706 em_notify_pd_created(dev->em_pd); 707 708 return 0; 709 } 710 EXPORT_SYMBOL_GPL(em_dev_register_pd_no_update); 711 712 /** 713 * em_dev_unregister_perf_domain() - Unregister Energy Model (EM) for a device 714 * @dev : Device for which the EM is registered 715 * 716 * Unregister the EM for the specified @dev (but not a CPU device). 717 */ 718 void em_dev_unregister_perf_domain(struct device *dev) 719 { 720 if (IS_ERR_OR_NULL(dev) || !dev->em_pd) 721 return; 722 723 if (_is_cpu_device(dev)) 724 return; 725 726 mutex_lock(&em_pd_list_mutex); 727 list_del_init(&dev->em_pd->node); 728 mutex_unlock(&em_pd_list_mutex); 729 730 em_notify_pd_deleted(dev->em_pd); 731 732 /* 733 * The mutex separates all register/unregister requests and protects 734 * from potential clean-up/setup issues in the debugfs directories. 735 * The debugfs directory name is the same as device's name. 736 */ 737 mutex_lock(&em_pd_mutex); 738 em_debug_remove_pd(dev); 739 740 em_table_free(rcu_dereference_protected(dev->em_pd->em_table, 741 lockdep_is_held(&em_pd_mutex))); 742 743 ida_free(&em_pd_ida, dev->em_pd->id); 744 745 kfree(dev->em_pd); 746 dev->em_pd = NULL; 747 mutex_unlock(&em_pd_mutex); 748 } 749 EXPORT_SYMBOL_GPL(em_dev_unregister_perf_domain); 750 751 static struct em_perf_table *em_table_dup(struct em_perf_domain *pd) 752 { 753 struct em_perf_table *em_table; 754 struct em_perf_state *ps, *new_ps; 755 int ps_size; 756 757 em_table = em_table_alloc(pd); 758 if (!em_table) 759 return NULL; 760 761 new_ps = em_table->state; 762 763 rcu_read_lock(); 764 ps = em_perf_state_from_pd(pd); 765 /* Initialize data based on old table */ 766 ps_size = sizeof(struct em_perf_state) * pd->nr_perf_states; 767 memcpy(new_ps, ps, ps_size); 768 769 rcu_read_unlock(); 770 771 return em_table; 772 } 773 774 static int em_recalc_and_update(struct device *dev, struct em_perf_domain *pd, 775 struct em_perf_table *em_table) 776 { 777 int ret; 778 779 if (!em_is_artificial(pd)) { 780 ret = em_compute_costs(dev, em_table->state, NULL, 781 pd->nr_perf_states, pd->flags); 782 if (ret) 783 goto free_em_table; 784 } 785 786 ret = em_dev_update_perf_domain(dev, em_table); 787 if (ret) 788 goto free_em_table; 789 790 /* 791 * This is one-time-update, so give up the ownership in this updater. 792 * The EM framework has incremented the usage counter and from now 793 * will keep the reference (then free the memory when needed). 794 */ 795 free_em_table: 796 em_table_free(em_table); 797 return ret; 798 } 799 800 /* 801 * Adjustment of CPU performance values after boot, when all CPUs capacites 802 * are correctly calculated. 803 */ 804 static void em_adjust_new_capacity(unsigned int cpu, struct device *dev, 805 struct em_perf_domain *pd) 806 { 807 unsigned long cpu_capacity = arch_scale_cpu_capacity(cpu); 808 struct em_perf_table *em_table; 809 struct em_perf_state *table; 810 unsigned long em_max_perf; 811 812 rcu_read_lock(); 813 table = em_perf_state_from_pd(pd); 814 em_max_perf = table[pd->nr_perf_states - 1].performance; 815 rcu_read_unlock(); 816 817 if (em_max_perf == cpu_capacity) 818 return; 819 820 pr_debug("updating cpu%d cpu_cap=%lu old capacity=%lu\n", cpu, 821 cpu_capacity, em_max_perf); 822 823 em_table = em_table_dup(pd); 824 if (!em_table) { 825 dev_warn(dev, "EM: allocation failed\n"); 826 return; 827 } 828 829 em_init_performance(dev, pd, em_table->state, pd->nr_perf_states); 830 831 em_recalc_and_update(dev, pd, em_table); 832 } 833 834 /** 835 * em_adjust_cpu_capacity() - Adjust the EM for a CPU after a capacity update. 836 * @cpu: Target CPU. 837 * 838 * Adjust the existing EM for @cpu after a capacity update under the assumption 839 * that the capacity has been updated in the same way for all of the CPUs in 840 * the same perf domain. 841 */ 842 void em_adjust_cpu_capacity(unsigned int cpu) 843 { 844 struct device *dev = get_cpu_device(cpu); 845 struct em_perf_domain *pd; 846 847 pd = em_pd_get(dev); 848 if (pd) 849 em_adjust_new_capacity(cpu, dev, pd); 850 } 851 852 static void em_check_capacity_update(void) 853 { 854 cpumask_var_t cpu_done_mask; 855 int cpu, failed_cpus = 0; 856 857 if (!zalloc_cpumask_var(&cpu_done_mask, GFP_KERNEL)) { 858 pr_warn("no free memory\n"); 859 return; 860 } 861 862 /* Check if CPUs capacity has changed than update EM */ 863 for_each_possible_cpu(cpu) { 864 struct cpufreq_policy *policy; 865 struct em_perf_domain *pd; 866 struct device *dev; 867 868 if (cpumask_test_cpu(cpu, cpu_done_mask)) 869 continue; 870 871 policy = cpufreq_cpu_get(cpu); 872 if (!policy) { 873 failed_cpus++; 874 continue; 875 } 876 cpufreq_cpu_put(policy); 877 878 dev = get_cpu_device(cpu); 879 pd = em_pd_get(dev); 880 if (!pd || em_is_artificial(pd)) 881 continue; 882 883 cpumask_or(cpu_done_mask, cpu_done_mask, 884 em_span_cpus(pd)); 885 886 em_adjust_new_capacity(cpu, dev, pd); 887 } 888 889 if (failed_cpus) 890 schedule_delayed_work(&em_update_work, msecs_to_jiffies(1000)); 891 892 free_cpumask_var(cpu_done_mask); 893 } 894 895 static void em_update_workfn(struct work_struct *work) 896 { 897 em_check_capacity_update(); 898 } 899 900 /** 901 * em_dev_update_chip_binning() - Update Energy Model after the new voltage 902 * information is present in the OPPs. 903 * @dev : Device for which the Energy Model has to be updated. 904 * 905 * This function allows to update easily the EM with new values available in 906 * the OPP framework and DT. It can be used after the chip has been properly 907 * verified by device drivers and the voltages adjusted for the 'chip binning'. 908 */ 909 int em_dev_update_chip_binning(struct device *dev) 910 { 911 struct em_perf_table *em_table; 912 struct em_perf_domain *pd; 913 int i, ret; 914 915 if (IS_ERR_OR_NULL(dev)) 916 return -EINVAL; 917 918 pd = em_pd_get(dev); 919 if (!pd) { 920 dev_warn(dev, "Couldn't find Energy Model\n"); 921 return -EINVAL; 922 } 923 924 em_table = em_table_dup(pd); 925 if (!em_table) { 926 dev_warn(dev, "EM: allocation failed\n"); 927 return -ENOMEM; 928 } 929 930 /* Update power values which might change due to new voltage in OPPs */ 931 for (i = 0; i < pd->nr_perf_states; i++) { 932 unsigned long freq = em_table->state[i].frequency; 933 unsigned long power; 934 935 ret = dev_pm_opp_calc_power(dev, &power, &freq); 936 if (ret) { 937 em_table_free(em_table); 938 return ret; 939 } 940 941 em_table->state[i].power = power; 942 } 943 944 return em_recalc_and_update(dev, pd, em_table); 945 } 946 EXPORT_SYMBOL_GPL(em_dev_update_chip_binning); 947 948 949 /** 950 * em_update_performance_limits() - Update Energy Model with performance 951 * limits information. 952 * @pd : Performance Domain with EM that has to be updated. 953 * @freq_min_khz : New minimum allowed frequency for this device. 954 * @freq_max_khz : New maximum allowed frequency for this device. 955 * 956 * This function allows to update the EM with information about available 957 * performance levels. It takes the minimum and maximum frequency in kHz 958 * and does internal translation to performance levels. 959 * Returns 0 on success or -EINVAL when failed. 960 */ 961 int em_update_performance_limits(struct em_perf_domain *pd, 962 unsigned long freq_min_khz, unsigned long freq_max_khz) 963 { 964 struct em_perf_state *table; 965 int min_ps = -1; 966 int max_ps = -1; 967 int i; 968 969 if (!pd) 970 return -EINVAL; 971 972 rcu_read_lock(); 973 table = em_perf_state_from_pd(pd); 974 975 for (i = 0; i < pd->nr_perf_states; i++) { 976 if (freq_min_khz == table[i].frequency) 977 min_ps = i; 978 if (freq_max_khz == table[i].frequency) 979 max_ps = i; 980 } 981 rcu_read_unlock(); 982 983 /* Only update when both are found and sane */ 984 if (min_ps < 0 || max_ps < 0 || max_ps < min_ps) 985 return -EINVAL; 986 987 988 /* Guard simultaneous updates and make them atomic */ 989 mutex_lock(&em_pd_mutex); 990 pd->min_perf_state = min_ps; 991 pd->max_perf_state = max_ps; 992 mutex_unlock(&em_pd_mutex); 993 994 return 0; 995 } 996 EXPORT_SYMBOL_GPL(em_update_performance_limits); 997 998 static void rebuild_sd_workfn(struct work_struct *work) 999 { 1000 rebuild_sched_domains_energy(); 1001 } 1002 1003 void em_rebuild_sched_domains(void) 1004 { 1005 static DECLARE_WORK(rebuild_sd_work, rebuild_sd_workfn); 1006 1007 /* 1008 * When called from the cpufreq_register_driver() path, the 1009 * cpu_hotplug_lock is already held, so use a work item to 1010 * avoid nested locking in rebuild_sched_domains(). 1011 */ 1012 schedule_work(&rebuild_sd_work); 1013 } 1014 1015 #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_NET) 1016 int for_each_em_perf_domain(int (*cb)(struct em_perf_domain*, void *), 1017 void *data) 1018 { 1019 struct em_perf_domain *pd; 1020 1021 lockdep_assert_not_held(&em_pd_mutex); 1022 guard(mutex)(&em_pd_list_mutex); 1023 1024 list_for_each_entry(pd, &em_pd_list, node) { 1025 int ret; 1026 1027 ret = cb(pd, data); 1028 if (ret) 1029 return ret; 1030 } 1031 1032 return 0; 1033 } 1034 1035 /* Run @cb on the matching domain with em_pd_list_mutex held. */ 1036 int em_perf_domain_for_id(int id, int (*cb)(struct em_perf_domain *, void *), 1037 void *data) 1038 { 1039 struct em_perf_domain *pd; 1040 1041 lockdep_assert_not_held(&em_pd_mutex); 1042 guard(mutex)(&em_pd_list_mutex); 1043 1044 list_for_each_entry(pd, &em_pd_list, node) { 1045 if (pd->id == id) 1046 return cb(pd, data); 1047 } 1048 1049 return -EINVAL; 1050 } 1051 #endif 1052