1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * CPPC (Collaborative Processor Performance Control) driver for 4 * interfacing with the CPUfreq layer and governors. See 5 * cppc_acpi.c for CPPC specific methods. 6 * 7 * (C) Copyright 2014, 2015 Linaro Ltd. 8 * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org> 9 */ 10 11 #define pr_fmt(fmt) "CPPC Cpufreq:" fmt 12 13 #include <linux/arch_topology.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/delay.h> 17 #include <linux/cpu.h> 18 #include <linux/cpufreq.h> 19 #include <linux/irq_work.h> 20 #include <linux/kthread.h> 21 #include <linux/time.h> 22 #include <linux/vmalloc.h> 23 #include <uapi/linux/sched/types.h> 24 25 #include <linux/unaligned.h> 26 27 #include <acpi/cppc_acpi.h> 28 29 static struct cpufreq_driver cppc_cpufreq_driver; 30 31 #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE 32 static enum { 33 FIE_UNSET = -1, 34 FIE_ENABLED, 35 FIE_DISABLED 36 } fie_disabled = FIE_UNSET; 37 38 module_param(fie_disabled, int, 0444); 39 MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)"); 40 41 /* Frequency invariance support */ 42 struct cppc_freq_invariance { 43 int cpu; 44 struct irq_work irq_work; 45 struct kthread_work work; 46 struct cppc_perf_fb_ctrs prev_perf_fb_ctrs; 47 struct cppc_cpudata *cpu_data; 48 }; 49 50 static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv); 51 static struct kthread_worker *kworker_fie; 52 53 static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0, 54 struct cppc_perf_fb_ctrs *fb_ctrs_t1); 55 56 /** 57 * cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance 58 * @work: The work item. 59 * 60 * The CPPC driver register itself with the topology core to provide its own 61 * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which 62 * gets called by the scheduler on every tick. 63 * 64 * Note that the arch specific counters have higher priority than CPPC counters, 65 * if available, though the CPPC driver doesn't need to have any special 66 * handling for that. 67 * 68 * On an invocation of cppc_scale_freq_tick(), we schedule an irq work (since we 69 * reach here from hard-irq context), which then schedules a normal work item 70 * and cppc_scale_freq_workfn() updates the per_cpu arch_freq_scale variable 71 * based on the counter updates since the last tick. 72 */ 73 static void cppc_scale_freq_workfn(struct kthread_work *work) 74 { 75 struct cppc_freq_invariance *cppc_fi; 76 struct cppc_perf_fb_ctrs fb_ctrs = {0}; 77 struct cppc_cpudata *cpu_data; 78 unsigned long local_freq_scale; 79 u64 perf; 80 81 cppc_fi = container_of(work, struct cppc_freq_invariance, work); 82 cpu_data = cppc_fi->cpu_data; 83 84 if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) { 85 pr_warn("%s: failed to read perf counters\n", __func__); 86 return; 87 } 88 89 perf = cppc_perf_from_fbctrs(&cppc_fi->prev_perf_fb_ctrs, &fb_ctrs); 90 if (!perf) 91 return; 92 93 cppc_fi->prev_perf_fb_ctrs = fb_ctrs; 94 95 perf <<= SCHED_CAPACITY_SHIFT; 96 local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf); 97 98 /* This can happen due to counter's overflow */ 99 if (unlikely(local_freq_scale > 1024)) 100 local_freq_scale = 1024; 101 102 per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale; 103 } 104 105 static void cppc_irq_work(struct irq_work *irq_work) 106 { 107 struct cppc_freq_invariance *cppc_fi; 108 109 cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work); 110 kthread_queue_work(kworker_fie, &cppc_fi->work); 111 } 112 113 static void cppc_scale_freq_tick(void) 114 { 115 struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id()); 116 117 /* 118 * cppc_get_perf_ctrs() can potentially sleep, call that from the right 119 * context. 120 */ 121 irq_work_queue(&cppc_fi->irq_work); 122 } 123 124 static struct scale_freq_data cppc_sftd = { 125 .source = SCALE_FREQ_SOURCE_CPPC, 126 .set_freq_scale = cppc_scale_freq_tick, 127 }; 128 129 static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) 130 { 131 struct cppc_freq_invariance *cppc_fi; 132 int cpu, ret; 133 134 if (fie_disabled) 135 return; 136 137 for_each_cpu(cpu, policy->cpus) { 138 cppc_fi = &per_cpu(cppc_freq_inv, cpu); 139 cppc_fi->cpu = cpu; 140 cppc_fi->cpu_data = policy->driver_data; 141 kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn); 142 init_irq_work(&cppc_fi->irq_work, cppc_irq_work); 143 144 ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs); 145 146 /* 147 * Don't abort as the CPU was offline while the driver was 148 * getting registered. 149 */ 150 if (ret && cpu_online(cpu)) { 151 pr_debug("%s: failed to read perf counters for cpu:%d: %d\n", 152 __func__, cpu, ret); 153 return; 154 } 155 } 156 157 /* Register for freq-invariance */ 158 topology_set_scale_freq_source(&cppc_sftd, policy->cpus); 159 } 160 161 /* 162 * We free all the resources on policy's removal and not on CPU removal as the 163 * irq-work are per-cpu and the hotplug core takes care of flushing the pending 164 * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work 165 * fires on another CPU after the concerned CPU is removed, it won't harm. 166 * 167 * We just need to make sure to remove them all on policy->exit(). 168 */ 169 static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) 170 { 171 struct cppc_freq_invariance *cppc_fi; 172 int cpu; 173 174 if (fie_disabled) 175 return; 176 177 /* policy->cpus will be empty here, use related_cpus instead */ 178 topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus); 179 180 for_each_cpu(cpu, policy->related_cpus) { 181 cppc_fi = &per_cpu(cppc_freq_inv, cpu); 182 irq_work_sync(&cppc_fi->irq_work); 183 kthread_cancel_work_sync(&cppc_fi->work); 184 } 185 } 186 187 static void __init cppc_freq_invariance_init(void) 188 { 189 struct sched_attr attr = { 190 .size = sizeof(struct sched_attr), 191 .sched_policy = SCHED_DEADLINE, 192 .sched_nice = 0, 193 .sched_priority = 0, 194 /* 195 * Fake (unused) bandwidth; workaround to "fix" 196 * priority inheritance. 197 */ 198 .sched_runtime = NSEC_PER_MSEC, 199 .sched_deadline = 10 * NSEC_PER_MSEC, 200 .sched_period = 10 * NSEC_PER_MSEC, 201 }; 202 int ret; 203 204 if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) { 205 fie_disabled = FIE_ENABLED; 206 if (cppc_perf_ctrs_in_pcc()) { 207 pr_info("FIE not enabled on systems with registers in PCC\n"); 208 fie_disabled = FIE_DISABLED; 209 } 210 } 211 212 if (fie_disabled) 213 return; 214 215 kworker_fie = kthread_run_worker(0, "cppc_fie"); 216 if (IS_ERR(kworker_fie)) { 217 pr_warn("%s: failed to create kworker_fie: %ld\n", __func__, 218 PTR_ERR(kworker_fie)); 219 fie_disabled = FIE_DISABLED; 220 return; 221 } 222 223 ret = sched_setattr_nocheck(kworker_fie->task, &attr); 224 if (ret) { 225 pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__, 226 ret); 227 kthread_destroy_worker(kworker_fie); 228 fie_disabled = FIE_DISABLED; 229 } 230 } 231 232 static void cppc_freq_invariance_exit(void) 233 { 234 if (fie_disabled) 235 return; 236 237 kthread_destroy_worker(kworker_fie); 238 } 239 240 #else 241 static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) 242 { 243 } 244 245 static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) 246 { 247 } 248 249 static inline void cppc_freq_invariance_init(void) 250 { 251 } 252 253 static inline void cppc_freq_invariance_exit(void) 254 { 255 } 256 #endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */ 257 258 static int cppc_cpufreq_set_target(struct cpufreq_policy *policy, 259 unsigned int target_freq, 260 unsigned int relation) 261 { 262 struct cppc_cpudata *cpu_data = policy->driver_data; 263 unsigned int cpu = policy->cpu; 264 struct cpufreq_freqs freqs; 265 int ret = 0; 266 267 cpu_data->perf_ctrls.desired_perf = 268 cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); 269 freqs.old = policy->cur; 270 freqs.new = target_freq; 271 272 cpufreq_freq_transition_begin(policy, &freqs); 273 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); 274 cpufreq_freq_transition_end(policy, &freqs, ret != 0); 275 276 if (ret) 277 pr_debug("Failed to set target on CPU:%d. ret:%d\n", 278 cpu, ret); 279 280 return ret; 281 } 282 283 static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy, 284 unsigned int target_freq) 285 { 286 struct cppc_cpudata *cpu_data = policy->driver_data; 287 unsigned int cpu = policy->cpu; 288 u32 desired_perf; 289 int ret; 290 291 desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); 292 cpu_data->perf_ctrls.desired_perf = desired_perf; 293 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); 294 295 if (ret) { 296 pr_debug("Failed to set target on CPU:%d. ret:%d\n", 297 cpu, ret); 298 return 0; 299 } 300 301 return target_freq; 302 } 303 304 static int cppc_verify_policy(struct cpufreq_policy_data *policy) 305 { 306 cpufreq_verify_within_cpu_limits(policy); 307 return 0; 308 } 309 310 static unsigned int __cppc_cpufreq_get_transition_delay_us(unsigned int cpu) 311 { 312 int transition_latency_ns = cppc_get_transition_latency(cpu); 313 314 if (transition_latency_ns < 0) 315 return CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS / NSEC_PER_USEC; 316 317 return transition_latency_ns / NSEC_PER_USEC; 318 } 319 320 /* 321 * The PCC subspace describes the rate at which platform can accept commands 322 * on the shared PCC channel (including READs which do not count towards freq 323 * transition requests), so ideally we need to use the PCC values as a fallback 324 * if we don't have a platform specific transition_delay_us 325 */ 326 #ifdef CONFIG_ARM64 327 #include <asm/cputype.h> 328 329 static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) 330 { 331 unsigned long implementor = read_cpuid_implementor(); 332 unsigned long part_num = read_cpuid_part_number(); 333 334 switch (implementor) { 335 case ARM_CPU_IMP_QCOM: 336 switch (part_num) { 337 case QCOM_CPU_PART_FALKOR_V1: 338 case QCOM_CPU_PART_FALKOR: 339 return 10000; 340 } 341 } 342 return __cppc_cpufreq_get_transition_delay_us(cpu); 343 } 344 #else 345 static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) 346 { 347 return __cppc_cpufreq_get_transition_delay_us(cpu); 348 } 349 #endif 350 351 #if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL) 352 353 static DEFINE_PER_CPU(unsigned int, efficiency_class); 354 355 /* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */ 356 #define CPPC_EM_CAP_STEP (20) 357 /* Increase the cost value by CPPC_EM_COST_STEP every performance state. */ 358 #define CPPC_EM_COST_STEP (1) 359 /* Add a cost gap correspnding to the energy of 4 CPUs. */ 360 #define CPPC_EM_COST_GAP (4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \ 361 / CPPC_EM_CAP_STEP) 362 363 static unsigned int get_perf_level_count(struct cpufreq_policy *policy) 364 { 365 struct cppc_perf_caps *perf_caps; 366 unsigned int min_cap, max_cap; 367 struct cppc_cpudata *cpu_data; 368 int cpu = policy->cpu; 369 370 cpu_data = policy->driver_data; 371 perf_caps = &cpu_data->perf_caps; 372 max_cap = arch_scale_cpu_capacity(cpu); 373 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf, 374 perf_caps->highest_perf); 375 if ((min_cap == 0) || (max_cap < min_cap)) 376 return 0; 377 return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP; 378 } 379 380 /* 381 * The cost is defined as: 382 * cost = power * max_frequency / frequency 383 */ 384 static inline unsigned long compute_cost(int cpu, int step) 385 { 386 return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) + 387 step * CPPC_EM_COST_STEP; 388 } 389 390 static int cppc_get_cpu_power(struct device *cpu_dev, 391 unsigned long *power, unsigned long *KHz) 392 { 393 unsigned long perf_step, perf_prev, perf, perf_check; 394 unsigned int min_step, max_step, step, step_check; 395 unsigned long prev_freq = *KHz; 396 unsigned int min_cap, max_cap; 397 struct cpufreq_policy *policy; 398 399 struct cppc_perf_caps *perf_caps; 400 struct cppc_cpudata *cpu_data; 401 402 policy = cpufreq_cpu_get_raw(cpu_dev->id); 403 if (!policy) 404 return -EINVAL; 405 406 cpu_data = policy->driver_data; 407 perf_caps = &cpu_data->perf_caps; 408 max_cap = arch_scale_cpu_capacity(cpu_dev->id); 409 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf, 410 perf_caps->highest_perf); 411 perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf, 412 max_cap); 413 min_step = min_cap / CPPC_EM_CAP_STEP; 414 max_step = max_cap / CPPC_EM_CAP_STEP; 415 416 perf_prev = cppc_khz_to_perf(perf_caps, *KHz); 417 step = perf_prev / perf_step; 418 419 if (step > max_step) 420 return -EINVAL; 421 422 if (min_step == max_step) { 423 step = max_step; 424 perf = perf_caps->highest_perf; 425 } else if (step < min_step) { 426 step = min_step; 427 perf = perf_caps->lowest_perf; 428 } else { 429 step++; 430 if (step == max_step) 431 perf = perf_caps->highest_perf; 432 else 433 perf = step * perf_step; 434 } 435 436 *KHz = cppc_perf_to_khz(perf_caps, perf); 437 perf_check = cppc_khz_to_perf(perf_caps, *KHz); 438 step_check = perf_check / perf_step; 439 440 /* 441 * To avoid bad integer approximation, check that new frequency value 442 * increased and that the new frequency will be converted to the 443 * desired step value. 444 */ 445 while ((*KHz == prev_freq) || (step_check != step)) { 446 perf++; 447 *KHz = cppc_perf_to_khz(perf_caps, perf); 448 perf_check = cppc_khz_to_perf(perf_caps, *KHz); 449 step_check = perf_check / perf_step; 450 } 451 452 /* 453 * With an artificial EM, only the cost value is used. Still the power 454 * is populated such as 0 < power < EM_MAX_POWER. This allows to add 455 * more sense to the artificial performance states. 456 */ 457 *power = compute_cost(cpu_dev->id, step); 458 459 return 0; 460 } 461 462 static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz, 463 unsigned long *cost) 464 { 465 unsigned long perf_step, perf_prev; 466 struct cppc_perf_caps *perf_caps; 467 struct cpufreq_policy *policy; 468 struct cppc_cpudata *cpu_data; 469 unsigned int max_cap; 470 int step; 471 472 policy = cpufreq_cpu_get_raw(cpu_dev->id); 473 if (!policy) 474 return -EINVAL; 475 476 cpu_data = policy->driver_data; 477 perf_caps = &cpu_data->perf_caps; 478 max_cap = arch_scale_cpu_capacity(cpu_dev->id); 479 480 perf_prev = cppc_khz_to_perf(perf_caps, KHz); 481 perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap; 482 step = perf_prev / perf_step; 483 484 *cost = compute_cost(cpu_dev->id, step); 485 486 return 0; 487 } 488 489 static void cppc_cpufreq_register_em(struct cpufreq_policy *policy) 490 { 491 struct cppc_cpudata *cpu_data; 492 struct em_data_callback em_cb = 493 EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost); 494 495 cpu_data = policy->driver_data; 496 em_dev_register_perf_domain(get_cpu_device(policy->cpu), 497 get_perf_level_count(policy), &em_cb, 498 cpu_data->shared_cpu_map, 0); 499 } 500 501 static void populate_efficiency_class(void) 502 { 503 struct acpi_madt_generic_interrupt *gicc; 504 DECLARE_BITMAP(used_classes, 256) = {}; 505 int class, cpu, index; 506 507 for_each_possible_cpu(cpu) { 508 gicc = acpi_cpu_get_madt_gicc(cpu); 509 class = gicc->efficiency_class; 510 bitmap_set(used_classes, class, 1); 511 } 512 513 if (bitmap_weight(used_classes, 256) <= 1) { 514 pr_debug("Efficiency classes are all equal (=%d). " 515 "No EM registered", class); 516 return; 517 } 518 519 /* 520 * Squeeze efficiency class values on [0:#efficiency_class-1]. 521 * Values are per spec in [0:255]. 522 */ 523 index = 0; 524 for_each_set_bit(class, used_classes, 256) { 525 for_each_possible_cpu(cpu) { 526 gicc = acpi_cpu_get_madt_gicc(cpu); 527 if (gicc->efficiency_class == class) 528 per_cpu(efficiency_class, cpu) = index; 529 } 530 index++; 531 } 532 cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em; 533 } 534 535 #else 536 static void populate_efficiency_class(void) 537 { 538 } 539 #endif 540 541 static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu) 542 { 543 struct cppc_cpudata *cpu_data; 544 int ret; 545 546 cpu_data = kzalloc(sizeof(struct cppc_cpudata), GFP_KERNEL); 547 if (!cpu_data) 548 goto out; 549 550 if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL)) 551 goto free_cpu; 552 553 ret = acpi_get_psd_map(cpu, cpu_data); 554 if (ret) { 555 pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret); 556 goto free_mask; 557 } 558 559 ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps); 560 if (ret) { 561 pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret); 562 goto free_mask; 563 } 564 565 return cpu_data; 566 567 free_mask: 568 free_cpumask_var(cpu_data->shared_cpu_map); 569 free_cpu: 570 kfree(cpu_data); 571 out: 572 return NULL; 573 } 574 575 static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy) 576 { 577 struct cppc_cpudata *cpu_data = policy->driver_data; 578 579 free_cpumask_var(cpu_data->shared_cpu_map); 580 kfree(cpu_data); 581 policy->driver_data = NULL; 582 } 583 584 static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) 585 { 586 unsigned int cpu = policy->cpu; 587 struct cppc_cpudata *cpu_data; 588 struct cppc_perf_caps *caps; 589 int ret; 590 591 cpu_data = cppc_cpufreq_get_cpu_data(cpu); 592 if (!cpu_data) { 593 pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu); 594 return -ENODEV; 595 } 596 caps = &cpu_data->perf_caps; 597 policy->driver_data = cpu_data; 598 599 /* 600 * Set min to lowest nonlinear perf to avoid any efficiency penalty (see 601 * Section 8.4.7.1.1.5 of ACPI 6.1 spec) 602 */ 603 policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf); 604 policy->max = cppc_perf_to_khz(caps, policy->boost_enabled ? 605 caps->highest_perf : caps->nominal_perf); 606 607 /* 608 * Set cpuinfo.min_freq to Lowest to make the full range of performance 609 * available if userspace wants to use any perf between lowest & lowest 610 * nonlinear perf 611 */ 612 policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf); 613 policy->cpuinfo.max_freq = policy->max; 614 615 policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu); 616 policy->shared_type = cpu_data->shared_type; 617 618 switch (policy->shared_type) { 619 case CPUFREQ_SHARED_TYPE_HW: 620 case CPUFREQ_SHARED_TYPE_NONE: 621 /* Nothing to be done - we'll have a policy for each CPU */ 622 break; 623 case CPUFREQ_SHARED_TYPE_ANY: 624 /* 625 * All CPUs in the domain will share a policy and all cpufreq 626 * operations will use a single cppc_cpudata structure stored 627 * in policy->driver_data. 628 */ 629 cpumask_copy(policy->cpus, cpu_data->shared_cpu_map); 630 break; 631 default: 632 pr_debug("Unsupported CPU co-ord type: %d\n", 633 policy->shared_type); 634 ret = -EFAULT; 635 goto out; 636 } 637 638 policy->fast_switch_possible = cppc_allow_fast_switch(); 639 policy->dvfs_possible_from_any_cpu = true; 640 641 /* 642 * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost 643 * is supported. 644 */ 645 if (caps->highest_perf > caps->nominal_perf) 646 policy->boost_supported = true; 647 648 /* Set policy->cur to max now. The governors will adjust later. */ 649 policy->cur = cppc_perf_to_khz(caps, caps->highest_perf); 650 cpu_data->perf_ctrls.desired_perf = caps->highest_perf; 651 652 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); 653 if (ret) { 654 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", 655 caps->highest_perf, cpu, ret); 656 goto out; 657 } 658 659 cppc_cpufreq_cpu_fie_init(policy); 660 return 0; 661 662 out: 663 cppc_cpufreq_put_cpu_data(policy); 664 return ret; 665 } 666 667 static void cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy) 668 { 669 struct cppc_cpudata *cpu_data = policy->driver_data; 670 struct cppc_perf_caps *caps = &cpu_data->perf_caps; 671 unsigned int cpu = policy->cpu; 672 int ret; 673 674 cppc_cpufreq_cpu_fie_exit(policy); 675 676 cpu_data->perf_ctrls.desired_perf = caps->lowest_perf; 677 678 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); 679 if (ret) 680 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", 681 caps->lowest_perf, cpu, ret); 682 683 cppc_cpufreq_put_cpu_data(policy); 684 } 685 686 static inline u64 get_delta(u64 t1, u64 t0) 687 { 688 if (t1 > t0 || t0 > ~(u32)0) 689 return t1 - t0; 690 691 return (u32)t1 - (u32)t0; 692 } 693 694 static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0, 695 struct cppc_perf_fb_ctrs *fb_ctrs_t1) 696 { 697 u64 delta_reference, delta_delivered; 698 u64 reference_perf; 699 700 reference_perf = fb_ctrs_t0->reference_perf; 701 702 delta_reference = get_delta(fb_ctrs_t1->reference, 703 fb_ctrs_t0->reference); 704 delta_delivered = get_delta(fb_ctrs_t1->delivered, 705 fb_ctrs_t0->delivered); 706 707 /* 708 * Avoid divide-by zero and unchanged feedback counters. 709 * Leave it for callers to handle. 710 */ 711 if (!delta_reference || !delta_delivered) 712 return 0; 713 714 return (reference_perf * delta_delivered) / delta_reference; 715 } 716 717 static int cppc_get_perf_ctrs_sample(int cpu, 718 struct cppc_perf_fb_ctrs *fb_ctrs_t0, 719 struct cppc_perf_fb_ctrs *fb_ctrs_t1) 720 { 721 int ret; 722 723 ret = cppc_get_perf_ctrs(cpu, fb_ctrs_t0); 724 if (ret) 725 return ret; 726 727 udelay(2); /* 2usec delay between sampling */ 728 729 return cppc_get_perf_ctrs(cpu, fb_ctrs_t1); 730 } 731 732 static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) 733 { 734 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 735 struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0}; 736 struct cppc_cpudata *cpu_data; 737 u64 delivered_perf; 738 int ret; 739 740 if (!policy) 741 return 0; 742 743 cpu_data = policy->driver_data; 744 745 ret = cppc_get_perf_ctrs_sample(cpu, &fb_ctrs_t0, &fb_ctrs_t1); 746 if (ret) { 747 if (ret == -EFAULT) 748 /* Any of the associated CPPC regs is 0. */ 749 goto out_invalid_counters; 750 else 751 return 0; 752 } 753 754 delivered_perf = cppc_perf_from_fbctrs(&fb_ctrs_t0, &fb_ctrs_t1); 755 if (!delivered_perf) 756 goto out_invalid_counters; 757 758 return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); 759 760 out_invalid_counters: 761 /* 762 * Feedback counters could be unchanged or 0 when a cpu enters a 763 * low-power idle state, e.g. clock-gated or power-gated. 764 * Use desired perf for reflecting frequency. Get the latest register 765 * value first as some platforms may update the actual delivered perf 766 * there; if failed, resort to the cached desired perf. 767 */ 768 if (cppc_get_desired_perf(cpu, &delivered_perf)) 769 delivered_perf = cpu_data->perf_ctrls.desired_perf; 770 771 return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); 772 } 773 774 static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state) 775 { 776 struct cppc_cpudata *cpu_data = policy->driver_data; 777 struct cppc_perf_caps *caps = &cpu_data->perf_caps; 778 int ret; 779 780 if (state) 781 policy->max = cppc_perf_to_khz(caps, caps->highest_perf); 782 else 783 policy->max = cppc_perf_to_khz(caps, caps->nominal_perf); 784 policy->cpuinfo.max_freq = policy->max; 785 786 ret = freq_qos_update_request(policy->max_freq_req, policy->max); 787 if (ret < 0) 788 return ret; 789 790 return 0; 791 } 792 793 static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) 794 { 795 struct cppc_cpudata *cpu_data = policy->driver_data; 796 797 return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf); 798 } 799 800 static ssize_t show_auto_select(struct cpufreq_policy *policy, char *buf) 801 { 802 bool val; 803 int ret; 804 805 ret = cppc_get_auto_sel(policy->cpu, &val); 806 807 /* show "<unsupported>" when this register is not supported by cpc */ 808 if (ret == -EOPNOTSUPP) 809 return sysfs_emit(buf, "<unsupported>\n"); 810 811 if (ret) 812 return ret; 813 814 return sysfs_emit(buf, "%d\n", val); 815 } 816 817 static ssize_t store_auto_select(struct cpufreq_policy *policy, 818 const char *buf, size_t count) 819 { 820 bool val; 821 int ret; 822 823 ret = kstrtobool(buf, &val); 824 if (ret) 825 return ret; 826 827 ret = cppc_set_auto_sel(policy->cpu, val); 828 if (ret) 829 return ret; 830 831 return count; 832 } 833 834 static ssize_t show_auto_act_window(struct cpufreq_policy *policy, char *buf) 835 { 836 u64 val; 837 int ret; 838 839 ret = cppc_get_auto_act_window(policy->cpu, &val); 840 841 /* show "<unsupported>" when this register is not supported by cpc */ 842 if (ret == -EOPNOTSUPP) 843 return sysfs_emit(buf, "<unsupported>\n"); 844 845 if (ret) 846 return ret; 847 848 return sysfs_emit(buf, "%llu\n", val); 849 } 850 851 static ssize_t store_auto_act_window(struct cpufreq_policy *policy, 852 const char *buf, size_t count) 853 { 854 u64 usec; 855 int ret; 856 857 ret = kstrtou64(buf, 0, &usec); 858 if (ret) 859 return ret; 860 861 ret = cppc_set_auto_act_window(policy->cpu, usec); 862 if (ret) 863 return ret; 864 865 return count; 866 } 867 868 static ssize_t show_energy_performance_preference_val(struct cpufreq_policy *policy, char *buf) 869 { 870 u64 val; 871 int ret; 872 873 ret = cppc_get_epp_perf(policy->cpu, &val); 874 875 /* show "<unsupported>" when this register is not supported by cpc */ 876 if (ret == -EOPNOTSUPP) 877 return sysfs_emit(buf, "<unsupported>\n"); 878 879 if (ret) 880 return ret; 881 882 return sysfs_emit(buf, "%llu\n", val); 883 } 884 885 static ssize_t store_energy_performance_preference_val(struct cpufreq_policy *policy, 886 const char *buf, size_t count) 887 { 888 u64 val; 889 int ret; 890 891 ret = kstrtou64(buf, 0, &val); 892 if (ret) 893 return ret; 894 895 ret = cppc_set_epp(policy->cpu, val); 896 if (ret) 897 return ret; 898 899 return count; 900 } 901 902 cpufreq_freq_attr_ro(freqdomain_cpus); 903 cpufreq_freq_attr_rw(auto_select); 904 cpufreq_freq_attr_rw(auto_act_window); 905 cpufreq_freq_attr_rw(energy_performance_preference_val); 906 907 static struct freq_attr *cppc_cpufreq_attr[] = { 908 &freqdomain_cpus, 909 &auto_select, 910 &auto_act_window, 911 &energy_performance_preference_val, 912 NULL, 913 }; 914 915 static struct cpufreq_driver cppc_cpufreq_driver = { 916 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 917 .verify = cppc_verify_policy, 918 .target = cppc_cpufreq_set_target, 919 .get = cppc_cpufreq_get_rate, 920 .fast_switch = cppc_cpufreq_fast_switch, 921 .init = cppc_cpufreq_cpu_init, 922 .exit = cppc_cpufreq_cpu_exit, 923 .set_boost = cppc_cpufreq_set_boost, 924 .attr = cppc_cpufreq_attr, 925 .name = "cppc_cpufreq", 926 }; 927 928 static int __init cppc_cpufreq_init(void) 929 { 930 int ret; 931 932 if (!acpi_cpc_valid()) 933 return -ENODEV; 934 935 cppc_freq_invariance_init(); 936 populate_efficiency_class(); 937 938 ret = cpufreq_register_driver(&cppc_cpufreq_driver); 939 if (ret) 940 cppc_freq_invariance_exit(); 941 942 return ret; 943 } 944 945 static void __exit cppc_cpufreq_exit(void) 946 { 947 cpufreq_unregister_driver(&cppc_cpufreq_driver); 948 cppc_freq_invariance_exit(); 949 } 950 951 module_exit(cppc_cpufreq_exit); 952 MODULE_AUTHOR("Ashwin Chaugule"); 953 MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec"); 954 MODULE_LICENSE("GPL"); 955 956 late_initcall(cppc_cpufreq_init); 957 958 static const struct acpi_device_id cppc_acpi_ids[] __used = { 959 {ACPI_PROCESSOR_DEVICE_HID, }, 960 {} 961 }; 962 963 MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids); 964