1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * CPPC (Collaborative Processor Performance Control) driver for 4 * interfacing with the CPUfreq layer and governors. See 5 * cppc_acpi.c for CPPC specific methods. 6 * 7 * (C) Copyright 2014, 2015 Linaro Ltd. 8 * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org> 9 */ 10 11 #define pr_fmt(fmt) "CPPC Cpufreq:" fmt 12 13 #include <linux/arch_topology.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/delay.h> 17 #include <linux/cpu.h> 18 #include <linux/cpufreq.h> 19 #include <linux/irq_work.h> 20 #include <linux/kthread.h> 21 #include <linux/mutex.h> 22 #include <linux/time.h> 23 #include <linux/vmalloc.h> 24 #include <uapi/linux/sched/types.h> 25 26 #include <linux/unaligned.h> 27 28 #include <acpi/cppc_acpi.h> 29 30 static struct cpufreq_driver cppc_cpufreq_driver; 31 32 #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE 33 static enum { 34 FIE_UNSET = -1, 35 FIE_ENABLED, 36 FIE_DISABLED 37 } fie_disabled = FIE_UNSET; 38 39 module_param(fie_disabled, int, 0444); 40 MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)"); 41 42 /* Frequency invariance support */ 43 struct cppc_freq_invariance { 44 int cpu; 45 bool pcc_work_initialized; 46 struct irq_work irq_work; 47 struct kthread_work work; 48 struct cppc_perf_fb_ctrs prev_perf_fb_ctrs; 49 struct cppc_cpudata *cpu_data; 50 }; 51 52 static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv); 53 static struct kthread_worker *kworker_fie; 54 static DEFINE_MUTEX(cppc_fie_lock); 55 56 static int cppc_perf_from_fbctrs(u64 reference_perf, 57 struct cppc_perf_fb_ctrs *fb_ctrs_t0, 58 struct cppc_perf_fb_ctrs *fb_ctrs_t1); 59 static int cppc_fie_kworker_init(void); 60 61 /** 62 * __cppc_scale_freq_tick - CPPC arch_freq_scale updater for frequency invariance 63 * @cppc_fi: per-cpu CPPC FIE data. 64 * 65 * The CPPC driver registers itself with the topology core to provide its own 66 * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which 67 * gets called by the scheduler on every tick. 68 * 69 * Note that the arch specific counters have higher priority than CPPC counters, 70 * if available, though the CPPC driver doesn't need to have any special 71 * handling for that. 72 */ 73 static void __cppc_scale_freq_tick(struct cppc_freq_invariance *cppc_fi) 74 { 75 struct cppc_perf_fb_ctrs fb_ctrs = {0}; 76 struct cppc_cpudata *cpu_data; 77 unsigned long local_freq_scale; 78 u64 perf, ref_perf; 79 80 cpu_data = cppc_fi->cpu_data; 81 82 if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) { 83 pr_warn("%s: failed to read perf counters\n", __func__); 84 return; 85 } 86 87 ref_perf = cpu_data->perf_caps.reference_perf; 88 perf = cppc_perf_from_fbctrs(ref_perf, 89 &cppc_fi->prev_perf_fb_ctrs, &fb_ctrs); 90 if (!perf) 91 return; 92 93 cppc_fi->prev_perf_fb_ctrs = fb_ctrs; 94 95 perf <<= SCHED_CAPACITY_SHIFT; 96 local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf); 97 98 /* This can happen due to counter's overflow */ 99 if (unlikely(local_freq_scale > 1024)) 100 local_freq_scale = 1024; 101 102 per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale; 103 } 104 105 static void cppc_scale_freq_tick(void) 106 { 107 __cppc_scale_freq_tick(&per_cpu(cppc_freq_inv, smp_processor_id())); 108 } 109 110 static struct scale_freq_data cppc_sftd = { 111 .source = SCALE_FREQ_SOURCE_CPPC, 112 .set_freq_scale = cppc_scale_freq_tick, 113 }; 114 115 static void cppc_scale_freq_workfn(struct kthread_work *work) 116 { 117 struct cppc_freq_invariance *cppc_fi; 118 119 cppc_fi = container_of(work, struct cppc_freq_invariance, work); 120 __cppc_scale_freq_tick(cppc_fi); 121 } 122 123 static void cppc_irq_work(struct irq_work *irq_work) 124 { 125 struct cppc_freq_invariance *cppc_fi; 126 127 cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work); 128 kthread_queue_work(kworker_fie, &cppc_fi->work); 129 } 130 131 /* 132 * Reading perf counters may sleep if the CPC regs are in PCC. Thus, we 133 * schedule an irq work in scale_freq_tick (since we reach here from hard-irq 134 * context), which then schedules a normal work item cppc_scale_freq_workfn() 135 * that updates the per_cpu arch_freq_scale variable based on the counter 136 * updates since the last tick. 137 */ 138 static void cppc_scale_freq_tick_pcc(void) 139 { 140 struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id()); 141 142 /* 143 * cppc_get_perf_ctrs() can potentially sleep, call that from the right 144 * context. 145 */ 146 irq_work_queue(&cppc_fi->irq_work); 147 } 148 149 static struct scale_freq_data cppc_sftd_pcc = { 150 .source = SCALE_FREQ_SOURCE_CPPC, 151 .set_freq_scale = cppc_scale_freq_tick_pcc, 152 }; 153 154 static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) 155 { 156 struct scale_freq_data *sftd; 157 struct cppc_freq_invariance *cppc_fi; 158 int cpu, ret; 159 160 if (fie_disabled) 161 return; 162 163 for_each_cpu(cpu, policy->cpus) { 164 cppc_fi = &per_cpu(cppc_freq_inv, cpu); 165 cppc_fi->cpu = cpu; 166 cppc_fi->cpu_data = policy->driver_data; 167 if (cppc_perf_ctrs_in_pcc_cpu(cpu)) { 168 if (cppc_fie_kworker_init()) 169 return; 170 171 kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn); 172 init_irq_work(&cppc_fi->irq_work, cppc_irq_work); 173 cppc_fi->pcc_work_initialized = true; 174 } 175 176 ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs); 177 178 /* 179 * Don't abort as the CPU was offline while the driver was 180 * getting registered. 181 */ 182 if (ret && cpu_online(cpu)) { 183 pr_debug("%s: failed to read perf counters for cpu:%d: %d\n", 184 __func__, cpu, ret); 185 return; 186 } 187 } 188 189 /* A shared policy may contain both PCC and non-PCC counters. */ 190 for_each_cpu(cpu, policy->cpus) { 191 cppc_fi = &per_cpu(cppc_freq_inv, cpu); 192 if (cppc_fi->pcc_work_initialized) 193 sftd = &cppc_sftd_pcc; 194 else 195 sftd = &cppc_sftd; 196 topology_set_scale_freq_source(sftd, cpumask_of(cpu)); 197 } 198 } 199 200 /* 201 * Drain work initialized by this policy even if processor removal has 202 * already unpublished the CPU's CPC descriptor. 203 */ 204 static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) 205 { 206 struct cppc_freq_invariance *cppc_fi; 207 int cpu; 208 209 if (fie_disabled) 210 return; 211 212 /* policy->cpus will be empty here, use related_cpus instead */ 213 topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus); 214 215 for_each_cpu(cpu, policy->related_cpus) { 216 cppc_fi = &per_cpu(cppc_freq_inv, cpu); 217 if (!cppc_fi->pcc_work_initialized) 218 continue; 219 irq_work_sync(&cppc_fi->irq_work); 220 kthread_cancel_work_sync(&cppc_fi->work); 221 cppc_fi->pcc_work_initialized = false; 222 } 223 } 224 225 static int cppc_fie_kworker_init(void) 226 { 227 struct kthread_worker *worker; 228 struct sched_attr attr = { 229 .size = sizeof(struct sched_attr), 230 .sched_policy = SCHED_DEADLINE, 231 .sched_nice = 0, 232 .sched_priority = 0, 233 /* 234 * Fake (unused) bandwidth; workaround to "fix" 235 * priority inheritance. 236 */ 237 .sched_runtime = NSEC_PER_MSEC, 238 .sched_deadline = 10 * NSEC_PER_MSEC, 239 .sched_period = 10 * NSEC_PER_MSEC, 240 }; 241 int ret; 242 243 guard(mutex)(&cppc_fie_lock); 244 245 if (kworker_fie) 246 return 0; 247 248 worker = kthread_run_worker(0, "cppc_fie"); 249 if (IS_ERR(worker)) { 250 pr_warn("%s: failed to create kworker_fie: %pe\n", __func__, worker); 251 return PTR_ERR(worker); 252 } 253 254 ret = sched_setattr_nocheck(worker->task, &attr); 255 if (ret) { 256 pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__, 257 ret); 258 kthread_destroy_worker(worker); 259 return ret; 260 } 261 262 kworker_fie = worker; 263 return 0; 264 } 265 266 static void __init cppc_freq_invariance_init(void) 267 { 268 bool perf_ctrs_in_pcc = cppc_perf_ctrs_in_pcc(); 269 270 if (fie_disabled == FIE_UNSET) { 271 if (perf_ctrs_in_pcc) { 272 pr_info("FIE not enabled on systems with registers in PCC\n"); 273 fie_disabled = FIE_DISABLED; 274 } else { 275 fie_disabled = FIE_ENABLED; 276 } 277 } 278 } 279 280 static void cppc_freq_invariance_exit(void) 281 { 282 if (kworker_fie) 283 kthread_destroy_worker(kworker_fie); 284 } 285 286 #else 287 static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) 288 { 289 } 290 291 static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) 292 { 293 } 294 295 static inline void cppc_freq_invariance_init(void) 296 { 297 } 298 299 static inline void cppc_freq_invariance_exit(void) 300 { 301 } 302 #endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */ 303 304 static void cppc_cpufreq_get_perf_limits(struct cppc_cpudata *cpu_data, 305 struct cpufreq_policy *policy, 306 u32 *min_perf, u32 *max_perf) 307 { 308 struct cppc_perf_caps *caps = &cpu_data->perf_caps; 309 unsigned int min_freq, max_freq; 310 u32 min, max; 311 312 min_freq = READ_ONCE(policy->min); 313 max_freq = READ_ONCE(policy->max); 314 if (unlikely(min_freq > max_freq)) 315 min_freq = max_freq; 316 317 min = cppc_khz_to_perf(caps, min_freq); 318 max = cppc_khz_to_perf(caps, max_freq); 319 320 *min_perf = clamp_t(u32, min, caps->lowest_perf, caps->highest_perf); 321 *max_perf = clamp_t(u32, max, caps->lowest_perf, caps->highest_perf); 322 } 323 324 static void cppc_cpufreq_update_perf_limits(struct cppc_cpudata *cpu_data, 325 struct cpufreq_policy *policy) 326 { 327 cppc_cpufreq_get_perf_limits(cpu_data, policy, 328 &cpu_data->perf_ctrls.min_perf, 329 &cpu_data->perf_ctrls.max_perf); 330 } 331 332 static int cppc_cpufreq_set_target(struct cpufreq_policy *policy, 333 unsigned int target_freq, 334 unsigned int relation) 335 { 336 struct cppc_cpudata *cpu_data = policy->driver_data; 337 unsigned int cpu = policy->cpu; 338 struct cpufreq_freqs freqs; 339 int ret = 0; 340 341 cpu_data->perf_ctrls.desired_perf = 342 cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); 343 cppc_cpufreq_update_perf_limits(cpu_data, policy); 344 345 freqs.old = policy->cur; 346 freqs.new = target_freq; 347 348 cpufreq_freq_transition_begin(policy, &freqs); 349 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); 350 cpufreq_freq_transition_end(policy, &freqs, ret != 0); 351 352 if (ret) 353 pr_debug("Failed to set target on CPU:%d. ret:%d\n", 354 cpu, ret); 355 356 return ret; 357 } 358 359 static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy, 360 unsigned int target_freq) 361 { 362 struct cppc_cpudata *cpu_data = policy->driver_data; 363 unsigned int cpu = policy->cpu; 364 u32 desired_perf; 365 int ret; 366 367 desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); 368 cpu_data->perf_ctrls.desired_perf = desired_perf; 369 cppc_cpufreq_update_perf_limits(cpu_data, policy); 370 371 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); 372 if (ret) { 373 pr_debug("Failed to set target on CPU:%d. ret:%d\n", 374 cpu, ret); 375 return 0; 376 } 377 378 return target_freq; 379 } 380 381 static int cppc_verify_policy(struct cpufreq_policy_data *policy) 382 { 383 cpufreq_verify_within_cpu_limits(policy); 384 return 0; 385 } 386 387 static unsigned int __cppc_cpufreq_get_transition_delay_us(unsigned int cpu) 388 { 389 int transition_latency_ns = cppc_get_transition_latency(cpu); 390 391 if (transition_latency_ns < 0) 392 return CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS / NSEC_PER_USEC; 393 394 return transition_latency_ns / NSEC_PER_USEC; 395 } 396 397 /* 398 * The PCC subspace describes the rate at which platform can accept commands 399 * on the shared PCC channel (including READs which do not count towards freq 400 * transition requests), so ideally we need to use the PCC values as a fallback 401 * if we don't have a platform specific transition_delay_us 402 */ 403 #ifdef CONFIG_ARM64 404 #include <asm/cputype.h> 405 406 static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) 407 { 408 unsigned long implementor = read_cpuid_implementor(); 409 unsigned long part_num = read_cpuid_part_number(); 410 411 switch (implementor) { 412 case ARM_CPU_IMP_QCOM: 413 switch (part_num) { 414 case QCOM_CPU_PART_FALKOR_V1: 415 case QCOM_CPU_PART_FALKOR: 416 return 10000; 417 } 418 } 419 return __cppc_cpufreq_get_transition_delay_us(cpu); 420 } 421 #else 422 static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) 423 { 424 return __cppc_cpufreq_get_transition_delay_us(cpu); 425 } 426 #endif 427 428 #if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL) 429 430 static DEFINE_PER_CPU(unsigned int, efficiency_class); 431 432 /* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */ 433 #define CPPC_EM_CAP_STEP (20) 434 /* Increase the cost value by CPPC_EM_COST_STEP every performance state. */ 435 #define CPPC_EM_COST_STEP (1) 436 /* Add a cost gap correspnding to the energy of 4 CPUs. */ 437 #define CPPC_EM_COST_GAP (4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \ 438 / CPPC_EM_CAP_STEP) 439 440 static unsigned int get_perf_level_count(struct cpufreq_policy *policy) 441 { 442 struct cppc_perf_caps *perf_caps; 443 unsigned int min_cap, max_cap; 444 struct cppc_cpudata *cpu_data; 445 int cpu = policy->cpu; 446 447 cpu_data = policy->driver_data; 448 perf_caps = &cpu_data->perf_caps; 449 max_cap = arch_scale_cpu_capacity(cpu); 450 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf, 451 perf_caps->highest_perf); 452 if ((min_cap == 0) || (max_cap < min_cap)) 453 return 0; 454 return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP; 455 } 456 457 /* 458 * The cost is defined as: 459 * cost = power * max_frequency / frequency 460 */ 461 static inline unsigned long compute_cost(int cpu, int step) 462 { 463 return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) + 464 step * CPPC_EM_COST_STEP; 465 } 466 467 static int cppc_get_cpu_power(struct device *cpu_dev, 468 unsigned long *power, unsigned long *KHz) 469 { 470 unsigned long perf_step, perf_prev, perf, perf_check; 471 unsigned int min_step, max_step, step, step_check; 472 unsigned long prev_freq = *KHz; 473 unsigned int min_cap, max_cap; 474 struct cpufreq_policy *policy; 475 476 struct cppc_perf_caps *perf_caps; 477 struct cppc_cpudata *cpu_data; 478 479 policy = cpufreq_cpu_get_raw(cpu_dev->id); 480 if (!policy) 481 return -EINVAL; 482 483 cpu_data = policy->driver_data; 484 perf_caps = &cpu_data->perf_caps; 485 max_cap = arch_scale_cpu_capacity(cpu_dev->id); 486 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf, 487 perf_caps->highest_perf); 488 perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf, 489 max_cap); 490 min_step = min_cap / CPPC_EM_CAP_STEP; 491 max_step = max_cap / CPPC_EM_CAP_STEP; 492 493 perf_prev = cppc_khz_to_perf(perf_caps, *KHz); 494 step = perf_prev / perf_step; 495 496 if (step > max_step) 497 return -EINVAL; 498 499 if (min_step == max_step) { 500 step = max_step; 501 perf = perf_caps->highest_perf; 502 } else if (step < min_step) { 503 step = min_step; 504 perf = perf_caps->lowest_perf; 505 } else { 506 step++; 507 if (step == max_step) 508 perf = perf_caps->highest_perf; 509 else 510 perf = step * perf_step; 511 } 512 513 *KHz = cppc_perf_to_khz(perf_caps, perf); 514 perf_check = cppc_khz_to_perf(perf_caps, *KHz); 515 step_check = perf_check / perf_step; 516 517 /* 518 * To avoid bad integer approximation, check that new frequency value 519 * increased and that the new frequency will be converted to the 520 * desired step value. 521 */ 522 while ((*KHz == prev_freq) || (step_check != step)) { 523 perf++; 524 *KHz = cppc_perf_to_khz(perf_caps, perf); 525 perf_check = cppc_khz_to_perf(perf_caps, *KHz); 526 step_check = perf_check / perf_step; 527 } 528 529 /* 530 * With an artificial EM, only the cost value is used. Still the power 531 * is populated such as 0 < power < EM_MAX_POWER. This allows to add 532 * more sense to the artificial performance states. 533 */ 534 *power = compute_cost(cpu_dev->id, step); 535 536 return 0; 537 } 538 539 static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz, 540 unsigned long *cost) 541 { 542 unsigned long perf_step, perf_prev; 543 struct cppc_perf_caps *perf_caps; 544 struct cpufreq_policy *policy; 545 struct cppc_cpudata *cpu_data; 546 unsigned int max_cap; 547 int step; 548 549 policy = cpufreq_cpu_get_raw(cpu_dev->id); 550 if (!policy) 551 return -EINVAL; 552 553 cpu_data = policy->driver_data; 554 perf_caps = &cpu_data->perf_caps; 555 max_cap = arch_scale_cpu_capacity(cpu_dev->id); 556 557 perf_prev = cppc_khz_to_perf(perf_caps, KHz); 558 perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap; 559 step = perf_prev / perf_step; 560 561 *cost = compute_cost(cpu_dev->id, step); 562 563 return 0; 564 } 565 566 static void cppc_cpufreq_register_em(struct cpufreq_policy *policy) 567 { 568 struct cppc_cpudata *cpu_data; 569 struct em_data_callback em_cb = 570 EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost); 571 572 cpu_data = policy->driver_data; 573 em_dev_register_perf_domain(get_cpu_device(policy->cpu), 574 get_perf_level_count(policy), &em_cb, 575 cpu_data->shared_cpu_map, 0); 576 } 577 578 static void populate_efficiency_class(void) 579 { 580 struct acpi_madt_generic_interrupt *gicc; 581 DECLARE_BITMAP(used_classes, 256) = {}; 582 int class, cpu, index; 583 584 for_each_possible_cpu(cpu) { 585 gicc = acpi_cpu_get_madt_gicc(cpu); 586 class = gicc->efficiency_class; 587 bitmap_set(used_classes, class, 1); 588 } 589 590 if (bitmap_weight(used_classes, 256) <= 1) { 591 pr_debug("Efficiency classes are all equal (=%d). " 592 "No EM registered", class); 593 return; 594 } 595 596 /* 597 * Squeeze efficiency class values on [0:#efficiency_class-1]. 598 * Values are per spec in [0:255]. 599 */ 600 index = 0; 601 for_each_set_bit(class, used_classes, 256) { 602 for_each_possible_cpu(cpu) { 603 gicc = acpi_cpu_get_madt_gicc(cpu); 604 if (gicc->efficiency_class == class) 605 per_cpu(efficiency_class, cpu) = index; 606 } 607 index++; 608 } 609 cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em; 610 } 611 612 #else 613 static void populate_efficiency_class(void) 614 { 615 } 616 #endif 617 618 static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu) 619 { 620 struct cppc_cpudata *cpu_data; 621 int ret; 622 623 cpu_data = kzalloc_obj(struct cppc_cpudata); 624 if (!cpu_data) 625 goto out; 626 627 if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL)) 628 goto free_cpu; 629 630 ret = acpi_get_psd_map(cpu, cpu_data); 631 if (ret) { 632 pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret); 633 goto free_mask; 634 } 635 636 ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps); 637 if (ret) { 638 pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret); 639 goto free_mask; 640 } 641 642 ret = cppc_get_perf(cpu, &cpu_data->perf_ctrls); 643 if (ret) { 644 pr_debug("Err reading CPU%d perf ctrls: ret:%d\n", cpu, ret); 645 goto free_mask; 646 } 647 648 return cpu_data; 649 650 free_mask: 651 free_cpumask_var(cpu_data->shared_cpu_map); 652 free_cpu: 653 kfree(cpu_data); 654 out: 655 return NULL; 656 } 657 658 static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy) 659 { 660 struct cppc_cpudata *cpu_data = policy->driver_data; 661 662 free_cpumask_var(cpu_data->shared_cpu_map); 663 kfree(cpu_data); 664 policy->driver_data = NULL; 665 } 666 667 static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) 668 { 669 unsigned int cpu = policy->cpu; 670 struct cppc_cpudata *cpu_data; 671 struct cppc_perf_caps *caps; 672 int ret; 673 674 cpu_data = cppc_cpufreq_get_cpu_data(cpu); 675 if (!cpu_data) { 676 pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu); 677 return -ENODEV; 678 } 679 caps = &cpu_data->perf_caps; 680 policy->driver_data = cpu_data; 681 682 /* 683 * Set min to lowest nonlinear perf to avoid any efficiency penalty (see 684 * Section 8.4.7.1.1.5 of ACPI 6.1 spec) 685 */ 686 policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf); 687 688 /* 689 * Set cpuinfo.min_freq to Lowest to make the full range of performance 690 * available if userspace wants to use any perf between lowest & lowest 691 * nonlinear perf 692 */ 693 policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf); 694 policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, policy->boost_enabled ? 695 caps->highest_perf : caps->nominal_perf); 696 697 policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu); 698 policy->shared_type = cpu_data->shared_type; 699 700 switch (policy->shared_type) { 701 case CPUFREQ_SHARED_TYPE_HW: 702 case CPUFREQ_SHARED_TYPE_NONE: 703 /* Nothing to be done - we'll have a policy for each CPU */ 704 break; 705 case CPUFREQ_SHARED_TYPE_ANY: 706 /* 707 * All CPUs in the domain will share a policy and all cpufreq 708 * operations will use a single cppc_cpudata structure stored 709 * in policy->driver_data. 710 */ 711 cpumask_copy(policy->cpus, cpu_data->shared_cpu_map); 712 break; 713 default: 714 pr_debug("Unsupported CPU co-ord type: %d\n", 715 policy->shared_type); 716 ret = -EFAULT; 717 goto out; 718 } 719 720 policy->fast_switch_possible = cppc_allow_fast_switch(policy->cpus); 721 policy->dvfs_possible_from_any_cpu = true; 722 723 /* 724 * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost 725 * is supported. 726 */ 727 if (caps->highest_perf > caps->nominal_perf) 728 policy->boost_supported = true; 729 730 /* Set policy->cur to max now. The governors will adjust later. */ 731 policy->cur = cppc_perf_to_khz(caps, caps->highest_perf); 732 cpu_data->perf_ctrls.desired_perf = caps->highest_perf; 733 734 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); 735 if (ret) { 736 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", 737 caps->highest_perf, cpu, ret); 738 goto out; 739 } 740 741 cppc_cpufreq_cpu_fie_init(policy); 742 return 0; 743 744 out: 745 cppc_cpufreq_put_cpu_data(policy); 746 return ret; 747 } 748 749 static void cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy) 750 { 751 struct cppc_cpudata *cpu_data = policy->driver_data; 752 struct cppc_perf_caps *caps = &cpu_data->perf_caps; 753 unsigned int cpu = policy->cpu; 754 int ret; 755 756 cppc_cpufreq_cpu_fie_exit(policy); 757 758 cpu_data->perf_ctrls.desired_perf = caps->lowest_perf; 759 760 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); 761 if (ret) 762 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", 763 caps->lowest_perf, cpu, ret); 764 765 cppc_cpufreq_put_cpu_data(policy); 766 } 767 768 static inline u64 get_delta(u64 t1, u64 t0) 769 { 770 if (t1 > t0 || t0 > ~(u32)0) 771 return t1 - t0; 772 773 return (u32)t1 - (u32)t0; 774 } 775 776 static int cppc_perf_from_fbctrs(u64 reference_perf, 777 struct cppc_perf_fb_ctrs *fb_ctrs_t0, 778 struct cppc_perf_fb_ctrs *fb_ctrs_t1) 779 { 780 u64 delta_reference, delta_delivered; 781 782 delta_reference = get_delta(fb_ctrs_t1->reference, 783 fb_ctrs_t0->reference); 784 delta_delivered = get_delta(fb_ctrs_t1->delivered, 785 fb_ctrs_t0->delivered); 786 787 /* 788 * Avoid divide-by zero and unchanged feedback counters. 789 * Leave it for callers to handle. 790 */ 791 if (!delta_reference || !delta_delivered) 792 return 0; 793 794 return (reference_perf * delta_delivered) / delta_reference; 795 } 796 797 static int cppc_get_perf_ctrs_sample(int cpu, 798 struct cppc_perf_fb_ctrs *fb_ctrs_t0, 799 struct cppc_perf_fb_ctrs *fb_ctrs_t1) 800 { 801 int ret; 802 803 ret = cppc_get_perf_ctrs(cpu, fb_ctrs_t0); 804 if (ret) 805 return ret; 806 807 udelay(2); /* 2usec delay between sampling */ 808 809 return cppc_get_perf_ctrs(cpu, fb_ctrs_t1); 810 } 811 812 static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) 813 { 814 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 815 struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0}; 816 struct cppc_cpudata *cpu_data; 817 u64 delivered_perf, reference_perf; 818 int ret; 819 820 if (!policy) 821 return 0; 822 823 cpu_data = policy->driver_data; 824 825 ret = cppc_get_perf_ctrs_sample(cpu, &fb_ctrs_t0, &fb_ctrs_t1); 826 if (ret) { 827 if (ret == -EFAULT) 828 /* Any of the associated CPPC regs is 0. */ 829 goto out_invalid_counters; 830 else 831 return 0; 832 } 833 834 reference_perf = cpu_data->perf_caps.reference_perf; 835 delivered_perf = cppc_perf_from_fbctrs(reference_perf, 836 &fb_ctrs_t0, &fb_ctrs_t1); 837 if (!delivered_perf) 838 goto out_invalid_counters; 839 840 return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); 841 842 out_invalid_counters: 843 /* 844 * Feedback counters could be unchanged or 0 when a cpu enters a 845 * low-power idle state, e.g. clock-gated or power-gated. 846 * Use desired perf for reflecting frequency. Get the latest register 847 * value first as some platforms may update the actual delivered perf 848 * there; if failed, resort to the cached desired perf. 849 */ 850 if (cppc_get_desired_perf(cpu, &delivered_perf) || !delivered_perf) 851 delivered_perf = cpu_data->perf_ctrls.desired_perf; 852 853 return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); 854 } 855 856 static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state) 857 { 858 struct cppc_cpudata *cpu_data = policy->driver_data; 859 struct cppc_perf_caps *caps = &cpu_data->perf_caps; 860 861 if (state) 862 policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->highest_perf); 863 else 864 policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->nominal_perf); 865 866 return 0; 867 } 868 869 static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) 870 { 871 struct cppc_cpudata *cpu_data = policy->driver_data; 872 873 return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf); 874 } 875 876 static ssize_t show_auto_select(struct cpufreq_policy *policy, char *buf) 877 { 878 bool val; 879 int ret; 880 881 ret = cppc_get_auto_sel(policy->cpu, &val); 882 883 /* show "<unsupported>" when this register is not supported by cpc */ 884 if (ret == -EOPNOTSUPP) 885 return sysfs_emit(buf, "<unsupported>\n"); 886 887 if (ret) 888 return ret; 889 890 return sysfs_emit(buf, "%d\n", val); 891 } 892 893 static ssize_t store_auto_select(struct cpufreq_policy *policy, 894 const char *buf, size_t count) 895 { 896 struct cppc_cpudata *cpu_data = policy->driver_data; 897 bool old_auto_sel = cpu_data->perf_ctrls.auto_sel; 898 bool val; 899 int ret; 900 901 ret = kstrtobool(buf, &val); 902 if (ret) 903 return ret; 904 905 ret = cppc_set_auto_sel(policy->cpu, val); 906 if (ret) 907 return ret; 908 909 cpu_data->perf_ctrls.auto_sel = val; 910 911 if (val) { 912 u32 old_min_perf = cpu_data->perf_ctrls.min_perf; 913 u32 old_max_perf = cpu_data->perf_ctrls.max_perf; 914 915 /* 916 * When enabling autonomous selection, program MIN_PERF and 917 * MAX_PERF from current policy limits so that the platform 918 * uses the correct performance bounds immediately. 919 */ 920 cppc_cpufreq_update_perf_limits(cpu_data, policy); 921 922 ret = cppc_set_perf(policy->cpu, &cpu_data->perf_ctrls); 923 if (ret) { 924 cpu_data->perf_ctrls.min_perf = old_min_perf; 925 cpu_data->perf_ctrls.max_perf = old_max_perf; 926 cppc_set_auto_sel(policy->cpu, old_auto_sel); 927 cpu_data->perf_ctrls.auto_sel = old_auto_sel; 928 return ret; 929 } 930 } 931 932 return count; 933 } 934 935 static ssize_t cppc_cpufreq_sysfs_show_u64(unsigned int cpu, 936 int (*get_func)(int, u64 *), 937 char *buf) 938 { 939 u64 val; 940 int ret = get_func((int)cpu, &val); 941 942 if (ret == -EOPNOTSUPP) 943 return sysfs_emit(buf, "<unsupported>\n"); 944 945 if (ret) 946 return ret; 947 948 return sysfs_emit(buf, "%llu\n", val); 949 } 950 951 static ssize_t cppc_cpufreq_sysfs_store_u64(unsigned int cpu, 952 int (*set_func)(int, u64), 953 const char *buf, size_t count) 954 { 955 u64 val; 956 int ret; 957 958 ret = kstrtou64(buf, 0, &val); 959 if (ret) 960 return ret; 961 962 ret = set_func((int)cpu, val); 963 964 return ret ? ret : count; 965 } 966 967 #define CPPC_CPUFREQ_ATTR_RW_U64(_name, _get_func, _set_func) \ 968 static ssize_t show_##_name(struct cpufreq_policy *policy, char *buf) \ 969 { \ 970 return cppc_cpufreq_sysfs_show_u64(policy->cpu, _get_func, buf);\ 971 } \ 972 static ssize_t store_##_name(struct cpufreq_policy *policy, \ 973 const char *buf, size_t count) \ 974 { \ 975 return cppc_cpufreq_sysfs_store_u64(policy->cpu, _set_func, \ 976 buf, count); \ 977 } 978 979 CPPC_CPUFREQ_ATTR_RW_U64(auto_act_window, cppc_get_auto_act_window, 980 cppc_set_auto_act_window) 981 982 static ssize_t 983 show_energy_performance_preference_val(struct cpufreq_policy *policy, char *buf) 984 { 985 return cppc_cpufreq_sysfs_show_u64(policy->cpu, cppc_get_epp_perf, buf); 986 } 987 988 static ssize_t 989 store_energy_performance_preference_val(struct cpufreq_policy *policy, 990 const char *buf, size_t count) 991 { 992 struct cppc_cpudata *cpu_data = policy->driver_data; 993 u64 val; 994 int ret; 995 996 ret = kstrtou64(buf, 0, &val); 997 if (ret) 998 return ret; 999 1000 ret = cppc_set_epp(policy->cpu, val); 1001 if (ret) 1002 return ret; 1003 1004 cpu_data->perf_ctrls.energy_perf = val; 1005 1006 return count; 1007 } 1008 1009 static int cppc_get_perf_limited_filtered(int cpu, u64 *perf_limited) 1010 { 1011 struct cpufreq_policy *policy; 1012 struct cppc_cpudata *cpu_data; 1013 int ret; 1014 1015 ret = cppc_get_perf_limited(cpu, perf_limited); 1016 if (ret) 1017 return ret; 1018 1019 policy = cpufreq_cpu_get_raw(cpu); 1020 if (!policy) 1021 return -EINVAL; 1022 1023 cpu_data = policy->driver_data; 1024 1025 /* 1026 * Desired Excursion is ignored when autonomous selection is 1027 * enabled. Clear the bit to avoid exposing meaningless state 1028 * to userspace. 1029 */ 1030 if (cpu_data && cpu_data->perf_ctrls.auto_sel) 1031 *perf_limited &= ~CPPC_PERF_LIMITED_DESIRED_EXCURSION; 1032 1033 return 0; 1034 } 1035 1036 CPPC_CPUFREQ_ATTR_RW_U64(perf_limited, cppc_get_perf_limited_filtered, 1037 cppc_set_perf_limited) 1038 1039 cpufreq_freq_attr_ro(freqdomain_cpus); 1040 cpufreq_freq_attr_rw(auto_select); 1041 cpufreq_freq_attr_rw(auto_act_window); 1042 cpufreq_freq_attr_rw(energy_performance_preference_val); 1043 cpufreq_freq_attr_rw(perf_limited); 1044 1045 static struct freq_attr *cppc_cpufreq_attr[] = { 1046 &freqdomain_cpus, 1047 &auto_select, 1048 &auto_act_window, 1049 &energy_performance_preference_val, 1050 &perf_limited, 1051 NULL, 1052 }; 1053 1054 static struct cpufreq_driver cppc_cpufreq_driver = { 1055 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 1056 .verify = cppc_verify_policy, 1057 .target = cppc_cpufreq_set_target, 1058 .get = cppc_cpufreq_get_rate, 1059 .fast_switch = cppc_cpufreq_fast_switch, 1060 .init = cppc_cpufreq_cpu_init, 1061 .exit = cppc_cpufreq_cpu_exit, 1062 .set_boost = cppc_cpufreq_set_boost, 1063 .attr = cppc_cpufreq_attr, 1064 .name = "cppc_cpufreq", 1065 }; 1066 1067 static int __init cppc_cpufreq_init(void) 1068 { 1069 int ret; 1070 1071 if (!acpi_cpc_valid()) 1072 return -ENODEV; 1073 1074 cppc_freq_invariance_init(); 1075 populate_efficiency_class(); 1076 1077 ret = cpufreq_register_driver(&cppc_cpufreq_driver); 1078 if (ret) 1079 cppc_freq_invariance_exit(); 1080 1081 return ret; 1082 } 1083 1084 static void __exit cppc_cpufreq_exit(void) 1085 { 1086 cpufreq_unregister_driver(&cppc_cpufreq_driver); 1087 cppc_freq_invariance_exit(); 1088 } 1089 1090 module_exit(cppc_cpufreq_exit); 1091 MODULE_AUTHOR("Ashwin Chaugule"); 1092 MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec"); 1093 MODULE_LICENSE("GPL"); 1094 1095 late_initcall(cppc_cpufreq_init); 1096 1097 static const struct acpi_device_id cppc_acpi_ids[] __used = { 1098 {ACPI_PROCESSOR_DEVICE_HID, }, 1099 {} 1100 }; 1101 1102 MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids); 1103