1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * CPPC (Collaborative Processor Performance Control) driver for
4 * interfacing with the CPUfreq layer and governors. See
5 * cppc_acpi.c for CPPC specific methods.
6 *
7 * (C) Copyright 2014, 2015 Linaro Ltd.
8 * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org>
9 */
10
11 #define pr_fmt(fmt) "CPPC Cpufreq:" fmt
12
13 #include <linux/arch_topology.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/delay.h>
17 #include <linux/cpu.h>
18 #include <linux/cpufreq.h>
19 #include <linux/irq_work.h>
20 #include <linux/kthread.h>
21 #include <linux/time.h>
22 #include <linux/vmalloc.h>
23 #include <uapi/linux/sched/types.h>
24
25 #include <linux/unaligned.h>
26
27 #include <acpi/cppc_acpi.h>
28
29 /*
30 * This list contains information parsed from per CPU ACPI _CPC and _PSD
31 * structures: e.g. the highest and lowest supported performance, capabilities,
32 * desired performance, level requested etc. Depending on the share_type, not
33 * all CPUs will have an entry in the list.
34 */
35 static LIST_HEAD(cpu_data_list);
36
37 static bool boost_supported;
38
39 struct cppc_workaround_oem_info {
40 char oem_id[ACPI_OEM_ID_SIZE + 1];
41 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
42 u32 oem_revision;
43 };
44
45 static struct cppc_workaround_oem_info wa_info[] = {
46 {
47 .oem_id = "HISI ",
48 .oem_table_id = "HIP07 ",
49 .oem_revision = 0,
50 }, {
51 .oem_id = "HISI ",
52 .oem_table_id = "HIP08 ",
53 .oem_revision = 0,
54 }
55 };
56
57 static struct cpufreq_driver cppc_cpufreq_driver;
58
59 static enum {
60 FIE_UNSET = -1,
61 FIE_ENABLED,
62 FIE_DISABLED
63 } fie_disabled = FIE_UNSET;
64
65 #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE
66 module_param(fie_disabled, int, 0444);
67 MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)");
68
69 /* Frequency invariance support */
70 struct cppc_freq_invariance {
71 int cpu;
72 struct irq_work irq_work;
73 struct kthread_work work;
74 struct cppc_perf_fb_ctrs prev_perf_fb_ctrs;
75 struct cppc_cpudata *cpu_data;
76 };
77
78 static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv);
79 static struct kthread_worker *kworker_fie;
80
81 static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu);
82 static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,
83 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
84 struct cppc_perf_fb_ctrs *fb_ctrs_t1);
85
86 /**
87 * cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance
88 * @work: The work item.
89 *
90 * The CPPC driver register itself with the topology core to provide its own
91 * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which
92 * gets called by the scheduler on every tick.
93 *
94 * Note that the arch specific counters have higher priority than CPPC counters,
95 * if available, though the CPPC driver doesn't need to have any special
96 * handling for that.
97 *
98 * On an invocation of cppc_scale_freq_tick(), we schedule an irq work (since we
99 * reach here from hard-irq context), which then schedules a normal work item
100 * and cppc_scale_freq_workfn() updates the per_cpu arch_freq_scale variable
101 * based on the counter updates since the last tick.
102 */
cppc_scale_freq_workfn(struct kthread_work * work)103 static void cppc_scale_freq_workfn(struct kthread_work *work)
104 {
105 struct cppc_freq_invariance *cppc_fi;
106 struct cppc_perf_fb_ctrs fb_ctrs = {0};
107 struct cppc_cpudata *cpu_data;
108 unsigned long local_freq_scale;
109 u64 perf;
110
111 cppc_fi = container_of(work, struct cppc_freq_invariance, work);
112 cpu_data = cppc_fi->cpu_data;
113
114 if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) {
115 pr_warn("%s: failed to read perf counters\n", __func__);
116 return;
117 }
118
119 perf = cppc_perf_from_fbctrs(cpu_data, &cppc_fi->prev_perf_fb_ctrs,
120 &fb_ctrs);
121 cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
122
123 perf <<= SCHED_CAPACITY_SHIFT;
124 local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf);
125
126 /* This can happen due to counter's overflow */
127 if (unlikely(local_freq_scale > 1024))
128 local_freq_scale = 1024;
129
130 per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale;
131 }
132
cppc_irq_work(struct irq_work * irq_work)133 static void cppc_irq_work(struct irq_work *irq_work)
134 {
135 struct cppc_freq_invariance *cppc_fi;
136
137 cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work);
138 kthread_queue_work(kworker_fie, &cppc_fi->work);
139 }
140
cppc_scale_freq_tick(void)141 static void cppc_scale_freq_tick(void)
142 {
143 struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id());
144
145 /*
146 * cppc_get_perf_ctrs() can potentially sleep, call that from the right
147 * context.
148 */
149 irq_work_queue(&cppc_fi->irq_work);
150 }
151
152 static struct scale_freq_data cppc_sftd = {
153 .source = SCALE_FREQ_SOURCE_CPPC,
154 .set_freq_scale = cppc_scale_freq_tick,
155 };
156
cppc_cpufreq_cpu_fie_init(struct cpufreq_policy * policy)157 static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
158 {
159 struct cppc_freq_invariance *cppc_fi;
160 int cpu, ret;
161
162 if (fie_disabled)
163 return;
164
165 for_each_cpu(cpu, policy->cpus) {
166 cppc_fi = &per_cpu(cppc_freq_inv, cpu);
167 cppc_fi->cpu = cpu;
168 cppc_fi->cpu_data = policy->driver_data;
169 kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn);
170 init_irq_work(&cppc_fi->irq_work, cppc_irq_work);
171
172 ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs);
173 if (ret) {
174 pr_warn("%s: failed to read perf counters for cpu:%d: %d\n",
175 __func__, cpu, ret);
176
177 /*
178 * Don't abort if the CPU was offline while the driver
179 * was getting registered.
180 */
181 if (cpu_online(cpu))
182 return;
183 }
184 }
185
186 /* Register for freq-invariance */
187 topology_set_scale_freq_source(&cppc_sftd, policy->cpus);
188 }
189
190 /*
191 * We free all the resources on policy's removal and not on CPU removal as the
192 * irq-work are per-cpu and the hotplug core takes care of flushing the pending
193 * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work
194 * fires on another CPU after the concerned CPU is removed, it won't harm.
195 *
196 * We just need to make sure to remove them all on policy->exit().
197 */
cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy * policy)198 static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
199 {
200 struct cppc_freq_invariance *cppc_fi;
201 int cpu;
202
203 if (fie_disabled)
204 return;
205
206 /* policy->cpus will be empty here, use related_cpus instead */
207 topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus);
208
209 for_each_cpu(cpu, policy->related_cpus) {
210 cppc_fi = &per_cpu(cppc_freq_inv, cpu);
211 irq_work_sync(&cppc_fi->irq_work);
212 kthread_cancel_work_sync(&cppc_fi->work);
213 }
214 }
215
cppc_freq_invariance_init(void)216 static void __init cppc_freq_invariance_init(void)
217 {
218 struct sched_attr attr = {
219 .size = sizeof(struct sched_attr),
220 .sched_policy = SCHED_DEADLINE,
221 .sched_nice = 0,
222 .sched_priority = 0,
223 /*
224 * Fake (unused) bandwidth; workaround to "fix"
225 * priority inheritance.
226 */
227 .sched_runtime = NSEC_PER_MSEC,
228 .sched_deadline = 10 * NSEC_PER_MSEC,
229 .sched_period = 10 * NSEC_PER_MSEC,
230 };
231 int ret;
232
233 if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) {
234 fie_disabled = FIE_ENABLED;
235 if (cppc_perf_ctrs_in_pcc()) {
236 pr_info("FIE not enabled on systems with registers in PCC\n");
237 fie_disabled = FIE_DISABLED;
238 }
239 }
240
241 if (fie_disabled)
242 return;
243
244 kworker_fie = kthread_create_worker(0, "cppc_fie");
245 if (IS_ERR(kworker_fie)) {
246 pr_warn("%s: failed to create kworker_fie: %ld\n", __func__,
247 PTR_ERR(kworker_fie));
248 fie_disabled = FIE_DISABLED;
249 return;
250 }
251
252 ret = sched_setattr_nocheck(kworker_fie->task, &attr);
253 if (ret) {
254 pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__,
255 ret);
256 kthread_destroy_worker(kworker_fie);
257 fie_disabled = FIE_DISABLED;
258 }
259 }
260
cppc_freq_invariance_exit(void)261 static void cppc_freq_invariance_exit(void)
262 {
263 if (fie_disabled)
264 return;
265
266 kthread_destroy_worker(kworker_fie);
267 }
268
269 #else
cppc_cpufreq_cpu_fie_init(struct cpufreq_policy * policy)270 static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
271 {
272 }
273
cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy * policy)274 static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
275 {
276 }
277
cppc_freq_invariance_init(void)278 static inline void cppc_freq_invariance_init(void)
279 {
280 }
281
cppc_freq_invariance_exit(void)282 static inline void cppc_freq_invariance_exit(void)
283 {
284 }
285 #endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */
286
cppc_cpufreq_set_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)287 static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
288 unsigned int target_freq,
289 unsigned int relation)
290 {
291 struct cppc_cpudata *cpu_data = policy->driver_data;
292 unsigned int cpu = policy->cpu;
293 struct cpufreq_freqs freqs;
294 int ret = 0;
295
296 cpu_data->perf_ctrls.desired_perf =
297 cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);
298 freqs.old = policy->cur;
299 freqs.new = target_freq;
300
301 cpufreq_freq_transition_begin(policy, &freqs);
302 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
303 cpufreq_freq_transition_end(policy, &freqs, ret != 0);
304
305 if (ret)
306 pr_debug("Failed to set target on CPU:%d. ret:%d\n",
307 cpu, ret);
308
309 return ret;
310 }
311
cppc_cpufreq_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)312 static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy,
313 unsigned int target_freq)
314 {
315 struct cppc_cpudata *cpu_data = policy->driver_data;
316 unsigned int cpu = policy->cpu;
317 u32 desired_perf;
318 int ret;
319
320 desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);
321 cpu_data->perf_ctrls.desired_perf = desired_perf;
322 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
323
324 if (ret) {
325 pr_debug("Failed to set target on CPU:%d. ret:%d\n",
326 cpu, ret);
327 return 0;
328 }
329
330 return target_freq;
331 }
332
cppc_verify_policy(struct cpufreq_policy_data * policy)333 static int cppc_verify_policy(struct cpufreq_policy_data *policy)
334 {
335 cpufreq_verify_within_cpu_limits(policy);
336 return 0;
337 }
338
339 /*
340 * The PCC subspace describes the rate at which platform can accept commands
341 * on the shared PCC channel (including READs which do not count towards freq
342 * transition requests), so ideally we need to use the PCC values as a fallback
343 * if we don't have a platform specific transition_delay_us
344 */
345 #ifdef CONFIG_ARM64
346 #include <asm/cputype.h>
347
cppc_cpufreq_get_transition_delay_us(unsigned int cpu)348 static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
349 {
350 unsigned long implementor = read_cpuid_implementor();
351 unsigned long part_num = read_cpuid_part_number();
352
353 switch (implementor) {
354 case ARM_CPU_IMP_QCOM:
355 switch (part_num) {
356 case QCOM_CPU_PART_FALKOR_V1:
357 case QCOM_CPU_PART_FALKOR:
358 return 10000;
359 }
360 }
361 return cppc_get_transition_latency(cpu) / NSEC_PER_USEC;
362 }
363 #else
cppc_cpufreq_get_transition_delay_us(unsigned int cpu)364 static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
365 {
366 return cppc_get_transition_latency(cpu) / NSEC_PER_USEC;
367 }
368 #endif
369
370 #if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL)
371
372 static DEFINE_PER_CPU(unsigned int, efficiency_class);
373 static void cppc_cpufreq_register_em(struct cpufreq_policy *policy);
374
375 /* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */
376 #define CPPC_EM_CAP_STEP (20)
377 /* Increase the cost value by CPPC_EM_COST_STEP every performance state. */
378 #define CPPC_EM_COST_STEP (1)
379 /* Add a cost gap correspnding to the energy of 4 CPUs. */
380 #define CPPC_EM_COST_GAP (4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \
381 / CPPC_EM_CAP_STEP)
382
get_perf_level_count(struct cpufreq_policy * policy)383 static unsigned int get_perf_level_count(struct cpufreq_policy *policy)
384 {
385 struct cppc_perf_caps *perf_caps;
386 unsigned int min_cap, max_cap;
387 struct cppc_cpudata *cpu_data;
388 int cpu = policy->cpu;
389
390 cpu_data = policy->driver_data;
391 perf_caps = &cpu_data->perf_caps;
392 max_cap = arch_scale_cpu_capacity(cpu);
393 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,
394 perf_caps->highest_perf);
395 if ((min_cap == 0) || (max_cap < min_cap))
396 return 0;
397 return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP;
398 }
399
400 /*
401 * The cost is defined as:
402 * cost = power * max_frequency / frequency
403 */
compute_cost(int cpu,int step)404 static inline unsigned long compute_cost(int cpu, int step)
405 {
406 return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) +
407 step * CPPC_EM_COST_STEP;
408 }
409
cppc_get_cpu_power(struct device * cpu_dev,unsigned long * power,unsigned long * KHz)410 static int cppc_get_cpu_power(struct device *cpu_dev,
411 unsigned long *power, unsigned long *KHz)
412 {
413 unsigned long perf_step, perf_prev, perf, perf_check;
414 unsigned int min_step, max_step, step, step_check;
415 unsigned long prev_freq = *KHz;
416 unsigned int min_cap, max_cap;
417 struct cpufreq_policy *policy;
418
419 struct cppc_perf_caps *perf_caps;
420 struct cppc_cpudata *cpu_data;
421
422 policy = cpufreq_cpu_get_raw(cpu_dev->id);
423 cpu_data = policy->driver_data;
424 perf_caps = &cpu_data->perf_caps;
425 max_cap = arch_scale_cpu_capacity(cpu_dev->id);
426 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,
427 perf_caps->highest_perf);
428 perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf,
429 max_cap);
430 min_step = min_cap / CPPC_EM_CAP_STEP;
431 max_step = max_cap / CPPC_EM_CAP_STEP;
432
433 perf_prev = cppc_khz_to_perf(perf_caps, *KHz);
434 step = perf_prev / perf_step;
435
436 if (step > max_step)
437 return -EINVAL;
438
439 if (min_step == max_step) {
440 step = max_step;
441 perf = perf_caps->highest_perf;
442 } else if (step < min_step) {
443 step = min_step;
444 perf = perf_caps->lowest_perf;
445 } else {
446 step++;
447 if (step == max_step)
448 perf = perf_caps->highest_perf;
449 else
450 perf = step * perf_step;
451 }
452
453 *KHz = cppc_perf_to_khz(perf_caps, perf);
454 perf_check = cppc_khz_to_perf(perf_caps, *KHz);
455 step_check = perf_check / perf_step;
456
457 /*
458 * To avoid bad integer approximation, check that new frequency value
459 * increased and that the new frequency will be converted to the
460 * desired step value.
461 */
462 while ((*KHz == prev_freq) || (step_check != step)) {
463 perf++;
464 *KHz = cppc_perf_to_khz(perf_caps, perf);
465 perf_check = cppc_khz_to_perf(perf_caps, *KHz);
466 step_check = perf_check / perf_step;
467 }
468
469 /*
470 * With an artificial EM, only the cost value is used. Still the power
471 * is populated such as 0 < power < EM_MAX_POWER. This allows to add
472 * more sense to the artificial performance states.
473 */
474 *power = compute_cost(cpu_dev->id, step);
475
476 return 0;
477 }
478
cppc_get_cpu_cost(struct device * cpu_dev,unsigned long KHz,unsigned long * cost)479 static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz,
480 unsigned long *cost)
481 {
482 unsigned long perf_step, perf_prev;
483 struct cppc_perf_caps *perf_caps;
484 struct cpufreq_policy *policy;
485 struct cppc_cpudata *cpu_data;
486 unsigned int max_cap;
487 int step;
488
489 policy = cpufreq_cpu_get_raw(cpu_dev->id);
490 cpu_data = policy->driver_data;
491 perf_caps = &cpu_data->perf_caps;
492 max_cap = arch_scale_cpu_capacity(cpu_dev->id);
493
494 perf_prev = cppc_khz_to_perf(perf_caps, KHz);
495 perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap;
496 step = perf_prev / perf_step;
497
498 *cost = compute_cost(cpu_dev->id, step);
499
500 return 0;
501 }
502
populate_efficiency_class(void)503 static int populate_efficiency_class(void)
504 {
505 struct acpi_madt_generic_interrupt *gicc;
506 DECLARE_BITMAP(used_classes, 256) = {};
507 int class, cpu, index;
508
509 for_each_possible_cpu(cpu) {
510 gicc = acpi_cpu_get_madt_gicc(cpu);
511 class = gicc->efficiency_class;
512 bitmap_set(used_classes, class, 1);
513 }
514
515 if (bitmap_weight(used_classes, 256) <= 1) {
516 pr_debug("Efficiency classes are all equal (=%d). "
517 "No EM registered", class);
518 return -EINVAL;
519 }
520
521 /*
522 * Squeeze efficiency class values on [0:#efficiency_class-1].
523 * Values are per spec in [0:255].
524 */
525 index = 0;
526 for_each_set_bit(class, used_classes, 256) {
527 for_each_possible_cpu(cpu) {
528 gicc = acpi_cpu_get_madt_gicc(cpu);
529 if (gicc->efficiency_class == class)
530 per_cpu(efficiency_class, cpu) = index;
531 }
532 index++;
533 }
534 cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em;
535
536 return 0;
537 }
538
cppc_cpufreq_register_em(struct cpufreq_policy * policy)539 static void cppc_cpufreq_register_em(struct cpufreq_policy *policy)
540 {
541 struct cppc_cpudata *cpu_data;
542 struct em_data_callback em_cb =
543 EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost);
544
545 cpu_data = policy->driver_data;
546 em_dev_register_perf_domain(get_cpu_device(policy->cpu),
547 get_perf_level_count(policy), &em_cb,
548 cpu_data->shared_cpu_map, 0);
549 }
550
551 #else
populate_efficiency_class(void)552 static int populate_efficiency_class(void)
553 {
554 return 0;
555 }
556 #endif
557
cppc_cpufreq_get_cpu_data(unsigned int cpu)558 static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu)
559 {
560 struct cppc_cpudata *cpu_data;
561 int ret;
562
563 cpu_data = kzalloc(sizeof(struct cppc_cpudata), GFP_KERNEL);
564 if (!cpu_data)
565 goto out;
566
567 if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL))
568 goto free_cpu;
569
570 ret = acpi_get_psd_map(cpu, cpu_data);
571 if (ret) {
572 pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret);
573 goto free_mask;
574 }
575
576 ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps);
577 if (ret) {
578 pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret);
579 goto free_mask;
580 }
581
582 list_add(&cpu_data->node, &cpu_data_list);
583
584 return cpu_data;
585
586 free_mask:
587 free_cpumask_var(cpu_data->shared_cpu_map);
588 free_cpu:
589 kfree(cpu_data);
590 out:
591 return NULL;
592 }
593
cppc_cpufreq_put_cpu_data(struct cpufreq_policy * policy)594 static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy)
595 {
596 struct cppc_cpudata *cpu_data = policy->driver_data;
597
598 list_del(&cpu_data->node);
599 free_cpumask_var(cpu_data->shared_cpu_map);
600 kfree(cpu_data);
601 policy->driver_data = NULL;
602 }
603
cppc_cpufreq_cpu_init(struct cpufreq_policy * policy)604 static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
605 {
606 unsigned int cpu = policy->cpu;
607 struct cppc_cpudata *cpu_data;
608 struct cppc_perf_caps *caps;
609 int ret;
610
611 cpu_data = cppc_cpufreq_get_cpu_data(cpu);
612 if (!cpu_data) {
613 pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu);
614 return -ENODEV;
615 }
616 caps = &cpu_data->perf_caps;
617 policy->driver_data = cpu_data;
618
619 /*
620 * Set min to lowest nonlinear perf to avoid any efficiency penalty (see
621 * Section 8.4.7.1.1.5 of ACPI 6.1 spec)
622 */
623 policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf);
624 policy->max = cppc_perf_to_khz(caps, caps->nominal_perf);
625
626 /*
627 * Set cpuinfo.min_freq to Lowest to make the full range of performance
628 * available if userspace wants to use any perf between lowest & lowest
629 * nonlinear perf
630 */
631 policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf);
632 policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->nominal_perf);
633
634 policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu);
635 policy->shared_type = cpu_data->shared_type;
636
637 switch (policy->shared_type) {
638 case CPUFREQ_SHARED_TYPE_HW:
639 case CPUFREQ_SHARED_TYPE_NONE:
640 /* Nothing to be done - we'll have a policy for each CPU */
641 break;
642 case CPUFREQ_SHARED_TYPE_ANY:
643 /*
644 * All CPUs in the domain will share a policy and all cpufreq
645 * operations will use a single cppc_cpudata structure stored
646 * in policy->driver_data.
647 */
648 cpumask_copy(policy->cpus, cpu_data->shared_cpu_map);
649 break;
650 default:
651 pr_debug("Unsupported CPU co-ord type: %d\n",
652 policy->shared_type);
653 ret = -EFAULT;
654 goto out;
655 }
656
657 policy->fast_switch_possible = cppc_allow_fast_switch();
658 policy->dvfs_possible_from_any_cpu = true;
659
660 /*
661 * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost
662 * is supported.
663 */
664 if (caps->highest_perf > caps->nominal_perf)
665 boost_supported = true;
666
667 /* Set policy->cur to max now. The governors will adjust later. */
668 policy->cur = cppc_perf_to_khz(caps, caps->highest_perf);
669 cpu_data->perf_ctrls.desired_perf = caps->highest_perf;
670
671 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
672 if (ret) {
673 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
674 caps->highest_perf, cpu, ret);
675 goto out;
676 }
677
678 cppc_cpufreq_cpu_fie_init(policy);
679 return 0;
680
681 out:
682 cppc_cpufreq_put_cpu_data(policy);
683 return ret;
684 }
685
cppc_cpufreq_cpu_exit(struct cpufreq_policy * policy)686 static void cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy)
687 {
688 struct cppc_cpudata *cpu_data = policy->driver_data;
689 struct cppc_perf_caps *caps = &cpu_data->perf_caps;
690 unsigned int cpu = policy->cpu;
691 int ret;
692
693 cppc_cpufreq_cpu_fie_exit(policy);
694
695 cpu_data->perf_ctrls.desired_perf = caps->lowest_perf;
696
697 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
698 if (ret)
699 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
700 caps->lowest_perf, cpu, ret);
701
702 cppc_cpufreq_put_cpu_data(policy);
703 }
704
get_delta(u64 t1,u64 t0)705 static inline u64 get_delta(u64 t1, u64 t0)
706 {
707 if (t1 > t0 || t0 > ~(u32)0)
708 return t1 - t0;
709
710 return (u32)t1 - (u32)t0;
711 }
712
cppc_perf_from_fbctrs(struct cppc_cpudata * cpu_data,struct cppc_perf_fb_ctrs * fb_ctrs_t0,struct cppc_perf_fb_ctrs * fb_ctrs_t1)713 static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,
714 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
715 struct cppc_perf_fb_ctrs *fb_ctrs_t1)
716 {
717 u64 delta_reference, delta_delivered;
718 u64 reference_perf;
719
720 reference_perf = fb_ctrs_t0->reference_perf;
721
722 delta_reference = get_delta(fb_ctrs_t1->reference,
723 fb_ctrs_t0->reference);
724 delta_delivered = get_delta(fb_ctrs_t1->delivered,
725 fb_ctrs_t0->delivered);
726
727 /* Check to avoid divide-by zero and invalid delivered_perf */
728 if (!delta_reference || !delta_delivered)
729 return cpu_data->perf_ctrls.desired_perf;
730
731 return (reference_perf * delta_delivered) / delta_reference;
732 }
733
cppc_cpufreq_get_rate(unsigned int cpu)734 static unsigned int cppc_cpufreq_get_rate(unsigned int cpu)
735 {
736 struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0};
737 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
738 struct cppc_cpudata *cpu_data;
739 u64 delivered_perf;
740 int ret;
741
742 if (!policy)
743 return -ENODEV;
744
745 cpu_data = policy->driver_data;
746
747 cpufreq_cpu_put(policy);
748
749 ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t0);
750 if (ret)
751 return 0;
752
753 udelay(2); /* 2usec delay between sampling */
754
755 ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t1);
756 if (ret)
757 return 0;
758
759 delivered_perf = cppc_perf_from_fbctrs(cpu_data, &fb_ctrs_t0,
760 &fb_ctrs_t1);
761
762 return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf);
763 }
764
cppc_cpufreq_set_boost(struct cpufreq_policy * policy,int state)765 static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
766 {
767 struct cppc_cpudata *cpu_data = policy->driver_data;
768 struct cppc_perf_caps *caps = &cpu_data->perf_caps;
769 int ret;
770
771 if (!boost_supported) {
772 pr_err("BOOST not supported by CPU or firmware\n");
773 return -EINVAL;
774 }
775
776 if (state)
777 policy->max = cppc_perf_to_khz(caps, caps->highest_perf);
778 else
779 policy->max = cppc_perf_to_khz(caps, caps->nominal_perf);
780 policy->cpuinfo.max_freq = policy->max;
781
782 ret = freq_qos_update_request(policy->max_freq_req, policy->max);
783 if (ret < 0)
784 return ret;
785
786 return 0;
787 }
788
show_freqdomain_cpus(struct cpufreq_policy * policy,char * buf)789 static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)
790 {
791 struct cppc_cpudata *cpu_data = policy->driver_data;
792
793 return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf);
794 }
795 cpufreq_freq_attr_ro(freqdomain_cpus);
796
797 static struct freq_attr *cppc_cpufreq_attr[] = {
798 &freqdomain_cpus,
799 NULL,
800 };
801
802 static struct cpufreq_driver cppc_cpufreq_driver = {
803 .flags = CPUFREQ_CONST_LOOPS,
804 .verify = cppc_verify_policy,
805 .target = cppc_cpufreq_set_target,
806 .get = cppc_cpufreq_get_rate,
807 .fast_switch = cppc_cpufreq_fast_switch,
808 .init = cppc_cpufreq_cpu_init,
809 .exit = cppc_cpufreq_cpu_exit,
810 .set_boost = cppc_cpufreq_set_boost,
811 .attr = cppc_cpufreq_attr,
812 .name = "cppc_cpufreq",
813 };
814
815 /*
816 * HISI platform does not support delivered performance counter and
817 * reference performance counter. It can calculate the performance using the
818 * platform specific mechanism. We reuse the desired performance register to
819 * store the real performance calculated by the platform.
820 */
hisi_cppc_cpufreq_get_rate(unsigned int cpu)821 static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu)
822 {
823 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
824 struct cppc_cpudata *cpu_data;
825 u64 desired_perf;
826 int ret;
827
828 if (!policy)
829 return -ENODEV;
830
831 cpu_data = policy->driver_data;
832
833 cpufreq_cpu_put(policy);
834
835 ret = cppc_get_desired_perf(cpu, &desired_perf);
836 if (ret < 0)
837 return -EIO;
838
839 return cppc_perf_to_khz(&cpu_data->perf_caps, desired_perf);
840 }
841
cppc_check_hisi_workaround(void)842 static void cppc_check_hisi_workaround(void)
843 {
844 struct acpi_table_header *tbl;
845 acpi_status status = AE_OK;
846 int i;
847
848 status = acpi_get_table(ACPI_SIG_PCCT, 0, &tbl);
849 if (ACPI_FAILURE(status) || !tbl)
850 return;
851
852 for (i = 0; i < ARRAY_SIZE(wa_info); i++) {
853 if (!memcmp(wa_info[i].oem_id, tbl->oem_id, ACPI_OEM_ID_SIZE) &&
854 !memcmp(wa_info[i].oem_table_id, tbl->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
855 wa_info[i].oem_revision == tbl->oem_revision) {
856 /* Overwrite the get() callback */
857 cppc_cpufreq_driver.get = hisi_cppc_cpufreq_get_rate;
858 fie_disabled = FIE_DISABLED;
859 break;
860 }
861 }
862
863 acpi_put_table(tbl);
864 }
865
cppc_cpufreq_init(void)866 static int __init cppc_cpufreq_init(void)
867 {
868 int ret;
869
870 if (!acpi_cpc_valid())
871 return -ENODEV;
872
873 cppc_check_hisi_workaround();
874 cppc_freq_invariance_init();
875 populate_efficiency_class();
876
877 ret = cpufreq_register_driver(&cppc_cpufreq_driver);
878 if (ret)
879 cppc_freq_invariance_exit();
880
881 return ret;
882 }
883
free_cpu_data(void)884 static inline void free_cpu_data(void)
885 {
886 struct cppc_cpudata *iter, *tmp;
887
888 list_for_each_entry_safe(iter, tmp, &cpu_data_list, node) {
889 free_cpumask_var(iter->shared_cpu_map);
890 list_del(&iter->node);
891 kfree(iter);
892 }
893
894 }
895
cppc_cpufreq_exit(void)896 static void __exit cppc_cpufreq_exit(void)
897 {
898 cpufreq_unregister_driver(&cppc_cpufreq_driver);
899 cppc_freq_invariance_exit();
900
901 free_cpu_data();
902 }
903
904 module_exit(cppc_cpufreq_exit);
905 MODULE_AUTHOR("Ashwin Chaugule");
906 MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec");
907 MODULE_LICENSE("GPL");
908
909 late_initcall(cppc_cpufreq_init);
910
911 static const struct acpi_device_id cppc_acpi_ids[] __used = {
912 {ACPI_PROCESSOR_DEVICE_HID, },
913 {}
914 };
915
916 MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids);
917