xref: /linux/drivers/cpufreq/amd-pstate.c (revision 6562c9acb43ac69ba5a956b0c3911b883d90541f)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * amd-pstate.c - AMD Processor P-state Frequency Driver
4  *
5  * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
6  *
7  * Author: Huang Rui <ray.huang@amd.com>
8  *
9  * AMD P-State introduces a new CPU performance scaling design for AMD
10  * processors using the ACPI Collaborative Performance and Power Control (CPPC)
11  * feature which works with the AMD SMU firmware providing a finer grained
12  * frequency control range. It is to replace the legacy ACPI P-States control,
13  * allows a flexible, low-latency interface for the Linux kernel to directly
14  * communicate the performance hints to hardware.
15  *
16  * AMD P-State is supported on recent AMD Zen base CPU series include some of
17  * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
18  * P-State supported system. And there are two types of hardware implementations
19  * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
20  * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
21  */
22 
23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
24 
25 #include <linux/kernel.h>
26 #include <linux/module.h>
27 #include <linux/init.h>
28 #include <linux/smp.h>
29 #include <linux/sched.h>
30 #include <linux/cpufreq.h>
31 #include <linux/compiler.h>
32 #include <linux/dmi.h>
33 #include <linux/slab.h>
34 #include <linux/acpi.h>
35 #include <linux/io.h>
36 #include <linux/delay.h>
37 #include <linux/uaccess.h>
38 #include <linux/static_call.h>
39 
40 #include <acpi/processor.h>
41 #include <acpi/cppc_acpi.h>
42 
43 #include <asm/msr.h>
44 #include <asm/processor.h>
45 #include <asm/cpufeature.h>
46 #include <asm/cpu_device_id.h>
47 #include "amd-pstate-trace.h"
48 
49 #define AMD_PSTATE_TRANSITION_LATENCY	20000
50 #define AMD_PSTATE_TRANSITION_DELAY	1000
51 
52 /*
53  * TODO: We need more time to fine tune processors with shared memory solution
54  * with community together.
55  *
56  * There are some performance drops on the CPU benchmarks which reports from
57  * Suse. We are co-working with them to fine tune the shared memory solution. So
58  * we disable it by default to go acpi-cpufreq on these processors and add a
59  * module parameter to be able to enable it manually for debugging.
60  */
61 static bool shared_mem = false;
62 module_param(shared_mem, bool, 0444);
63 MODULE_PARM_DESC(shared_mem,
64 		 "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)");
65 
66 static struct cpufreq_driver amd_pstate_driver;
67 
68 /**
69  * struct  amd_aperf_mperf
70  * @aperf: actual performance frequency clock count
71  * @mperf: maximum performance frequency clock count
72  * @tsc:   time stamp counter
73  */
74 struct amd_aperf_mperf {
75 	u64 aperf;
76 	u64 mperf;
77 	u64 tsc;
78 };
79 
80 /**
81  * struct amd_cpudata - private CPU data for AMD P-State
82  * @cpu: CPU number
83  * @req: constraint request to apply
84  * @cppc_req_cached: cached performance request hints
85  * @highest_perf: the maximum performance an individual processor may reach,
86  *		  assuming ideal conditions
87  * @nominal_perf: the maximum sustained performance level of the processor,
88  *		  assuming ideal operating conditions
89  * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power
90  *			   savings are achieved
91  * @lowest_perf: the absolute lowest performance level of the processor
92  * @max_freq: the frequency that mapped to highest_perf
93  * @min_freq: the frequency that mapped to lowest_perf
94  * @nominal_freq: the frequency that mapped to nominal_perf
95  * @lowest_nonlinear_freq: the frequency that mapped to lowest_nonlinear_perf
96  * @cur: Difference of Aperf/Mperf/tsc count between last and current sample
97  * @prev: Last Aperf/Mperf/tsc count value read from register
98  * @freq: current cpu frequency value
99  * @boost_supported: check whether the Processor or SBIOS supports boost mode
100  *
101  * The amd_cpudata is key private data for each CPU thread in AMD P-State, and
102  * represents all the attributes and goals that AMD P-State requests at runtime.
103  */
104 struct amd_cpudata {
105 	int	cpu;
106 
107 	struct	freq_qos_request req[2];
108 	u64	cppc_req_cached;
109 
110 	u32	highest_perf;
111 	u32	nominal_perf;
112 	u32	lowest_nonlinear_perf;
113 	u32	lowest_perf;
114 
115 	u32	max_freq;
116 	u32	min_freq;
117 	u32	nominal_freq;
118 	u32	lowest_nonlinear_freq;
119 
120 	struct amd_aperf_mperf cur;
121 	struct amd_aperf_mperf prev;
122 
123 	u64 	freq;
124 	bool	boost_supported;
125 };
126 
127 static inline int pstate_enable(bool enable)
128 {
129 	return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable);
130 }
131 
132 static int cppc_enable(bool enable)
133 {
134 	int cpu, ret = 0;
135 
136 	for_each_present_cpu(cpu) {
137 		ret = cppc_set_enable(cpu, enable);
138 		if (ret)
139 			return ret;
140 	}
141 
142 	return ret;
143 }
144 
145 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable);
146 
147 static inline int amd_pstate_enable(bool enable)
148 {
149 	return static_call(amd_pstate_enable)(enable);
150 }
151 
152 static int pstate_init_perf(struct amd_cpudata *cpudata)
153 {
154 	u64 cap1;
155 	u32 highest_perf;
156 
157 	int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
158 				     &cap1);
159 	if (ret)
160 		return ret;
161 
162 	/*
163 	 * TODO: Introduce AMD specific power feature.
164 	 *
165 	 * CPPC entry doesn't indicate the highest performance in some ASICs.
166 	 */
167 	highest_perf = amd_get_highest_perf();
168 	if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1))
169 		highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
170 
171 	WRITE_ONCE(cpudata->highest_perf, highest_perf);
172 
173 	WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
174 	WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
175 	WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));
176 
177 	return 0;
178 }
179 
180 static int cppc_init_perf(struct amd_cpudata *cpudata)
181 {
182 	struct cppc_perf_caps cppc_perf;
183 	u32 highest_perf;
184 
185 	int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
186 	if (ret)
187 		return ret;
188 
189 	highest_perf = amd_get_highest_perf();
190 	if (highest_perf > cppc_perf.highest_perf)
191 		highest_perf = cppc_perf.highest_perf;
192 
193 	WRITE_ONCE(cpudata->highest_perf, highest_perf);
194 
195 	WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
196 	WRITE_ONCE(cpudata->lowest_nonlinear_perf,
197 		   cppc_perf.lowest_nonlinear_perf);
198 	WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
199 
200 	return 0;
201 }
202 
203 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
204 
205 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
206 {
207 	return static_call(amd_pstate_init_perf)(cpudata);
208 }
209 
210 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,
211 			       u32 des_perf, u32 max_perf, bool fast_switch)
212 {
213 	if (fast_switch)
214 		wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));
215 	else
216 		wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
217 			      READ_ONCE(cpudata->cppc_req_cached));
218 }
219 
220 static void cppc_update_perf(struct amd_cpudata *cpudata,
221 			     u32 min_perf, u32 des_perf,
222 			     u32 max_perf, bool fast_switch)
223 {
224 	struct cppc_perf_ctrls perf_ctrls;
225 
226 	perf_ctrls.max_perf = max_perf;
227 	perf_ctrls.min_perf = min_perf;
228 	perf_ctrls.desired_perf = des_perf;
229 
230 	cppc_set_perf(cpudata->cpu, &perf_ctrls);
231 }
232 
233 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf);
234 
235 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,
236 					  u32 min_perf, u32 des_perf,
237 					  u32 max_perf, bool fast_switch)
238 {
239 	static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,
240 					    max_perf, fast_switch);
241 }
242 
243 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
244 {
245 	u64 aperf, mperf, tsc;
246 	unsigned long flags;
247 
248 	local_irq_save(flags);
249 	rdmsrl(MSR_IA32_APERF, aperf);
250 	rdmsrl(MSR_IA32_MPERF, mperf);
251 	tsc = rdtsc();
252 
253 	if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
254 		local_irq_restore(flags);
255 		return false;
256 	}
257 
258 	local_irq_restore(flags);
259 
260 	cpudata->cur.aperf = aperf;
261 	cpudata->cur.mperf = mperf;
262 	cpudata->cur.tsc =  tsc;
263 	cpudata->cur.aperf -= cpudata->prev.aperf;
264 	cpudata->cur.mperf -= cpudata->prev.mperf;
265 	cpudata->cur.tsc -= cpudata->prev.tsc;
266 
267 	cpudata->prev.aperf = aperf;
268 	cpudata->prev.mperf = mperf;
269 	cpudata->prev.tsc = tsc;
270 
271 	cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
272 
273 	return true;
274 }
275 
276 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
277 			      u32 des_perf, u32 max_perf, bool fast_switch)
278 {
279 	u64 prev = READ_ONCE(cpudata->cppc_req_cached);
280 	u64 value = prev;
281 
282 	des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
283 	value &= ~AMD_CPPC_MIN_PERF(~0L);
284 	value |= AMD_CPPC_MIN_PERF(min_perf);
285 
286 	value &= ~AMD_CPPC_DES_PERF(~0L);
287 	value |= AMD_CPPC_DES_PERF(des_perf);
288 
289 	value &= ~AMD_CPPC_MAX_PERF(~0L);
290 	value |= AMD_CPPC_MAX_PERF(max_perf);
291 
292 	if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
293 		trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
294 			cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
295 				cpudata->cpu, (value != prev), fast_switch);
296 	}
297 
298 	if (value == prev)
299 		return;
300 
301 	WRITE_ONCE(cpudata->cppc_req_cached, value);
302 
303 	amd_pstate_update_perf(cpudata, min_perf, des_perf,
304 			       max_perf, fast_switch);
305 }
306 
307 static int amd_pstate_verify(struct cpufreq_policy_data *policy)
308 {
309 	cpufreq_verify_within_cpu_limits(policy);
310 
311 	return 0;
312 }
313 
314 static int amd_pstate_target(struct cpufreq_policy *policy,
315 			     unsigned int target_freq,
316 			     unsigned int relation)
317 {
318 	struct cpufreq_freqs freqs;
319 	struct amd_cpudata *cpudata = policy->driver_data;
320 	unsigned long max_perf, min_perf, des_perf, cap_perf;
321 
322 	if (!cpudata->max_freq)
323 		return -ENODEV;
324 
325 	cap_perf = READ_ONCE(cpudata->highest_perf);
326 	min_perf = READ_ONCE(cpudata->lowest_perf);
327 	max_perf = cap_perf;
328 
329 	freqs.old = policy->cur;
330 	freqs.new = target_freq;
331 
332 	des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf,
333 				     cpudata->max_freq);
334 
335 	cpufreq_freq_transition_begin(policy, &freqs);
336 	amd_pstate_update(cpudata, min_perf, des_perf,
337 			  max_perf, false);
338 	cpufreq_freq_transition_end(policy, &freqs, false);
339 
340 	return 0;
341 }
342 
343 static void amd_pstate_adjust_perf(unsigned int cpu,
344 				   unsigned long _min_perf,
345 				   unsigned long target_perf,
346 				   unsigned long capacity)
347 {
348 	unsigned long max_perf, min_perf, des_perf,
349 		      cap_perf, lowest_nonlinear_perf;
350 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
351 	struct amd_cpudata *cpudata = policy->driver_data;
352 
353 	cap_perf = READ_ONCE(cpudata->highest_perf);
354 	lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
355 
356 	des_perf = cap_perf;
357 	if (target_perf < capacity)
358 		des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
359 
360 	min_perf = READ_ONCE(cpudata->highest_perf);
361 	if (_min_perf < capacity)
362 		min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
363 
364 	if (min_perf < lowest_nonlinear_perf)
365 		min_perf = lowest_nonlinear_perf;
366 
367 	max_perf = cap_perf;
368 	if (max_perf < min_perf)
369 		max_perf = min_perf;
370 
371 	amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true);
372 }
373 
374 static int amd_get_min_freq(struct amd_cpudata *cpudata)
375 {
376 	struct cppc_perf_caps cppc_perf;
377 
378 	int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
379 	if (ret)
380 		return ret;
381 
382 	/* Switch to khz */
383 	return cppc_perf.lowest_freq * 1000;
384 }
385 
386 static int amd_get_max_freq(struct amd_cpudata *cpudata)
387 {
388 	struct cppc_perf_caps cppc_perf;
389 	u32 max_perf, max_freq, nominal_freq, nominal_perf;
390 	u64 boost_ratio;
391 
392 	int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
393 	if (ret)
394 		return ret;
395 
396 	nominal_freq = cppc_perf.nominal_freq;
397 	nominal_perf = READ_ONCE(cpudata->nominal_perf);
398 	max_perf = READ_ONCE(cpudata->highest_perf);
399 
400 	boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT,
401 			      nominal_perf);
402 
403 	max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT;
404 
405 	/* Switch to khz */
406 	return max_freq * 1000;
407 }
408 
409 static int amd_get_nominal_freq(struct amd_cpudata *cpudata)
410 {
411 	struct cppc_perf_caps cppc_perf;
412 
413 	int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
414 	if (ret)
415 		return ret;
416 
417 	/* Switch to khz */
418 	return cppc_perf.nominal_freq * 1000;
419 }
420 
421 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata)
422 {
423 	struct cppc_perf_caps cppc_perf;
424 	u32 lowest_nonlinear_freq, lowest_nonlinear_perf,
425 	    nominal_freq, nominal_perf;
426 	u64 lowest_nonlinear_ratio;
427 
428 	int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
429 	if (ret)
430 		return ret;
431 
432 	nominal_freq = cppc_perf.nominal_freq;
433 	nominal_perf = READ_ONCE(cpudata->nominal_perf);
434 
435 	lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
436 
437 	lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT,
438 					 nominal_perf);
439 
440 	lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT;
441 
442 	/* Switch to khz */
443 	return lowest_nonlinear_freq * 1000;
444 }
445 
446 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
447 {
448 	struct amd_cpudata *cpudata = policy->driver_data;
449 	int ret;
450 
451 	if (!cpudata->boost_supported) {
452 		pr_err("Boost mode is not supported by this processor or SBIOS\n");
453 		return -EINVAL;
454 	}
455 
456 	if (state)
457 		policy->cpuinfo.max_freq = cpudata->max_freq;
458 	else
459 		policy->cpuinfo.max_freq = cpudata->nominal_freq;
460 
461 	policy->max = policy->cpuinfo.max_freq;
462 
463 	ret = freq_qos_update_request(&cpudata->req[1],
464 				      policy->cpuinfo.max_freq);
465 	if (ret < 0)
466 		return ret;
467 
468 	return 0;
469 }
470 
471 static void amd_pstate_boost_init(struct amd_cpudata *cpudata)
472 {
473 	u32 highest_perf, nominal_perf;
474 
475 	highest_perf = READ_ONCE(cpudata->highest_perf);
476 	nominal_perf = READ_ONCE(cpudata->nominal_perf);
477 
478 	if (highest_perf <= nominal_perf)
479 		return;
480 
481 	cpudata->boost_supported = true;
482 	amd_pstate_driver.boost_enabled = true;
483 }
484 
485 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
486 {
487 	int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
488 	struct device *dev;
489 	struct amd_cpudata *cpudata;
490 
491 	dev = get_cpu_device(policy->cpu);
492 	if (!dev)
493 		return -ENODEV;
494 
495 	cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
496 	if (!cpudata)
497 		return -ENOMEM;
498 
499 	cpudata->cpu = policy->cpu;
500 
501 	ret = amd_pstate_init_perf(cpudata);
502 	if (ret)
503 		goto free_cpudata1;
504 
505 	min_freq = amd_get_min_freq(cpudata);
506 	max_freq = amd_get_max_freq(cpudata);
507 	nominal_freq = amd_get_nominal_freq(cpudata);
508 	lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
509 
510 	if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
511 		dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
512 			min_freq, max_freq);
513 		ret = -EINVAL;
514 		goto free_cpudata1;
515 	}
516 
517 	policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY;
518 	policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY;
519 
520 	policy->min = min_freq;
521 	policy->max = max_freq;
522 
523 	policy->cpuinfo.min_freq = min_freq;
524 	policy->cpuinfo.max_freq = max_freq;
525 
526 	/* It will be updated by governor */
527 	policy->cur = policy->cpuinfo.min_freq;
528 
529 	if (boot_cpu_has(X86_FEATURE_CPPC))
530 		policy->fast_switch_possible = true;
531 
532 	ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
533 				   FREQ_QOS_MIN, policy->cpuinfo.min_freq);
534 	if (ret < 0) {
535 		dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
536 		goto free_cpudata1;
537 	}
538 
539 	ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
540 				   FREQ_QOS_MAX, policy->cpuinfo.max_freq);
541 	if (ret < 0) {
542 		dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
543 		goto free_cpudata2;
544 	}
545 
546 	/* Initial processor data capability frequencies */
547 	cpudata->max_freq = max_freq;
548 	cpudata->min_freq = min_freq;
549 	cpudata->nominal_freq = nominal_freq;
550 	cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
551 
552 	policy->driver_data = cpudata;
553 
554 	amd_pstate_boost_init(cpudata);
555 
556 	return 0;
557 
558 free_cpudata2:
559 	freq_qos_remove_request(&cpudata->req[0]);
560 free_cpudata1:
561 	kfree(cpudata);
562 	return ret;
563 }
564 
565 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy)
566 {
567 	struct amd_cpudata *cpudata = policy->driver_data;
568 
569 	freq_qos_remove_request(&cpudata->req[1]);
570 	freq_qos_remove_request(&cpudata->req[0]);
571 	kfree(cpudata);
572 
573 	return 0;
574 }
575 
576 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)
577 {
578 	int ret;
579 
580 	ret = amd_pstate_enable(true);
581 	if (ret)
582 		pr_err("failed to enable amd-pstate during resume, return %d\n", ret);
583 
584 	return ret;
585 }
586 
587 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)
588 {
589 	int ret;
590 
591 	ret = amd_pstate_enable(false);
592 	if (ret)
593 		pr_err("failed to disable amd-pstate during suspend, return %d\n", ret);
594 
595 	return ret;
596 }
597 
598 /* Sysfs attributes */
599 
600 /*
601  * This frequency is to indicate the maximum hardware frequency.
602  * If boost is not active but supported, the frequency will be larger than the
603  * one in cpuinfo.
604  */
605 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
606 					char *buf)
607 {
608 	int max_freq;
609 	struct amd_cpudata *cpudata = policy->driver_data;
610 
611 	max_freq = amd_get_max_freq(cpudata);
612 	if (max_freq < 0)
613 		return max_freq;
614 
615 	return sprintf(&buf[0], "%u\n", max_freq);
616 }
617 
618 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
619 						     char *buf)
620 {
621 	int freq;
622 	struct amd_cpudata *cpudata = policy->driver_data;
623 
624 	freq = amd_get_lowest_nonlinear_freq(cpudata);
625 	if (freq < 0)
626 		return freq;
627 
628 	return sprintf(&buf[0], "%u\n", freq);
629 }
630 
631 /*
632  * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
633  * need to expose it to sysfs.
634  */
635 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
636 					    char *buf)
637 {
638 	u32 perf;
639 	struct amd_cpudata *cpudata = policy->driver_data;
640 
641 	perf = READ_ONCE(cpudata->highest_perf);
642 
643 	return sprintf(&buf[0], "%u\n", perf);
644 }
645 
646 cpufreq_freq_attr_ro(amd_pstate_max_freq);
647 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
648 
649 cpufreq_freq_attr_ro(amd_pstate_highest_perf);
650 
651 static struct freq_attr *amd_pstate_attr[] = {
652 	&amd_pstate_max_freq,
653 	&amd_pstate_lowest_nonlinear_freq,
654 	&amd_pstate_highest_perf,
655 	NULL,
656 };
657 
658 static struct cpufreq_driver amd_pstate_driver = {
659 	.flags		= CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
660 	.verify		= amd_pstate_verify,
661 	.target		= amd_pstate_target,
662 	.init		= amd_pstate_cpu_init,
663 	.exit		= amd_pstate_cpu_exit,
664 	.suspend	= amd_pstate_cpu_suspend,
665 	.resume		= amd_pstate_cpu_resume,
666 	.set_boost	= amd_pstate_set_boost,
667 	.name		= "amd-pstate",
668 	.attr		= amd_pstate_attr,
669 };
670 
671 static int __init amd_pstate_init(void)
672 {
673 	int ret;
674 
675 	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
676 		return -ENODEV;
677 
678 	if (!acpi_cpc_valid()) {
679 		pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
680 		return -ENODEV;
681 	}
682 
683 	/* don't keep reloading if cpufreq_driver exists */
684 	if (cpufreq_get_current_driver())
685 		return -EEXIST;
686 
687 	/* capability check */
688 	if (boot_cpu_has(X86_FEATURE_CPPC)) {
689 		pr_debug("AMD CPPC MSR based functionality is supported\n");
690 		amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf;
691 	} else if (shared_mem) {
692 		static_call_update(amd_pstate_enable, cppc_enable);
693 		static_call_update(amd_pstate_init_perf, cppc_init_perf);
694 		static_call_update(amd_pstate_update_perf, cppc_update_perf);
695 	} else {
696 		pr_info("This processor supports shared memory solution, you can enable it with amd_pstate.shared_mem=1\n");
697 		return -ENODEV;
698 	}
699 
700 	/* enable amd pstate feature */
701 	ret = amd_pstate_enable(true);
702 	if (ret) {
703 		pr_err("failed to enable amd-pstate with return %d\n", ret);
704 		return ret;
705 	}
706 
707 	ret = cpufreq_register_driver(&amd_pstate_driver);
708 	if (ret)
709 		pr_err("failed to register amd_pstate_driver with return %d\n",
710 		       ret);
711 
712 	return ret;
713 }
714 
715 static void __exit amd_pstate_exit(void)
716 {
717 	cpufreq_unregister_driver(&amd_pstate_driver);
718 
719 	amd_pstate_enable(false);
720 }
721 
722 module_init(amd_pstate_init);
723 module_exit(amd_pstate_exit);
724 
725 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
726 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");
727 MODULE_LICENSE("GPL");
728