1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * amd-pstate.c - AMD Processor P-state Frequency Driver
4 *
5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
6 *
7 * Author: Huang Rui <ray.huang@amd.com>
8 *
9 * AMD P-State introduces a new CPU performance scaling design for AMD
10 * processors using the ACPI Collaborative Performance and Power Control (CPPC)
11 * feature which works with the AMD SMU firmware providing a finer grained
12 * frequency control range. It is to replace the legacy ACPI P-States control,
13 * allows a flexible, low-latency interface for the Linux kernel to directly
14 * communicate the performance hints to hardware.
15 *
16 * AMD P-State is supported on recent AMD Zen base CPU series include some of
17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
18 * P-State supported system. And there are two types of hardware implementations
19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
21 */
22
23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
24
25 #include <linux/bitfield.h>
26 #include <linux/kernel.h>
27 #include <linux/module.h>
28 #include <linux/init.h>
29 #include <linux/smp.h>
30 #include <linux/sched.h>
31 #include <linux/cpufreq.h>
32 #include <linux/compiler.h>
33 #include <linux/dmi.h>
34 #include <linux/slab.h>
35 #include <linux/acpi.h>
36 #include <linux/io.h>
37 #include <linux/delay.h>
38 #include <linux/uaccess.h>
39 #include <linux/power_supply.h>
40 #include <linux/static_call.h>
41 #include <linux/topology.h>
42
43 #include <acpi/processor.h>
44 #include <acpi/cppc_acpi.h>
45
46 #include <asm/msr.h>
47 #include <asm/processor.h>
48 #include <asm/cpufeature.h>
49 #include <asm/cpu_device_id.h>
50
51 #include "amd-pstate.h"
52 #include "amd-pstate-trace.h"
53
54 #define AMD_PSTATE_TRANSITION_LATENCY 20000
55 #define AMD_PSTATE_TRANSITION_DELAY 1000
56 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600
57
58 #define AMD_CPPC_EPP_PERFORMANCE 0x00
59 #define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80
60 #define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF
61 #define AMD_CPPC_EPP_POWERSAVE 0xFF
62
63 static const char * const amd_pstate_mode_string[] = {
64 [AMD_PSTATE_UNDEFINED] = "undefined",
65 [AMD_PSTATE_DISABLE] = "disable",
66 [AMD_PSTATE_PASSIVE] = "passive",
67 [AMD_PSTATE_ACTIVE] = "active",
68 [AMD_PSTATE_GUIDED] = "guided",
69 };
70 static_assert(ARRAY_SIZE(amd_pstate_mode_string) == AMD_PSTATE_MAX);
71
amd_pstate_get_mode_string(enum amd_pstate_mode mode)72 const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode)
73 {
74 if (mode < AMD_PSTATE_UNDEFINED || mode >= AMD_PSTATE_MAX)
75 mode = AMD_PSTATE_UNDEFINED;
76 return amd_pstate_mode_string[mode];
77 }
78 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_mode_string);
79
80 struct quirk_entry {
81 u32 nominal_freq;
82 u32 lowest_freq;
83 };
84
85 static struct cpufreq_driver *current_pstate_driver;
86 static struct cpufreq_driver amd_pstate_driver;
87 static struct cpufreq_driver amd_pstate_epp_driver;
88 static int cppc_state = AMD_PSTATE_UNDEFINED;
89 static bool amd_pstate_prefcore = true;
90 static struct quirk_entry *quirks;
91
92 /*
93 * AMD Energy Preference Performance (EPP)
94 * The EPP is used in the CCLK DPM controller to drive
95 * the frequency that a core is going to operate during
96 * short periods of activity. EPP values will be utilized for
97 * different OS profiles (balanced, performance, power savings)
98 * display strings corresponding to EPP index in the
99 * energy_perf_strings[]
100 * index String
101 *-------------------------------------
102 * 0 default
103 * 1 performance
104 * 2 balance_performance
105 * 3 balance_power
106 * 4 power
107 * 5 custom (for raw EPP values)
108 * 6 dynamic (platform profile driven selection)
109 */
110 enum energy_perf_value_index {
111 EPP_INDEX_DEFAULT = 0,
112 EPP_INDEX_PERFORMANCE,
113 EPP_INDEX_BALANCE_PERFORMANCE,
114 EPP_INDEX_BALANCE_POWERSAVE,
115 EPP_INDEX_POWERSAVE,
116 EPP_INDEX_CUSTOM,
117 EPP_INDEX_DYNAMIC,
118 EPP_INDEX_MAX,
119 };
120
121 static const char * const energy_perf_strings[] = {
122 [EPP_INDEX_DEFAULT] = "default",
123 [EPP_INDEX_PERFORMANCE] = "performance",
124 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
125 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
126 [EPP_INDEX_POWERSAVE] = "power",
127 [EPP_INDEX_CUSTOM] = "custom",
128 [EPP_INDEX_DYNAMIC] = "dynamic",
129 };
130 static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX);
131
132 static unsigned int epp_values[] = {
133 [EPP_INDEX_DEFAULT] = 0,
134 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE,
135 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE,
136 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE,
137 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE,
138 };
139 static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX - 2);
140
141 typedef int (*cppc_mode_transition_fn)(int);
142
143 static struct quirk_entry quirk_amd_7k62 = {
144 .nominal_freq = 2600,
145 .lowest_freq = 550,
146 };
147
freq_to_perf(union perf_cached perf,u32 nominal_freq,unsigned int freq_val)148 static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val)
149 {
150 u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq);
151
152 return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf);
153 }
154
perf_to_freq(union perf_cached perf,u32 nominal_freq,u8 perf_val)155 static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val)
156 {
157 return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val,
158 perf.nominal_perf);
159 }
160
dmi_matched_7k62_bios_bug(const struct dmi_system_id * dmi)161 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi)
162 {
163 /**
164 * match the broken bios for family 17h processor support CPPC V2
165 * broken BIOS lack of nominal_freq and lowest_freq capabilities
166 * definition in ACPI tables
167 */
168 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) {
169 quirks = dmi->driver_data;
170 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident);
171 return 1;
172 }
173
174 return 0;
175 }
176
177 static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = {
178 {
179 .callback = dmi_matched_7k62_bios_bug,
180 .ident = "AMD EPYC 7K62",
181 .matches = {
182 DMI_MATCH(DMI_BIOS_VERSION, "5.14"),
183 DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"),
184 },
185 .driver_data = &quirk_amd_7k62,
186 },
187 {}
188 };
189 MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table);
190
get_mode_idx_from_str(const char * str,size_t size)191 static inline int get_mode_idx_from_str(const char *str, size_t size)
192 {
193 int i;
194
195 for (i = 0; i < AMD_PSTATE_MAX; i++) {
196 if (!strncmp(str, amd_pstate_mode_string[i], size))
197 return i;
198 }
199 return -EINVAL;
200 }
201
202 static DEFINE_MUTEX(amd_pstate_driver_lock);
203
msr_get_epp(struct amd_cpudata * cpudata)204 static int msr_get_epp(struct amd_cpudata *cpudata)
205 {
206 u64 value;
207 int ret;
208
209 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value);
210 if (ret < 0) {
211 pr_debug("Could not retrieve energy perf value (%d)\n", ret);
212 return ret;
213 }
214
215 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value);
216 }
217
218 DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp);
219
amd_pstate_get_epp(struct amd_cpudata * cpudata)220 static inline int amd_pstate_get_epp(struct amd_cpudata *cpudata)
221 {
222 return static_call(amd_pstate_get_epp)(cpudata);
223 }
224
shmem_get_epp(struct amd_cpudata * cpudata)225 static int shmem_get_epp(struct amd_cpudata *cpudata)
226 {
227 u64 epp;
228 int ret;
229
230 ret = cppc_get_epp_perf(cpudata->cpu, &epp);
231 if (ret < 0) {
232 pr_debug("Could not retrieve energy perf value (%d)\n", ret);
233 return ret;
234 }
235
236 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp);
237 }
238
msr_update_perf(struct cpufreq_policy * policy,u8 min_perf,u8 des_perf,u8 max_perf,u8 epp,bool fast_switch)239 static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf,
240 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch)
241 {
242 struct amd_cpudata *cpudata = policy->driver_data;
243 u64 value, prev;
244
245 value = prev = READ_ONCE(cpudata->cppc_req_cached);
246
247 FIELD_MODIFY(AMD_CPPC_MAX_PERF_MASK, &value, max_perf);
248 FIELD_MODIFY(AMD_CPPC_DES_PERF_MASK, &value, des_perf);
249 FIELD_MODIFY(AMD_CPPC_MIN_PERF_MASK, &value, min_perf);
250 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp);
251
252 if (trace_amd_pstate_epp_perf_enabled()) {
253 union perf_cached perf = READ_ONCE(cpudata->perf);
254
255 trace_call__amd_pstate_epp_perf(cpudata->cpu,
256 perf.highest_perf,
257 epp,
258 min_perf,
259 max_perf,
260 policy->boost_enabled,
261 value != prev);
262 }
263
264 if (value == prev)
265 return 0;
266
267 if (fast_switch) {
268 wrmsrq(MSR_AMD_CPPC_REQ, value);
269 } else {
270 int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
271
272 if (ret)
273 return ret;
274 }
275
276 WRITE_ONCE(cpudata->cppc_req_cached, value);
277
278 return 0;
279 }
280
281 DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf);
282
amd_pstate_update_perf(struct cpufreq_policy * policy,u8 min_perf,u8 des_perf,u8 max_perf,u8 epp,bool fast_switch)283 static inline int amd_pstate_update_perf(struct cpufreq_policy *policy,
284 u8 min_perf, u8 des_perf,
285 u8 max_perf, u8 epp,
286 bool fast_switch)
287 {
288 return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf,
289 max_perf, epp, fast_switch);
290 }
291
msr_set_epp(struct cpufreq_policy * policy,u8 epp)292 static int msr_set_epp(struct cpufreq_policy *policy, u8 epp)
293 {
294 struct amd_cpudata *cpudata = policy->driver_data;
295 u64 value, prev;
296 int ret;
297
298 value = prev = READ_ONCE(cpudata->cppc_req_cached);
299 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp);
300
301 if (trace_amd_pstate_epp_perf_enabled()) {
302 union perf_cached perf = cpudata->perf;
303
304 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf,
305 epp,
306 FIELD_GET(AMD_CPPC_MIN_PERF_MASK,
307 cpudata->cppc_req_cached),
308 FIELD_GET(AMD_CPPC_MAX_PERF_MASK,
309 cpudata->cppc_req_cached),
310 policy->boost_enabled,
311 value != prev);
312 }
313
314 if (value == prev)
315 return 0;
316
317 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
318 if (ret) {
319 pr_err("failed to set energy perf value (%d)\n", ret);
320 return ret;
321 }
322
323 /* update both so that msr_update_perf() can effectively check */
324 WRITE_ONCE(cpudata->cppc_req_cached, value);
325
326 return ret;
327 }
328
329 DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp);
330
amd_pstate_set_epp(struct cpufreq_policy * policy,u8 epp)331 static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp)
332 {
333 return static_call(amd_pstate_set_epp)(policy, epp);
334 }
335
amd_pstate_set_floor_perf(struct cpufreq_policy * policy,u8 perf)336 static int amd_pstate_set_floor_perf(struct cpufreq_policy *policy, u8 perf)
337 {
338 struct amd_cpudata *cpudata = policy->driver_data;
339 u64 value, prev;
340 bool changed;
341 int ret;
342
343 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO))
344 return 0;
345
346 value = prev = READ_ONCE(cpudata->cppc_req2_cached);
347 FIELD_MODIFY(AMD_CPPC_FLOOR_PERF_MASK, &value, perf);
348
349 changed = value != prev;
350 if (!changed) {
351 ret = 0;
352 goto out_trace;
353 }
354
355 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, value);
356 if (ret) {
357 changed = false;
358 pr_err("failed to set CPPC REQ2 value. Error (%d)\n", ret);
359 goto out_trace;
360 }
361
362 WRITE_ONCE(cpudata->cppc_req2_cached, value);
363
364 out_trace:
365 if (trace_amd_pstate_cppc_req2_enabled())
366 trace_call__amd_pstate_cppc_req2(cpudata->cpu, perf, changed,
367 ret);
368 return ret;
369 }
370
amd_pstate_init_floor_perf(struct cpufreq_policy * policy)371 static int amd_pstate_init_floor_perf(struct cpufreq_policy *policy)
372 {
373 struct amd_cpudata *cpudata = policy->driver_data;
374 u8 floor_perf;
375 u64 value;
376 int ret;
377
378 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO))
379 return 0;
380
381 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, &value);
382 if (ret) {
383 pr_err("failed to read CPPC REQ2 value. Error (%d)\n", ret);
384 return ret;
385 }
386
387 WRITE_ONCE(cpudata->cppc_req2_cached, value);
388 floor_perf = FIELD_GET(AMD_CPPC_FLOOR_PERF_MASK,
389 cpudata->cppc_req2_cached);
390
391 /* Set a sane value for floor_perf if the default value is invalid */
392 if (floor_perf < cpudata->perf.lowest_perf) {
393 floor_perf = cpudata->perf.nominal_perf;
394 ret = amd_pstate_set_floor_perf(policy, floor_perf);
395 if (ret)
396 return ret;
397 }
398
399
400 cpudata->bios_floor_perf = floor_perf;
401 cpudata->floor_freq = perf_to_freq(cpudata->perf, cpudata->nominal_freq,
402 floor_perf);
403 return 0;
404 }
405
shmem_set_epp(struct cpufreq_policy * policy,u8 epp)406 static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp)
407 {
408 struct amd_cpudata *cpudata = policy->driver_data;
409 struct cppc_perf_ctrls perf_ctrls;
410 u8 epp_cached;
411 u64 value;
412 int ret;
413
414
415 epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached);
416 if (trace_amd_pstate_epp_perf_enabled()) {
417 union perf_cached perf = cpudata->perf;
418
419 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf,
420 epp,
421 FIELD_GET(AMD_CPPC_MIN_PERF_MASK,
422 cpudata->cppc_req_cached),
423 FIELD_GET(AMD_CPPC_MAX_PERF_MASK,
424 cpudata->cppc_req_cached),
425 policy->boost_enabled,
426 epp != epp_cached);
427 }
428
429 if (epp == epp_cached)
430 return 0;
431
432 perf_ctrls.energy_perf = epp;
433 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1);
434 if (ret) {
435 pr_debug("failed to set energy perf value (%d)\n", ret);
436 return ret;
437 }
438
439 value = READ_ONCE(cpudata->cppc_req_cached);
440 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp);
441 WRITE_ONCE(cpudata->cppc_req_cached, value);
442
443 return ret;
444 }
445
msr_cppc_enable(struct cpufreq_policy * policy)446 static inline int msr_cppc_enable(struct cpufreq_policy *policy)
447 {
448 return wrmsrq_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1);
449 }
450
shmem_cppc_enable(struct cpufreq_policy * policy)451 static int shmem_cppc_enable(struct cpufreq_policy *policy)
452 {
453 return cppc_set_enable(policy->cpu, 1);
454 }
455
456 DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable);
457
amd_pstate_cppc_enable(struct cpufreq_policy * policy)458 static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy)
459 {
460 return static_call(amd_pstate_cppc_enable)(policy);
461 }
462
msr_init_perf(struct amd_cpudata * cpudata)463 static int msr_init_perf(struct amd_cpudata *cpudata)
464 {
465 union perf_cached perf = READ_ONCE(cpudata->perf);
466 u64 cap1, numerator, cppc_req;
467
468 int ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
469 &cap1);
470 if (ret)
471 return ret;
472
473 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator);
474 if (ret)
475 return ret;
476
477 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req);
478 if (ret)
479 return ret;
480
481 WRITE_ONCE(cpudata->cppc_req_cached, cppc_req);
482
483 perf.highest_perf = numerator;
484 perf.max_limit_perf = numerator;
485 perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1);
486 perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1);
487 perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1);
488 perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1);
489 perf.bios_min_perf = FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cppc_req);
490 WRITE_ONCE(cpudata->perf, perf);
491 WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1));
492 WRITE_ONCE(cpudata->floor_perf_cnt, FIELD_GET(AMD_CPPC_FLOOR_PERF_CNT_MASK, cap1));
493
494 return 0;
495 }
496
shmem_init_perf(struct amd_cpudata * cpudata)497 static int shmem_init_perf(struct amd_cpudata *cpudata)
498 {
499 struct cppc_perf_caps cppc_perf;
500 union perf_cached perf = READ_ONCE(cpudata->perf);
501 u64 numerator;
502 bool auto_sel;
503
504 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
505 if (ret)
506 return ret;
507
508 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator);
509 if (ret)
510 return ret;
511
512 perf.highest_perf = numerator;
513 perf.max_limit_perf = numerator;
514 perf.min_limit_perf = cppc_perf.lowest_perf;
515 perf.nominal_perf = cppc_perf.nominal_perf;
516 perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
517 perf.lowest_perf = cppc_perf.lowest_perf;
518 WRITE_ONCE(cpudata->perf, perf);
519 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf);
520
521 ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel);
522 if (ret) {
523 pr_warn("failed to get auto_sel, ret: %d\n", ret);
524 return 0;
525 }
526
527 ret = cppc_set_auto_sel(cpudata->cpu,
528 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
529
530 if (ret)
531 pr_warn("failed to set auto_sel, ret: %d\n", ret);
532
533 return ret;
534 }
535
536 DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf);
537
amd_pstate_init_perf(struct amd_cpudata * cpudata)538 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
539 {
540 return static_call(amd_pstate_init_perf)(cpudata);
541 }
542
shmem_update_perf(struct cpufreq_policy * policy,u8 min_perf,u8 des_perf,u8 max_perf,u8 epp,bool fast_switch)543 static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf,
544 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch)
545 {
546 struct amd_cpudata *cpudata = policy->driver_data;
547 struct cppc_perf_ctrls perf_ctrls;
548 u64 value, prev;
549 int ret;
550
551 if (cppc_state == AMD_PSTATE_ACTIVE) {
552 int ret = shmem_set_epp(policy, epp);
553
554 if (ret)
555 return ret;
556 }
557
558 value = prev = READ_ONCE(cpudata->cppc_req_cached);
559
560 FIELD_MODIFY(AMD_CPPC_MAX_PERF_MASK, &value, max_perf);
561 FIELD_MODIFY(AMD_CPPC_DES_PERF_MASK, &value, des_perf);
562 FIELD_MODIFY(AMD_CPPC_MIN_PERF_MASK, &value, min_perf);
563 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp);
564
565 if (trace_amd_pstate_epp_perf_enabled()) {
566 union perf_cached perf = READ_ONCE(cpudata->perf);
567
568 trace_call__amd_pstate_epp_perf(cpudata->cpu,
569 perf.highest_perf,
570 epp,
571 min_perf,
572 max_perf,
573 policy->boost_enabled,
574 value != prev);
575 }
576
577 if (value == prev)
578 return 0;
579
580 perf_ctrls.max_perf = max_perf;
581 perf_ctrls.min_perf = min_perf;
582 perf_ctrls.desired_perf = des_perf;
583
584 ret = cppc_set_perf(cpudata->cpu, &perf_ctrls);
585 if (ret)
586 return ret;
587
588 WRITE_ONCE(cpudata->cppc_req_cached, value);
589
590 return 0;
591 }
592
amd_pstate_sample(struct amd_cpudata * cpudata)593 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
594 {
595 u64 aperf, mperf, tsc;
596 unsigned long flags;
597
598 local_irq_save(flags);
599 rdmsrq(MSR_IA32_APERF, aperf);
600 rdmsrq(MSR_IA32_MPERF, mperf);
601 tsc = rdtsc();
602
603 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
604 local_irq_restore(flags);
605 return false;
606 }
607
608 local_irq_restore(flags);
609
610 cpudata->cur.aperf = aperf;
611 cpudata->cur.mperf = mperf;
612 cpudata->cur.tsc = tsc;
613 cpudata->cur.aperf -= cpudata->prev.aperf;
614 cpudata->cur.mperf -= cpudata->prev.mperf;
615 cpudata->cur.tsc -= cpudata->prev.tsc;
616
617 cpudata->prev.aperf = aperf;
618 cpudata->prev.mperf = mperf;
619 cpudata->prev.tsc = tsc;
620
621 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
622
623 return true;
624 }
625
amd_pstate_update(struct cpufreq_policy * policy,u8 min_perf,u8 des_perf,u8 max_perf,bool fast_switch,int gov_flags)626 static void amd_pstate_update(struct cpufreq_policy *policy, u8 min_perf,
627 u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags)
628 {
629 struct amd_cpudata *cpudata = policy->driver_data;
630 union perf_cached perf = READ_ONCE(cpudata->perf);
631
632 /* limit the max perf when core performance boost feature is disabled */
633 if (!cpudata->boost_supported)
634 max_perf = min_t(u8, perf.nominal_perf, max_perf);
635
636 des_perf = clamp_t(u8, des_perf, min_perf, max_perf);
637
638 policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf);
639
640 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) {
641 min_perf = des_perf;
642 des_perf = 0;
643 }
644
645 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
646 trace_call__amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
647 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
648 cpudata->cpu, fast_switch);
649 }
650
651 amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch);
652 }
653
amd_pstate_verify(struct cpufreq_policy_data * policy_data)654 static int amd_pstate_verify(struct cpufreq_policy_data *policy_data)
655 {
656 /*
657 * Initialize lower frequency limit (i.e.policy->min) with
658 * lowest_nonlinear_frequency or the min frequency (if) specified in BIOS,
659 * Override the initial value set by cpufreq core and amd-pstate qos_requests.
660 */
661 if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) {
662 struct cpufreq_policy *policy __free(put_cpufreq_policy) =
663 cpufreq_cpu_get(policy_data->cpu);
664 struct amd_cpudata *cpudata;
665 union perf_cached perf;
666
667 if (!policy)
668 return -EINVAL;
669
670 cpudata = policy->driver_data;
671 perf = READ_ONCE(cpudata->perf);
672
673 if (perf.bios_min_perf)
674 policy_data->min = perf_to_freq(perf, cpudata->nominal_freq,
675 perf.bios_min_perf);
676 else
677 policy_data->min = cpudata->lowest_nonlinear_freq;
678 }
679
680 cpufreq_verify_within_cpu_limits(policy_data);
681
682 return 0;
683 }
684
amd_pstate_update_min_max_limit(struct cpufreq_policy * policy)685 static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy)
686 {
687 struct amd_cpudata *cpudata = policy->driver_data;
688 union perf_cached perf = READ_ONCE(cpudata->perf);
689
690 perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max);
691 WRITE_ONCE(cpudata->max_limit_freq, policy->max);
692
693 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
694 u8 min_limit_perf = perf.bios_min_perf ?: perf.nominal_perf;
695 u32 min_limit_freq;
696
697 /*
698 * For performance policy, set MinPerf to nominal_perf / bios_min_perf
699 * rather than highest_perf or lowest_nonlinear_perf.
700 *
701 * Per commit 0c411b39e4f4c, using highest_perf was observed
702 * to cause frequency throttling on power-limited platforms, leading to
703 * performance regressions. Using lowest_nonlinear_perf would limit
704 * performance too much for HPC workloads requiring high frequency
705 * operation and minimal wakeup latency from idle states.
706 *
707 * nominal_perf therefore provides a balanced default by avoiding
708 * throttling while still maintaining enough performance for HPC
709 * workloads when bios_min_perf is not available.
710 *
711 * When bios_min_perf is available, users have profiled their workloads
712 * to understand the best idling frequency. Use that instead.
713 */
714 min_limit_perf = min(min_limit_perf, perf.max_limit_perf);
715 min_limit_freq = perf_to_freq(perf, cpudata->nominal_freq, min_limit_perf);
716 perf.min_limit_perf = min_limit_perf;
717
718 WRITE_ONCE(cpudata->min_limit_freq, min(min_limit_freq, cpudata->max_limit_freq));
719 } else {
720 perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min);
721 WRITE_ONCE(cpudata->min_limit_freq, policy->min);
722 }
723
724 WRITE_ONCE(cpudata->perf, perf);
725 }
726
amd_pstate_update_freq(struct cpufreq_policy * policy,unsigned int target_freq,bool fast_switch)727 static int amd_pstate_update_freq(struct cpufreq_policy *policy,
728 unsigned int target_freq, bool fast_switch)
729 {
730 struct cpufreq_freqs freqs;
731 struct amd_cpudata *cpudata;
732 union perf_cached perf;
733 u8 des_perf;
734
735 cpudata = policy->driver_data;
736
737 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)
738 amd_pstate_update_min_max_limit(policy);
739
740 perf = READ_ONCE(cpudata->perf);
741
742 freqs.old = policy->cur;
743 freqs.new = target_freq;
744
745 des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq);
746
747 WARN_ON(fast_switch && !policy->fast_switch_enabled);
748 /*
749 * If fast_switch is desired, then there aren't any registered
750 * transition notifiers. See comment for
751 * cpufreq_enable_fast_switch().
752 */
753 if (!fast_switch)
754 cpufreq_freq_transition_begin(policy, &freqs);
755
756 amd_pstate_update(policy, perf.min_limit_perf, des_perf,
757 perf.max_limit_perf, fast_switch,
758 policy->governor->flags);
759
760 if (!fast_switch)
761 cpufreq_freq_transition_end(policy, &freqs, false);
762
763 return 0;
764 }
765
amd_pstate_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)766 static int amd_pstate_target(struct cpufreq_policy *policy,
767 unsigned int target_freq,
768 unsigned int relation)
769 {
770 return amd_pstate_update_freq(policy, target_freq, false);
771 }
772
amd_pstate_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)773 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy,
774 unsigned int target_freq)
775 {
776 if (!amd_pstate_update_freq(policy, target_freq, true))
777 return target_freq;
778 return policy->cur;
779 }
780
amd_pstate_adjust_perf(struct cpufreq_policy * policy,unsigned long _min_perf,unsigned long target_perf,unsigned long _max_perf,unsigned long capacity)781 static void amd_pstate_adjust_perf(struct cpufreq_policy *policy,
782 unsigned long _min_perf,
783 unsigned long target_perf,
784 unsigned long _max_perf,
785 unsigned long capacity)
786 {
787 u8 max_perf, min_perf, des_perf, cap_perf;
788 struct amd_cpudata *cpudata;
789 union perf_cached perf;
790
791 if (!policy)
792 return;
793
794 cpudata = policy->driver_data;
795
796 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)
797 amd_pstate_update_min_max_limit(policy);
798
799 perf = READ_ONCE(cpudata->perf);
800 cap_perf = perf.highest_perf;
801
802 des_perf = cap_perf;
803 if (target_perf < capacity)
804 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
805
806 if (_min_perf < capacity)
807 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
808 else
809 min_perf = cap_perf;
810
811 if (min_perf < perf.min_limit_perf)
812 min_perf = perf.min_limit_perf;
813
814 max_perf = perf.max_limit_perf;
815 if (max_perf < min_perf)
816 max_perf = min_perf;
817
818 amd_pstate_update(policy, min_perf, des_perf, max_perf, true,
819 policy->governor->flags);
820 }
821
amd_pstate_cpu_boost_update(struct cpufreq_policy * policy,bool on)822 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on)
823 {
824 struct amd_cpudata *cpudata = policy->driver_data;
825 u32 nominal_freq;
826 int ret = 0;
827
828 nominal_freq = READ_ONCE(cpudata->nominal_freq);
829
830 if (on)
831 policy->cpuinfo.max_freq = cpudata->max_freq;
832 else if (policy->cpuinfo.max_freq > nominal_freq)
833 policy->cpuinfo.max_freq = nominal_freq;
834
835 if (cppc_state == AMD_PSTATE_PASSIVE) {
836 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq);
837 if (ret < 0)
838 pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu);
839 }
840
841 return ret < 0 ? ret : 0;
842 }
843
amd_pstate_set_boost(struct cpufreq_policy * policy,int state)844 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
845 {
846 struct amd_cpudata *cpudata = policy->driver_data;
847 int ret;
848
849 if (!cpudata->boost_supported) {
850 pr_err("Boost mode is not supported by this processor or SBIOS\n");
851 return -EOPNOTSUPP;
852 }
853
854 ret = amd_pstate_cpu_boost_update(policy, state);
855 refresh_frequency_limits(policy);
856
857 return ret;
858 }
859
amd_pstate_init_boost_support(struct amd_cpudata * cpudata)860 static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata)
861 {
862 u64 boost_val;
863 int ret = -1;
864
865 /*
866 * If platform has no CPB support or disable it, initialize current driver
867 * boost_enabled state to be false, it is not an error for cpufreq core to handle.
868 */
869 if (!cpu_feature_enabled(X86_FEATURE_CPB)) {
870 pr_debug_once("Boost CPB capabilities not present in the processor\n");
871 ret = 0;
872 goto exit_err;
873 }
874
875 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val);
876 if (ret) {
877 pr_err_once("failed to read initial CPU boost state!\n");
878 ret = -EIO;
879 goto exit_err;
880 }
881
882 if (!(boost_val & MSR_K7_HWCR_CPB_DIS))
883 cpudata->boost_supported = true;
884
885 return 0;
886
887 exit_err:
888 cpudata->boost_supported = false;
889 return ret;
890 }
891
amd_perf_ctl_reset(unsigned int cpu)892 static void amd_perf_ctl_reset(unsigned int cpu)
893 {
894 wrmsrq_on_cpu(cpu, MSR_AMD_PERF_CTL, 0);
895 }
896
897 #define CPPC_MAX_PERF U8_MAX
898
amd_pstate_init_prefcore(struct amd_cpudata * cpudata)899 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata)
900 {
901 /* user disabled or not detected */
902 if (!amd_pstate_prefcore)
903 return;
904
905 /* should use amd-hfi instead */
906 if (cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS) &&
907 IS_ENABLED(CONFIG_AMD_HFI)) {
908 amd_pstate_prefcore = false;
909 return;
910 }
911
912 cpudata->hw_prefcore = true;
913
914 /* Priorities must be initialized before ITMT support can be toggled on. */
915 sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu);
916 }
917
amd_pstate_update_limits(struct cpufreq_policy * policy)918 static void amd_pstate_update_limits(struct cpufreq_policy *policy)
919 {
920 struct amd_cpudata *cpudata;
921 u32 prev_high = 0, cur_high = 0;
922 bool highest_perf_changed = false;
923 unsigned int cpu = policy->cpu;
924
925 if (!amd_pstate_prefcore)
926 return;
927
928 if (amd_get_highest_perf(cpu, &cur_high))
929 return;
930
931 cpudata = policy->driver_data;
932
933 prev_high = READ_ONCE(cpudata->prefcore_ranking);
934 highest_perf_changed = (prev_high != cur_high);
935 if (highest_perf_changed) {
936 WRITE_ONCE(cpudata->prefcore_ranking, cur_high);
937
938 if (cur_high < CPPC_MAX_PERF) {
939 sched_set_itmt_core_prio((int)cur_high, cpu);
940 sched_update_asym_prefer_cpu(cpu, prev_high, cur_high);
941 }
942 }
943 }
944
945 /*
946 * Get pstate transition delay time from ACPI tables that firmware set
947 * instead of using hardcode value directly.
948 */
amd_pstate_get_transition_delay_us(unsigned int cpu)949 static u32 amd_pstate_get_transition_delay_us(unsigned int cpu)
950 {
951 int transition_delay_ns;
952
953 transition_delay_ns = cppc_get_transition_latency(cpu);
954 if (transition_delay_ns < 0) {
955 if (cpu_feature_enabled(X86_FEATURE_AMD_FAST_CPPC))
956 return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY;
957 else
958 return AMD_PSTATE_TRANSITION_DELAY;
959 }
960
961 return transition_delay_ns / NSEC_PER_USEC;
962 }
963
964 /*
965 * Get pstate transition latency value from ACPI tables that firmware
966 * set instead of using hardcode value directly.
967 */
amd_pstate_get_transition_latency(unsigned int cpu)968 static u32 amd_pstate_get_transition_latency(unsigned int cpu)
969 {
970 int transition_latency;
971
972 transition_latency = cppc_get_transition_latency(cpu);
973 if (transition_latency < 0)
974 return AMD_PSTATE_TRANSITION_LATENCY;
975
976 return transition_latency;
977 }
978
979 /*
980 * amd_pstate_init_freq: Initialize the nominal_freq and lowest_nonlinear_freq
981 * for the @cpudata object.
982 *
983 * Requires: all perf members of @cpudata to be initialized.
984 *
985 * Returns 0 on success, non-zero value on failure.
986 */
amd_pstate_init_freq(struct amd_cpudata * cpudata)987 static int amd_pstate_init_freq(struct amd_cpudata *cpudata)
988 {
989 u32 min_freq, max_freq, nominal_freq, lowest_nonlinear_freq;
990 struct cppc_perf_caps cppc_perf;
991 union perf_cached perf;
992 int ret;
993
994 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
995 if (ret)
996 return ret;
997 perf = READ_ONCE(cpudata->perf);
998
999 if (quirks && quirks->nominal_freq)
1000 nominal_freq = quirks->nominal_freq;
1001 else
1002 nominal_freq = cppc_perf.nominal_freq;
1003 nominal_freq *= 1000;
1004
1005 if (quirks && quirks->lowest_freq) {
1006 min_freq = quirks->lowest_freq;
1007 perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq);
1008 WRITE_ONCE(cpudata->perf, perf);
1009 } else
1010 min_freq = cppc_perf.lowest_freq;
1011
1012 min_freq *= 1000;
1013
1014 WRITE_ONCE(cpudata->nominal_freq, nominal_freq);
1015
1016 /* max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf */
1017 max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf);
1018 WRITE_ONCE(cpudata->max_freq, max_freq);
1019
1020 lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf);
1021 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq);
1022
1023 /**
1024 * Below values need to be initialized correctly, otherwise driver will fail to load
1025 * lowest_nonlinear_freq is a value between [min_freq, nominal_freq]
1026 * Check _CPC in ACPI table objects if any values are incorrect
1027 */
1028 if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) {
1029 pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n",
1030 min_freq, max_freq, nominal_freq);
1031 return -EINVAL;
1032 }
1033
1034 if (lowest_nonlinear_freq < min_freq || lowest_nonlinear_freq > nominal_freq) {
1035 pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n",
1036 lowest_nonlinear_freq, min_freq, nominal_freq);
1037 return -EINVAL;
1038 }
1039
1040 if (perf.bios_min_perf) {
1041 u32 bios_min_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.bios_min_perf);
1042
1043 pr_debug("Found Requested CPU Min Frequency of %uKHz on CPU%d\n",
1044 bios_min_freq, cpudata->cpu);
1045 }
1046
1047 return 0;
1048 }
1049
amd_pstate_cpu_init(struct cpufreq_policy * policy)1050 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
1051 {
1052 struct amd_cpudata *cpudata;
1053 union perf_cached perf;
1054 struct device *dev;
1055 int ret;
1056
1057 /*
1058 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
1059 * which is ideal for initialization process.
1060 */
1061 amd_perf_ctl_reset(policy->cpu);
1062 dev = get_cpu_device(policy->cpu);
1063 if (!dev)
1064 return -ENODEV;
1065
1066 cpudata = kzalloc_obj(*cpudata);
1067 if (!cpudata)
1068 return -ENOMEM;
1069
1070 cpudata->cpu = policy->cpu;
1071
1072 ret = amd_pstate_init_perf(cpudata);
1073 if (ret)
1074 goto free_cpudata1;
1075
1076 amd_pstate_init_prefcore(cpudata);
1077
1078 ret = amd_pstate_init_freq(cpudata);
1079 if (ret)
1080 goto free_cpudata1;
1081
1082 ret = amd_pstate_init_boost_support(cpudata);
1083 if (ret)
1084 goto free_cpudata1;
1085
1086 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu);
1087 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu);
1088
1089 perf = READ_ONCE(cpudata->perf);
1090
1091 policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq,
1092 perf.lowest_perf);
1093 policy->cpuinfo.max_freq = cpudata->max_freq;
1094
1095 policy->driver_data = cpudata;
1096 ret = amd_pstate_cppc_enable(policy);
1097 if (ret)
1098 goto free_cpudata1;
1099
1100 policy->boost_supported = READ_ONCE(cpudata->boost_supported);
1101
1102 /* It will be updated by governor */
1103 policy->cur = policy->cpuinfo.min_freq;
1104
1105 if (cpu_feature_enabled(X86_FEATURE_CPPC))
1106 policy->fast_switch_possible = true;
1107
1108 ret = amd_pstate_init_floor_perf(policy);
1109 if (ret) {
1110 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret);
1111 goto free_cpudata1;
1112 }
1113
1114 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
1115 FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE);
1116 if (ret < 0) {
1117 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
1118 goto free_cpudata1;
1119 }
1120
1121 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
1122 FREQ_QOS_MAX, policy->cpuinfo.max_freq);
1123 if (ret < 0) {
1124 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
1125 goto free_cpudata2;
1126 }
1127
1128
1129 if (!current_pstate_driver->adjust_perf)
1130 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
1131
1132 return 0;
1133
1134 free_cpudata2:
1135 freq_qos_remove_request(&cpudata->req[0]);
1136 free_cpudata1:
1137 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret);
1138 kfree(cpudata);
1139 policy->driver_data = NULL;
1140 return ret;
1141 }
1142
amd_pstate_cpu_exit(struct cpufreq_policy * policy)1143 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy)
1144 {
1145 struct amd_cpudata *cpudata = policy->driver_data;
1146 union perf_cached perf = READ_ONCE(cpudata->perf);
1147
1148 /* Reset CPPC_REQ MSR to the BIOS value */
1149 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false);
1150 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf);
1151
1152 freq_qos_remove_request(&cpudata->req[1]);
1153 freq_qos_remove_request(&cpudata->req[0]);
1154 policy->fast_switch_possible = false;
1155 kfree(cpudata);
1156 }
1157
amd_pstate_get_balanced_epp(struct cpufreq_policy * policy)1158 static int amd_pstate_get_balanced_epp(struct cpufreq_policy *policy)
1159 {
1160 struct amd_cpudata *cpudata = policy->driver_data;
1161
1162 if (power_supply_is_system_supplied())
1163 return cpudata->epp_default_ac;
1164 else
1165 return cpudata->epp_default_dc;
1166 }
1167
amd_pstate_power_supply_notifier(struct notifier_block * nb,unsigned long event,void * data)1168 static int amd_pstate_power_supply_notifier(struct notifier_block *nb,
1169 unsigned long event, void *data)
1170 {
1171 struct amd_cpudata *cpudata = container_of(nb, struct amd_cpudata, power_nb);
1172 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu);
1173 u8 epp;
1174 int ret;
1175
1176 if (event != PSY_EVENT_PROP_CHANGED)
1177 return NOTIFY_OK;
1178
1179 /* dynamic actions are only applied while platform profile is in balanced */
1180 if (cpudata->current_profile != PLATFORM_PROFILE_BALANCED)
1181 return 0;
1182
1183 if (!policy)
1184 return NOTIFY_OK;
1185
1186 epp = amd_pstate_get_balanced_epp(policy);
1187
1188 ret = amd_pstate_set_epp(policy, epp);
1189 if (ret)
1190 pr_warn("Failed to set CPU %d EPP %u: %d\n", cpudata->cpu, epp, ret);
1191
1192 return NOTIFY_OK;
1193 }
1194
amd_pstate_get_epp_from_platform_profile(struct cpufreq_policy * policy,enum platform_profile_option profile)1195 static int amd_pstate_get_epp_from_platform_profile(struct cpufreq_policy *policy,
1196 enum platform_profile_option profile)
1197 {
1198 switch (profile) {
1199 case PLATFORM_PROFILE_PERFORMANCE:
1200 return AMD_CPPC_EPP_PERFORMANCE;
1201 case PLATFORM_PROFILE_BALANCED:
1202 return amd_pstate_get_balanced_epp(policy);
1203 case PLATFORM_PROFILE_LOW_POWER:
1204 return AMD_CPPC_EPP_POWERSAVE;
1205 default:
1206 break;
1207 }
1208
1209 pr_err("Unknown Platform Profile %d\n", profile);
1210 return -EOPNOTSUPP;
1211 }
1212
amd_pstate_profile_probe(void * drvdata,unsigned long * choices)1213 static int amd_pstate_profile_probe(void *drvdata, unsigned long *choices)
1214 {
1215 set_bit(PLATFORM_PROFILE_LOW_POWER, choices);
1216 set_bit(PLATFORM_PROFILE_BALANCED, choices);
1217 set_bit(PLATFORM_PROFILE_PERFORMANCE, choices);
1218
1219 return 0;
1220 }
1221
amd_pstate_profile_get(struct device * dev,enum platform_profile_option * profile)1222 static int amd_pstate_profile_get(struct device *dev,
1223 enum platform_profile_option *profile)
1224 {
1225 struct amd_cpudata *cpudata = dev_get_drvdata(dev);
1226
1227 *profile = cpudata->current_profile;
1228
1229 return 0;
1230 }
1231
amd_pstate_profile_set(struct device * dev,enum platform_profile_option profile)1232 static int amd_pstate_profile_set(struct device *dev,
1233 enum platform_profile_option profile)
1234 {
1235 struct amd_cpudata *cpudata = dev_get_drvdata(dev);
1236 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu);
1237 int ret;
1238 u8 epp;
1239
1240 if (!policy)
1241 return -ENODEV;
1242
1243 ret = amd_pstate_get_epp_from_platform_profile(policy, profile);
1244 if (ret < 0)
1245 return ret;
1246
1247 epp = (u8)ret;
1248 ret = amd_pstate_set_epp(policy, epp);
1249 if (ret)
1250 return ret;
1251
1252 cpudata->current_profile = profile;
1253
1254 return 0;
1255 }
1256
1257 static const struct platform_profile_ops amd_pstate_profile_ops = {
1258 .probe = amd_pstate_profile_probe,
1259 .profile_set = amd_pstate_profile_set,
1260 .profile_get = amd_pstate_profile_get,
1261 };
1262
amd_pstate_clear_dynamic_epp(struct cpufreq_policy * policy)1263 void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy)
1264 {
1265 struct amd_cpudata *cpudata = policy->driver_data;
1266
1267 if (cpudata->power_nb.notifier_call)
1268 power_supply_unreg_notifier(&cpudata->power_nb);
1269 if (cpudata->ppdev) {
1270 platform_profile_remove(cpudata->ppdev);
1271 cpudata->ppdev = NULL;
1272 }
1273 kfree(cpudata->profile_name);
1274 cpudata->dynamic_epp = false;
1275 }
1276 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_clear_dynamic_epp);
1277
amd_pstate_set_dynamic_epp(struct cpufreq_policy * policy)1278 static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy)
1279 {
1280 struct amd_cpudata *cpudata = policy->driver_data;
1281 u64 prev = READ_ONCE(cpudata->cppc_req_cached);
1282 int ret;
1283 u8 epp;
1284
1285 ret = amd_pstate_get_epp_from_platform_profile(policy, cpudata->current_profile);
1286 if (ret < 0)
1287 return ret;
1288
1289 epp = (u8)ret;
1290 ret = amd_pstate_set_epp(policy, epp);
1291 if (ret)
1292 return ret;
1293
1294 cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu);
1295 if (!cpudata->profile_name)
1296 return -ENOMEM;
1297
1298 cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu),
1299 cpudata->profile_name,
1300 policy->driver_data,
1301 &amd_pstate_profile_ops);
1302 if (IS_ERR(cpudata->ppdev)) {
1303 ret = PTR_ERR(cpudata->ppdev);
1304 goto cleanup;
1305 }
1306
1307 /* only enable notifier if things will actually change */
1308 if (cpudata->epp_default_ac != cpudata->epp_default_dc) {
1309 cpudata->power_nb.notifier_call = amd_pstate_power_supply_notifier;
1310 ret = power_supply_reg_notifier(&cpudata->power_nb);
1311 if (ret)
1312 goto cleanup;
1313 }
1314
1315 cpudata->dynamic_epp = true;
1316
1317 return 0;
1318
1319 cleanup:
1320 amd_pstate_clear_dynamic_epp(policy);
1321
1322 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, prev);
1323 /* Restore previous EPP if toggling Dynamic EPP failed. */
1324 amd_pstate_set_epp(policy, epp);
1325 return ret;
1326 }
1327
1328 /* Sysfs attributes */
1329
1330 /*
1331 * This frequency is to indicate the maximum hardware frequency.
1332 * If boost is not active but supported, the frequency will be larger than the
1333 * one in cpuinfo.
1334 */
show_amd_pstate_max_freq(struct cpufreq_policy * policy,char * buf)1335 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
1336 char *buf)
1337 {
1338 struct amd_cpudata *cpudata = policy->driver_data;
1339
1340 return sysfs_emit(buf, "%u\n", cpudata->max_freq);
1341 }
1342
show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy * policy,char * buf)1343 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
1344 char *buf)
1345 {
1346 struct amd_cpudata *cpudata;
1347 union perf_cached perf;
1348
1349 cpudata = policy->driver_data;
1350 perf = READ_ONCE(cpudata->perf);
1351
1352 return sysfs_emit(buf, "%u\n",
1353 perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf));
1354 }
1355
1356 /*
1357 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
1358 * need to expose it to sysfs.
1359 */
show_amd_pstate_highest_perf(struct cpufreq_policy * policy,char * buf)1360 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
1361 char *buf)
1362 {
1363 struct amd_cpudata *cpudata;
1364
1365 cpudata = policy->driver_data;
1366
1367 return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf);
1368 }
1369
show_amd_pstate_prefcore_ranking(struct cpufreq_policy * policy,char * buf)1370 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy,
1371 char *buf)
1372 {
1373 u8 perf;
1374 struct amd_cpudata *cpudata = policy->driver_data;
1375
1376 perf = READ_ONCE(cpudata->prefcore_ranking);
1377
1378 return sysfs_emit(buf, "%u\n", perf);
1379 }
1380
show_amd_pstate_hw_prefcore(struct cpufreq_policy * policy,char * buf)1381 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy,
1382 char *buf)
1383 {
1384 bool hw_prefcore;
1385 struct amd_cpudata *cpudata = policy->driver_data;
1386
1387 hw_prefcore = READ_ONCE(cpudata->hw_prefcore);
1388
1389 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore));
1390 }
1391
show_energy_performance_available_preferences(struct cpufreq_policy * policy,char * buf)1392 static ssize_t show_energy_performance_available_preferences(
1393 struct cpufreq_policy *policy, char *buf)
1394 {
1395 int i, offset = 0;
1396 struct amd_cpudata *cpudata = policy->driver_data;
1397
1398 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1399 return sysfs_emit_at(buf, offset, "%s\n",
1400 energy_perf_strings[EPP_INDEX_PERFORMANCE]);
1401
1402 for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++)
1403 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]);
1404
1405 offset += sysfs_emit_at(buf, offset, "\n");
1406
1407 return offset;
1408 }
1409
store_energy_performance_preference(struct cpufreq_policy * policy,const char * buf,size_t count)1410 ssize_t store_energy_performance_preference(struct cpufreq_policy *policy,
1411 const char *buf, size_t count)
1412 {
1413 struct amd_cpudata *cpudata = policy->driver_data;
1414 ssize_t ret;
1415 bool raw_epp = false;
1416 u8 epp;
1417
1418 /*
1419 * if the value matches a number, use that, otherwise see if
1420 * matches an index in the energy_perf_strings array
1421 */
1422 ret = kstrtou8(buf, 0, &epp);
1423 raw_epp = !ret;
1424 if (ret) {
1425 ret = sysfs_match_string(energy_perf_strings, buf);
1426 if (ret < 0 || ret == EPP_INDEX_CUSTOM)
1427 return -EINVAL;
1428
1429 if (ret == EPP_INDEX_DYNAMIC) {
1430 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1431 return -EBUSY;
1432 /*
1433 * Dynamic EPP was already enabled for this CPU.
1434 * Nothing to do.
1435 */
1436 if (cpudata->dynamic_epp)
1437 return count;
1438
1439 cpudata->current_profile = PLATFORM_PROFILE_BALANCED;
1440 ret = amd_pstate_set_dynamic_epp(policy);
1441 if (ret)
1442 return ret;
1443
1444 return count;
1445 }
1446
1447 if (ret)
1448 epp = epp_values[ret];
1449 else
1450 epp = cpudata->epp_default_dc;
1451 }
1452
1453 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
1454 pr_debug("EPP cannot be set under performance policy\n");
1455 return -EBUSY;
1456 }
1457
1458 /*
1459 * Dynamic EPP was enabled previously!
1460 * Switch back to the static EPP mode.
1461 */
1462 if (cpudata->dynamic_epp)
1463 amd_pstate_clear_dynamic_epp(policy);
1464
1465 ret = amd_pstate_set_epp(policy, epp);
1466 if (ret)
1467 return ret;
1468
1469 cpudata->raw_epp = raw_epp;
1470
1471 return count;
1472 }
1473 EXPORT_SYMBOL_FOR_PSTATE_UT(store_energy_performance_preference);
1474
show_energy_performance_preference(struct cpufreq_policy * policy,char * buf)1475 ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf)
1476 {
1477 struct amd_cpudata *cpudata = policy->driver_data;
1478 u8 preference, epp;
1479
1480 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached);
1481
1482 if (!cpudata->dynamic_epp && cpudata->raw_epp)
1483 return sysfs_emit(buf, "%u\n", epp);
1484
1485 switch (epp) {
1486 case AMD_CPPC_EPP_PERFORMANCE:
1487 preference = EPP_INDEX_PERFORMANCE;
1488 break;
1489 case AMD_CPPC_EPP_BALANCE_PERFORMANCE:
1490 preference = EPP_INDEX_BALANCE_PERFORMANCE;
1491 break;
1492 case AMD_CPPC_EPP_BALANCE_POWERSAVE:
1493 preference = EPP_INDEX_BALANCE_POWERSAVE;
1494 break;
1495 case AMD_CPPC_EPP_POWERSAVE:
1496 preference = EPP_INDEX_POWERSAVE;
1497 break;
1498 default:
1499 return -EINVAL;
1500 }
1501
1502 if (cpudata->dynamic_epp)
1503 return sysfs_emit(buf, "dynamic(profile:%s)\n", energy_perf_strings[preference]);
1504
1505 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]);
1506 }
1507 EXPORT_SYMBOL_FOR_PSTATE_UT(show_energy_performance_preference);
1508
store_amd_pstate_floor_freq(struct cpufreq_policy * policy,const char * buf,size_t count)1509 ssize_t store_amd_pstate_floor_freq(struct cpufreq_policy *policy, const char *buf, size_t count)
1510 {
1511 struct amd_cpudata *cpudata = policy->driver_data;
1512 union perf_cached perf = READ_ONCE(cpudata->perf);
1513 unsigned int freq;
1514 u8 floor_perf;
1515 int ret;
1516
1517 ret = kstrtouint(buf, 0, &freq);
1518 if (ret)
1519 return ret;
1520
1521 if (freq < policy->cpuinfo.min_freq || freq > policy->max)
1522 return -EINVAL;
1523
1524 floor_perf = freq_to_perf(perf, cpudata->nominal_freq, freq);
1525 ret = amd_pstate_set_floor_perf(policy, floor_perf);
1526
1527 if (!ret)
1528 cpudata->floor_freq = freq;
1529
1530 return ret ?: count;
1531 }
1532 EXPORT_SYMBOL_FOR_PSTATE_UT(store_amd_pstate_floor_freq);
1533
show_amd_pstate_floor_freq(struct cpufreq_policy * policy,char * buf)1534 ssize_t show_amd_pstate_floor_freq(struct cpufreq_policy *policy, char *buf)
1535 {
1536 struct amd_cpudata *cpudata = policy->driver_data;
1537
1538 return sysfs_emit(buf, "%u\n", cpudata->floor_freq);
1539 }
1540 EXPORT_SYMBOL_FOR_PSTATE_UT(show_amd_pstate_floor_freq);
1541
show_amd_pstate_floor_count(struct cpufreq_policy * policy,char * buf)1542 static ssize_t show_amd_pstate_floor_count(struct cpufreq_policy *policy, char *buf)
1543 {
1544 struct amd_cpudata *cpudata = policy->driver_data;
1545 u8 count = cpudata->floor_perf_cnt;
1546
1547 return sysfs_emit(buf, "%u\n", count);
1548 }
1549
1550 cpufreq_freq_attr_ro(amd_pstate_max_freq);
1551 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
1552
1553 cpufreq_freq_attr_ro(amd_pstate_highest_perf);
1554 cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking);
1555 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore);
1556 cpufreq_freq_attr_rw(energy_performance_preference);
1557 cpufreq_freq_attr_ro(energy_performance_available_preferences);
1558 cpufreq_freq_attr_rw(amd_pstate_floor_freq);
1559 cpufreq_freq_attr_ro(amd_pstate_floor_count);
1560
1561 struct freq_attr_visibility {
1562 struct freq_attr *attr;
1563 bool (*visibility_fn)(void);
1564 };
1565
1566 /* For attributes which are always visible */
always_visible(void)1567 static bool always_visible(void)
1568 {
1569 return true;
1570 }
1571
1572 /* Determines whether prefcore related attributes should be visible */
prefcore_visibility(void)1573 static bool prefcore_visibility(void)
1574 {
1575 return amd_pstate_prefcore;
1576 }
1577
1578 /* Determines whether energy performance preference should be visible */
epp_visibility(void)1579 static bool epp_visibility(void)
1580 {
1581 return cppc_state == AMD_PSTATE_ACTIVE;
1582 }
1583
1584 /* Determines whether amd_pstate_floor_freq related attributes should be visible */
floor_freq_visibility(void)1585 static bool floor_freq_visibility(void)
1586 {
1587 return cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO);
1588 }
1589
1590 static struct freq_attr_visibility amd_pstate_attr_visibility[] = {
1591 {&amd_pstate_max_freq, always_visible},
1592 {&amd_pstate_lowest_nonlinear_freq, always_visible},
1593 {&amd_pstate_highest_perf, always_visible},
1594 {&amd_pstate_prefcore_ranking, prefcore_visibility},
1595 {&amd_pstate_hw_prefcore, prefcore_visibility},
1596 {&energy_performance_preference, epp_visibility},
1597 {&energy_performance_available_preferences, epp_visibility},
1598 {&amd_pstate_floor_freq, floor_freq_visibility},
1599 {&amd_pstate_floor_count, floor_freq_visibility},
1600 };
1601
amd_pstate_get_current_attrs(void)1602 struct freq_attr **amd_pstate_get_current_attrs(void)
1603 {
1604 if (!current_pstate_driver)
1605 return NULL;
1606 return current_pstate_driver->attr;
1607 }
1608 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_current_attrs);
1609
get_freq_attrs(void)1610 static struct freq_attr **get_freq_attrs(void)
1611 {
1612 bool attr_visible[ARRAY_SIZE(amd_pstate_attr_visibility)];
1613 struct freq_attr **attrs;
1614 int i, j, count;
1615
1616 for (i = 0, count = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) {
1617 struct freq_attr_visibility *v = &amd_pstate_attr_visibility[i];
1618
1619 attr_visible[i] = v->visibility_fn();
1620 if (attr_visible[i])
1621 count++;
1622 }
1623
1624 /* amd_pstate_{max_freq, lowest_nonlinear_freq, highest_perf} should always be visible */
1625 BUG_ON(!count);
1626
1627 attrs = kcalloc(count + 1, sizeof(struct freq_attr *), GFP_KERNEL);
1628 if (!attrs)
1629 return ERR_PTR(-ENOMEM);
1630
1631 for (i = 0, j = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) {
1632 if (!attr_visible[i])
1633 continue;
1634
1635 attrs[j++] = amd_pstate_attr_visibility[i].attr;
1636 }
1637
1638 return attrs;
1639 }
1640
amd_pstate_driver_cleanup(void)1641 static void amd_pstate_driver_cleanup(void)
1642 {
1643 if (amd_pstate_prefcore)
1644 sched_clear_itmt_support();
1645
1646 cppc_state = AMD_PSTATE_DISABLE;
1647 kfree(current_pstate_driver->attr);
1648 current_pstate_driver->attr = NULL;
1649 current_pstate_driver = NULL;
1650 }
1651
amd_pstate_set_driver(int mode_idx)1652 static int amd_pstate_set_driver(int mode_idx)
1653 {
1654 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) {
1655 cppc_state = mode_idx;
1656 if (cppc_state == AMD_PSTATE_DISABLE)
1657 pr_info("driver is explicitly disabled\n");
1658
1659 if (cppc_state == AMD_PSTATE_ACTIVE)
1660 current_pstate_driver = &amd_pstate_epp_driver;
1661
1662 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED)
1663 current_pstate_driver = &amd_pstate_driver;
1664
1665 return 0;
1666 }
1667
1668 return -EINVAL;
1669 }
1670
amd_pstate_register_driver(int mode)1671 static int amd_pstate_register_driver(int mode)
1672 {
1673 struct freq_attr **attr = NULL;
1674 int ret;
1675
1676 ret = amd_pstate_set_driver(mode);
1677 if (ret)
1678 return ret;
1679
1680 cppc_state = mode;
1681
1682 /*
1683 * Note: It is important to compute the attrs _after_
1684 * re-initializing the cppc_state. Some attributes become
1685 * visible only when cppc_state is AMD_PSTATE_ACTIVE.
1686 */
1687 attr = get_freq_attrs();
1688 if (IS_ERR(attr)) {
1689 ret = (int) PTR_ERR(attr);
1690 pr_err("Couldn't compute freq_attrs for current mode %s [%d]\n",
1691 amd_pstate_get_mode_string(cppc_state), ret);
1692 amd_pstate_driver_cleanup();
1693 return ret;
1694 }
1695
1696 current_pstate_driver->attr = attr;
1697
1698 /* at least one CPU supports CPB */
1699 current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB);
1700
1701 ret = cpufreq_register_driver(current_pstate_driver);
1702 if (ret) {
1703 amd_pstate_driver_cleanup();
1704 return ret;
1705 }
1706
1707 /* Enable ITMT support once all CPUs have initialized their asym priorities. */
1708 if (amd_pstate_prefcore)
1709 sched_set_itmt_support();
1710
1711 return 0;
1712 }
1713
amd_pstate_unregister_driver(int dummy)1714 static int amd_pstate_unregister_driver(int dummy)
1715 {
1716 cpufreq_unregister_driver(current_pstate_driver);
1717 amd_pstate_driver_cleanup();
1718 return 0;
1719 }
1720
amd_pstate_change_mode_without_dvr_change(int mode)1721 static int amd_pstate_change_mode_without_dvr_change(int mode)
1722 {
1723 int cpu = 0;
1724
1725 cppc_state = mode;
1726
1727 if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE)
1728 return 0;
1729
1730 for_each_online_cpu(cpu) {
1731 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
1732 }
1733
1734 return 0;
1735 }
1736
amd_pstate_change_driver_mode(int mode)1737 static int amd_pstate_change_driver_mode(int mode)
1738 {
1739 int ret;
1740
1741 lockdep_assert_held(&amd_pstate_driver_lock);
1742
1743 ret = amd_pstate_unregister_driver(0);
1744 if (ret)
1745 return ret;
1746
1747 ret = amd_pstate_register_driver(mode);
1748 if (ret)
1749 return ret;
1750
1751 return 0;
1752 }
1753
1754 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = {
1755 [AMD_PSTATE_DISABLE] = {
1756 [AMD_PSTATE_DISABLE] = NULL,
1757 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver,
1758 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver,
1759 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver,
1760 },
1761 [AMD_PSTATE_PASSIVE] = {
1762 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
1763 [AMD_PSTATE_PASSIVE] = NULL,
1764 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode,
1765 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change,
1766 },
1767 [AMD_PSTATE_ACTIVE] = {
1768 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
1769 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode,
1770 [AMD_PSTATE_ACTIVE] = NULL,
1771 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode,
1772 },
1773 [AMD_PSTATE_GUIDED] = {
1774 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
1775 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change,
1776 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode,
1777 [AMD_PSTATE_GUIDED] = NULL,
1778 },
1779 };
1780
amd_pstate_show_status(char * buf)1781 static ssize_t amd_pstate_show_status(char *buf)
1782 {
1783 if (!current_pstate_driver)
1784 return sysfs_emit(buf, "disable\n");
1785
1786 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]);
1787 }
1788
amd_pstate_get_status(void)1789 int amd_pstate_get_status(void)
1790 {
1791 return cppc_state;
1792 }
1793 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_status);
1794
amd_pstate_update_status(const char * buf,size_t size)1795 int amd_pstate_update_status(const char *buf, size_t size)
1796 {
1797 int mode_idx;
1798
1799 if (size > strlen("passive") || size < strlen("active"))
1800 return -EINVAL;
1801
1802 mode_idx = get_mode_idx_from_str(buf, size);
1803 if (mode_idx < 0)
1804 return mode_idx;
1805
1806 if (mode_state_machine[cppc_state][mode_idx]) {
1807 guard(mutex)(&amd_pstate_driver_lock);
1808 return mode_state_machine[cppc_state][mode_idx](mode_idx);
1809 }
1810
1811 return 0;
1812 }
1813 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_update_status);
1814
status_show(struct device * dev,struct device_attribute * attr,char * buf)1815 static ssize_t status_show(struct device *dev,
1816 struct device_attribute *attr, char *buf)
1817 {
1818
1819 guard(mutex)(&amd_pstate_driver_lock);
1820
1821 return amd_pstate_show_status(buf);
1822 }
1823
status_store(struct device * a,struct device_attribute * b,const char * buf,size_t count)1824 static ssize_t status_store(struct device *a, struct device_attribute *b,
1825 const char *buf, size_t count)
1826 {
1827 char *p = memchr(buf, '\n', count);
1828 int ret;
1829
1830 ret = amd_pstate_update_status(buf, p ? p - buf : count);
1831
1832 return ret < 0 ? ret : count;
1833 }
1834
prefcore_show(struct device * dev,struct device_attribute * attr,char * buf)1835 static ssize_t prefcore_show(struct device *dev,
1836 struct device_attribute *attr, char *buf)
1837 {
1838 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore));
1839 }
1840
1841 static DEVICE_ATTR_RW(status);
1842 static DEVICE_ATTR_RO(prefcore);
1843
1844 static struct attribute *pstate_global_attributes[] = {
1845 &dev_attr_status.attr,
1846 &dev_attr_prefcore.attr,
1847 NULL
1848 };
1849
1850 static const struct attribute_group amd_pstate_global_attr_group = {
1851 .name = "amd_pstate",
1852 .attrs = pstate_global_attributes,
1853 };
1854
amd_pstate_acpi_pm_profile_server(void)1855 static bool amd_pstate_acpi_pm_profile_server(void)
1856 {
1857 switch (acpi_gbl_FADT.preferred_profile) {
1858 case PM_ENTERPRISE_SERVER:
1859 case PM_SOHO_SERVER:
1860 case PM_PERFORMANCE_SERVER:
1861 return true;
1862 }
1863 return false;
1864 }
1865
amd_pstate_acpi_pm_profile_undefined(void)1866 static bool amd_pstate_acpi_pm_profile_undefined(void)
1867 {
1868 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED)
1869 return true;
1870 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES)
1871 return true;
1872 return false;
1873 }
1874
amd_pstate_epp_cpu_init(struct cpufreq_policy * policy)1875 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
1876 {
1877 struct amd_cpudata *cpudata;
1878 union perf_cached perf;
1879 struct device *dev;
1880 int default_epp;
1881 int ret;
1882
1883 /*
1884 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
1885 * which is ideal for initialization process.
1886 */
1887 amd_perf_ctl_reset(policy->cpu);
1888 dev = get_cpu_device(policy->cpu);
1889 if (!dev)
1890 return -ENODEV;
1891
1892 cpudata = kzalloc_obj(*cpudata);
1893 if (!cpudata)
1894 return -ENOMEM;
1895
1896 cpudata->cpu = policy->cpu;
1897
1898 ret = amd_pstate_init_perf(cpudata);
1899 if (ret)
1900 goto free_cpudata1;
1901
1902 amd_pstate_init_prefcore(cpudata);
1903
1904 ret = amd_pstate_init_freq(cpudata);
1905 if (ret)
1906 goto free_cpudata1;
1907
1908 ret = amd_pstate_init_boost_support(cpudata);
1909 if (ret)
1910 goto free_cpudata1;
1911
1912 perf = READ_ONCE(cpudata->perf);
1913
1914 policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq,
1915 perf.lowest_perf);
1916 policy->cpuinfo.max_freq = cpudata->max_freq;
1917 policy->driver_data = cpudata;
1918
1919 ret = amd_pstate_cppc_enable(policy);
1920 if (ret)
1921 goto free_cpudata1;
1922
1923 /* It will be updated by governor */
1924 policy->cur = policy->cpuinfo.min_freq;
1925
1926
1927 policy->boost_supported = READ_ONCE(cpudata->boost_supported);
1928
1929 /* Cache the firmware programmed EPP */
1930 default_epp = amd_pstate_get_epp(cpudata);
1931 if (default_epp < 0) {
1932 ret = default_epp;
1933 goto free_cpudata1;
1934 }
1935 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &cpudata->cppc_req_cached, default_epp);
1936
1937 /*
1938 * Set the policy to provide a valid fallback value in case
1939 * the default cpufreq governor is neither powersave nor performance.
1940 */
1941 if (amd_pstate_acpi_pm_profile_server() ||
1942 amd_pstate_acpi_pm_profile_undefined()) {
1943 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
1944 cpudata->epp_default_ac = cpudata->epp_default_dc = default_epp;
1945 cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE;
1946 } else {
1947 policy->policy = CPUFREQ_POLICY_POWERSAVE;
1948 cpudata->epp_default_ac = AMD_CPPC_EPP_PERFORMANCE;
1949 cpudata->epp_default_dc = AMD_CPPC_EPP_BALANCE_PERFORMANCE;
1950 cpudata->current_profile = PLATFORM_PROFILE_BALANCED;
1951 }
1952
1953 ret = amd_pstate_set_epp(policy, cpudata->epp_default_dc);
1954 if (ret)
1955 goto free_cpudata1;
1956
1957 ret = amd_pstate_init_floor_perf(policy);
1958 if (ret) {
1959 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret);
1960 goto free_cpudata1;
1961 }
1962
1963 current_pstate_driver->adjust_perf = NULL;
1964
1965 return 0;
1966
1967 free_cpudata1:
1968 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret);
1969 kfree(cpudata);
1970 policy->driver_data = NULL;
1971 return ret;
1972 }
1973
amd_pstate_epp_cpu_exit(struct cpufreq_policy * policy)1974 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)
1975 {
1976 struct amd_cpudata *cpudata = policy->driver_data;
1977
1978 if (cpudata) {
1979 union perf_cached perf = READ_ONCE(cpudata->perf);
1980
1981 if (cpudata->dynamic_epp)
1982 amd_pstate_clear_dynamic_epp(policy);
1983
1984 /* Reset CPPC_REQ MSR to the BIOS value */
1985 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false);
1986 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf);
1987
1988 kfree(cpudata);
1989 policy->driver_data = NULL;
1990 }
1991
1992 pr_debug("CPU %d exiting\n", policy->cpu);
1993 }
1994
amd_pstate_epp_update_limit(struct cpufreq_policy * policy,bool policy_change)1995 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy, bool policy_change)
1996 {
1997 struct amd_cpudata *cpudata = policy->driver_data;
1998 union perf_cached perf;
1999 u8 epp;
2000
2001 if (policy_change ||
2002 policy->min != cpudata->min_limit_freq ||
2003 policy->max != cpudata->max_limit_freq)
2004 amd_pstate_update_min_max_limit(policy);
2005
2006 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
2007 epp = 0;
2008 else
2009 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached);
2010
2011 perf = READ_ONCE(cpudata->perf);
2012
2013 return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U,
2014 perf.max_limit_perf, epp, false);
2015 }
2016
amd_pstate_epp_set_policy(struct cpufreq_policy * policy)2017 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy)
2018 {
2019 struct amd_cpudata *cpudata = policy->driver_data;
2020 int ret;
2021
2022 if (!policy->cpuinfo.max_freq)
2023 return -ENODEV;
2024
2025 /* Must be a switch between PERFORMANCE and POWERSAVE */
2026 if (cpudata->policy != policy->policy) {
2027 /*
2028 * Disable dynamic_epp when switching
2029 * out of CPUFREQ_POLICY_POWERSAVE.
2030 */
2031 if (cpudata->dynamic_epp) {
2032 WARN_ON_ONCE(cpudata->policy != CPUFREQ_POLICY_POWERSAVE);
2033 amd_pstate_clear_dynamic_epp(policy);
2034 }
2035 }
2036
2037 cpudata->policy = policy->policy;
2038
2039 ret = amd_pstate_epp_update_limit(policy, true);
2040 if (ret)
2041 return ret;
2042
2043 /*
2044 * policy->cur is never updated with the amd_pstate_epp driver, but it
2045 * is used as a stale frequency value. So, keep it within limits.
2046 */
2047 policy->cur = policy->min;
2048
2049 return 0;
2050 }
2051
amd_pstate_cpu_online(struct cpufreq_policy * policy)2052 static int amd_pstate_cpu_online(struct cpufreq_policy *policy)
2053 {
2054 struct amd_cpudata *cpudata = policy->driver_data;
2055 union perf_cached perf = READ_ONCE(cpudata->perf);
2056 u8 cached_floor_perf;
2057 int ret;
2058
2059 ret = amd_pstate_cppc_enable(policy);
2060 if (ret)
2061 return ret;
2062
2063 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq);
2064 return amd_pstate_set_floor_perf(policy, cached_floor_perf);
2065 }
2066
amd_pstate_cpu_offline(struct cpufreq_policy * policy)2067 static int amd_pstate_cpu_offline(struct cpufreq_policy *policy)
2068 {
2069 struct amd_cpudata *cpudata = policy->driver_data;
2070 union perf_cached perf = READ_ONCE(cpudata->perf);
2071 int ret;
2072
2073 /*
2074 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified
2075 * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the
2076 * limits, epp and desired perf will get reset to the cached values in cpudata struct
2077 */
2078 ret = amd_pstate_update_perf(policy, perf.bios_min_perf,
2079 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached),
2080 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached),
2081 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached),
2082 false);
2083 if (ret)
2084 return ret;
2085
2086 return amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf);
2087 }
2088
amd_pstate_suspend(struct cpufreq_policy * policy)2089 static int amd_pstate_suspend(struct cpufreq_policy *policy)
2090 {
2091 struct amd_cpudata *cpudata = policy->driver_data;
2092 union perf_cached perf = READ_ONCE(cpudata->perf);
2093 int ret;
2094
2095 /*
2096 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified
2097 * min_perf value across kexec reboots. If this CPU is just resumed back without kexec,
2098 * the limits, epp and desired perf will get reset to the cached values in cpudata struct
2099 */
2100 ret = amd_pstate_update_perf(policy, perf.bios_min_perf,
2101 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached),
2102 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached),
2103 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached),
2104 false);
2105 if (ret)
2106 return ret;
2107
2108 ret = amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf);
2109 if (ret)
2110 return ret;
2111
2112 /* set this flag to avoid setting core offline*/
2113 cpudata->suspended = true;
2114
2115 return 0;
2116 }
2117
amd_pstate_resume(struct cpufreq_policy * policy)2118 static int amd_pstate_resume(struct cpufreq_policy *policy)
2119 {
2120 struct amd_cpudata *cpudata = policy->driver_data;
2121 union perf_cached perf = READ_ONCE(cpudata->perf);
2122 int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur);
2123 u8 cached_floor_perf;
2124 int ret;
2125
2126 /* Set CPPC_REQ to last sane value until the governor updates it */
2127 ret = amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf,
2128 0U, false);
2129 if (ret)
2130 return ret;
2131
2132 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq);
2133 return amd_pstate_set_floor_perf(policy, cached_floor_perf);
2134 }
2135
amd_pstate_epp_resume(struct cpufreq_policy * policy)2136 static int amd_pstate_epp_resume(struct cpufreq_policy *policy)
2137 {
2138 struct amd_cpudata *cpudata = policy->driver_data;
2139 union perf_cached perf = READ_ONCE(cpudata->perf);
2140 u8 cached_floor_perf;
2141
2142 if (cpudata->suspended) {
2143 int ret;
2144
2145 /* enable amd pstate from suspend state*/
2146 ret = amd_pstate_epp_update_limit(policy, false);
2147 if (ret)
2148 return ret;
2149
2150 cpudata->suspended = false;
2151 }
2152
2153 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq);
2154 return amd_pstate_set_floor_perf(policy, cached_floor_perf);
2155 }
2156
2157 static struct cpufreq_driver amd_pstate_driver = {
2158 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
2159 .verify = amd_pstate_verify,
2160 .target = amd_pstate_target,
2161 .fast_switch = amd_pstate_fast_switch,
2162 .init = amd_pstate_cpu_init,
2163 .exit = amd_pstate_cpu_exit,
2164 .online = amd_pstate_cpu_online,
2165 .offline = amd_pstate_cpu_offline,
2166 .suspend = amd_pstate_suspend,
2167 .resume = amd_pstate_resume,
2168 .set_boost = amd_pstate_set_boost,
2169 .update_limits = amd_pstate_update_limits,
2170 .name = "amd-pstate",
2171 };
2172
2173 static struct cpufreq_driver amd_pstate_epp_driver = {
2174 .flags = CPUFREQ_CONST_LOOPS,
2175 .verify = amd_pstate_verify,
2176 .setpolicy = amd_pstate_epp_set_policy,
2177 .init = amd_pstate_epp_cpu_init,
2178 .exit = amd_pstate_epp_cpu_exit,
2179 .offline = amd_pstate_cpu_offline,
2180 .online = amd_pstate_cpu_online,
2181 .suspend = amd_pstate_suspend,
2182 .resume = amd_pstate_epp_resume,
2183 .update_limits = amd_pstate_update_limits,
2184 .set_boost = amd_pstate_set_boost,
2185 .name = "amd-pstate-epp",
2186 };
2187
2188 /*
2189 * Processors without frequency scaling support can't do CPPC.
2190 * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F.
2191 * show the debug message that helps to check if the CPU has CPPC support for loading issue.
2192 */
amd_cppc_supported(void)2193 static bool amd_cppc_supported(void)
2194 {
2195 struct cpuinfo_x86 *c = &cpu_data(0);
2196 bool warn = false;
2197
2198 if (!cpu_feature_enabled(X86_FEATURE_HW_PSTATE)) {
2199 pr_debug_once("frequency scaling is not supported by the processor\n");
2200 return false;
2201 }
2202
2203 if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) {
2204 pr_debug_once("CPPC feature is not supported by the processor\n");
2205 return false;
2206 }
2207
2208 /*
2209 * If the CPPC feature is disabled in the BIOS for processors
2210 * that support MSR-based CPPC, the AMD Pstate driver may not
2211 * function correctly.
2212 *
2213 * For such processors, check the CPPC flag and display a
2214 * warning message if the platform supports CPPC.
2215 *
2216 * Note: The code check below will not abort the driver
2217 * registration process because of the code is added for
2218 * debugging purposes. Besides, it may still be possible for
2219 * the driver to work using the shared-memory mechanism.
2220 */
2221 if (!cpu_feature_enabled(X86_FEATURE_CPPC)) {
2222 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) {
2223 switch (c->x86_model) {
2224 case 0x60 ... 0x6F:
2225 case 0x80 ... 0xAF:
2226 warn = true;
2227 break;
2228 }
2229 } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) ||
2230 cpu_feature_enabled(X86_FEATURE_ZEN4)) {
2231 switch (c->x86_model) {
2232 case 0x10 ... 0x1F:
2233 case 0x40 ... 0xAF:
2234 warn = true;
2235 break;
2236 }
2237 } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) {
2238 warn = true;
2239 }
2240 }
2241
2242 if (warn)
2243 pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n"
2244 "Please enable it if your BIOS has the CPPC option.\n");
2245 return true;
2246 }
2247
amd_pstate_init(void)2248 static int __init amd_pstate_init(void)
2249 {
2250 struct device *dev_root;
2251 int ret;
2252
2253 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
2254 return -ENODEV;
2255
2256 /* show debug message only if CPPC is not supported */
2257 if (!amd_cppc_supported())
2258 return -EOPNOTSUPP;
2259
2260 /* show warning message when BIOS broken or ACPI disabled */
2261 if (!acpi_cpc_valid()) {
2262 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
2263 return -ENODEV;
2264 }
2265
2266 /* don't keep reloading if cpufreq_driver exists */
2267 if (cpufreq_get_current_driver())
2268 return -EEXIST;
2269
2270 quirks = NULL;
2271
2272 /* check if this machine need CPPC quirks */
2273 dmi_check_system(amd_pstate_quirks_table);
2274
2275 /*
2276 * determine the driver mode from the command line or kernel config.
2277 * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED.
2278 * command line options will override the kernel config settings.
2279 */
2280
2281 if (cppc_state == AMD_PSTATE_UNDEFINED) {
2282 /* Disable on the following configs by default:
2283 * 1. Undefined platforms
2284 * 2. Server platforms with CPUs older than Family 0x1A.
2285 */
2286 if (amd_pstate_acpi_pm_profile_undefined() ||
2287 (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) {
2288 pr_info("driver load is disabled, boot with specific mode to enable this\n");
2289 return -ENODEV;
2290 }
2291 /* get driver mode from kernel config option [1:4] */
2292 cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE;
2293 }
2294
2295 if (cppc_state == AMD_PSTATE_DISABLE) {
2296 pr_info("driver load is disabled, boot with specific mode to enable this\n");
2297 return -ENODEV;
2298 }
2299
2300 /* capability check */
2301 if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
2302 pr_debug("AMD CPPC MSR based functionality is supported\n");
2303 } else {
2304 pr_debug("AMD CPPC shared memory based functionality is supported\n");
2305 static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable);
2306 static_call_update(amd_pstate_init_perf, shmem_init_perf);
2307 static_call_update(amd_pstate_update_perf, shmem_update_perf);
2308 static_call_update(amd_pstate_get_epp, shmem_get_epp);
2309 static_call_update(amd_pstate_set_epp, shmem_set_epp);
2310 }
2311
2312 if (amd_pstate_prefcore) {
2313 ret = amd_detect_prefcore(&amd_pstate_prefcore);
2314 if (ret)
2315 return ret;
2316 }
2317
2318 ret = amd_pstate_register_driver(cppc_state);
2319 if (ret) {
2320 pr_err("failed to register with return %d\n", ret);
2321 return ret;
2322 }
2323
2324 dev_root = bus_get_dev_root(&cpu_subsys);
2325 if (dev_root) {
2326 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group);
2327 put_device(dev_root);
2328 if (ret) {
2329 pr_err("sysfs attribute export failed with error %d.\n", ret);
2330 goto global_attr_free;
2331 }
2332 }
2333
2334 return ret;
2335
2336 global_attr_free:
2337 amd_pstate_unregister_driver(0);
2338 return ret;
2339 }
2340 device_initcall(amd_pstate_init);
2341
amd_pstate_param(char * str)2342 static int __init amd_pstate_param(char *str)
2343 {
2344 size_t size;
2345 int mode_idx;
2346
2347 if (!str)
2348 return -EINVAL;
2349
2350 size = strlen(str);
2351 mode_idx = get_mode_idx_from_str(str, size);
2352
2353 return amd_pstate_set_driver(mode_idx);
2354 }
2355
amd_prefcore_param(char * str)2356 static int __init amd_prefcore_param(char *str)
2357 {
2358 if (!strcmp(str, "disable"))
2359 amd_pstate_prefcore = false;
2360
2361 return 0;
2362 }
2363
2364 early_param("amd_pstate", amd_pstate_param);
2365 early_param("amd_prefcore", amd_prefcore_param);
2366
2367 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
2368 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");
2369