1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/kernel.h> 26 #include <linux/module.h> 27 #include <linux/init.h> 28 #include <linux/smp.h> 29 #include <linux/sched.h> 30 #include <linux/cpufreq.h> 31 #include <linux/compiler.h> 32 #include <linux/dmi.h> 33 #include <linux/slab.h> 34 #include <linux/acpi.h> 35 #include <linux/io.h> 36 #include <linux/delay.h> 37 #include <linux/uaccess.h> 38 #include <linux/static_call.h> 39 #include <linux/amd-pstate.h> 40 #include <linux/topology.h> 41 42 #include <acpi/processor.h> 43 #include <acpi/cppc_acpi.h> 44 45 #include <asm/msr.h> 46 #include <asm/processor.h> 47 #include <asm/cpufeature.h> 48 #include <asm/cpu_device_id.h> 49 #include "amd-pstate-trace.h" 50 51 #define AMD_PSTATE_TRANSITION_LATENCY 20000 52 #define AMD_PSTATE_TRANSITION_DELAY 1000 53 #define CPPC_HIGHEST_PERF_PERFORMANCE 196 54 #define CPPC_HIGHEST_PERF_DEFAULT 166 55 56 /* 57 * TODO: We need more time to fine tune processors with shared memory solution 58 * with community together. 59 * 60 * There are some performance drops on the CPU benchmarks which reports from 61 * Suse. We are co-working with them to fine tune the shared memory solution. So 62 * we disable it by default to go acpi-cpufreq on these processors and add a 63 * module parameter to be able to enable it manually for debugging. 64 */ 65 static struct cpufreq_driver *current_pstate_driver; 66 static struct cpufreq_driver amd_pstate_driver; 67 static struct cpufreq_driver amd_pstate_epp_driver; 68 static int cppc_state = AMD_PSTATE_UNDEFINED; 69 static bool cppc_enabled; 70 static bool amd_pstate_prefcore = true; 71 static struct quirk_entry *quirks; 72 73 /* 74 * AMD Energy Preference Performance (EPP) 75 * The EPP is used in the CCLK DPM controller to drive 76 * the frequency that a core is going to operate during 77 * short periods of activity. EPP values will be utilized for 78 * different OS profiles (balanced, performance, power savings) 79 * display strings corresponding to EPP index in the 80 * energy_perf_strings[] 81 * index String 82 *------------------------------------- 83 * 0 default 84 * 1 performance 85 * 2 balance_performance 86 * 3 balance_power 87 * 4 power 88 */ 89 enum energy_perf_value_index { 90 EPP_INDEX_DEFAULT = 0, 91 EPP_INDEX_PERFORMANCE, 92 EPP_INDEX_BALANCE_PERFORMANCE, 93 EPP_INDEX_BALANCE_POWERSAVE, 94 EPP_INDEX_POWERSAVE, 95 }; 96 97 static const char * const energy_perf_strings[] = { 98 [EPP_INDEX_DEFAULT] = "default", 99 [EPP_INDEX_PERFORMANCE] = "performance", 100 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 101 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 102 [EPP_INDEX_POWERSAVE] = "power", 103 NULL 104 }; 105 106 static unsigned int epp_values[] = { 107 [EPP_INDEX_DEFAULT] = 0, 108 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 109 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 110 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 111 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 112 }; 113 114 typedef int (*cppc_mode_transition_fn)(int); 115 116 static struct quirk_entry quirk_amd_7k62 = { 117 .nominal_freq = 2600, 118 .lowest_freq = 550, 119 }; 120 121 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) 122 { 123 /** 124 * match the broken bios for family 17h processor support CPPC V2 125 * broken BIOS lack of nominal_freq and lowest_freq capabilities 126 * definition in ACPI tables 127 */ 128 if (boot_cpu_has(X86_FEATURE_ZEN2)) { 129 quirks = dmi->driver_data; 130 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); 131 return 1; 132 } 133 134 return 0; 135 } 136 137 static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { 138 { 139 .callback = dmi_matched_7k62_bios_bug, 140 .ident = "AMD EPYC 7K62", 141 .matches = { 142 DMI_MATCH(DMI_BIOS_VERSION, "5.14"), 143 DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), 144 }, 145 .driver_data = &quirk_amd_7k62, 146 }, 147 {} 148 }; 149 MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); 150 151 static inline int get_mode_idx_from_str(const char *str, size_t size) 152 { 153 int i; 154 155 for (i=0; i < AMD_PSTATE_MAX; i++) { 156 if (!strncmp(str, amd_pstate_mode_string[i], size)) 157 return i; 158 } 159 return -EINVAL; 160 } 161 162 static DEFINE_MUTEX(amd_pstate_limits_lock); 163 static DEFINE_MUTEX(amd_pstate_driver_lock); 164 165 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) 166 { 167 u64 epp; 168 int ret; 169 170 if (boot_cpu_has(X86_FEATURE_CPPC)) { 171 if (!cppc_req_cached) { 172 epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 173 &cppc_req_cached); 174 if (epp) 175 return epp; 176 } 177 epp = (cppc_req_cached >> 24) & 0xFF; 178 } else { 179 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 180 if (ret < 0) { 181 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 182 return -EIO; 183 } 184 } 185 186 return (s16)(epp & 0xff); 187 } 188 189 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) 190 { 191 s16 epp; 192 int index = -EINVAL; 193 194 epp = amd_pstate_get_epp(cpudata, 0); 195 if (epp < 0) 196 return epp; 197 198 switch (epp) { 199 case AMD_CPPC_EPP_PERFORMANCE: 200 index = EPP_INDEX_PERFORMANCE; 201 break; 202 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 203 index = EPP_INDEX_BALANCE_PERFORMANCE; 204 break; 205 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 206 index = EPP_INDEX_BALANCE_POWERSAVE; 207 break; 208 case AMD_CPPC_EPP_POWERSAVE: 209 index = EPP_INDEX_POWERSAVE; 210 break; 211 default: 212 break; 213 } 214 215 return index; 216 } 217 218 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) 219 { 220 int ret; 221 struct cppc_perf_ctrls perf_ctrls; 222 223 if (boot_cpu_has(X86_FEATURE_CPPC)) { 224 u64 value = READ_ONCE(cpudata->cppc_req_cached); 225 226 value &= ~GENMASK_ULL(31, 24); 227 value |= (u64)epp << 24; 228 WRITE_ONCE(cpudata->cppc_req_cached, value); 229 230 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 231 if (!ret) 232 cpudata->epp_cached = epp; 233 } else { 234 perf_ctrls.energy_perf = epp; 235 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 236 if (ret) { 237 pr_debug("failed to set energy perf value (%d)\n", ret); 238 return ret; 239 } 240 cpudata->epp_cached = epp; 241 } 242 243 return ret; 244 } 245 246 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, 247 int pref_index) 248 { 249 int epp = -EINVAL; 250 int ret; 251 252 if (!pref_index) { 253 pr_debug("EPP pref_index is invalid\n"); 254 return -EINVAL; 255 } 256 257 if (epp == -EINVAL) 258 epp = epp_values[pref_index]; 259 260 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 261 pr_debug("EPP cannot be set under performance policy\n"); 262 return -EBUSY; 263 } 264 265 ret = amd_pstate_set_epp(cpudata, epp); 266 267 return ret; 268 } 269 270 static inline int pstate_enable(bool enable) 271 { 272 int ret, cpu; 273 unsigned long logical_proc_id_mask = 0; 274 275 if (enable == cppc_enabled) 276 return 0; 277 278 for_each_present_cpu(cpu) { 279 unsigned long logical_id = topology_logical_die_id(cpu); 280 281 if (test_bit(logical_id, &logical_proc_id_mask)) 282 continue; 283 284 set_bit(logical_id, &logical_proc_id_mask); 285 286 ret = wrmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_ENABLE, 287 enable); 288 if (ret) 289 return ret; 290 } 291 292 cppc_enabled = enable; 293 return 0; 294 } 295 296 static int cppc_enable(bool enable) 297 { 298 int cpu, ret = 0; 299 struct cppc_perf_ctrls perf_ctrls; 300 301 if (enable == cppc_enabled) 302 return 0; 303 304 for_each_present_cpu(cpu) { 305 ret = cppc_set_enable(cpu, enable); 306 if (ret) 307 return ret; 308 309 /* Enable autonomous mode for EPP */ 310 if (cppc_state == AMD_PSTATE_ACTIVE) { 311 /* Set desired perf as zero to allow EPP firmware control */ 312 perf_ctrls.desired_perf = 0; 313 ret = cppc_set_perf(cpu, &perf_ctrls); 314 if (ret) 315 return ret; 316 } 317 } 318 319 cppc_enabled = enable; 320 return ret; 321 } 322 323 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); 324 325 static inline int amd_pstate_enable(bool enable) 326 { 327 return static_call(amd_pstate_enable)(enable); 328 } 329 330 static u32 amd_pstate_highest_perf_set(struct amd_cpudata *cpudata) 331 { 332 struct cpuinfo_x86 *c = &cpu_data(0); 333 334 /* 335 * For AMD CPUs with Family ID 19H and Model ID range 0x70 to 0x7f, 336 * the highest performance level is set to 196. 337 * https://bugzilla.kernel.org/show_bug.cgi?id=218759 338 */ 339 if (c->x86 == 0x19 && (c->x86_model >= 0x70 && c->x86_model <= 0x7f)) 340 return CPPC_HIGHEST_PERF_PERFORMANCE; 341 342 return CPPC_HIGHEST_PERF_DEFAULT; 343 } 344 345 static int pstate_init_perf(struct amd_cpudata *cpudata) 346 { 347 u64 cap1; 348 u32 highest_perf; 349 350 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 351 &cap1); 352 if (ret) 353 return ret; 354 355 /* For platforms that do not support the preferred core feature, the 356 * highest_pef may be configured with 166 or 255, to avoid max frequency 357 * calculated wrongly. we take the AMD_CPPC_HIGHEST_PERF(cap1) value as 358 * the default max perf. 359 */ 360 if (cpudata->hw_prefcore) 361 highest_perf = amd_pstate_highest_perf_set(cpudata); 362 else 363 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); 364 365 WRITE_ONCE(cpudata->highest_perf, highest_perf); 366 WRITE_ONCE(cpudata->max_limit_perf, highest_perf); 367 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); 368 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); 369 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); 370 WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1)); 371 WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1)); 372 return 0; 373 } 374 375 static int cppc_init_perf(struct amd_cpudata *cpudata) 376 { 377 struct cppc_perf_caps cppc_perf; 378 u32 highest_perf; 379 380 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 381 if (ret) 382 return ret; 383 384 if (cpudata->hw_prefcore) 385 highest_perf = amd_pstate_highest_perf_set(cpudata); 386 else 387 highest_perf = cppc_perf.highest_perf; 388 389 WRITE_ONCE(cpudata->highest_perf, highest_perf); 390 WRITE_ONCE(cpudata->max_limit_perf, highest_perf); 391 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); 392 WRITE_ONCE(cpudata->lowest_nonlinear_perf, 393 cppc_perf.lowest_nonlinear_perf); 394 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); 395 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); 396 WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf); 397 398 if (cppc_state == AMD_PSTATE_ACTIVE) 399 return 0; 400 401 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); 402 if (ret) { 403 pr_warn("failed to get auto_sel, ret: %d\n", ret); 404 return 0; 405 } 406 407 ret = cppc_set_auto_sel(cpudata->cpu, 408 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 409 410 if (ret) 411 pr_warn("failed to set auto_sel, ret: %d\n", ret); 412 413 return ret; 414 } 415 416 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); 417 418 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 419 { 420 return static_call(amd_pstate_init_perf)(cpudata); 421 } 422 423 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, 424 u32 des_perf, u32 max_perf, bool fast_switch) 425 { 426 if (fast_switch) 427 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); 428 else 429 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 430 READ_ONCE(cpudata->cppc_req_cached)); 431 } 432 433 static void cppc_update_perf(struct amd_cpudata *cpudata, 434 u32 min_perf, u32 des_perf, 435 u32 max_perf, bool fast_switch) 436 { 437 struct cppc_perf_ctrls perf_ctrls; 438 439 perf_ctrls.max_perf = max_perf; 440 perf_ctrls.min_perf = min_perf; 441 perf_ctrls.desired_perf = des_perf; 442 443 cppc_set_perf(cpudata->cpu, &perf_ctrls); 444 } 445 446 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); 447 448 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, 449 u32 min_perf, u32 des_perf, 450 u32 max_perf, bool fast_switch) 451 { 452 static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, 453 max_perf, fast_switch); 454 } 455 456 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 457 { 458 u64 aperf, mperf, tsc; 459 unsigned long flags; 460 461 local_irq_save(flags); 462 rdmsrl(MSR_IA32_APERF, aperf); 463 rdmsrl(MSR_IA32_MPERF, mperf); 464 tsc = rdtsc(); 465 466 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 467 local_irq_restore(flags); 468 return false; 469 } 470 471 local_irq_restore(flags); 472 473 cpudata->cur.aperf = aperf; 474 cpudata->cur.mperf = mperf; 475 cpudata->cur.tsc = tsc; 476 cpudata->cur.aperf -= cpudata->prev.aperf; 477 cpudata->cur.mperf -= cpudata->prev.mperf; 478 cpudata->cur.tsc -= cpudata->prev.tsc; 479 480 cpudata->prev.aperf = aperf; 481 cpudata->prev.mperf = mperf; 482 cpudata->prev.tsc = tsc; 483 484 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 485 486 return true; 487 } 488 489 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, 490 u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags) 491 { 492 u64 prev = READ_ONCE(cpudata->cppc_req_cached); 493 u64 value = prev; 494 495 min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, 496 cpudata->max_limit_perf); 497 max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, 498 cpudata->max_limit_perf); 499 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 500 501 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 502 min_perf = des_perf; 503 des_perf = 0; 504 } 505 506 value &= ~AMD_CPPC_MIN_PERF(~0L); 507 value |= AMD_CPPC_MIN_PERF(min_perf); 508 509 value &= ~AMD_CPPC_DES_PERF(~0L); 510 value |= AMD_CPPC_DES_PERF(des_perf); 511 512 value &= ~AMD_CPPC_MAX_PERF(~0L); 513 value |= AMD_CPPC_MAX_PERF(max_perf); 514 515 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 516 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 517 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 518 cpudata->cpu, (value != prev), fast_switch); 519 } 520 521 if (value == prev) 522 return; 523 524 WRITE_ONCE(cpudata->cppc_req_cached, value); 525 526 amd_pstate_update_perf(cpudata, min_perf, des_perf, 527 max_perf, fast_switch); 528 } 529 530 static int amd_pstate_verify(struct cpufreq_policy_data *policy) 531 { 532 cpufreq_verify_within_cpu_limits(policy); 533 534 return 0; 535 } 536 537 static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) 538 { 539 u32 max_limit_perf, min_limit_perf, lowest_perf; 540 struct amd_cpudata *cpudata = policy->driver_data; 541 542 max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); 543 min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); 544 545 lowest_perf = READ_ONCE(cpudata->lowest_perf); 546 if (min_limit_perf < lowest_perf) 547 min_limit_perf = lowest_perf; 548 549 if (max_limit_perf < min_limit_perf) 550 max_limit_perf = min_limit_perf; 551 552 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); 553 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); 554 WRITE_ONCE(cpudata->max_limit_freq, policy->max); 555 WRITE_ONCE(cpudata->min_limit_freq, policy->min); 556 557 return 0; 558 } 559 560 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 561 unsigned int target_freq, bool fast_switch) 562 { 563 struct cpufreq_freqs freqs; 564 struct amd_cpudata *cpudata = policy->driver_data; 565 unsigned long max_perf, min_perf, des_perf, cap_perf; 566 567 if (!cpudata->max_freq) 568 return -ENODEV; 569 570 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 571 amd_pstate_update_min_max_limit(policy); 572 573 cap_perf = READ_ONCE(cpudata->highest_perf); 574 min_perf = READ_ONCE(cpudata->lowest_perf); 575 max_perf = cap_perf; 576 577 freqs.old = policy->cur; 578 freqs.new = target_freq; 579 580 des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, 581 cpudata->max_freq); 582 583 WARN_ON(fast_switch && !policy->fast_switch_enabled); 584 /* 585 * If fast_switch is desired, then there aren't any registered 586 * transition notifiers. See comment for 587 * cpufreq_enable_fast_switch(). 588 */ 589 if (!fast_switch) 590 cpufreq_freq_transition_begin(policy, &freqs); 591 592 amd_pstate_update(cpudata, min_perf, des_perf, 593 max_perf, fast_switch, policy->governor->flags); 594 595 if (!fast_switch) 596 cpufreq_freq_transition_end(policy, &freqs, false); 597 598 return 0; 599 } 600 601 static int amd_pstate_target(struct cpufreq_policy *policy, 602 unsigned int target_freq, 603 unsigned int relation) 604 { 605 return amd_pstate_update_freq(policy, target_freq, false); 606 } 607 608 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 609 unsigned int target_freq) 610 { 611 if (!amd_pstate_update_freq(policy, target_freq, true)) 612 return target_freq; 613 return policy->cur; 614 } 615 616 static void amd_pstate_adjust_perf(unsigned int cpu, 617 unsigned long _min_perf, 618 unsigned long target_perf, 619 unsigned long capacity) 620 { 621 unsigned long max_perf, min_perf, des_perf, 622 cap_perf, lowest_nonlinear_perf, max_freq; 623 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 624 struct amd_cpudata *cpudata = policy->driver_data; 625 unsigned int target_freq; 626 627 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 628 amd_pstate_update_min_max_limit(policy); 629 630 631 cap_perf = READ_ONCE(cpudata->highest_perf); 632 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 633 max_freq = READ_ONCE(cpudata->max_freq); 634 635 des_perf = cap_perf; 636 if (target_perf < capacity) 637 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 638 639 min_perf = READ_ONCE(cpudata->lowest_perf); 640 if (_min_perf < capacity) 641 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 642 643 if (min_perf < lowest_nonlinear_perf) 644 min_perf = lowest_nonlinear_perf; 645 646 max_perf = cap_perf; 647 if (max_perf < min_perf) 648 max_perf = min_perf; 649 650 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 651 target_freq = div_u64(des_perf * max_freq, max_perf); 652 policy->cur = target_freq; 653 654 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, 655 policy->governor->flags); 656 cpufreq_cpu_put(policy); 657 } 658 659 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 660 { 661 struct amd_cpudata *cpudata = policy->driver_data; 662 int ret; 663 664 if (!cpudata->boost_supported) { 665 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 666 return -EINVAL; 667 } 668 669 if (state) 670 policy->cpuinfo.max_freq = cpudata->max_freq; 671 else 672 policy->cpuinfo.max_freq = cpudata->nominal_freq; 673 674 policy->max = policy->cpuinfo.max_freq; 675 676 ret = freq_qos_update_request(&cpudata->req[1], 677 policy->cpuinfo.max_freq); 678 if (ret < 0) 679 return ret; 680 681 return 0; 682 } 683 684 static void amd_pstate_boost_init(struct amd_cpudata *cpudata) 685 { 686 u32 highest_perf, nominal_perf; 687 688 highest_perf = READ_ONCE(cpudata->highest_perf); 689 nominal_perf = READ_ONCE(cpudata->nominal_perf); 690 691 if (highest_perf <= nominal_perf) 692 return; 693 694 cpudata->boost_supported = true; 695 current_pstate_driver->boost_enabled = true; 696 } 697 698 static void amd_perf_ctl_reset(unsigned int cpu) 699 { 700 wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 701 } 702 703 /* 704 * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks 705 * due to locking, so queue the work for later. 706 */ 707 static void amd_pstste_sched_prefcore_workfn(struct work_struct *work) 708 { 709 sched_set_itmt_support(); 710 } 711 static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn); 712 713 /* 714 * Get the highest performance register value. 715 * @cpu: CPU from which to get highest performance. 716 * @highest_perf: Return address. 717 * 718 * Return: 0 for success, -EIO otherwise. 719 */ 720 static int amd_pstate_get_highest_perf(int cpu, u32 *highest_perf) 721 { 722 int ret; 723 724 if (boot_cpu_has(X86_FEATURE_CPPC)) { 725 u64 cap1; 726 727 ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); 728 if (ret) 729 return ret; 730 WRITE_ONCE(*highest_perf, AMD_CPPC_HIGHEST_PERF(cap1)); 731 } else { 732 u64 cppc_highest_perf; 733 734 ret = cppc_get_highest_perf(cpu, &cppc_highest_perf); 735 if (ret) 736 return ret; 737 WRITE_ONCE(*highest_perf, cppc_highest_perf); 738 } 739 740 return (ret); 741 } 742 743 #define CPPC_MAX_PERF U8_MAX 744 745 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) 746 { 747 int ret, prio; 748 u32 highest_perf; 749 750 ret = amd_pstate_get_highest_perf(cpudata->cpu, &highest_perf); 751 if (ret) 752 return; 753 754 cpudata->hw_prefcore = true; 755 /* check if CPPC preferred core feature is enabled*/ 756 if (highest_perf < CPPC_MAX_PERF) 757 prio = (int)highest_perf; 758 else { 759 pr_debug("AMD CPPC preferred core is unsupported!\n"); 760 cpudata->hw_prefcore = false; 761 return; 762 } 763 764 if (!amd_pstate_prefcore) 765 return; 766 767 /* 768 * The priorities can be set regardless of whether or not 769 * sched_set_itmt_support(true) has been called and it is valid to 770 * update them at any time after it has been called. 771 */ 772 sched_set_itmt_core_prio(prio, cpudata->cpu); 773 774 schedule_work(&sched_prefcore_work); 775 } 776 777 static void amd_pstate_update_limits(unsigned int cpu) 778 { 779 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 780 struct amd_cpudata *cpudata = policy->driver_data; 781 u32 prev_high = 0, cur_high = 0; 782 int ret; 783 bool highest_perf_changed = false; 784 785 mutex_lock(&amd_pstate_driver_lock); 786 if ((!amd_pstate_prefcore) || (!cpudata->hw_prefcore)) 787 goto free_cpufreq_put; 788 789 ret = amd_pstate_get_highest_perf(cpu, &cur_high); 790 if (ret) 791 goto free_cpufreq_put; 792 793 prev_high = READ_ONCE(cpudata->prefcore_ranking); 794 if (prev_high != cur_high) { 795 highest_perf_changed = true; 796 WRITE_ONCE(cpudata->prefcore_ranking, cur_high); 797 798 if (cur_high < CPPC_MAX_PERF) 799 sched_set_itmt_core_prio((int)cur_high, cpu); 800 } 801 802 free_cpufreq_put: 803 cpufreq_cpu_put(policy); 804 805 if (!highest_perf_changed) 806 cpufreq_update_policy(cpu); 807 808 mutex_unlock(&amd_pstate_driver_lock); 809 } 810 811 /* 812 * Get pstate transition delay time from ACPI tables that firmware set 813 * instead of using hardcode value directly. 814 */ 815 static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) 816 { 817 u32 transition_delay_ns; 818 819 transition_delay_ns = cppc_get_transition_latency(cpu); 820 if (transition_delay_ns == CPUFREQ_ETERNAL) 821 return AMD_PSTATE_TRANSITION_DELAY; 822 823 return transition_delay_ns / NSEC_PER_USEC; 824 } 825 826 /* 827 * Get pstate transition latency value from ACPI tables that firmware 828 * set instead of using hardcode value directly. 829 */ 830 static u32 amd_pstate_get_transition_latency(unsigned int cpu) 831 { 832 u32 transition_latency; 833 834 transition_latency = cppc_get_transition_latency(cpu); 835 if (transition_latency == CPUFREQ_ETERNAL) 836 return AMD_PSTATE_TRANSITION_LATENCY; 837 838 return transition_latency; 839 } 840 841 /* 842 * amd_pstate_init_freq: Initialize the max_freq, min_freq, 843 * nominal_freq and lowest_nonlinear_freq for 844 * the @cpudata object. 845 * 846 * Requires: highest_perf, lowest_perf, nominal_perf and 847 * lowest_nonlinear_perf members of @cpudata to be 848 * initialized. 849 * 850 * Returns 0 on success, non-zero value on failure. 851 */ 852 static int amd_pstate_init_freq(struct amd_cpudata *cpudata) 853 { 854 int ret; 855 u32 min_freq; 856 u32 highest_perf, max_freq; 857 u32 nominal_perf, nominal_freq; 858 u32 lowest_nonlinear_perf, lowest_nonlinear_freq; 859 u32 boost_ratio, lowest_nonlinear_ratio; 860 struct cppc_perf_caps cppc_perf; 861 862 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 863 if (ret) 864 return ret; 865 866 if (quirks && quirks->lowest_freq) 867 min_freq = quirks->lowest_freq * 1000; 868 else 869 min_freq = cppc_perf.lowest_freq * 1000; 870 871 if (quirks && quirks->nominal_freq) 872 nominal_freq = quirks->nominal_freq ; 873 else 874 nominal_freq = cppc_perf.nominal_freq; 875 876 nominal_perf = READ_ONCE(cpudata->nominal_perf); 877 878 highest_perf = READ_ONCE(cpudata->highest_perf); 879 boost_ratio = div_u64(highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); 880 max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT) * 1000; 881 882 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 883 lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, 884 nominal_perf); 885 lowest_nonlinear_freq = (nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT) * 1000; 886 887 WRITE_ONCE(cpudata->min_freq, min_freq); 888 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); 889 WRITE_ONCE(cpudata->nominal_freq, nominal_freq); 890 WRITE_ONCE(cpudata->max_freq, max_freq); 891 892 return 0; 893 } 894 895 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 896 { 897 int min_freq, max_freq, nominal_freq, ret; 898 struct device *dev; 899 struct amd_cpudata *cpudata; 900 901 /* 902 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 903 * which is ideal for initialization process. 904 */ 905 amd_perf_ctl_reset(policy->cpu); 906 dev = get_cpu_device(policy->cpu); 907 if (!dev) 908 return -ENODEV; 909 910 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 911 if (!cpudata) 912 return -ENOMEM; 913 914 cpudata->cpu = policy->cpu; 915 916 amd_pstate_init_prefcore(cpudata); 917 918 ret = amd_pstate_init_perf(cpudata); 919 if (ret) 920 goto free_cpudata1; 921 922 ret = amd_pstate_init_freq(cpudata); 923 if (ret) 924 goto free_cpudata1; 925 926 min_freq = READ_ONCE(cpudata->min_freq); 927 max_freq = READ_ONCE(cpudata->max_freq); 928 nominal_freq = READ_ONCE(cpudata->nominal_freq); 929 930 if (min_freq <= 0 || max_freq <= 0 || 931 nominal_freq <= 0 || min_freq > max_freq) { 932 dev_err(dev, 933 "min_freq(%d) or max_freq(%d) or nominal_freq (%d) value is incorrect, check _CPC in ACPI tables\n", 934 min_freq, max_freq, nominal_freq); 935 ret = -EINVAL; 936 goto free_cpudata1; 937 } 938 939 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); 940 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); 941 942 policy->min = min_freq; 943 policy->max = max_freq; 944 945 policy->cpuinfo.min_freq = min_freq; 946 policy->cpuinfo.max_freq = max_freq; 947 948 /* It will be updated by governor */ 949 policy->cur = policy->cpuinfo.min_freq; 950 951 if (boot_cpu_has(X86_FEATURE_CPPC)) 952 policy->fast_switch_possible = true; 953 954 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 955 FREQ_QOS_MIN, policy->cpuinfo.min_freq); 956 if (ret < 0) { 957 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 958 goto free_cpudata1; 959 } 960 961 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 962 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 963 if (ret < 0) { 964 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 965 goto free_cpudata2; 966 } 967 968 cpudata->max_limit_freq = max_freq; 969 cpudata->min_limit_freq = min_freq; 970 971 policy->driver_data = cpudata; 972 973 amd_pstate_boost_init(cpudata); 974 if (!current_pstate_driver->adjust_perf) 975 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 976 977 return 0; 978 979 free_cpudata2: 980 freq_qos_remove_request(&cpudata->req[0]); 981 free_cpudata1: 982 kfree(cpudata); 983 return ret; 984 } 985 986 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) 987 { 988 struct amd_cpudata *cpudata = policy->driver_data; 989 990 freq_qos_remove_request(&cpudata->req[1]); 991 freq_qos_remove_request(&cpudata->req[0]); 992 policy->fast_switch_possible = false; 993 kfree(cpudata); 994 995 return 0; 996 } 997 998 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) 999 { 1000 int ret; 1001 1002 ret = amd_pstate_enable(true); 1003 if (ret) 1004 pr_err("failed to enable amd-pstate during resume, return %d\n", ret); 1005 1006 return ret; 1007 } 1008 1009 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) 1010 { 1011 int ret; 1012 1013 ret = amd_pstate_enable(false); 1014 if (ret) 1015 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); 1016 1017 return ret; 1018 } 1019 1020 /* Sysfs attributes */ 1021 1022 /* 1023 * This frequency is to indicate the maximum hardware frequency. 1024 * If boost is not active but supported, the frequency will be larger than the 1025 * one in cpuinfo. 1026 */ 1027 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 1028 char *buf) 1029 { 1030 int max_freq; 1031 struct amd_cpudata *cpudata = policy->driver_data; 1032 1033 max_freq = READ_ONCE(cpudata->max_freq); 1034 if (max_freq < 0) 1035 return max_freq; 1036 1037 return sysfs_emit(buf, "%u\n", max_freq); 1038 } 1039 1040 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 1041 char *buf) 1042 { 1043 int freq; 1044 struct amd_cpudata *cpudata = policy->driver_data; 1045 1046 freq = READ_ONCE(cpudata->lowest_nonlinear_freq); 1047 if (freq < 0) 1048 return freq; 1049 1050 return sysfs_emit(buf, "%u\n", freq); 1051 } 1052 1053 /* 1054 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 1055 * need to expose it to sysfs. 1056 */ 1057 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 1058 char *buf) 1059 { 1060 u32 perf; 1061 struct amd_cpudata *cpudata = policy->driver_data; 1062 1063 perf = READ_ONCE(cpudata->highest_perf); 1064 1065 return sysfs_emit(buf, "%u\n", perf); 1066 } 1067 1068 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, 1069 char *buf) 1070 { 1071 u32 perf; 1072 struct amd_cpudata *cpudata = policy->driver_data; 1073 1074 perf = READ_ONCE(cpudata->prefcore_ranking); 1075 1076 return sysfs_emit(buf, "%u\n", perf); 1077 } 1078 1079 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, 1080 char *buf) 1081 { 1082 bool hw_prefcore; 1083 struct amd_cpudata *cpudata = policy->driver_data; 1084 1085 hw_prefcore = READ_ONCE(cpudata->hw_prefcore); 1086 1087 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); 1088 } 1089 1090 static ssize_t show_energy_performance_available_preferences( 1091 struct cpufreq_policy *policy, char *buf) 1092 { 1093 int i = 0; 1094 int offset = 0; 1095 struct amd_cpudata *cpudata = policy->driver_data; 1096 1097 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1098 return sysfs_emit_at(buf, offset, "%s\n", 1099 energy_perf_strings[EPP_INDEX_PERFORMANCE]); 1100 1101 while (energy_perf_strings[i] != NULL) 1102 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); 1103 1104 offset += sysfs_emit_at(buf, offset, "\n"); 1105 1106 return offset; 1107 } 1108 1109 static ssize_t store_energy_performance_preference( 1110 struct cpufreq_policy *policy, const char *buf, size_t count) 1111 { 1112 struct amd_cpudata *cpudata = policy->driver_data; 1113 char str_preference[21]; 1114 ssize_t ret; 1115 1116 ret = sscanf(buf, "%20s", str_preference); 1117 if (ret != 1) 1118 return -EINVAL; 1119 1120 ret = match_string(energy_perf_strings, -1, str_preference); 1121 if (ret < 0) 1122 return -EINVAL; 1123 1124 mutex_lock(&amd_pstate_limits_lock); 1125 ret = amd_pstate_set_energy_pref_index(cpudata, ret); 1126 mutex_unlock(&amd_pstate_limits_lock); 1127 1128 return ret ?: count; 1129 } 1130 1131 static ssize_t show_energy_performance_preference( 1132 struct cpufreq_policy *policy, char *buf) 1133 { 1134 struct amd_cpudata *cpudata = policy->driver_data; 1135 int preference; 1136 1137 preference = amd_pstate_get_energy_pref_index(cpudata); 1138 if (preference < 0) 1139 return preference; 1140 1141 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 1142 } 1143 1144 static void amd_pstate_driver_cleanup(void) 1145 { 1146 amd_pstate_enable(false); 1147 cppc_state = AMD_PSTATE_DISABLE; 1148 current_pstate_driver = NULL; 1149 } 1150 1151 static int amd_pstate_register_driver(int mode) 1152 { 1153 int ret; 1154 1155 if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED) 1156 current_pstate_driver = &amd_pstate_driver; 1157 else if (mode == AMD_PSTATE_ACTIVE) 1158 current_pstate_driver = &amd_pstate_epp_driver; 1159 else 1160 return -EINVAL; 1161 1162 cppc_state = mode; 1163 ret = cpufreq_register_driver(current_pstate_driver); 1164 if (ret) { 1165 amd_pstate_driver_cleanup(); 1166 return ret; 1167 } 1168 return 0; 1169 } 1170 1171 static int amd_pstate_unregister_driver(int dummy) 1172 { 1173 cpufreq_unregister_driver(current_pstate_driver); 1174 amd_pstate_driver_cleanup(); 1175 return 0; 1176 } 1177 1178 static int amd_pstate_change_mode_without_dvr_change(int mode) 1179 { 1180 int cpu = 0; 1181 1182 cppc_state = mode; 1183 1184 if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 1185 return 0; 1186 1187 for_each_present_cpu(cpu) { 1188 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 1189 } 1190 1191 return 0; 1192 } 1193 1194 static int amd_pstate_change_driver_mode(int mode) 1195 { 1196 int ret; 1197 1198 ret = amd_pstate_unregister_driver(0); 1199 if (ret) 1200 return ret; 1201 1202 ret = amd_pstate_register_driver(mode); 1203 if (ret) 1204 return ret; 1205 1206 return 0; 1207 } 1208 1209 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 1210 [AMD_PSTATE_DISABLE] = { 1211 [AMD_PSTATE_DISABLE] = NULL, 1212 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 1213 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 1214 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 1215 }, 1216 [AMD_PSTATE_PASSIVE] = { 1217 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1218 [AMD_PSTATE_PASSIVE] = NULL, 1219 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1220 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 1221 }, 1222 [AMD_PSTATE_ACTIVE] = { 1223 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1224 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 1225 [AMD_PSTATE_ACTIVE] = NULL, 1226 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 1227 }, 1228 [AMD_PSTATE_GUIDED] = { 1229 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1230 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 1231 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1232 [AMD_PSTATE_GUIDED] = NULL, 1233 }, 1234 }; 1235 1236 static ssize_t amd_pstate_show_status(char *buf) 1237 { 1238 if (!current_pstate_driver) 1239 return sysfs_emit(buf, "disable\n"); 1240 1241 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 1242 } 1243 1244 static int amd_pstate_update_status(const char *buf, size_t size) 1245 { 1246 int mode_idx; 1247 1248 if (size > strlen("passive") || size < strlen("active")) 1249 return -EINVAL; 1250 1251 mode_idx = get_mode_idx_from_str(buf, size); 1252 1253 if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) 1254 return -EINVAL; 1255 1256 if (mode_state_machine[cppc_state][mode_idx]) 1257 return mode_state_machine[cppc_state][mode_idx](mode_idx); 1258 1259 return 0; 1260 } 1261 1262 static ssize_t status_show(struct device *dev, 1263 struct device_attribute *attr, char *buf) 1264 { 1265 ssize_t ret; 1266 1267 mutex_lock(&amd_pstate_driver_lock); 1268 ret = amd_pstate_show_status(buf); 1269 mutex_unlock(&amd_pstate_driver_lock); 1270 1271 return ret; 1272 } 1273 1274 static ssize_t status_store(struct device *a, struct device_attribute *b, 1275 const char *buf, size_t count) 1276 { 1277 char *p = memchr(buf, '\n', count); 1278 int ret; 1279 1280 mutex_lock(&amd_pstate_driver_lock); 1281 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1282 mutex_unlock(&amd_pstate_driver_lock); 1283 1284 return ret < 0 ? ret : count; 1285 } 1286 1287 static ssize_t prefcore_show(struct device *dev, 1288 struct device_attribute *attr, char *buf) 1289 { 1290 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); 1291 } 1292 1293 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1294 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1295 1296 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1297 cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); 1298 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); 1299 cpufreq_freq_attr_rw(energy_performance_preference); 1300 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1301 static DEVICE_ATTR_RW(status); 1302 static DEVICE_ATTR_RO(prefcore); 1303 1304 static struct freq_attr *amd_pstate_attr[] = { 1305 &amd_pstate_max_freq, 1306 &amd_pstate_lowest_nonlinear_freq, 1307 &amd_pstate_highest_perf, 1308 &amd_pstate_prefcore_ranking, 1309 &amd_pstate_hw_prefcore, 1310 NULL, 1311 }; 1312 1313 static struct freq_attr *amd_pstate_epp_attr[] = { 1314 &amd_pstate_max_freq, 1315 &amd_pstate_lowest_nonlinear_freq, 1316 &amd_pstate_highest_perf, 1317 &amd_pstate_prefcore_ranking, 1318 &amd_pstate_hw_prefcore, 1319 &energy_performance_preference, 1320 &energy_performance_available_preferences, 1321 NULL, 1322 }; 1323 1324 static struct attribute *pstate_global_attributes[] = { 1325 &dev_attr_status.attr, 1326 &dev_attr_prefcore.attr, 1327 NULL 1328 }; 1329 1330 static const struct attribute_group amd_pstate_global_attr_group = { 1331 .name = "amd_pstate", 1332 .attrs = pstate_global_attributes, 1333 }; 1334 1335 static bool amd_pstate_acpi_pm_profile_server(void) 1336 { 1337 switch (acpi_gbl_FADT.preferred_profile) { 1338 case PM_ENTERPRISE_SERVER: 1339 case PM_SOHO_SERVER: 1340 case PM_PERFORMANCE_SERVER: 1341 return true; 1342 } 1343 return false; 1344 } 1345 1346 static bool amd_pstate_acpi_pm_profile_undefined(void) 1347 { 1348 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) 1349 return true; 1350 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) 1351 return true; 1352 return false; 1353 } 1354 1355 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1356 { 1357 int min_freq, max_freq, nominal_freq, ret; 1358 struct amd_cpudata *cpudata; 1359 struct device *dev; 1360 u64 value; 1361 1362 /* 1363 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1364 * which is ideal for initialization process. 1365 */ 1366 amd_perf_ctl_reset(policy->cpu); 1367 dev = get_cpu_device(policy->cpu); 1368 if (!dev) 1369 return -ENODEV; 1370 1371 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 1372 if (!cpudata) 1373 return -ENOMEM; 1374 1375 cpudata->cpu = policy->cpu; 1376 cpudata->epp_policy = 0; 1377 1378 amd_pstate_init_prefcore(cpudata); 1379 1380 ret = amd_pstate_init_perf(cpudata); 1381 if (ret) 1382 goto free_cpudata1; 1383 1384 ret = amd_pstate_init_freq(cpudata); 1385 if (ret) 1386 goto free_cpudata1; 1387 1388 min_freq = READ_ONCE(cpudata->min_freq); 1389 max_freq = READ_ONCE(cpudata->max_freq); 1390 nominal_freq = READ_ONCE(cpudata->nominal_freq); 1391 if (min_freq <= 0 || max_freq <= 0 || 1392 nominal_freq <= 0 || min_freq > max_freq) { 1393 dev_err(dev, 1394 "min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect, check _CPC in ACPI tables\n", 1395 min_freq, max_freq, nominal_freq); 1396 ret = -EINVAL; 1397 goto free_cpudata1; 1398 } 1399 1400 policy->cpuinfo.min_freq = min_freq; 1401 policy->cpuinfo.max_freq = max_freq; 1402 /* It will be updated by governor */ 1403 policy->cur = policy->cpuinfo.min_freq; 1404 1405 policy->driver_data = cpudata; 1406 1407 cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); 1408 1409 policy->min = policy->cpuinfo.min_freq; 1410 policy->max = policy->cpuinfo.max_freq; 1411 1412 /* 1413 * Set the policy to provide a valid fallback value in case 1414 * the default cpufreq governor is neither powersave nor performance. 1415 */ 1416 if (amd_pstate_acpi_pm_profile_server() || 1417 amd_pstate_acpi_pm_profile_undefined()) 1418 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 1419 else 1420 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1421 1422 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1423 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 1424 if (ret) 1425 return ret; 1426 WRITE_ONCE(cpudata->cppc_req_cached, value); 1427 1428 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); 1429 if (ret) 1430 return ret; 1431 WRITE_ONCE(cpudata->cppc_cap1_cached, value); 1432 } 1433 amd_pstate_boost_init(cpudata); 1434 1435 return 0; 1436 1437 free_cpudata1: 1438 kfree(cpudata); 1439 return ret; 1440 } 1441 1442 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1443 { 1444 struct amd_cpudata *cpudata = policy->driver_data; 1445 1446 if (cpudata) { 1447 kfree(cpudata); 1448 policy->driver_data = NULL; 1449 } 1450 1451 pr_debug("CPU %d exiting\n", policy->cpu); 1452 return 0; 1453 } 1454 1455 static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy) 1456 { 1457 struct amd_cpudata *cpudata = policy->driver_data; 1458 u32 max_perf, min_perf, min_limit_perf, max_limit_perf; 1459 u64 value; 1460 s16 epp; 1461 1462 max_perf = READ_ONCE(cpudata->highest_perf); 1463 min_perf = READ_ONCE(cpudata->lowest_perf); 1464 max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); 1465 min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); 1466 1467 if (min_limit_perf < min_perf) 1468 min_limit_perf = min_perf; 1469 1470 if (max_limit_perf < min_limit_perf) 1471 max_limit_perf = min_limit_perf; 1472 1473 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); 1474 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); 1475 1476 max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, 1477 cpudata->max_limit_perf); 1478 min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, 1479 cpudata->max_limit_perf); 1480 value = READ_ONCE(cpudata->cppc_req_cached); 1481 1482 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1483 min_perf = max_perf; 1484 1485 /* Initial min/max values for CPPC Performance Controls Register */ 1486 value &= ~AMD_CPPC_MIN_PERF(~0L); 1487 value |= AMD_CPPC_MIN_PERF(min_perf); 1488 1489 value &= ~AMD_CPPC_MAX_PERF(~0L); 1490 value |= AMD_CPPC_MAX_PERF(max_perf); 1491 1492 /* CPPC EPP feature require to set zero to the desire perf bit */ 1493 value &= ~AMD_CPPC_DES_PERF(~0L); 1494 value |= AMD_CPPC_DES_PERF(0); 1495 1496 cpudata->epp_policy = cpudata->policy; 1497 1498 /* Get BIOS pre-defined epp value */ 1499 epp = amd_pstate_get_epp(cpudata, value); 1500 if (epp < 0) { 1501 /** 1502 * This return value can only be negative for shared_memory 1503 * systems where EPP register read/write not supported. 1504 */ 1505 return; 1506 } 1507 1508 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1509 epp = 0; 1510 1511 /* Set initial EPP value */ 1512 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1513 value &= ~GENMASK_ULL(31, 24); 1514 value |= (u64)epp << 24; 1515 } 1516 1517 WRITE_ONCE(cpudata->cppc_req_cached, value); 1518 amd_pstate_set_epp(cpudata, epp); 1519 } 1520 1521 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 1522 { 1523 struct amd_cpudata *cpudata = policy->driver_data; 1524 1525 if (!policy->cpuinfo.max_freq) 1526 return -ENODEV; 1527 1528 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", 1529 policy->cpuinfo.max_freq, policy->max); 1530 1531 cpudata->policy = policy->policy; 1532 1533 amd_pstate_epp_update_limit(policy); 1534 1535 return 0; 1536 } 1537 1538 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) 1539 { 1540 struct cppc_perf_ctrls perf_ctrls; 1541 u64 value, max_perf; 1542 int ret; 1543 1544 ret = amd_pstate_enable(true); 1545 if (ret) 1546 pr_err("failed to enable amd pstate during resume, return %d\n", ret); 1547 1548 value = READ_ONCE(cpudata->cppc_req_cached); 1549 max_perf = READ_ONCE(cpudata->highest_perf); 1550 1551 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1552 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1553 } else { 1554 perf_ctrls.max_perf = max_perf; 1555 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); 1556 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1557 } 1558 } 1559 1560 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) 1561 { 1562 struct amd_cpudata *cpudata = policy->driver_data; 1563 1564 pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); 1565 1566 if (cppc_state == AMD_PSTATE_ACTIVE) { 1567 amd_pstate_epp_reenable(cpudata); 1568 cpudata->suspended = false; 1569 } 1570 1571 return 0; 1572 } 1573 1574 static void amd_pstate_epp_offline(struct cpufreq_policy *policy) 1575 { 1576 struct amd_cpudata *cpudata = policy->driver_data; 1577 struct cppc_perf_ctrls perf_ctrls; 1578 int min_perf; 1579 u64 value; 1580 1581 min_perf = READ_ONCE(cpudata->lowest_perf); 1582 value = READ_ONCE(cpudata->cppc_req_cached); 1583 1584 mutex_lock(&amd_pstate_limits_lock); 1585 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1586 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; 1587 1588 /* Set max perf same as min perf */ 1589 value &= ~AMD_CPPC_MAX_PERF(~0L); 1590 value |= AMD_CPPC_MAX_PERF(min_perf); 1591 value &= ~AMD_CPPC_MIN_PERF(~0L); 1592 value |= AMD_CPPC_MIN_PERF(min_perf); 1593 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1594 } else { 1595 perf_ctrls.desired_perf = 0; 1596 perf_ctrls.max_perf = min_perf; 1597 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); 1598 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1599 } 1600 mutex_unlock(&amd_pstate_limits_lock); 1601 } 1602 1603 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) 1604 { 1605 struct amd_cpudata *cpudata = policy->driver_data; 1606 1607 pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); 1608 1609 if (cpudata->suspended) 1610 return 0; 1611 1612 if (cppc_state == AMD_PSTATE_ACTIVE) 1613 amd_pstate_epp_offline(policy); 1614 1615 return 0; 1616 } 1617 1618 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) 1619 { 1620 cpufreq_verify_within_cpu_limits(policy); 1621 pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); 1622 return 0; 1623 } 1624 1625 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) 1626 { 1627 struct amd_cpudata *cpudata = policy->driver_data; 1628 int ret; 1629 1630 /* avoid suspending when EPP is not enabled */ 1631 if (cppc_state != AMD_PSTATE_ACTIVE) 1632 return 0; 1633 1634 /* set this flag to avoid setting core offline*/ 1635 cpudata->suspended = true; 1636 1637 /* disable CPPC in lowlevel firmware */ 1638 ret = amd_pstate_enable(false); 1639 if (ret) 1640 pr_err("failed to suspend, return %d\n", ret); 1641 1642 return 0; 1643 } 1644 1645 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 1646 { 1647 struct amd_cpudata *cpudata = policy->driver_data; 1648 1649 if (cpudata->suspended) { 1650 mutex_lock(&amd_pstate_limits_lock); 1651 1652 /* enable amd pstate from suspend state*/ 1653 amd_pstate_epp_reenable(cpudata); 1654 1655 mutex_unlock(&amd_pstate_limits_lock); 1656 1657 cpudata->suspended = false; 1658 } 1659 1660 return 0; 1661 } 1662 1663 static struct cpufreq_driver amd_pstate_driver = { 1664 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 1665 .verify = amd_pstate_verify, 1666 .target = amd_pstate_target, 1667 .fast_switch = amd_pstate_fast_switch, 1668 .init = amd_pstate_cpu_init, 1669 .exit = amd_pstate_cpu_exit, 1670 .suspend = amd_pstate_cpu_suspend, 1671 .resume = amd_pstate_cpu_resume, 1672 .set_boost = amd_pstate_set_boost, 1673 .update_limits = amd_pstate_update_limits, 1674 .name = "amd-pstate", 1675 .attr = amd_pstate_attr, 1676 }; 1677 1678 static struct cpufreq_driver amd_pstate_epp_driver = { 1679 .flags = CPUFREQ_CONST_LOOPS, 1680 .verify = amd_pstate_epp_verify_policy, 1681 .setpolicy = amd_pstate_epp_set_policy, 1682 .init = amd_pstate_epp_cpu_init, 1683 .exit = amd_pstate_epp_cpu_exit, 1684 .offline = amd_pstate_epp_cpu_offline, 1685 .online = amd_pstate_epp_cpu_online, 1686 .suspend = amd_pstate_epp_suspend, 1687 .resume = amd_pstate_epp_resume, 1688 .update_limits = amd_pstate_update_limits, 1689 .name = "amd-pstate-epp", 1690 .attr = amd_pstate_epp_attr, 1691 }; 1692 1693 static int __init amd_pstate_set_driver(int mode_idx) 1694 { 1695 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1696 cppc_state = mode_idx; 1697 if (cppc_state == AMD_PSTATE_DISABLE) 1698 pr_info("driver is explicitly disabled\n"); 1699 1700 if (cppc_state == AMD_PSTATE_ACTIVE) 1701 current_pstate_driver = &amd_pstate_epp_driver; 1702 1703 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1704 current_pstate_driver = &amd_pstate_driver; 1705 1706 return 0; 1707 } 1708 1709 return -EINVAL; 1710 } 1711 1712 static int __init amd_pstate_init(void) 1713 { 1714 struct device *dev_root; 1715 int ret; 1716 1717 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 1718 return -ENODEV; 1719 1720 if (!acpi_cpc_valid()) { 1721 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 1722 return -ENODEV; 1723 } 1724 1725 /* don't keep reloading if cpufreq_driver exists */ 1726 if (cpufreq_get_current_driver()) 1727 return -EEXIST; 1728 1729 quirks = NULL; 1730 1731 /* check if this machine need CPPC quirks */ 1732 dmi_check_system(amd_pstate_quirks_table); 1733 1734 switch (cppc_state) { 1735 case AMD_PSTATE_UNDEFINED: 1736 /* Disable on the following configs by default: 1737 * 1. Undefined platforms 1738 * 2. Server platforms 1739 * 3. Shared memory designs 1740 */ 1741 if (amd_pstate_acpi_pm_profile_undefined() || 1742 amd_pstate_acpi_pm_profile_server() || 1743 !boot_cpu_has(X86_FEATURE_CPPC)) { 1744 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 1745 return -ENODEV; 1746 } 1747 ret = amd_pstate_set_driver(CONFIG_X86_AMD_PSTATE_DEFAULT_MODE); 1748 if (ret) 1749 return ret; 1750 break; 1751 case AMD_PSTATE_DISABLE: 1752 return -ENODEV; 1753 case AMD_PSTATE_PASSIVE: 1754 case AMD_PSTATE_ACTIVE: 1755 case AMD_PSTATE_GUIDED: 1756 break; 1757 default: 1758 return -EINVAL; 1759 } 1760 1761 /* capability check */ 1762 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1763 pr_debug("AMD CPPC MSR based functionality is supported\n"); 1764 if (cppc_state != AMD_PSTATE_ACTIVE) 1765 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1766 } else { 1767 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 1768 static_call_update(amd_pstate_enable, cppc_enable); 1769 static_call_update(amd_pstate_init_perf, cppc_init_perf); 1770 static_call_update(amd_pstate_update_perf, cppc_update_perf); 1771 } 1772 1773 /* enable amd pstate feature */ 1774 ret = amd_pstate_enable(true); 1775 if (ret) { 1776 pr_err("failed to enable with return %d\n", ret); 1777 return ret; 1778 } 1779 1780 ret = cpufreq_register_driver(current_pstate_driver); 1781 if (ret) 1782 pr_err("failed to register with return %d\n", ret); 1783 1784 dev_root = bus_get_dev_root(&cpu_subsys); 1785 if (dev_root) { 1786 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 1787 put_device(dev_root); 1788 if (ret) { 1789 pr_err("sysfs attribute export failed with error %d.\n", ret); 1790 goto global_attr_free; 1791 } 1792 } 1793 1794 return ret; 1795 1796 global_attr_free: 1797 cpufreq_unregister_driver(current_pstate_driver); 1798 return ret; 1799 } 1800 device_initcall(amd_pstate_init); 1801 1802 static int __init amd_pstate_param(char *str) 1803 { 1804 size_t size; 1805 int mode_idx; 1806 1807 if (!str) 1808 return -EINVAL; 1809 1810 size = strlen(str); 1811 mode_idx = get_mode_idx_from_str(str, size); 1812 1813 return amd_pstate_set_driver(mode_idx); 1814 } 1815 1816 static int __init amd_prefcore_param(char *str) 1817 { 1818 if (!strcmp(str, "disable")) 1819 amd_pstate_prefcore = false; 1820 1821 return 0; 1822 } 1823 1824 early_param("amd_pstate", amd_pstate_param); 1825 early_param("amd_prefcore", amd_prefcore_param); 1826 1827 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 1828 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 1829