1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/bitfield.h> 26 #include <linux/kernel.h> 27 #include <linux/module.h> 28 #include <linux/init.h> 29 #include <linux/smp.h> 30 #include <linux/sched.h> 31 #include <linux/cpufreq.h> 32 #include <linux/compiler.h> 33 #include <linux/dmi.h> 34 #include <linux/slab.h> 35 #include <linux/acpi.h> 36 #include <linux/io.h> 37 #include <linux/delay.h> 38 #include <linux/uaccess.h> 39 #include <linux/static_call.h> 40 #include <linux/topology.h> 41 42 #include <acpi/processor.h> 43 #include <acpi/cppc_acpi.h> 44 45 #include <asm/msr.h> 46 #include <asm/processor.h> 47 #include <asm/cpufeature.h> 48 #include <asm/cpu_device_id.h> 49 50 #include "amd-pstate.h" 51 #include "amd-pstate-trace.h" 52 53 #define AMD_PSTATE_TRANSITION_LATENCY 20000 54 #define AMD_PSTATE_TRANSITION_DELAY 1000 55 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600 56 57 #define AMD_CPPC_EPP_PERFORMANCE 0x00 58 #define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 59 #define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF 60 #define AMD_CPPC_EPP_POWERSAVE 0xFF 61 62 static const char * const amd_pstate_mode_string[] = { 63 [AMD_PSTATE_UNDEFINED] = "undefined", 64 [AMD_PSTATE_DISABLE] = "disable", 65 [AMD_PSTATE_PASSIVE] = "passive", 66 [AMD_PSTATE_ACTIVE] = "active", 67 [AMD_PSTATE_GUIDED] = "guided", 68 NULL, 69 }; 70 71 const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode) 72 { 73 if (mode < 0 || mode >= AMD_PSTATE_MAX) 74 return NULL; 75 return amd_pstate_mode_string[mode]; 76 } 77 EXPORT_SYMBOL_GPL(amd_pstate_get_mode_string); 78 79 struct quirk_entry { 80 u32 nominal_freq; 81 u32 lowest_freq; 82 }; 83 84 static struct cpufreq_driver *current_pstate_driver; 85 static struct cpufreq_driver amd_pstate_driver; 86 static struct cpufreq_driver amd_pstate_epp_driver; 87 static int cppc_state = AMD_PSTATE_UNDEFINED; 88 static bool amd_pstate_prefcore = true; 89 static struct quirk_entry *quirks; 90 91 /* 92 * AMD Energy Preference Performance (EPP) 93 * The EPP is used in the CCLK DPM controller to drive 94 * the frequency that a core is going to operate during 95 * short periods of activity. EPP values will be utilized for 96 * different OS profiles (balanced, performance, power savings) 97 * display strings corresponding to EPP index in the 98 * energy_perf_strings[] 99 * index String 100 *------------------------------------- 101 * 0 default 102 * 1 performance 103 * 2 balance_performance 104 * 3 balance_power 105 * 4 power 106 */ 107 enum energy_perf_value_index { 108 EPP_INDEX_DEFAULT = 0, 109 EPP_INDEX_PERFORMANCE, 110 EPP_INDEX_BALANCE_PERFORMANCE, 111 EPP_INDEX_BALANCE_POWERSAVE, 112 EPP_INDEX_POWERSAVE, 113 }; 114 115 static const char * const energy_perf_strings[] = { 116 [EPP_INDEX_DEFAULT] = "default", 117 [EPP_INDEX_PERFORMANCE] = "performance", 118 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 119 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 120 [EPP_INDEX_POWERSAVE] = "power", 121 NULL 122 }; 123 124 static unsigned int epp_values[] = { 125 [EPP_INDEX_DEFAULT] = 0, 126 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 127 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 128 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 129 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 130 }; 131 132 typedef int (*cppc_mode_transition_fn)(int); 133 134 static struct quirk_entry quirk_amd_7k62 = { 135 .nominal_freq = 2600, 136 .lowest_freq = 550, 137 }; 138 139 static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) 140 { 141 u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); 142 143 return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf); 144 } 145 146 static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val) 147 { 148 return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val, 149 perf.nominal_perf); 150 } 151 152 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) 153 { 154 /** 155 * match the broken bios for family 17h processor support CPPC V2 156 * broken BIOS lack of nominal_freq and lowest_freq capabilities 157 * definition in ACPI tables 158 */ 159 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 160 quirks = dmi->driver_data; 161 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); 162 return 1; 163 } 164 165 return 0; 166 } 167 168 static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { 169 { 170 .callback = dmi_matched_7k62_bios_bug, 171 .ident = "AMD EPYC 7K62", 172 .matches = { 173 DMI_MATCH(DMI_BIOS_VERSION, "5.14"), 174 DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), 175 }, 176 .driver_data = &quirk_amd_7k62, 177 }, 178 {} 179 }; 180 MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); 181 182 static inline int get_mode_idx_from_str(const char *str, size_t size) 183 { 184 int i; 185 186 for (i=0; i < AMD_PSTATE_MAX; i++) { 187 if (!strncmp(str, amd_pstate_mode_string[i], size)) 188 return i; 189 } 190 return -EINVAL; 191 } 192 193 static DEFINE_MUTEX(amd_pstate_driver_lock); 194 195 static u8 msr_get_epp(struct amd_cpudata *cpudata) 196 { 197 u64 value; 198 int ret; 199 200 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 201 if (ret < 0) { 202 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 203 return ret; 204 } 205 206 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value); 207 } 208 209 DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); 210 211 static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) 212 { 213 return static_call(amd_pstate_get_epp)(cpudata); 214 } 215 216 static u8 shmem_get_epp(struct amd_cpudata *cpudata) 217 { 218 u64 epp; 219 int ret; 220 221 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 222 if (ret < 0) { 223 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 224 return ret; 225 } 226 227 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp); 228 } 229 230 static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, 231 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 232 { 233 struct amd_cpudata *cpudata = policy->driver_data; 234 u64 value, prev; 235 236 value = prev = READ_ONCE(cpudata->cppc_req_cached); 237 238 value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | 239 AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); 240 value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); 241 value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); 242 value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); 243 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 244 245 if (trace_amd_pstate_epp_perf_enabled()) { 246 union perf_cached perf = READ_ONCE(cpudata->perf); 247 248 trace_amd_pstate_epp_perf(cpudata->cpu, 249 perf.highest_perf, 250 epp, 251 min_perf, 252 max_perf, 253 policy->boost_enabled, 254 value != prev); 255 } 256 257 if (value == prev) 258 return 0; 259 260 if (fast_switch) { 261 wrmsrl(MSR_AMD_CPPC_REQ, value); 262 return 0; 263 } else { 264 int ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 265 266 if (ret) 267 return ret; 268 } 269 270 WRITE_ONCE(cpudata->cppc_req_cached, value); 271 272 return 0; 273 } 274 275 DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); 276 277 static inline int amd_pstate_update_perf(struct cpufreq_policy *policy, 278 u8 min_perf, u8 des_perf, 279 u8 max_perf, u8 epp, 280 bool fast_switch) 281 { 282 return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf, 283 max_perf, epp, fast_switch); 284 } 285 286 static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) 287 { 288 struct amd_cpudata *cpudata = policy->driver_data; 289 u64 value, prev; 290 int ret; 291 292 value = prev = READ_ONCE(cpudata->cppc_req_cached); 293 value &= ~AMD_CPPC_EPP_PERF_MASK; 294 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 295 296 if (trace_amd_pstate_epp_perf_enabled()) { 297 union perf_cached perf = cpudata->perf; 298 299 trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 300 epp, 301 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 302 cpudata->cppc_req_cached), 303 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 304 cpudata->cppc_req_cached), 305 policy->boost_enabled, 306 value != prev); 307 } 308 309 if (value == prev) 310 return 0; 311 312 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 313 if (ret) { 314 pr_err("failed to set energy perf value (%d)\n", ret); 315 return ret; 316 } 317 318 /* update both so that msr_update_perf() can effectively check */ 319 WRITE_ONCE(cpudata->cppc_req_cached, value); 320 321 return ret; 322 } 323 324 DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); 325 326 static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp) 327 { 328 return static_call(amd_pstate_set_epp)(policy, epp); 329 } 330 331 static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) 332 { 333 struct amd_cpudata *cpudata = policy->driver_data; 334 struct cppc_perf_ctrls perf_ctrls; 335 u8 epp_cached; 336 u64 value; 337 int ret; 338 339 340 epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 341 if (trace_amd_pstate_epp_perf_enabled()) { 342 union perf_cached perf = cpudata->perf; 343 344 trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 345 epp, 346 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 347 cpudata->cppc_req_cached), 348 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 349 cpudata->cppc_req_cached), 350 policy->boost_enabled, 351 epp != epp_cached); 352 } 353 354 if (epp == epp_cached) 355 return 0; 356 357 perf_ctrls.energy_perf = epp; 358 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 359 if (ret) { 360 pr_debug("failed to set energy perf value (%d)\n", ret); 361 return ret; 362 } 363 364 value = READ_ONCE(cpudata->cppc_req_cached); 365 value &= ~AMD_CPPC_EPP_PERF_MASK; 366 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 367 WRITE_ONCE(cpudata->cppc_req_cached, value); 368 369 return ret; 370 } 371 372 static inline int msr_cppc_enable(struct cpufreq_policy *policy) 373 { 374 return wrmsrl_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); 375 } 376 377 static int shmem_cppc_enable(struct cpufreq_policy *policy) 378 { 379 return cppc_set_enable(policy->cpu, 1); 380 } 381 382 DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); 383 384 static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) 385 { 386 return static_call(amd_pstate_cppc_enable)(policy); 387 } 388 389 static int msr_init_perf(struct amd_cpudata *cpudata) 390 { 391 union perf_cached perf = READ_ONCE(cpudata->perf); 392 u64 cap1, numerator; 393 394 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 395 &cap1); 396 if (ret) 397 return ret; 398 399 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 400 if (ret) 401 return ret; 402 403 perf.highest_perf = numerator; 404 perf.max_limit_perf = numerator; 405 perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 406 perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); 407 perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); 408 perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 409 WRITE_ONCE(cpudata->perf, perf); 410 WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1)); 411 412 return 0; 413 } 414 415 static int shmem_init_perf(struct amd_cpudata *cpudata) 416 { 417 struct cppc_perf_caps cppc_perf; 418 union perf_cached perf = READ_ONCE(cpudata->perf); 419 u64 numerator; 420 421 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 422 if (ret) 423 return ret; 424 425 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 426 if (ret) 427 return ret; 428 429 perf.highest_perf = numerator; 430 perf.max_limit_perf = numerator; 431 perf.min_limit_perf = cppc_perf.lowest_perf; 432 perf.nominal_perf = cppc_perf.nominal_perf; 433 perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 434 perf.lowest_perf = cppc_perf.lowest_perf; 435 WRITE_ONCE(cpudata->perf, perf); 436 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); 437 438 if (cppc_state == AMD_PSTATE_ACTIVE) 439 return 0; 440 441 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); 442 if (ret) { 443 pr_warn("failed to get auto_sel, ret: %d\n", ret); 444 return 0; 445 } 446 447 ret = cppc_set_auto_sel(cpudata->cpu, 448 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 449 450 if (ret) 451 pr_warn("failed to set auto_sel, ret: %d\n", ret); 452 453 return ret; 454 } 455 456 DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf); 457 458 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 459 { 460 return static_call(amd_pstate_init_perf)(cpudata); 461 } 462 463 static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf, 464 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 465 { 466 struct amd_cpudata *cpudata = policy->driver_data; 467 struct cppc_perf_ctrls perf_ctrls; 468 u64 value, prev; 469 int ret; 470 471 if (cppc_state == AMD_PSTATE_ACTIVE) { 472 int ret = shmem_set_epp(policy, epp); 473 474 if (ret) 475 return ret; 476 } 477 478 value = prev = READ_ONCE(cpudata->cppc_req_cached); 479 480 value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | 481 AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); 482 value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); 483 value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); 484 value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); 485 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 486 487 if (trace_amd_pstate_epp_perf_enabled()) { 488 union perf_cached perf = READ_ONCE(cpudata->perf); 489 490 trace_amd_pstate_epp_perf(cpudata->cpu, 491 perf.highest_perf, 492 epp, 493 min_perf, 494 max_perf, 495 policy->boost_enabled, 496 value != prev); 497 } 498 499 if (value == prev) 500 return 0; 501 502 perf_ctrls.max_perf = max_perf; 503 perf_ctrls.min_perf = min_perf; 504 perf_ctrls.desired_perf = des_perf; 505 506 ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); 507 if (ret) 508 return ret; 509 510 WRITE_ONCE(cpudata->cppc_req_cached, value); 511 512 return 0; 513 } 514 515 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 516 { 517 u64 aperf, mperf, tsc; 518 unsigned long flags; 519 520 local_irq_save(flags); 521 rdmsrl(MSR_IA32_APERF, aperf); 522 rdmsrl(MSR_IA32_MPERF, mperf); 523 tsc = rdtsc(); 524 525 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 526 local_irq_restore(flags); 527 return false; 528 } 529 530 local_irq_restore(flags); 531 532 cpudata->cur.aperf = aperf; 533 cpudata->cur.mperf = mperf; 534 cpudata->cur.tsc = tsc; 535 cpudata->cur.aperf -= cpudata->prev.aperf; 536 cpudata->cur.mperf -= cpudata->prev.mperf; 537 cpudata->cur.tsc -= cpudata->prev.tsc; 538 539 cpudata->prev.aperf = aperf; 540 cpudata->prev.mperf = mperf; 541 cpudata->prev.tsc = tsc; 542 543 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 544 545 return true; 546 } 547 548 static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, 549 u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) 550 { 551 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 552 union perf_cached perf = READ_ONCE(cpudata->perf); 553 554 if (!policy) 555 return; 556 557 des_perf = clamp_t(u8, des_perf, min_perf, max_perf); 558 559 policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); 560 561 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 562 min_perf = des_perf; 563 des_perf = 0; 564 } 565 566 /* limit the max perf when core performance boost feature is disabled */ 567 if (!cpudata->boost_supported) 568 max_perf = min_t(u8, perf.nominal_perf, max_perf); 569 570 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 571 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 572 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 573 cpudata->cpu, fast_switch); 574 } 575 576 amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch); 577 } 578 579 static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) 580 { 581 /* 582 * Initialize lower frequency limit (i.e.policy->min) with 583 * lowest_nonlinear_frequency which is the most energy efficient 584 * frequency. Override the initial value set by cpufreq core and 585 * amd-pstate qos_requests. 586 */ 587 if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { 588 struct cpufreq_policy *policy __free(put_cpufreq_policy) = 589 cpufreq_cpu_get(policy_data->cpu); 590 struct amd_cpudata *cpudata; 591 592 if (!policy) 593 return -EINVAL; 594 595 cpudata = policy->driver_data; 596 policy_data->min = cpudata->lowest_nonlinear_freq; 597 } 598 599 cpufreq_verify_within_cpu_limits(policy_data); 600 601 return 0; 602 } 603 604 static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) 605 { 606 struct amd_cpudata *cpudata = policy->driver_data; 607 union perf_cached perf = READ_ONCE(cpudata->perf); 608 609 perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); 610 perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); 611 612 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 613 perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); 614 615 WRITE_ONCE(cpudata->max_limit_freq, policy->max); 616 WRITE_ONCE(cpudata->min_limit_freq, policy->min); 617 WRITE_ONCE(cpudata->perf, perf); 618 } 619 620 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 621 unsigned int target_freq, bool fast_switch) 622 { 623 struct cpufreq_freqs freqs; 624 struct amd_cpudata *cpudata; 625 union perf_cached perf; 626 u8 des_perf; 627 628 cpudata = policy->driver_data; 629 630 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 631 amd_pstate_update_min_max_limit(policy); 632 633 perf = READ_ONCE(cpudata->perf); 634 635 freqs.old = policy->cur; 636 freqs.new = target_freq; 637 638 des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq); 639 640 WARN_ON(fast_switch && !policy->fast_switch_enabled); 641 /* 642 * If fast_switch is desired, then there aren't any registered 643 * transition notifiers. See comment for 644 * cpufreq_enable_fast_switch(). 645 */ 646 if (!fast_switch) 647 cpufreq_freq_transition_begin(policy, &freqs); 648 649 amd_pstate_update(cpudata, perf.min_limit_perf, des_perf, 650 perf.max_limit_perf, fast_switch, 651 policy->governor->flags); 652 653 if (!fast_switch) 654 cpufreq_freq_transition_end(policy, &freqs, false); 655 656 return 0; 657 } 658 659 static int amd_pstate_target(struct cpufreq_policy *policy, 660 unsigned int target_freq, 661 unsigned int relation) 662 { 663 return amd_pstate_update_freq(policy, target_freq, false); 664 } 665 666 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 667 unsigned int target_freq) 668 { 669 if (!amd_pstate_update_freq(policy, target_freq, true)) 670 return target_freq; 671 return policy->cur; 672 } 673 674 static void amd_pstate_adjust_perf(unsigned int cpu, 675 unsigned long _min_perf, 676 unsigned long target_perf, 677 unsigned long capacity) 678 { 679 u8 max_perf, min_perf, des_perf, cap_perf; 680 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 681 struct amd_cpudata *cpudata; 682 union perf_cached perf; 683 684 if (!policy) 685 return; 686 687 cpudata = policy->driver_data; 688 689 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 690 amd_pstate_update_min_max_limit(policy); 691 692 perf = READ_ONCE(cpudata->perf); 693 cap_perf = perf.highest_perf; 694 695 des_perf = cap_perf; 696 if (target_perf < capacity) 697 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 698 699 if (_min_perf < capacity) 700 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 701 else 702 min_perf = cap_perf; 703 704 if (min_perf < perf.min_limit_perf) 705 min_perf = perf.min_limit_perf; 706 707 max_perf = perf.max_limit_perf; 708 if (max_perf < min_perf) 709 max_perf = min_perf; 710 711 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, 712 policy->governor->flags); 713 } 714 715 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) 716 { 717 struct amd_cpudata *cpudata = policy->driver_data; 718 union perf_cached perf = READ_ONCE(cpudata->perf); 719 u32 nominal_freq, max_freq; 720 int ret = 0; 721 722 nominal_freq = READ_ONCE(cpudata->nominal_freq); 723 max_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf); 724 725 if (on) 726 policy->cpuinfo.max_freq = max_freq; 727 else if (policy->cpuinfo.max_freq > nominal_freq) 728 policy->cpuinfo.max_freq = nominal_freq; 729 730 policy->max = policy->cpuinfo.max_freq; 731 732 if (cppc_state == AMD_PSTATE_PASSIVE) { 733 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); 734 if (ret < 0) 735 pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu); 736 } 737 738 return ret < 0 ? ret : 0; 739 } 740 741 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 742 { 743 struct amd_cpudata *cpudata = policy->driver_data; 744 int ret; 745 746 if (!cpudata->boost_supported) { 747 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 748 return -EOPNOTSUPP; 749 } 750 751 ret = amd_pstate_cpu_boost_update(policy, state); 752 refresh_frequency_limits(policy); 753 754 return ret; 755 } 756 757 static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) 758 { 759 u64 boost_val; 760 int ret = -1; 761 762 /* 763 * If platform has no CPB support or disable it, initialize current driver 764 * boost_enabled state to be false, it is not an error for cpufreq core to handle. 765 */ 766 if (!cpu_feature_enabled(X86_FEATURE_CPB)) { 767 pr_debug_once("Boost CPB capabilities not present in the processor\n"); 768 ret = 0; 769 goto exit_err; 770 } 771 772 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); 773 if (ret) { 774 pr_err_once("failed to read initial CPU boost state!\n"); 775 ret = -EIO; 776 goto exit_err; 777 } 778 779 if (!(boost_val & MSR_K7_HWCR_CPB_DIS)) 780 cpudata->boost_supported = true; 781 782 return 0; 783 784 exit_err: 785 cpudata->boost_supported = false; 786 return ret; 787 } 788 789 static void amd_perf_ctl_reset(unsigned int cpu) 790 { 791 wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 792 } 793 794 /* 795 * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks 796 * due to locking, so queue the work for later. 797 */ 798 static void amd_pstste_sched_prefcore_workfn(struct work_struct *work) 799 { 800 sched_set_itmt_support(); 801 } 802 static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn); 803 804 #define CPPC_MAX_PERF U8_MAX 805 806 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) 807 { 808 /* user disabled or not detected */ 809 if (!amd_pstate_prefcore) 810 return; 811 812 cpudata->hw_prefcore = true; 813 814 /* 815 * The priorities can be set regardless of whether or not 816 * sched_set_itmt_support(true) has been called and it is valid to 817 * update them at any time after it has been called. 818 */ 819 sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); 820 821 schedule_work(&sched_prefcore_work); 822 } 823 824 static void amd_pstate_update_limits(unsigned int cpu) 825 { 826 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 827 struct amd_cpudata *cpudata; 828 u32 prev_high = 0, cur_high = 0; 829 bool highest_perf_changed = false; 830 831 if (!amd_pstate_prefcore) 832 return; 833 834 if (!policy) 835 return; 836 837 if (amd_get_highest_perf(cpu, &cur_high)) 838 return; 839 840 cpudata = policy->driver_data; 841 842 prev_high = READ_ONCE(cpudata->prefcore_ranking); 843 highest_perf_changed = (prev_high != cur_high); 844 if (highest_perf_changed) { 845 WRITE_ONCE(cpudata->prefcore_ranking, cur_high); 846 847 if (cur_high < CPPC_MAX_PERF) 848 sched_set_itmt_core_prio((int)cur_high, cpu); 849 } 850 } 851 852 /* 853 * Get pstate transition delay time from ACPI tables that firmware set 854 * instead of using hardcode value directly. 855 */ 856 static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) 857 { 858 u32 transition_delay_ns; 859 860 transition_delay_ns = cppc_get_transition_latency(cpu); 861 if (transition_delay_ns == CPUFREQ_ETERNAL) { 862 if (cpu_feature_enabled(X86_FEATURE_AMD_FAST_CPPC)) 863 return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY; 864 else 865 return AMD_PSTATE_TRANSITION_DELAY; 866 } 867 868 return transition_delay_ns / NSEC_PER_USEC; 869 } 870 871 /* 872 * Get pstate transition latency value from ACPI tables that firmware 873 * set instead of using hardcode value directly. 874 */ 875 static u32 amd_pstate_get_transition_latency(unsigned int cpu) 876 { 877 u32 transition_latency; 878 879 transition_latency = cppc_get_transition_latency(cpu); 880 if (transition_latency == CPUFREQ_ETERNAL) 881 return AMD_PSTATE_TRANSITION_LATENCY; 882 883 return transition_latency; 884 } 885 886 /* 887 * amd_pstate_init_freq: Initialize the nominal_freq and lowest_nonlinear_freq 888 * for the @cpudata object. 889 * 890 * Requires: all perf members of @cpudata to be initialized. 891 * 892 * Returns 0 on success, non-zero value on failure. 893 */ 894 static int amd_pstate_init_freq(struct amd_cpudata *cpudata) 895 { 896 u32 min_freq, max_freq, nominal_freq, lowest_nonlinear_freq; 897 struct cppc_perf_caps cppc_perf; 898 union perf_cached perf; 899 int ret; 900 901 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 902 if (ret) 903 return ret; 904 perf = READ_ONCE(cpudata->perf); 905 906 if (quirks && quirks->nominal_freq) 907 nominal_freq = quirks->nominal_freq; 908 else 909 nominal_freq = cppc_perf.nominal_freq; 910 nominal_freq *= 1000; 911 912 if (quirks && quirks->lowest_freq) { 913 min_freq = quirks->lowest_freq; 914 perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq); 915 WRITE_ONCE(cpudata->perf, perf); 916 } else 917 min_freq = cppc_perf.lowest_freq; 918 919 min_freq *= 1000; 920 921 WRITE_ONCE(cpudata->nominal_freq, nominal_freq); 922 923 max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); 924 lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); 925 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); 926 927 /** 928 * Below values need to be initialized correctly, otherwise driver will fail to load 929 * max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf 930 * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] 931 * Check _CPC in ACPI table objects if any values are incorrect 932 */ 933 if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { 934 pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", 935 min_freq, max_freq, nominal_freq); 936 return -EINVAL; 937 } 938 939 if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq) { 940 pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", 941 lowest_nonlinear_freq, min_freq, nominal_freq); 942 return -EINVAL; 943 } 944 945 return 0; 946 } 947 948 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 949 { 950 struct amd_cpudata *cpudata; 951 union perf_cached perf; 952 struct device *dev; 953 int ret; 954 955 /* 956 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 957 * which is ideal for initialization process. 958 */ 959 amd_perf_ctl_reset(policy->cpu); 960 dev = get_cpu_device(policy->cpu); 961 if (!dev) 962 return -ENODEV; 963 964 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 965 if (!cpudata) 966 return -ENOMEM; 967 968 cpudata->cpu = policy->cpu; 969 970 ret = amd_pstate_init_perf(cpudata); 971 if (ret) 972 goto free_cpudata1; 973 974 amd_pstate_init_prefcore(cpudata); 975 976 ret = amd_pstate_init_freq(cpudata); 977 if (ret) 978 goto free_cpudata1; 979 980 ret = amd_pstate_init_boost_support(cpudata); 981 if (ret) 982 goto free_cpudata1; 983 984 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); 985 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); 986 987 perf = READ_ONCE(cpudata->perf); 988 989 policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, 990 cpudata->nominal_freq, 991 perf.lowest_perf); 992 policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, 993 cpudata->nominal_freq, 994 perf.highest_perf); 995 996 ret = amd_pstate_cppc_enable(policy); 997 if (ret) 998 goto free_cpudata1; 999 1000 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1001 1002 /* It will be updated by governor */ 1003 policy->cur = policy->cpuinfo.min_freq; 1004 1005 if (cpu_feature_enabled(X86_FEATURE_CPPC)) 1006 policy->fast_switch_possible = true; 1007 1008 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 1009 FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); 1010 if (ret < 0) { 1011 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 1012 goto free_cpudata1; 1013 } 1014 1015 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 1016 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 1017 if (ret < 0) { 1018 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 1019 goto free_cpudata2; 1020 } 1021 1022 policy->driver_data = cpudata; 1023 1024 if (!current_pstate_driver->adjust_perf) 1025 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1026 1027 return 0; 1028 1029 free_cpudata2: 1030 freq_qos_remove_request(&cpudata->req[0]); 1031 free_cpudata1: 1032 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1033 kfree(cpudata); 1034 return ret; 1035 } 1036 1037 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) 1038 { 1039 struct amd_cpudata *cpudata = policy->driver_data; 1040 1041 freq_qos_remove_request(&cpudata->req[1]); 1042 freq_qos_remove_request(&cpudata->req[0]); 1043 policy->fast_switch_possible = false; 1044 kfree(cpudata); 1045 } 1046 1047 /* Sysfs attributes */ 1048 1049 /* 1050 * This frequency is to indicate the maximum hardware frequency. 1051 * If boost is not active but supported, the frequency will be larger than the 1052 * one in cpuinfo. 1053 */ 1054 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 1055 char *buf) 1056 { 1057 struct amd_cpudata *cpudata; 1058 union perf_cached perf; 1059 1060 cpudata = policy->driver_data; 1061 perf = READ_ONCE(cpudata->perf); 1062 1063 return sysfs_emit(buf, "%u\n", 1064 perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf)); 1065 } 1066 1067 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 1068 char *buf) 1069 { 1070 struct amd_cpudata *cpudata; 1071 union perf_cached perf; 1072 1073 cpudata = policy->driver_data; 1074 perf = READ_ONCE(cpudata->perf); 1075 1076 return sysfs_emit(buf, "%u\n", 1077 perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf)); 1078 } 1079 1080 /* 1081 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 1082 * need to expose it to sysfs. 1083 */ 1084 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 1085 char *buf) 1086 { 1087 struct amd_cpudata *cpudata; 1088 1089 cpudata = policy->driver_data; 1090 1091 return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf); 1092 } 1093 1094 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, 1095 char *buf) 1096 { 1097 u8 perf; 1098 struct amd_cpudata *cpudata = policy->driver_data; 1099 1100 perf = READ_ONCE(cpudata->prefcore_ranking); 1101 1102 return sysfs_emit(buf, "%u\n", perf); 1103 } 1104 1105 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, 1106 char *buf) 1107 { 1108 bool hw_prefcore; 1109 struct amd_cpudata *cpudata = policy->driver_data; 1110 1111 hw_prefcore = READ_ONCE(cpudata->hw_prefcore); 1112 1113 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); 1114 } 1115 1116 static ssize_t show_energy_performance_available_preferences( 1117 struct cpufreq_policy *policy, char *buf) 1118 { 1119 int i = 0; 1120 int offset = 0; 1121 struct amd_cpudata *cpudata = policy->driver_data; 1122 1123 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1124 return sysfs_emit_at(buf, offset, "%s\n", 1125 energy_perf_strings[EPP_INDEX_PERFORMANCE]); 1126 1127 while (energy_perf_strings[i] != NULL) 1128 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); 1129 1130 offset += sysfs_emit_at(buf, offset, "\n"); 1131 1132 return offset; 1133 } 1134 1135 static ssize_t store_energy_performance_preference( 1136 struct cpufreq_policy *policy, const char *buf, size_t count) 1137 { 1138 struct amd_cpudata *cpudata = policy->driver_data; 1139 char str_preference[21]; 1140 ssize_t ret; 1141 u8 epp; 1142 1143 ret = sscanf(buf, "%20s", str_preference); 1144 if (ret != 1) 1145 return -EINVAL; 1146 1147 ret = match_string(energy_perf_strings, -1, str_preference); 1148 if (ret < 0) 1149 return -EINVAL; 1150 1151 if (!ret) 1152 epp = cpudata->epp_default; 1153 else 1154 epp = epp_values[ret]; 1155 1156 if (epp > 0 && policy->policy == CPUFREQ_POLICY_PERFORMANCE) { 1157 pr_debug("EPP cannot be set under performance policy\n"); 1158 return -EBUSY; 1159 } 1160 1161 ret = amd_pstate_set_epp(policy, epp); 1162 1163 return ret ? ret : count; 1164 } 1165 1166 static ssize_t show_energy_performance_preference( 1167 struct cpufreq_policy *policy, char *buf) 1168 { 1169 struct amd_cpudata *cpudata = policy->driver_data; 1170 u8 preference, epp; 1171 1172 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 1173 1174 switch (epp) { 1175 case AMD_CPPC_EPP_PERFORMANCE: 1176 preference = EPP_INDEX_PERFORMANCE; 1177 break; 1178 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 1179 preference = EPP_INDEX_BALANCE_PERFORMANCE; 1180 break; 1181 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 1182 preference = EPP_INDEX_BALANCE_POWERSAVE; 1183 break; 1184 case AMD_CPPC_EPP_POWERSAVE: 1185 preference = EPP_INDEX_POWERSAVE; 1186 break; 1187 default: 1188 return -EINVAL; 1189 } 1190 1191 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 1192 } 1193 1194 static void amd_pstate_driver_cleanup(void) 1195 { 1196 cppc_state = AMD_PSTATE_DISABLE; 1197 current_pstate_driver = NULL; 1198 } 1199 1200 static int amd_pstate_set_driver(int mode_idx) 1201 { 1202 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1203 cppc_state = mode_idx; 1204 if (cppc_state == AMD_PSTATE_DISABLE) 1205 pr_info("driver is explicitly disabled\n"); 1206 1207 if (cppc_state == AMD_PSTATE_ACTIVE) 1208 current_pstate_driver = &amd_pstate_epp_driver; 1209 1210 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1211 current_pstate_driver = &amd_pstate_driver; 1212 1213 return 0; 1214 } 1215 1216 return -EINVAL; 1217 } 1218 1219 static int amd_pstate_register_driver(int mode) 1220 { 1221 int ret; 1222 1223 ret = amd_pstate_set_driver(mode); 1224 if (ret) 1225 return ret; 1226 1227 cppc_state = mode; 1228 1229 /* at least one CPU supports CPB */ 1230 current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); 1231 1232 ret = cpufreq_register_driver(current_pstate_driver); 1233 if (ret) { 1234 amd_pstate_driver_cleanup(); 1235 return ret; 1236 } 1237 1238 return 0; 1239 } 1240 1241 static int amd_pstate_unregister_driver(int dummy) 1242 { 1243 cpufreq_unregister_driver(current_pstate_driver); 1244 amd_pstate_driver_cleanup(); 1245 return 0; 1246 } 1247 1248 static int amd_pstate_change_mode_without_dvr_change(int mode) 1249 { 1250 int cpu = 0; 1251 1252 cppc_state = mode; 1253 1254 if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 1255 return 0; 1256 1257 for_each_present_cpu(cpu) { 1258 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 1259 } 1260 1261 return 0; 1262 } 1263 1264 static int amd_pstate_change_driver_mode(int mode) 1265 { 1266 int ret; 1267 1268 ret = amd_pstate_unregister_driver(0); 1269 if (ret) 1270 return ret; 1271 1272 ret = amd_pstate_register_driver(mode); 1273 if (ret) 1274 return ret; 1275 1276 return 0; 1277 } 1278 1279 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 1280 [AMD_PSTATE_DISABLE] = { 1281 [AMD_PSTATE_DISABLE] = NULL, 1282 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 1283 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 1284 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 1285 }, 1286 [AMD_PSTATE_PASSIVE] = { 1287 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1288 [AMD_PSTATE_PASSIVE] = NULL, 1289 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1290 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 1291 }, 1292 [AMD_PSTATE_ACTIVE] = { 1293 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1294 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 1295 [AMD_PSTATE_ACTIVE] = NULL, 1296 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 1297 }, 1298 [AMD_PSTATE_GUIDED] = { 1299 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1300 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 1301 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1302 [AMD_PSTATE_GUIDED] = NULL, 1303 }, 1304 }; 1305 1306 static ssize_t amd_pstate_show_status(char *buf) 1307 { 1308 if (!current_pstate_driver) 1309 return sysfs_emit(buf, "disable\n"); 1310 1311 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 1312 } 1313 1314 int amd_pstate_update_status(const char *buf, size_t size) 1315 { 1316 int mode_idx; 1317 1318 if (size > strlen("passive") || size < strlen("active")) 1319 return -EINVAL; 1320 1321 mode_idx = get_mode_idx_from_str(buf, size); 1322 1323 if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) 1324 return -EINVAL; 1325 1326 if (mode_state_machine[cppc_state][mode_idx]) { 1327 guard(mutex)(&amd_pstate_driver_lock); 1328 return mode_state_machine[cppc_state][mode_idx](mode_idx); 1329 } 1330 1331 return 0; 1332 } 1333 EXPORT_SYMBOL_GPL(amd_pstate_update_status); 1334 1335 static ssize_t status_show(struct device *dev, 1336 struct device_attribute *attr, char *buf) 1337 { 1338 1339 guard(mutex)(&amd_pstate_driver_lock); 1340 1341 return amd_pstate_show_status(buf); 1342 } 1343 1344 static ssize_t status_store(struct device *a, struct device_attribute *b, 1345 const char *buf, size_t count) 1346 { 1347 char *p = memchr(buf, '\n', count); 1348 int ret; 1349 1350 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1351 1352 return ret < 0 ? ret : count; 1353 } 1354 1355 static ssize_t prefcore_show(struct device *dev, 1356 struct device_attribute *attr, char *buf) 1357 { 1358 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); 1359 } 1360 1361 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1362 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1363 1364 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1365 cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); 1366 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); 1367 cpufreq_freq_attr_rw(energy_performance_preference); 1368 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1369 static DEVICE_ATTR_RW(status); 1370 static DEVICE_ATTR_RO(prefcore); 1371 1372 static struct freq_attr *amd_pstate_attr[] = { 1373 &amd_pstate_max_freq, 1374 &amd_pstate_lowest_nonlinear_freq, 1375 &amd_pstate_highest_perf, 1376 &amd_pstate_prefcore_ranking, 1377 &amd_pstate_hw_prefcore, 1378 NULL, 1379 }; 1380 1381 static struct freq_attr *amd_pstate_epp_attr[] = { 1382 &amd_pstate_max_freq, 1383 &amd_pstate_lowest_nonlinear_freq, 1384 &amd_pstate_highest_perf, 1385 &amd_pstate_prefcore_ranking, 1386 &amd_pstate_hw_prefcore, 1387 &energy_performance_preference, 1388 &energy_performance_available_preferences, 1389 NULL, 1390 }; 1391 1392 static struct attribute *pstate_global_attributes[] = { 1393 &dev_attr_status.attr, 1394 &dev_attr_prefcore.attr, 1395 NULL 1396 }; 1397 1398 static const struct attribute_group amd_pstate_global_attr_group = { 1399 .name = "amd_pstate", 1400 .attrs = pstate_global_attributes, 1401 }; 1402 1403 static bool amd_pstate_acpi_pm_profile_server(void) 1404 { 1405 switch (acpi_gbl_FADT.preferred_profile) { 1406 case PM_ENTERPRISE_SERVER: 1407 case PM_SOHO_SERVER: 1408 case PM_PERFORMANCE_SERVER: 1409 return true; 1410 } 1411 return false; 1412 } 1413 1414 static bool amd_pstate_acpi_pm_profile_undefined(void) 1415 { 1416 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) 1417 return true; 1418 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) 1419 return true; 1420 return false; 1421 } 1422 1423 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1424 { 1425 struct amd_cpudata *cpudata; 1426 union perf_cached perf; 1427 struct device *dev; 1428 u64 value; 1429 int ret; 1430 1431 /* 1432 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1433 * which is ideal for initialization process. 1434 */ 1435 amd_perf_ctl_reset(policy->cpu); 1436 dev = get_cpu_device(policy->cpu); 1437 if (!dev) 1438 return -ENODEV; 1439 1440 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 1441 if (!cpudata) 1442 return -ENOMEM; 1443 1444 cpudata->cpu = policy->cpu; 1445 1446 ret = amd_pstate_init_perf(cpudata); 1447 if (ret) 1448 goto free_cpudata1; 1449 1450 amd_pstate_init_prefcore(cpudata); 1451 1452 ret = amd_pstate_init_freq(cpudata); 1453 if (ret) 1454 goto free_cpudata1; 1455 1456 ret = amd_pstate_init_boost_support(cpudata); 1457 if (ret) 1458 goto free_cpudata1; 1459 1460 perf = READ_ONCE(cpudata->perf); 1461 1462 policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, 1463 cpudata->nominal_freq, 1464 perf.lowest_perf); 1465 policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, 1466 cpudata->nominal_freq, 1467 perf.highest_perf); 1468 policy->driver_data = cpudata; 1469 1470 ret = amd_pstate_cppc_enable(policy); 1471 if (ret) 1472 goto free_cpudata1; 1473 1474 /* It will be updated by governor */ 1475 policy->cur = policy->cpuinfo.min_freq; 1476 1477 1478 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1479 1480 /* 1481 * Set the policy to provide a valid fallback value in case 1482 * the default cpufreq governor is neither powersave nor performance. 1483 */ 1484 if (amd_pstate_acpi_pm_profile_server() || 1485 amd_pstate_acpi_pm_profile_undefined()) { 1486 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 1487 cpudata->epp_default = amd_pstate_get_epp(cpudata); 1488 } else { 1489 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1490 cpudata->epp_default = AMD_CPPC_EPP_BALANCE_PERFORMANCE; 1491 } 1492 1493 if (cpu_feature_enabled(X86_FEATURE_CPPC)) { 1494 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 1495 if (ret) 1496 return ret; 1497 WRITE_ONCE(cpudata->cppc_req_cached, value); 1498 } 1499 ret = amd_pstate_set_epp(policy, cpudata->epp_default); 1500 if (ret) 1501 return ret; 1502 1503 current_pstate_driver->adjust_perf = NULL; 1504 1505 return 0; 1506 1507 free_cpudata1: 1508 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1509 kfree(cpudata); 1510 return ret; 1511 } 1512 1513 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1514 { 1515 struct amd_cpudata *cpudata = policy->driver_data; 1516 1517 if (cpudata) { 1518 kfree(cpudata); 1519 policy->driver_data = NULL; 1520 } 1521 1522 pr_debug("CPU %d exiting\n", policy->cpu); 1523 } 1524 1525 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) 1526 { 1527 struct amd_cpudata *cpudata = policy->driver_data; 1528 union perf_cached perf; 1529 u8 epp; 1530 1531 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 1532 amd_pstate_update_min_max_limit(policy); 1533 1534 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1535 epp = 0; 1536 else 1537 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 1538 1539 perf = READ_ONCE(cpudata->perf); 1540 1541 return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U, 1542 perf.max_limit_perf, epp, false); 1543 } 1544 1545 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 1546 { 1547 struct amd_cpudata *cpudata = policy->driver_data; 1548 int ret; 1549 1550 if (!policy->cpuinfo.max_freq) 1551 return -ENODEV; 1552 1553 cpudata->policy = policy->policy; 1554 1555 ret = amd_pstate_epp_update_limit(policy); 1556 if (ret) 1557 return ret; 1558 1559 /* 1560 * policy->cur is never updated with the amd_pstate_epp driver, but it 1561 * is used as a stale frequency value. So, keep it within limits. 1562 */ 1563 policy->cur = policy->min; 1564 1565 return 0; 1566 } 1567 1568 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) 1569 { 1570 pr_debug("AMD CPU Core %d going online\n", policy->cpu); 1571 1572 return amd_pstate_cppc_enable(policy); 1573 } 1574 1575 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) 1576 { 1577 return 0; 1578 } 1579 1580 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) 1581 { 1582 struct amd_cpudata *cpudata = policy->driver_data; 1583 1584 /* invalidate to ensure it's rewritten during resume */ 1585 cpudata->cppc_req_cached = 0; 1586 1587 /* set this flag to avoid setting core offline*/ 1588 cpudata->suspended = true; 1589 1590 return 0; 1591 } 1592 1593 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 1594 { 1595 struct amd_cpudata *cpudata = policy->driver_data; 1596 1597 if (cpudata->suspended) { 1598 int ret; 1599 1600 /* enable amd pstate from suspend state*/ 1601 ret = amd_pstate_epp_update_limit(policy); 1602 if (ret) 1603 return ret; 1604 1605 cpudata->suspended = false; 1606 } 1607 1608 return 0; 1609 } 1610 1611 static struct cpufreq_driver amd_pstate_driver = { 1612 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 1613 .verify = amd_pstate_verify, 1614 .target = amd_pstate_target, 1615 .fast_switch = amd_pstate_fast_switch, 1616 .init = amd_pstate_cpu_init, 1617 .exit = amd_pstate_cpu_exit, 1618 .set_boost = amd_pstate_set_boost, 1619 .update_limits = amd_pstate_update_limits, 1620 .name = "amd-pstate", 1621 .attr = amd_pstate_attr, 1622 }; 1623 1624 static struct cpufreq_driver amd_pstate_epp_driver = { 1625 .flags = CPUFREQ_CONST_LOOPS, 1626 .verify = amd_pstate_verify, 1627 .setpolicy = amd_pstate_epp_set_policy, 1628 .init = amd_pstate_epp_cpu_init, 1629 .exit = amd_pstate_epp_cpu_exit, 1630 .offline = amd_pstate_epp_cpu_offline, 1631 .online = amd_pstate_epp_cpu_online, 1632 .suspend = amd_pstate_epp_suspend, 1633 .resume = amd_pstate_epp_resume, 1634 .update_limits = amd_pstate_update_limits, 1635 .set_boost = amd_pstate_set_boost, 1636 .name = "amd-pstate-epp", 1637 .attr = amd_pstate_epp_attr, 1638 }; 1639 1640 /* 1641 * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. 1642 * show the debug message that helps to check if the CPU has CPPC support for loading issue. 1643 */ 1644 static bool amd_cppc_supported(void) 1645 { 1646 struct cpuinfo_x86 *c = &cpu_data(0); 1647 bool warn = false; 1648 1649 if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) { 1650 pr_debug_once("CPPC feature is not supported by the processor\n"); 1651 return false; 1652 } 1653 1654 /* 1655 * If the CPPC feature is disabled in the BIOS for processors 1656 * that support MSR-based CPPC, the AMD Pstate driver may not 1657 * function correctly. 1658 * 1659 * For such processors, check the CPPC flag and display a 1660 * warning message if the platform supports CPPC. 1661 * 1662 * Note: The code check below will not abort the driver 1663 * registration process because of the code is added for 1664 * debugging purposes. Besides, it may still be possible for 1665 * the driver to work using the shared-memory mechanism. 1666 */ 1667 if (!cpu_feature_enabled(X86_FEATURE_CPPC)) { 1668 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 1669 switch (c->x86_model) { 1670 case 0x60 ... 0x6F: 1671 case 0x80 ... 0xAF: 1672 warn = true; 1673 break; 1674 } 1675 } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || 1676 cpu_feature_enabled(X86_FEATURE_ZEN4)) { 1677 switch (c->x86_model) { 1678 case 0x10 ... 0x1F: 1679 case 0x40 ... 0xAF: 1680 warn = true; 1681 break; 1682 } 1683 } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) { 1684 warn = true; 1685 } 1686 } 1687 1688 if (warn) 1689 pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n" 1690 "Please enable it if your BIOS has the CPPC option.\n"); 1691 return true; 1692 } 1693 1694 static int __init amd_pstate_init(void) 1695 { 1696 struct device *dev_root; 1697 int ret; 1698 1699 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 1700 return -ENODEV; 1701 1702 /* show debug message only if CPPC is not supported */ 1703 if (!amd_cppc_supported()) 1704 return -EOPNOTSUPP; 1705 1706 /* show warning message when BIOS broken or ACPI disabled */ 1707 if (!acpi_cpc_valid()) { 1708 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 1709 return -ENODEV; 1710 } 1711 1712 /* don't keep reloading if cpufreq_driver exists */ 1713 if (cpufreq_get_current_driver()) 1714 return -EEXIST; 1715 1716 quirks = NULL; 1717 1718 /* check if this machine need CPPC quirks */ 1719 dmi_check_system(amd_pstate_quirks_table); 1720 1721 /* 1722 * determine the driver mode from the command line or kernel config. 1723 * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED. 1724 * command line options will override the kernel config settings. 1725 */ 1726 1727 if (cppc_state == AMD_PSTATE_UNDEFINED) { 1728 /* Disable on the following configs by default: 1729 * 1. Undefined platforms 1730 * 2. Server platforms with CPUs older than Family 0x1A. 1731 */ 1732 if (amd_pstate_acpi_pm_profile_undefined() || 1733 (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) { 1734 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 1735 return -ENODEV; 1736 } 1737 /* get driver mode from kernel config option [1:4] */ 1738 cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; 1739 } 1740 1741 if (cppc_state == AMD_PSTATE_DISABLE) { 1742 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 1743 return -ENODEV; 1744 } 1745 1746 /* capability check */ 1747 if (cpu_feature_enabled(X86_FEATURE_CPPC)) { 1748 pr_debug("AMD CPPC MSR based functionality is supported\n"); 1749 } else { 1750 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 1751 static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); 1752 static_call_update(amd_pstate_init_perf, shmem_init_perf); 1753 static_call_update(amd_pstate_update_perf, shmem_update_perf); 1754 static_call_update(amd_pstate_get_epp, shmem_get_epp); 1755 static_call_update(amd_pstate_set_epp, shmem_set_epp); 1756 } 1757 1758 if (amd_pstate_prefcore) { 1759 ret = amd_detect_prefcore(&amd_pstate_prefcore); 1760 if (ret) 1761 return ret; 1762 } 1763 1764 ret = amd_pstate_register_driver(cppc_state); 1765 if (ret) { 1766 pr_err("failed to register with return %d\n", ret); 1767 return ret; 1768 } 1769 1770 dev_root = bus_get_dev_root(&cpu_subsys); 1771 if (dev_root) { 1772 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 1773 put_device(dev_root); 1774 if (ret) { 1775 pr_err("sysfs attribute export failed with error %d.\n", ret); 1776 goto global_attr_free; 1777 } 1778 } 1779 1780 return ret; 1781 1782 global_attr_free: 1783 cpufreq_unregister_driver(current_pstate_driver); 1784 return ret; 1785 } 1786 device_initcall(amd_pstate_init); 1787 1788 static int __init amd_pstate_param(char *str) 1789 { 1790 size_t size; 1791 int mode_idx; 1792 1793 if (!str) 1794 return -EINVAL; 1795 1796 size = strlen(str); 1797 mode_idx = get_mode_idx_from_str(str, size); 1798 1799 return amd_pstate_set_driver(mode_idx); 1800 } 1801 1802 static int __init amd_prefcore_param(char *str) 1803 { 1804 if (!strcmp(str, "disable")) 1805 amd_pstate_prefcore = false; 1806 1807 return 0; 1808 } 1809 1810 early_param("amd_pstate", amd_pstate_param); 1811 early_param("amd_prefcore", amd_prefcore_param); 1812 1813 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 1814 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 1815