1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/bitfield.h> 26 #include <linux/kernel.h> 27 #include <linux/module.h> 28 #include <linux/init.h> 29 #include <linux/smp.h> 30 #include <linux/sched.h> 31 #include <linux/cpufreq.h> 32 #include <linux/compiler.h> 33 #include <linux/dmi.h> 34 #include <linux/slab.h> 35 #include <linux/acpi.h> 36 #include <linux/io.h> 37 #include <linux/delay.h> 38 #include <linux/uaccess.h> 39 #include <linux/static_call.h> 40 #include <linux/topology.h> 41 42 #include <acpi/processor.h> 43 #include <acpi/cppc_acpi.h> 44 45 #include <asm/msr.h> 46 #include <asm/processor.h> 47 #include <asm/cpufeature.h> 48 #include <asm/cpu_device_id.h> 49 50 #include "amd-pstate.h" 51 #include "amd-pstate-trace.h" 52 53 #define AMD_PSTATE_TRANSITION_LATENCY 20000 54 #define AMD_PSTATE_TRANSITION_DELAY 1000 55 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600 56 57 #define AMD_CPPC_EPP_PERFORMANCE 0x00 58 #define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 59 #define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF 60 #define AMD_CPPC_EPP_POWERSAVE 0xFF 61 62 static const char * const amd_pstate_mode_string[] = { 63 [AMD_PSTATE_UNDEFINED] = "undefined", 64 [AMD_PSTATE_DISABLE] = "disable", 65 [AMD_PSTATE_PASSIVE] = "passive", 66 [AMD_PSTATE_ACTIVE] = "active", 67 [AMD_PSTATE_GUIDED] = "guided", 68 }; 69 static_assert(ARRAY_SIZE(amd_pstate_mode_string) == AMD_PSTATE_MAX); 70 71 const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode) 72 { 73 if (mode < AMD_PSTATE_UNDEFINED || mode >= AMD_PSTATE_MAX) 74 mode = AMD_PSTATE_UNDEFINED; 75 return amd_pstate_mode_string[mode]; 76 } 77 EXPORT_SYMBOL_GPL(amd_pstate_get_mode_string); 78 79 struct quirk_entry { 80 u32 nominal_freq; 81 u32 lowest_freq; 82 }; 83 84 static struct cpufreq_driver *current_pstate_driver; 85 static struct cpufreq_driver amd_pstate_driver; 86 static struct cpufreq_driver amd_pstate_epp_driver; 87 static int cppc_state = AMD_PSTATE_UNDEFINED; 88 static bool amd_pstate_prefcore = true; 89 static struct quirk_entry *quirks; 90 91 /* 92 * AMD Energy Preference Performance (EPP) 93 * The EPP is used in the CCLK DPM controller to drive 94 * the frequency that a core is going to operate during 95 * short periods of activity. EPP values will be utilized for 96 * different OS profiles (balanced, performance, power savings) 97 * display strings corresponding to EPP index in the 98 * energy_perf_strings[] 99 * index String 100 *------------------------------------- 101 * 0 default 102 * 1 performance 103 * 2 balance_performance 104 * 3 balance_power 105 * 4 power 106 */ 107 enum energy_perf_value_index { 108 EPP_INDEX_DEFAULT = 0, 109 EPP_INDEX_PERFORMANCE, 110 EPP_INDEX_BALANCE_PERFORMANCE, 111 EPP_INDEX_BALANCE_POWERSAVE, 112 EPP_INDEX_POWERSAVE, 113 EPP_INDEX_MAX, 114 }; 115 116 static const char * const energy_perf_strings[] = { 117 [EPP_INDEX_DEFAULT] = "default", 118 [EPP_INDEX_PERFORMANCE] = "performance", 119 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 120 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 121 [EPP_INDEX_POWERSAVE] = "power", 122 }; 123 static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); 124 125 static unsigned int epp_values[] = { 126 [EPP_INDEX_DEFAULT] = 0, 127 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 128 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 129 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 130 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 131 }; 132 static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX); 133 134 typedef int (*cppc_mode_transition_fn)(int); 135 136 static struct quirk_entry quirk_amd_7k62 = { 137 .nominal_freq = 2600, 138 .lowest_freq = 550, 139 }; 140 141 static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) 142 { 143 u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); 144 145 return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf); 146 } 147 148 static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val) 149 { 150 return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val, 151 perf.nominal_perf); 152 } 153 154 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) 155 { 156 /** 157 * match the broken bios for family 17h processor support CPPC V2 158 * broken BIOS lack of nominal_freq and lowest_freq capabilities 159 * definition in ACPI tables 160 */ 161 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 162 quirks = dmi->driver_data; 163 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); 164 return 1; 165 } 166 167 return 0; 168 } 169 170 static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { 171 { 172 .callback = dmi_matched_7k62_bios_bug, 173 .ident = "AMD EPYC 7K62", 174 .matches = { 175 DMI_MATCH(DMI_BIOS_VERSION, "5.14"), 176 DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), 177 }, 178 .driver_data = &quirk_amd_7k62, 179 }, 180 {} 181 }; 182 MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); 183 184 static inline int get_mode_idx_from_str(const char *str, size_t size) 185 { 186 int i; 187 188 for (i = 0; i < AMD_PSTATE_MAX; i++) { 189 if (!strncmp(str, amd_pstate_mode_string[i], size)) 190 return i; 191 } 192 return -EINVAL; 193 } 194 195 static DEFINE_MUTEX(amd_pstate_driver_lock); 196 197 static u8 msr_get_epp(struct amd_cpudata *cpudata) 198 { 199 u64 value; 200 int ret; 201 202 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 203 if (ret < 0) { 204 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 205 return ret; 206 } 207 208 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value); 209 } 210 211 DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); 212 213 static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) 214 { 215 return static_call(amd_pstate_get_epp)(cpudata); 216 } 217 218 static u8 shmem_get_epp(struct amd_cpudata *cpudata) 219 { 220 u64 epp; 221 int ret; 222 223 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 224 if (ret < 0) { 225 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 226 return ret; 227 } 228 229 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp); 230 } 231 232 static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, 233 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 234 { 235 struct amd_cpudata *cpudata = policy->driver_data; 236 u64 value, prev; 237 238 value = prev = READ_ONCE(cpudata->cppc_req_cached); 239 240 value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | 241 AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); 242 value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); 243 value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); 244 value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); 245 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 246 247 if (trace_amd_pstate_epp_perf_enabled()) { 248 union perf_cached perf = READ_ONCE(cpudata->perf); 249 250 trace_amd_pstate_epp_perf(cpudata->cpu, 251 perf.highest_perf, 252 epp, 253 min_perf, 254 max_perf, 255 policy->boost_enabled, 256 value != prev); 257 } 258 259 if (value == prev) 260 return 0; 261 262 if (fast_switch) { 263 wrmsrq(MSR_AMD_CPPC_REQ, value); 264 return 0; 265 } else { 266 int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 267 268 if (ret) 269 return ret; 270 } 271 272 WRITE_ONCE(cpudata->cppc_req_cached, value); 273 274 return 0; 275 } 276 277 DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); 278 279 static inline int amd_pstate_update_perf(struct cpufreq_policy *policy, 280 u8 min_perf, u8 des_perf, 281 u8 max_perf, u8 epp, 282 bool fast_switch) 283 { 284 return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf, 285 max_perf, epp, fast_switch); 286 } 287 288 static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) 289 { 290 struct amd_cpudata *cpudata = policy->driver_data; 291 u64 value, prev; 292 int ret; 293 294 value = prev = READ_ONCE(cpudata->cppc_req_cached); 295 value &= ~AMD_CPPC_EPP_PERF_MASK; 296 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 297 298 if (trace_amd_pstate_epp_perf_enabled()) { 299 union perf_cached perf = cpudata->perf; 300 301 trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 302 epp, 303 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 304 cpudata->cppc_req_cached), 305 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 306 cpudata->cppc_req_cached), 307 policy->boost_enabled, 308 value != prev); 309 } 310 311 if (value == prev) 312 return 0; 313 314 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 315 if (ret) { 316 pr_err("failed to set energy perf value (%d)\n", ret); 317 return ret; 318 } 319 320 /* update both so that msr_update_perf() can effectively check */ 321 WRITE_ONCE(cpudata->cppc_req_cached, value); 322 323 return ret; 324 } 325 326 DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); 327 328 static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp) 329 { 330 return static_call(amd_pstate_set_epp)(policy, epp); 331 } 332 333 static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) 334 { 335 struct amd_cpudata *cpudata = policy->driver_data; 336 struct cppc_perf_ctrls perf_ctrls; 337 u8 epp_cached; 338 u64 value; 339 int ret; 340 341 342 epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 343 if (trace_amd_pstate_epp_perf_enabled()) { 344 union perf_cached perf = cpudata->perf; 345 346 trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 347 epp, 348 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 349 cpudata->cppc_req_cached), 350 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 351 cpudata->cppc_req_cached), 352 policy->boost_enabled, 353 epp != epp_cached); 354 } 355 356 if (epp == epp_cached) 357 return 0; 358 359 perf_ctrls.energy_perf = epp; 360 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 361 if (ret) { 362 pr_debug("failed to set energy perf value (%d)\n", ret); 363 return ret; 364 } 365 366 value = READ_ONCE(cpudata->cppc_req_cached); 367 value &= ~AMD_CPPC_EPP_PERF_MASK; 368 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 369 WRITE_ONCE(cpudata->cppc_req_cached, value); 370 371 return ret; 372 } 373 374 static inline int msr_cppc_enable(struct cpufreq_policy *policy) 375 { 376 return wrmsrq_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); 377 } 378 379 static int shmem_cppc_enable(struct cpufreq_policy *policy) 380 { 381 return cppc_set_enable(policy->cpu, 1); 382 } 383 384 DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); 385 386 static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) 387 { 388 return static_call(amd_pstate_cppc_enable)(policy); 389 } 390 391 static int msr_init_perf(struct amd_cpudata *cpudata) 392 { 393 union perf_cached perf = READ_ONCE(cpudata->perf); 394 u64 cap1, numerator, cppc_req; 395 u8 min_perf; 396 397 int ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 398 &cap1); 399 if (ret) 400 return ret; 401 402 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 403 if (ret) 404 return ret; 405 406 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); 407 if (ret) 408 return ret; 409 410 WRITE_ONCE(cpudata->cppc_req_cached, cppc_req); 411 min_perf = FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cppc_req); 412 413 /* 414 * Clear out the min_perf part to check if the rest of the MSR is 0, if yes, this is an 415 * indication that the min_perf value is the one specified through the BIOS option 416 */ 417 cppc_req &= ~(AMD_CPPC_MIN_PERF_MASK); 418 419 if (!cppc_req) 420 perf.bios_min_perf = min_perf; 421 422 perf.highest_perf = numerator; 423 perf.max_limit_perf = numerator; 424 perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 425 perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); 426 perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); 427 perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 428 WRITE_ONCE(cpudata->perf, perf); 429 WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1)); 430 431 return 0; 432 } 433 434 static int shmem_init_perf(struct amd_cpudata *cpudata) 435 { 436 struct cppc_perf_caps cppc_perf; 437 union perf_cached perf = READ_ONCE(cpudata->perf); 438 u64 numerator; 439 bool auto_sel; 440 441 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 442 if (ret) 443 return ret; 444 445 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 446 if (ret) 447 return ret; 448 449 perf.highest_perf = numerator; 450 perf.max_limit_perf = numerator; 451 perf.min_limit_perf = cppc_perf.lowest_perf; 452 perf.nominal_perf = cppc_perf.nominal_perf; 453 perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 454 perf.lowest_perf = cppc_perf.lowest_perf; 455 WRITE_ONCE(cpudata->perf, perf); 456 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); 457 458 if (cppc_state == AMD_PSTATE_ACTIVE) 459 return 0; 460 461 ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel); 462 if (ret) { 463 pr_warn("failed to get auto_sel, ret: %d\n", ret); 464 return 0; 465 } 466 467 ret = cppc_set_auto_sel(cpudata->cpu, 468 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 469 470 if (ret) 471 pr_warn("failed to set auto_sel, ret: %d\n", ret); 472 473 return ret; 474 } 475 476 DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf); 477 478 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 479 { 480 return static_call(amd_pstate_init_perf)(cpudata); 481 } 482 483 static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf, 484 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 485 { 486 struct amd_cpudata *cpudata = policy->driver_data; 487 struct cppc_perf_ctrls perf_ctrls; 488 u64 value, prev; 489 int ret; 490 491 if (cppc_state == AMD_PSTATE_ACTIVE) { 492 int ret = shmem_set_epp(policy, epp); 493 494 if (ret) 495 return ret; 496 } 497 498 value = prev = READ_ONCE(cpudata->cppc_req_cached); 499 500 value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | 501 AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); 502 value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); 503 value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); 504 value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); 505 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 506 507 if (trace_amd_pstate_epp_perf_enabled()) { 508 union perf_cached perf = READ_ONCE(cpudata->perf); 509 510 trace_amd_pstate_epp_perf(cpudata->cpu, 511 perf.highest_perf, 512 epp, 513 min_perf, 514 max_perf, 515 policy->boost_enabled, 516 value != prev); 517 } 518 519 if (value == prev) 520 return 0; 521 522 perf_ctrls.max_perf = max_perf; 523 perf_ctrls.min_perf = min_perf; 524 perf_ctrls.desired_perf = des_perf; 525 526 ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); 527 if (ret) 528 return ret; 529 530 WRITE_ONCE(cpudata->cppc_req_cached, value); 531 532 return 0; 533 } 534 535 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 536 { 537 u64 aperf, mperf, tsc; 538 unsigned long flags; 539 540 local_irq_save(flags); 541 rdmsrq(MSR_IA32_APERF, aperf); 542 rdmsrq(MSR_IA32_MPERF, mperf); 543 tsc = rdtsc(); 544 545 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 546 local_irq_restore(flags); 547 return false; 548 } 549 550 local_irq_restore(flags); 551 552 cpudata->cur.aperf = aperf; 553 cpudata->cur.mperf = mperf; 554 cpudata->cur.tsc = tsc; 555 cpudata->cur.aperf -= cpudata->prev.aperf; 556 cpudata->cur.mperf -= cpudata->prev.mperf; 557 cpudata->cur.tsc -= cpudata->prev.tsc; 558 559 cpudata->prev.aperf = aperf; 560 cpudata->prev.mperf = mperf; 561 cpudata->prev.tsc = tsc; 562 563 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 564 565 return true; 566 } 567 568 static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, 569 u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) 570 { 571 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 572 union perf_cached perf = READ_ONCE(cpudata->perf); 573 574 if (!policy) 575 return; 576 577 /* limit the max perf when core performance boost feature is disabled */ 578 if (!cpudata->boost_supported) 579 max_perf = min_t(u8, perf.nominal_perf, max_perf); 580 581 des_perf = clamp_t(u8, des_perf, min_perf, max_perf); 582 583 policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); 584 585 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 586 min_perf = des_perf; 587 des_perf = 0; 588 } 589 590 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 591 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 592 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 593 cpudata->cpu, fast_switch); 594 } 595 596 amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch); 597 } 598 599 static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) 600 { 601 /* 602 * Initialize lower frequency limit (i.e.policy->min) with 603 * lowest_nonlinear_frequency or the min frequency (if) specified in BIOS, 604 * Override the initial value set by cpufreq core and amd-pstate qos_requests. 605 */ 606 if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { 607 struct cpufreq_policy *policy __free(put_cpufreq_policy) = 608 cpufreq_cpu_get(policy_data->cpu); 609 struct amd_cpudata *cpudata; 610 union perf_cached perf; 611 612 if (!policy) 613 return -EINVAL; 614 615 cpudata = policy->driver_data; 616 perf = READ_ONCE(cpudata->perf); 617 618 if (perf.bios_min_perf) 619 policy_data->min = perf_to_freq(perf, cpudata->nominal_freq, 620 perf.bios_min_perf); 621 else 622 policy_data->min = cpudata->lowest_nonlinear_freq; 623 } 624 625 cpufreq_verify_within_cpu_limits(policy_data); 626 627 return 0; 628 } 629 630 static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) 631 { 632 struct amd_cpudata *cpudata = policy->driver_data; 633 union perf_cached perf = READ_ONCE(cpudata->perf); 634 635 perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); 636 WRITE_ONCE(cpudata->max_limit_freq, policy->max); 637 638 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 639 /* 640 * For performance policy, set MinPerf to nominal_perf rather than 641 * highest_perf or lowest_nonlinear_perf. 642 * 643 * Per commit 0c411b39e4f4c, using highest_perf was observed 644 * to cause frequency throttling on power-limited platforms, leading to 645 * performance regressions. Using lowest_nonlinear_perf would limit 646 * performance too much for HPC workloads requiring high frequency 647 * operation and minimal wakeup latency from idle states. 648 * 649 * nominal_perf therefore provides a balance by avoiding throttling 650 * while still maintaining enough performance for HPC workloads. 651 */ 652 perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); 653 WRITE_ONCE(cpudata->min_limit_freq, min(cpudata->nominal_freq, cpudata->max_limit_freq)); 654 } else { 655 perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); 656 WRITE_ONCE(cpudata->min_limit_freq, policy->min); 657 } 658 659 WRITE_ONCE(cpudata->perf, perf); 660 } 661 662 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 663 unsigned int target_freq, bool fast_switch) 664 { 665 struct cpufreq_freqs freqs; 666 struct amd_cpudata *cpudata; 667 union perf_cached perf; 668 u8 des_perf; 669 670 cpudata = policy->driver_data; 671 672 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 673 amd_pstate_update_min_max_limit(policy); 674 675 perf = READ_ONCE(cpudata->perf); 676 677 freqs.old = policy->cur; 678 freqs.new = target_freq; 679 680 des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq); 681 682 WARN_ON(fast_switch && !policy->fast_switch_enabled); 683 /* 684 * If fast_switch is desired, then there aren't any registered 685 * transition notifiers. See comment for 686 * cpufreq_enable_fast_switch(). 687 */ 688 if (!fast_switch) 689 cpufreq_freq_transition_begin(policy, &freqs); 690 691 amd_pstate_update(cpudata, perf.min_limit_perf, des_perf, 692 perf.max_limit_perf, fast_switch, 693 policy->governor->flags); 694 695 if (!fast_switch) 696 cpufreq_freq_transition_end(policy, &freqs, false); 697 698 return 0; 699 } 700 701 static int amd_pstate_target(struct cpufreq_policy *policy, 702 unsigned int target_freq, 703 unsigned int relation) 704 { 705 return amd_pstate_update_freq(policy, target_freq, false); 706 } 707 708 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 709 unsigned int target_freq) 710 { 711 if (!amd_pstate_update_freq(policy, target_freq, true)) 712 return target_freq; 713 return policy->cur; 714 } 715 716 static void amd_pstate_adjust_perf(unsigned int cpu, 717 unsigned long _min_perf, 718 unsigned long target_perf, 719 unsigned long capacity) 720 { 721 u8 max_perf, min_perf, des_perf, cap_perf; 722 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); 723 struct amd_cpudata *cpudata; 724 union perf_cached perf; 725 726 if (!policy) 727 return; 728 729 cpudata = policy->driver_data; 730 731 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 732 amd_pstate_update_min_max_limit(policy); 733 734 perf = READ_ONCE(cpudata->perf); 735 cap_perf = perf.highest_perf; 736 737 des_perf = cap_perf; 738 if (target_perf < capacity) 739 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 740 741 if (_min_perf < capacity) 742 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 743 else 744 min_perf = cap_perf; 745 746 if (min_perf < perf.min_limit_perf) 747 min_perf = perf.min_limit_perf; 748 749 max_perf = perf.max_limit_perf; 750 if (max_perf < min_perf) 751 max_perf = min_perf; 752 753 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, 754 policy->governor->flags); 755 } 756 757 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) 758 { 759 struct amd_cpudata *cpudata = policy->driver_data; 760 union perf_cached perf = READ_ONCE(cpudata->perf); 761 u32 nominal_freq, max_freq; 762 int ret = 0; 763 764 nominal_freq = READ_ONCE(cpudata->nominal_freq); 765 max_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf); 766 767 if (on) 768 policy->cpuinfo.max_freq = max_freq; 769 else if (policy->cpuinfo.max_freq > nominal_freq) 770 policy->cpuinfo.max_freq = nominal_freq; 771 772 policy->max = policy->cpuinfo.max_freq; 773 774 if (cppc_state == AMD_PSTATE_PASSIVE) { 775 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); 776 if (ret < 0) 777 pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu); 778 } 779 780 return ret < 0 ? ret : 0; 781 } 782 783 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 784 { 785 struct amd_cpudata *cpudata = policy->driver_data; 786 int ret; 787 788 if (!cpudata->boost_supported) { 789 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 790 return -EOPNOTSUPP; 791 } 792 793 ret = amd_pstate_cpu_boost_update(policy, state); 794 refresh_frequency_limits(policy); 795 796 return ret; 797 } 798 799 static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) 800 { 801 u64 boost_val; 802 int ret = -1; 803 804 /* 805 * If platform has no CPB support or disable it, initialize current driver 806 * boost_enabled state to be false, it is not an error for cpufreq core to handle. 807 */ 808 if (!cpu_feature_enabled(X86_FEATURE_CPB)) { 809 pr_debug_once("Boost CPB capabilities not present in the processor\n"); 810 ret = 0; 811 goto exit_err; 812 } 813 814 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); 815 if (ret) { 816 pr_err_once("failed to read initial CPU boost state!\n"); 817 ret = -EIO; 818 goto exit_err; 819 } 820 821 if (!(boost_val & MSR_K7_HWCR_CPB_DIS)) 822 cpudata->boost_supported = true; 823 824 return 0; 825 826 exit_err: 827 cpudata->boost_supported = false; 828 return ret; 829 } 830 831 static void amd_perf_ctl_reset(unsigned int cpu) 832 { 833 wrmsrq_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 834 } 835 836 #define CPPC_MAX_PERF U8_MAX 837 838 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) 839 { 840 /* user disabled or not detected */ 841 if (!amd_pstate_prefcore) 842 return; 843 844 /* should use amd-hfi instead */ 845 if (cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS) && 846 IS_ENABLED(CONFIG_AMD_HFI)) { 847 amd_pstate_prefcore = false; 848 return; 849 } 850 851 cpudata->hw_prefcore = true; 852 853 /* Priorities must be initialized before ITMT support can be toggled on. */ 854 sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); 855 } 856 857 static void amd_pstate_update_limits(struct cpufreq_policy *policy) 858 { 859 struct amd_cpudata *cpudata; 860 u32 prev_high = 0, cur_high = 0; 861 bool highest_perf_changed = false; 862 unsigned int cpu = policy->cpu; 863 864 if (!amd_pstate_prefcore) 865 return; 866 867 if (amd_get_highest_perf(cpu, &cur_high)) 868 return; 869 870 cpudata = policy->driver_data; 871 872 prev_high = READ_ONCE(cpudata->prefcore_ranking); 873 highest_perf_changed = (prev_high != cur_high); 874 if (highest_perf_changed) { 875 WRITE_ONCE(cpudata->prefcore_ranking, cur_high); 876 877 if (cur_high < CPPC_MAX_PERF) { 878 sched_set_itmt_core_prio((int)cur_high, cpu); 879 sched_update_asym_prefer_cpu(cpu, prev_high, cur_high); 880 } 881 } 882 } 883 884 /* 885 * Get pstate transition delay time from ACPI tables that firmware set 886 * instead of using hardcode value directly. 887 */ 888 static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) 889 { 890 int transition_delay_ns; 891 892 transition_delay_ns = cppc_get_transition_latency(cpu); 893 if (transition_delay_ns < 0) { 894 if (cpu_feature_enabled(X86_FEATURE_AMD_FAST_CPPC)) 895 return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY; 896 else 897 return AMD_PSTATE_TRANSITION_DELAY; 898 } 899 900 return transition_delay_ns / NSEC_PER_USEC; 901 } 902 903 /* 904 * Get pstate transition latency value from ACPI tables that firmware 905 * set instead of using hardcode value directly. 906 */ 907 static u32 amd_pstate_get_transition_latency(unsigned int cpu) 908 { 909 int transition_latency; 910 911 transition_latency = cppc_get_transition_latency(cpu); 912 if (transition_latency < 0) 913 return AMD_PSTATE_TRANSITION_LATENCY; 914 915 return transition_latency; 916 } 917 918 /* 919 * amd_pstate_init_freq: Initialize the nominal_freq and lowest_nonlinear_freq 920 * for the @cpudata object. 921 * 922 * Requires: all perf members of @cpudata to be initialized. 923 * 924 * Returns 0 on success, non-zero value on failure. 925 */ 926 static int amd_pstate_init_freq(struct amd_cpudata *cpudata) 927 { 928 u32 min_freq, max_freq, nominal_freq, lowest_nonlinear_freq; 929 struct cppc_perf_caps cppc_perf; 930 union perf_cached perf; 931 int ret; 932 933 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 934 if (ret) 935 return ret; 936 perf = READ_ONCE(cpudata->perf); 937 938 if (quirks && quirks->nominal_freq) 939 nominal_freq = quirks->nominal_freq; 940 else 941 nominal_freq = cppc_perf.nominal_freq; 942 nominal_freq *= 1000; 943 944 if (quirks && quirks->lowest_freq) { 945 min_freq = quirks->lowest_freq; 946 perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq); 947 WRITE_ONCE(cpudata->perf, perf); 948 } else 949 min_freq = cppc_perf.lowest_freq; 950 951 min_freq *= 1000; 952 953 WRITE_ONCE(cpudata->nominal_freq, nominal_freq); 954 955 max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); 956 lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); 957 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); 958 959 /** 960 * Below values need to be initialized correctly, otherwise driver will fail to load 961 * max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf 962 * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] 963 * Check _CPC in ACPI table objects if any values are incorrect 964 */ 965 if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { 966 pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", 967 min_freq, max_freq, nominal_freq); 968 return -EINVAL; 969 } 970 971 if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq) { 972 pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", 973 lowest_nonlinear_freq, min_freq, nominal_freq); 974 return -EINVAL; 975 } 976 977 return 0; 978 } 979 980 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 981 { 982 struct amd_cpudata *cpudata; 983 union perf_cached perf; 984 struct device *dev; 985 int ret; 986 987 /* 988 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 989 * which is ideal for initialization process. 990 */ 991 amd_perf_ctl_reset(policy->cpu); 992 dev = get_cpu_device(policy->cpu); 993 if (!dev) 994 return -ENODEV; 995 996 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 997 if (!cpudata) 998 return -ENOMEM; 999 1000 cpudata->cpu = policy->cpu; 1001 1002 ret = amd_pstate_init_perf(cpudata); 1003 if (ret) 1004 goto free_cpudata1; 1005 1006 amd_pstate_init_prefcore(cpudata); 1007 1008 ret = amd_pstate_init_freq(cpudata); 1009 if (ret) 1010 goto free_cpudata1; 1011 1012 ret = amd_pstate_init_boost_support(cpudata); 1013 if (ret) 1014 goto free_cpudata1; 1015 1016 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); 1017 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); 1018 1019 perf = READ_ONCE(cpudata->perf); 1020 1021 policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, 1022 cpudata->nominal_freq, 1023 perf.lowest_perf); 1024 policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, 1025 cpudata->nominal_freq, 1026 perf.highest_perf); 1027 1028 ret = amd_pstate_cppc_enable(policy); 1029 if (ret) 1030 goto free_cpudata1; 1031 1032 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1033 1034 /* It will be updated by governor */ 1035 policy->cur = policy->cpuinfo.min_freq; 1036 1037 if (cpu_feature_enabled(X86_FEATURE_CPPC)) 1038 policy->fast_switch_possible = true; 1039 1040 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 1041 FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); 1042 if (ret < 0) { 1043 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 1044 goto free_cpudata1; 1045 } 1046 1047 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 1048 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 1049 if (ret < 0) { 1050 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 1051 goto free_cpudata2; 1052 } 1053 1054 policy->driver_data = cpudata; 1055 1056 if (!current_pstate_driver->adjust_perf) 1057 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1058 1059 return 0; 1060 1061 free_cpudata2: 1062 freq_qos_remove_request(&cpudata->req[0]); 1063 free_cpudata1: 1064 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1065 kfree(cpudata); 1066 return ret; 1067 } 1068 1069 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) 1070 { 1071 struct amd_cpudata *cpudata = policy->driver_data; 1072 union perf_cached perf = READ_ONCE(cpudata->perf); 1073 1074 /* Reset CPPC_REQ MSR to the BIOS value */ 1075 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1076 1077 freq_qos_remove_request(&cpudata->req[1]); 1078 freq_qos_remove_request(&cpudata->req[0]); 1079 policy->fast_switch_possible = false; 1080 kfree(cpudata); 1081 } 1082 1083 /* Sysfs attributes */ 1084 1085 /* 1086 * This frequency is to indicate the maximum hardware frequency. 1087 * If boost is not active but supported, the frequency will be larger than the 1088 * one in cpuinfo. 1089 */ 1090 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 1091 char *buf) 1092 { 1093 struct amd_cpudata *cpudata; 1094 union perf_cached perf; 1095 1096 cpudata = policy->driver_data; 1097 perf = READ_ONCE(cpudata->perf); 1098 1099 return sysfs_emit(buf, "%u\n", 1100 perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf)); 1101 } 1102 1103 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 1104 char *buf) 1105 { 1106 struct amd_cpudata *cpudata; 1107 union perf_cached perf; 1108 1109 cpudata = policy->driver_data; 1110 perf = READ_ONCE(cpudata->perf); 1111 1112 return sysfs_emit(buf, "%u\n", 1113 perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf)); 1114 } 1115 1116 /* 1117 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 1118 * need to expose it to sysfs. 1119 */ 1120 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 1121 char *buf) 1122 { 1123 struct amd_cpudata *cpudata; 1124 1125 cpudata = policy->driver_data; 1126 1127 return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf); 1128 } 1129 1130 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, 1131 char *buf) 1132 { 1133 u8 perf; 1134 struct amd_cpudata *cpudata = policy->driver_data; 1135 1136 perf = READ_ONCE(cpudata->prefcore_ranking); 1137 1138 return sysfs_emit(buf, "%u\n", perf); 1139 } 1140 1141 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, 1142 char *buf) 1143 { 1144 bool hw_prefcore; 1145 struct amd_cpudata *cpudata = policy->driver_data; 1146 1147 hw_prefcore = READ_ONCE(cpudata->hw_prefcore); 1148 1149 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); 1150 } 1151 1152 static ssize_t show_energy_performance_available_preferences( 1153 struct cpufreq_policy *policy, char *buf) 1154 { 1155 int offset = 0, i; 1156 struct amd_cpudata *cpudata = policy->driver_data; 1157 1158 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1159 return sysfs_emit_at(buf, offset, "%s\n", 1160 energy_perf_strings[EPP_INDEX_PERFORMANCE]); 1161 1162 for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++) 1163 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]); 1164 1165 offset += sysfs_emit_at(buf, offset, "\n"); 1166 1167 return offset; 1168 } 1169 1170 static ssize_t store_energy_performance_preference( 1171 struct cpufreq_policy *policy, const char *buf, size_t count) 1172 { 1173 struct amd_cpudata *cpudata = policy->driver_data; 1174 ssize_t ret; 1175 u8 epp; 1176 1177 ret = sysfs_match_string(energy_perf_strings, buf); 1178 if (ret < 0) 1179 return -EINVAL; 1180 1181 if (!ret) 1182 epp = cpudata->epp_default; 1183 else 1184 epp = epp_values[ret]; 1185 1186 if (epp > 0 && policy->policy == CPUFREQ_POLICY_PERFORMANCE) { 1187 pr_debug("EPP cannot be set under performance policy\n"); 1188 return -EBUSY; 1189 } 1190 1191 ret = amd_pstate_set_epp(policy, epp); 1192 1193 return ret ? ret : count; 1194 } 1195 1196 static ssize_t show_energy_performance_preference( 1197 struct cpufreq_policy *policy, char *buf) 1198 { 1199 struct amd_cpudata *cpudata = policy->driver_data; 1200 u8 preference, epp; 1201 1202 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 1203 1204 switch (epp) { 1205 case AMD_CPPC_EPP_PERFORMANCE: 1206 preference = EPP_INDEX_PERFORMANCE; 1207 break; 1208 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 1209 preference = EPP_INDEX_BALANCE_PERFORMANCE; 1210 break; 1211 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 1212 preference = EPP_INDEX_BALANCE_POWERSAVE; 1213 break; 1214 case AMD_CPPC_EPP_POWERSAVE: 1215 preference = EPP_INDEX_POWERSAVE; 1216 break; 1217 default: 1218 return -EINVAL; 1219 } 1220 1221 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 1222 } 1223 1224 static void amd_pstate_driver_cleanup(void) 1225 { 1226 if (amd_pstate_prefcore) 1227 sched_clear_itmt_support(); 1228 1229 cppc_state = AMD_PSTATE_DISABLE; 1230 current_pstate_driver = NULL; 1231 } 1232 1233 static int amd_pstate_set_driver(int mode_idx) 1234 { 1235 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1236 cppc_state = mode_idx; 1237 if (cppc_state == AMD_PSTATE_DISABLE) 1238 pr_info("driver is explicitly disabled\n"); 1239 1240 if (cppc_state == AMD_PSTATE_ACTIVE) 1241 current_pstate_driver = &amd_pstate_epp_driver; 1242 1243 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1244 current_pstate_driver = &amd_pstate_driver; 1245 1246 return 0; 1247 } 1248 1249 return -EINVAL; 1250 } 1251 1252 static int amd_pstate_register_driver(int mode) 1253 { 1254 int ret; 1255 1256 ret = amd_pstate_set_driver(mode); 1257 if (ret) 1258 return ret; 1259 1260 cppc_state = mode; 1261 1262 /* at least one CPU supports CPB */ 1263 current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); 1264 1265 ret = cpufreq_register_driver(current_pstate_driver); 1266 if (ret) { 1267 amd_pstate_driver_cleanup(); 1268 return ret; 1269 } 1270 1271 /* Enable ITMT support once all CPUs have initialized their asym priorities. */ 1272 if (amd_pstate_prefcore) 1273 sched_set_itmt_support(); 1274 1275 return 0; 1276 } 1277 1278 static int amd_pstate_unregister_driver(int dummy) 1279 { 1280 cpufreq_unregister_driver(current_pstate_driver); 1281 amd_pstate_driver_cleanup(); 1282 return 0; 1283 } 1284 1285 static int amd_pstate_change_mode_without_dvr_change(int mode) 1286 { 1287 int cpu = 0; 1288 1289 cppc_state = mode; 1290 1291 if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 1292 return 0; 1293 1294 for_each_online_cpu(cpu) { 1295 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 1296 } 1297 1298 return 0; 1299 } 1300 1301 static int amd_pstate_change_driver_mode(int mode) 1302 { 1303 int ret; 1304 1305 ret = amd_pstate_unregister_driver(0); 1306 if (ret) 1307 return ret; 1308 1309 ret = amd_pstate_register_driver(mode); 1310 if (ret) 1311 return ret; 1312 1313 return 0; 1314 } 1315 1316 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 1317 [AMD_PSTATE_DISABLE] = { 1318 [AMD_PSTATE_DISABLE] = NULL, 1319 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 1320 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 1321 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 1322 }, 1323 [AMD_PSTATE_PASSIVE] = { 1324 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1325 [AMD_PSTATE_PASSIVE] = NULL, 1326 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1327 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 1328 }, 1329 [AMD_PSTATE_ACTIVE] = { 1330 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1331 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 1332 [AMD_PSTATE_ACTIVE] = NULL, 1333 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 1334 }, 1335 [AMD_PSTATE_GUIDED] = { 1336 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1337 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 1338 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1339 [AMD_PSTATE_GUIDED] = NULL, 1340 }, 1341 }; 1342 1343 static ssize_t amd_pstate_show_status(char *buf) 1344 { 1345 if (!current_pstate_driver) 1346 return sysfs_emit(buf, "disable\n"); 1347 1348 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 1349 } 1350 1351 int amd_pstate_get_status(void) 1352 { 1353 return cppc_state; 1354 } 1355 EXPORT_SYMBOL_GPL(amd_pstate_get_status); 1356 1357 int amd_pstate_update_status(const char *buf, size_t size) 1358 { 1359 int mode_idx; 1360 1361 if (size > strlen("passive") || size < strlen("active")) 1362 return -EINVAL; 1363 1364 mode_idx = get_mode_idx_from_str(buf, size); 1365 if (mode_idx < 0) 1366 return mode_idx; 1367 1368 if (mode_state_machine[cppc_state][mode_idx]) { 1369 guard(mutex)(&amd_pstate_driver_lock); 1370 return mode_state_machine[cppc_state][mode_idx](mode_idx); 1371 } 1372 1373 return 0; 1374 } 1375 EXPORT_SYMBOL_GPL(amd_pstate_update_status); 1376 1377 static ssize_t status_show(struct device *dev, 1378 struct device_attribute *attr, char *buf) 1379 { 1380 1381 guard(mutex)(&amd_pstate_driver_lock); 1382 1383 return amd_pstate_show_status(buf); 1384 } 1385 1386 static ssize_t status_store(struct device *a, struct device_attribute *b, 1387 const char *buf, size_t count) 1388 { 1389 char *p = memchr(buf, '\n', count); 1390 int ret; 1391 1392 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1393 1394 return ret < 0 ? ret : count; 1395 } 1396 1397 static ssize_t prefcore_show(struct device *dev, 1398 struct device_attribute *attr, char *buf) 1399 { 1400 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); 1401 } 1402 1403 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1404 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1405 1406 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1407 cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); 1408 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); 1409 cpufreq_freq_attr_rw(energy_performance_preference); 1410 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1411 static DEVICE_ATTR_RW(status); 1412 static DEVICE_ATTR_RO(prefcore); 1413 1414 static struct freq_attr *amd_pstate_attr[] = { 1415 &amd_pstate_max_freq, 1416 &amd_pstate_lowest_nonlinear_freq, 1417 &amd_pstate_highest_perf, 1418 &amd_pstate_prefcore_ranking, 1419 &amd_pstate_hw_prefcore, 1420 NULL, 1421 }; 1422 1423 static struct freq_attr *amd_pstate_epp_attr[] = { 1424 &amd_pstate_max_freq, 1425 &amd_pstate_lowest_nonlinear_freq, 1426 &amd_pstate_highest_perf, 1427 &amd_pstate_prefcore_ranking, 1428 &amd_pstate_hw_prefcore, 1429 &energy_performance_preference, 1430 &energy_performance_available_preferences, 1431 NULL, 1432 }; 1433 1434 static struct attribute *pstate_global_attributes[] = { 1435 &dev_attr_status.attr, 1436 &dev_attr_prefcore.attr, 1437 NULL 1438 }; 1439 1440 static const struct attribute_group amd_pstate_global_attr_group = { 1441 .name = "amd_pstate", 1442 .attrs = pstate_global_attributes, 1443 }; 1444 1445 static bool amd_pstate_acpi_pm_profile_server(void) 1446 { 1447 switch (acpi_gbl_FADT.preferred_profile) { 1448 case PM_ENTERPRISE_SERVER: 1449 case PM_SOHO_SERVER: 1450 case PM_PERFORMANCE_SERVER: 1451 return true; 1452 } 1453 return false; 1454 } 1455 1456 static bool amd_pstate_acpi_pm_profile_undefined(void) 1457 { 1458 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) 1459 return true; 1460 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) 1461 return true; 1462 return false; 1463 } 1464 1465 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1466 { 1467 struct amd_cpudata *cpudata; 1468 union perf_cached perf; 1469 struct device *dev; 1470 int ret; 1471 1472 /* 1473 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1474 * which is ideal for initialization process. 1475 */ 1476 amd_perf_ctl_reset(policy->cpu); 1477 dev = get_cpu_device(policy->cpu); 1478 if (!dev) 1479 return -ENODEV; 1480 1481 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 1482 if (!cpudata) 1483 return -ENOMEM; 1484 1485 cpudata->cpu = policy->cpu; 1486 1487 ret = amd_pstate_init_perf(cpudata); 1488 if (ret) 1489 goto free_cpudata1; 1490 1491 amd_pstate_init_prefcore(cpudata); 1492 1493 ret = amd_pstate_init_freq(cpudata); 1494 if (ret) 1495 goto free_cpudata1; 1496 1497 ret = amd_pstate_init_boost_support(cpudata); 1498 if (ret) 1499 goto free_cpudata1; 1500 1501 perf = READ_ONCE(cpudata->perf); 1502 1503 policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, 1504 cpudata->nominal_freq, 1505 perf.lowest_perf); 1506 policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, 1507 cpudata->nominal_freq, 1508 perf.highest_perf); 1509 policy->driver_data = cpudata; 1510 1511 ret = amd_pstate_cppc_enable(policy); 1512 if (ret) 1513 goto free_cpudata1; 1514 1515 /* It will be updated by governor */ 1516 policy->cur = policy->cpuinfo.min_freq; 1517 1518 1519 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1520 1521 /* 1522 * Set the policy to provide a valid fallback value in case 1523 * the default cpufreq governor is neither powersave nor performance. 1524 */ 1525 if (amd_pstate_acpi_pm_profile_server() || 1526 amd_pstate_acpi_pm_profile_undefined()) { 1527 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 1528 cpudata->epp_default = amd_pstate_get_epp(cpudata); 1529 } else { 1530 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1531 cpudata->epp_default = AMD_CPPC_EPP_BALANCE_PERFORMANCE; 1532 } 1533 1534 ret = amd_pstate_set_epp(policy, cpudata->epp_default); 1535 if (ret) 1536 return ret; 1537 1538 current_pstate_driver->adjust_perf = NULL; 1539 1540 return 0; 1541 1542 free_cpudata1: 1543 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1544 kfree(cpudata); 1545 return ret; 1546 } 1547 1548 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1549 { 1550 struct amd_cpudata *cpudata = policy->driver_data; 1551 1552 if (cpudata) { 1553 union perf_cached perf = READ_ONCE(cpudata->perf); 1554 1555 /* Reset CPPC_REQ MSR to the BIOS value */ 1556 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1557 1558 kfree(cpudata); 1559 policy->driver_data = NULL; 1560 } 1561 1562 pr_debug("CPU %d exiting\n", policy->cpu); 1563 } 1564 1565 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy, bool policy_change) 1566 { 1567 struct amd_cpudata *cpudata = policy->driver_data; 1568 union perf_cached perf; 1569 u8 epp; 1570 1571 if (policy_change || 1572 policy->min != cpudata->min_limit_freq || 1573 policy->max != cpudata->max_limit_freq) 1574 amd_pstate_update_min_max_limit(policy); 1575 1576 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1577 epp = 0; 1578 else 1579 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 1580 1581 perf = READ_ONCE(cpudata->perf); 1582 1583 return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U, 1584 perf.max_limit_perf, epp, false); 1585 } 1586 1587 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 1588 { 1589 struct amd_cpudata *cpudata = policy->driver_data; 1590 int ret; 1591 1592 if (!policy->cpuinfo.max_freq) 1593 return -ENODEV; 1594 1595 cpudata->policy = policy->policy; 1596 1597 ret = amd_pstate_epp_update_limit(policy, true); 1598 if (ret) 1599 return ret; 1600 1601 /* 1602 * policy->cur is never updated with the amd_pstate_epp driver, but it 1603 * is used as a stale frequency value. So, keep it within limits. 1604 */ 1605 policy->cur = policy->min; 1606 1607 return 0; 1608 } 1609 1610 static int amd_pstate_cpu_online(struct cpufreq_policy *policy) 1611 { 1612 return amd_pstate_cppc_enable(policy); 1613 } 1614 1615 static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) 1616 { 1617 struct amd_cpudata *cpudata = policy->driver_data; 1618 union perf_cached perf = READ_ONCE(cpudata->perf); 1619 1620 /* 1621 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 1622 * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the 1623 * limits, epp and desired perf will get reset to the cached values in cpudata struct 1624 */ 1625 return amd_pstate_update_perf(policy, perf.bios_min_perf, 1626 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 1627 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 1628 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 1629 false); 1630 } 1631 1632 static int amd_pstate_suspend(struct cpufreq_policy *policy) 1633 { 1634 struct amd_cpudata *cpudata = policy->driver_data; 1635 union perf_cached perf = READ_ONCE(cpudata->perf); 1636 int ret; 1637 1638 /* 1639 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 1640 * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, 1641 * the limits, epp and desired perf will get reset to the cached values in cpudata struct 1642 */ 1643 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 1644 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 1645 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 1646 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 1647 false); 1648 if (ret) 1649 return ret; 1650 1651 /* set this flag to avoid setting core offline*/ 1652 cpudata->suspended = true; 1653 1654 return 0; 1655 } 1656 1657 static int amd_pstate_resume(struct cpufreq_policy *policy) 1658 { 1659 struct amd_cpudata *cpudata = policy->driver_data; 1660 union perf_cached perf = READ_ONCE(cpudata->perf); 1661 int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur); 1662 1663 /* Set CPPC_REQ to last sane value until the governor updates it */ 1664 return amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, 1665 0U, false); 1666 } 1667 1668 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 1669 { 1670 struct amd_cpudata *cpudata = policy->driver_data; 1671 1672 if (cpudata->suspended) { 1673 int ret; 1674 1675 /* enable amd pstate from suspend state*/ 1676 ret = amd_pstate_epp_update_limit(policy, false); 1677 if (ret) 1678 return ret; 1679 1680 cpudata->suspended = false; 1681 } 1682 1683 return 0; 1684 } 1685 1686 static struct cpufreq_driver amd_pstate_driver = { 1687 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 1688 .verify = amd_pstate_verify, 1689 .target = amd_pstate_target, 1690 .fast_switch = amd_pstate_fast_switch, 1691 .init = amd_pstate_cpu_init, 1692 .exit = amd_pstate_cpu_exit, 1693 .online = amd_pstate_cpu_online, 1694 .offline = amd_pstate_cpu_offline, 1695 .suspend = amd_pstate_suspend, 1696 .resume = amd_pstate_resume, 1697 .set_boost = amd_pstate_set_boost, 1698 .update_limits = amd_pstate_update_limits, 1699 .name = "amd-pstate", 1700 .attr = amd_pstate_attr, 1701 }; 1702 1703 static struct cpufreq_driver amd_pstate_epp_driver = { 1704 .flags = CPUFREQ_CONST_LOOPS, 1705 .verify = amd_pstate_verify, 1706 .setpolicy = amd_pstate_epp_set_policy, 1707 .init = amd_pstate_epp_cpu_init, 1708 .exit = amd_pstate_epp_cpu_exit, 1709 .offline = amd_pstate_cpu_offline, 1710 .online = amd_pstate_cpu_online, 1711 .suspend = amd_pstate_suspend, 1712 .resume = amd_pstate_epp_resume, 1713 .update_limits = amd_pstate_update_limits, 1714 .set_boost = amd_pstate_set_boost, 1715 .name = "amd-pstate-epp", 1716 .attr = amd_pstate_epp_attr, 1717 }; 1718 1719 /* 1720 * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. 1721 * show the debug message that helps to check if the CPU has CPPC support for loading issue. 1722 */ 1723 static bool amd_cppc_supported(void) 1724 { 1725 struct cpuinfo_x86 *c = &cpu_data(0); 1726 bool warn = false; 1727 1728 if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) { 1729 pr_debug_once("CPPC feature is not supported by the processor\n"); 1730 return false; 1731 } 1732 1733 /* 1734 * If the CPPC feature is disabled in the BIOS for processors 1735 * that support MSR-based CPPC, the AMD Pstate driver may not 1736 * function correctly. 1737 * 1738 * For such processors, check the CPPC flag and display a 1739 * warning message if the platform supports CPPC. 1740 * 1741 * Note: The code check below will not abort the driver 1742 * registration process because of the code is added for 1743 * debugging purposes. Besides, it may still be possible for 1744 * the driver to work using the shared-memory mechanism. 1745 */ 1746 if (!cpu_feature_enabled(X86_FEATURE_CPPC)) { 1747 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 1748 switch (c->x86_model) { 1749 case 0x60 ... 0x6F: 1750 case 0x80 ... 0xAF: 1751 warn = true; 1752 break; 1753 } 1754 } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || 1755 cpu_feature_enabled(X86_FEATURE_ZEN4)) { 1756 switch (c->x86_model) { 1757 case 0x10 ... 0x1F: 1758 case 0x40 ... 0xAF: 1759 warn = true; 1760 break; 1761 } 1762 } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) { 1763 warn = true; 1764 } 1765 } 1766 1767 if (warn) 1768 pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n" 1769 "Please enable it if your BIOS has the CPPC option.\n"); 1770 return true; 1771 } 1772 1773 static int __init amd_pstate_init(void) 1774 { 1775 struct device *dev_root; 1776 int ret; 1777 1778 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 1779 return -ENODEV; 1780 1781 /* show debug message only if CPPC is not supported */ 1782 if (!amd_cppc_supported()) 1783 return -EOPNOTSUPP; 1784 1785 /* show warning message when BIOS broken or ACPI disabled */ 1786 if (!acpi_cpc_valid()) { 1787 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 1788 return -ENODEV; 1789 } 1790 1791 /* don't keep reloading if cpufreq_driver exists */ 1792 if (cpufreq_get_current_driver()) 1793 return -EEXIST; 1794 1795 quirks = NULL; 1796 1797 /* check if this machine need CPPC quirks */ 1798 dmi_check_system(amd_pstate_quirks_table); 1799 1800 /* 1801 * determine the driver mode from the command line or kernel config. 1802 * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED. 1803 * command line options will override the kernel config settings. 1804 */ 1805 1806 if (cppc_state == AMD_PSTATE_UNDEFINED) { 1807 /* Disable on the following configs by default: 1808 * 1. Undefined platforms 1809 * 2. Server platforms with CPUs older than Family 0x1A. 1810 */ 1811 if (amd_pstate_acpi_pm_profile_undefined() || 1812 (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) { 1813 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 1814 return -ENODEV; 1815 } 1816 /* get driver mode from kernel config option [1:4] */ 1817 cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; 1818 } 1819 1820 if (cppc_state == AMD_PSTATE_DISABLE) { 1821 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 1822 return -ENODEV; 1823 } 1824 1825 /* capability check */ 1826 if (cpu_feature_enabled(X86_FEATURE_CPPC)) { 1827 pr_debug("AMD CPPC MSR based functionality is supported\n"); 1828 } else { 1829 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 1830 static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); 1831 static_call_update(amd_pstate_init_perf, shmem_init_perf); 1832 static_call_update(amd_pstate_update_perf, shmem_update_perf); 1833 static_call_update(amd_pstate_get_epp, shmem_get_epp); 1834 static_call_update(amd_pstate_set_epp, shmem_set_epp); 1835 } 1836 1837 if (amd_pstate_prefcore) { 1838 ret = amd_detect_prefcore(&amd_pstate_prefcore); 1839 if (ret) 1840 return ret; 1841 } 1842 1843 ret = amd_pstate_register_driver(cppc_state); 1844 if (ret) { 1845 pr_err("failed to register with return %d\n", ret); 1846 return ret; 1847 } 1848 1849 dev_root = bus_get_dev_root(&cpu_subsys); 1850 if (dev_root) { 1851 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 1852 put_device(dev_root); 1853 if (ret) { 1854 pr_err("sysfs attribute export failed with error %d.\n", ret); 1855 goto global_attr_free; 1856 } 1857 } 1858 1859 return ret; 1860 1861 global_attr_free: 1862 cpufreq_unregister_driver(current_pstate_driver); 1863 return ret; 1864 } 1865 device_initcall(amd_pstate_init); 1866 1867 static int __init amd_pstate_param(char *str) 1868 { 1869 size_t size; 1870 int mode_idx; 1871 1872 if (!str) 1873 return -EINVAL; 1874 1875 size = strlen(str); 1876 mode_idx = get_mode_idx_from_str(str, size); 1877 1878 return amd_pstate_set_driver(mode_idx); 1879 } 1880 1881 static int __init amd_prefcore_param(char *str) 1882 { 1883 if (!strcmp(str, "disable")) 1884 amd_pstate_prefcore = false; 1885 1886 return 0; 1887 } 1888 1889 early_param("amd_pstate", amd_pstate_param); 1890 early_param("amd_prefcore", amd_prefcore_param); 1891 1892 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 1893 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 1894