1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/bitfield.h> 26 #include <linux/kernel.h> 27 #include <linux/module.h> 28 #include <linux/init.h> 29 #include <linux/smp.h> 30 #include <linux/sched.h> 31 #include <linux/cpufreq.h> 32 #include <linux/compiler.h> 33 #include <linux/dmi.h> 34 #include <linux/slab.h> 35 #include <linux/acpi.h> 36 #include <linux/io.h> 37 #include <linux/delay.h> 38 #include <linux/uaccess.h> 39 #include <linux/power_supply.h> 40 #include <linux/static_call.h> 41 #include <linux/topology.h> 42 43 #include <acpi/processor.h> 44 #include <acpi/cppc_acpi.h> 45 46 #include <asm/msr.h> 47 #include <asm/processor.h> 48 #include <asm/cpufeature.h> 49 #include <asm/cpu_device_id.h> 50 51 #include "amd-pstate.h" 52 #include "amd-pstate-trace.h" 53 54 #define AMD_PSTATE_TRANSITION_LATENCY 20000 55 #define AMD_PSTATE_TRANSITION_DELAY 1000 56 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600 57 58 #define AMD_CPPC_EPP_PERFORMANCE 0x00 59 #define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 60 #define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF 61 #define AMD_CPPC_EPP_POWERSAVE 0xFF 62 63 static const char * const amd_pstate_mode_string[] = { 64 [AMD_PSTATE_UNDEFINED] = "undefined", 65 [AMD_PSTATE_DISABLE] = "disable", 66 [AMD_PSTATE_PASSIVE] = "passive", 67 [AMD_PSTATE_ACTIVE] = "active", 68 [AMD_PSTATE_GUIDED] = "guided", 69 }; 70 static_assert(ARRAY_SIZE(amd_pstate_mode_string) == AMD_PSTATE_MAX); 71 72 const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode) 73 { 74 if (mode < AMD_PSTATE_UNDEFINED || mode >= AMD_PSTATE_MAX) 75 mode = AMD_PSTATE_UNDEFINED; 76 return amd_pstate_mode_string[mode]; 77 } 78 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_mode_string); 79 80 struct quirk_entry { 81 u32 nominal_freq; 82 u32 lowest_freq; 83 }; 84 85 static struct cpufreq_driver *current_pstate_driver; 86 static struct cpufreq_driver amd_pstate_driver; 87 static struct cpufreq_driver amd_pstate_epp_driver; 88 static int cppc_state = AMD_PSTATE_UNDEFINED; 89 static bool amd_pstate_prefcore = true; 90 static struct quirk_entry *quirks; 91 92 /* 93 * AMD Energy Preference Performance (EPP) 94 * The EPP is used in the CCLK DPM controller to drive 95 * the frequency that a core is going to operate during 96 * short periods of activity. EPP values will be utilized for 97 * different OS profiles (balanced, performance, power savings) 98 * display strings corresponding to EPP index in the 99 * energy_perf_strings[] 100 * index String 101 *------------------------------------- 102 * 0 default 103 * 1 performance 104 * 2 balance_performance 105 * 3 balance_power 106 * 4 power 107 * 5 custom (for raw EPP values) 108 * 6 dynamic (platform profile driven selection) 109 */ 110 enum energy_perf_value_index { 111 EPP_INDEX_DEFAULT = 0, 112 EPP_INDEX_PERFORMANCE, 113 EPP_INDEX_BALANCE_PERFORMANCE, 114 EPP_INDEX_BALANCE_POWERSAVE, 115 EPP_INDEX_POWERSAVE, 116 EPP_INDEX_CUSTOM, 117 EPP_INDEX_DYNAMIC, 118 EPP_INDEX_MAX, 119 }; 120 121 static const char * const energy_perf_strings[] = { 122 [EPP_INDEX_DEFAULT] = "default", 123 [EPP_INDEX_PERFORMANCE] = "performance", 124 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 125 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 126 [EPP_INDEX_POWERSAVE] = "power", 127 [EPP_INDEX_CUSTOM] = "custom", 128 [EPP_INDEX_DYNAMIC] = "dynamic", 129 }; 130 static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); 131 132 static unsigned int epp_values[] = { 133 [EPP_INDEX_DEFAULT] = 0, 134 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 135 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 136 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 137 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 138 }; 139 static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX - 2); 140 141 typedef int (*cppc_mode_transition_fn)(int); 142 143 static struct quirk_entry quirk_amd_7k62 = { 144 .nominal_freq = 2600, 145 .lowest_freq = 550, 146 }; 147 148 static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) 149 { 150 u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); 151 152 return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf); 153 } 154 155 static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val) 156 { 157 return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val, 158 perf.nominal_perf); 159 } 160 161 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) 162 { 163 /** 164 * match the broken bios for family 17h processor support CPPC V2 165 * broken BIOS lack of nominal_freq and lowest_freq capabilities 166 * definition in ACPI tables 167 */ 168 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 169 quirks = dmi->driver_data; 170 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); 171 return 1; 172 } 173 174 return 0; 175 } 176 177 static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { 178 { 179 .callback = dmi_matched_7k62_bios_bug, 180 .ident = "AMD EPYC 7K62", 181 .matches = { 182 DMI_MATCH(DMI_BIOS_VERSION, "5.14"), 183 DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), 184 }, 185 .driver_data = &quirk_amd_7k62, 186 }, 187 {} 188 }; 189 MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); 190 191 static inline int get_mode_idx_from_str(const char *str, size_t size) 192 { 193 int i; 194 195 for (i = 0; i < AMD_PSTATE_MAX; i++) { 196 if (!strncmp(str, amd_pstate_mode_string[i], size)) 197 return i; 198 } 199 return -EINVAL; 200 } 201 202 static DEFINE_MUTEX(amd_pstate_driver_lock); 203 204 static int msr_get_epp(struct amd_cpudata *cpudata) 205 { 206 u64 value; 207 int ret; 208 209 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 210 if (ret < 0) { 211 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 212 return ret; 213 } 214 215 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value); 216 } 217 218 DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); 219 220 static inline int amd_pstate_get_epp(struct amd_cpudata *cpudata) 221 { 222 return static_call(amd_pstate_get_epp)(cpudata); 223 } 224 225 static int shmem_get_epp(struct amd_cpudata *cpudata) 226 { 227 u64 epp; 228 int ret; 229 230 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 231 if (ret < 0) { 232 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 233 return ret; 234 } 235 236 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp); 237 } 238 239 static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, 240 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 241 { 242 struct amd_cpudata *cpudata = policy->driver_data; 243 u64 value, prev; 244 245 value = prev = READ_ONCE(cpudata->cppc_req_cached); 246 247 FIELD_MODIFY(AMD_CPPC_MAX_PERF_MASK, &value, max_perf); 248 FIELD_MODIFY(AMD_CPPC_DES_PERF_MASK, &value, des_perf); 249 FIELD_MODIFY(AMD_CPPC_MIN_PERF_MASK, &value, min_perf); 250 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 251 252 if (trace_amd_pstate_epp_perf_enabled()) { 253 union perf_cached perf = READ_ONCE(cpudata->perf); 254 255 trace_call__amd_pstate_epp_perf(cpudata->cpu, 256 perf.highest_perf, 257 epp, 258 min_perf, 259 max_perf, 260 policy->boost_enabled, 261 value != prev); 262 } 263 264 if (value == prev) 265 return 0; 266 267 if (fast_switch) { 268 wrmsrq(MSR_AMD_CPPC_REQ, value); 269 } else { 270 int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 271 272 if (ret) 273 return ret; 274 } 275 276 WRITE_ONCE(cpudata->cppc_req_cached, value); 277 278 return 0; 279 } 280 281 DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); 282 283 static inline int amd_pstate_update_perf(struct cpufreq_policy *policy, 284 u8 min_perf, u8 des_perf, 285 u8 max_perf, u8 epp, 286 bool fast_switch) 287 { 288 return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf, 289 max_perf, epp, fast_switch); 290 } 291 292 static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) 293 { 294 struct amd_cpudata *cpudata = policy->driver_data; 295 u64 value, prev; 296 int ret; 297 298 value = prev = READ_ONCE(cpudata->cppc_req_cached); 299 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 300 301 if (trace_amd_pstate_epp_perf_enabled()) { 302 union perf_cached perf = cpudata->perf; 303 304 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 305 epp, 306 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 307 cpudata->cppc_req_cached), 308 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 309 cpudata->cppc_req_cached), 310 policy->boost_enabled, 311 value != prev); 312 } 313 314 if (value == prev) 315 return 0; 316 317 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 318 if (ret) { 319 pr_err("failed to set energy perf value (%d)\n", ret); 320 return ret; 321 } 322 323 /* update both so that msr_update_perf() can effectively check */ 324 WRITE_ONCE(cpudata->cppc_req_cached, value); 325 326 return ret; 327 } 328 329 DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); 330 331 static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp) 332 { 333 return static_call(amd_pstate_set_epp)(policy, epp); 334 } 335 336 static int amd_pstate_set_floor_perf(struct cpufreq_policy *policy, u8 perf) 337 { 338 struct amd_cpudata *cpudata = policy->driver_data; 339 u64 value, prev; 340 bool changed; 341 int ret; 342 343 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) 344 return 0; 345 346 value = prev = READ_ONCE(cpudata->cppc_req2_cached); 347 FIELD_MODIFY(AMD_CPPC_FLOOR_PERF_MASK, &value, perf); 348 349 changed = value != prev; 350 if (!changed) { 351 ret = 0; 352 goto out_trace; 353 } 354 355 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, value); 356 if (ret) { 357 changed = false; 358 pr_err("failed to set CPPC REQ2 value. Error (%d)\n", ret); 359 goto out_trace; 360 } 361 362 WRITE_ONCE(cpudata->cppc_req2_cached, value); 363 364 out_trace: 365 if (trace_amd_pstate_cppc_req2_enabled()) 366 trace_call__amd_pstate_cppc_req2(cpudata->cpu, perf, changed, 367 ret); 368 return ret; 369 } 370 371 static int amd_pstate_init_floor_perf(struct cpufreq_policy *policy) 372 { 373 struct amd_cpudata *cpudata = policy->driver_data; 374 u8 floor_perf; 375 u64 value; 376 int ret; 377 378 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) 379 return 0; 380 381 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, &value); 382 if (ret) { 383 pr_err("failed to read CPPC REQ2 value. Error (%d)\n", ret); 384 return ret; 385 } 386 387 WRITE_ONCE(cpudata->cppc_req2_cached, value); 388 floor_perf = FIELD_GET(AMD_CPPC_FLOOR_PERF_MASK, 389 cpudata->cppc_req2_cached); 390 391 /* Set a sane value for floor_perf if the default value is invalid */ 392 if (floor_perf < cpudata->perf.lowest_perf) { 393 floor_perf = cpudata->perf.nominal_perf; 394 ret = amd_pstate_set_floor_perf(policy, floor_perf); 395 if (ret) 396 return ret; 397 } 398 399 400 cpudata->bios_floor_perf = floor_perf; 401 cpudata->floor_freq = perf_to_freq(cpudata->perf, cpudata->nominal_freq, 402 floor_perf); 403 return 0; 404 } 405 406 static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) 407 { 408 struct amd_cpudata *cpudata = policy->driver_data; 409 struct cppc_perf_ctrls perf_ctrls; 410 u8 epp_cached; 411 u64 value; 412 int ret; 413 414 415 epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 416 if (trace_amd_pstate_epp_perf_enabled()) { 417 union perf_cached perf = cpudata->perf; 418 419 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 420 epp, 421 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 422 cpudata->cppc_req_cached), 423 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 424 cpudata->cppc_req_cached), 425 policy->boost_enabled, 426 epp != epp_cached); 427 } 428 429 if (epp == epp_cached) 430 return 0; 431 432 perf_ctrls.energy_perf = epp; 433 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 434 if (ret) { 435 pr_debug("failed to set energy perf value (%d)\n", ret); 436 return ret; 437 } 438 439 value = READ_ONCE(cpudata->cppc_req_cached); 440 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 441 WRITE_ONCE(cpudata->cppc_req_cached, value); 442 443 return ret; 444 } 445 446 static inline int msr_cppc_enable(struct cpufreq_policy *policy) 447 { 448 return wrmsrq_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); 449 } 450 451 static int shmem_cppc_enable(struct cpufreq_policy *policy) 452 { 453 return cppc_set_enable(policy->cpu, 1); 454 } 455 456 DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); 457 458 static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) 459 { 460 return static_call(amd_pstate_cppc_enable)(policy); 461 } 462 463 static int msr_init_perf(struct amd_cpudata *cpudata) 464 { 465 union perf_cached perf = READ_ONCE(cpudata->perf); 466 u64 cap1, numerator, cppc_req; 467 468 int ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 469 &cap1); 470 if (ret) 471 return ret; 472 473 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 474 if (ret) 475 return ret; 476 477 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); 478 if (ret) 479 return ret; 480 481 WRITE_ONCE(cpudata->cppc_req_cached, cppc_req); 482 483 perf.highest_perf = numerator; 484 perf.max_limit_perf = numerator; 485 perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 486 perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); 487 perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); 488 perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 489 perf.bios_min_perf = FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cppc_req); 490 WRITE_ONCE(cpudata->perf, perf); 491 WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1)); 492 WRITE_ONCE(cpudata->floor_perf_cnt, FIELD_GET(AMD_CPPC_FLOOR_PERF_CNT_MASK, cap1)); 493 494 return 0; 495 } 496 497 static int shmem_init_perf(struct amd_cpudata *cpudata) 498 { 499 struct cppc_perf_caps cppc_perf; 500 union perf_cached perf = READ_ONCE(cpudata->perf); 501 u64 numerator; 502 bool auto_sel; 503 504 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 505 if (ret) 506 return ret; 507 508 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 509 if (ret) 510 return ret; 511 512 perf.highest_perf = numerator; 513 perf.max_limit_perf = numerator; 514 perf.min_limit_perf = cppc_perf.lowest_perf; 515 perf.nominal_perf = cppc_perf.nominal_perf; 516 perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 517 perf.lowest_perf = cppc_perf.lowest_perf; 518 WRITE_ONCE(cpudata->perf, perf); 519 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); 520 521 ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel); 522 if (ret) { 523 pr_warn("failed to get auto_sel, ret: %d\n", ret); 524 return 0; 525 } 526 527 ret = cppc_set_auto_sel(cpudata->cpu, 528 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 529 530 if (ret) 531 pr_warn("failed to set auto_sel, ret: %d\n", ret); 532 533 return ret; 534 } 535 536 DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf); 537 538 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 539 { 540 return static_call(amd_pstate_init_perf)(cpudata); 541 } 542 543 static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf, 544 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 545 { 546 struct amd_cpudata *cpudata = policy->driver_data; 547 struct cppc_perf_ctrls perf_ctrls; 548 u64 value, prev; 549 int ret; 550 551 if (cppc_state == AMD_PSTATE_ACTIVE) { 552 int ret = shmem_set_epp(policy, epp); 553 554 if (ret) 555 return ret; 556 } 557 558 value = prev = READ_ONCE(cpudata->cppc_req_cached); 559 560 FIELD_MODIFY(AMD_CPPC_MAX_PERF_MASK, &value, max_perf); 561 FIELD_MODIFY(AMD_CPPC_DES_PERF_MASK, &value, des_perf); 562 FIELD_MODIFY(AMD_CPPC_MIN_PERF_MASK, &value, min_perf); 563 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 564 565 if (trace_amd_pstate_epp_perf_enabled()) { 566 union perf_cached perf = READ_ONCE(cpudata->perf); 567 568 trace_call__amd_pstate_epp_perf(cpudata->cpu, 569 perf.highest_perf, 570 epp, 571 min_perf, 572 max_perf, 573 policy->boost_enabled, 574 value != prev); 575 } 576 577 if (value == prev) 578 return 0; 579 580 perf_ctrls.max_perf = max_perf; 581 perf_ctrls.min_perf = min_perf; 582 perf_ctrls.desired_perf = des_perf; 583 584 ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); 585 if (ret) 586 return ret; 587 588 WRITE_ONCE(cpudata->cppc_req_cached, value); 589 590 return 0; 591 } 592 593 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 594 { 595 u64 aperf, mperf, tsc; 596 unsigned long flags; 597 598 local_irq_save(flags); 599 rdmsrq(MSR_IA32_APERF, aperf); 600 rdmsrq(MSR_IA32_MPERF, mperf); 601 tsc = rdtsc(); 602 603 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 604 local_irq_restore(flags); 605 return false; 606 } 607 608 local_irq_restore(flags); 609 610 cpudata->cur.aperf = aperf; 611 cpudata->cur.mperf = mperf; 612 cpudata->cur.tsc = tsc; 613 cpudata->cur.aperf -= cpudata->prev.aperf; 614 cpudata->cur.mperf -= cpudata->prev.mperf; 615 cpudata->cur.tsc -= cpudata->prev.tsc; 616 617 cpudata->prev.aperf = aperf; 618 cpudata->prev.mperf = mperf; 619 cpudata->prev.tsc = tsc; 620 621 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 622 623 return true; 624 } 625 626 static void amd_pstate_update(struct cpufreq_policy *policy, u8 min_perf, 627 u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) 628 { 629 struct amd_cpudata *cpudata = policy->driver_data; 630 union perf_cached perf = READ_ONCE(cpudata->perf); 631 632 /* limit the max perf when core performance boost feature is disabled */ 633 if (!cpudata->boost_supported) 634 max_perf = min_t(u8, perf.nominal_perf, max_perf); 635 636 des_perf = clamp_t(u8, des_perf, min_perf, max_perf); 637 638 policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); 639 640 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 641 min_perf = des_perf; 642 des_perf = 0; 643 } 644 645 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 646 trace_call__amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 647 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 648 cpudata->cpu, fast_switch); 649 } 650 651 amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch); 652 } 653 654 static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) 655 { 656 /* 657 * Initialize lower frequency limit (i.e.policy->min) with 658 * lowest_nonlinear_frequency or the min frequency (if) specified in BIOS, 659 * Override the initial value set by cpufreq core and amd-pstate qos_requests. 660 */ 661 if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { 662 struct cpufreq_policy *policy __free(put_cpufreq_policy) = 663 cpufreq_cpu_get(policy_data->cpu); 664 struct amd_cpudata *cpudata; 665 union perf_cached perf; 666 667 if (!policy) 668 return -EINVAL; 669 670 cpudata = policy->driver_data; 671 perf = READ_ONCE(cpudata->perf); 672 673 if (perf.bios_min_perf) 674 policy_data->min = perf_to_freq(perf, cpudata->nominal_freq, 675 perf.bios_min_perf); 676 else 677 policy_data->min = cpudata->lowest_nonlinear_freq; 678 } 679 680 cpufreq_verify_within_cpu_limits(policy_data); 681 682 return 0; 683 } 684 685 static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) 686 { 687 struct amd_cpudata *cpudata = policy->driver_data; 688 union perf_cached perf = READ_ONCE(cpudata->perf); 689 690 perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); 691 WRITE_ONCE(cpudata->max_limit_freq, policy->max); 692 693 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 694 u8 min_limit_perf = perf.bios_min_perf ?: perf.nominal_perf; 695 u32 min_limit_freq; 696 697 /* 698 * For performance policy, set MinPerf to nominal_perf / bios_min_perf 699 * rather than highest_perf or lowest_nonlinear_perf. 700 * 701 * Per commit 0c411b39e4f4c, using highest_perf was observed 702 * to cause frequency throttling on power-limited platforms, leading to 703 * performance regressions. Using lowest_nonlinear_perf would limit 704 * performance too much for HPC workloads requiring high frequency 705 * operation and minimal wakeup latency from idle states. 706 * 707 * nominal_perf therefore provides a balanced default by avoiding 708 * throttling while still maintaining enough performance for HPC 709 * workloads when bios_min_perf is not available. 710 * 711 * When bios_min_perf is available, users have profiled their workloads 712 * to understand the best idling frequency. Use that instead. 713 */ 714 min_limit_perf = min(min_limit_perf, perf.max_limit_perf); 715 min_limit_freq = perf_to_freq(perf, cpudata->nominal_freq, min_limit_perf); 716 perf.min_limit_perf = min_limit_perf; 717 718 WRITE_ONCE(cpudata->min_limit_freq, min(min_limit_freq, cpudata->max_limit_freq)); 719 } else { 720 perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); 721 WRITE_ONCE(cpudata->min_limit_freq, policy->min); 722 } 723 724 WRITE_ONCE(cpudata->perf, perf); 725 } 726 727 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 728 unsigned int target_freq, bool fast_switch) 729 { 730 struct cpufreq_freqs freqs; 731 struct amd_cpudata *cpudata; 732 union perf_cached perf; 733 u8 des_perf; 734 735 cpudata = policy->driver_data; 736 737 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 738 amd_pstate_update_min_max_limit(policy); 739 740 perf = READ_ONCE(cpudata->perf); 741 742 freqs.old = policy->cur; 743 freqs.new = target_freq; 744 745 des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq); 746 747 WARN_ON(fast_switch && !policy->fast_switch_enabled); 748 /* 749 * If fast_switch is desired, then there aren't any registered 750 * transition notifiers. See comment for 751 * cpufreq_enable_fast_switch(). 752 */ 753 if (!fast_switch) 754 cpufreq_freq_transition_begin(policy, &freqs); 755 756 amd_pstate_update(policy, perf.min_limit_perf, des_perf, 757 perf.max_limit_perf, fast_switch, 758 policy->governor->flags); 759 760 if (!fast_switch) 761 cpufreq_freq_transition_end(policy, &freqs, false); 762 763 return 0; 764 } 765 766 static int amd_pstate_target(struct cpufreq_policy *policy, 767 unsigned int target_freq, 768 unsigned int relation) 769 { 770 return amd_pstate_update_freq(policy, target_freq, false); 771 } 772 773 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 774 unsigned int target_freq) 775 { 776 if (!amd_pstate_update_freq(policy, target_freq, true)) 777 return target_freq; 778 return policy->cur; 779 } 780 781 static void amd_pstate_adjust_perf(struct cpufreq_policy *policy, 782 unsigned long _min_perf, 783 unsigned long target_perf, 784 unsigned long _max_perf, 785 unsigned long capacity) 786 { 787 u8 max_perf, min_perf, des_perf, cap_perf; 788 struct amd_cpudata *cpudata; 789 union perf_cached perf; 790 791 if (!policy) 792 return; 793 794 cpudata = policy->driver_data; 795 796 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 797 amd_pstate_update_min_max_limit(policy); 798 799 perf = READ_ONCE(cpudata->perf); 800 cap_perf = perf.highest_perf; 801 802 des_perf = cap_perf; 803 if (target_perf < capacity) 804 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 805 806 if (_min_perf < capacity) 807 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 808 else 809 min_perf = cap_perf; 810 811 if (min_perf < perf.min_limit_perf) 812 min_perf = perf.min_limit_perf; 813 814 max_perf = perf.max_limit_perf; 815 if (max_perf < min_perf) 816 max_perf = min_perf; 817 818 amd_pstate_update(policy, min_perf, des_perf, max_perf, true, 819 policy->governor->flags); 820 } 821 822 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) 823 { 824 struct amd_cpudata *cpudata = policy->driver_data; 825 u32 nominal_freq; 826 int ret = 0; 827 828 nominal_freq = READ_ONCE(cpudata->nominal_freq); 829 830 if (on) 831 policy->cpuinfo.max_freq = cpudata->max_freq; 832 else if (policy->cpuinfo.max_freq > nominal_freq) 833 policy->cpuinfo.max_freq = nominal_freq; 834 835 if (cppc_state == AMD_PSTATE_PASSIVE) { 836 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); 837 if (ret < 0) 838 pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu); 839 } 840 841 return ret < 0 ? ret : 0; 842 } 843 844 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 845 { 846 struct amd_cpudata *cpudata = policy->driver_data; 847 int ret; 848 849 if (!cpudata->boost_supported) { 850 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 851 return -EOPNOTSUPP; 852 } 853 854 ret = amd_pstate_cpu_boost_update(policy, state); 855 refresh_frequency_limits(policy); 856 857 return ret; 858 } 859 860 static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) 861 { 862 u64 boost_val; 863 int ret = -1; 864 865 /* 866 * If platform has no CPB support or disable it, initialize current driver 867 * boost_enabled state to be false, it is not an error for cpufreq core to handle. 868 */ 869 if (!cpu_feature_enabled(X86_FEATURE_CPB)) { 870 pr_debug_once("Boost CPB capabilities not present in the processor\n"); 871 ret = 0; 872 goto exit_err; 873 } 874 875 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); 876 if (ret) { 877 pr_err_once("failed to read initial CPU boost state!\n"); 878 ret = -EIO; 879 goto exit_err; 880 } 881 882 if (!(boost_val & MSR_K7_HWCR_CPB_DIS)) 883 cpudata->boost_supported = true; 884 885 return 0; 886 887 exit_err: 888 cpudata->boost_supported = false; 889 return ret; 890 } 891 892 static void amd_perf_ctl_reset(unsigned int cpu) 893 { 894 wrmsrq_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 895 } 896 897 #define CPPC_MAX_PERF U8_MAX 898 899 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) 900 { 901 /* user disabled or not detected */ 902 if (!amd_pstate_prefcore) 903 return; 904 905 /* should use amd-hfi instead */ 906 if (cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS) && 907 IS_ENABLED(CONFIG_AMD_HFI)) { 908 amd_pstate_prefcore = false; 909 return; 910 } 911 912 cpudata->hw_prefcore = true; 913 914 /* Priorities must be initialized before ITMT support can be toggled on. */ 915 sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); 916 } 917 918 static void amd_pstate_update_limits(struct cpufreq_policy *policy) 919 { 920 struct amd_cpudata *cpudata; 921 u32 prev_high = 0, cur_high = 0; 922 bool highest_perf_changed = false; 923 unsigned int cpu = policy->cpu; 924 925 if (!amd_pstate_prefcore) 926 return; 927 928 if (amd_get_highest_perf(cpu, &cur_high)) 929 return; 930 931 cpudata = policy->driver_data; 932 933 prev_high = READ_ONCE(cpudata->prefcore_ranking); 934 highest_perf_changed = (prev_high != cur_high); 935 if (highest_perf_changed) { 936 WRITE_ONCE(cpudata->prefcore_ranking, cur_high); 937 938 if (cur_high < CPPC_MAX_PERF) { 939 sched_set_itmt_core_prio((int)cur_high, cpu); 940 sched_update_asym_prefer_cpu(cpu, prev_high, cur_high); 941 } 942 } 943 } 944 945 /* 946 * Get pstate transition delay time from ACPI tables that firmware set 947 * instead of using hardcode value directly. 948 */ 949 static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) 950 { 951 int transition_delay_ns; 952 953 transition_delay_ns = cppc_get_transition_latency(cpu); 954 if (transition_delay_ns < 0) { 955 if (cpu_feature_enabled(X86_FEATURE_AMD_FAST_CPPC)) 956 return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY; 957 else 958 return AMD_PSTATE_TRANSITION_DELAY; 959 } 960 961 return transition_delay_ns / NSEC_PER_USEC; 962 } 963 964 /* 965 * Get pstate transition latency value from ACPI tables that firmware 966 * set instead of using hardcode value directly. 967 */ 968 static u32 amd_pstate_get_transition_latency(unsigned int cpu) 969 { 970 int transition_latency; 971 972 transition_latency = cppc_get_transition_latency(cpu); 973 if (transition_latency < 0) 974 return AMD_PSTATE_TRANSITION_LATENCY; 975 976 return transition_latency; 977 } 978 979 /* 980 * amd_pstate_init_freq: Initialize the nominal_freq and lowest_nonlinear_freq 981 * for the @cpudata object. 982 * 983 * Requires: all perf members of @cpudata to be initialized. 984 * 985 * Returns 0 on success, non-zero value on failure. 986 */ 987 static int amd_pstate_init_freq(struct amd_cpudata *cpudata) 988 { 989 u32 min_freq, max_freq, nominal_freq, lowest_nonlinear_freq; 990 struct cppc_perf_caps cppc_perf; 991 union perf_cached perf; 992 int ret; 993 994 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 995 if (ret) 996 return ret; 997 perf = READ_ONCE(cpudata->perf); 998 999 if (quirks && quirks->nominal_freq) 1000 nominal_freq = quirks->nominal_freq; 1001 else 1002 nominal_freq = cppc_perf.nominal_freq; 1003 nominal_freq *= 1000; 1004 1005 if (quirks && quirks->lowest_freq) { 1006 min_freq = quirks->lowest_freq; 1007 perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq); 1008 WRITE_ONCE(cpudata->perf, perf); 1009 } else 1010 min_freq = cppc_perf.lowest_freq; 1011 1012 min_freq *= 1000; 1013 1014 WRITE_ONCE(cpudata->nominal_freq, nominal_freq); 1015 1016 /* max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf */ 1017 max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); 1018 WRITE_ONCE(cpudata->max_freq, max_freq); 1019 1020 lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); 1021 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); 1022 1023 /** 1024 * Below values need to be initialized correctly, otherwise driver will fail to load 1025 * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] 1026 * Check _CPC in ACPI table objects if any values are incorrect 1027 */ 1028 if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { 1029 pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", 1030 min_freq, max_freq, nominal_freq); 1031 return -EINVAL; 1032 } 1033 1034 if (lowest_nonlinear_freq < min_freq || lowest_nonlinear_freq > nominal_freq) { 1035 pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", 1036 lowest_nonlinear_freq, min_freq, nominal_freq); 1037 return -EINVAL; 1038 } 1039 1040 if (perf.bios_min_perf) { 1041 u32 bios_min_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.bios_min_perf); 1042 1043 pr_debug("Found Requested CPU Min Frequency of %uKHz on CPU%d\n", 1044 bios_min_freq, cpudata->cpu); 1045 } 1046 1047 return 0; 1048 } 1049 1050 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 1051 { 1052 struct amd_cpudata *cpudata; 1053 union perf_cached perf; 1054 struct device *dev; 1055 int ret; 1056 1057 /* 1058 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1059 * which is ideal for initialization process. 1060 */ 1061 amd_perf_ctl_reset(policy->cpu); 1062 dev = get_cpu_device(policy->cpu); 1063 if (!dev) 1064 return -ENODEV; 1065 1066 cpudata = kzalloc_obj(*cpudata); 1067 if (!cpudata) 1068 return -ENOMEM; 1069 1070 cpudata->cpu = policy->cpu; 1071 1072 ret = amd_pstate_init_perf(cpudata); 1073 if (ret) 1074 goto free_cpudata1; 1075 1076 amd_pstate_init_prefcore(cpudata); 1077 1078 ret = amd_pstate_init_freq(cpudata); 1079 if (ret) 1080 goto free_cpudata1; 1081 1082 ret = amd_pstate_init_boost_support(cpudata); 1083 if (ret) 1084 goto free_cpudata1; 1085 1086 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); 1087 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); 1088 1089 perf = READ_ONCE(cpudata->perf); 1090 1091 policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq, 1092 perf.lowest_perf); 1093 policy->cpuinfo.max_freq = cpudata->max_freq; 1094 1095 policy->driver_data = cpudata; 1096 ret = amd_pstate_cppc_enable(policy); 1097 if (ret) 1098 goto free_cpudata1; 1099 1100 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1101 1102 /* It will be updated by governor */ 1103 policy->cur = policy->cpuinfo.min_freq; 1104 1105 if (cpu_feature_enabled(X86_FEATURE_CPPC)) 1106 policy->fast_switch_possible = true; 1107 1108 ret = amd_pstate_init_floor_perf(policy); 1109 if (ret) { 1110 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); 1111 goto free_cpudata1; 1112 } 1113 1114 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 1115 FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); 1116 if (ret < 0) { 1117 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 1118 goto free_cpudata1; 1119 } 1120 1121 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 1122 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 1123 if (ret < 0) { 1124 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 1125 goto free_cpudata2; 1126 } 1127 1128 1129 if (!current_pstate_driver->adjust_perf) 1130 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1131 1132 return 0; 1133 1134 free_cpudata2: 1135 freq_qos_remove_request(&cpudata->req[0]); 1136 free_cpudata1: 1137 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1138 kfree(cpudata); 1139 policy->driver_data = NULL; 1140 return ret; 1141 } 1142 1143 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) 1144 { 1145 struct amd_cpudata *cpudata = policy->driver_data; 1146 union perf_cached perf = READ_ONCE(cpudata->perf); 1147 1148 /* Reset CPPC_REQ MSR to the BIOS value */ 1149 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1150 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 1151 1152 freq_qos_remove_request(&cpudata->req[1]); 1153 freq_qos_remove_request(&cpudata->req[0]); 1154 policy->fast_switch_possible = false; 1155 kfree(cpudata); 1156 } 1157 1158 static int amd_pstate_get_balanced_epp(struct cpufreq_policy *policy) 1159 { 1160 struct amd_cpudata *cpudata = policy->driver_data; 1161 1162 if (power_supply_is_system_supplied()) 1163 return cpudata->epp_default_ac; 1164 else 1165 return cpudata->epp_default_dc; 1166 } 1167 1168 static int amd_pstate_power_supply_notifier(struct notifier_block *nb, 1169 unsigned long event, void *data) 1170 { 1171 struct amd_cpudata *cpudata = container_of(nb, struct amd_cpudata, power_nb); 1172 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 1173 u8 epp; 1174 int ret; 1175 1176 if (event != PSY_EVENT_PROP_CHANGED) 1177 return NOTIFY_OK; 1178 1179 /* dynamic actions are only applied while platform profile is in balanced */ 1180 if (cpudata->current_profile != PLATFORM_PROFILE_BALANCED) 1181 return 0; 1182 1183 if (!policy) 1184 return NOTIFY_OK; 1185 1186 epp = amd_pstate_get_balanced_epp(policy); 1187 1188 ret = amd_pstate_set_epp(policy, epp); 1189 if (ret) 1190 pr_warn("Failed to set CPU %d EPP %u: %d\n", cpudata->cpu, epp, ret); 1191 1192 return NOTIFY_OK; 1193 } 1194 1195 static int amd_pstate_get_epp_from_platform_profile(struct cpufreq_policy *policy, 1196 enum platform_profile_option profile) 1197 { 1198 switch (profile) { 1199 case PLATFORM_PROFILE_PERFORMANCE: 1200 return AMD_CPPC_EPP_PERFORMANCE; 1201 case PLATFORM_PROFILE_BALANCED: 1202 return amd_pstate_get_balanced_epp(policy); 1203 case PLATFORM_PROFILE_LOW_POWER: 1204 return AMD_CPPC_EPP_POWERSAVE; 1205 default: 1206 break; 1207 } 1208 1209 pr_err("Unknown Platform Profile %d\n", profile); 1210 return -EOPNOTSUPP; 1211 } 1212 1213 static int amd_pstate_profile_probe(void *drvdata, unsigned long *choices) 1214 { 1215 set_bit(PLATFORM_PROFILE_LOW_POWER, choices); 1216 set_bit(PLATFORM_PROFILE_BALANCED, choices); 1217 set_bit(PLATFORM_PROFILE_PERFORMANCE, choices); 1218 1219 return 0; 1220 } 1221 1222 static int amd_pstate_profile_get(struct device *dev, 1223 enum platform_profile_option *profile) 1224 { 1225 struct amd_cpudata *cpudata = dev_get_drvdata(dev); 1226 1227 *profile = cpudata->current_profile; 1228 1229 return 0; 1230 } 1231 1232 static int amd_pstate_profile_set(struct device *dev, 1233 enum platform_profile_option profile) 1234 { 1235 struct amd_cpudata *cpudata = dev_get_drvdata(dev); 1236 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 1237 int ret; 1238 u8 epp; 1239 1240 if (!policy) 1241 return -ENODEV; 1242 1243 ret = amd_pstate_get_epp_from_platform_profile(policy, profile); 1244 if (ret < 0) 1245 return ret; 1246 1247 epp = (u8)ret; 1248 ret = amd_pstate_set_epp(policy, epp); 1249 if (ret) 1250 return ret; 1251 1252 cpudata->current_profile = profile; 1253 1254 return 0; 1255 } 1256 1257 static const struct platform_profile_ops amd_pstate_profile_ops = { 1258 .probe = amd_pstate_profile_probe, 1259 .profile_set = amd_pstate_profile_set, 1260 .profile_get = amd_pstate_profile_get, 1261 }; 1262 1263 void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy) 1264 { 1265 struct amd_cpudata *cpudata = policy->driver_data; 1266 1267 if (cpudata->power_nb.notifier_call) 1268 power_supply_unreg_notifier(&cpudata->power_nb); 1269 if (cpudata->ppdev) { 1270 platform_profile_remove(cpudata->ppdev); 1271 cpudata->ppdev = NULL; 1272 } 1273 kfree(cpudata->profile_name); 1274 cpudata->dynamic_epp = false; 1275 } 1276 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_clear_dynamic_epp); 1277 1278 static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy) 1279 { 1280 struct amd_cpudata *cpudata = policy->driver_data; 1281 u64 prev = READ_ONCE(cpudata->cppc_req_cached); 1282 int ret; 1283 u8 epp; 1284 1285 ret = amd_pstate_get_epp_from_platform_profile(policy, cpudata->current_profile); 1286 if (ret < 0) 1287 return ret; 1288 1289 epp = (u8)ret; 1290 ret = amd_pstate_set_epp(policy, epp); 1291 if (ret) 1292 return ret; 1293 1294 cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu); 1295 if (!cpudata->profile_name) 1296 return -ENOMEM; 1297 1298 cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu), 1299 cpudata->profile_name, 1300 policy->driver_data, 1301 &amd_pstate_profile_ops); 1302 if (IS_ERR(cpudata->ppdev)) { 1303 ret = PTR_ERR(cpudata->ppdev); 1304 goto cleanup; 1305 } 1306 1307 /* only enable notifier if things will actually change */ 1308 if (cpudata->epp_default_ac != cpudata->epp_default_dc) { 1309 cpudata->power_nb.notifier_call = amd_pstate_power_supply_notifier; 1310 ret = power_supply_reg_notifier(&cpudata->power_nb); 1311 if (ret) 1312 goto cleanup; 1313 } 1314 1315 cpudata->dynamic_epp = true; 1316 1317 return 0; 1318 1319 cleanup: 1320 amd_pstate_clear_dynamic_epp(policy); 1321 1322 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, prev); 1323 /* Restore previous EPP if toggling Dynamic EPP failed. */ 1324 amd_pstate_set_epp(policy, epp); 1325 return ret; 1326 } 1327 1328 /* Sysfs attributes */ 1329 1330 /* 1331 * This frequency is to indicate the maximum hardware frequency. 1332 * If boost is not active but supported, the frequency will be larger than the 1333 * one in cpuinfo. 1334 */ 1335 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 1336 char *buf) 1337 { 1338 struct amd_cpudata *cpudata = policy->driver_data; 1339 1340 return sysfs_emit(buf, "%u\n", cpudata->max_freq); 1341 } 1342 1343 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 1344 char *buf) 1345 { 1346 struct amd_cpudata *cpudata; 1347 union perf_cached perf; 1348 1349 cpudata = policy->driver_data; 1350 perf = READ_ONCE(cpudata->perf); 1351 1352 return sysfs_emit(buf, "%u\n", 1353 perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf)); 1354 } 1355 1356 /* 1357 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 1358 * need to expose it to sysfs. 1359 */ 1360 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 1361 char *buf) 1362 { 1363 struct amd_cpudata *cpudata; 1364 1365 cpudata = policy->driver_data; 1366 1367 return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf); 1368 } 1369 1370 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, 1371 char *buf) 1372 { 1373 u8 perf; 1374 struct amd_cpudata *cpudata = policy->driver_data; 1375 1376 perf = READ_ONCE(cpudata->prefcore_ranking); 1377 1378 return sysfs_emit(buf, "%u\n", perf); 1379 } 1380 1381 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, 1382 char *buf) 1383 { 1384 bool hw_prefcore; 1385 struct amd_cpudata *cpudata = policy->driver_data; 1386 1387 hw_prefcore = READ_ONCE(cpudata->hw_prefcore); 1388 1389 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); 1390 } 1391 1392 static ssize_t show_energy_performance_available_preferences( 1393 struct cpufreq_policy *policy, char *buf) 1394 { 1395 int i, offset = 0; 1396 struct amd_cpudata *cpudata = policy->driver_data; 1397 1398 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1399 return sysfs_emit_at(buf, offset, "%s\n", 1400 energy_perf_strings[EPP_INDEX_PERFORMANCE]); 1401 1402 for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++) 1403 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]); 1404 1405 offset += sysfs_emit_at(buf, offset, "\n"); 1406 1407 return offset; 1408 } 1409 1410 ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, 1411 const char *buf, size_t count) 1412 { 1413 struct amd_cpudata *cpudata = policy->driver_data; 1414 ssize_t ret; 1415 bool raw_epp = false; 1416 u8 epp; 1417 1418 /* 1419 * if the value matches a number, use that, otherwise see if 1420 * matches an index in the energy_perf_strings array 1421 */ 1422 ret = kstrtou8(buf, 0, &epp); 1423 raw_epp = !ret; 1424 if (ret) { 1425 ret = sysfs_match_string(energy_perf_strings, buf); 1426 if (ret < 0 || ret == EPP_INDEX_CUSTOM) 1427 return -EINVAL; 1428 1429 if (ret == EPP_INDEX_DYNAMIC) { 1430 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1431 return -EBUSY; 1432 /* 1433 * Dynamic EPP was already enabled for this CPU. 1434 * Nothing to do. 1435 */ 1436 if (cpudata->dynamic_epp) 1437 return count; 1438 1439 cpudata->current_profile = PLATFORM_PROFILE_BALANCED; 1440 ret = amd_pstate_set_dynamic_epp(policy); 1441 if (ret) 1442 return ret; 1443 1444 return count; 1445 } 1446 1447 if (ret) 1448 epp = epp_values[ret]; 1449 else 1450 epp = cpudata->epp_default_dc; 1451 } 1452 1453 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 1454 pr_debug("EPP cannot be set under performance policy\n"); 1455 return -EBUSY; 1456 } 1457 1458 /* 1459 * Dynamic EPP was enabled previously! 1460 * Switch back to the static EPP mode. 1461 */ 1462 if (cpudata->dynamic_epp) 1463 amd_pstate_clear_dynamic_epp(policy); 1464 1465 ret = amd_pstate_set_epp(policy, epp); 1466 if (ret) 1467 return ret; 1468 1469 cpudata->raw_epp = raw_epp; 1470 1471 return count; 1472 } 1473 EXPORT_SYMBOL_FOR_PSTATE_UT(store_energy_performance_preference); 1474 1475 ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf) 1476 { 1477 struct amd_cpudata *cpudata = policy->driver_data; 1478 u8 preference, epp; 1479 1480 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 1481 1482 if (!cpudata->dynamic_epp && cpudata->raw_epp) 1483 return sysfs_emit(buf, "%u\n", epp); 1484 1485 switch (epp) { 1486 case AMD_CPPC_EPP_PERFORMANCE: 1487 preference = EPP_INDEX_PERFORMANCE; 1488 break; 1489 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 1490 preference = EPP_INDEX_BALANCE_PERFORMANCE; 1491 break; 1492 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 1493 preference = EPP_INDEX_BALANCE_POWERSAVE; 1494 break; 1495 case AMD_CPPC_EPP_POWERSAVE: 1496 preference = EPP_INDEX_POWERSAVE; 1497 break; 1498 default: 1499 return -EINVAL; 1500 } 1501 1502 if (cpudata->dynamic_epp) 1503 return sysfs_emit(buf, "dynamic(profile:%s)\n", energy_perf_strings[preference]); 1504 1505 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 1506 } 1507 EXPORT_SYMBOL_FOR_PSTATE_UT(show_energy_performance_preference); 1508 1509 ssize_t store_amd_pstate_floor_freq(struct cpufreq_policy *policy, const char *buf, size_t count) 1510 { 1511 struct amd_cpudata *cpudata = policy->driver_data; 1512 union perf_cached perf = READ_ONCE(cpudata->perf); 1513 unsigned int freq; 1514 u8 floor_perf; 1515 int ret; 1516 1517 ret = kstrtouint(buf, 0, &freq); 1518 if (ret) 1519 return ret; 1520 1521 if (freq < policy->cpuinfo.min_freq || freq > policy->max) 1522 return -EINVAL; 1523 1524 floor_perf = freq_to_perf(perf, cpudata->nominal_freq, freq); 1525 ret = amd_pstate_set_floor_perf(policy, floor_perf); 1526 1527 if (!ret) 1528 cpudata->floor_freq = freq; 1529 1530 return ret ?: count; 1531 } 1532 EXPORT_SYMBOL_FOR_PSTATE_UT(store_amd_pstate_floor_freq); 1533 1534 ssize_t show_amd_pstate_floor_freq(struct cpufreq_policy *policy, char *buf) 1535 { 1536 struct amd_cpudata *cpudata = policy->driver_data; 1537 1538 return sysfs_emit(buf, "%u\n", cpudata->floor_freq); 1539 } 1540 EXPORT_SYMBOL_FOR_PSTATE_UT(show_amd_pstate_floor_freq); 1541 1542 static ssize_t show_amd_pstate_floor_count(struct cpufreq_policy *policy, char *buf) 1543 { 1544 struct amd_cpudata *cpudata = policy->driver_data; 1545 u8 count = cpudata->floor_perf_cnt; 1546 1547 return sysfs_emit(buf, "%u\n", count); 1548 } 1549 1550 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1551 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1552 1553 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1554 cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); 1555 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); 1556 cpufreq_freq_attr_rw(energy_performance_preference); 1557 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1558 cpufreq_freq_attr_rw(amd_pstate_floor_freq); 1559 cpufreq_freq_attr_ro(amd_pstate_floor_count); 1560 1561 struct freq_attr_visibility { 1562 struct freq_attr *attr; 1563 bool (*visibility_fn)(void); 1564 }; 1565 1566 /* For attributes which are always visible */ 1567 static bool always_visible(void) 1568 { 1569 return true; 1570 } 1571 1572 /* Determines whether prefcore related attributes should be visible */ 1573 static bool prefcore_visibility(void) 1574 { 1575 return amd_pstate_prefcore; 1576 } 1577 1578 /* Determines whether energy performance preference should be visible */ 1579 static bool epp_visibility(void) 1580 { 1581 return cppc_state == AMD_PSTATE_ACTIVE; 1582 } 1583 1584 /* Determines whether amd_pstate_floor_freq related attributes should be visible */ 1585 static bool floor_freq_visibility(void) 1586 { 1587 return cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO); 1588 } 1589 1590 static struct freq_attr_visibility amd_pstate_attr_visibility[] = { 1591 {&amd_pstate_max_freq, always_visible}, 1592 {&amd_pstate_lowest_nonlinear_freq, always_visible}, 1593 {&amd_pstate_highest_perf, always_visible}, 1594 {&amd_pstate_prefcore_ranking, prefcore_visibility}, 1595 {&amd_pstate_hw_prefcore, prefcore_visibility}, 1596 {&energy_performance_preference, epp_visibility}, 1597 {&energy_performance_available_preferences, epp_visibility}, 1598 {&amd_pstate_floor_freq, floor_freq_visibility}, 1599 {&amd_pstate_floor_count, floor_freq_visibility}, 1600 }; 1601 1602 struct freq_attr **amd_pstate_get_current_attrs(void) 1603 { 1604 if (!current_pstate_driver) 1605 return NULL; 1606 return current_pstate_driver->attr; 1607 } 1608 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_current_attrs); 1609 1610 static struct freq_attr **get_freq_attrs(void) 1611 { 1612 bool attr_visible[ARRAY_SIZE(amd_pstate_attr_visibility)]; 1613 struct freq_attr **attrs; 1614 int i, j, count; 1615 1616 for (i = 0, count = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { 1617 struct freq_attr_visibility *v = &amd_pstate_attr_visibility[i]; 1618 1619 attr_visible[i] = v->visibility_fn(); 1620 if (attr_visible[i]) 1621 count++; 1622 } 1623 1624 /* amd_pstate_{max_freq, lowest_nonlinear_freq, highest_perf} should always be visible */ 1625 BUG_ON(!count); 1626 1627 attrs = kcalloc(count + 1, sizeof(struct freq_attr *), GFP_KERNEL); 1628 if (!attrs) 1629 return ERR_PTR(-ENOMEM); 1630 1631 for (i = 0, j = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { 1632 if (!attr_visible[i]) 1633 continue; 1634 1635 attrs[j++] = amd_pstate_attr_visibility[i].attr; 1636 } 1637 1638 return attrs; 1639 } 1640 1641 static void amd_pstate_driver_cleanup(void) 1642 { 1643 if (amd_pstate_prefcore) 1644 sched_clear_itmt_support(); 1645 1646 cppc_state = AMD_PSTATE_DISABLE; 1647 kfree(current_pstate_driver->attr); 1648 current_pstate_driver->attr = NULL; 1649 current_pstate_driver = NULL; 1650 } 1651 1652 static int amd_pstate_set_driver(int mode_idx) 1653 { 1654 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1655 cppc_state = mode_idx; 1656 if (cppc_state == AMD_PSTATE_DISABLE) 1657 pr_info("driver is explicitly disabled\n"); 1658 1659 if (cppc_state == AMD_PSTATE_ACTIVE) 1660 current_pstate_driver = &amd_pstate_epp_driver; 1661 1662 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1663 current_pstate_driver = &amd_pstate_driver; 1664 1665 return 0; 1666 } 1667 1668 return -EINVAL; 1669 } 1670 1671 static int amd_pstate_register_driver(int mode) 1672 { 1673 struct freq_attr **attr = NULL; 1674 int ret; 1675 1676 ret = amd_pstate_set_driver(mode); 1677 if (ret) 1678 return ret; 1679 1680 cppc_state = mode; 1681 1682 /* 1683 * Note: It is important to compute the attrs _after_ 1684 * re-initializing the cppc_state. Some attributes become 1685 * visible only when cppc_state is AMD_PSTATE_ACTIVE. 1686 */ 1687 attr = get_freq_attrs(); 1688 if (IS_ERR(attr)) { 1689 ret = (int) PTR_ERR(attr); 1690 pr_err("Couldn't compute freq_attrs for current mode %s [%d]\n", 1691 amd_pstate_get_mode_string(cppc_state), ret); 1692 amd_pstate_driver_cleanup(); 1693 return ret; 1694 } 1695 1696 current_pstate_driver->attr = attr; 1697 1698 /* at least one CPU supports CPB */ 1699 current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); 1700 1701 ret = cpufreq_register_driver(current_pstate_driver); 1702 if (ret) { 1703 amd_pstate_driver_cleanup(); 1704 return ret; 1705 } 1706 1707 /* Enable ITMT support once all CPUs have initialized their asym priorities. */ 1708 if (amd_pstate_prefcore) 1709 sched_set_itmt_support(); 1710 1711 return 0; 1712 } 1713 1714 static int amd_pstate_unregister_driver(int dummy) 1715 { 1716 cpufreq_unregister_driver(current_pstate_driver); 1717 amd_pstate_driver_cleanup(); 1718 return 0; 1719 } 1720 1721 static int amd_pstate_change_mode_without_dvr_change(int mode) 1722 { 1723 int cpu = 0; 1724 1725 cppc_state = mode; 1726 1727 if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 1728 return 0; 1729 1730 for_each_online_cpu(cpu) { 1731 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 1732 } 1733 1734 return 0; 1735 } 1736 1737 static int amd_pstate_change_driver_mode(int mode) 1738 { 1739 int ret; 1740 1741 lockdep_assert_held(&amd_pstate_driver_lock); 1742 1743 ret = amd_pstate_unregister_driver(0); 1744 if (ret) 1745 return ret; 1746 1747 ret = amd_pstate_register_driver(mode); 1748 if (ret) 1749 return ret; 1750 1751 return 0; 1752 } 1753 1754 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 1755 [AMD_PSTATE_DISABLE] = { 1756 [AMD_PSTATE_DISABLE] = NULL, 1757 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 1758 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 1759 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 1760 }, 1761 [AMD_PSTATE_PASSIVE] = { 1762 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1763 [AMD_PSTATE_PASSIVE] = NULL, 1764 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1765 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 1766 }, 1767 [AMD_PSTATE_ACTIVE] = { 1768 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1769 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 1770 [AMD_PSTATE_ACTIVE] = NULL, 1771 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 1772 }, 1773 [AMD_PSTATE_GUIDED] = { 1774 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1775 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 1776 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1777 [AMD_PSTATE_GUIDED] = NULL, 1778 }, 1779 }; 1780 1781 static ssize_t amd_pstate_show_status(char *buf) 1782 { 1783 if (!current_pstate_driver) 1784 return sysfs_emit(buf, "disable\n"); 1785 1786 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 1787 } 1788 1789 int amd_pstate_get_status(void) 1790 { 1791 return cppc_state; 1792 } 1793 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_status); 1794 1795 int amd_pstate_update_status(const char *buf, size_t size) 1796 { 1797 int mode_idx; 1798 1799 if (size > strlen("passive") || size < strlen("active")) 1800 return -EINVAL; 1801 1802 mode_idx = get_mode_idx_from_str(buf, size); 1803 if (mode_idx < 0) 1804 return mode_idx; 1805 1806 if (mode_state_machine[cppc_state][mode_idx]) { 1807 guard(mutex)(&amd_pstate_driver_lock); 1808 return mode_state_machine[cppc_state][mode_idx](mode_idx); 1809 } 1810 1811 return 0; 1812 } 1813 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_update_status); 1814 1815 static ssize_t status_show(struct device *dev, 1816 struct device_attribute *attr, char *buf) 1817 { 1818 1819 guard(mutex)(&amd_pstate_driver_lock); 1820 1821 return amd_pstate_show_status(buf); 1822 } 1823 1824 static ssize_t status_store(struct device *a, struct device_attribute *b, 1825 const char *buf, size_t count) 1826 { 1827 char *p = memchr(buf, '\n', count); 1828 int ret; 1829 1830 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1831 1832 return ret < 0 ? ret : count; 1833 } 1834 1835 static ssize_t prefcore_show(struct device *dev, 1836 struct device_attribute *attr, char *buf) 1837 { 1838 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); 1839 } 1840 1841 static DEVICE_ATTR_RW(status); 1842 static DEVICE_ATTR_RO(prefcore); 1843 1844 static struct attribute *pstate_global_attributes[] = { 1845 &dev_attr_status.attr, 1846 &dev_attr_prefcore.attr, 1847 NULL 1848 }; 1849 1850 static const struct attribute_group amd_pstate_global_attr_group = { 1851 .name = "amd_pstate", 1852 .attrs = pstate_global_attributes, 1853 }; 1854 1855 static bool amd_pstate_acpi_pm_profile_server(void) 1856 { 1857 switch (acpi_gbl_FADT.preferred_profile) { 1858 case PM_ENTERPRISE_SERVER: 1859 case PM_SOHO_SERVER: 1860 case PM_PERFORMANCE_SERVER: 1861 return true; 1862 } 1863 return false; 1864 } 1865 1866 static bool amd_pstate_acpi_pm_profile_undefined(void) 1867 { 1868 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) 1869 return true; 1870 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) 1871 return true; 1872 return false; 1873 } 1874 1875 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1876 { 1877 struct amd_cpudata *cpudata; 1878 union perf_cached perf; 1879 struct device *dev; 1880 int default_epp; 1881 int ret; 1882 1883 /* 1884 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1885 * which is ideal for initialization process. 1886 */ 1887 amd_perf_ctl_reset(policy->cpu); 1888 dev = get_cpu_device(policy->cpu); 1889 if (!dev) 1890 return -ENODEV; 1891 1892 cpudata = kzalloc_obj(*cpudata); 1893 if (!cpudata) 1894 return -ENOMEM; 1895 1896 cpudata->cpu = policy->cpu; 1897 1898 ret = amd_pstate_init_perf(cpudata); 1899 if (ret) 1900 goto free_cpudata1; 1901 1902 amd_pstate_init_prefcore(cpudata); 1903 1904 ret = amd_pstate_init_freq(cpudata); 1905 if (ret) 1906 goto free_cpudata1; 1907 1908 ret = amd_pstate_init_boost_support(cpudata); 1909 if (ret) 1910 goto free_cpudata1; 1911 1912 perf = READ_ONCE(cpudata->perf); 1913 1914 policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq, 1915 perf.lowest_perf); 1916 policy->cpuinfo.max_freq = cpudata->max_freq; 1917 policy->driver_data = cpudata; 1918 1919 ret = amd_pstate_cppc_enable(policy); 1920 if (ret) 1921 goto free_cpudata1; 1922 1923 /* It will be updated by governor */ 1924 policy->cur = policy->cpuinfo.min_freq; 1925 1926 1927 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1928 1929 /* Cache the firmware programmed EPP */ 1930 default_epp = amd_pstate_get_epp(cpudata); 1931 if (default_epp < 0) { 1932 ret = default_epp; 1933 goto free_cpudata1; 1934 } 1935 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &cpudata->cppc_req_cached, default_epp); 1936 1937 /* 1938 * Set the policy to provide a valid fallback value in case 1939 * the default cpufreq governor is neither powersave nor performance. 1940 */ 1941 if (amd_pstate_acpi_pm_profile_server() || 1942 amd_pstate_acpi_pm_profile_undefined()) { 1943 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 1944 cpudata->epp_default_ac = cpudata->epp_default_dc = default_epp; 1945 cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE; 1946 } else { 1947 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1948 cpudata->epp_default_ac = AMD_CPPC_EPP_PERFORMANCE; 1949 cpudata->epp_default_dc = AMD_CPPC_EPP_BALANCE_PERFORMANCE; 1950 cpudata->current_profile = PLATFORM_PROFILE_BALANCED; 1951 } 1952 1953 ret = amd_pstate_set_epp(policy, cpudata->epp_default_dc); 1954 if (ret) 1955 goto free_cpudata1; 1956 1957 ret = amd_pstate_init_floor_perf(policy); 1958 if (ret) { 1959 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); 1960 goto free_cpudata1; 1961 } 1962 1963 current_pstate_driver->adjust_perf = NULL; 1964 1965 return 0; 1966 1967 free_cpudata1: 1968 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1969 kfree(cpudata); 1970 policy->driver_data = NULL; 1971 return ret; 1972 } 1973 1974 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1975 { 1976 struct amd_cpudata *cpudata = policy->driver_data; 1977 1978 if (cpudata) { 1979 union perf_cached perf = READ_ONCE(cpudata->perf); 1980 1981 if (cpudata->dynamic_epp) 1982 amd_pstate_clear_dynamic_epp(policy); 1983 1984 /* Reset CPPC_REQ MSR to the BIOS value */ 1985 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1986 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 1987 1988 kfree(cpudata); 1989 policy->driver_data = NULL; 1990 } 1991 1992 pr_debug("CPU %d exiting\n", policy->cpu); 1993 } 1994 1995 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy, bool policy_change) 1996 { 1997 struct amd_cpudata *cpudata = policy->driver_data; 1998 union perf_cached perf; 1999 u8 epp; 2000 2001 if (policy_change || 2002 policy->min != cpudata->min_limit_freq || 2003 policy->max != cpudata->max_limit_freq) 2004 amd_pstate_update_min_max_limit(policy); 2005 2006 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 2007 epp = 0; 2008 else 2009 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 2010 2011 perf = READ_ONCE(cpudata->perf); 2012 2013 return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U, 2014 perf.max_limit_perf, epp, false); 2015 } 2016 2017 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 2018 { 2019 struct amd_cpudata *cpudata = policy->driver_data; 2020 int ret; 2021 2022 if (!policy->cpuinfo.max_freq) 2023 return -ENODEV; 2024 2025 /* Must be a switch between PERFORMANCE and POWERSAVE */ 2026 if (cpudata->policy != policy->policy) { 2027 /* 2028 * Disable dynamic_epp when switching 2029 * out of CPUFREQ_POLICY_POWERSAVE. 2030 */ 2031 if (cpudata->dynamic_epp) { 2032 WARN_ON_ONCE(cpudata->policy != CPUFREQ_POLICY_POWERSAVE); 2033 amd_pstate_clear_dynamic_epp(policy); 2034 } 2035 } 2036 2037 cpudata->policy = policy->policy; 2038 2039 ret = amd_pstate_epp_update_limit(policy, true); 2040 if (ret) 2041 return ret; 2042 2043 /* 2044 * policy->cur is never updated with the amd_pstate_epp driver, but it 2045 * is used as a stale frequency value. So, keep it within limits. 2046 */ 2047 policy->cur = policy->min; 2048 2049 return 0; 2050 } 2051 2052 static int amd_pstate_cpu_online(struct cpufreq_policy *policy) 2053 { 2054 struct amd_cpudata *cpudata = policy->driver_data; 2055 union perf_cached perf = READ_ONCE(cpudata->perf); 2056 u8 cached_floor_perf; 2057 int ret; 2058 2059 ret = amd_pstate_cppc_enable(policy); 2060 if (ret) 2061 return ret; 2062 2063 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2064 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2065 } 2066 2067 static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) 2068 { 2069 struct amd_cpudata *cpudata = policy->driver_data; 2070 union perf_cached perf = READ_ONCE(cpudata->perf); 2071 int ret; 2072 2073 /* 2074 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 2075 * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the 2076 * limits, epp and desired perf will get reset to the cached values in cpudata struct 2077 */ 2078 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 2079 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 2080 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 2081 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 2082 false); 2083 if (ret) 2084 return ret; 2085 2086 return amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2087 } 2088 2089 static int amd_pstate_suspend(struct cpufreq_policy *policy) 2090 { 2091 struct amd_cpudata *cpudata = policy->driver_data; 2092 union perf_cached perf = READ_ONCE(cpudata->perf); 2093 int ret; 2094 2095 /* 2096 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 2097 * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, 2098 * the limits, epp and desired perf will get reset to the cached values in cpudata struct 2099 */ 2100 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 2101 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 2102 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 2103 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 2104 false); 2105 if (ret) 2106 return ret; 2107 2108 ret = amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2109 if (ret) 2110 return ret; 2111 2112 /* set this flag to avoid setting core offline*/ 2113 cpudata->suspended = true; 2114 2115 return 0; 2116 } 2117 2118 static int amd_pstate_resume(struct cpufreq_policy *policy) 2119 { 2120 struct amd_cpudata *cpudata = policy->driver_data; 2121 union perf_cached perf = READ_ONCE(cpudata->perf); 2122 int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur); 2123 u8 cached_floor_perf; 2124 int ret; 2125 2126 /* Set CPPC_REQ to last sane value until the governor updates it */ 2127 ret = amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, 2128 0U, false); 2129 if (ret) 2130 return ret; 2131 2132 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2133 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2134 } 2135 2136 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 2137 { 2138 struct amd_cpudata *cpudata = policy->driver_data; 2139 union perf_cached perf = READ_ONCE(cpudata->perf); 2140 u8 cached_floor_perf; 2141 2142 if (cpudata->suspended) { 2143 int ret; 2144 2145 /* enable amd pstate from suspend state*/ 2146 ret = amd_pstate_epp_update_limit(policy, false); 2147 if (ret) 2148 return ret; 2149 2150 cpudata->suspended = false; 2151 } 2152 2153 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2154 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2155 } 2156 2157 static struct cpufreq_driver amd_pstate_driver = { 2158 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 2159 .verify = amd_pstate_verify, 2160 .target = amd_pstate_target, 2161 .fast_switch = amd_pstate_fast_switch, 2162 .init = amd_pstate_cpu_init, 2163 .exit = amd_pstate_cpu_exit, 2164 .online = amd_pstate_cpu_online, 2165 .offline = amd_pstate_cpu_offline, 2166 .suspend = amd_pstate_suspend, 2167 .resume = amd_pstate_resume, 2168 .set_boost = amd_pstate_set_boost, 2169 .update_limits = amd_pstate_update_limits, 2170 .name = "amd-pstate", 2171 }; 2172 2173 static struct cpufreq_driver amd_pstate_epp_driver = { 2174 .flags = CPUFREQ_CONST_LOOPS, 2175 .verify = amd_pstate_verify, 2176 .setpolicy = amd_pstate_epp_set_policy, 2177 .init = amd_pstate_epp_cpu_init, 2178 .exit = amd_pstate_epp_cpu_exit, 2179 .offline = amd_pstate_cpu_offline, 2180 .online = amd_pstate_cpu_online, 2181 .suspend = amd_pstate_suspend, 2182 .resume = amd_pstate_epp_resume, 2183 .update_limits = amd_pstate_update_limits, 2184 .set_boost = amd_pstate_set_boost, 2185 .name = "amd-pstate-epp", 2186 }; 2187 2188 /* 2189 * Processors without frequency scaling support can't do CPPC. 2190 * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. 2191 * show the debug message that helps to check if the CPU has CPPC support for loading issue. 2192 */ 2193 static bool amd_cppc_supported(void) 2194 { 2195 struct cpuinfo_x86 *c = &cpu_data(0); 2196 bool warn = false; 2197 2198 if (!cpu_feature_enabled(X86_FEATURE_HW_PSTATE)) { 2199 pr_debug_once("frequency scaling is not supported by the processor\n"); 2200 return false; 2201 } 2202 2203 if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) { 2204 pr_debug_once("CPPC feature is not supported by the processor\n"); 2205 return false; 2206 } 2207 2208 /* 2209 * If the CPPC feature is disabled in the BIOS for processors 2210 * that support MSR-based CPPC, the AMD Pstate driver may not 2211 * function correctly. 2212 * 2213 * For such processors, check the CPPC flag and display a 2214 * warning message if the platform supports CPPC. 2215 * 2216 * Note: The code check below will not abort the driver 2217 * registration process because of the code is added for 2218 * debugging purposes. Besides, it may still be possible for 2219 * the driver to work using the shared-memory mechanism. 2220 */ 2221 if (!cpu_feature_enabled(X86_FEATURE_CPPC)) { 2222 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 2223 switch (c->x86_model) { 2224 case 0x60 ... 0x6F: 2225 case 0x80 ... 0xAF: 2226 warn = true; 2227 break; 2228 } 2229 } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || 2230 cpu_feature_enabled(X86_FEATURE_ZEN4)) { 2231 switch (c->x86_model) { 2232 case 0x10 ... 0x1F: 2233 case 0x40 ... 0xAF: 2234 warn = true; 2235 break; 2236 } 2237 } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) { 2238 warn = true; 2239 } 2240 } 2241 2242 if (warn) 2243 pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n" 2244 "Please enable it if your BIOS has the CPPC option.\n"); 2245 return true; 2246 } 2247 2248 static int __init amd_pstate_init(void) 2249 { 2250 struct device *dev_root; 2251 int ret; 2252 2253 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 2254 return -ENODEV; 2255 2256 /* show debug message only if CPPC is not supported */ 2257 if (!amd_cppc_supported()) 2258 return -EOPNOTSUPP; 2259 2260 /* show warning message when BIOS broken or ACPI disabled */ 2261 if (!acpi_cpc_valid()) { 2262 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 2263 return -ENODEV; 2264 } 2265 2266 /* don't keep reloading if cpufreq_driver exists */ 2267 if (cpufreq_get_current_driver()) 2268 return -EEXIST; 2269 2270 quirks = NULL; 2271 2272 /* check if this machine need CPPC quirks */ 2273 dmi_check_system(amd_pstate_quirks_table); 2274 2275 /* 2276 * determine the driver mode from the command line or kernel config. 2277 * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED. 2278 * command line options will override the kernel config settings. 2279 */ 2280 2281 if (cppc_state == AMD_PSTATE_UNDEFINED) { 2282 /* Disable on the following configs by default: 2283 * 1. Undefined platforms 2284 * 2. Server platforms with CPUs older than Family 0x1A. 2285 */ 2286 if (amd_pstate_acpi_pm_profile_undefined() || 2287 (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) { 2288 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 2289 return -ENODEV; 2290 } 2291 /* get driver mode from kernel config option [1:4] */ 2292 cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; 2293 } 2294 2295 if (cppc_state == AMD_PSTATE_DISABLE) { 2296 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 2297 return -ENODEV; 2298 } 2299 2300 /* capability check */ 2301 if (cpu_feature_enabled(X86_FEATURE_CPPC)) { 2302 pr_debug("AMD CPPC MSR based functionality is supported\n"); 2303 } else { 2304 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 2305 static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); 2306 static_call_update(amd_pstate_init_perf, shmem_init_perf); 2307 static_call_update(amd_pstate_update_perf, shmem_update_perf); 2308 static_call_update(amd_pstate_get_epp, shmem_get_epp); 2309 static_call_update(amd_pstate_set_epp, shmem_set_epp); 2310 } 2311 2312 if (amd_pstate_prefcore) { 2313 ret = amd_detect_prefcore(&amd_pstate_prefcore); 2314 if (ret) 2315 return ret; 2316 } 2317 2318 ret = amd_pstate_register_driver(cppc_state); 2319 if (ret) { 2320 pr_err("failed to register with return %d\n", ret); 2321 return ret; 2322 } 2323 2324 dev_root = bus_get_dev_root(&cpu_subsys); 2325 if (dev_root) { 2326 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 2327 put_device(dev_root); 2328 if (ret) { 2329 pr_err("sysfs attribute export failed with error %d.\n", ret); 2330 goto global_attr_free; 2331 } 2332 } 2333 2334 return ret; 2335 2336 global_attr_free: 2337 amd_pstate_unregister_driver(0); 2338 return ret; 2339 } 2340 device_initcall(amd_pstate_init); 2341 2342 static int __init amd_pstate_param(char *str) 2343 { 2344 size_t size; 2345 int mode_idx; 2346 2347 if (!str) 2348 return -EINVAL; 2349 2350 size = strlen(str); 2351 mode_idx = get_mode_idx_from_str(str, size); 2352 2353 return amd_pstate_set_driver(mode_idx); 2354 } 2355 2356 static int __init amd_prefcore_param(char *str) 2357 { 2358 if (!strcmp(str, "disable")) 2359 amd_pstate_prefcore = false; 2360 2361 return 0; 2362 } 2363 2364 early_param("amd_pstate", amd_pstate_param); 2365 early_param("amd_prefcore", amd_prefcore_param); 2366 2367 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 2368 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 2369