1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/bitfield.h> 26 #include <linux/kernel.h> 27 #include <linux/module.h> 28 #include <linux/init.h> 29 #include <linux/smp.h> 30 #include <linux/sched.h> 31 #include <linux/cpufreq.h> 32 #include <linux/compiler.h> 33 #include <linux/dmi.h> 34 #include <linux/slab.h> 35 #include <linux/acpi.h> 36 #include <linux/io.h> 37 #include <linux/delay.h> 38 #include <linux/uaccess.h> 39 #include <linux/power_supply.h> 40 #include <linux/static_call.h> 41 #include <linux/topology.h> 42 43 #include <acpi/processor.h> 44 #include <acpi/cppc_acpi.h> 45 46 #include <asm/msr.h> 47 #include <asm/processor.h> 48 #include <asm/cpufeature.h> 49 #include <asm/cpu_device_id.h> 50 51 #include "amd-pstate.h" 52 #include "amd-pstate-trace.h" 53 54 #define AMD_PSTATE_TRANSITION_LATENCY 20000 55 #define AMD_PSTATE_TRANSITION_DELAY 1000 56 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600 57 58 #define AMD_CPPC_EPP_PERFORMANCE 0x00 59 #define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 60 #define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF 61 #define AMD_CPPC_EPP_POWERSAVE 0xFF 62 63 static const char * const amd_pstate_mode_string[] = { 64 [AMD_PSTATE_UNDEFINED] = "undefined", 65 [AMD_PSTATE_DISABLE] = "disable", 66 [AMD_PSTATE_PASSIVE] = "passive", 67 [AMD_PSTATE_ACTIVE] = "active", 68 [AMD_PSTATE_GUIDED] = "guided", 69 }; 70 static_assert(ARRAY_SIZE(amd_pstate_mode_string) == AMD_PSTATE_MAX); 71 72 const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode) 73 { 74 if (mode < AMD_PSTATE_UNDEFINED || mode >= AMD_PSTATE_MAX) 75 mode = AMD_PSTATE_UNDEFINED; 76 return amd_pstate_mode_string[mode]; 77 } 78 EXPORT_SYMBOL_GPL(amd_pstate_get_mode_string); 79 80 struct quirk_entry { 81 u32 nominal_freq; 82 u32 lowest_freq; 83 }; 84 85 static struct cpufreq_driver *current_pstate_driver; 86 static struct cpufreq_driver amd_pstate_driver; 87 static struct cpufreq_driver amd_pstate_epp_driver; 88 static int cppc_state = AMD_PSTATE_UNDEFINED; 89 static bool amd_pstate_prefcore = true; 90 static bool dynamic_epp; 91 static struct quirk_entry *quirks; 92 93 /* 94 * AMD Energy Preference Performance (EPP) 95 * The EPP is used in the CCLK DPM controller to drive 96 * the frequency that a core is going to operate during 97 * short periods of activity. EPP values will be utilized for 98 * different OS profiles (balanced, performance, power savings) 99 * display strings corresponding to EPP index in the 100 * energy_perf_strings[] 101 * index String 102 *------------------------------------- 103 * 0 default 104 * 1 performance 105 * 2 balance_performance 106 * 3 balance_power 107 * 4 power 108 * 5 custom (for raw EPP values) 109 */ 110 enum energy_perf_value_index { 111 EPP_INDEX_DEFAULT = 0, 112 EPP_INDEX_PERFORMANCE, 113 EPP_INDEX_BALANCE_PERFORMANCE, 114 EPP_INDEX_BALANCE_POWERSAVE, 115 EPP_INDEX_POWERSAVE, 116 EPP_INDEX_CUSTOM, 117 EPP_INDEX_MAX, 118 }; 119 120 static const char * const energy_perf_strings[] = { 121 [EPP_INDEX_DEFAULT] = "default", 122 [EPP_INDEX_PERFORMANCE] = "performance", 123 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 124 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 125 [EPP_INDEX_POWERSAVE] = "power", 126 [EPP_INDEX_CUSTOM] = "custom", 127 }; 128 static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); 129 130 static unsigned int epp_values[] = { 131 [EPP_INDEX_DEFAULT] = 0, 132 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 133 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 134 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 135 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 136 }; 137 static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX - 1); 138 139 typedef int (*cppc_mode_transition_fn)(int); 140 141 static struct quirk_entry quirk_amd_7k62 = { 142 .nominal_freq = 2600, 143 .lowest_freq = 550, 144 }; 145 146 static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) 147 { 148 u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); 149 150 return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf); 151 } 152 153 static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val) 154 { 155 return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val, 156 perf.nominal_perf); 157 } 158 159 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) 160 { 161 /** 162 * match the broken bios for family 17h processor support CPPC V2 163 * broken BIOS lack of nominal_freq and lowest_freq capabilities 164 * definition in ACPI tables 165 */ 166 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 167 quirks = dmi->driver_data; 168 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); 169 return 1; 170 } 171 172 return 0; 173 } 174 175 static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { 176 { 177 .callback = dmi_matched_7k62_bios_bug, 178 .ident = "AMD EPYC 7K62", 179 .matches = { 180 DMI_MATCH(DMI_BIOS_VERSION, "5.14"), 181 DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), 182 }, 183 .driver_data = &quirk_amd_7k62, 184 }, 185 {} 186 }; 187 MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); 188 189 static inline int get_mode_idx_from_str(const char *str, size_t size) 190 { 191 int i; 192 193 for (i = 0; i < AMD_PSTATE_MAX; i++) { 194 if (!strncmp(str, amd_pstate_mode_string[i], size)) 195 return i; 196 } 197 return -EINVAL; 198 } 199 200 static DEFINE_MUTEX(amd_pstate_driver_lock); 201 202 static u8 msr_get_epp(struct amd_cpudata *cpudata) 203 { 204 u64 value; 205 int ret; 206 207 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 208 if (ret < 0) { 209 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 210 return ret; 211 } 212 213 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value); 214 } 215 216 DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); 217 218 static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) 219 { 220 return static_call(amd_pstate_get_epp)(cpudata); 221 } 222 223 static u8 shmem_get_epp(struct amd_cpudata *cpudata) 224 { 225 u64 epp; 226 int ret; 227 228 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 229 if (ret < 0) { 230 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 231 return ret; 232 } 233 234 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp); 235 } 236 237 static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, 238 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 239 { 240 struct amd_cpudata *cpudata = policy->driver_data; 241 u64 value, prev; 242 243 value = prev = READ_ONCE(cpudata->cppc_req_cached); 244 245 FIELD_MODIFY(AMD_CPPC_MAX_PERF_MASK, &value, max_perf); 246 FIELD_MODIFY(AMD_CPPC_DES_PERF_MASK, &value, des_perf); 247 FIELD_MODIFY(AMD_CPPC_MIN_PERF_MASK, &value, min_perf); 248 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 249 250 if (trace_amd_pstate_epp_perf_enabled()) { 251 union perf_cached perf = READ_ONCE(cpudata->perf); 252 253 trace_call__amd_pstate_epp_perf(cpudata->cpu, 254 perf.highest_perf, 255 epp, 256 min_perf, 257 max_perf, 258 policy->boost_enabled, 259 value != prev); 260 } 261 262 if (value == prev) 263 return 0; 264 265 if (fast_switch) { 266 wrmsrq(MSR_AMD_CPPC_REQ, value); 267 } else { 268 int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 269 270 if (ret) 271 return ret; 272 } 273 274 WRITE_ONCE(cpudata->cppc_req_cached, value); 275 276 return 0; 277 } 278 279 DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); 280 281 static inline int amd_pstate_update_perf(struct cpufreq_policy *policy, 282 u8 min_perf, u8 des_perf, 283 u8 max_perf, u8 epp, 284 bool fast_switch) 285 { 286 return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf, 287 max_perf, epp, fast_switch); 288 } 289 290 static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) 291 { 292 struct amd_cpudata *cpudata = policy->driver_data; 293 u64 value, prev; 294 int ret; 295 296 value = prev = READ_ONCE(cpudata->cppc_req_cached); 297 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 298 299 if (trace_amd_pstate_epp_perf_enabled()) { 300 union perf_cached perf = cpudata->perf; 301 302 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 303 epp, 304 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 305 cpudata->cppc_req_cached), 306 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 307 cpudata->cppc_req_cached), 308 policy->boost_enabled, 309 value != prev); 310 } 311 312 if (value == prev) 313 return 0; 314 315 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 316 if (ret) { 317 pr_err("failed to set energy perf value (%d)\n", ret); 318 return ret; 319 } 320 321 /* update both so that msr_update_perf() can effectively check */ 322 WRITE_ONCE(cpudata->cppc_req_cached, value); 323 324 return ret; 325 } 326 327 DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); 328 329 static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp) 330 { 331 return static_call(amd_pstate_set_epp)(policy, epp); 332 } 333 334 static int amd_pstate_set_floor_perf(struct cpufreq_policy *policy, u8 perf) 335 { 336 struct amd_cpudata *cpudata = policy->driver_data; 337 u64 value, prev; 338 bool changed; 339 int ret; 340 341 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) 342 return 0; 343 344 value = prev = READ_ONCE(cpudata->cppc_req2_cached); 345 FIELD_MODIFY(AMD_CPPC_FLOOR_PERF_MASK, &value, perf); 346 347 changed = value != prev; 348 if (!changed) { 349 ret = 0; 350 goto out_trace; 351 } 352 353 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, value); 354 if (ret) { 355 changed = false; 356 pr_err("failed to set CPPC REQ2 value. Error (%d)\n", ret); 357 goto out_trace; 358 } 359 360 WRITE_ONCE(cpudata->cppc_req2_cached, value); 361 362 out_trace: 363 if (trace_amd_pstate_cppc_req2_enabled()) 364 trace_call__amd_pstate_cppc_req2(cpudata->cpu, perf, changed, 365 ret); 366 return ret; 367 } 368 369 static int amd_pstate_init_floor_perf(struct cpufreq_policy *policy) 370 { 371 struct amd_cpudata *cpudata = policy->driver_data; 372 u8 floor_perf; 373 u64 value; 374 int ret; 375 376 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) 377 return 0; 378 379 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, &value); 380 if (ret) { 381 pr_err("failed to read CPPC REQ2 value. Error (%d)\n", ret); 382 return ret; 383 } 384 385 WRITE_ONCE(cpudata->cppc_req2_cached, value); 386 floor_perf = FIELD_GET(AMD_CPPC_FLOOR_PERF_MASK, 387 cpudata->cppc_req2_cached); 388 389 /* Set a sane value for floor_perf if the default value is invalid */ 390 if (floor_perf < cpudata->perf.lowest_perf) { 391 floor_perf = cpudata->perf.nominal_perf; 392 ret = amd_pstate_set_floor_perf(policy, floor_perf); 393 if (ret) 394 return ret; 395 } 396 397 398 cpudata->bios_floor_perf = floor_perf; 399 cpudata->floor_freq = perf_to_freq(cpudata->perf, cpudata->nominal_freq, 400 floor_perf); 401 return 0; 402 } 403 404 static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) 405 { 406 struct amd_cpudata *cpudata = policy->driver_data; 407 struct cppc_perf_ctrls perf_ctrls; 408 u8 epp_cached; 409 u64 value; 410 int ret; 411 412 413 epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 414 if (trace_amd_pstate_epp_perf_enabled()) { 415 union perf_cached perf = cpudata->perf; 416 417 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 418 epp, 419 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 420 cpudata->cppc_req_cached), 421 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 422 cpudata->cppc_req_cached), 423 policy->boost_enabled, 424 epp != epp_cached); 425 } 426 427 if (epp == epp_cached) 428 return 0; 429 430 perf_ctrls.energy_perf = epp; 431 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 432 if (ret) { 433 pr_debug("failed to set energy perf value (%d)\n", ret); 434 return ret; 435 } 436 437 value = READ_ONCE(cpudata->cppc_req_cached); 438 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 439 WRITE_ONCE(cpudata->cppc_req_cached, value); 440 441 return ret; 442 } 443 444 static inline int msr_cppc_enable(struct cpufreq_policy *policy) 445 { 446 return wrmsrq_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); 447 } 448 449 static int shmem_cppc_enable(struct cpufreq_policy *policy) 450 { 451 return cppc_set_enable(policy->cpu, 1); 452 } 453 454 DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); 455 456 static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) 457 { 458 return static_call(amd_pstate_cppc_enable)(policy); 459 } 460 461 static int msr_init_perf(struct amd_cpudata *cpudata) 462 { 463 union perf_cached perf = READ_ONCE(cpudata->perf); 464 u64 cap1, numerator, cppc_req; 465 u8 min_perf; 466 467 int ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 468 &cap1); 469 if (ret) 470 return ret; 471 472 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 473 if (ret) 474 return ret; 475 476 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); 477 if (ret) 478 return ret; 479 480 WRITE_ONCE(cpudata->cppc_req_cached, cppc_req); 481 min_perf = FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cppc_req); 482 483 /* 484 * Clear out the min_perf part to check if the rest of the MSR is 0, if yes, this is an 485 * indication that the min_perf value is the one specified through the BIOS option 486 */ 487 cppc_req &= ~(AMD_CPPC_MIN_PERF_MASK); 488 489 if (!cppc_req) 490 perf.bios_min_perf = min_perf; 491 492 perf.highest_perf = numerator; 493 perf.max_limit_perf = numerator; 494 perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 495 perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); 496 perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); 497 perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 498 WRITE_ONCE(cpudata->perf, perf); 499 WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1)); 500 WRITE_ONCE(cpudata->floor_perf_cnt, FIELD_GET(AMD_CPPC_FLOOR_PERF_CNT_MASK, cap1)); 501 502 return 0; 503 } 504 505 static int shmem_init_perf(struct amd_cpudata *cpudata) 506 { 507 struct cppc_perf_caps cppc_perf; 508 union perf_cached perf = READ_ONCE(cpudata->perf); 509 u64 numerator; 510 bool auto_sel; 511 512 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 513 if (ret) 514 return ret; 515 516 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 517 if (ret) 518 return ret; 519 520 perf.highest_perf = numerator; 521 perf.max_limit_perf = numerator; 522 perf.min_limit_perf = cppc_perf.lowest_perf; 523 perf.nominal_perf = cppc_perf.nominal_perf; 524 perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 525 perf.lowest_perf = cppc_perf.lowest_perf; 526 WRITE_ONCE(cpudata->perf, perf); 527 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); 528 529 if (cppc_state == AMD_PSTATE_ACTIVE) 530 return 0; 531 532 ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel); 533 if (ret) { 534 pr_warn("failed to get auto_sel, ret: %d\n", ret); 535 return 0; 536 } 537 538 ret = cppc_set_auto_sel(cpudata->cpu, 539 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 540 541 if (ret) 542 pr_warn("failed to set auto_sel, ret: %d\n", ret); 543 544 return ret; 545 } 546 547 DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf); 548 549 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 550 { 551 return static_call(amd_pstate_init_perf)(cpudata); 552 } 553 554 static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf, 555 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 556 { 557 struct amd_cpudata *cpudata = policy->driver_data; 558 struct cppc_perf_ctrls perf_ctrls; 559 u64 value, prev; 560 int ret; 561 562 if (cppc_state == AMD_PSTATE_ACTIVE) { 563 int ret = shmem_set_epp(policy, epp); 564 565 if (ret) 566 return ret; 567 } 568 569 value = prev = READ_ONCE(cpudata->cppc_req_cached); 570 571 FIELD_MODIFY(AMD_CPPC_MAX_PERF_MASK, &value, max_perf); 572 FIELD_MODIFY(AMD_CPPC_DES_PERF_MASK, &value, des_perf); 573 FIELD_MODIFY(AMD_CPPC_MIN_PERF_MASK, &value, min_perf); 574 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 575 576 if (trace_amd_pstate_epp_perf_enabled()) { 577 union perf_cached perf = READ_ONCE(cpudata->perf); 578 579 trace_call__amd_pstate_epp_perf(cpudata->cpu, 580 perf.highest_perf, 581 epp, 582 min_perf, 583 max_perf, 584 policy->boost_enabled, 585 value != prev); 586 } 587 588 if (value == prev) 589 return 0; 590 591 perf_ctrls.max_perf = max_perf; 592 perf_ctrls.min_perf = min_perf; 593 perf_ctrls.desired_perf = des_perf; 594 595 ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); 596 if (ret) 597 return ret; 598 599 WRITE_ONCE(cpudata->cppc_req_cached, value); 600 601 return 0; 602 } 603 604 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 605 { 606 u64 aperf, mperf, tsc; 607 unsigned long flags; 608 609 local_irq_save(flags); 610 rdmsrq(MSR_IA32_APERF, aperf); 611 rdmsrq(MSR_IA32_MPERF, mperf); 612 tsc = rdtsc(); 613 614 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 615 local_irq_restore(flags); 616 return false; 617 } 618 619 local_irq_restore(flags); 620 621 cpudata->cur.aperf = aperf; 622 cpudata->cur.mperf = mperf; 623 cpudata->cur.tsc = tsc; 624 cpudata->cur.aperf -= cpudata->prev.aperf; 625 cpudata->cur.mperf -= cpudata->prev.mperf; 626 cpudata->cur.tsc -= cpudata->prev.tsc; 627 628 cpudata->prev.aperf = aperf; 629 cpudata->prev.mperf = mperf; 630 cpudata->prev.tsc = tsc; 631 632 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 633 634 return true; 635 } 636 637 static void amd_pstate_update(struct cpufreq_policy *policy, u8 min_perf, 638 u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) 639 { 640 struct amd_cpudata *cpudata = policy->driver_data; 641 union perf_cached perf = READ_ONCE(cpudata->perf); 642 643 /* limit the max perf when core performance boost feature is disabled */ 644 if (!cpudata->boost_supported) 645 max_perf = min_t(u8, perf.nominal_perf, max_perf); 646 647 des_perf = clamp_t(u8, des_perf, min_perf, max_perf); 648 649 policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); 650 651 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 652 min_perf = des_perf; 653 des_perf = 0; 654 } 655 656 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 657 trace_call__amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 658 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 659 cpudata->cpu, fast_switch); 660 } 661 662 amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch); 663 } 664 665 static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) 666 { 667 /* 668 * Initialize lower frequency limit (i.e.policy->min) with 669 * lowest_nonlinear_frequency or the min frequency (if) specified in BIOS, 670 * Override the initial value set by cpufreq core and amd-pstate qos_requests. 671 */ 672 if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { 673 struct cpufreq_policy *policy __free(put_cpufreq_policy) = 674 cpufreq_cpu_get(policy_data->cpu); 675 struct amd_cpudata *cpudata; 676 union perf_cached perf; 677 678 if (!policy) 679 return -EINVAL; 680 681 cpudata = policy->driver_data; 682 perf = READ_ONCE(cpudata->perf); 683 684 if (perf.bios_min_perf) 685 policy_data->min = perf_to_freq(perf, cpudata->nominal_freq, 686 perf.bios_min_perf); 687 else 688 policy_data->min = cpudata->lowest_nonlinear_freq; 689 } 690 691 cpufreq_verify_within_cpu_limits(policy_data); 692 693 return 0; 694 } 695 696 static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) 697 { 698 struct amd_cpudata *cpudata = policy->driver_data; 699 union perf_cached perf = READ_ONCE(cpudata->perf); 700 701 perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); 702 WRITE_ONCE(cpudata->max_limit_freq, policy->max); 703 704 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 705 /* 706 * For performance policy, set MinPerf to nominal_perf rather than 707 * highest_perf or lowest_nonlinear_perf. 708 * 709 * Per commit 0c411b39e4f4c, using highest_perf was observed 710 * to cause frequency throttling on power-limited platforms, leading to 711 * performance regressions. Using lowest_nonlinear_perf would limit 712 * performance too much for HPC workloads requiring high frequency 713 * operation and minimal wakeup latency from idle states. 714 * 715 * nominal_perf therefore provides a balance by avoiding throttling 716 * while still maintaining enough performance for HPC workloads. 717 */ 718 perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); 719 WRITE_ONCE(cpudata->min_limit_freq, min(cpudata->nominal_freq, cpudata->max_limit_freq)); 720 } else { 721 perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); 722 WRITE_ONCE(cpudata->min_limit_freq, policy->min); 723 } 724 725 WRITE_ONCE(cpudata->perf, perf); 726 } 727 728 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 729 unsigned int target_freq, bool fast_switch) 730 { 731 struct cpufreq_freqs freqs; 732 struct amd_cpudata *cpudata; 733 union perf_cached perf; 734 u8 des_perf; 735 736 cpudata = policy->driver_data; 737 738 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 739 amd_pstate_update_min_max_limit(policy); 740 741 perf = READ_ONCE(cpudata->perf); 742 743 freqs.old = policy->cur; 744 freqs.new = target_freq; 745 746 des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq); 747 748 WARN_ON(fast_switch && !policy->fast_switch_enabled); 749 /* 750 * If fast_switch is desired, then there aren't any registered 751 * transition notifiers. See comment for 752 * cpufreq_enable_fast_switch(). 753 */ 754 if (!fast_switch) 755 cpufreq_freq_transition_begin(policy, &freqs); 756 757 amd_pstate_update(policy, perf.min_limit_perf, des_perf, 758 perf.max_limit_perf, fast_switch, 759 policy->governor->flags); 760 761 if (!fast_switch) 762 cpufreq_freq_transition_end(policy, &freqs, false); 763 764 return 0; 765 } 766 767 static int amd_pstate_target(struct cpufreq_policy *policy, 768 unsigned int target_freq, 769 unsigned int relation) 770 { 771 return amd_pstate_update_freq(policy, target_freq, false); 772 } 773 774 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 775 unsigned int target_freq) 776 { 777 if (!amd_pstate_update_freq(policy, target_freq, true)) 778 return target_freq; 779 return policy->cur; 780 } 781 782 static void amd_pstate_adjust_perf(struct cpufreq_policy *policy, 783 unsigned long _min_perf, 784 unsigned long target_perf, 785 unsigned long capacity) 786 { 787 u8 max_perf, min_perf, des_perf, cap_perf; 788 struct amd_cpudata *cpudata; 789 union perf_cached perf; 790 791 if (!policy) 792 return; 793 794 cpudata = policy->driver_data; 795 796 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 797 amd_pstate_update_min_max_limit(policy); 798 799 perf = READ_ONCE(cpudata->perf); 800 cap_perf = perf.highest_perf; 801 802 des_perf = cap_perf; 803 if (target_perf < capacity) 804 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 805 806 if (_min_perf < capacity) 807 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 808 else 809 min_perf = cap_perf; 810 811 if (min_perf < perf.min_limit_perf) 812 min_perf = perf.min_limit_perf; 813 814 max_perf = perf.max_limit_perf; 815 if (max_perf < min_perf) 816 max_perf = min_perf; 817 818 amd_pstate_update(policy, min_perf, des_perf, max_perf, true, 819 policy->governor->flags); 820 } 821 822 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) 823 { 824 struct amd_cpudata *cpudata = policy->driver_data; 825 u32 nominal_freq; 826 int ret = 0; 827 828 nominal_freq = READ_ONCE(cpudata->nominal_freq); 829 830 if (on) 831 policy->cpuinfo.max_freq = cpudata->max_freq; 832 else if (policy->cpuinfo.max_freq > nominal_freq) 833 policy->cpuinfo.max_freq = nominal_freq; 834 835 if (cppc_state == AMD_PSTATE_PASSIVE) { 836 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); 837 if (ret < 0) 838 pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu); 839 } 840 841 return ret < 0 ? ret : 0; 842 } 843 844 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 845 { 846 struct amd_cpudata *cpudata = policy->driver_data; 847 int ret; 848 849 if (!cpudata->boost_supported) { 850 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 851 return -EOPNOTSUPP; 852 } 853 854 ret = amd_pstate_cpu_boost_update(policy, state); 855 refresh_frequency_limits(policy); 856 857 return ret; 858 } 859 860 static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) 861 { 862 u64 boost_val; 863 int ret = -1; 864 865 /* 866 * If platform has no CPB support or disable it, initialize current driver 867 * boost_enabled state to be false, it is not an error for cpufreq core to handle. 868 */ 869 if (!cpu_feature_enabled(X86_FEATURE_CPB)) { 870 pr_debug_once("Boost CPB capabilities not present in the processor\n"); 871 ret = 0; 872 goto exit_err; 873 } 874 875 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); 876 if (ret) { 877 pr_err_once("failed to read initial CPU boost state!\n"); 878 ret = -EIO; 879 goto exit_err; 880 } 881 882 if (!(boost_val & MSR_K7_HWCR_CPB_DIS)) 883 cpudata->boost_supported = true; 884 885 return 0; 886 887 exit_err: 888 cpudata->boost_supported = false; 889 return ret; 890 } 891 892 static void amd_perf_ctl_reset(unsigned int cpu) 893 { 894 wrmsrq_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 895 } 896 897 #define CPPC_MAX_PERF U8_MAX 898 899 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) 900 { 901 /* user disabled or not detected */ 902 if (!amd_pstate_prefcore) 903 return; 904 905 /* should use amd-hfi instead */ 906 if (cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS) && 907 IS_ENABLED(CONFIG_AMD_HFI)) { 908 amd_pstate_prefcore = false; 909 return; 910 } 911 912 cpudata->hw_prefcore = true; 913 914 /* Priorities must be initialized before ITMT support can be toggled on. */ 915 sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); 916 } 917 918 static void amd_pstate_update_limits(struct cpufreq_policy *policy) 919 { 920 struct amd_cpudata *cpudata; 921 u32 prev_high = 0, cur_high = 0; 922 bool highest_perf_changed = false; 923 unsigned int cpu = policy->cpu; 924 925 if (!amd_pstate_prefcore) 926 return; 927 928 if (amd_get_highest_perf(cpu, &cur_high)) 929 return; 930 931 cpudata = policy->driver_data; 932 933 prev_high = READ_ONCE(cpudata->prefcore_ranking); 934 highest_perf_changed = (prev_high != cur_high); 935 if (highest_perf_changed) { 936 WRITE_ONCE(cpudata->prefcore_ranking, cur_high); 937 938 if (cur_high < CPPC_MAX_PERF) { 939 sched_set_itmt_core_prio((int)cur_high, cpu); 940 sched_update_asym_prefer_cpu(cpu, prev_high, cur_high); 941 } 942 } 943 } 944 945 /* 946 * Get pstate transition delay time from ACPI tables that firmware set 947 * instead of using hardcode value directly. 948 */ 949 static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) 950 { 951 int transition_delay_ns; 952 953 transition_delay_ns = cppc_get_transition_latency(cpu); 954 if (transition_delay_ns < 0) { 955 if (cpu_feature_enabled(X86_FEATURE_AMD_FAST_CPPC)) 956 return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY; 957 else 958 return AMD_PSTATE_TRANSITION_DELAY; 959 } 960 961 return transition_delay_ns / NSEC_PER_USEC; 962 } 963 964 /* 965 * Get pstate transition latency value from ACPI tables that firmware 966 * set instead of using hardcode value directly. 967 */ 968 static u32 amd_pstate_get_transition_latency(unsigned int cpu) 969 { 970 int transition_latency; 971 972 transition_latency = cppc_get_transition_latency(cpu); 973 if (transition_latency < 0) 974 return AMD_PSTATE_TRANSITION_LATENCY; 975 976 return transition_latency; 977 } 978 979 /* 980 * amd_pstate_init_freq: Initialize the nominal_freq and lowest_nonlinear_freq 981 * for the @cpudata object. 982 * 983 * Requires: all perf members of @cpudata to be initialized. 984 * 985 * Returns 0 on success, non-zero value on failure. 986 */ 987 static int amd_pstate_init_freq(struct amd_cpudata *cpudata) 988 { 989 u32 min_freq, max_freq, nominal_freq, lowest_nonlinear_freq; 990 struct cppc_perf_caps cppc_perf; 991 union perf_cached perf; 992 int ret; 993 994 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 995 if (ret) 996 return ret; 997 perf = READ_ONCE(cpudata->perf); 998 999 if (quirks && quirks->nominal_freq) 1000 nominal_freq = quirks->nominal_freq; 1001 else 1002 nominal_freq = cppc_perf.nominal_freq; 1003 nominal_freq *= 1000; 1004 1005 if (quirks && quirks->lowest_freq) { 1006 min_freq = quirks->lowest_freq; 1007 perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq); 1008 WRITE_ONCE(cpudata->perf, perf); 1009 } else 1010 min_freq = cppc_perf.lowest_freq; 1011 1012 min_freq *= 1000; 1013 1014 WRITE_ONCE(cpudata->nominal_freq, nominal_freq); 1015 1016 /* max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf */ 1017 max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); 1018 WRITE_ONCE(cpudata->max_freq, max_freq); 1019 1020 lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); 1021 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); 1022 1023 /** 1024 * Below values need to be initialized correctly, otherwise driver will fail to load 1025 * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] 1026 * Check _CPC in ACPI table objects if any values are incorrect 1027 */ 1028 if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { 1029 pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", 1030 min_freq, max_freq, nominal_freq); 1031 return -EINVAL; 1032 } 1033 1034 if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq) { 1035 pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", 1036 lowest_nonlinear_freq, min_freq, nominal_freq); 1037 return -EINVAL; 1038 } 1039 1040 return 0; 1041 } 1042 1043 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 1044 { 1045 struct amd_cpudata *cpudata; 1046 union perf_cached perf; 1047 struct device *dev; 1048 int ret; 1049 1050 /* 1051 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1052 * which is ideal for initialization process. 1053 */ 1054 amd_perf_ctl_reset(policy->cpu); 1055 dev = get_cpu_device(policy->cpu); 1056 if (!dev) 1057 return -ENODEV; 1058 1059 cpudata = kzalloc_obj(*cpudata); 1060 if (!cpudata) 1061 return -ENOMEM; 1062 1063 cpudata->cpu = policy->cpu; 1064 1065 ret = amd_pstate_init_perf(cpudata); 1066 if (ret) 1067 goto free_cpudata1; 1068 1069 amd_pstate_init_prefcore(cpudata); 1070 1071 ret = amd_pstate_init_freq(cpudata); 1072 if (ret) 1073 goto free_cpudata1; 1074 1075 ret = amd_pstate_init_boost_support(cpudata); 1076 if (ret) 1077 goto free_cpudata1; 1078 1079 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); 1080 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); 1081 1082 perf = READ_ONCE(cpudata->perf); 1083 1084 policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq, 1085 perf.lowest_perf); 1086 policy->cpuinfo.max_freq = cpudata->max_freq; 1087 1088 policy->driver_data = cpudata; 1089 ret = amd_pstate_cppc_enable(policy); 1090 if (ret) 1091 goto free_cpudata1; 1092 1093 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1094 1095 /* It will be updated by governor */ 1096 policy->cur = policy->cpuinfo.min_freq; 1097 1098 if (cpu_feature_enabled(X86_FEATURE_CPPC)) 1099 policy->fast_switch_possible = true; 1100 1101 ret = amd_pstate_init_floor_perf(policy); 1102 if (ret) { 1103 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); 1104 goto free_cpudata1; 1105 } 1106 1107 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 1108 FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); 1109 if (ret < 0) { 1110 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 1111 goto free_cpudata1; 1112 } 1113 1114 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 1115 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 1116 if (ret < 0) { 1117 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 1118 goto free_cpudata2; 1119 } 1120 1121 1122 if (!current_pstate_driver->adjust_perf) 1123 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1124 1125 return 0; 1126 1127 free_cpudata2: 1128 freq_qos_remove_request(&cpudata->req[0]); 1129 free_cpudata1: 1130 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1131 kfree(cpudata); 1132 policy->driver_data = NULL; 1133 return ret; 1134 } 1135 1136 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) 1137 { 1138 struct amd_cpudata *cpudata = policy->driver_data; 1139 union perf_cached perf = READ_ONCE(cpudata->perf); 1140 1141 /* Reset CPPC_REQ MSR to the BIOS value */ 1142 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1143 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 1144 1145 freq_qos_remove_request(&cpudata->req[1]); 1146 freq_qos_remove_request(&cpudata->req[0]); 1147 policy->fast_switch_possible = false; 1148 kfree(cpudata); 1149 } 1150 1151 static int amd_pstate_get_balanced_epp(struct cpufreq_policy *policy) 1152 { 1153 struct amd_cpudata *cpudata = policy->driver_data; 1154 1155 if (power_supply_is_system_supplied()) 1156 return cpudata->epp_default_ac; 1157 else 1158 return cpudata->epp_default_dc; 1159 } 1160 1161 static int amd_pstate_power_supply_notifier(struct notifier_block *nb, 1162 unsigned long event, void *data) 1163 { 1164 struct amd_cpudata *cpudata = container_of(nb, struct amd_cpudata, power_nb); 1165 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 1166 u8 epp; 1167 int ret; 1168 1169 if (event != PSY_EVENT_PROP_CHANGED) 1170 return NOTIFY_OK; 1171 1172 /* dynamic actions are only applied while platform profile is in balanced */ 1173 if (cpudata->current_profile != PLATFORM_PROFILE_BALANCED) 1174 return 0; 1175 1176 epp = amd_pstate_get_balanced_epp(policy); 1177 1178 ret = amd_pstate_set_epp(policy, epp); 1179 if (ret) 1180 pr_warn("Failed to set CPU %d EPP %u: %d\n", cpudata->cpu, epp, ret); 1181 1182 return NOTIFY_OK; 1183 } 1184 1185 static int amd_pstate_profile_probe(void *drvdata, unsigned long *choices) 1186 { 1187 set_bit(PLATFORM_PROFILE_LOW_POWER, choices); 1188 set_bit(PLATFORM_PROFILE_BALANCED, choices); 1189 set_bit(PLATFORM_PROFILE_PERFORMANCE, choices); 1190 1191 return 0; 1192 } 1193 1194 static int amd_pstate_profile_get(struct device *dev, 1195 enum platform_profile_option *profile) 1196 { 1197 struct amd_cpudata *cpudata = dev_get_drvdata(dev); 1198 1199 *profile = cpudata->current_profile; 1200 1201 return 0; 1202 } 1203 1204 static int amd_pstate_profile_set(struct device *dev, 1205 enum platform_profile_option profile) 1206 { 1207 struct amd_cpudata *cpudata = dev_get_drvdata(dev); 1208 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 1209 int ret; 1210 1211 switch (profile) { 1212 case PLATFORM_PROFILE_LOW_POWER: 1213 ret = amd_pstate_set_epp(policy, AMD_CPPC_EPP_POWERSAVE); 1214 if (ret) 1215 return ret; 1216 break; 1217 case PLATFORM_PROFILE_BALANCED: 1218 ret = amd_pstate_set_epp(policy, 1219 amd_pstate_get_balanced_epp(policy)); 1220 if (ret) 1221 return ret; 1222 break; 1223 case PLATFORM_PROFILE_PERFORMANCE: 1224 ret = amd_pstate_set_epp(policy, AMD_CPPC_EPP_PERFORMANCE); 1225 if (ret) 1226 return ret; 1227 break; 1228 default: 1229 pr_err("Unknown Platform Profile %d\n", profile); 1230 return -EOPNOTSUPP; 1231 } 1232 1233 cpudata->current_profile = profile; 1234 1235 return 0; 1236 } 1237 1238 static const struct platform_profile_ops amd_pstate_profile_ops = { 1239 .probe = amd_pstate_profile_probe, 1240 .profile_set = amd_pstate_profile_set, 1241 .profile_get = amd_pstate_profile_get, 1242 }; 1243 1244 void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy) 1245 { 1246 struct amd_cpudata *cpudata = policy->driver_data; 1247 1248 if (cpudata->power_nb.notifier_call) 1249 power_supply_unreg_notifier(&cpudata->power_nb); 1250 if (cpudata->ppdev) { 1251 platform_profile_remove(cpudata->ppdev); 1252 cpudata->ppdev = NULL; 1253 } 1254 kfree(cpudata->profile_name); 1255 cpudata->dynamic_epp = false; 1256 } 1257 EXPORT_SYMBOL_GPL(amd_pstate_clear_dynamic_epp); 1258 1259 static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy) 1260 { 1261 struct amd_cpudata *cpudata = policy->driver_data; 1262 int ret; 1263 u8 epp; 1264 1265 switch (cpudata->current_profile) { 1266 case PLATFORM_PROFILE_PERFORMANCE: 1267 epp = AMD_CPPC_EPP_PERFORMANCE; 1268 break; 1269 case PLATFORM_PROFILE_LOW_POWER: 1270 epp = AMD_CPPC_EPP_POWERSAVE; 1271 break; 1272 case PLATFORM_PROFILE_BALANCED: 1273 epp = amd_pstate_get_balanced_epp(policy); 1274 break; 1275 default: 1276 pr_err("Unknown Platform Profile %d\n", cpudata->current_profile); 1277 return -EOPNOTSUPP; 1278 } 1279 ret = amd_pstate_set_epp(policy, epp); 1280 if (ret) 1281 return ret; 1282 1283 cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu); 1284 if (!cpudata->profile_name) 1285 return -ENOMEM; 1286 1287 cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu), 1288 cpudata->profile_name, 1289 policy->driver_data, 1290 &amd_pstate_profile_ops); 1291 if (IS_ERR(cpudata->ppdev)) { 1292 ret = PTR_ERR(cpudata->ppdev); 1293 goto cleanup; 1294 } 1295 1296 /* only enable notifier if things will actually change */ 1297 if (cpudata->epp_default_ac != cpudata->epp_default_dc) { 1298 cpudata->power_nb.notifier_call = amd_pstate_power_supply_notifier; 1299 ret = power_supply_reg_notifier(&cpudata->power_nb); 1300 if (ret) 1301 goto cleanup; 1302 } 1303 1304 cpudata->dynamic_epp = true; 1305 1306 return 0; 1307 1308 cleanup: 1309 amd_pstate_clear_dynamic_epp(policy); 1310 1311 return ret; 1312 } 1313 1314 /* Sysfs attributes */ 1315 1316 /* 1317 * This frequency is to indicate the maximum hardware frequency. 1318 * If boost is not active but supported, the frequency will be larger than the 1319 * one in cpuinfo. 1320 */ 1321 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 1322 char *buf) 1323 { 1324 struct amd_cpudata *cpudata = policy->driver_data; 1325 1326 return sysfs_emit(buf, "%u\n", cpudata->max_freq); 1327 } 1328 1329 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 1330 char *buf) 1331 { 1332 struct amd_cpudata *cpudata; 1333 union perf_cached perf; 1334 1335 cpudata = policy->driver_data; 1336 perf = READ_ONCE(cpudata->perf); 1337 1338 return sysfs_emit(buf, "%u\n", 1339 perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf)); 1340 } 1341 1342 /* 1343 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 1344 * need to expose it to sysfs. 1345 */ 1346 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 1347 char *buf) 1348 { 1349 struct amd_cpudata *cpudata; 1350 1351 cpudata = policy->driver_data; 1352 1353 return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf); 1354 } 1355 1356 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, 1357 char *buf) 1358 { 1359 u8 perf; 1360 struct amd_cpudata *cpudata = policy->driver_data; 1361 1362 perf = READ_ONCE(cpudata->prefcore_ranking); 1363 1364 return sysfs_emit(buf, "%u\n", perf); 1365 } 1366 1367 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, 1368 char *buf) 1369 { 1370 bool hw_prefcore; 1371 struct amd_cpudata *cpudata = policy->driver_data; 1372 1373 hw_prefcore = READ_ONCE(cpudata->hw_prefcore); 1374 1375 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); 1376 } 1377 1378 static ssize_t show_energy_performance_available_preferences( 1379 struct cpufreq_policy *policy, char *buf) 1380 { 1381 int offset = 0, i; 1382 struct amd_cpudata *cpudata = policy->driver_data; 1383 1384 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1385 return sysfs_emit_at(buf, offset, "%s\n", 1386 energy_perf_strings[EPP_INDEX_PERFORMANCE]); 1387 1388 for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++) 1389 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]); 1390 1391 offset += sysfs_emit_at(buf, offset, "\n"); 1392 1393 return offset; 1394 } 1395 1396 ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, 1397 const char *buf, size_t count) 1398 { 1399 struct amd_cpudata *cpudata = policy->driver_data; 1400 ssize_t ret; 1401 bool raw_epp = false; 1402 u8 epp; 1403 1404 if (cpudata->dynamic_epp) { 1405 pr_debug("EPP cannot be set when dynamic EPP is enabled\n"); 1406 return -EBUSY; 1407 } 1408 1409 /* 1410 * if the value matches a number, use that, otherwise see if 1411 * matches an index in the energy_perf_strings array 1412 */ 1413 ret = kstrtou8(buf, 0, &epp); 1414 raw_epp = !ret; 1415 if (ret) { 1416 ret = sysfs_match_string(energy_perf_strings, buf); 1417 if (ret < 0 || ret == EPP_INDEX_CUSTOM) 1418 return -EINVAL; 1419 if (ret) 1420 epp = epp_values[ret]; 1421 else 1422 epp = cpudata->epp_default_dc; 1423 } 1424 1425 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 1426 pr_debug("EPP cannot be set under performance policy\n"); 1427 return -EBUSY; 1428 } 1429 1430 ret = amd_pstate_set_epp(policy, epp); 1431 if (ret) 1432 return ret; 1433 1434 cpudata->raw_epp = raw_epp; 1435 1436 return count; 1437 } 1438 EXPORT_SYMBOL_GPL(store_energy_performance_preference); 1439 1440 ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf) 1441 { 1442 struct amd_cpudata *cpudata = policy->driver_data; 1443 u8 preference, epp; 1444 1445 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 1446 1447 if (cpudata->raw_epp) 1448 return sysfs_emit(buf, "%u\n", epp); 1449 1450 switch (epp) { 1451 case AMD_CPPC_EPP_PERFORMANCE: 1452 preference = EPP_INDEX_PERFORMANCE; 1453 break; 1454 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 1455 preference = EPP_INDEX_BALANCE_PERFORMANCE; 1456 break; 1457 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 1458 preference = EPP_INDEX_BALANCE_POWERSAVE; 1459 break; 1460 case AMD_CPPC_EPP_POWERSAVE: 1461 preference = EPP_INDEX_POWERSAVE; 1462 break; 1463 default: 1464 return -EINVAL; 1465 } 1466 1467 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 1468 } 1469 EXPORT_SYMBOL_GPL(show_energy_performance_preference); 1470 1471 static ssize_t store_amd_pstate_floor_freq(struct cpufreq_policy *policy, 1472 const char *buf, size_t count) 1473 { 1474 struct amd_cpudata *cpudata = policy->driver_data; 1475 union perf_cached perf = READ_ONCE(cpudata->perf); 1476 unsigned int freq; 1477 u8 floor_perf; 1478 int ret; 1479 1480 ret = kstrtouint(buf, 0, &freq); 1481 if (ret) 1482 return ret; 1483 1484 if (freq < policy->cpuinfo.min_freq || freq > policy->max) 1485 return -EINVAL; 1486 1487 floor_perf = freq_to_perf(perf, cpudata->nominal_freq, freq); 1488 ret = amd_pstate_set_floor_perf(policy, floor_perf); 1489 1490 if (!ret) 1491 cpudata->floor_freq = freq; 1492 1493 return ret ?: count; 1494 } 1495 1496 static ssize_t show_amd_pstate_floor_freq(struct cpufreq_policy *policy, char *buf) 1497 { 1498 struct amd_cpudata *cpudata = policy->driver_data; 1499 1500 return sysfs_emit(buf, "%u\n", cpudata->floor_freq); 1501 } 1502 1503 static ssize_t show_amd_pstate_floor_count(struct cpufreq_policy *policy, char *buf) 1504 { 1505 struct amd_cpudata *cpudata = policy->driver_data; 1506 u8 count = cpudata->floor_perf_cnt; 1507 1508 return sysfs_emit(buf, "%u\n", count); 1509 } 1510 1511 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1512 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1513 1514 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1515 cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); 1516 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); 1517 cpufreq_freq_attr_rw(energy_performance_preference); 1518 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1519 cpufreq_freq_attr_rw(amd_pstate_floor_freq); 1520 cpufreq_freq_attr_ro(amd_pstate_floor_count); 1521 1522 struct freq_attr_visibility { 1523 struct freq_attr *attr; 1524 bool (*visibility_fn)(void); 1525 }; 1526 1527 /* For attributes which are always visible */ 1528 static bool always_visible(void) 1529 { 1530 return true; 1531 } 1532 1533 /* Determines whether prefcore related attributes should be visible */ 1534 static bool prefcore_visibility(void) 1535 { 1536 return amd_pstate_prefcore; 1537 } 1538 1539 /* Determines whether energy performance preference should be visible */ 1540 static bool epp_visibility(void) 1541 { 1542 return cppc_state == AMD_PSTATE_ACTIVE; 1543 } 1544 1545 /* Determines whether amd_pstate_floor_freq related attributes should be visible */ 1546 static bool floor_freq_visibility(void) 1547 { 1548 return cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO); 1549 } 1550 1551 static struct freq_attr_visibility amd_pstate_attr_visibility[] = { 1552 {&amd_pstate_max_freq, always_visible}, 1553 {&amd_pstate_lowest_nonlinear_freq, always_visible}, 1554 {&amd_pstate_highest_perf, always_visible}, 1555 {&amd_pstate_prefcore_ranking, prefcore_visibility}, 1556 {&amd_pstate_hw_prefcore, prefcore_visibility}, 1557 {&energy_performance_preference, epp_visibility}, 1558 {&energy_performance_available_preferences, epp_visibility}, 1559 {&amd_pstate_floor_freq, floor_freq_visibility}, 1560 {&amd_pstate_floor_count, floor_freq_visibility}, 1561 }; 1562 1563 struct freq_attr **amd_pstate_get_current_attrs(void) 1564 { 1565 if (!current_pstate_driver) 1566 return NULL; 1567 return current_pstate_driver->attr; 1568 } 1569 EXPORT_SYMBOL_GPL(amd_pstate_get_current_attrs); 1570 1571 static struct freq_attr **get_freq_attrs(void) 1572 { 1573 bool attr_visible[ARRAY_SIZE(amd_pstate_attr_visibility)]; 1574 struct freq_attr **attrs; 1575 int i, j, count; 1576 1577 for (i = 0, count = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { 1578 struct freq_attr_visibility *v = &amd_pstate_attr_visibility[i]; 1579 1580 attr_visible[i] = v->visibility_fn(); 1581 if (attr_visible[i]) 1582 count++; 1583 } 1584 1585 /* amd_pstate_{max_freq, lowest_nonlinear_freq, highest_perf} should always be visible */ 1586 BUG_ON(!count); 1587 1588 attrs = kcalloc(count + 1, sizeof(struct freq_attr *), GFP_KERNEL); 1589 if (!attrs) 1590 return ERR_PTR(-ENOMEM); 1591 1592 for (i = 0, j = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { 1593 if (!attr_visible[i]) 1594 continue; 1595 1596 attrs[j++] = amd_pstate_attr_visibility[i].attr; 1597 } 1598 1599 return attrs; 1600 } 1601 1602 static void amd_pstate_driver_cleanup(void) 1603 { 1604 if (amd_pstate_prefcore) 1605 sched_clear_itmt_support(); 1606 1607 cppc_state = AMD_PSTATE_DISABLE; 1608 kfree(current_pstate_driver->attr); 1609 current_pstate_driver->attr = NULL; 1610 current_pstate_driver = NULL; 1611 } 1612 1613 static int amd_pstate_set_driver(int mode_idx) 1614 { 1615 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1616 cppc_state = mode_idx; 1617 if (cppc_state == AMD_PSTATE_DISABLE) 1618 pr_info("driver is explicitly disabled\n"); 1619 1620 if (cppc_state == AMD_PSTATE_ACTIVE) 1621 current_pstate_driver = &amd_pstate_epp_driver; 1622 1623 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1624 current_pstate_driver = &amd_pstate_driver; 1625 1626 return 0; 1627 } 1628 1629 return -EINVAL; 1630 } 1631 1632 static int amd_pstate_register_driver(int mode) 1633 { 1634 struct freq_attr **attr = NULL; 1635 int ret; 1636 1637 ret = amd_pstate_set_driver(mode); 1638 if (ret) 1639 return ret; 1640 1641 cppc_state = mode; 1642 1643 /* 1644 * Note: It is important to compute the attrs _after_ 1645 * re-initializing the cppc_state. Some attributes become 1646 * visible only when cppc_state is AMD_PSTATE_ACTIVE. 1647 */ 1648 attr = get_freq_attrs(); 1649 if (IS_ERR(attr)) { 1650 ret = (int) PTR_ERR(attr); 1651 pr_err("Couldn't compute freq_attrs for current mode %s [%d]\n", 1652 amd_pstate_get_mode_string(cppc_state), ret); 1653 amd_pstate_driver_cleanup(); 1654 return ret; 1655 } 1656 1657 current_pstate_driver->attr = attr; 1658 1659 /* at least one CPU supports CPB */ 1660 current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); 1661 1662 ret = cpufreq_register_driver(current_pstate_driver); 1663 if (ret) { 1664 amd_pstate_driver_cleanup(); 1665 return ret; 1666 } 1667 1668 /* Enable ITMT support once all CPUs have initialized their asym priorities. */ 1669 if (amd_pstate_prefcore) 1670 sched_set_itmt_support(); 1671 1672 return 0; 1673 } 1674 1675 static int amd_pstate_unregister_driver(int dummy) 1676 { 1677 cpufreq_unregister_driver(current_pstate_driver); 1678 amd_pstate_driver_cleanup(); 1679 return 0; 1680 } 1681 1682 static int amd_pstate_change_mode_without_dvr_change(int mode) 1683 { 1684 int cpu = 0; 1685 1686 cppc_state = mode; 1687 1688 if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 1689 return 0; 1690 1691 for_each_online_cpu(cpu) { 1692 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 1693 } 1694 1695 return 0; 1696 } 1697 1698 static int amd_pstate_change_driver_mode(int mode) 1699 { 1700 int ret; 1701 1702 lockdep_assert_held(&amd_pstate_driver_lock); 1703 1704 ret = amd_pstate_unregister_driver(0); 1705 if (ret) 1706 return ret; 1707 1708 ret = amd_pstate_register_driver(mode); 1709 if (ret) 1710 return ret; 1711 1712 return 0; 1713 } 1714 1715 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 1716 [AMD_PSTATE_DISABLE] = { 1717 [AMD_PSTATE_DISABLE] = NULL, 1718 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 1719 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 1720 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 1721 }, 1722 [AMD_PSTATE_PASSIVE] = { 1723 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1724 [AMD_PSTATE_PASSIVE] = NULL, 1725 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1726 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 1727 }, 1728 [AMD_PSTATE_ACTIVE] = { 1729 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1730 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 1731 [AMD_PSTATE_ACTIVE] = NULL, 1732 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 1733 }, 1734 [AMD_PSTATE_GUIDED] = { 1735 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1736 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 1737 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1738 [AMD_PSTATE_GUIDED] = NULL, 1739 }, 1740 }; 1741 1742 static ssize_t amd_pstate_show_status(char *buf) 1743 { 1744 if (!current_pstate_driver) 1745 return sysfs_emit(buf, "disable\n"); 1746 1747 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 1748 } 1749 1750 int amd_pstate_get_status(void) 1751 { 1752 return cppc_state; 1753 } 1754 EXPORT_SYMBOL_GPL(amd_pstate_get_status); 1755 1756 int amd_pstate_update_status(const char *buf, size_t size) 1757 { 1758 int mode_idx; 1759 1760 if (size > strlen("passive") || size < strlen("active")) 1761 return -EINVAL; 1762 1763 mode_idx = get_mode_idx_from_str(buf, size); 1764 if (mode_idx < 0) 1765 return mode_idx; 1766 1767 if (mode_state_machine[cppc_state][mode_idx]) { 1768 guard(mutex)(&amd_pstate_driver_lock); 1769 return mode_state_machine[cppc_state][mode_idx](mode_idx); 1770 } 1771 1772 return 0; 1773 } 1774 EXPORT_SYMBOL_GPL(amd_pstate_update_status); 1775 1776 static ssize_t status_show(struct device *dev, 1777 struct device_attribute *attr, char *buf) 1778 { 1779 1780 guard(mutex)(&amd_pstate_driver_lock); 1781 1782 return amd_pstate_show_status(buf); 1783 } 1784 1785 static ssize_t status_store(struct device *a, struct device_attribute *b, 1786 const char *buf, size_t count) 1787 { 1788 char *p = memchr(buf, '\n', count); 1789 int ret; 1790 1791 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1792 1793 return ret < 0 ? ret : count; 1794 } 1795 1796 static ssize_t prefcore_show(struct device *dev, 1797 struct device_attribute *attr, char *buf) 1798 { 1799 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); 1800 } 1801 1802 static ssize_t dynamic_epp_show(struct device *dev, 1803 struct device_attribute *attr, char *buf) 1804 { 1805 return sysfs_emit(buf, "%s\n", str_enabled_disabled(dynamic_epp)); 1806 } 1807 1808 static ssize_t dynamic_epp_store(struct device *a, struct device_attribute *b, 1809 const char *buf, size_t count) 1810 { 1811 bool enabled; 1812 int ret; 1813 1814 ret = kstrtobool(buf, &enabled); 1815 if (ret) 1816 return ret; 1817 1818 guard(mutex)(&amd_pstate_driver_lock); 1819 1820 if (cppc_state != AMD_PSTATE_ACTIVE) { 1821 pr_debug("dynamic_epp can only be toggled in active mode\n"); 1822 return -EINVAL; 1823 } 1824 1825 /* Nothing to do */ 1826 if (dynamic_epp == enabled) 1827 return count; 1828 1829 /* reinitialize with desired dynamic EPP value */ 1830 dynamic_epp = enabled; 1831 ret = amd_pstate_change_driver_mode(cppc_state); 1832 if (ret) 1833 dynamic_epp = false; 1834 1835 return ret ? ret : count; 1836 } 1837 1838 static DEVICE_ATTR_RW(status); 1839 static DEVICE_ATTR_RO(prefcore); 1840 static DEVICE_ATTR_RW(dynamic_epp); 1841 1842 static struct attribute *pstate_global_attributes[] = { 1843 &dev_attr_status.attr, 1844 &dev_attr_prefcore.attr, 1845 &dev_attr_dynamic_epp.attr, 1846 NULL 1847 }; 1848 1849 static const struct attribute_group amd_pstate_global_attr_group = { 1850 .name = "amd_pstate", 1851 .attrs = pstate_global_attributes, 1852 }; 1853 1854 static bool amd_pstate_acpi_pm_profile_server(void) 1855 { 1856 switch (acpi_gbl_FADT.preferred_profile) { 1857 case PM_ENTERPRISE_SERVER: 1858 case PM_SOHO_SERVER: 1859 case PM_PERFORMANCE_SERVER: 1860 return true; 1861 } 1862 return false; 1863 } 1864 1865 static bool amd_pstate_acpi_pm_profile_undefined(void) 1866 { 1867 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) 1868 return true; 1869 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) 1870 return true; 1871 return false; 1872 } 1873 1874 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1875 { 1876 struct amd_cpudata *cpudata; 1877 union perf_cached perf; 1878 struct device *dev; 1879 int ret; 1880 1881 /* 1882 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1883 * which is ideal for initialization process. 1884 */ 1885 amd_perf_ctl_reset(policy->cpu); 1886 dev = get_cpu_device(policy->cpu); 1887 if (!dev) 1888 return -ENODEV; 1889 1890 cpudata = kzalloc_obj(*cpudata); 1891 if (!cpudata) 1892 return -ENOMEM; 1893 1894 cpudata->cpu = policy->cpu; 1895 1896 ret = amd_pstate_init_perf(cpudata); 1897 if (ret) 1898 goto free_cpudata1; 1899 1900 amd_pstate_init_prefcore(cpudata); 1901 1902 ret = amd_pstate_init_freq(cpudata); 1903 if (ret) 1904 goto free_cpudata1; 1905 1906 ret = amd_pstate_init_boost_support(cpudata); 1907 if (ret) 1908 goto free_cpudata1; 1909 1910 perf = READ_ONCE(cpudata->perf); 1911 1912 policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq, 1913 perf.lowest_perf); 1914 policy->cpuinfo.max_freq = cpudata->max_freq; 1915 policy->driver_data = cpudata; 1916 1917 ret = amd_pstate_cppc_enable(policy); 1918 if (ret) 1919 goto free_cpudata1; 1920 1921 /* It will be updated by governor */ 1922 policy->cur = policy->cpuinfo.min_freq; 1923 1924 1925 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1926 1927 /* 1928 * Set the policy to provide a valid fallback value in case 1929 * the default cpufreq governor is neither powersave nor performance. 1930 */ 1931 if (amd_pstate_acpi_pm_profile_server() || 1932 amd_pstate_acpi_pm_profile_undefined()) { 1933 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 1934 cpudata->epp_default_ac = cpudata->epp_default_dc = amd_pstate_get_epp(cpudata); 1935 cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE; 1936 } else { 1937 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1938 cpudata->epp_default_ac = AMD_CPPC_EPP_PERFORMANCE; 1939 cpudata->epp_default_dc = AMD_CPPC_EPP_BALANCE_PERFORMANCE; 1940 cpudata->current_profile = PLATFORM_PROFILE_BALANCED; 1941 } 1942 1943 if (dynamic_epp) 1944 ret = amd_pstate_set_dynamic_epp(policy); 1945 else 1946 ret = amd_pstate_set_epp(policy, cpudata->epp_default_dc); 1947 if (ret) 1948 goto free_cpudata1; 1949 1950 ret = amd_pstate_init_floor_perf(policy); 1951 if (ret) { 1952 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); 1953 goto free_cpudata1; 1954 } 1955 1956 current_pstate_driver->adjust_perf = NULL; 1957 1958 return 0; 1959 1960 free_cpudata1: 1961 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1962 kfree(cpudata); 1963 policy->driver_data = NULL; 1964 return ret; 1965 } 1966 1967 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1968 { 1969 struct amd_cpudata *cpudata = policy->driver_data; 1970 1971 if (cpudata) { 1972 union perf_cached perf = READ_ONCE(cpudata->perf); 1973 1974 if (cpudata->dynamic_epp) 1975 amd_pstate_clear_dynamic_epp(policy); 1976 1977 /* Reset CPPC_REQ MSR to the BIOS value */ 1978 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1979 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 1980 1981 kfree(cpudata); 1982 policy->driver_data = NULL; 1983 } 1984 1985 pr_debug("CPU %d exiting\n", policy->cpu); 1986 } 1987 1988 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy, bool policy_change) 1989 { 1990 struct amd_cpudata *cpudata = policy->driver_data; 1991 union perf_cached perf; 1992 u8 epp; 1993 1994 if (policy_change || 1995 policy->min != cpudata->min_limit_freq || 1996 policy->max != cpudata->max_limit_freq) 1997 amd_pstate_update_min_max_limit(policy); 1998 1999 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 2000 epp = 0; 2001 else 2002 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 2003 2004 perf = READ_ONCE(cpudata->perf); 2005 2006 return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U, 2007 perf.max_limit_perf, epp, false); 2008 } 2009 2010 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 2011 { 2012 struct amd_cpudata *cpudata = policy->driver_data; 2013 int ret; 2014 2015 if (!policy->cpuinfo.max_freq) 2016 return -ENODEV; 2017 2018 cpudata->policy = policy->policy; 2019 2020 ret = amd_pstate_epp_update_limit(policy, true); 2021 if (ret) 2022 return ret; 2023 2024 /* 2025 * policy->cur is never updated with the amd_pstate_epp driver, but it 2026 * is used as a stale frequency value. So, keep it within limits. 2027 */ 2028 policy->cur = policy->min; 2029 2030 return 0; 2031 } 2032 2033 static int amd_pstate_cpu_online(struct cpufreq_policy *policy) 2034 { 2035 struct amd_cpudata *cpudata = policy->driver_data; 2036 union perf_cached perf = READ_ONCE(cpudata->perf); 2037 u8 cached_floor_perf; 2038 int ret; 2039 2040 ret = amd_pstate_cppc_enable(policy); 2041 if (ret) 2042 return ret; 2043 2044 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2045 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2046 } 2047 2048 static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) 2049 { 2050 struct amd_cpudata *cpudata = policy->driver_data; 2051 union perf_cached perf = READ_ONCE(cpudata->perf); 2052 int ret; 2053 2054 /* 2055 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 2056 * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the 2057 * limits, epp and desired perf will get reset to the cached values in cpudata struct 2058 */ 2059 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 2060 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 2061 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 2062 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 2063 false); 2064 if (ret) 2065 return ret; 2066 2067 return amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2068 } 2069 2070 static int amd_pstate_suspend(struct cpufreq_policy *policy) 2071 { 2072 struct amd_cpudata *cpudata = policy->driver_data; 2073 union perf_cached perf = READ_ONCE(cpudata->perf); 2074 int ret; 2075 2076 /* 2077 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 2078 * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, 2079 * the limits, epp and desired perf will get reset to the cached values in cpudata struct 2080 */ 2081 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 2082 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 2083 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 2084 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 2085 false); 2086 if (ret) 2087 return ret; 2088 2089 ret = amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2090 if (ret) 2091 return ret; 2092 2093 /* set this flag to avoid setting core offline*/ 2094 cpudata->suspended = true; 2095 2096 return 0; 2097 } 2098 2099 static int amd_pstate_resume(struct cpufreq_policy *policy) 2100 { 2101 struct amd_cpudata *cpudata = policy->driver_data; 2102 union perf_cached perf = READ_ONCE(cpudata->perf); 2103 int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur); 2104 u8 cached_floor_perf; 2105 int ret; 2106 2107 /* Set CPPC_REQ to last sane value until the governor updates it */ 2108 ret = amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, 2109 0U, false); 2110 if (ret) 2111 return ret; 2112 2113 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2114 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2115 } 2116 2117 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 2118 { 2119 struct amd_cpudata *cpudata = policy->driver_data; 2120 union perf_cached perf = READ_ONCE(cpudata->perf); 2121 u8 cached_floor_perf; 2122 2123 if (cpudata->suspended) { 2124 int ret; 2125 2126 /* enable amd pstate from suspend state*/ 2127 ret = amd_pstate_epp_update_limit(policy, false); 2128 if (ret) 2129 return ret; 2130 2131 cpudata->suspended = false; 2132 } 2133 2134 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2135 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2136 } 2137 2138 static struct cpufreq_driver amd_pstate_driver = { 2139 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 2140 .verify = amd_pstate_verify, 2141 .target = amd_pstate_target, 2142 .fast_switch = amd_pstate_fast_switch, 2143 .init = amd_pstate_cpu_init, 2144 .exit = amd_pstate_cpu_exit, 2145 .online = amd_pstate_cpu_online, 2146 .offline = amd_pstate_cpu_offline, 2147 .suspend = amd_pstate_suspend, 2148 .resume = amd_pstate_resume, 2149 .set_boost = amd_pstate_set_boost, 2150 .update_limits = amd_pstate_update_limits, 2151 .name = "amd-pstate", 2152 }; 2153 2154 static struct cpufreq_driver amd_pstate_epp_driver = { 2155 .flags = CPUFREQ_CONST_LOOPS, 2156 .verify = amd_pstate_verify, 2157 .setpolicy = amd_pstate_epp_set_policy, 2158 .init = amd_pstate_epp_cpu_init, 2159 .exit = amd_pstate_epp_cpu_exit, 2160 .offline = amd_pstate_cpu_offline, 2161 .online = amd_pstate_cpu_online, 2162 .suspend = amd_pstate_suspend, 2163 .resume = amd_pstate_epp_resume, 2164 .update_limits = amd_pstate_update_limits, 2165 .set_boost = amd_pstate_set_boost, 2166 .name = "amd-pstate-epp", 2167 }; 2168 2169 /* 2170 * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. 2171 * show the debug message that helps to check if the CPU has CPPC support for loading issue. 2172 */ 2173 static bool amd_cppc_supported(void) 2174 { 2175 struct cpuinfo_x86 *c = &cpu_data(0); 2176 bool warn = false; 2177 2178 if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) { 2179 pr_debug_once("CPPC feature is not supported by the processor\n"); 2180 return false; 2181 } 2182 2183 /* 2184 * If the CPPC feature is disabled in the BIOS for processors 2185 * that support MSR-based CPPC, the AMD Pstate driver may not 2186 * function correctly. 2187 * 2188 * For such processors, check the CPPC flag and display a 2189 * warning message if the platform supports CPPC. 2190 * 2191 * Note: The code check below will not abort the driver 2192 * registration process because of the code is added for 2193 * debugging purposes. Besides, it may still be possible for 2194 * the driver to work using the shared-memory mechanism. 2195 */ 2196 if (!cpu_feature_enabled(X86_FEATURE_CPPC)) { 2197 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 2198 switch (c->x86_model) { 2199 case 0x60 ... 0x6F: 2200 case 0x80 ... 0xAF: 2201 warn = true; 2202 break; 2203 } 2204 } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || 2205 cpu_feature_enabled(X86_FEATURE_ZEN4)) { 2206 switch (c->x86_model) { 2207 case 0x10 ... 0x1F: 2208 case 0x40 ... 0xAF: 2209 warn = true; 2210 break; 2211 } 2212 } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) { 2213 warn = true; 2214 } 2215 } 2216 2217 if (warn) 2218 pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n" 2219 "Please enable it if your BIOS has the CPPC option.\n"); 2220 return true; 2221 } 2222 2223 static int __init amd_pstate_init(void) 2224 { 2225 struct device *dev_root; 2226 int ret; 2227 2228 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 2229 return -ENODEV; 2230 2231 /* show debug message only if CPPC is not supported */ 2232 if (!amd_cppc_supported()) 2233 return -EOPNOTSUPP; 2234 2235 /* show warning message when BIOS broken or ACPI disabled */ 2236 if (!acpi_cpc_valid()) { 2237 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 2238 return -ENODEV; 2239 } 2240 2241 /* don't keep reloading if cpufreq_driver exists */ 2242 if (cpufreq_get_current_driver()) 2243 return -EEXIST; 2244 2245 quirks = NULL; 2246 2247 /* check if this machine need CPPC quirks */ 2248 dmi_check_system(amd_pstate_quirks_table); 2249 2250 /* 2251 * determine the driver mode from the command line or kernel config. 2252 * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED. 2253 * command line options will override the kernel config settings. 2254 */ 2255 2256 if (cppc_state == AMD_PSTATE_UNDEFINED) { 2257 /* Disable on the following configs by default: 2258 * 1. Undefined platforms 2259 * 2. Server platforms with CPUs older than Family 0x1A. 2260 */ 2261 if (amd_pstate_acpi_pm_profile_undefined() || 2262 (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) { 2263 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 2264 return -ENODEV; 2265 } 2266 /* get driver mode from kernel config option [1:4] */ 2267 cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; 2268 } 2269 2270 if (cppc_state == AMD_PSTATE_DISABLE) { 2271 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 2272 return -ENODEV; 2273 } 2274 2275 /* capability check */ 2276 if (cpu_feature_enabled(X86_FEATURE_CPPC)) { 2277 pr_debug("AMD CPPC MSR based functionality is supported\n"); 2278 } else { 2279 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 2280 static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); 2281 static_call_update(amd_pstate_init_perf, shmem_init_perf); 2282 static_call_update(amd_pstate_update_perf, shmem_update_perf); 2283 static_call_update(amd_pstate_get_epp, shmem_get_epp); 2284 static_call_update(amd_pstate_set_epp, shmem_set_epp); 2285 } 2286 2287 if (amd_pstate_prefcore) { 2288 ret = amd_detect_prefcore(&amd_pstate_prefcore); 2289 if (ret) 2290 return ret; 2291 } 2292 2293 ret = amd_pstate_register_driver(cppc_state); 2294 if (ret) { 2295 pr_err("failed to register with return %d\n", ret); 2296 return ret; 2297 } 2298 2299 dev_root = bus_get_dev_root(&cpu_subsys); 2300 if (dev_root) { 2301 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 2302 put_device(dev_root); 2303 if (ret) { 2304 pr_err("sysfs attribute export failed with error %d.\n", ret); 2305 goto global_attr_free; 2306 } 2307 } 2308 2309 return ret; 2310 2311 global_attr_free: 2312 amd_pstate_unregister_driver(0); 2313 return ret; 2314 } 2315 device_initcall(amd_pstate_init); 2316 2317 static int __init amd_pstate_param(char *str) 2318 { 2319 size_t size; 2320 int mode_idx; 2321 2322 if (!str) 2323 return -EINVAL; 2324 2325 size = strlen(str); 2326 mode_idx = get_mode_idx_from_str(str, size); 2327 2328 return amd_pstate_set_driver(mode_idx); 2329 } 2330 2331 static int __init amd_prefcore_param(char *str) 2332 { 2333 if (!strcmp(str, "disable")) 2334 amd_pstate_prefcore = false; 2335 2336 return 0; 2337 } 2338 2339 static int __init amd_dynamic_epp_param(char *str) 2340 { 2341 if (!strcmp(str, "disable")) 2342 dynamic_epp = false; 2343 if (!strcmp(str, "enable")) 2344 dynamic_epp = true; 2345 2346 return 0; 2347 } 2348 2349 early_param("amd_pstate", amd_pstate_param); 2350 early_param("amd_prefcore", amd_prefcore_param); 2351 early_param("amd_dynamic_epp", amd_dynamic_epp_param); 2352 2353 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 2354 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 2355