1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/bitfield.h> 26 #include <linux/kernel.h> 27 #include <linux/module.h> 28 #include <linux/init.h> 29 #include <linux/smp.h> 30 #include <linux/sched.h> 31 #include <linux/cpufreq.h> 32 #include <linux/compiler.h> 33 #include <linux/dmi.h> 34 #include <linux/slab.h> 35 #include <linux/acpi.h> 36 #include <linux/io.h> 37 #include <linux/delay.h> 38 #include <linux/uaccess.h> 39 #include <linux/power_supply.h> 40 #include <linux/static_call.h> 41 #include <linux/topology.h> 42 43 #include <acpi/processor.h> 44 #include <acpi/cppc_acpi.h> 45 46 #include <asm/msr.h> 47 #include <asm/processor.h> 48 #include <asm/cpufeature.h> 49 #include <asm/cpu_device_id.h> 50 51 #include "amd-pstate.h" 52 #include "amd-pstate-trace.h" 53 54 #define AMD_PSTATE_TRANSITION_LATENCY 20000 55 #define AMD_PSTATE_TRANSITION_DELAY 1000 56 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600 57 58 #define AMD_CPPC_EPP_PERFORMANCE 0x00 59 #define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 60 #define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF 61 #define AMD_CPPC_EPP_POWERSAVE 0xFF 62 63 static const char * const amd_pstate_mode_string[] = { 64 [AMD_PSTATE_UNDEFINED] = "undefined", 65 [AMD_PSTATE_DISABLE] = "disable", 66 [AMD_PSTATE_PASSIVE] = "passive", 67 [AMD_PSTATE_ACTIVE] = "active", 68 [AMD_PSTATE_GUIDED] = "guided", 69 }; 70 static_assert(ARRAY_SIZE(amd_pstate_mode_string) == AMD_PSTATE_MAX); 71 72 const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode) 73 { 74 if (mode < AMD_PSTATE_UNDEFINED || mode >= AMD_PSTATE_MAX) 75 mode = AMD_PSTATE_UNDEFINED; 76 return amd_pstate_mode_string[mode]; 77 } 78 EXPORT_SYMBOL_GPL(amd_pstate_get_mode_string); 79 80 struct quirk_entry { 81 u32 nominal_freq; 82 u32 lowest_freq; 83 }; 84 85 static struct cpufreq_driver *current_pstate_driver; 86 static struct cpufreq_driver amd_pstate_driver; 87 static struct cpufreq_driver amd_pstate_epp_driver; 88 static int cppc_state = AMD_PSTATE_UNDEFINED; 89 static bool amd_pstate_prefcore = true; 90 static bool dynamic_epp; 91 static struct quirk_entry *quirks; 92 93 /* 94 * AMD Energy Preference Performance (EPP) 95 * The EPP is used in the CCLK DPM controller to drive 96 * the frequency that a core is going to operate during 97 * short periods of activity. EPP values will be utilized for 98 * different OS profiles (balanced, performance, power savings) 99 * display strings corresponding to EPP index in the 100 * energy_perf_strings[] 101 * index String 102 *------------------------------------- 103 * 0 default 104 * 1 performance 105 * 2 balance_performance 106 * 3 balance_power 107 * 4 power 108 * 5 custom (for raw EPP values) 109 */ 110 enum energy_perf_value_index { 111 EPP_INDEX_DEFAULT = 0, 112 EPP_INDEX_PERFORMANCE, 113 EPP_INDEX_BALANCE_PERFORMANCE, 114 EPP_INDEX_BALANCE_POWERSAVE, 115 EPP_INDEX_POWERSAVE, 116 EPP_INDEX_CUSTOM, 117 EPP_INDEX_MAX, 118 }; 119 120 static const char * const energy_perf_strings[] = { 121 [EPP_INDEX_DEFAULT] = "default", 122 [EPP_INDEX_PERFORMANCE] = "performance", 123 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 124 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 125 [EPP_INDEX_POWERSAVE] = "power", 126 [EPP_INDEX_CUSTOM] = "custom", 127 }; 128 static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); 129 130 static unsigned int epp_values[] = { 131 [EPP_INDEX_DEFAULT] = 0, 132 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 133 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 134 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 135 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 136 }; 137 static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX - 1); 138 139 typedef int (*cppc_mode_transition_fn)(int); 140 141 static struct quirk_entry quirk_amd_7k62 = { 142 .nominal_freq = 2600, 143 .lowest_freq = 550, 144 }; 145 146 static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) 147 { 148 u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); 149 150 return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf); 151 } 152 153 static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val) 154 { 155 return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val, 156 perf.nominal_perf); 157 } 158 159 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) 160 { 161 /** 162 * match the broken bios for family 17h processor support CPPC V2 163 * broken BIOS lack of nominal_freq and lowest_freq capabilities 164 * definition in ACPI tables 165 */ 166 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 167 quirks = dmi->driver_data; 168 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); 169 return 1; 170 } 171 172 return 0; 173 } 174 175 static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { 176 { 177 .callback = dmi_matched_7k62_bios_bug, 178 .ident = "AMD EPYC 7K62", 179 .matches = { 180 DMI_MATCH(DMI_BIOS_VERSION, "5.14"), 181 DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), 182 }, 183 .driver_data = &quirk_amd_7k62, 184 }, 185 {} 186 }; 187 MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); 188 189 static inline int get_mode_idx_from_str(const char *str, size_t size) 190 { 191 int i; 192 193 for (i = 0; i < AMD_PSTATE_MAX; i++) { 194 if (!strncmp(str, amd_pstate_mode_string[i], size)) 195 return i; 196 } 197 return -EINVAL; 198 } 199 200 static DEFINE_MUTEX(amd_pstate_driver_lock); 201 202 static u8 msr_get_epp(struct amd_cpudata *cpudata) 203 { 204 u64 value; 205 int ret; 206 207 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 208 if (ret < 0) { 209 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 210 return ret; 211 } 212 213 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value); 214 } 215 216 DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); 217 218 static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) 219 { 220 return static_call(amd_pstate_get_epp)(cpudata); 221 } 222 223 static u8 shmem_get_epp(struct amd_cpudata *cpudata) 224 { 225 u64 epp; 226 int ret; 227 228 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 229 if (ret < 0) { 230 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 231 return ret; 232 } 233 234 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp); 235 } 236 237 static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, 238 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 239 { 240 struct amd_cpudata *cpudata = policy->driver_data; 241 u64 value, prev; 242 243 value = prev = READ_ONCE(cpudata->cppc_req_cached); 244 245 value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | 246 AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); 247 value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); 248 value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); 249 value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); 250 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 251 252 if (trace_amd_pstate_epp_perf_enabled()) { 253 union perf_cached perf = READ_ONCE(cpudata->perf); 254 255 trace_call__amd_pstate_epp_perf(cpudata->cpu, 256 perf.highest_perf, 257 epp, 258 min_perf, 259 max_perf, 260 policy->boost_enabled, 261 value != prev); 262 } 263 264 if (value == prev) 265 return 0; 266 267 if (fast_switch) { 268 wrmsrq(MSR_AMD_CPPC_REQ, value); 269 } else { 270 int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 271 272 if (ret) 273 return ret; 274 } 275 276 WRITE_ONCE(cpudata->cppc_req_cached, value); 277 278 return 0; 279 } 280 281 DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); 282 283 static inline int amd_pstate_update_perf(struct cpufreq_policy *policy, 284 u8 min_perf, u8 des_perf, 285 u8 max_perf, u8 epp, 286 bool fast_switch) 287 { 288 return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf, 289 max_perf, epp, fast_switch); 290 } 291 292 static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) 293 { 294 struct amd_cpudata *cpudata = policy->driver_data; 295 u64 value, prev; 296 int ret; 297 298 value = prev = READ_ONCE(cpudata->cppc_req_cached); 299 value &= ~AMD_CPPC_EPP_PERF_MASK; 300 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 301 302 if (trace_amd_pstate_epp_perf_enabled()) { 303 union perf_cached perf = cpudata->perf; 304 305 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 306 epp, 307 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 308 cpudata->cppc_req_cached), 309 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 310 cpudata->cppc_req_cached), 311 policy->boost_enabled, 312 value != prev); 313 } 314 315 if (value == prev) 316 return 0; 317 318 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 319 if (ret) { 320 pr_err("failed to set energy perf value (%d)\n", ret); 321 return ret; 322 } 323 324 /* update both so that msr_update_perf() can effectively check */ 325 WRITE_ONCE(cpudata->cppc_req_cached, value); 326 327 return ret; 328 } 329 330 DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); 331 332 static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp) 333 { 334 return static_call(amd_pstate_set_epp)(policy, epp); 335 } 336 337 static int amd_pstate_set_floor_perf(struct cpufreq_policy *policy, u8 perf) 338 { 339 struct amd_cpudata *cpudata = policy->driver_data; 340 u64 value, prev; 341 bool changed; 342 int ret; 343 344 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) 345 return 0; 346 347 value = prev = READ_ONCE(cpudata->cppc_req2_cached); 348 FIELD_MODIFY(AMD_CPPC_FLOOR_PERF_MASK, &value, perf); 349 350 changed = value != prev; 351 if (!changed) { 352 ret = 0; 353 goto out_trace; 354 } 355 356 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, value); 357 if (ret) { 358 changed = false; 359 pr_err("failed to set CPPC REQ2 value. Error (%d)\n", ret); 360 goto out_trace; 361 } 362 363 WRITE_ONCE(cpudata->cppc_req2_cached, value); 364 365 out_trace: 366 if (trace_amd_pstate_cppc_req2_enabled()) 367 trace_amd_pstate_cppc_req2(cpudata->cpu, perf, changed, ret); 368 return ret; 369 } 370 371 static int amd_pstate_init_floor_perf(struct cpufreq_policy *policy) 372 { 373 struct amd_cpudata *cpudata = policy->driver_data; 374 u8 floor_perf; 375 u64 value; 376 int ret; 377 378 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) 379 return 0; 380 381 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, &value); 382 if (ret) { 383 pr_err("failed to read CPPC REQ2 value. Error (%d)\n", ret); 384 return ret; 385 } 386 387 WRITE_ONCE(cpudata->cppc_req2_cached, value); 388 floor_perf = FIELD_GET(AMD_CPPC_FLOOR_PERF_MASK, 389 cpudata->cppc_req2_cached); 390 391 /* Set a sane value for floor_perf if the default value is invalid */ 392 if (floor_perf < cpudata->perf.lowest_perf) { 393 floor_perf = cpudata->perf.nominal_perf; 394 ret = amd_pstate_set_floor_perf(policy, floor_perf); 395 if (ret) 396 return ret; 397 } 398 399 400 cpudata->bios_floor_perf = floor_perf; 401 cpudata->floor_freq = perf_to_freq(cpudata->perf, cpudata->nominal_freq, 402 floor_perf); 403 return 0; 404 } 405 406 static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) 407 { 408 struct amd_cpudata *cpudata = policy->driver_data; 409 struct cppc_perf_ctrls perf_ctrls; 410 u8 epp_cached; 411 u64 value; 412 int ret; 413 414 415 epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 416 if (trace_amd_pstate_epp_perf_enabled()) { 417 union perf_cached perf = cpudata->perf; 418 419 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 420 epp, 421 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 422 cpudata->cppc_req_cached), 423 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 424 cpudata->cppc_req_cached), 425 policy->boost_enabled, 426 epp != epp_cached); 427 } 428 429 if (epp == epp_cached) 430 return 0; 431 432 perf_ctrls.energy_perf = epp; 433 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 434 if (ret) { 435 pr_debug("failed to set energy perf value (%d)\n", ret); 436 return ret; 437 } 438 439 value = READ_ONCE(cpudata->cppc_req_cached); 440 value &= ~AMD_CPPC_EPP_PERF_MASK; 441 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 442 WRITE_ONCE(cpudata->cppc_req_cached, value); 443 444 return ret; 445 } 446 447 static inline int msr_cppc_enable(struct cpufreq_policy *policy) 448 { 449 return wrmsrq_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); 450 } 451 452 static int shmem_cppc_enable(struct cpufreq_policy *policy) 453 { 454 return cppc_set_enable(policy->cpu, 1); 455 } 456 457 DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); 458 459 static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) 460 { 461 return static_call(amd_pstate_cppc_enable)(policy); 462 } 463 464 static int msr_init_perf(struct amd_cpudata *cpudata) 465 { 466 union perf_cached perf = READ_ONCE(cpudata->perf); 467 u64 cap1, numerator, cppc_req; 468 u8 min_perf; 469 470 int ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 471 &cap1); 472 if (ret) 473 return ret; 474 475 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 476 if (ret) 477 return ret; 478 479 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); 480 if (ret) 481 return ret; 482 483 WRITE_ONCE(cpudata->cppc_req_cached, cppc_req); 484 min_perf = FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cppc_req); 485 486 /* 487 * Clear out the min_perf part to check if the rest of the MSR is 0, if yes, this is an 488 * indication that the min_perf value is the one specified through the BIOS option 489 */ 490 cppc_req &= ~(AMD_CPPC_MIN_PERF_MASK); 491 492 if (!cppc_req) 493 perf.bios_min_perf = min_perf; 494 495 perf.highest_perf = numerator; 496 perf.max_limit_perf = numerator; 497 perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 498 perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); 499 perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); 500 perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 501 WRITE_ONCE(cpudata->perf, perf); 502 WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1)); 503 WRITE_ONCE(cpudata->floor_perf_cnt, FIELD_GET(AMD_CPPC_FLOOR_PERF_CNT_MASK, cap1)); 504 505 return 0; 506 } 507 508 static int shmem_init_perf(struct amd_cpudata *cpudata) 509 { 510 struct cppc_perf_caps cppc_perf; 511 union perf_cached perf = READ_ONCE(cpudata->perf); 512 u64 numerator; 513 bool auto_sel; 514 515 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 516 if (ret) 517 return ret; 518 519 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 520 if (ret) 521 return ret; 522 523 perf.highest_perf = numerator; 524 perf.max_limit_perf = numerator; 525 perf.min_limit_perf = cppc_perf.lowest_perf; 526 perf.nominal_perf = cppc_perf.nominal_perf; 527 perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 528 perf.lowest_perf = cppc_perf.lowest_perf; 529 WRITE_ONCE(cpudata->perf, perf); 530 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); 531 532 if (cppc_state == AMD_PSTATE_ACTIVE) 533 return 0; 534 535 ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel); 536 if (ret) { 537 pr_warn("failed to get auto_sel, ret: %d\n", ret); 538 return 0; 539 } 540 541 ret = cppc_set_auto_sel(cpudata->cpu, 542 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 543 544 if (ret) 545 pr_warn("failed to set auto_sel, ret: %d\n", ret); 546 547 return ret; 548 } 549 550 DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf); 551 552 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 553 { 554 return static_call(amd_pstate_init_perf)(cpudata); 555 } 556 557 static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf, 558 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 559 { 560 struct amd_cpudata *cpudata = policy->driver_data; 561 struct cppc_perf_ctrls perf_ctrls; 562 u64 value, prev; 563 int ret; 564 565 if (cppc_state == AMD_PSTATE_ACTIVE) { 566 int ret = shmem_set_epp(policy, epp); 567 568 if (ret) 569 return ret; 570 } 571 572 value = prev = READ_ONCE(cpudata->cppc_req_cached); 573 574 value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | 575 AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); 576 value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); 577 value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); 578 value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); 579 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 580 581 if (trace_amd_pstate_epp_perf_enabled()) { 582 union perf_cached perf = READ_ONCE(cpudata->perf); 583 584 trace_call__amd_pstate_epp_perf(cpudata->cpu, 585 perf.highest_perf, 586 epp, 587 min_perf, 588 max_perf, 589 policy->boost_enabled, 590 value != prev); 591 } 592 593 if (value == prev) 594 return 0; 595 596 perf_ctrls.max_perf = max_perf; 597 perf_ctrls.min_perf = min_perf; 598 perf_ctrls.desired_perf = des_perf; 599 600 ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); 601 if (ret) 602 return ret; 603 604 WRITE_ONCE(cpudata->cppc_req_cached, value); 605 606 return 0; 607 } 608 609 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 610 { 611 u64 aperf, mperf, tsc; 612 unsigned long flags; 613 614 local_irq_save(flags); 615 rdmsrq(MSR_IA32_APERF, aperf); 616 rdmsrq(MSR_IA32_MPERF, mperf); 617 tsc = rdtsc(); 618 619 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 620 local_irq_restore(flags); 621 return false; 622 } 623 624 local_irq_restore(flags); 625 626 cpudata->cur.aperf = aperf; 627 cpudata->cur.mperf = mperf; 628 cpudata->cur.tsc = tsc; 629 cpudata->cur.aperf -= cpudata->prev.aperf; 630 cpudata->cur.mperf -= cpudata->prev.mperf; 631 cpudata->cur.tsc -= cpudata->prev.tsc; 632 633 cpudata->prev.aperf = aperf; 634 cpudata->prev.mperf = mperf; 635 cpudata->prev.tsc = tsc; 636 637 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 638 639 return true; 640 } 641 642 static void amd_pstate_update(struct cpufreq_policy *policy, u8 min_perf, 643 u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) 644 { 645 struct amd_cpudata *cpudata = policy->driver_data; 646 union perf_cached perf = READ_ONCE(cpudata->perf); 647 648 /* limit the max perf when core performance boost feature is disabled */ 649 if (!cpudata->boost_supported) 650 max_perf = min_t(u8, perf.nominal_perf, max_perf); 651 652 des_perf = clamp_t(u8, des_perf, min_perf, max_perf); 653 654 policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); 655 656 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 657 min_perf = des_perf; 658 des_perf = 0; 659 } 660 661 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 662 trace_call__amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 663 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 664 cpudata->cpu, fast_switch); 665 } 666 667 amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch); 668 } 669 670 static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) 671 { 672 /* 673 * Initialize lower frequency limit (i.e.policy->min) with 674 * lowest_nonlinear_frequency or the min frequency (if) specified in BIOS, 675 * Override the initial value set by cpufreq core and amd-pstate qos_requests. 676 */ 677 if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { 678 struct cpufreq_policy *policy __free(put_cpufreq_policy) = 679 cpufreq_cpu_get(policy_data->cpu); 680 struct amd_cpudata *cpudata; 681 union perf_cached perf; 682 683 if (!policy) 684 return -EINVAL; 685 686 cpudata = policy->driver_data; 687 perf = READ_ONCE(cpudata->perf); 688 689 if (perf.bios_min_perf) 690 policy_data->min = perf_to_freq(perf, cpudata->nominal_freq, 691 perf.bios_min_perf); 692 else 693 policy_data->min = cpudata->lowest_nonlinear_freq; 694 } 695 696 cpufreq_verify_within_cpu_limits(policy_data); 697 698 return 0; 699 } 700 701 static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) 702 { 703 struct amd_cpudata *cpudata = policy->driver_data; 704 union perf_cached perf = READ_ONCE(cpudata->perf); 705 706 perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); 707 WRITE_ONCE(cpudata->max_limit_freq, policy->max); 708 709 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 710 /* 711 * For performance policy, set MinPerf to nominal_perf rather than 712 * highest_perf or lowest_nonlinear_perf. 713 * 714 * Per commit 0c411b39e4f4c, using highest_perf was observed 715 * to cause frequency throttling on power-limited platforms, leading to 716 * performance regressions. Using lowest_nonlinear_perf would limit 717 * performance too much for HPC workloads requiring high frequency 718 * operation and minimal wakeup latency from idle states. 719 * 720 * nominal_perf therefore provides a balance by avoiding throttling 721 * while still maintaining enough performance for HPC workloads. 722 */ 723 perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); 724 WRITE_ONCE(cpudata->min_limit_freq, min(cpudata->nominal_freq, cpudata->max_limit_freq)); 725 } else { 726 perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); 727 WRITE_ONCE(cpudata->min_limit_freq, policy->min); 728 } 729 730 WRITE_ONCE(cpudata->perf, perf); 731 } 732 733 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 734 unsigned int target_freq, bool fast_switch) 735 { 736 struct cpufreq_freqs freqs; 737 struct amd_cpudata *cpudata; 738 union perf_cached perf; 739 u8 des_perf; 740 741 cpudata = policy->driver_data; 742 743 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 744 amd_pstate_update_min_max_limit(policy); 745 746 perf = READ_ONCE(cpudata->perf); 747 748 freqs.old = policy->cur; 749 freqs.new = target_freq; 750 751 des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq); 752 753 WARN_ON(fast_switch && !policy->fast_switch_enabled); 754 /* 755 * If fast_switch is desired, then there aren't any registered 756 * transition notifiers. See comment for 757 * cpufreq_enable_fast_switch(). 758 */ 759 if (!fast_switch) 760 cpufreq_freq_transition_begin(policy, &freqs); 761 762 amd_pstate_update(policy, perf.min_limit_perf, des_perf, 763 perf.max_limit_perf, fast_switch, 764 policy->governor->flags); 765 766 if (!fast_switch) 767 cpufreq_freq_transition_end(policy, &freqs, false); 768 769 return 0; 770 } 771 772 static int amd_pstate_target(struct cpufreq_policy *policy, 773 unsigned int target_freq, 774 unsigned int relation) 775 { 776 return amd_pstate_update_freq(policy, target_freq, false); 777 } 778 779 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 780 unsigned int target_freq) 781 { 782 if (!amd_pstate_update_freq(policy, target_freq, true)) 783 return target_freq; 784 return policy->cur; 785 } 786 787 static void amd_pstate_adjust_perf(struct cpufreq_policy *policy, 788 unsigned long _min_perf, 789 unsigned long target_perf, 790 unsigned long capacity) 791 { 792 u8 max_perf, min_perf, des_perf, cap_perf; 793 struct amd_cpudata *cpudata; 794 union perf_cached perf; 795 796 if (!policy) 797 return; 798 799 cpudata = policy->driver_data; 800 801 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 802 amd_pstate_update_min_max_limit(policy); 803 804 perf = READ_ONCE(cpudata->perf); 805 cap_perf = perf.highest_perf; 806 807 des_perf = cap_perf; 808 if (target_perf < capacity) 809 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 810 811 if (_min_perf < capacity) 812 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 813 else 814 min_perf = cap_perf; 815 816 if (min_perf < perf.min_limit_perf) 817 min_perf = perf.min_limit_perf; 818 819 max_perf = perf.max_limit_perf; 820 if (max_perf < min_perf) 821 max_perf = min_perf; 822 823 amd_pstate_update(policy, min_perf, des_perf, max_perf, true, 824 policy->governor->flags); 825 } 826 827 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) 828 { 829 struct amd_cpudata *cpudata = policy->driver_data; 830 u32 nominal_freq; 831 int ret = 0; 832 833 nominal_freq = READ_ONCE(cpudata->nominal_freq); 834 835 if (on) 836 policy->cpuinfo.max_freq = cpudata->max_freq; 837 else if (policy->cpuinfo.max_freq > nominal_freq) 838 policy->cpuinfo.max_freq = nominal_freq; 839 840 if (cppc_state == AMD_PSTATE_PASSIVE) { 841 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); 842 if (ret < 0) 843 pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu); 844 } 845 846 return ret < 0 ? ret : 0; 847 } 848 849 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 850 { 851 struct amd_cpudata *cpudata = policy->driver_data; 852 int ret; 853 854 if (!cpudata->boost_supported) { 855 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 856 return -EOPNOTSUPP; 857 } 858 859 ret = amd_pstate_cpu_boost_update(policy, state); 860 refresh_frequency_limits(policy); 861 862 return ret; 863 } 864 865 static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) 866 { 867 u64 boost_val; 868 int ret = -1; 869 870 /* 871 * If platform has no CPB support or disable it, initialize current driver 872 * boost_enabled state to be false, it is not an error for cpufreq core to handle. 873 */ 874 if (!cpu_feature_enabled(X86_FEATURE_CPB)) { 875 pr_debug_once("Boost CPB capabilities not present in the processor\n"); 876 ret = 0; 877 goto exit_err; 878 } 879 880 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); 881 if (ret) { 882 pr_err_once("failed to read initial CPU boost state!\n"); 883 ret = -EIO; 884 goto exit_err; 885 } 886 887 if (!(boost_val & MSR_K7_HWCR_CPB_DIS)) 888 cpudata->boost_supported = true; 889 890 return 0; 891 892 exit_err: 893 cpudata->boost_supported = false; 894 return ret; 895 } 896 897 static void amd_perf_ctl_reset(unsigned int cpu) 898 { 899 wrmsrq_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 900 } 901 902 #define CPPC_MAX_PERF U8_MAX 903 904 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) 905 { 906 /* user disabled or not detected */ 907 if (!amd_pstate_prefcore) 908 return; 909 910 /* should use amd-hfi instead */ 911 if (cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS) && 912 IS_ENABLED(CONFIG_AMD_HFI)) { 913 amd_pstate_prefcore = false; 914 return; 915 } 916 917 cpudata->hw_prefcore = true; 918 919 /* Priorities must be initialized before ITMT support can be toggled on. */ 920 sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); 921 } 922 923 static void amd_pstate_update_limits(struct cpufreq_policy *policy) 924 { 925 struct amd_cpudata *cpudata; 926 u32 prev_high = 0, cur_high = 0; 927 bool highest_perf_changed = false; 928 unsigned int cpu = policy->cpu; 929 930 if (!amd_pstate_prefcore) 931 return; 932 933 if (amd_get_highest_perf(cpu, &cur_high)) 934 return; 935 936 cpudata = policy->driver_data; 937 938 prev_high = READ_ONCE(cpudata->prefcore_ranking); 939 highest_perf_changed = (prev_high != cur_high); 940 if (highest_perf_changed) { 941 WRITE_ONCE(cpudata->prefcore_ranking, cur_high); 942 943 if (cur_high < CPPC_MAX_PERF) { 944 sched_set_itmt_core_prio((int)cur_high, cpu); 945 sched_update_asym_prefer_cpu(cpu, prev_high, cur_high); 946 } 947 } 948 } 949 950 /* 951 * Get pstate transition delay time from ACPI tables that firmware set 952 * instead of using hardcode value directly. 953 */ 954 static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) 955 { 956 int transition_delay_ns; 957 958 transition_delay_ns = cppc_get_transition_latency(cpu); 959 if (transition_delay_ns < 0) { 960 if (cpu_feature_enabled(X86_FEATURE_AMD_FAST_CPPC)) 961 return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY; 962 else 963 return AMD_PSTATE_TRANSITION_DELAY; 964 } 965 966 return transition_delay_ns / NSEC_PER_USEC; 967 } 968 969 /* 970 * Get pstate transition latency value from ACPI tables that firmware 971 * set instead of using hardcode value directly. 972 */ 973 static u32 amd_pstate_get_transition_latency(unsigned int cpu) 974 { 975 int transition_latency; 976 977 transition_latency = cppc_get_transition_latency(cpu); 978 if (transition_latency < 0) 979 return AMD_PSTATE_TRANSITION_LATENCY; 980 981 return transition_latency; 982 } 983 984 /* 985 * amd_pstate_init_freq: Initialize the nominal_freq and lowest_nonlinear_freq 986 * for the @cpudata object. 987 * 988 * Requires: all perf members of @cpudata to be initialized. 989 * 990 * Returns 0 on success, non-zero value on failure. 991 */ 992 static int amd_pstate_init_freq(struct amd_cpudata *cpudata) 993 { 994 u32 min_freq, max_freq, nominal_freq, lowest_nonlinear_freq; 995 struct cppc_perf_caps cppc_perf; 996 union perf_cached perf; 997 int ret; 998 999 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 1000 if (ret) 1001 return ret; 1002 perf = READ_ONCE(cpudata->perf); 1003 1004 if (quirks && quirks->nominal_freq) 1005 nominal_freq = quirks->nominal_freq; 1006 else 1007 nominal_freq = cppc_perf.nominal_freq; 1008 nominal_freq *= 1000; 1009 1010 if (quirks && quirks->lowest_freq) { 1011 min_freq = quirks->lowest_freq; 1012 perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq); 1013 WRITE_ONCE(cpudata->perf, perf); 1014 } else 1015 min_freq = cppc_perf.lowest_freq; 1016 1017 min_freq *= 1000; 1018 1019 WRITE_ONCE(cpudata->nominal_freq, nominal_freq); 1020 1021 /* max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf */ 1022 max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); 1023 WRITE_ONCE(cpudata->max_freq, max_freq); 1024 1025 lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); 1026 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); 1027 1028 /** 1029 * Below values need to be initialized correctly, otherwise driver will fail to load 1030 * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] 1031 * Check _CPC in ACPI table objects if any values are incorrect 1032 */ 1033 if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { 1034 pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", 1035 min_freq, max_freq, nominal_freq); 1036 return -EINVAL; 1037 } 1038 1039 if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq) { 1040 pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", 1041 lowest_nonlinear_freq, min_freq, nominal_freq); 1042 return -EINVAL; 1043 } 1044 1045 return 0; 1046 } 1047 1048 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 1049 { 1050 struct amd_cpudata *cpudata; 1051 union perf_cached perf; 1052 struct device *dev; 1053 int ret; 1054 1055 /* 1056 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1057 * which is ideal for initialization process. 1058 */ 1059 amd_perf_ctl_reset(policy->cpu); 1060 dev = get_cpu_device(policy->cpu); 1061 if (!dev) 1062 return -ENODEV; 1063 1064 cpudata = kzalloc_obj(*cpudata); 1065 if (!cpudata) 1066 return -ENOMEM; 1067 1068 cpudata->cpu = policy->cpu; 1069 1070 ret = amd_pstate_init_perf(cpudata); 1071 if (ret) 1072 goto free_cpudata1; 1073 1074 amd_pstate_init_prefcore(cpudata); 1075 1076 ret = amd_pstate_init_freq(cpudata); 1077 if (ret) 1078 goto free_cpudata1; 1079 1080 ret = amd_pstate_init_boost_support(cpudata); 1081 if (ret) 1082 goto free_cpudata1; 1083 1084 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); 1085 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); 1086 1087 perf = READ_ONCE(cpudata->perf); 1088 1089 policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, 1090 cpudata->nominal_freq, 1091 perf.lowest_perf); 1092 policy->cpuinfo.max_freq = policy->max = cpudata->max_freq; 1093 1094 policy->driver_data = cpudata; 1095 ret = amd_pstate_cppc_enable(policy); 1096 if (ret) 1097 goto free_cpudata1; 1098 1099 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1100 1101 /* It will be updated by governor */ 1102 policy->cur = policy->cpuinfo.min_freq; 1103 1104 if (cpu_feature_enabled(X86_FEATURE_CPPC)) 1105 policy->fast_switch_possible = true; 1106 1107 ret = amd_pstate_init_floor_perf(policy); 1108 if (ret) { 1109 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); 1110 goto free_cpudata1; 1111 } 1112 1113 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 1114 FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); 1115 if (ret < 0) { 1116 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 1117 goto free_cpudata1; 1118 } 1119 1120 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 1121 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 1122 if (ret < 0) { 1123 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 1124 goto free_cpudata2; 1125 } 1126 1127 1128 if (!current_pstate_driver->adjust_perf) 1129 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1130 1131 return 0; 1132 1133 free_cpudata2: 1134 freq_qos_remove_request(&cpudata->req[0]); 1135 free_cpudata1: 1136 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1137 kfree(cpudata); 1138 policy->driver_data = NULL; 1139 return ret; 1140 } 1141 1142 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) 1143 { 1144 struct amd_cpudata *cpudata = policy->driver_data; 1145 union perf_cached perf = READ_ONCE(cpudata->perf); 1146 1147 /* Reset CPPC_REQ MSR to the BIOS value */ 1148 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1149 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 1150 1151 freq_qos_remove_request(&cpudata->req[1]); 1152 freq_qos_remove_request(&cpudata->req[0]); 1153 policy->fast_switch_possible = false; 1154 kfree(cpudata); 1155 } 1156 1157 static int amd_pstate_get_balanced_epp(struct cpufreq_policy *policy) 1158 { 1159 struct amd_cpudata *cpudata = policy->driver_data; 1160 1161 if (power_supply_is_system_supplied()) 1162 return cpudata->epp_default_ac; 1163 else 1164 return cpudata->epp_default_dc; 1165 } 1166 1167 static int amd_pstate_power_supply_notifier(struct notifier_block *nb, 1168 unsigned long event, void *data) 1169 { 1170 struct amd_cpudata *cpudata = container_of(nb, struct amd_cpudata, power_nb); 1171 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 1172 u8 epp; 1173 int ret; 1174 1175 if (event != PSY_EVENT_PROP_CHANGED) 1176 return NOTIFY_OK; 1177 1178 /* dynamic actions are only applied while platform profile is in balanced */ 1179 if (cpudata->current_profile != PLATFORM_PROFILE_BALANCED) 1180 return 0; 1181 1182 epp = amd_pstate_get_balanced_epp(policy); 1183 1184 ret = amd_pstate_set_epp(policy, epp); 1185 if (ret) 1186 pr_warn("Failed to set CPU %d EPP %u: %d\n", cpudata->cpu, epp, ret); 1187 1188 return NOTIFY_OK; 1189 } 1190 1191 static int amd_pstate_profile_probe(void *drvdata, unsigned long *choices) 1192 { 1193 set_bit(PLATFORM_PROFILE_LOW_POWER, choices); 1194 set_bit(PLATFORM_PROFILE_BALANCED, choices); 1195 set_bit(PLATFORM_PROFILE_PERFORMANCE, choices); 1196 1197 return 0; 1198 } 1199 1200 static int amd_pstate_profile_get(struct device *dev, 1201 enum platform_profile_option *profile) 1202 { 1203 struct amd_cpudata *cpudata = dev_get_drvdata(dev); 1204 1205 *profile = cpudata->current_profile; 1206 1207 return 0; 1208 } 1209 1210 static int amd_pstate_profile_set(struct device *dev, 1211 enum platform_profile_option profile) 1212 { 1213 struct amd_cpudata *cpudata = dev_get_drvdata(dev); 1214 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 1215 int ret; 1216 1217 switch (profile) { 1218 case PLATFORM_PROFILE_LOW_POWER: 1219 ret = amd_pstate_set_epp(policy, AMD_CPPC_EPP_POWERSAVE); 1220 if (ret) 1221 return ret; 1222 break; 1223 case PLATFORM_PROFILE_BALANCED: 1224 ret = amd_pstate_set_epp(policy, 1225 amd_pstate_get_balanced_epp(policy)); 1226 if (ret) 1227 return ret; 1228 break; 1229 case PLATFORM_PROFILE_PERFORMANCE: 1230 ret = amd_pstate_set_epp(policy, AMD_CPPC_EPP_PERFORMANCE); 1231 if (ret) 1232 return ret; 1233 break; 1234 default: 1235 pr_err("Unknown Platform Profile %d\n", profile); 1236 return -EOPNOTSUPP; 1237 } 1238 1239 cpudata->current_profile = profile; 1240 1241 return 0; 1242 } 1243 1244 static const struct platform_profile_ops amd_pstate_profile_ops = { 1245 .probe = amd_pstate_profile_probe, 1246 .profile_set = amd_pstate_profile_set, 1247 .profile_get = amd_pstate_profile_get, 1248 }; 1249 1250 void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy) 1251 { 1252 struct amd_cpudata *cpudata = policy->driver_data; 1253 1254 if (cpudata->power_nb.notifier_call) 1255 power_supply_unreg_notifier(&cpudata->power_nb); 1256 if (cpudata->ppdev) { 1257 platform_profile_remove(cpudata->ppdev); 1258 cpudata->ppdev = NULL; 1259 } 1260 kfree(cpudata->profile_name); 1261 cpudata->dynamic_epp = false; 1262 } 1263 EXPORT_SYMBOL_GPL(amd_pstate_clear_dynamic_epp); 1264 1265 static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy) 1266 { 1267 struct amd_cpudata *cpudata = policy->driver_data; 1268 int ret; 1269 u8 epp; 1270 1271 switch (cpudata->current_profile) { 1272 case PLATFORM_PROFILE_PERFORMANCE: 1273 epp = AMD_CPPC_EPP_PERFORMANCE; 1274 break; 1275 case PLATFORM_PROFILE_LOW_POWER: 1276 epp = AMD_CPPC_EPP_POWERSAVE; 1277 break; 1278 case PLATFORM_PROFILE_BALANCED: 1279 epp = amd_pstate_get_balanced_epp(policy); 1280 break; 1281 default: 1282 pr_err("Unknown Platform Profile %d\n", cpudata->current_profile); 1283 return -EOPNOTSUPP; 1284 } 1285 ret = amd_pstate_set_epp(policy, epp); 1286 if (ret) 1287 return ret; 1288 1289 cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu); 1290 if (!cpudata->profile_name) 1291 return -ENOMEM; 1292 1293 cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu), 1294 cpudata->profile_name, 1295 policy->driver_data, 1296 &amd_pstate_profile_ops); 1297 if (IS_ERR(cpudata->ppdev)) { 1298 ret = PTR_ERR(cpudata->ppdev); 1299 goto cleanup; 1300 } 1301 1302 /* only enable notifier if things will actually change */ 1303 if (cpudata->epp_default_ac != cpudata->epp_default_dc) { 1304 cpudata->power_nb.notifier_call = amd_pstate_power_supply_notifier; 1305 ret = power_supply_reg_notifier(&cpudata->power_nb); 1306 if (ret) 1307 goto cleanup; 1308 } 1309 1310 cpudata->dynamic_epp = true; 1311 1312 return 0; 1313 1314 cleanup: 1315 amd_pstate_clear_dynamic_epp(policy); 1316 1317 return ret; 1318 } 1319 1320 /* Sysfs attributes */ 1321 1322 /* 1323 * This frequency is to indicate the maximum hardware frequency. 1324 * If boost is not active but supported, the frequency will be larger than the 1325 * one in cpuinfo. 1326 */ 1327 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 1328 char *buf) 1329 { 1330 struct amd_cpudata *cpudata = policy->driver_data; 1331 1332 return sysfs_emit(buf, "%u\n", cpudata->max_freq); 1333 } 1334 1335 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 1336 char *buf) 1337 { 1338 struct amd_cpudata *cpudata; 1339 union perf_cached perf; 1340 1341 cpudata = policy->driver_data; 1342 perf = READ_ONCE(cpudata->perf); 1343 1344 return sysfs_emit(buf, "%u\n", 1345 perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf)); 1346 } 1347 1348 /* 1349 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 1350 * need to expose it to sysfs. 1351 */ 1352 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 1353 char *buf) 1354 { 1355 struct amd_cpudata *cpudata; 1356 1357 cpudata = policy->driver_data; 1358 1359 return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf); 1360 } 1361 1362 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, 1363 char *buf) 1364 { 1365 u8 perf; 1366 struct amd_cpudata *cpudata = policy->driver_data; 1367 1368 perf = READ_ONCE(cpudata->prefcore_ranking); 1369 1370 return sysfs_emit(buf, "%u\n", perf); 1371 } 1372 1373 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, 1374 char *buf) 1375 { 1376 bool hw_prefcore; 1377 struct amd_cpudata *cpudata = policy->driver_data; 1378 1379 hw_prefcore = READ_ONCE(cpudata->hw_prefcore); 1380 1381 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); 1382 } 1383 1384 static ssize_t show_energy_performance_available_preferences( 1385 struct cpufreq_policy *policy, char *buf) 1386 { 1387 int offset = 0, i; 1388 struct amd_cpudata *cpudata = policy->driver_data; 1389 1390 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1391 return sysfs_emit_at(buf, offset, "%s\n", 1392 energy_perf_strings[EPP_INDEX_PERFORMANCE]); 1393 1394 for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++) 1395 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]); 1396 1397 offset += sysfs_emit_at(buf, offset, "\n"); 1398 1399 return offset; 1400 } 1401 1402 ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, 1403 const char *buf, size_t count) 1404 { 1405 struct amd_cpudata *cpudata = policy->driver_data; 1406 ssize_t ret; 1407 bool raw_epp = false; 1408 u8 epp; 1409 1410 if (cpudata->dynamic_epp) { 1411 pr_debug("EPP cannot be set when dynamic EPP is enabled\n"); 1412 return -EBUSY; 1413 } 1414 1415 /* 1416 * if the value matches a number, use that, otherwise see if 1417 * matches an index in the energy_perf_strings array 1418 */ 1419 ret = kstrtou8(buf, 0, &epp); 1420 raw_epp = !ret; 1421 if (ret) { 1422 ret = sysfs_match_string(energy_perf_strings, buf); 1423 if (ret < 0 || ret == EPP_INDEX_CUSTOM) 1424 return -EINVAL; 1425 if (ret) 1426 epp = epp_values[ret]; 1427 else 1428 epp = cpudata->epp_default_dc; 1429 } 1430 1431 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 1432 pr_debug("EPP cannot be set under performance policy\n"); 1433 return -EBUSY; 1434 } 1435 1436 ret = amd_pstate_set_epp(policy, epp); 1437 if (ret) 1438 return ret; 1439 1440 cpudata->raw_epp = raw_epp; 1441 1442 return count; 1443 } 1444 EXPORT_SYMBOL_GPL(store_energy_performance_preference); 1445 1446 ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf) 1447 { 1448 struct amd_cpudata *cpudata = policy->driver_data; 1449 u8 preference, epp; 1450 1451 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 1452 1453 if (cpudata->raw_epp) 1454 return sysfs_emit(buf, "%u\n", epp); 1455 1456 switch (epp) { 1457 case AMD_CPPC_EPP_PERFORMANCE: 1458 preference = EPP_INDEX_PERFORMANCE; 1459 break; 1460 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 1461 preference = EPP_INDEX_BALANCE_PERFORMANCE; 1462 break; 1463 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 1464 preference = EPP_INDEX_BALANCE_POWERSAVE; 1465 break; 1466 case AMD_CPPC_EPP_POWERSAVE: 1467 preference = EPP_INDEX_POWERSAVE; 1468 break; 1469 default: 1470 return -EINVAL; 1471 } 1472 1473 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 1474 } 1475 EXPORT_SYMBOL_GPL(show_energy_performance_preference); 1476 1477 static ssize_t store_amd_pstate_floor_freq(struct cpufreq_policy *policy, 1478 const char *buf, size_t count) 1479 { 1480 struct amd_cpudata *cpudata = policy->driver_data; 1481 union perf_cached perf = READ_ONCE(cpudata->perf); 1482 unsigned int freq; 1483 u8 floor_perf; 1484 int ret; 1485 1486 ret = kstrtouint(buf, 0, &freq); 1487 if (ret) 1488 return ret; 1489 1490 if (freq < policy->cpuinfo.min_freq || freq > policy->max) 1491 return -EINVAL; 1492 1493 floor_perf = freq_to_perf(perf, cpudata->nominal_freq, freq); 1494 ret = amd_pstate_set_floor_perf(policy, floor_perf); 1495 1496 if (!ret) 1497 cpudata->floor_freq = freq; 1498 1499 return ret ?: count; 1500 } 1501 1502 static ssize_t show_amd_pstate_floor_freq(struct cpufreq_policy *policy, char *buf) 1503 { 1504 struct amd_cpudata *cpudata = policy->driver_data; 1505 1506 return sysfs_emit(buf, "%u\n", cpudata->floor_freq); 1507 } 1508 1509 static ssize_t show_amd_pstate_floor_count(struct cpufreq_policy *policy, char *buf) 1510 { 1511 struct amd_cpudata *cpudata = policy->driver_data; 1512 u8 count = cpudata->floor_perf_cnt; 1513 1514 return sysfs_emit(buf, "%u\n", count); 1515 } 1516 1517 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1518 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1519 1520 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1521 cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); 1522 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); 1523 cpufreq_freq_attr_rw(energy_performance_preference); 1524 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1525 cpufreq_freq_attr_rw(amd_pstate_floor_freq); 1526 cpufreq_freq_attr_ro(amd_pstate_floor_count); 1527 1528 struct freq_attr_visibility { 1529 struct freq_attr *attr; 1530 bool (*visibility_fn)(void); 1531 }; 1532 1533 /* For attributes which are always visible */ 1534 static bool always_visible(void) 1535 { 1536 return true; 1537 } 1538 1539 /* Determines whether prefcore related attributes should be visible */ 1540 static bool prefcore_visibility(void) 1541 { 1542 return amd_pstate_prefcore; 1543 } 1544 1545 /* Determines whether energy performance preference should be visible */ 1546 static bool epp_visibility(void) 1547 { 1548 return cppc_state == AMD_PSTATE_ACTIVE; 1549 } 1550 1551 /* Determines whether amd_pstate_floor_freq related attributes should be visible */ 1552 static bool floor_freq_visibility(void) 1553 { 1554 return cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO); 1555 } 1556 1557 static struct freq_attr_visibility amd_pstate_attr_visibility[] = { 1558 {&amd_pstate_max_freq, always_visible}, 1559 {&amd_pstate_lowest_nonlinear_freq, always_visible}, 1560 {&amd_pstate_highest_perf, always_visible}, 1561 {&amd_pstate_prefcore_ranking, prefcore_visibility}, 1562 {&amd_pstate_hw_prefcore, prefcore_visibility}, 1563 {&energy_performance_preference, epp_visibility}, 1564 {&energy_performance_available_preferences, epp_visibility}, 1565 {&amd_pstate_floor_freq, floor_freq_visibility}, 1566 {&amd_pstate_floor_count, floor_freq_visibility}, 1567 }; 1568 1569 struct freq_attr **amd_pstate_get_current_attrs(void) 1570 { 1571 if (!current_pstate_driver) 1572 return NULL; 1573 return current_pstate_driver->attr; 1574 } 1575 EXPORT_SYMBOL_GPL(amd_pstate_get_current_attrs); 1576 1577 static struct freq_attr **get_freq_attrs(void) 1578 { 1579 bool attr_visible[ARRAY_SIZE(amd_pstate_attr_visibility)]; 1580 struct freq_attr **attrs; 1581 int i, j, count; 1582 1583 for (i = 0, count = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { 1584 struct freq_attr_visibility *v = &amd_pstate_attr_visibility[i]; 1585 1586 attr_visible[i] = v->visibility_fn(); 1587 if (attr_visible[i]) 1588 count++; 1589 } 1590 1591 /* amd_pstate_{max_freq, lowest_nonlinear_freq, highest_perf} should always be visible */ 1592 BUG_ON(!count); 1593 1594 attrs = kcalloc(count + 1, sizeof(struct freq_attr *), GFP_KERNEL); 1595 if (!attrs) 1596 return ERR_PTR(-ENOMEM); 1597 1598 for (i = 0, j = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { 1599 if (!attr_visible[i]) 1600 continue; 1601 1602 attrs[j++] = amd_pstate_attr_visibility[i].attr; 1603 } 1604 1605 return attrs; 1606 } 1607 1608 static void amd_pstate_driver_cleanup(void) 1609 { 1610 if (amd_pstate_prefcore) 1611 sched_clear_itmt_support(); 1612 1613 cppc_state = AMD_PSTATE_DISABLE; 1614 kfree(current_pstate_driver->attr); 1615 current_pstate_driver->attr = NULL; 1616 current_pstate_driver = NULL; 1617 } 1618 1619 static int amd_pstate_set_driver(int mode_idx) 1620 { 1621 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1622 cppc_state = mode_idx; 1623 if (cppc_state == AMD_PSTATE_DISABLE) 1624 pr_info("driver is explicitly disabled\n"); 1625 1626 if (cppc_state == AMD_PSTATE_ACTIVE) 1627 current_pstate_driver = &amd_pstate_epp_driver; 1628 1629 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1630 current_pstate_driver = &amd_pstate_driver; 1631 1632 return 0; 1633 } 1634 1635 return -EINVAL; 1636 } 1637 1638 static int amd_pstate_register_driver(int mode) 1639 { 1640 struct freq_attr **attr = NULL; 1641 int ret; 1642 1643 ret = amd_pstate_set_driver(mode); 1644 if (ret) 1645 return ret; 1646 1647 cppc_state = mode; 1648 1649 /* 1650 * Note: It is important to compute the attrs _after_ 1651 * re-initializing the cppc_state. Some attributes become 1652 * visible only when cppc_state is AMD_PSTATE_ACTIVE. 1653 */ 1654 attr = get_freq_attrs(); 1655 if (IS_ERR(attr)) { 1656 ret = (int) PTR_ERR(attr); 1657 pr_err("Couldn't compute freq_attrs for current mode %s [%d]\n", 1658 amd_pstate_get_mode_string(cppc_state), ret); 1659 amd_pstate_driver_cleanup(); 1660 return ret; 1661 } 1662 1663 current_pstate_driver->attr = attr; 1664 1665 /* at least one CPU supports CPB */ 1666 current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); 1667 1668 ret = cpufreq_register_driver(current_pstate_driver); 1669 if (ret) { 1670 amd_pstate_driver_cleanup(); 1671 return ret; 1672 } 1673 1674 /* Enable ITMT support once all CPUs have initialized their asym priorities. */ 1675 if (amd_pstate_prefcore) 1676 sched_set_itmt_support(); 1677 1678 return 0; 1679 } 1680 1681 static int amd_pstate_unregister_driver(int dummy) 1682 { 1683 cpufreq_unregister_driver(current_pstate_driver); 1684 amd_pstate_driver_cleanup(); 1685 return 0; 1686 } 1687 1688 static int amd_pstate_change_mode_without_dvr_change(int mode) 1689 { 1690 int cpu = 0; 1691 1692 cppc_state = mode; 1693 1694 if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 1695 return 0; 1696 1697 for_each_online_cpu(cpu) { 1698 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 1699 } 1700 1701 return 0; 1702 } 1703 1704 static int amd_pstate_change_driver_mode(int mode) 1705 { 1706 int ret; 1707 1708 lockdep_assert_held(&amd_pstate_driver_lock); 1709 1710 ret = amd_pstate_unregister_driver(0); 1711 if (ret) 1712 return ret; 1713 1714 ret = amd_pstate_register_driver(mode); 1715 if (ret) 1716 return ret; 1717 1718 return 0; 1719 } 1720 1721 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 1722 [AMD_PSTATE_DISABLE] = { 1723 [AMD_PSTATE_DISABLE] = NULL, 1724 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 1725 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 1726 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 1727 }, 1728 [AMD_PSTATE_PASSIVE] = { 1729 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1730 [AMD_PSTATE_PASSIVE] = NULL, 1731 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1732 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 1733 }, 1734 [AMD_PSTATE_ACTIVE] = { 1735 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1736 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 1737 [AMD_PSTATE_ACTIVE] = NULL, 1738 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 1739 }, 1740 [AMD_PSTATE_GUIDED] = { 1741 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1742 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 1743 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1744 [AMD_PSTATE_GUIDED] = NULL, 1745 }, 1746 }; 1747 1748 static ssize_t amd_pstate_show_status(char *buf) 1749 { 1750 if (!current_pstate_driver) 1751 return sysfs_emit(buf, "disable\n"); 1752 1753 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 1754 } 1755 1756 int amd_pstate_get_status(void) 1757 { 1758 return cppc_state; 1759 } 1760 EXPORT_SYMBOL_GPL(amd_pstate_get_status); 1761 1762 int amd_pstate_update_status(const char *buf, size_t size) 1763 { 1764 int mode_idx; 1765 1766 if (size > strlen("passive") || size < strlen("active")) 1767 return -EINVAL; 1768 1769 mode_idx = get_mode_idx_from_str(buf, size); 1770 if (mode_idx < 0) 1771 return mode_idx; 1772 1773 if (mode_state_machine[cppc_state][mode_idx]) { 1774 guard(mutex)(&amd_pstate_driver_lock); 1775 return mode_state_machine[cppc_state][mode_idx](mode_idx); 1776 } 1777 1778 return 0; 1779 } 1780 EXPORT_SYMBOL_GPL(amd_pstate_update_status); 1781 1782 static ssize_t status_show(struct device *dev, 1783 struct device_attribute *attr, char *buf) 1784 { 1785 1786 guard(mutex)(&amd_pstate_driver_lock); 1787 1788 return amd_pstate_show_status(buf); 1789 } 1790 1791 static ssize_t status_store(struct device *a, struct device_attribute *b, 1792 const char *buf, size_t count) 1793 { 1794 char *p = memchr(buf, '\n', count); 1795 int ret; 1796 1797 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1798 1799 return ret < 0 ? ret : count; 1800 } 1801 1802 static ssize_t prefcore_show(struct device *dev, 1803 struct device_attribute *attr, char *buf) 1804 { 1805 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); 1806 } 1807 1808 static ssize_t dynamic_epp_show(struct device *dev, 1809 struct device_attribute *attr, char *buf) 1810 { 1811 return sysfs_emit(buf, "%s\n", str_enabled_disabled(dynamic_epp)); 1812 } 1813 1814 static ssize_t dynamic_epp_store(struct device *a, struct device_attribute *b, 1815 const char *buf, size_t count) 1816 { 1817 bool enabled; 1818 int ret; 1819 1820 ret = kstrtobool(buf, &enabled); 1821 if (ret) 1822 return ret; 1823 1824 guard(mutex)(&amd_pstate_driver_lock); 1825 1826 if (cppc_state != AMD_PSTATE_ACTIVE) { 1827 pr_debug("dynamic_epp can only be toggled in active mode\n"); 1828 return -EINVAL; 1829 } 1830 1831 /* Nothing to do */ 1832 if (dynamic_epp == enabled) 1833 return count; 1834 1835 /* reinitialize with desired dynamic EPP value */ 1836 dynamic_epp = enabled; 1837 ret = amd_pstate_change_driver_mode(cppc_state); 1838 if (ret) 1839 dynamic_epp = false; 1840 1841 return ret ? ret : count; 1842 } 1843 1844 static DEVICE_ATTR_RW(status); 1845 static DEVICE_ATTR_RO(prefcore); 1846 static DEVICE_ATTR_RW(dynamic_epp); 1847 1848 static struct attribute *pstate_global_attributes[] = { 1849 &dev_attr_status.attr, 1850 &dev_attr_prefcore.attr, 1851 &dev_attr_dynamic_epp.attr, 1852 NULL 1853 }; 1854 1855 static const struct attribute_group amd_pstate_global_attr_group = { 1856 .name = "amd_pstate", 1857 .attrs = pstate_global_attributes, 1858 }; 1859 1860 static bool amd_pstate_acpi_pm_profile_server(void) 1861 { 1862 switch (acpi_gbl_FADT.preferred_profile) { 1863 case PM_ENTERPRISE_SERVER: 1864 case PM_SOHO_SERVER: 1865 case PM_PERFORMANCE_SERVER: 1866 return true; 1867 } 1868 return false; 1869 } 1870 1871 static bool amd_pstate_acpi_pm_profile_undefined(void) 1872 { 1873 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) 1874 return true; 1875 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) 1876 return true; 1877 return false; 1878 } 1879 1880 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1881 { 1882 struct amd_cpudata *cpudata; 1883 union perf_cached perf; 1884 struct device *dev; 1885 int ret; 1886 1887 /* 1888 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1889 * which is ideal for initialization process. 1890 */ 1891 amd_perf_ctl_reset(policy->cpu); 1892 dev = get_cpu_device(policy->cpu); 1893 if (!dev) 1894 return -ENODEV; 1895 1896 cpudata = kzalloc_obj(*cpudata); 1897 if (!cpudata) 1898 return -ENOMEM; 1899 1900 cpudata->cpu = policy->cpu; 1901 1902 ret = amd_pstate_init_perf(cpudata); 1903 if (ret) 1904 goto free_cpudata1; 1905 1906 amd_pstate_init_prefcore(cpudata); 1907 1908 ret = amd_pstate_init_freq(cpudata); 1909 if (ret) 1910 goto free_cpudata1; 1911 1912 ret = amd_pstate_init_boost_support(cpudata); 1913 if (ret) 1914 goto free_cpudata1; 1915 1916 perf = READ_ONCE(cpudata->perf); 1917 1918 policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, 1919 cpudata->nominal_freq, 1920 perf.lowest_perf); 1921 policy->cpuinfo.max_freq = policy->max = cpudata->max_freq; 1922 policy->driver_data = cpudata; 1923 1924 ret = amd_pstate_cppc_enable(policy); 1925 if (ret) 1926 goto free_cpudata1; 1927 1928 /* It will be updated by governor */ 1929 policy->cur = policy->cpuinfo.min_freq; 1930 1931 1932 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1933 1934 /* 1935 * Set the policy to provide a valid fallback value in case 1936 * the default cpufreq governor is neither powersave nor performance. 1937 */ 1938 if (amd_pstate_acpi_pm_profile_server() || 1939 amd_pstate_acpi_pm_profile_undefined()) { 1940 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 1941 cpudata->epp_default_ac = cpudata->epp_default_dc = amd_pstate_get_epp(cpudata); 1942 cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE; 1943 } else { 1944 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1945 cpudata->epp_default_ac = AMD_CPPC_EPP_PERFORMANCE; 1946 cpudata->epp_default_dc = AMD_CPPC_EPP_BALANCE_PERFORMANCE; 1947 cpudata->current_profile = PLATFORM_PROFILE_BALANCED; 1948 } 1949 1950 if (dynamic_epp) 1951 ret = amd_pstate_set_dynamic_epp(policy); 1952 else 1953 ret = amd_pstate_set_epp(policy, cpudata->epp_default_dc); 1954 if (ret) 1955 goto free_cpudata1; 1956 1957 ret = amd_pstate_init_floor_perf(policy); 1958 if (ret) { 1959 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); 1960 goto free_cpudata1; 1961 } 1962 1963 current_pstate_driver->adjust_perf = NULL; 1964 1965 return 0; 1966 1967 free_cpudata1: 1968 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1969 kfree(cpudata); 1970 policy->driver_data = NULL; 1971 return ret; 1972 } 1973 1974 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1975 { 1976 struct amd_cpudata *cpudata = policy->driver_data; 1977 1978 if (cpudata) { 1979 union perf_cached perf = READ_ONCE(cpudata->perf); 1980 1981 if (cpudata->dynamic_epp) 1982 amd_pstate_clear_dynamic_epp(policy); 1983 1984 /* Reset CPPC_REQ MSR to the BIOS value */ 1985 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1986 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 1987 1988 kfree(cpudata); 1989 policy->driver_data = NULL; 1990 } 1991 1992 pr_debug("CPU %d exiting\n", policy->cpu); 1993 } 1994 1995 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy, bool policy_change) 1996 { 1997 struct amd_cpudata *cpudata = policy->driver_data; 1998 union perf_cached perf; 1999 u8 epp; 2000 2001 if (policy_change || 2002 policy->min != cpudata->min_limit_freq || 2003 policy->max != cpudata->max_limit_freq) 2004 amd_pstate_update_min_max_limit(policy); 2005 2006 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 2007 epp = 0; 2008 else 2009 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 2010 2011 perf = READ_ONCE(cpudata->perf); 2012 2013 return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U, 2014 perf.max_limit_perf, epp, false); 2015 } 2016 2017 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 2018 { 2019 struct amd_cpudata *cpudata = policy->driver_data; 2020 int ret; 2021 2022 if (!policy->cpuinfo.max_freq) 2023 return -ENODEV; 2024 2025 cpudata->policy = policy->policy; 2026 2027 ret = amd_pstate_epp_update_limit(policy, true); 2028 if (ret) 2029 return ret; 2030 2031 /* 2032 * policy->cur is never updated with the amd_pstate_epp driver, but it 2033 * is used as a stale frequency value. So, keep it within limits. 2034 */ 2035 policy->cur = policy->min; 2036 2037 return 0; 2038 } 2039 2040 static int amd_pstate_cpu_online(struct cpufreq_policy *policy) 2041 { 2042 struct amd_cpudata *cpudata = policy->driver_data; 2043 union perf_cached perf = READ_ONCE(cpudata->perf); 2044 u8 cached_floor_perf; 2045 int ret; 2046 2047 ret = amd_pstate_cppc_enable(policy); 2048 if (ret) 2049 return ret; 2050 2051 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2052 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2053 } 2054 2055 static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) 2056 { 2057 struct amd_cpudata *cpudata = policy->driver_data; 2058 union perf_cached perf = READ_ONCE(cpudata->perf); 2059 int ret; 2060 2061 /* 2062 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 2063 * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the 2064 * limits, epp and desired perf will get reset to the cached values in cpudata struct 2065 */ 2066 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 2067 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 2068 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 2069 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 2070 false); 2071 if (ret) 2072 return ret; 2073 2074 return amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2075 } 2076 2077 static int amd_pstate_suspend(struct cpufreq_policy *policy) 2078 { 2079 struct amd_cpudata *cpudata = policy->driver_data; 2080 union perf_cached perf = READ_ONCE(cpudata->perf); 2081 int ret; 2082 2083 /* 2084 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 2085 * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, 2086 * the limits, epp and desired perf will get reset to the cached values in cpudata struct 2087 */ 2088 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 2089 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 2090 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 2091 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 2092 false); 2093 if (ret) 2094 return ret; 2095 2096 ret = amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2097 if (ret) 2098 return ret; 2099 2100 /* set this flag to avoid setting core offline*/ 2101 cpudata->suspended = true; 2102 2103 return 0; 2104 } 2105 2106 static int amd_pstate_resume(struct cpufreq_policy *policy) 2107 { 2108 struct amd_cpudata *cpudata = policy->driver_data; 2109 union perf_cached perf = READ_ONCE(cpudata->perf); 2110 int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur); 2111 u8 cached_floor_perf; 2112 int ret; 2113 2114 /* Set CPPC_REQ to last sane value until the governor updates it */ 2115 ret = amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, 2116 0U, false); 2117 if (ret) 2118 return ret; 2119 2120 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2121 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2122 } 2123 2124 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 2125 { 2126 struct amd_cpudata *cpudata = policy->driver_data; 2127 union perf_cached perf = READ_ONCE(cpudata->perf); 2128 u8 cached_floor_perf; 2129 2130 if (cpudata->suspended) { 2131 int ret; 2132 2133 /* enable amd pstate from suspend state*/ 2134 ret = amd_pstate_epp_update_limit(policy, false); 2135 if (ret) 2136 return ret; 2137 2138 cpudata->suspended = false; 2139 } 2140 2141 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2142 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2143 } 2144 2145 static struct cpufreq_driver amd_pstate_driver = { 2146 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 2147 .verify = amd_pstate_verify, 2148 .target = amd_pstate_target, 2149 .fast_switch = amd_pstate_fast_switch, 2150 .init = amd_pstate_cpu_init, 2151 .exit = amd_pstate_cpu_exit, 2152 .online = amd_pstate_cpu_online, 2153 .offline = amd_pstate_cpu_offline, 2154 .suspend = amd_pstate_suspend, 2155 .resume = amd_pstate_resume, 2156 .set_boost = amd_pstate_set_boost, 2157 .update_limits = amd_pstate_update_limits, 2158 .name = "amd-pstate", 2159 }; 2160 2161 static struct cpufreq_driver amd_pstate_epp_driver = { 2162 .flags = CPUFREQ_CONST_LOOPS, 2163 .verify = amd_pstate_verify, 2164 .setpolicy = amd_pstate_epp_set_policy, 2165 .init = amd_pstate_epp_cpu_init, 2166 .exit = amd_pstate_epp_cpu_exit, 2167 .offline = amd_pstate_cpu_offline, 2168 .online = amd_pstate_cpu_online, 2169 .suspend = amd_pstate_suspend, 2170 .resume = amd_pstate_epp_resume, 2171 .update_limits = amd_pstate_update_limits, 2172 .set_boost = amd_pstate_set_boost, 2173 .name = "amd-pstate-epp", 2174 }; 2175 2176 /* 2177 * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. 2178 * show the debug message that helps to check if the CPU has CPPC support for loading issue. 2179 */ 2180 static bool amd_cppc_supported(void) 2181 { 2182 struct cpuinfo_x86 *c = &cpu_data(0); 2183 bool warn = false; 2184 2185 if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) { 2186 pr_debug_once("CPPC feature is not supported by the processor\n"); 2187 return false; 2188 } 2189 2190 /* 2191 * If the CPPC feature is disabled in the BIOS for processors 2192 * that support MSR-based CPPC, the AMD Pstate driver may not 2193 * function correctly. 2194 * 2195 * For such processors, check the CPPC flag and display a 2196 * warning message if the platform supports CPPC. 2197 * 2198 * Note: The code check below will not abort the driver 2199 * registration process because of the code is added for 2200 * debugging purposes. Besides, it may still be possible for 2201 * the driver to work using the shared-memory mechanism. 2202 */ 2203 if (!cpu_feature_enabled(X86_FEATURE_CPPC)) { 2204 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 2205 switch (c->x86_model) { 2206 case 0x60 ... 0x6F: 2207 case 0x80 ... 0xAF: 2208 warn = true; 2209 break; 2210 } 2211 } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || 2212 cpu_feature_enabled(X86_FEATURE_ZEN4)) { 2213 switch (c->x86_model) { 2214 case 0x10 ... 0x1F: 2215 case 0x40 ... 0xAF: 2216 warn = true; 2217 break; 2218 } 2219 } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) { 2220 warn = true; 2221 } 2222 } 2223 2224 if (warn) 2225 pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n" 2226 "Please enable it if your BIOS has the CPPC option.\n"); 2227 return true; 2228 } 2229 2230 static int __init amd_pstate_init(void) 2231 { 2232 struct device *dev_root; 2233 int ret; 2234 2235 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 2236 return -ENODEV; 2237 2238 /* show debug message only if CPPC is not supported */ 2239 if (!amd_cppc_supported()) 2240 return -EOPNOTSUPP; 2241 2242 /* show warning message when BIOS broken or ACPI disabled */ 2243 if (!acpi_cpc_valid()) { 2244 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 2245 return -ENODEV; 2246 } 2247 2248 /* don't keep reloading if cpufreq_driver exists */ 2249 if (cpufreq_get_current_driver()) 2250 return -EEXIST; 2251 2252 quirks = NULL; 2253 2254 /* check if this machine need CPPC quirks */ 2255 dmi_check_system(amd_pstate_quirks_table); 2256 2257 /* 2258 * determine the driver mode from the command line or kernel config. 2259 * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED. 2260 * command line options will override the kernel config settings. 2261 */ 2262 2263 if (cppc_state == AMD_PSTATE_UNDEFINED) { 2264 /* Disable on the following configs by default: 2265 * 1. Undefined platforms 2266 * 2. Server platforms with CPUs older than Family 0x1A. 2267 */ 2268 if (amd_pstate_acpi_pm_profile_undefined() || 2269 (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) { 2270 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 2271 return -ENODEV; 2272 } 2273 /* get driver mode from kernel config option [1:4] */ 2274 cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; 2275 } 2276 2277 if (cppc_state == AMD_PSTATE_DISABLE) { 2278 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 2279 return -ENODEV; 2280 } 2281 2282 /* capability check */ 2283 if (cpu_feature_enabled(X86_FEATURE_CPPC)) { 2284 pr_debug("AMD CPPC MSR based functionality is supported\n"); 2285 } else { 2286 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 2287 static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); 2288 static_call_update(amd_pstate_init_perf, shmem_init_perf); 2289 static_call_update(amd_pstate_update_perf, shmem_update_perf); 2290 static_call_update(amd_pstate_get_epp, shmem_get_epp); 2291 static_call_update(amd_pstate_set_epp, shmem_set_epp); 2292 } 2293 2294 if (amd_pstate_prefcore) { 2295 ret = amd_detect_prefcore(&amd_pstate_prefcore); 2296 if (ret) 2297 return ret; 2298 } 2299 2300 ret = amd_pstate_register_driver(cppc_state); 2301 if (ret) { 2302 pr_err("failed to register with return %d\n", ret); 2303 return ret; 2304 } 2305 2306 dev_root = bus_get_dev_root(&cpu_subsys); 2307 if (dev_root) { 2308 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 2309 put_device(dev_root); 2310 if (ret) { 2311 pr_err("sysfs attribute export failed with error %d.\n", ret); 2312 goto global_attr_free; 2313 } 2314 } 2315 2316 return ret; 2317 2318 global_attr_free: 2319 amd_pstate_unregister_driver(0); 2320 return ret; 2321 } 2322 device_initcall(amd_pstate_init); 2323 2324 static int __init amd_pstate_param(char *str) 2325 { 2326 size_t size; 2327 int mode_idx; 2328 2329 if (!str) 2330 return -EINVAL; 2331 2332 size = strlen(str); 2333 mode_idx = get_mode_idx_from_str(str, size); 2334 2335 return amd_pstate_set_driver(mode_idx); 2336 } 2337 2338 static int __init amd_prefcore_param(char *str) 2339 { 2340 if (!strcmp(str, "disable")) 2341 amd_pstate_prefcore = false; 2342 2343 return 0; 2344 } 2345 2346 static int __init amd_dynamic_epp_param(char *str) 2347 { 2348 if (!strcmp(str, "disable")) 2349 dynamic_epp = false; 2350 if (!strcmp(str, "enable")) 2351 dynamic_epp = true; 2352 2353 return 0; 2354 } 2355 2356 early_param("amd_pstate", amd_pstate_param); 2357 early_param("amd_prefcore", amd_prefcore_param); 2358 early_param("amd_dynamic_epp", amd_dynamic_epp_param); 2359 2360 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 2361 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 2362