1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/bitfield.h> 26 #include <linux/kernel.h> 27 #include <linux/module.h> 28 #include <linux/init.h> 29 #include <linux/smp.h> 30 #include <linux/sched.h> 31 #include <linux/cpufreq.h> 32 #include <linux/compiler.h> 33 #include <linux/dmi.h> 34 #include <linux/slab.h> 35 #include <linux/acpi.h> 36 #include <linux/io.h> 37 #include <linux/delay.h> 38 #include <linux/uaccess.h> 39 #include <linux/power_supply.h> 40 #include <linux/static_call.h> 41 #include <linux/topology.h> 42 43 #include <acpi/processor.h> 44 #include <acpi/cppc_acpi.h> 45 46 #include <asm/msr.h> 47 #include <asm/processor.h> 48 #include <asm/cpufeature.h> 49 #include <asm/cpu_device_id.h> 50 51 #include "amd-pstate.h" 52 #include "amd-pstate-trace.h" 53 54 #define AMD_PSTATE_TRANSITION_LATENCY 20000 55 #define AMD_PSTATE_TRANSITION_DELAY 1000 56 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600 57 58 #define AMD_CPPC_EPP_PERFORMANCE 0x00 59 #define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 60 #define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF 61 #define AMD_CPPC_EPP_POWERSAVE 0xFF 62 63 static const char * const amd_pstate_mode_string[] = { 64 [AMD_PSTATE_UNDEFINED] = "undefined", 65 [AMD_PSTATE_DISABLE] = "disable", 66 [AMD_PSTATE_PASSIVE] = "passive", 67 [AMD_PSTATE_ACTIVE] = "active", 68 [AMD_PSTATE_GUIDED] = "guided", 69 }; 70 static_assert(ARRAY_SIZE(amd_pstate_mode_string) == AMD_PSTATE_MAX); 71 72 const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode) 73 { 74 if (mode < AMD_PSTATE_UNDEFINED || mode >= AMD_PSTATE_MAX) 75 mode = AMD_PSTATE_UNDEFINED; 76 return amd_pstate_mode_string[mode]; 77 } 78 EXPORT_SYMBOL_GPL(amd_pstate_get_mode_string); 79 80 struct quirk_entry { 81 u32 nominal_freq; 82 u32 lowest_freq; 83 }; 84 85 static struct cpufreq_driver *current_pstate_driver; 86 static struct cpufreq_driver amd_pstate_driver; 87 static struct cpufreq_driver amd_pstate_epp_driver; 88 static int cppc_state = AMD_PSTATE_UNDEFINED; 89 static bool amd_pstate_prefcore = true; 90 #ifdef CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP 91 static bool dynamic_epp = CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP; 92 #else 93 static bool dynamic_epp; 94 #endif 95 static struct quirk_entry *quirks; 96 97 /* 98 * AMD Energy Preference Performance (EPP) 99 * The EPP is used in the CCLK DPM controller to drive 100 * the frequency that a core is going to operate during 101 * short periods of activity. EPP values will be utilized for 102 * different OS profiles (balanced, performance, power savings) 103 * display strings corresponding to EPP index in the 104 * energy_perf_strings[] 105 * index String 106 *------------------------------------- 107 * 0 default 108 * 1 performance 109 * 2 balance_performance 110 * 3 balance_power 111 * 4 power 112 * 5 custom (for raw EPP values) 113 */ 114 enum energy_perf_value_index { 115 EPP_INDEX_DEFAULT = 0, 116 EPP_INDEX_PERFORMANCE, 117 EPP_INDEX_BALANCE_PERFORMANCE, 118 EPP_INDEX_BALANCE_POWERSAVE, 119 EPP_INDEX_POWERSAVE, 120 EPP_INDEX_CUSTOM, 121 EPP_INDEX_MAX, 122 }; 123 124 static const char * const energy_perf_strings[] = { 125 [EPP_INDEX_DEFAULT] = "default", 126 [EPP_INDEX_PERFORMANCE] = "performance", 127 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 128 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 129 [EPP_INDEX_POWERSAVE] = "power", 130 [EPP_INDEX_CUSTOM] = "custom", 131 }; 132 static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); 133 134 static unsigned int epp_values[] = { 135 [EPP_INDEX_DEFAULT] = 0, 136 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 137 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 138 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 139 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 140 }; 141 static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX - 1); 142 143 typedef int (*cppc_mode_transition_fn)(int); 144 145 static struct quirk_entry quirk_amd_7k62 = { 146 .nominal_freq = 2600, 147 .lowest_freq = 550, 148 }; 149 150 static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) 151 { 152 u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); 153 154 return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf); 155 } 156 157 static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val) 158 { 159 return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val, 160 perf.nominal_perf); 161 } 162 163 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) 164 { 165 /** 166 * match the broken bios for family 17h processor support CPPC V2 167 * broken BIOS lack of nominal_freq and lowest_freq capabilities 168 * definition in ACPI tables 169 */ 170 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 171 quirks = dmi->driver_data; 172 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); 173 return 1; 174 } 175 176 return 0; 177 } 178 179 static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { 180 { 181 .callback = dmi_matched_7k62_bios_bug, 182 .ident = "AMD EPYC 7K62", 183 .matches = { 184 DMI_MATCH(DMI_BIOS_VERSION, "5.14"), 185 DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), 186 }, 187 .driver_data = &quirk_amd_7k62, 188 }, 189 {} 190 }; 191 MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); 192 193 static inline int get_mode_idx_from_str(const char *str, size_t size) 194 { 195 int i; 196 197 for (i = 0; i < AMD_PSTATE_MAX; i++) { 198 if (!strncmp(str, amd_pstate_mode_string[i], size)) 199 return i; 200 } 201 return -EINVAL; 202 } 203 204 static DEFINE_MUTEX(amd_pstate_driver_lock); 205 206 static u8 msr_get_epp(struct amd_cpudata *cpudata) 207 { 208 u64 value; 209 int ret; 210 211 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 212 if (ret < 0) { 213 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 214 return ret; 215 } 216 217 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value); 218 } 219 220 DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); 221 222 static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) 223 { 224 return static_call(amd_pstate_get_epp)(cpudata); 225 } 226 227 static u8 shmem_get_epp(struct amd_cpudata *cpudata) 228 { 229 u64 epp; 230 int ret; 231 232 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 233 if (ret < 0) { 234 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 235 return ret; 236 } 237 238 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp); 239 } 240 241 static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, 242 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 243 { 244 struct amd_cpudata *cpudata = policy->driver_data; 245 u64 value, prev; 246 247 value = prev = READ_ONCE(cpudata->cppc_req_cached); 248 249 value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | 250 AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); 251 value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); 252 value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); 253 value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); 254 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 255 256 if (trace_amd_pstate_epp_perf_enabled()) { 257 union perf_cached perf = READ_ONCE(cpudata->perf); 258 259 trace_call__amd_pstate_epp_perf(cpudata->cpu, 260 perf.highest_perf, 261 epp, 262 min_perf, 263 max_perf, 264 policy->boost_enabled, 265 value != prev); 266 } 267 268 if (value == prev) 269 return 0; 270 271 if (fast_switch) { 272 wrmsrq(MSR_AMD_CPPC_REQ, value); 273 } else { 274 int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 275 276 if (ret) 277 return ret; 278 } 279 280 WRITE_ONCE(cpudata->cppc_req_cached, value); 281 282 return 0; 283 } 284 285 DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); 286 287 static inline int amd_pstate_update_perf(struct cpufreq_policy *policy, 288 u8 min_perf, u8 des_perf, 289 u8 max_perf, u8 epp, 290 bool fast_switch) 291 { 292 return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf, 293 max_perf, epp, fast_switch); 294 } 295 296 static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) 297 { 298 struct amd_cpudata *cpudata = policy->driver_data; 299 u64 value, prev; 300 int ret; 301 302 value = prev = READ_ONCE(cpudata->cppc_req_cached); 303 value &= ~AMD_CPPC_EPP_PERF_MASK; 304 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 305 306 if (trace_amd_pstate_epp_perf_enabled()) { 307 union perf_cached perf = cpudata->perf; 308 309 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 310 epp, 311 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 312 cpudata->cppc_req_cached), 313 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 314 cpudata->cppc_req_cached), 315 policy->boost_enabled, 316 value != prev); 317 } 318 319 if (value == prev) 320 return 0; 321 322 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 323 if (ret) { 324 pr_err("failed to set energy perf value (%d)\n", ret); 325 return ret; 326 } 327 328 /* update both so that msr_update_perf() can effectively check */ 329 WRITE_ONCE(cpudata->cppc_req_cached, value); 330 331 return ret; 332 } 333 334 DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); 335 336 static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp) 337 { 338 return static_call(amd_pstate_set_epp)(policy, epp); 339 } 340 341 static int amd_pstate_set_floor_perf(struct cpufreq_policy *policy, u8 perf) 342 { 343 struct amd_cpudata *cpudata = policy->driver_data; 344 u64 value, prev; 345 bool changed; 346 int ret; 347 348 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) 349 return 0; 350 351 value = prev = READ_ONCE(cpudata->cppc_req2_cached); 352 FIELD_MODIFY(AMD_CPPC_FLOOR_PERF_MASK, &value, perf); 353 354 changed = value != prev; 355 if (!changed) { 356 ret = 0; 357 goto out_trace; 358 } 359 360 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, value); 361 if (ret) { 362 changed = false; 363 pr_err("failed to set CPPC REQ2 value. Error (%d)\n", ret); 364 goto out_trace; 365 } 366 367 WRITE_ONCE(cpudata->cppc_req2_cached, value); 368 369 out_trace: 370 if (trace_amd_pstate_cppc_req2_enabled()) 371 trace_amd_pstate_cppc_req2(cpudata->cpu, perf, changed, ret); 372 return ret; 373 } 374 375 static int amd_pstate_init_floor_perf(struct cpufreq_policy *policy) 376 { 377 struct amd_cpudata *cpudata = policy->driver_data; 378 u8 floor_perf; 379 u64 value; 380 int ret; 381 382 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) 383 return 0; 384 385 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, &value); 386 if (ret) { 387 pr_err("failed to read CPPC REQ2 value. Error (%d)\n", ret); 388 return ret; 389 } 390 391 WRITE_ONCE(cpudata->cppc_req2_cached, value); 392 floor_perf = FIELD_GET(AMD_CPPC_FLOOR_PERF_MASK, 393 cpudata->cppc_req2_cached); 394 395 /* Set a sane value for floor_perf if the default value is invalid */ 396 if (floor_perf < cpudata->perf.lowest_perf) { 397 floor_perf = cpudata->perf.nominal_perf; 398 ret = amd_pstate_set_floor_perf(policy, floor_perf); 399 if (ret) 400 return ret; 401 } 402 403 404 cpudata->bios_floor_perf = floor_perf; 405 cpudata->floor_freq = perf_to_freq(cpudata->perf, cpudata->nominal_freq, 406 floor_perf); 407 return 0; 408 } 409 410 static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) 411 { 412 struct amd_cpudata *cpudata = policy->driver_data; 413 struct cppc_perf_ctrls perf_ctrls; 414 u8 epp_cached; 415 u64 value; 416 int ret; 417 418 419 epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 420 if (trace_amd_pstate_epp_perf_enabled()) { 421 union perf_cached perf = cpudata->perf; 422 423 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 424 epp, 425 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 426 cpudata->cppc_req_cached), 427 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 428 cpudata->cppc_req_cached), 429 policy->boost_enabled, 430 epp != epp_cached); 431 } 432 433 if (epp == epp_cached) 434 return 0; 435 436 perf_ctrls.energy_perf = epp; 437 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 438 if (ret) { 439 pr_debug("failed to set energy perf value (%d)\n", ret); 440 return ret; 441 } 442 443 value = READ_ONCE(cpudata->cppc_req_cached); 444 value &= ~AMD_CPPC_EPP_PERF_MASK; 445 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 446 WRITE_ONCE(cpudata->cppc_req_cached, value); 447 448 return ret; 449 } 450 451 static inline int msr_cppc_enable(struct cpufreq_policy *policy) 452 { 453 return wrmsrq_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); 454 } 455 456 static int shmem_cppc_enable(struct cpufreq_policy *policy) 457 { 458 return cppc_set_enable(policy->cpu, 1); 459 } 460 461 DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); 462 463 static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) 464 { 465 return static_call(amd_pstate_cppc_enable)(policy); 466 } 467 468 static int msr_init_perf(struct amd_cpudata *cpudata) 469 { 470 union perf_cached perf = READ_ONCE(cpudata->perf); 471 u64 cap1, numerator, cppc_req; 472 u8 min_perf; 473 474 int ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 475 &cap1); 476 if (ret) 477 return ret; 478 479 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 480 if (ret) 481 return ret; 482 483 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); 484 if (ret) 485 return ret; 486 487 WRITE_ONCE(cpudata->cppc_req_cached, cppc_req); 488 min_perf = FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cppc_req); 489 490 /* 491 * Clear out the min_perf part to check if the rest of the MSR is 0, if yes, this is an 492 * indication that the min_perf value is the one specified through the BIOS option 493 */ 494 cppc_req &= ~(AMD_CPPC_MIN_PERF_MASK); 495 496 if (!cppc_req) 497 perf.bios_min_perf = min_perf; 498 499 perf.highest_perf = numerator; 500 perf.max_limit_perf = numerator; 501 perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 502 perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); 503 perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); 504 perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 505 WRITE_ONCE(cpudata->perf, perf); 506 WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1)); 507 WRITE_ONCE(cpudata->floor_perf_cnt, FIELD_GET(AMD_CPPC_FLOOR_PERF_CNT_MASK, cap1)); 508 509 return 0; 510 } 511 512 static int shmem_init_perf(struct amd_cpudata *cpudata) 513 { 514 struct cppc_perf_caps cppc_perf; 515 union perf_cached perf = READ_ONCE(cpudata->perf); 516 u64 numerator; 517 bool auto_sel; 518 519 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 520 if (ret) 521 return ret; 522 523 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); 524 if (ret) 525 return ret; 526 527 perf.highest_perf = numerator; 528 perf.max_limit_perf = numerator; 529 perf.min_limit_perf = cppc_perf.lowest_perf; 530 perf.nominal_perf = cppc_perf.nominal_perf; 531 perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 532 perf.lowest_perf = cppc_perf.lowest_perf; 533 WRITE_ONCE(cpudata->perf, perf); 534 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); 535 536 if (cppc_state == AMD_PSTATE_ACTIVE) 537 return 0; 538 539 ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel); 540 if (ret) { 541 pr_warn("failed to get auto_sel, ret: %d\n", ret); 542 return 0; 543 } 544 545 ret = cppc_set_auto_sel(cpudata->cpu, 546 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 547 548 if (ret) 549 pr_warn("failed to set auto_sel, ret: %d\n", ret); 550 551 return ret; 552 } 553 554 DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf); 555 556 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 557 { 558 return static_call(amd_pstate_init_perf)(cpudata); 559 } 560 561 static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf, 562 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 563 { 564 struct amd_cpudata *cpudata = policy->driver_data; 565 struct cppc_perf_ctrls perf_ctrls; 566 u64 value, prev; 567 int ret; 568 569 if (cppc_state == AMD_PSTATE_ACTIVE) { 570 int ret = shmem_set_epp(policy, epp); 571 572 if (ret) 573 return ret; 574 } 575 576 value = prev = READ_ONCE(cpudata->cppc_req_cached); 577 578 value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | 579 AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); 580 value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); 581 value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); 582 value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); 583 value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); 584 585 if (trace_amd_pstate_epp_perf_enabled()) { 586 union perf_cached perf = READ_ONCE(cpudata->perf); 587 588 trace_call__amd_pstate_epp_perf(cpudata->cpu, 589 perf.highest_perf, 590 epp, 591 min_perf, 592 max_perf, 593 policy->boost_enabled, 594 value != prev); 595 } 596 597 if (value == prev) 598 return 0; 599 600 perf_ctrls.max_perf = max_perf; 601 perf_ctrls.min_perf = min_perf; 602 perf_ctrls.desired_perf = des_perf; 603 604 ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); 605 if (ret) 606 return ret; 607 608 WRITE_ONCE(cpudata->cppc_req_cached, value); 609 610 return 0; 611 } 612 613 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 614 { 615 u64 aperf, mperf, tsc; 616 unsigned long flags; 617 618 local_irq_save(flags); 619 rdmsrq(MSR_IA32_APERF, aperf); 620 rdmsrq(MSR_IA32_MPERF, mperf); 621 tsc = rdtsc(); 622 623 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 624 local_irq_restore(flags); 625 return false; 626 } 627 628 local_irq_restore(flags); 629 630 cpudata->cur.aperf = aperf; 631 cpudata->cur.mperf = mperf; 632 cpudata->cur.tsc = tsc; 633 cpudata->cur.aperf -= cpudata->prev.aperf; 634 cpudata->cur.mperf -= cpudata->prev.mperf; 635 cpudata->cur.tsc -= cpudata->prev.tsc; 636 637 cpudata->prev.aperf = aperf; 638 cpudata->prev.mperf = mperf; 639 cpudata->prev.tsc = tsc; 640 641 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 642 643 return true; 644 } 645 646 static void amd_pstate_update(struct cpufreq_policy *policy, u8 min_perf, 647 u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) 648 { 649 struct amd_cpudata *cpudata = policy->driver_data; 650 union perf_cached perf = READ_ONCE(cpudata->perf); 651 652 /* limit the max perf when core performance boost feature is disabled */ 653 if (!cpudata->boost_supported) 654 max_perf = min_t(u8, perf.nominal_perf, max_perf); 655 656 des_perf = clamp_t(u8, des_perf, min_perf, max_perf); 657 658 policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); 659 660 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 661 min_perf = des_perf; 662 des_perf = 0; 663 } 664 665 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 666 trace_call__amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 667 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 668 cpudata->cpu, fast_switch); 669 } 670 671 amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch); 672 } 673 674 static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) 675 { 676 /* 677 * Initialize lower frequency limit (i.e.policy->min) with 678 * lowest_nonlinear_frequency or the min frequency (if) specified in BIOS, 679 * Override the initial value set by cpufreq core and amd-pstate qos_requests. 680 */ 681 if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { 682 struct cpufreq_policy *policy __free(put_cpufreq_policy) = 683 cpufreq_cpu_get(policy_data->cpu); 684 struct amd_cpudata *cpudata; 685 union perf_cached perf; 686 687 if (!policy) 688 return -EINVAL; 689 690 cpudata = policy->driver_data; 691 perf = READ_ONCE(cpudata->perf); 692 693 if (perf.bios_min_perf) 694 policy_data->min = perf_to_freq(perf, cpudata->nominal_freq, 695 perf.bios_min_perf); 696 else 697 policy_data->min = cpudata->lowest_nonlinear_freq; 698 } 699 700 cpufreq_verify_within_cpu_limits(policy_data); 701 702 return 0; 703 } 704 705 static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) 706 { 707 struct amd_cpudata *cpudata = policy->driver_data; 708 union perf_cached perf = READ_ONCE(cpudata->perf); 709 710 perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); 711 WRITE_ONCE(cpudata->max_limit_freq, policy->max); 712 713 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 714 /* 715 * For performance policy, set MinPerf to nominal_perf rather than 716 * highest_perf or lowest_nonlinear_perf. 717 * 718 * Per commit 0c411b39e4f4c, using highest_perf was observed 719 * to cause frequency throttling on power-limited platforms, leading to 720 * performance regressions. Using lowest_nonlinear_perf would limit 721 * performance too much for HPC workloads requiring high frequency 722 * operation and minimal wakeup latency from idle states. 723 * 724 * nominal_perf therefore provides a balance by avoiding throttling 725 * while still maintaining enough performance for HPC workloads. 726 */ 727 perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); 728 WRITE_ONCE(cpudata->min_limit_freq, min(cpudata->nominal_freq, cpudata->max_limit_freq)); 729 } else { 730 perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); 731 WRITE_ONCE(cpudata->min_limit_freq, policy->min); 732 } 733 734 WRITE_ONCE(cpudata->perf, perf); 735 } 736 737 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 738 unsigned int target_freq, bool fast_switch) 739 { 740 struct cpufreq_freqs freqs; 741 struct amd_cpudata *cpudata; 742 union perf_cached perf; 743 u8 des_perf; 744 745 cpudata = policy->driver_data; 746 747 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 748 amd_pstate_update_min_max_limit(policy); 749 750 perf = READ_ONCE(cpudata->perf); 751 752 freqs.old = policy->cur; 753 freqs.new = target_freq; 754 755 des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq); 756 757 WARN_ON(fast_switch && !policy->fast_switch_enabled); 758 /* 759 * If fast_switch is desired, then there aren't any registered 760 * transition notifiers. See comment for 761 * cpufreq_enable_fast_switch(). 762 */ 763 if (!fast_switch) 764 cpufreq_freq_transition_begin(policy, &freqs); 765 766 amd_pstate_update(policy, perf.min_limit_perf, des_perf, 767 perf.max_limit_perf, fast_switch, 768 policy->governor->flags); 769 770 if (!fast_switch) 771 cpufreq_freq_transition_end(policy, &freqs, false); 772 773 return 0; 774 } 775 776 static int amd_pstate_target(struct cpufreq_policy *policy, 777 unsigned int target_freq, 778 unsigned int relation) 779 { 780 return amd_pstate_update_freq(policy, target_freq, false); 781 } 782 783 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 784 unsigned int target_freq) 785 { 786 if (!amd_pstate_update_freq(policy, target_freq, true)) 787 return target_freq; 788 return policy->cur; 789 } 790 791 static void amd_pstate_adjust_perf(struct cpufreq_policy *policy, 792 unsigned long _min_perf, 793 unsigned long target_perf, 794 unsigned long capacity) 795 { 796 u8 max_perf, min_perf, des_perf, cap_perf; 797 struct amd_cpudata *cpudata; 798 union perf_cached perf; 799 800 if (!policy) 801 return; 802 803 cpudata = policy->driver_data; 804 805 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 806 amd_pstate_update_min_max_limit(policy); 807 808 perf = READ_ONCE(cpudata->perf); 809 cap_perf = perf.highest_perf; 810 811 des_perf = cap_perf; 812 if (target_perf < capacity) 813 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 814 815 if (_min_perf < capacity) 816 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 817 else 818 min_perf = cap_perf; 819 820 if (min_perf < perf.min_limit_perf) 821 min_perf = perf.min_limit_perf; 822 823 max_perf = perf.max_limit_perf; 824 if (max_perf < min_perf) 825 max_perf = min_perf; 826 827 amd_pstate_update(policy, min_perf, des_perf, max_perf, true, 828 policy->governor->flags); 829 } 830 831 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) 832 { 833 struct amd_cpudata *cpudata = policy->driver_data; 834 u32 nominal_freq; 835 int ret = 0; 836 837 nominal_freq = READ_ONCE(cpudata->nominal_freq); 838 839 if (on) 840 policy->cpuinfo.max_freq = cpudata->max_freq; 841 else if (policy->cpuinfo.max_freq > nominal_freq) 842 policy->cpuinfo.max_freq = nominal_freq; 843 844 if (cppc_state == AMD_PSTATE_PASSIVE) { 845 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); 846 if (ret < 0) 847 pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu); 848 } 849 850 return ret < 0 ? ret : 0; 851 } 852 853 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 854 { 855 struct amd_cpudata *cpudata = policy->driver_data; 856 int ret; 857 858 if (!cpudata->boost_supported) { 859 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 860 return -EOPNOTSUPP; 861 } 862 863 ret = amd_pstate_cpu_boost_update(policy, state); 864 refresh_frequency_limits(policy); 865 866 return ret; 867 } 868 869 static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) 870 { 871 u64 boost_val; 872 int ret = -1; 873 874 /* 875 * If platform has no CPB support or disable it, initialize current driver 876 * boost_enabled state to be false, it is not an error for cpufreq core to handle. 877 */ 878 if (!cpu_feature_enabled(X86_FEATURE_CPB)) { 879 pr_debug_once("Boost CPB capabilities not present in the processor\n"); 880 ret = 0; 881 goto exit_err; 882 } 883 884 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); 885 if (ret) { 886 pr_err_once("failed to read initial CPU boost state!\n"); 887 ret = -EIO; 888 goto exit_err; 889 } 890 891 if (!(boost_val & MSR_K7_HWCR_CPB_DIS)) 892 cpudata->boost_supported = true; 893 894 return 0; 895 896 exit_err: 897 cpudata->boost_supported = false; 898 return ret; 899 } 900 901 static void amd_perf_ctl_reset(unsigned int cpu) 902 { 903 wrmsrq_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 904 } 905 906 #define CPPC_MAX_PERF U8_MAX 907 908 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) 909 { 910 /* user disabled or not detected */ 911 if (!amd_pstate_prefcore) 912 return; 913 914 /* should use amd-hfi instead */ 915 if (cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS) && 916 IS_ENABLED(CONFIG_AMD_HFI)) { 917 amd_pstate_prefcore = false; 918 return; 919 } 920 921 cpudata->hw_prefcore = true; 922 923 /* Priorities must be initialized before ITMT support can be toggled on. */ 924 sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); 925 } 926 927 static void amd_pstate_update_limits(struct cpufreq_policy *policy) 928 { 929 struct amd_cpudata *cpudata; 930 u32 prev_high = 0, cur_high = 0; 931 bool highest_perf_changed = false; 932 unsigned int cpu = policy->cpu; 933 934 if (!amd_pstate_prefcore) 935 return; 936 937 if (amd_get_highest_perf(cpu, &cur_high)) 938 return; 939 940 cpudata = policy->driver_data; 941 942 prev_high = READ_ONCE(cpudata->prefcore_ranking); 943 highest_perf_changed = (prev_high != cur_high); 944 if (highest_perf_changed) { 945 WRITE_ONCE(cpudata->prefcore_ranking, cur_high); 946 947 if (cur_high < CPPC_MAX_PERF) { 948 sched_set_itmt_core_prio((int)cur_high, cpu); 949 sched_update_asym_prefer_cpu(cpu, prev_high, cur_high); 950 } 951 } 952 } 953 954 /* 955 * Get pstate transition delay time from ACPI tables that firmware set 956 * instead of using hardcode value directly. 957 */ 958 static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) 959 { 960 int transition_delay_ns; 961 962 transition_delay_ns = cppc_get_transition_latency(cpu); 963 if (transition_delay_ns < 0) { 964 if (cpu_feature_enabled(X86_FEATURE_AMD_FAST_CPPC)) 965 return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY; 966 else 967 return AMD_PSTATE_TRANSITION_DELAY; 968 } 969 970 return transition_delay_ns / NSEC_PER_USEC; 971 } 972 973 /* 974 * Get pstate transition latency value from ACPI tables that firmware 975 * set instead of using hardcode value directly. 976 */ 977 static u32 amd_pstate_get_transition_latency(unsigned int cpu) 978 { 979 int transition_latency; 980 981 transition_latency = cppc_get_transition_latency(cpu); 982 if (transition_latency < 0) 983 return AMD_PSTATE_TRANSITION_LATENCY; 984 985 return transition_latency; 986 } 987 988 /* 989 * amd_pstate_init_freq: Initialize the nominal_freq and lowest_nonlinear_freq 990 * for the @cpudata object. 991 * 992 * Requires: all perf members of @cpudata to be initialized. 993 * 994 * Returns 0 on success, non-zero value on failure. 995 */ 996 static int amd_pstate_init_freq(struct amd_cpudata *cpudata) 997 { 998 u32 min_freq, max_freq, nominal_freq, lowest_nonlinear_freq; 999 struct cppc_perf_caps cppc_perf; 1000 union perf_cached perf; 1001 int ret; 1002 1003 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 1004 if (ret) 1005 return ret; 1006 perf = READ_ONCE(cpudata->perf); 1007 1008 if (quirks && quirks->nominal_freq) 1009 nominal_freq = quirks->nominal_freq; 1010 else 1011 nominal_freq = cppc_perf.nominal_freq; 1012 nominal_freq *= 1000; 1013 1014 if (quirks && quirks->lowest_freq) { 1015 min_freq = quirks->lowest_freq; 1016 perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq); 1017 WRITE_ONCE(cpudata->perf, perf); 1018 } else 1019 min_freq = cppc_perf.lowest_freq; 1020 1021 min_freq *= 1000; 1022 1023 WRITE_ONCE(cpudata->nominal_freq, nominal_freq); 1024 1025 /* max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf */ 1026 max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); 1027 WRITE_ONCE(cpudata->max_freq, max_freq); 1028 1029 lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); 1030 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); 1031 1032 /** 1033 * Below values need to be initialized correctly, otherwise driver will fail to load 1034 * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] 1035 * Check _CPC in ACPI table objects if any values are incorrect 1036 */ 1037 if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { 1038 pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", 1039 min_freq, max_freq, nominal_freq); 1040 return -EINVAL; 1041 } 1042 1043 if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq) { 1044 pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", 1045 lowest_nonlinear_freq, min_freq, nominal_freq); 1046 return -EINVAL; 1047 } 1048 1049 return 0; 1050 } 1051 1052 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 1053 { 1054 struct amd_cpudata *cpudata; 1055 union perf_cached perf; 1056 struct device *dev; 1057 int ret; 1058 1059 /* 1060 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1061 * which is ideal for initialization process. 1062 */ 1063 amd_perf_ctl_reset(policy->cpu); 1064 dev = get_cpu_device(policy->cpu); 1065 if (!dev) 1066 return -ENODEV; 1067 1068 cpudata = kzalloc_obj(*cpudata); 1069 if (!cpudata) 1070 return -ENOMEM; 1071 1072 cpudata->cpu = policy->cpu; 1073 1074 ret = amd_pstate_init_perf(cpudata); 1075 if (ret) 1076 goto free_cpudata1; 1077 1078 amd_pstate_init_prefcore(cpudata); 1079 1080 ret = amd_pstate_init_freq(cpudata); 1081 if (ret) 1082 goto free_cpudata1; 1083 1084 ret = amd_pstate_init_boost_support(cpudata); 1085 if (ret) 1086 goto free_cpudata1; 1087 1088 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); 1089 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); 1090 1091 perf = READ_ONCE(cpudata->perf); 1092 1093 policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, 1094 cpudata->nominal_freq, 1095 perf.lowest_perf); 1096 policy->cpuinfo.max_freq = policy->max = cpudata->max_freq; 1097 1098 policy->driver_data = cpudata; 1099 ret = amd_pstate_cppc_enable(policy); 1100 if (ret) 1101 goto free_cpudata1; 1102 1103 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1104 1105 /* It will be updated by governor */ 1106 policy->cur = policy->cpuinfo.min_freq; 1107 1108 if (cpu_feature_enabled(X86_FEATURE_CPPC)) 1109 policy->fast_switch_possible = true; 1110 1111 ret = amd_pstate_init_floor_perf(policy); 1112 if (ret) { 1113 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); 1114 goto free_cpudata1; 1115 } 1116 1117 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 1118 FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); 1119 if (ret < 0) { 1120 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 1121 goto free_cpudata1; 1122 } 1123 1124 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 1125 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 1126 if (ret < 0) { 1127 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 1128 goto free_cpudata2; 1129 } 1130 1131 1132 if (!current_pstate_driver->adjust_perf) 1133 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1134 1135 return 0; 1136 1137 free_cpudata2: 1138 freq_qos_remove_request(&cpudata->req[0]); 1139 free_cpudata1: 1140 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1141 kfree(cpudata); 1142 policy->driver_data = NULL; 1143 return ret; 1144 } 1145 1146 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) 1147 { 1148 struct amd_cpudata *cpudata = policy->driver_data; 1149 union perf_cached perf = READ_ONCE(cpudata->perf); 1150 1151 /* Reset CPPC_REQ MSR to the BIOS value */ 1152 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1153 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 1154 1155 freq_qos_remove_request(&cpudata->req[1]); 1156 freq_qos_remove_request(&cpudata->req[0]); 1157 policy->fast_switch_possible = false; 1158 kfree(cpudata); 1159 } 1160 1161 static int amd_pstate_get_balanced_epp(struct cpufreq_policy *policy) 1162 { 1163 struct amd_cpudata *cpudata = policy->driver_data; 1164 1165 if (power_supply_is_system_supplied()) 1166 return cpudata->epp_default_ac; 1167 else 1168 return cpudata->epp_default_dc; 1169 } 1170 1171 static int amd_pstate_power_supply_notifier(struct notifier_block *nb, 1172 unsigned long event, void *data) 1173 { 1174 struct amd_cpudata *cpudata = container_of(nb, struct amd_cpudata, power_nb); 1175 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 1176 u8 epp; 1177 int ret; 1178 1179 if (event != PSY_EVENT_PROP_CHANGED) 1180 return NOTIFY_OK; 1181 1182 /* dynamic actions are only applied while platform profile is in balanced */ 1183 if (cpudata->current_profile != PLATFORM_PROFILE_BALANCED) 1184 return 0; 1185 1186 epp = amd_pstate_get_balanced_epp(policy); 1187 1188 ret = amd_pstate_set_epp(policy, epp); 1189 if (ret) 1190 pr_warn("Failed to set CPU %d EPP %u: %d\n", cpudata->cpu, epp, ret); 1191 1192 return NOTIFY_OK; 1193 } 1194 1195 static int amd_pstate_profile_probe(void *drvdata, unsigned long *choices) 1196 { 1197 set_bit(PLATFORM_PROFILE_LOW_POWER, choices); 1198 set_bit(PLATFORM_PROFILE_BALANCED, choices); 1199 set_bit(PLATFORM_PROFILE_PERFORMANCE, choices); 1200 1201 return 0; 1202 } 1203 1204 static int amd_pstate_profile_get(struct device *dev, 1205 enum platform_profile_option *profile) 1206 { 1207 struct amd_cpudata *cpudata = dev_get_drvdata(dev); 1208 1209 *profile = cpudata->current_profile; 1210 1211 return 0; 1212 } 1213 1214 static int amd_pstate_profile_set(struct device *dev, 1215 enum platform_profile_option profile) 1216 { 1217 struct amd_cpudata *cpudata = dev_get_drvdata(dev); 1218 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 1219 int ret; 1220 1221 switch (profile) { 1222 case PLATFORM_PROFILE_LOW_POWER: 1223 ret = amd_pstate_set_epp(policy, AMD_CPPC_EPP_POWERSAVE); 1224 if (ret) 1225 return ret; 1226 break; 1227 case PLATFORM_PROFILE_BALANCED: 1228 ret = amd_pstate_set_epp(policy, 1229 amd_pstate_get_balanced_epp(policy)); 1230 if (ret) 1231 return ret; 1232 break; 1233 case PLATFORM_PROFILE_PERFORMANCE: 1234 ret = amd_pstate_set_epp(policy, AMD_CPPC_EPP_PERFORMANCE); 1235 if (ret) 1236 return ret; 1237 break; 1238 default: 1239 pr_err("Unknown Platform Profile %d\n", profile); 1240 return -EOPNOTSUPP; 1241 } 1242 1243 cpudata->current_profile = profile; 1244 1245 return 0; 1246 } 1247 1248 static const struct platform_profile_ops amd_pstate_profile_ops = { 1249 .probe = amd_pstate_profile_probe, 1250 .profile_set = amd_pstate_profile_set, 1251 .profile_get = amd_pstate_profile_get, 1252 }; 1253 1254 void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy) 1255 { 1256 struct amd_cpudata *cpudata = policy->driver_data; 1257 1258 if (cpudata->power_nb.notifier_call) 1259 power_supply_unreg_notifier(&cpudata->power_nb); 1260 if (cpudata->ppdev) { 1261 platform_profile_remove(cpudata->ppdev); 1262 cpudata->ppdev = NULL; 1263 } 1264 kfree(cpudata->profile_name); 1265 cpudata->dynamic_epp = false; 1266 } 1267 EXPORT_SYMBOL_GPL(amd_pstate_clear_dynamic_epp); 1268 1269 static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy) 1270 { 1271 struct amd_cpudata *cpudata = policy->driver_data; 1272 int ret; 1273 u8 epp; 1274 1275 switch (cpudata->current_profile) { 1276 case PLATFORM_PROFILE_PERFORMANCE: 1277 epp = AMD_CPPC_EPP_PERFORMANCE; 1278 break; 1279 case PLATFORM_PROFILE_LOW_POWER: 1280 epp = AMD_CPPC_EPP_POWERSAVE; 1281 break; 1282 case PLATFORM_PROFILE_BALANCED: 1283 epp = amd_pstate_get_balanced_epp(policy); 1284 break; 1285 default: 1286 pr_err("Unknown Platform Profile %d\n", cpudata->current_profile); 1287 return -EOPNOTSUPP; 1288 } 1289 ret = amd_pstate_set_epp(policy, epp); 1290 if (ret) 1291 return ret; 1292 1293 cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu); 1294 1295 cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu), 1296 cpudata->profile_name, 1297 policy->driver_data, 1298 &amd_pstate_profile_ops); 1299 if (IS_ERR(cpudata->ppdev)) { 1300 ret = PTR_ERR(cpudata->ppdev); 1301 goto cleanup; 1302 } 1303 1304 /* only enable notifier if things will actually change */ 1305 if (cpudata->epp_default_ac != cpudata->epp_default_dc) { 1306 cpudata->power_nb.notifier_call = amd_pstate_power_supply_notifier; 1307 ret = power_supply_reg_notifier(&cpudata->power_nb); 1308 if (ret) 1309 goto cleanup; 1310 } 1311 1312 cpudata->dynamic_epp = true; 1313 1314 return 0; 1315 1316 cleanup: 1317 amd_pstate_clear_dynamic_epp(policy); 1318 1319 return ret; 1320 } 1321 1322 /* Sysfs attributes */ 1323 1324 /* 1325 * This frequency is to indicate the maximum hardware frequency. 1326 * If boost is not active but supported, the frequency will be larger than the 1327 * one in cpuinfo. 1328 */ 1329 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 1330 char *buf) 1331 { 1332 struct amd_cpudata *cpudata = policy->driver_data; 1333 1334 return sysfs_emit(buf, "%u\n", cpudata->max_freq); 1335 } 1336 1337 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 1338 char *buf) 1339 { 1340 struct amd_cpudata *cpudata; 1341 union perf_cached perf; 1342 1343 cpudata = policy->driver_data; 1344 perf = READ_ONCE(cpudata->perf); 1345 1346 return sysfs_emit(buf, "%u\n", 1347 perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf)); 1348 } 1349 1350 /* 1351 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 1352 * need to expose it to sysfs. 1353 */ 1354 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 1355 char *buf) 1356 { 1357 struct amd_cpudata *cpudata; 1358 1359 cpudata = policy->driver_data; 1360 1361 return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf); 1362 } 1363 1364 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, 1365 char *buf) 1366 { 1367 u8 perf; 1368 struct amd_cpudata *cpudata = policy->driver_data; 1369 1370 perf = READ_ONCE(cpudata->prefcore_ranking); 1371 1372 return sysfs_emit(buf, "%u\n", perf); 1373 } 1374 1375 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, 1376 char *buf) 1377 { 1378 bool hw_prefcore; 1379 struct amd_cpudata *cpudata = policy->driver_data; 1380 1381 hw_prefcore = READ_ONCE(cpudata->hw_prefcore); 1382 1383 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); 1384 } 1385 1386 static ssize_t show_energy_performance_available_preferences( 1387 struct cpufreq_policy *policy, char *buf) 1388 { 1389 int offset = 0, i; 1390 struct amd_cpudata *cpudata = policy->driver_data; 1391 1392 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1393 return sysfs_emit_at(buf, offset, "%s\n", 1394 energy_perf_strings[EPP_INDEX_PERFORMANCE]); 1395 1396 for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++) 1397 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]); 1398 1399 offset += sysfs_emit_at(buf, offset, "\n"); 1400 1401 return offset; 1402 } 1403 1404 ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, 1405 const char *buf, size_t count) 1406 { 1407 struct amd_cpudata *cpudata = policy->driver_data; 1408 ssize_t ret; 1409 bool raw_epp = false; 1410 u8 epp; 1411 1412 if (cpudata->dynamic_epp) { 1413 pr_debug("EPP cannot be set when dynamic EPP is enabled\n"); 1414 return -EBUSY; 1415 } 1416 1417 /* 1418 * if the value matches a number, use that, otherwise see if 1419 * matches an index in the energy_perf_strings array 1420 */ 1421 ret = kstrtou8(buf, 0, &epp); 1422 raw_epp = !ret; 1423 if (ret) { 1424 ret = sysfs_match_string(energy_perf_strings, buf); 1425 if (ret < 0 || ret == EPP_INDEX_CUSTOM) 1426 return -EINVAL; 1427 if (ret) 1428 epp = epp_values[ret]; 1429 else 1430 epp = amd_pstate_get_balanced_epp(policy); 1431 } 1432 1433 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 1434 pr_debug("EPP cannot be set under performance policy\n"); 1435 return -EBUSY; 1436 } 1437 1438 ret = amd_pstate_set_epp(policy, epp); 1439 if (ret) 1440 return ret; 1441 1442 cpudata->raw_epp = raw_epp; 1443 1444 return count; 1445 } 1446 EXPORT_SYMBOL_GPL(store_energy_performance_preference); 1447 1448 ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf) 1449 { 1450 struct amd_cpudata *cpudata = policy->driver_data; 1451 u8 preference, epp; 1452 1453 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 1454 1455 if (cpudata->raw_epp) 1456 return sysfs_emit(buf, "%u\n", epp); 1457 1458 switch (epp) { 1459 case AMD_CPPC_EPP_PERFORMANCE: 1460 preference = EPP_INDEX_PERFORMANCE; 1461 break; 1462 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 1463 preference = EPP_INDEX_BALANCE_PERFORMANCE; 1464 break; 1465 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 1466 preference = EPP_INDEX_BALANCE_POWERSAVE; 1467 break; 1468 case AMD_CPPC_EPP_POWERSAVE: 1469 preference = EPP_INDEX_POWERSAVE; 1470 break; 1471 default: 1472 return -EINVAL; 1473 } 1474 1475 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 1476 } 1477 EXPORT_SYMBOL_GPL(show_energy_performance_preference); 1478 1479 static ssize_t store_amd_pstate_floor_freq(struct cpufreq_policy *policy, 1480 const char *buf, size_t count) 1481 { 1482 struct amd_cpudata *cpudata = policy->driver_data; 1483 union perf_cached perf = READ_ONCE(cpudata->perf); 1484 unsigned int freq; 1485 u8 floor_perf; 1486 int ret; 1487 1488 ret = kstrtouint(buf, 0, &freq); 1489 if (ret) 1490 return ret; 1491 1492 if (freq < policy->cpuinfo.min_freq || freq > policy->max) 1493 return -EINVAL; 1494 1495 floor_perf = freq_to_perf(perf, cpudata->nominal_freq, freq); 1496 ret = amd_pstate_set_floor_perf(policy, floor_perf); 1497 1498 if (!ret) 1499 cpudata->floor_freq = freq; 1500 1501 return ret ?: count; 1502 } 1503 1504 static ssize_t show_amd_pstate_floor_freq(struct cpufreq_policy *policy, char *buf) 1505 { 1506 struct amd_cpudata *cpudata = policy->driver_data; 1507 1508 return sysfs_emit(buf, "%u\n", cpudata->floor_freq); 1509 } 1510 1511 static ssize_t show_amd_pstate_floor_count(struct cpufreq_policy *policy, char *buf) 1512 { 1513 struct amd_cpudata *cpudata = policy->driver_data; 1514 u8 count = cpudata->floor_perf_cnt; 1515 1516 return sysfs_emit(buf, "%u\n", count); 1517 } 1518 1519 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1520 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1521 1522 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1523 cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); 1524 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); 1525 cpufreq_freq_attr_rw(energy_performance_preference); 1526 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1527 cpufreq_freq_attr_rw(amd_pstate_floor_freq); 1528 cpufreq_freq_attr_ro(amd_pstate_floor_count); 1529 1530 struct freq_attr_visibility { 1531 struct freq_attr *attr; 1532 bool (*visibility_fn)(void); 1533 }; 1534 1535 /* For attributes which are always visible */ 1536 static bool always_visible(void) 1537 { 1538 return true; 1539 } 1540 1541 /* Determines whether prefcore related attributes should be visible */ 1542 static bool prefcore_visibility(void) 1543 { 1544 return amd_pstate_prefcore; 1545 } 1546 1547 /* Determines whether energy performance preference should be visible */ 1548 static bool epp_visibility(void) 1549 { 1550 return cppc_state == AMD_PSTATE_ACTIVE; 1551 } 1552 1553 /* Determines whether amd_pstate_floor_freq related attributes should be visible */ 1554 static bool floor_freq_visibility(void) 1555 { 1556 return cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO); 1557 } 1558 1559 static struct freq_attr_visibility amd_pstate_attr_visibility[] = { 1560 {&amd_pstate_max_freq, always_visible}, 1561 {&amd_pstate_lowest_nonlinear_freq, always_visible}, 1562 {&amd_pstate_highest_perf, always_visible}, 1563 {&amd_pstate_prefcore_ranking, prefcore_visibility}, 1564 {&amd_pstate_hw_prefcore, prefcore_visibility}, 1565 {&energy_performance_preference, epp_visibility}, 1566 {&energy_performance_available_preferences, epp_visibility}, 1567 {&amd_pstate_floor_freq, floor_freq_visibility}, 1568 {&amd_pstate_floor_count, floor_freq_visibility}, 1569 }; 1570 1571 struct freq_attr **amd_pstate_get_current_attrs(void) 1572 { 1573 if (!current_pstate_driver) 1574 return NULL; 1575 return current_pstate_driver->attr; 1576 } 1577 EXPORT_SYMBOL_GPL(amd_pstate_get_current_attrs); 1578 1579 static struct freq_attr **get_freq_attrs(void) 1580 { 1581 bool attr_visible[ARRAY_SIZE(amd_pstate_attr_visibility)]; 1582 struct freq_attr **attrs; 1583 int i, j, count; 1584 1585 for (i = 0, count = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { 1586 struct freq_attr_visibility *v = &amd_pstate_attr_visibility[i]; 1587 1588 attr_visible[i] = v->visibility_fn(); 1589 if (attr_visible[i]) 1590 count++; 1591 } 1592 1593 /* amd_pstate_{max_freq, lowest_nonlinear_freq, highest_perf} should always be visible */ 1594 BUG_ON(!count); 1595 1596 attrs = kcalloc(count + 1, sizeof(struct freq_attr *), GFP_KERNEL); 1597 if (!attrs) 1598 return ERR_PTR(-ENOMEM); 1599 1600 for (i = 0, j = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { 1601 if (!attr_visible[i]) 1602 continue; 1603 1604 attrs[j++] = amd_pstate_attr_visibility[i].attr; 1605 } 1606 1607 return attrs; 1608 } 1609 1610 static void amd_pstate_driver_cleanup(void) 1611 { 1612 if (amd_pstate_prefcore) 1613 sched_clear_itmt_support(); 1614 1615 cppc_state = AMD_PSTATE_DISABLE; 1616 kfree(current_pstate_driver->attr); 1617 current_pstate_driver->attr = NULL; 1618 current_pstate_driver = NULL; 1619 } 1620 1621 static int amd_pstate_set_driver(int mode_idx) 1622 { 1623 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1624 cppc_state = mode_idx; 1625 if (cppc_state == AMD_PSTATE_DISABLE) 1626 pr_info("driver is explicitly disabled\n"); 1627 1628 if (cppc_state == AMD_PSTATE_ACTIVE) 1629 current_pstate_driver = &amd_pstate_epp_driver; 1630 1631 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1632 current_pstate_driver = &amd_pstate_driver; 1633 1634 return 0; 1635 } 1636 1637 return -EINVAL; 1638 } 1639 1640 static int amd_pstate_register_driver(int mode) 1641 { 1642 struct freq_attr **attr = NULL; 1643 int ret; 1644 1645 ret = amd_pstate_set_driver(mode); 1646 if (ret) 1647 return ret; 1648 1649 cppc_state = mode; 1650 1651 /* 1652 * Note: It is important to compute the attrs _after_ 1653 * re-initializing the cppc_state. Some attributes become 1654 * visible only when cppc_state is AMD_PSTATE_ACTIVE. 1655 */ 1656 attr = get_freq_attrs(); 1657 if (IS_ERR(attr)) { 1658 ret = (int) PTR_ERR(attr); 1659 pr_err("Couldn't compute freq_attrs for current mode %s [%d]\n", 1660 amd_pstate_get_mode_string(cppc_state), ret); 1661 amd_pstate_driver_cleanup(); 1662 return ret; 1663 } 1664 1665 current_pstate_driver->attr = attr; 1666 1667 /* at least one CPU supports CPB */ 1668 current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); 1669 1670 ret = cpufreq_register_driver(current_pstate_driver); 1671 if (ret) { 1672 amd_pstate_driver_cleanup(); 1673 return ret; 1674 } 1675 1676 /* Enable ITMT support once all CPUs have initialized their asym priorities. */ 1677 if (amd_pstate_prefcore) 1678 sched_set_itmt_support(); 1679 1680 return 0; 1681 } 1682 1683 static int amd_pstate_unregister_driver(int dummy) 1684 { 1685 cpufreq_unregister_driver(current_pstate_driver); 1686 amd_pstate_driver_cleanup(); 1687 return 0; 1688 } 1689 1690 static int amd_pstate_change_mode_without_dvr_change(int mode) 1691 { 1692 int cpu = 0; 1693 1694 cppc_state = mode; 1695 1696 if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 1697 return 0; 1698 1699 for_each_online_cpu(cpu) { 1700 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 1701 } 1702 1703 return 0; 1704 } 1705 1706 static int amd_pstate_change_driver_mode(int mode) 1707 { 1708 int ret; 1709 1710 ret = amd_pstate_unregister_driver(0); 1711 if (ret) 1712 return ret; 1713 1714 ret = amd_pstate_register_driver(mode); 1715 if (ret) 1716 return ret; 1717 1718 return 0; 1719 } 1720 1721 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 1722 [AMD_PSTATE_DISABLE] = { 1723 [AMD_PSTATE_DISABLE] = NULL, 1724 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 1725 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 1726 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 1727 }, 1728 [AMD_PSTATE_PASSIVE] = { 1729 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1730 [AMD_PSTATE_PASSIVE] = NULL, 1731 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1732 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 1733 }, 1734 [AMD_PSTATE_ACTIVE] = { 1735 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1736 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 1737 [AMD_PSTATE_ACTIVE] = NULL, 1738 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 1739 }, 1740 [AMD_PSTATE_GUIDED] = { 1741 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1742 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 1743 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1744 [AMD_PSTATE_GUIDED] = NULL, 1745 }, 1746 }; 1747 1748 static ssize_t amd_pstate_show_status(char *buf) 1749 { 1750 if (!current_pstate_driver) 1751 return sysfs_emit(buf, "disable\n"); 1752 1753 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 1754 } 1755 1756 int amd_pstate_get_status(void) 1757 { 1758 return cppc_state; 1759 } 1760 EXPORT_SYMBOL_GPL(amd_pstate_get_status); 1761 1762 int amd_pstate_update_status(const char *buf, size_t size) 1763 { 1764 int mode_idx; 1765 1766 if (size > strlen("passive") || size < strlen("active")) 1767 return -EINVAL; 1768 1769 mode_idx = get_mode_idx_from_str(buf, size); 1770 if (mode_idx < 0) 1771 return mode_idx; 1772 1773 if (mode_state_machine[cppc_state][mode_idx]) { 1774 guard(mutex)(&amd_pstate_driver_lock); 1775 return mode_state_machine[cppc_state][mode_idx](mode_idx); 1776 } 1777 1778 return 0; 1779 } 1780 EXPORT_SYMBOL_GPL(amd_pstate_update_status); 1781 1782 static ssize_t status_show(struct device *dev, 1783 struct device_attribute *attr, char *buf) 1784 { 1785 1786 guard(mutex)(&amd_pstate_driver_lock); 1787 1788 return amd_pstate_show_status(buf); 1789 } 1790 1791 static ssize_t status_store(struct device *a, struct device_attribute *b, 1792 const char *buf, size_t count) 1793 { 1794 char *p = memchr(buf, '\n', count); 1795 int ret; 1796 1797 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1798 1799 return ret < 0 ? ret : count; 1800 } 1801 1802 static ssize_t prefcore_show(struct device *dev, 1803 struct device_attribute *attr, char *buf) 1804 { 1805 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); 1806 } 1807 1808 static ssize_t dynamic_epp_show(struct device *dev, 1809 struct device_attribute *attr, char *buf) 1810 { 1811 return sysfs_emit(buf, "%s\n", str_enabled_disabled(dynamic_epp)); 1812 } 1813 1814 static ssize_t dynamic_epp_store(struct device *a, struct device_attribute *b, 1815 const char *buf, size_t count) 1816 { 1817 bool enabled; 1818 int ret; 1819 1820 ret = kstrtobool(buf, &enabled); 1821 if (ret) 1822 return ret; 1823 1824 if (dynamic_epp == enabled) 1825 return -EINVAL; 1826 1827 /* reinitialize with desired dynamic EPP value */ 1828 dynamic_epp = enabled; 1829 ret = amd_pstate_change_driver_mode(cppc_state); 1830 if (ret) 1831 dynamic_epp = false; 1832 1833 return ret ? ret : count; 1834 } 1835 1836 static DEVICE_ATTR_RW(status); 1837 static DEVICE_ATTR_RO(prefcore); 1838 static DEVICE_ATTR_RW(dynamic_epp); 1839 1840 static struct attribute *pstate_global_attributes[] = { 1841 &dev_attr_status.attr, 1842 &dev_attr_prefcore.attr, 1843 &dev_attr_dynamic_epp.attr, 1844 NULL 1845 }; 1846 1847 static const struct attribute_group amd_pstate_global_attr_group = { 1848 .name = "amd_pstate", 1849 .attrs = pstate_global_attributes, 1850 }; 1851 1852 static bool amd_pstate_acpi_pm_profile_server(void) 1853 { 1854 switch (acpi_gbl_FADT.preferred_profile) { 1855 case PM_ENTERPRISE_SERVER: 1856 case PM_SOHO_SERVER: 1857 case PM_PERFORMANCE_SERVER: 1858 return true; 1859 } 1860 return false; 1861 } 1862 1863 static bool amd_pstate_acpi_pm_profile_undefined(void) 1864 { 1865 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) 1866 return true; 1867 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) 1868 return true; 1869 return false; 1870 } 1871 1872 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1873 { 1874 struct amd_cpudata *cpudata; 1875 union perf_cached perf; 1876 struct device *dev; 1877 int ret; 1878 1879 /* 1880 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1881 * which is ideal for initialization process. 1882 */ 1883 amd_perf_ctl_reset(policy->cpu); 1884 dev = get_cpu_device(policy->cpu); 1885 if (!dev) 1886 return -ENODEV; 1887 1888 cpudata = kzalloc_obj(*cpudata); 1889 if (!cpudata) 1890 return -ENOMEM; 1891 1892 cpudata->cpu = policy->cpu; 1893 1894 ret = amd_pstate_init_perf(cpudata); 1895 if (ret) 1896 goto free_cpudata1; 1897 1898 amd_pstate_init_prefcore(cpudata); 1899 1900 ret = amd_pstate_init_freq(cpudata); 1901 if (ret) 1902 goto free_cpudata1; 1903 1904 ret = amd_pstate_init_boost_support(cpudata); 1905 if (ret) 1906 goto free_cpudata1; 1907 1908 perf = READ_ONCE(cpudata->perf); 1909 1910 policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, 1911 cpudata->nominal_freq, 1912 perf.lowest_perf); 1913 policy->cpuinfo.max_freq = policy->max = cpudata->max_freq; 1914 policy->driver_data = cpudata; 1915 1916 ret = amd_pstate_cppc_enable(policy); 1917 if (ret) 1918 goto free_cpudata1; 1919 1920 /* It will be updated by governor */ 1921 policy->cur = policy->cpuinfo.min_freq; 1922 1923 1924 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1925 1926 /* 1927 * Set the policy to provide a valid fallback value in case 1928 * the default cpufreq governor is neither powersave nor performance. 1929 */ 1930 if (amd_pstate_acpi_pm_profile_server() || 1931 amd_pstate_acpi_pm_profile_undefined()) { 1932 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 1933 cpudata->epp_default_ac = cpudata->epp_default_dc = amd_pstate_get_epp(cpudata); 1934 cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE; 1935 } else { 1936 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1937 cpudata->epp_default_ac = AMD_CPPC_EPP_PERFORMANCE; 1938 cpudata->epp_default_dc = AMD_CPPC_EPP_BALANCE_PERFORMANCE; 1939 cpudata->current_profile = PLATFORM_PROFILE_BALANCED; 1940 } 1941 1942 if (dynamic_epp) 1943 ret = amd_pstate_set_dynamic_epp(policy); 1944 else 1945 ret = amd_pstate_set_epp(policy, amd_pstate_get_balanced_epp(policy)); 1946 if (ret) 1947 goto free_cpudata1; 1948 1949 ret = amd_pstate_init_floor_perf(policy); 1950 if (ret) { 1951 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); 1952 goto free_cpudata1; 1953 } 1954 1955 current_pstate_driver->adjust_perf = NULL; 1956 1957 return 0; 1958 1959 free_cpudata1: 1960 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1961 kfree(cpudata); 1962 policy->driver_data = NULL; 1963 return ret; 1964 } 1965 1966 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1967 { 1968 struct amd_cpudata *cpudata = policy->driver_data; 1969 1970 if (cpudata) { 1971 union perf_cached perf = READ_ONCE(cpudata->perf); 1972 1973 /* Reset CPPC_REQ MSR to the BIOS value */ 1974 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1975 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 1976 1977 if (cpudata->dynamic_epp) 1978 amd_pstate_clear_dynamic_epp(policy); 1979 kfree(cpudata); 1980 policy->driver_data = NULL; 1981 } 1982 1983 pr_debug("CPU %d exiting\n", policy->cpu); 1984 } 1985 1986 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy, bool policy_change) 1987 { 1988 struct amd_cpudata *cpudata = policy->driver_data; 1989 union perf_cached perf; 1990 u8 epp; 1991 1992 if (policy_change || 1993 policy->min != cpudata->min_limit_freq || 1994 policy->max != cpudata->max_limit_freq) 1995 amd_pstate_update_min_max_limit(policy); 1996 1997 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1998 epp = 0; 1999 else 2000 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 2001 2002 perf = READ_ONCE(cpudata->perf); 2003 2004 return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U, 2005 perf.max_limit_perf, epp, false); 2006 } 2007 2008 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 2009 { 2010 struct amd_cpudata *cpudata = policy->driver_data; 2011 int ret; 2012 2013 if (!policy->cpuinfo.max_freq) 2014 return -ENODEV; 2015 2016 cpudata->policy = policy->policy; 2017 2018 ret = amd_pstate_epp_update_limit(policy, true); 2019 if (ret) 2020 return ret; 2021 2022 /* 2023 * policy->cur is never updated with the amd_pstate_epp driver, but it 2024 * is used as a stale frequency value. So, keep it within limits. 2025 */ 2026 policy->cur = policy->min; 2027 2028 return 0; 2029 } 2030 2031 static int amd_pstate_cpu_online(struct cpufreq_policy *policy) 2032 { 2033 struct amd_cpudata *cpudata = policy->driver_data; 2034 union perf_cached perf = READ_ONCE(cpudata->perf); 2035 u8 cached_floor_perf; 2036 int ret; 2037 2038 ret = amd_pstate_cppc_enable(policy); 2039 if (ret) 2040 return ret; 2041 2042 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2043 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2044 } 2045 2046 static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) 2047 { 2048 struct amd_cpudata *cpudata = policy->driver_data; 2049 union perf_cached perf = READ_ONCE(cpudata->perf); 2050 int ret; 2051 2052 /* 2053 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 2054 * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the 2055 * limits, epp and desired perf will get reset to the cached values in cpudata struct 2056 */ 2057 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 2058 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 2059 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 2060 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 2061 false); 2062 if (ret) 2063 return ret; 2064 2065 return amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2066 } 2067 2068 static int amd_pstate_suspend(struct cpufreq_policy *policy) 2069 { 2070 struct amd_cpudata *cpudata = policy->driver_data; 2071 union perf_cached perf = READ_ONCE(cpudata->perf); 2072 int ret; 2073 2074 /* 2075 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 2076 * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, 2077 * the limits, epp and desired perf will get reset to the cached values in cpudata struct 2078 */ 2079 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 2080 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 2081 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 2082 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 2083 false); 2084 if (ret) 2085 return ret; 2086 2087 ret = amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2088 if (ret) 2089 return ret; 2090 2091 /* set this flag to avoid setting core offline*/ 2092 cpudata->suspended = true; 2093 2094 return 0; 2095 } 2096 2097 static int amd_pstate_resume(struct cpufreq_policy *policy) 2098 { 2099 struct amd_cpudata *cpudata = policy->driver_data; 2100 union perf_cached perf = READ_ONCE(cpudata->perf); 2101 int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur); 2102 u8 cached_floor_perf; 2103 int ret; 2104 2105 /* Set CPPC_REQ to last sane value until the governor updates it */ 2106 ret = amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, 2107 0U, false); 2108 if (ret) 2109 return ret; 2110 2111 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2112 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2113 } 2114 2115 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 2116 { 2117 struct amd_cpudata *cpudata = policy->driver_data; 2118 union perf_cached perf = READ_ONCE(cpudata->perf); 2119 u8 cached_floor_perf; 2120 2121 if (cpudata->suspended) { 2122 int ret; 2123 2124 /* enable amd pstate from suspend state*/ 2125 ret = amd_pstate_epp_update_limit(policy, false); 2126 if (ret) 2127 return ret; 2128 2129 cpudata->suspended = false; 2130 } 2131 2132 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2133 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2134 } 2135 2136 static struct cpufreq_driver amd_pstate_driver = { 2137 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 2138 .verify = amd_pstate_verify, 2139 .target = amd_pstate_target, 2140 .fast_switch = amd_pstate_fast_switch, 2141 .init = amd_pstate_cpu_init, 2142 .exit = amd_pstate_cpu_exit, 2143 .online = amd_pstate_cpu_online, 2144 .offline = amd_pstate_cpu_offline, 2145 .suspend = amd_pstate_suspend, 2146 .resume = amd_pstate_resume, 2147 .set_boost = amd_pstate_set_boost, 2148 .update_limits = amd_pstate_update_limits, 2149 .name = "amd-pstate", 2150 }; 2151 2152 static struct cpufreq_driver amd_pstate_epp_driver = { 2153 .flags = CPUFREQ_CONST_LOOPS, 2154 .verify = amd_pstate_verify, 2155 .setpolicy = amd_pstate_epp_set_policy, 2156 .init = amd_pstate_epp_cpu_init, 2157 .exit = amd_pstate_epp_cpu_exit, 2158 .offline = amd_pstate_cpu_offline, 2159 .online = amd_pstate_cpu_online, 2160 .suspend = amd_pstate_suspend, 2161 .resume = amd_pstate_epp_resume, 2162 .update_limits = amd_pstate_update_limits, 2163 .set_boost = amd_pstate_set_boost, 2164 .name = "amd-pstate-epp", 2165 }; 2166 2167 /* 2168 * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. 2169 * show the debug message that helps to check if the CPU has CPPC support for loading issue. 2170 */ 2171 static bool amd_cppc_supported(void) 2172 { 2173 struct cpuinfo_x86 *c = &cpu_data(0); 2174 bool warn = false; 2175 2176 if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) { 2177 pr_debug_once("CPPC feature is not supported by the processor\n"); 2178 return false; 2179 } 2180 2181 /* 2182 * If the CPPC feature is disabled in the BIOS for processors 2183 * that support MSR-based CPPC, the AMD Pstate driver may not 2184 * function correctly. 2185 * 2186 * For such processors, check the CPPC flag and display a 2187 * warning message if the platform supports CPPC. 2188 * 2189 * Note: The code check below will not abort the driver 2190 * registration process because of the code is added for 2191 * debugging purposes. Besides, it may still be possible for 2192 * the driver to work using the shared-memory mechanism. 2193 */ 2194 if (!cpu_feature_enabled(X86_FEATURE_CPPC)) { 2195 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 2196 switch (c->x86_model) { 2197 case 0x60 ... 0x6F: 2198 case 0x80 ... 0xAF: 2199 warn = true; 2200 break; 2201 } 2202 } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || 2203 cpu_feature_enabled(X86_FEATURE_ZEN4)) { 2204 switch (c->x86_model) { 2205 case 0x10 ... 0x1F: 2206 case 0x40 ... 0xAF: 2207 warn = true; 2208 break; 2209 } 2210 } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) { 2211 warn = true; 2212 } 2213 } 2214 2215 if (warn) 2216 pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n" 2217 "Please enable it if your BIOS has the CPPC option.\n"); 2218 return true; 2219 } 2220 2221 static int __init amd_pstate_init(void) 2222 { 2223 struct device *dev_root; 2224 int ret; 2225 2226 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 2227 return -ENODEV; 2228 2229 /* show debug message only if CPPC is not supported */ 2230 if (!amd_cppc_supported()) 2231 return -EOPNOTSUPP; 2232 2233 /* show warning message when BIOS broken or ACPI disabled */ 2234 if (!acpi_cpc_valid()) { 2235 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 2236 return -ENODEV; 2237 } 2238 2239 /* don't keep reloading if cpufreq_driver exists */ 2240 if (cpufreq_get_current_driver()) 2241 return -EEXIST; 2242 2243 quirks = NULL; 2244 2245 /* check if this machine need CPPC quirks */ 2246 dmi_check_system(amd_pstate_quirks_table); 2247 2248 /* 2249 * determine the driver mode from the command line or kernel config. 2250 * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED. 2251 * command line options will override the kernel config settings. 2252 */ 2253 2254 if (cppc_state == AMD_PSTATE_UNDEFINED) { 2255 /* Disable on the following configs by default: 2256 * 1. Undefined platforms 2257 * 2. Server platforms with CPUs older than Family 0x1A. 2258 */ 2259 if (amd_pstate_acpi_pm_profile_undefined() || 2260 (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) { 2261 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 2262 return -ENODEV; 2263 } 2264 /* get driver mode from kernel config option [1:4] */ 2265 cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; 2266 } 2267 2268 if (cppc_state == AMD_PSTATE_DISABLE) { 2269 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 2270 return -ENODEV; 2271 } 2272 2273 /* capability check */ 2274 if (cpu_feature_enabled(X86_FEATURE_CPPC)) { 2275 pr_debug("AMD CPPC MSR based functionality is supported\n"); 2276 } else { 2277 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 2278 static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); 2279 static_call_update(amd_pstate_init_perf, shmem_init_perf); 2280 static_call_update(amd_pstate_update_perf, shmem_update_perf); 2281 static_call_update(amd_pstate_get_epp, shmem_get_epp); 2282 static_call_update(amd_pstate_set_epp, shmem_set_epp); 2283 } 2284 2285 if (amd_pstate_prefcore) { 2286 ret = amd_detect_prefcore(&amd_pstate_prefcore); 2287 if (ret) 2288 return ret; 2289 } 2290 2291 ret = amd_pstate_register_driver(cppc_state); 2292 if (ret) { 2293 pr_err("failed to register with return %d\n", ret); 2294 return ret; 2295 } 2296 2297 dev_root = bus_get_dev_root(&cpu_subsys); 2298 if (dev_root) { 2299 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 2300 put_device(dev_root); 2301 if (ret) { 2302 pr_err("sysfs attribute export failed with error %d.\n", ret); 2303 goto global_attr_free; 2304 } 2305 } 2306 2307 return ret; 2308 2309 global_attr_free: 2310 amd_pstate_unregister_driver(0); 2311 return ret; 2312 } 2313 device_initcall(amd_pstate_init); 2314 2315 static int __init amd_pstate_param(char *str) 2316 { 2317 size_t size; 2318 int mode_idx; 2319 2320 if (!str) 2321 return -EINVAL; 2322 2323 size = strlen(str); 2324 mode_idx = get_mode_idx_from_str(str, size); 2325 2326 return amd_pstate_set_driver(mode_idx); 2327 } 2328 2329 static int __init amd_prefcore_param(char *str) 2330 { 2331 if (!strcmp(str, "disable")) 2332 amd_pstate_prefcore = false; 2333 2334 return 0; 2335 } 2336 2337 static int __init amd_dynamic_epp_param(char *str) 2338 { 2339 if (!strcmp(str, "disable")) 2340 dynamic_epp = false; 2341 if (!strcmp(str, "enable")) 2342 dynamic_epp = true; 2343 2344 return 0; 2345 } 2346 2347 early_param("amd_pstate", amd_pstate_param); 2348 early_param("amd_prefcore", amd_prefcore_param); 2349 early_param("amd_dynamic_epp", amd_dynamic_epp_param); 2350 2351 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 2352 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 2353