1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/bitfield.h> 26 #include <linux/kernel.h> 27 #include <linux/module.h> 28 #include <linux/init.h> 29 #include <linux/smp.h> 30 #include <linux/sched.h> 31 #include <linux/cpufreq.h> 32 #include <linux/compiler.h> 33 #include <linux/dmi.h> 34 #include <linux/slab.h> 35 #include <linux/acpi.h> 36 #include <linux/io.h> 37 #include <linux/delay.h> 38 #include <linux/uaccess.h> 39 #include <linux/power_supply.h> 40 #include <linux/static_call.h> 41 #include <linux/topology.h> 42 43 #include <acpi/processor.h> 44 #include <acpi/cppc_acpi.h> 45 46 #include <asm/msr.h> 47 #include <asm/cpufeature.h> 48 #include <asm/cpu_device_id.h> 49 50 #include "amd-pstate.h" 51 #include "amd-pstate-trace.h" 52 53 #define AMD_PSTATE_TRANSITION_LATENCY 20000 54 #define AMD_PSTATE_TRANSITION_DELAY 1000 55 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600 56 57 #define AMD_CPPC_EPP_LEGACY_PERFORMANCE 0x00 58 #define AMD_CPPC_EPP_LEGACY_BALANCE_PERFORMANCE 0x80 59 #define AMD_CPPC_EPP_LEGACY_BALANCE_POWERSAVE 0xBF 60 #define AMD_CPPC_EPP_LEGACY_POWERSAVE 0xFF 61 62 static const char * const amd_pstate_mode_string[] = { 63 [AMD_PSTATE_UNDEFINED] = "undefined", 64 [AMD_PSTATE_DISABLE] = "disable", 65 [AMD_PSTATE_PASSIVE] = "passive", 66 [AMD_PSTATE_ACTIVE] = "active", 67 [AMD_PSTATE_GUIDED] = "guided", 68 }; 69 static_assert(ARRAY_SIZE(amd_pstate_mode_string) == AMD_PSTATE_MAX); 70 71 const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode) 72 { 73 if (mode < AMD_PSTATE_UNDEFINED || mode >= AMD_PSTATE_MAX) 74 mode = AMD_PSTATE_UNDEFINED; 75 return amd_pstate_mode_string[mode]; 76 } 77 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_mode_string); 78 79 struct quirk_entry { 80 u32 nominal_freq; 81 u32 lowest_freq; 82 }; 83 84 static struct cpufreq_driver *current_pstate_driver; 85 static struct cpufreq_driver amd_pstate_driver; 86 static struct cpufreq_driver amd_pstate_epp_driver; 87 static int cppc_state = AMD_PSTATE_UNDEFINED; 88 static bool amd_pstate_prefcore = true; 89 static struct quirk_entry *quirks; 90 91 /* 92 * The numeric EPP value programmed for each named preference. First dimension 93 * is CPU type (TOPO_CPU_TYPE_ANY for non-hybrid, TOPO_CPU_TYPE_PERFORMANCE/ 94 * EFFICIENCY/LOW_POWER for hybrid). The initializer holds the legacy values 95 * used as the fallback on any platform not listed in amd_pstate_epp_soc_ids[]; 96 * amd_pstate_init_epp_values() overwrites slots at boot when the running SoC 97 * has a per-SoC (and potentially per-CPU-type) override. 98 */ 99 static u8 epp_values[][EPP_INDEX_MAX] = { 100 /* 101 * Initialize all CPU types to legacy defaults. 102 * amd_pstate_init_epp_values() will fix these up 103 * based on the platform during boot. 104 */ 105 [TOPO_CPU_TYPE_ANY ... TOPO_CPU_TYPE_LOW_POWER] = { 106 [EPP_INDEX_DEFAULT] = 0, 107 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_LEGACY_PERFORMANCE, 108 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_LEGACY_BALANCE_PERFORMANCE, 109 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_LEGACY_BALANCE_POWERSAVE, 110 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_LEGACY_POWERSAVE, 111 }, 112 }; 113 static_assert(ARRAY_SIZE(epp_values) == TOPO_CPU_TYPE_LOW_POWER + 1, 114 "epp_values must have entries for all CPU types up to TOPO_CPU_TYPE_LOW_POWER"); 115 116 /* 117 * Get the EPP value row for a given CPU, accounting for hybrid CPU types. 118 * Non-hybrid systems use TOPO_CPU_TYPE_ANY; hybrid systems use the CPU's 119 * actual type (PERFORMANCE/EFFICIENCY/LOW_POWER). 120 */ 121 u8 *amd_pstate_cpu_epp_values(enum x86_topology_cpu_type cpu_type) 122 { 123 switch (cpu_type) { 124 case TOPO_CPU_TYPE_PERFORMANCE: 125 case TOPO_CPU_TYPE_EFFICIENCY: 126 case TOPO_CPU_TYPE_LOW_POWER: 127 return epp_values[cpu_type]; 128 default: 129 return epp_values[TOPO_CPU_TYPE_ANY]; 130 } 131 } 132 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_cpu_epp_values); 133 134 /** 135 * struct amd_pstate_epp_values - EPP values for the four named preferences 136 * @performance: value for the "performance" preference 137 * @balance_performance: value for the "balance_performance" preference 138 * @balance_power: value for the "balance_power" preference 139 * @power: value for the "power" preference 140 */ 141 struct amd_pstate_epp_values { 142 u8 performance; 143 u8 balance_performance; 144 u8 balance_power; 145 u8 power; 146 }; 147 148 /** 149 * struct amd_pstate_epp_soc - per-CPU-type EPP overrides for hybrid systems 150 * @performance_core: values for TOPO_CPU_TYPE_PERFORMANCE cores 151 * @efficiency_core: values for TOPO_CPU_TYPE_EFFICIENCY cores 152 * @low_power_core: values for TOPO_CPU_TYPE_LOW_POWER cores 153 * 154 * Referenced from amd_pstate_epp_soc_ids[] to give a hybrid platform its own 155 * numeric EPP values for the four named preferences, with distinct values per 156 * CPU type. Non-hybrid systems are not listed in the table and always use the 157 * legacy defaults. 158 */ 159 struct amd_pstate_epp_soc { 160 struct amd_pstate_epp_values performance_core; 161 struct amd_pstate_epp_values efficiency_core; 162 struct amd_pstate_epp_values low_power_core; 163 }; 164 165 /* 166 * Per-CPU-type EPP overrides for hybrid systems. Only hybrid SoCs should be 167 * listed here; non-hybrid systems always use the legacy defaults. 168 */ 169 static const struct amd_pstate_epp_soc epp_soc_zen6_client __initconst = { 170 .performance_core = { 171 .performance = 25, 172 .balance_performance = 51, 173 .balance_power = 64, 174 .power = 115, 175 }, 176 .efficiency_core = { 177 .performance = 25, 178 .balance_performance = 51, 179 .balance_power = 64, 180 .power = 115, 181 }, 182 .low_power_core = { 183 .performance = 25, 184 .balance_performance = 64, 185 .balance_power = 64, 186 .power = 64, 187 }, 188 }; 189 190 static const struct x86_cpu_id amd_pstate_epp_soc_ids[] __initconst = { 191 X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x80, &epp_soc_zen6_client), 192 X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x81, &epp_soc_zen6_client), 193 X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x84, &epp_soc_zen6_client), 194 X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x85, &epp_soc_zen6_client), 195 X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0xe0, &epp_soc_zen6_client), 196 {} 197 }; 198 199 typedef int (*cppc_mode_transition_fn)(int); 200 201 static struct quirk_entry quirk_amd_7k62 = { 202 .nominal_freq = 2600, 203 .lowest_freq = 550, 204 }; 205 206 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) 207 { 208 /** 209 * match the broken bios for family 17h processor support CPPC V2 210 * broken BIOS lack of nominal_freq and lowest_freq capabilities 211 * definition in ACPI tables 212 */ 213 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 214 quirks = dmi->driver_data; 215 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); 216 return 1; 217 } 218 219 return 0; 220 } 221 222 static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { 223 { 224 .callback = dmi_matched_7k62_bios_bug, 225 .ident = "AMD EPYC 7K62", 226 .matches = { 227 DMI_MATCH(DMI_BIOS_VERSION, "5.14"), 228 DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), 229 }, 230 .driver_data = &quirk_amd_7k62, 231 }, 232 {} 233 }; 234 MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); 235 236 static inline int get_mode_idx_from_str(const char *str, size_t size) 237 { 238 int i; 239 240 for (i = 0; i < AMD_PSTATE_MAX; i++) { 241 if (!strncmp(str, amd_pstate_mode_string[i], size)) 242 return i; 243 } 244 return -EINVAL; 245 } 246 247 static DEFINE_MUTEX(amd_pstate_driver_lock); 248 249 static int msr_get_epp(struct amd_cpudata *cpudata) 250 { 251 u64 value; 252 int ret; 253 254 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 255 if (ret < 0) { 256 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 257 return ret; 258 } 259 260 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value); 261 } 262 263 DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); 264 265 static inline int amd_pstate_get_epp(struct amd_cpudata *cpudata) 266 { 267 return static_call(amd_pstate_get_epp)(cpudata); 268 } 269 270 static int shmem_get_epp(struct amd_cpudata *cpudata) 271 { 272 u64 epp; 273 int ret; 274 275 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 276 if (ret < 0) { 277 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 278 return ret; 279 } 280 281 return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp); 282 } 283 284 static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, 285 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 286 { 287 struct amd_cpudata *cpudata = policy->driver_data; 288 u64 value, prev; 289 290 value = prev = READ_ONCE(cpudata->cppc_req_cached); 291 292 FIELD_MODIFY(AMD_CPPC_MAX_PERF_MASK, &value, max_perf); 293 FIELD_MODIFY(AMD_CPPC_DES_PERF_MASK, &value, des_perf); 294 FIELD_MODIFY(AMD_CPPC_MIN_PERF_MASK, &value, min_perf); 295 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 296 297 if (trace_amd_pstate_epp_perf_enabled()) { 298 union perf_cached perf = READ_ONCE(cpudata->perf); 299 300 trace_call__amd_pstate_epp_perf(cpudata->cpu, 301 perf.highest_perf, 302 epp, 303 min_perf, 304 max_perf, 305 policy->boost_enabled, 306 value != prev); 307 } 308 309 if (value == prev) 310 return 0; 311 312 if (fast_switch) { 313 wrmsrq(MSR_AMD_CPPC_REQ, value); 314 } else { 315 int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 316 317 if (ret) 318 return ret; 319 } 320 321 WRITE_ONCE(cpudata->cppc_req_cached, value); 322 323 return 0; 324 } 325 326 DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); 327 328 static inline int amd_pstate_update_perf(struct cpufreq_policy *policy, 329 u8 min_perf, u8 des_perf, 330 u8 max_perf, u8 epp, 331 bool fast_switch) 332 { 333 return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf, 334 max_perf, epp, fast_switch); 335 } 336 337 static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) 338 { 339 struct amd_cpudata *cpudata = policy->driver_data; 340 u64 value, prev; 341 int ret; 342 343 value = prev = READ_ONCE(cpudata->cppc_req_cached); 344 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 345 346 if (trace_amd_pstate_epp_perf_enabled()) { 347 union perf_cached perf = cpudata->perf; 348 349 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 350 epp, 351 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 352 cpudata->cppc_req_cached), 353 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 354 cpudata->cppc_req_cached), 355 policy->boost_enabled, 356 value != prev); 357 } 358 359 if (value == prev) 360 return 0; 361 362 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 363 if (ret) { 364 pr_err("failed to set energy perf value (%d)\n", ret); 365 return ret; 366 } 367 368 /* update both so that msr_update_perf() can effectively check */ 369 WRITE_ONCE(cpudata->cppc_req_cached, value); 370 371 return ret; 372 } 373 374 DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); 375 376 static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp) 377 { 378 return static_call(amd_pstate_set_epp)(policy, epp); 379 } 380 381 static int amd_pstate_set_floor_perf(struct cpufreq_policy *policy, u8 perf) 382 { 383 struct amd_cpudata *cpudata = policy->driver_data; 384 u64 value, prev; 385 bool changed; 386 int ret; 387 388 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) 389 return 0; 390 391 value = prev = READ_ONCE(cpudata->cppc_req2_cached); 392 FIELD_MODIFY(AMD_CPPC_FLOOR_PERF_MASK, &value, perf); 393 394 changed = value != prev; 395 if (!changed) { 396 ret = 0; 397 goto out_trace; 398 } 399 400 ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, value); 401 if (ret) { 402 changed = false; 403 pr_err("failed to set CPPC REQ2 value. Error (%d)\n", ret); 404 goto out_trace; 405 } 406 407 WRITE_ONCE(cpudata->cppc_req2_cached, value); 408 409 out_trace: 410 if (trace_amd_pstate_cppc_req2_enabled()) 411 trace_call__amd_pstate_cppc_req2(cpudata->cpu, perf, changed, 412 ret); 413 return ret; 414 } 415 416 static int amd_pstate_init_floor_perf(struct cpufreq_policy *policy) 417 { 418 struct amd_cpudata *cpudata = policy->driver_data; 419 u8 floor_perf; 420 u64 value; 421 int ret; 422 423 if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) 424 return 0; 425 426 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, &value); 427 if (ret) { 428 pr_err("failed to read CPPC REQ2 value. Error (%d)\n", ret); 429 return ret; 430 } 431 432 WRITE_ONCE(cpudata->cppc_req2_cached, value); 433 floor_perf = FIELD_GET(AMD_CPPC_FLOOR_PERF_MASK, 434 cpudata->cppc_req2_cached); 435 436 /* Set a sane value for floor_perf if the default value is invalid */ 437 if (floor_perf < cpudata->perf.lowest_perf) { 438 floor_perf = cpudata->perf.nominal_perf; 439 ret = amd_pstate_set_floor_perf(policy, floor_perf); 440 if (ret) 441 return ret; 442 } 443 444 445 cpudata->bios_floor_perf = floor_perf; 446 cpudata->floor_freq = perf_to_freq(cpudata->perf, cpudata->nominal_freq, 447 floor_perf); 448 return 0; 449 } 450 451 static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) 452 { 453 struct amd_cpudata *cpudata = policy->driver_data; 454 struct cppc_perf_ctrls perf_ctrls; 455 u8 epp_cached; 456 u64 value; 457 int ret; 458 459 460 epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 461 if (trace_amd_pstate_epp_perf_enabled()) { 462 union perf_cached perf = cpudata->perf; 463 464 trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, 465 epp, 466 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, 467 cpudata->cppc_req_cached), 468 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, 469 cpudata->cppc_req_cached), 470 policy->boost_enabled, 471 epp != epp_cached); 472 } 473 474 if (epp == epp_cached) 475 return 0; 476 477 perf_ctrls.energy_perf = epp; 478 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 479 if (ret) { 480 pr_debug("failed to set energy perf value (%d)\n", ret); 481 return ret; 482 } 483 484 value = READ_ONCE(cpudata->cppc_req_cached); 485 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 486 WRITE_ONCE(cpudata->cppc_req_cached, value); 487 488 return ret; 489 } 490 491 static inline int msr_cppc_enable(struct cpufreq_policy *policy) 492 { 493 return wrmsrq_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); 494 } 495 496 static int shmem_cppc_enable(struct cpufreq_policy *policy) 497 { 498 return cppc_set_enable(policy->cpu, 1); 499 } 500 501 DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); 502 503 static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) 504 { 505 return static_call(amd_pstate_cppc_enable)(policy); 506 } 507 508 static int msr_init_perf(struct amd_cpudata *cpudata) 509 { 510 union perf_cached perf = READ_ONCE(cpudata->perf); 511 u64 cap1, numerator, cppc_req; 512 513 int ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 514 &cap1); 515 if (ret) 516 return ret; 517 518 ret = amd_get_effective_highest_perf(cpudata->cpu); 519 if (ret < 0) 520 return ret; 521 522 numerator = ret; 523 524 ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); 525 if (ret) 526 return ret; 527 528 WRITE_ONCE(cpudata->cppc_req_cached, cppc_req); 529 530 perf.highest_perf = numerator; 531 perf.max_limit_perf = numerator; 532 perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 533 perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); 534 perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); 535 perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); 536 perf.bios_min_perf = FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cppc_req); 537 WRITE_ONCE(cpudata->perf, perf); 538 WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1)); 539 WRITE_ONCE(cpudata->floor_perf_cnt, FIELD_GET(AMD_CPPC_FLOOR_PERF_CNT_MASK, cap1)); 540 541 return 0; 542 } 543 544 static int shmem_init_perf(struct amd_cpudata *cpudata) 545 { 546 struct cppc_perf_caps cppc_perf; 547 union perf_cached perf = READ_ONCE(cpudata->perf); 548 u64 numerator; 549 bool auto_sel; 550 551 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 552 if (ret) 553 return ret; 554 555 ret = amd_get_effective_highest_perf(cpudata->cpu); 556 if (ret < 0) 557 return ret; 558 559 numerator = ret; 560 561 perf.highest_perf = numerator; 562 perf.max_limit_perf = numerator; 563 perf.min_limit_perf = cppc_perf.lowest_perf; 564 perf.nominal_perf = cppc_perf.nominal_perf; 565 perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 566 perf.lowest_perf = cppc_perf.lowest_perf; 567 WRITE_ONCE(cpudata->perf, perf); 568 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); 569 570 ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel); 571 if (ret) { 572 pr_warn("failed to get auto_sel, ret: %d\n", ret); 573 return 0; 574 } 575 576 if ((cppc_state != AMD_PSTATE_PASSIVE) == auto_sel) 577 return 0; 578 579 ret = cppc_set_auto_sel(cpudata->cpu, !auto_sel); 580 581 if (ret) 582 pr_warn("failed to set auto_sel, ret: %d\n", ret); 583 584 return ret; 585 } 586 587 DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf); 588 589 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 590 { 591 return static_call(amd_pstate_init_perf)(cpudata); 592 } 593 594 static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf, 595 u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) 596 { 597 struct amd_cpudata *cpudata = policy->driver_data; 598 struct cppc_perf_ctrls perf_ctrls = { 599 .max_perf = max_perf, 600 .min_perf = min_perf, 601 .desired_perf = des_perf, 602 }; 603 u64 value, prev; 604 int ret; 605 606 if (cppc_state == AMD_PSTATE_ACTIVE) { 607 int ret = shmem_set_epp(policy, epp); 608 609 if (ret) 610 return ret; 611 } 612 613 value = prev = READ_ONCE(cpudata->cppc_req_cached); 614 615 FIELD_MODIFY(AMD_CPPC_MAX_PERF_MASK, &value, max_perf); 616 FIELD_MODIFY(AMD_CPPC_DES_PERF_MASK, &value, des_perf); 617 FIELD_MODIFY(AMD_CPPC_MIN_PERF_MASK, &value, min_perf); 618 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp); 619 620 if (trace_amd_pstate_epp_perf_enabled()) { 621 union perf_cached perf = READ_ONCE(cpudata->perf); 622 623 trace_call__amd_pstate_epp_perf(cpudata->cpu, 624 perf.highest_perf, 625 epp, 626 min_perf, 627 max_perf, 628 policy->boost_enabled, 629 value != prev); 630 } 631 632 if (value == prev) 633 return 0; 634 635 ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); 636 if (ret) 637 return ret; 638 639 WRITE_ONCE(cpudata->cppc_req_cached, value); 640 641 return 0; 642 } 643 644 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 645 { 646 u64 aperf, mperf, tsc; 647 unsigned long flags; 648 649 local_irq_save(flags); 650 rdmsrq(MSR_IA32_APERF, aperf); 651 rdmsrq(MSR_IA32_MPERF, mperf); 652 tsc = rdtsc(); 653 654 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 655 local_irq_restore(flags); 656 return false; 657 } 658 659 local_irq_restore(flags); 660 661 cpudata->cur.aperf = aperf; 662 cpudata->cur.mperf = mperf; 663 cpudata->cur.tsc = tsc; 664 cpudata->cur.aperf -= cpudata->prev.aperf; 665 cpudata->cur.mperf -= cpudata->prev.mperf; 666 cpudata->cur.tsc -= cpudata->prev.tsc; 667 668 cpudata->prev.aperf = aperf; 669 cpudata->prev.mperf = mperf; 670 cpudata->prev.tsc = tsc; 671 672 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 673 674 return true; 675 } 676 677 static void amd_pstate_update(struct cpufreq_policy *policy, u8 min_perf, 678 u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) 679 { 680 struct amd_cpudata *cpudata = policy->driver_data; 681 union perf_cached perf = READ_ONCE(cpudata->perf); 682 683 /* limit the max perf when core performance boost feature is disabled */ 684 if (!cpudata->boost_supported) 685 max_perf = min_t(u8, perf.nominal_perf, max_perf); 686 687 des_perf = clamp_t(u8, des_perf, min_perf, max_perf); 688 689 policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); 690 691 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 692 min_perf = des_perf; 693 des_perf = 0; 694 } 695 696 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 697 trace_call__amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 698 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 699 cpudata->cpu, fast_switch); 700 } 701 702 amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch); 703 } 704 705 static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) 706 { 707 /* 708 * Initialize lower frequency limit (i.e.policy->min) with 709 * lowest_nonlinear_frequency or the min frequency (if) specified in BIOS, 710 * Override the initial value set by cpufreq core and amd-pstate qos_requests. 711 */ 712 if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { 713 struct cpufreq_policy *policy __free(put_cpufreq_policy) = 714 cpufreq_cpu_get(policy_data->cpu); 715 struct amd_cpudata *cpudata; 716 union perf_cached perf; 717 718 if (!policy) 719 return -EINVAL; 720 721 cpudata = policy->driver_data; 722 perf = READ_ONCE(cpudata->perf); 723 724 if (perf.bios_min_perf) 725 policy_data->min = perf_to_freq(perf, cpudata->nominal_freq, 726 perf.bios_min_perf); 727 else 728 policy_data->min = cpudata->lowest_nonlinear_freq; 729 } 730 731 cpufreq_verify_within_cpu_limits(policy_data); 732 733 return 0; 734 } 735 736 static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) 737 { 738 struct amd_cpudata *cpudata = policy->driver_data; 739 union perf_cached perf = READ_ONCE(cpudata->perf); 740 741 if (policy->max >= READ_ONCE(cpudata->max_freq)) 742 perf.max_limit_perf = perf.highest_perf; 743 else 744 perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); 745 WRITE_ONCE(cpudata->max_limit_freq, policy->max); 746 747 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 748 u8 min_limit_perf = perf.bios_min_perf ?: perf.nominal_perf; 749 u32 min_limit_freq; 750 751 /* 752 * For performance policy, set MinPerf to nominal_perf / bios_min_perf 753 * rather than highest_perf or lowest_nonlinear_perf. 754 * 755 * Per commit 0c411b39e4f4c, using highest_perf was observed 756 * to cause frequency throttling on power-limited platforms, leading to 757 * performance regressions. Using lowest_nonlinear_perf would limit 758 * performance too much for HPC workloads requiring high frequency 759 * operation and minimal wakeup latency from idle states. 760 * 761 * nominal_perf therefore provides a balanced default by avoiding 762 * throttling while still maintaining enough performance for HPC 763 * workloads when bios_min_perf is not available. 764 * 765 * When bios_min_perf is available, users have profiled their workloads 766 * to understand the best idling frequency. Use that instead. 767 */ 768 min_limit_perf = min(min_limit_perf, perf.max_limit_perf); 769 min_limit_freq = perf_to_freq(perf, cpudata->nominal_freq, min_limit_perf); 770 perf.min_limit_perf = min_limit_perf; 771 772 WRITE_ONCE(cpudata->min_limit_freq, min(min_limit_freq, cpudata->max_limit_freq)); 773 } else { 774 perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); 775 WRITE_ONCE(cpudata->min_limit_freq, policy->min); 776 } 777 778 WRITE_ONCE(cpudata->perf, perf); 779 } 780 781 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 782 unsigned int target_freq, bool fast_switch) 783 { 784 struct cpufreq_freqs freqs; 785 struct amd_cpudata *cpudata; 786 union perf_cached perf; 787 u8 des_perf; 788 789 cpudata = policy->driver_data; 790 791 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 792 amd_pstate_update_min_max_limit(policy); 793 794 perf = READ_ONCE(cpudata->perf); 795 796 freqs.old = policy->cur; 797 freqs.new = target_freq; 798 799 des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq); 800 801 WARN_ON(fast_switch && !policy->fast_switch_enabled); 802 /* 803 * If fast_switch is desired, then there aren't any registered 804 * transition notifiers. See comment for 805 * cpufreq_enable_fast_switch(). 806 */ 807 if (!fast_switch) 808 cpufreq_freq_transition_begin(policy, &freqs); 809 810 amd_pstate_update(policy, perf.min_limit_perf, des_perf, 811 perf.max_limit_perf, fast_switch, 812 policy->governor->flags); 813 814 if (!fast_switch) 815 cpufreq_freq_transition_end(policy, &freqs, false); 816 817 return 0; 818 } 819 820 static int amd_pstate_target(struct cpufreq_policy *policy, 821 unsigned int target_freq, 822 unsigned int relation) 823 { 824 return amd_pstate_update_freq(policy, target_freq, false); 825 } 826 827 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 828 unsigned int target_freq) 829 { 830 if (!amd_pstate_update_freq(policy, target_freq, true)) 831 return target_freq; 832 return policy->cur; 833 } 834 835 static void amd_pstate_adjust_perf(struct cpufreq_policy *policy, 836 unsigned long _min_perf, 837 unsigned long target_perf, 838 unsigned long _max_perf, 839 unsigned long capacity) 840 { 841 u8 max_perf, min_perf, des_perf, cap_perf; 842 struct amd_cpudata *cpudata; 843 union perf_cached perf; 844 845 if (!policy) 846 return; 847 848 cpudata = policy->driver_data; 849 850 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 851 amd_pstate_update_min_max_limit(policy); 852 853 perf = READ_ONCE(cpudata->perf); 854 cap_perf = perf.highest_perf; 855 856 des_perf = cap_perf; 857 if (target_perf < capacity) 858 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 859 860 if (_min_perf < capacity) 861 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 862 else 863 min_perf = cap_perf; 864 865 if (min_perf < perf.min_limit_perf) 866 min_perf = perf.min_limit_perf; 867 868 max_perf = perf.max_limit_perf; 869 if (max_perf < min_perf) 870 max_perf = min_perf; 871 872 amd_pstate_update(policy, min_perf, des_perf, max_perf, true, 873 policy->governor->flags); 874 } 875 876 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) 877 { 878 struct amd_cpudata *cpudata = policy->driver_data; 879 u32 nominal_freq; 880 int ret = 0; 881 882 nominal_freq = READ_ONCE(cpudata->nominal_freq); 883 884 if (on) 885 policy->cpuinfo.max_freq = cpudata->max_freq; 886 else if (policy->cpuinfo.max_freq > nominal_freq) 887 policy->cpuinfo.max_freq = nominal_freq; 888 889 if (cppc_state == AMD_PSTATE_PASSIVE) { 890 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); 891 if (ret < 0) 892 pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu); 893 } 894 895 return ret < 0 ? ret : 0; 896 } 897 898 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 899 { 900 struct amd_cpudata *cpudata = policy->driver_data; 901 int ret; 902 903 if (!cpudata->boost_supported) { 904 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 905 return -EOPNOTSUPP; 906 } 907 908 ret = amd_pstate_cpu_boost_update(policy, state); 909 refresh_frequency_limits(policy); 910 911 return ret; 912 } 913 914 static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) 915 { 916 u64 boost_val; 917 int ret = -1; 918 919 /* 920 * If platform has no CPB support or disable it, initialize current driver 921 * boost_enabled state to be false, it is not an error for cpufreq core to handle. 922 */ 923 if (!cpu_feature_enabled(X86_FEATURE_CPB)) { 924 pr_debug_once("Boost CPB capabilities not present in the processor\n"); 925 ret = 0; 926 goto exit_err; 927 } 928 929 ret = rdmsrq_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); 930 if (ret) { 931 pr_err_once("failed to read initial CPU boost state!\n"); 932 ret = -EIO; 933 goto exit_err; 934 } 935 936 if (!(boost_val & MSR_K7_HWCR_CPB_DIS)) 937 cpudata->boost_supported = true; 938 939 return 0; 940 941 exit_err: 942 cpudata->boost_supported = false; 943 return ret; 944 } 945 946 static void amd_perf_ctl_reset(unsigned int cpu) 947 { 948 wrmsrq_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 949 } 950 951 #define CPPC_MAX_PERF U8_MAX 952 953 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) 954 { 955 /* user disabled or not detected */ 956 if (!amd_pstate_prefcore) 957 return; 958 959 /* should use amd-hfi instead */ 960 if (cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS) && 961 IS_ENABLED(CONFIG_AMD_HFI)) { 962 amd_pstate_prefcore = false; 963 return; 964 } 965 966 cpudata->hw_prefcore = true; 967 968 /* Priorities must be initialized before ITMT support can be toggled on. */ 969 sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); 970 } 971 972 static void amd_pstate_update_limits(struct cpufreq_policy *policy) 973 { 974 struct amd_cpudata *cpudata; 975 u32 prev_high = 0, cur_high = 0; 976 bool highest_perf_changed = false; 977 unsigned int cpu = policy->cpu; 978 979 if (!amd_pstate_prefcore) 980 return; 981 982 if (amd_get_highest_perf(cpu, &cur_high)) 983 return; 984 985 cpudata = policy->driver_data; 986 987 prev_high = READ_ONCE(cpudata->prefcore_ranking); 988 highest_perf_changed = (prev_high != cur_high); 989 if (highest_perf_changed) { 990 WRITE_ONCE(cpudata->prefcore_ranking, cur_high); 991 992 if (cur_high < CPPC_MAX_PERF) { 993 sched_set_itmt_core_prio((int)cur_high, cpu); 994 sched_update_asym_prefer_cpu(cpu, prev_high, cur_high); 995 } 996 } 997 } 998 999 /* 1000 * Get pstate transition delay time from ACPI tables that firmware set 1001 * instead of using hardcode value directly. 1002 */ 1003 static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) 1004 { 1005 int transition_delay_ns; 1006 1007 transition_delay_ns = cppc_get_transition_latency(cpu); 1008 if (transition_delay_ns < 0) { 1009 if (cpu_feature_enabled(X86_FEATURE_AMD_FAST_CPPC)) 1010 return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY; 1011 else 1012 return AMD_PSTATE_TRANSITION_DELAY; 1013 } 1014 1015 return transition_delay_ns / NSEC_PER_USEC; 1016 } 1017 1018 /* 1019 * Get pstate transition latency value from ACPI tables that firmware 1020 * set instead of using hardcode value directly. 1021 */ 1022 static u32 amd_pstate_get_transition_latency(unsigned int cpu) 1023 { 1024 int transition_latency; 1025 1026 transition_latency = cppc_get_transition_latency(cpu); 1027 if (transition_latency < 0) 1028 return AMD_PSTATE_TRANSITION_LATENCY; 1029 1030 return transition_latency; 1031 } 1032 1033 /* 1034 * amd_pstate_init_freq: Initialize the nominal_freq and lowest_nonlinear_freq 1035 * for the @cpudata object. 1036 * 1037 * Requires: all perf members of @cpudata to be initialized. 1038 * 1039 * Returns 0 on success, non-zero value on failure. 1040 */ 1041 static int amd_pstate_init_freq(struct amd_cpudata *cpudata) 1042 { 1043 u32 min_freq, max_freq, nominal_freq, lowest_nonlinear_freq; 1044 struct cppc_perf_caps cppc_perf; 1045 union perf_cached perf; 1046 int ret; 1047 1048 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 1049 if (ret) 1050 return ret; 1051 perf = READ_ONCE(cpudata->perf); 1052 1053 if (quirks && quirks->nominal_freq) 1054 nominal_freq = quirks->nominal_freq; 1055 else 1056 nominal_freq = cppc_perf.nominal_freq; 1057 nominal_freq *= 1000; 1058 1059 if (quirks && quirks->lowest_freq) { 1060 min_freq = quirks->lowest_freq; 1061 perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq); 1062 WRITE_ONCE(cpudata->perf, perf); 1063 } else 1064 min_freq = cppc_perf.lowest_freq; 1065 1066 min_freq *= 1000; 1067 1068 WRITE_ONCE(cpudata->nominal_freq, nominal_freq); 1069 1070 /* try to look up from BIOS/quirk first, fall back to (nominal_freq * highest_perf)/nominal_perf */ 1071 max_freq = amd_get_max_frequency(cpudata->cpu) * 1000; 1072 if (!max_freq) 1073 max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); 1074 WRITE_ONCE(cpudata->max_freq, max_freq); 1075 1076 lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); 1077 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); 1078 1079 /** 1080 * Below values need to be initialized correctly, otherwise driver will fail to load 1081 * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] 1082 * Check _CPC in ACPI table objects if any values are incorrect 1083 */ 1084 if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { 1085 pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", 1086 min_freq, max_freq, nominal_freq); 1087 return -EINVAL; 1088 } 1089 1090 if (lowest_nonlinear_freq < min_freq || lowest_nonlinear_freq > nominal_freq) { 1091 pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", 1092 lowest_nonlinear_freq, min_freq, nominal_freq); 1093 return -EINVAL; 1094 } 1095 1096 if (perf.bios_min_perf) { 1097 u32 bios_min_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.bios_min_perf); 1098 1099 pr_debug("Found Requested CPU Min Frequency of %uKHz on CPU%d\n", 1100 bios_min_freq, cpudata->cpu); 1101 } 1102 1103 return 0; 1104 } 1105 1106 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 1107 { 1108 struct amd_cpudata *cpudata; 1109 union perf_cached perf; 1110 struct device *dev; 1111 int ret; 1112 1113 /* 1114 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1115 * which is ideal for initialization process. 1116 */ 1117 amd_perf_ctl_reset(policy->cpu); 1118 dev = get_cpu_device(policy->cpu); 1119 if (!dev) 1120 return -ENODEV; 1121 1122 cpudata = kzalloc_obj(*cpudata); 1123 if (!cpudata) 1124 return -ENOMEM; 1125 1126 cpudata->cpu = policy->cpu; 1127 cpudata->cpu_type = cpu_data(policy->cpu).topo.cpu_type; 1128 1129 ret = amd_pstate_init_perf(cpudata); 1130 if (ret) 1131 goto free_cpudata1; 1132 1133 amd_pstate_init_prefcore(cpudata); 1134 1135 ret = amd_pstate_init_freq(cpudata); 1136 if (ret) 1137 goto free_cpudata1; 1138 1139 ret = amd_pstate_init_boost_support(cpudata); 1140 if (ret) 1141 goto free_cpudata1; 1142 1143 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); 1144 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); 1145 1146 perf = READ_ONCE(cpudata->perf); 1147 1148 policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq, 1149 perf.lowest_perf); 1150 policy->cpuinfo.max_freq = cpudata->max_freq; 1151 1152 policy->driver_data = cpudata; 1153 ret = amd_pstate_cppc_enable(policy); 1154 if (ret) 1155 goto free_cpudata1; 1156 1157 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 1158 1159 /* It will be updated by governor */ 1160 policy->cur = policy->cpuinfo.min_freq; 1161 1162 if (cpu_feature_enabled(X86_FEATURE_CPPC)) 1163 policy->fast_switch_possible = true; 1164 1165 ret = amd_pstate_init_floor_perf(policy); 1166 if (ret) { 1167 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); 1168 goto free_cpudata1; 1169 } 1170 1171 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 1172 FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); 1173 if (ret < 0) { 1174 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 1175 goto free_cpudata1; 1176 } 1177 1178 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 1179 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 1180 if (ret < 0) { 1181 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 1182 goto free_cpudata2; 1183 } 1184 1185 1186 if (!current_pstate_driver->adjust_perf) 1187 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1188 1189 return 0; 1190 1191 free_cpudata2: 1192 freq_qos_remove_request(&cpudata->req[0]); 1193 free_cpudata1: 1194 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 1195 kfree(cpudata); 1196 policy->driver_data = NULL; 1197 return ret; 1198 } 1199 1200 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) 1201 { 1202 struct amd_cpudata *cpudata = policy->driver_data; 1203 union perf_cached perf = READ_ONCE(cpudata->perf); 1204 1205 /* Reset CPPC_REQ MSR to the BIOS value */ 1206 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 1207 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 1208 1209 freq_qos_remove_request(&cpudata->req[1]); 1210 freq_qos_remove_request(&cpudata->req[0]); 1211 policy->fast_switch_possible = false; 1212 kfree(cpudata); 1213 } 1214 1215 static int amd_pstate_get_balanced_epp(struct cpufreq_policy *policy) 1216 { 1217 struct amd_cpudata *cpudata = policy->driver_data; 1218 1219 if (power_supply_is_system_supplied()) 1220 return cpudata->epp_default_ac; 1221 else 1222 return cpudata->epp_default_dc; 1223 } 1224 1225 static int amd_pstate_power_supply_notifier(struct notifier_block *nb, 1226 unsigned long event, void *data) 1227 { 1228 struct amd_cpudata *cpudata = container_of(nb, struct amd_cpudata, power_nb); 1229 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 1230 u8 epp; 1231 int ret; 1232 1233 if (event != PSY_EVENT_PROP_CHANGED) 1234 return NOTIFY_OK; 1235 1236 /* dynamic actions are only applied while platform profile is in balanced */ 1237 if (cpudata->current_profile != PLATFORM_PROFILE_BALANCED) 1238 return 0; 1239 1240 if (!policy) 1241 return NOTIFY_OK; 1242 1243 epp = amd_pstate_get_balanced_epp(policy); 1244 1245 ret = amd_pstate_set_epp(policy, epp); 1246 if (ret) 1247 pr_warn("Failed to set CPU %d EPP %u: %d\n", cpudata->cpu, epp, ret); 1248 1249 return NOTIFY_OK; 1250 } 1251 1252 static int amd_pstate_get_epp_from_platform_profile(struct cpufreq_policy *policy, 1253 enum platform_profile_option profile) 1254 { 1255 struct amd_cpudata *cpudata = policy->driver_data; 1256 u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type); 1257 1258 switch (profile) { 1259 case PLATFORM_PROFILE_PERFORMANCE: 1260 return values[EPP_INDEX_PERFORMANCE]; 1261 case PLATFORM_PROFILE_BALANCED: 1262 return amd_pstate_get_balanced_epp(policy); 1263 case PLATFORM_PROFILE_LOW_POWER: 1264 return values[EPP_INDEX_POWERSAVE]; 1265 default: 1266 break; 1267 } 1268 1269 pr_err("Unknown Platform Profile %d\n", profile); 1270 return -EOPNOTSUPP; 1271 } 1272 1273 static int amd_pstate_profile_probe(void *drvdata, unsigned long *choices) 1274 { 1275 set_bit(PLATFORM_PROFILE_LOW_POWER, choices); 1276 set_bit(PLATFORM_PROFILE_BALANCED, choices); 1277 set_bit(PLATFORM_PROFILE_PERFORMANCE, choices); 1278 1279 return 0; 1280 } 1281 1282 static int amd_pstate_profile_get(struct device *dev, 1283 enum platform_profile_option *profile) 1284 { 1285 struct amd_cpudata *cpudata = dev_get_drvdata(dev); 1286 1287 *profile = cpudata->current_profile; 1288 1289 return 0; 1290 } 1291 1292 static int amd_pstate_profile_set(struct device *dev, 1293 enum platform_profile_option profile) 1294 { 1295 struct amd_cpudata *cpudata = dev_get_drvdata(dev); 1296 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); 1297 int ret; 1298 u8 epp; 1299 1300 if (!policy) 1301 return -ENODEV; 1302 1303 ret = amd_pstate_get_epp_from_platform_profile(policy, profile); 1304 if (ret < 0) 1305 return ret; 1306 1307 epp = (u8)ret; 1308 ret = amd_pstate_set_epp(policy, epp); 1309 if (ret) 1310 return ret; 1311 1312 cpudata->current_profile = profile; 1313 1314 return 0; 1315 } 1316 1317 static const struct platform_profile_ops amd_pstate_profile_ops = { 1318 .probe = amd_pstate_profile_probe, 1319 .profile_set = amd_pstate_profile_set, 1320 .profile_get = amd_pstate_profile_get, 1321 }; 1322 1323 void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy) 1324 { 1325 struct amd_cpudata *cpudata = policy->driver_data; 1326 1327 if (cpudata->power_nb.notifier_call) 1328 power_supply_unreg_notifier(&cpudata->power_nb); 1329 if (cpudata->ppdev) { 1330 platform_profile_remove(cpudata->ppdev); 1331 cpudata->ppdev = NULL; 1332 } 1333 kfree(cpudata->profile_name); 1334 cpudata->dynamic_epp = false; 1335 } 1336 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_clear_dynamic_epp); 1337 1338 static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy) 1339 { 1340 struct amd_cpudata *cpudata = policy->driver_data; 1341 u64 prev = READ_ONCE(cpudata->cppc_req_cached); 1342 int ret; 1343 u8 epp; 1344 1345 ret = amd_pstate_get_epp_from_platform_profile(policy, cpudata->current_profile); 1346 if (ret < 0) 1347 return ret; 1348 1349 epp = (u8)ret; 1350 ret = amd_pstate_set_epp(policy, epp); 1351 if (ret) 1352 return ret; 1353 1354 cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu); 1355 if (!cpudata->profile_name) { 1356 ret = -ENOMEM; 1357 goto cleanup; 1358 } 1359 1360 cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu), 1361 cpudata->profile_name, 1362 policy->driver_data, 1363 &amd_pstate_profile_ops); 1364 if (IS_ERR(cpudata->ppdev)) { 1365 ret = PTR_ERR(cpudata->ppdev); 1366 goto cleanup; 1367 } 1368 1369 /* only enable notifier if things will actually change */ 1370 if (cpudata->epp_default_ac != cpudata->epp_default_dc) { 1371 cpudata->power_nb.notifier_call = amd_pstate_power_supply_notifier; 1372 ret = power_supply_reg_notifier(&cpudata->power_nb); 1373 if (ret) 1374 goto cleanup; 1375 } 1376 1377 cpudata->dynamic_epp = true; 1378 1379 return 0; 1380 1381 cleanup: 1382 amd_pstate_clear_dynamic_epp(policy); 1383 1384 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, prev); 1385 /* Restore previous EPP if toggling Dynamic EPP failed. */ 1386 amd_pstate_set_epp(policy, epp); 1387 return ret; 1388 } 1389 1390 /* Sysfs attributes */ 1391 1392 /* 1393 * This frequency is to indicate the maximum hardware frequency. 1394 * If boost is not active but supported, the frequency will be larger than the 1395 * one in cpuinfo. 1396 */ 1397 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 1398 char *buf) 1399 { 1400 struct amd_cpudata *cpudata = policy->driver_data; 1401 1402 return sysfs_emit(buf, "%u\n", cpudata->max_freq); 1403 } 1404 1405 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 1406 char *buf) 1407 { 1408 struct amd_cpudata *cpudata; 1409 union perf_cached perf; 1410 1411 cpudata = policy->driver_data; 1412 perf = READ_ONCE(cpudata->perf); 1413 1414 return sysfs_emit(buf, "%u\n", 1415 perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf)); 1416 } 1417 1418 /* 1419 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 1420 * need to expose it to sysfs. 1421 */ 1422 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 1423 char *buf) 1424 { 1425 struct amd_cpudata *cpudata; 1426 1427 cpudata = policy->driver_data; 1428 1429 return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf); 1430 } 1431 1432 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, 1433 char *buf) 1434 { 1435 u8 perf; 1436 struct amd_cpudata *cpudata = policy->driver_data; 1437 1438 perf = READ_ONCE(cpudata->prefcore_ranking); 1439 1440 return sysfs_emit(buf, "%u\n", perf); 1441 } 1442 1443 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, 1444 char *buf) 1445 { 1446 bool hw_prefcore; 1447 struct amd_cpudata *cpudata = policy->driver_data; 1448 1449 hw_prefcore = READ_ONCE(cpudata->hw_prefcore); 1450 1451 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); 1452 } 1453 1454 static ssize_t show_energy_performance_available_preferences( 1455 struct cpufreq_policy *policy, char *buf) 1456 { 1457 int i, offset = 0; 1458 struct amd_cpudata *cpudata = policy->driver_data; 1459 1460 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1461 return sysfs_emit_at(buf, offset, "%s\n", 1462 energy_perf_strings[EPP_INDEX_PERFORMANCE]); 1463 1464 for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++) 1465 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]); 1466 1467 offset += sysfs_emit_at(buf, offset, "\n"); 1468 1469 return offset; 1470 } 1471 1472 ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, 1473 const char *buf, size_t count) 1474 { 1475 struct amd_cpudata *cpudata = policy->driver_data; 1476 u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type); 1477 ssize_t ret; 1478 bool raw_epp = false; 1479 u8 epp; 1480 1481 /* 1482 * if the value matches a number, use that, otherwise see if 1483 * matches an index in the energy_perf_strings array 1484 */ 1485 ret = kstrtou8(buf, 0, &epp); 1486 raw_epp = !ret; 1487 if (ret) { 1488 ret = sysfs_match_string(energy_perf_strings, buf); 1489 if (ret < 0 || ret == EPP_INDEX_CUSTOM) 1490 return -EINVAL; 1491 1492 if (ret == EPP_INDEX_DYNAMIC) { 1493 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1494 return -EBUSY; 1495 /* 1496 * Dynamic EPP was already enabled for this CPU. 1497 * Nothing to do. 1498 */ 1499 if (cpudata->dynamic_epp) 1500 return count; 1501 1502 cpudata->current_profile = PLATFORM_PROFILE_BALANCED; 1503 ret = amd_pstate_set_dynamic_epp(policy); 1504 if (ret) 1505 return ret; 1506 1507 return count; 1508 } 1509 1510 if (ret) 1511 epp = values[ret]; 1512 else 1513 epp = cpudata->epp_default_dc; 1514 } 1515 1516 if (epp > 0 && epp != values[EPP_INDEX_PERFORMANCE] && 1517 cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 1518 pr_debug("EPP cannot be set under performance policy\n"); 1519 return -EBUSY; 1520 } 1521 1522 /* 1523 * Dynamic EPP was enabled previously! 1524 * Switch back to the static EPP mode. 1525 */ 1526 if (cpudata->dynamic_epp) 1527 amd_pstate_clear_dynamic_epp(policy); 1528 1529 ret = amd_pstate_set_epp(policy, epp); 1530 if (ret) 1531 return ret; 1532 1533 cpudata->raw_epp = raw_epp; 1534 1535 return count; 1536 } 1537 EXPORT_SYMBOL_FOR_PSTATE_UT(store_energy_performance_preference); 1538 1539 ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf) 1540 { 1541 struct amd_cpudata *cpudata = policy->driver_data; 1542 u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type); 1543 u8 epp; 1544 int i; 1545 1546 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 1547 1548 if (!cpudata->dynamic_epp && cpudata->raw_epp) 1549 return sysfs_emit(buf, "%u\n", epp); 1550 1551 /* 1552 * Map the cached EPP value back to a named preference. Skip the 1553 * "default" slot (index 0) so an EPP of 0 reports as "performance". 1554 * Stop at POWERSAVE; CUSTOM and DYNAMIC are not initialized in epp_values. 1555 */ 1556 for (i = EPP_INDEX_PERFORMANCE; i <= EPP_INDEX_POWERSAVE; i++) { 1557 const char *name = energy_perf_strings[i]; 1558 1559 if (epp == values[i]) { 1560 if (cpudata->dynamic_epp) 1561 return sysfs_emit(buf, "dynamic(profile:%s)\n", name); 1562 return sysfs_emit(buf, "%s\n", name); 1563 } 1564 } 1565 1566 return sysfs_emit(buf, "%u\n", epp); 1567 } 1568 EXPORT_SYMBOL_FOR_PSTATE_UT(show_energy_performance_preference); 1569 1570 ssize_t store_amd_pstate_floor_freq(struct cpufreq_policy *policy, const char *buf, size_t count) 1571 { 1572 struct amd_cpudata *cpudata = policy->driver_data; 1573 union perf_cached perf = READ_ONCE(cpudata->perf); 1574 unsigned int freq; 1575 u8 floor_perf; 1576 int ret; 1577 1578 ret = kstrtouint(buf, 0, &freq); 1579 if (ret) 1580 return ret; 1581 1582 if (freq < policy->cpuinfo.min_freq || freq > policy->max) 1583 return -EINVAL; 1584 1585 floor_perf = freq_to_perf(perf, cpudata->nominal_freq, freq); 1586 ret = amd_pstate_set_floor_perf(policy, floor_perf); 1587 1588 if (!ret) 1589 cpudata->floor_freq = freq; 1590 1591 return ret ?: count; 1592 } 1593 EXPORT_SYMBOL_FOR_PSTATE_UT(store_amd_pstate_floor_freq); 1594 1595 ssize_t show_amd_pstate_floor_freq(struct cpufreq_policy *policy, char *buf) 1596 { 1597 struct amd_cpudata *cpudata = policy->driver_data; 1598 1599 return sysfs_emit(buf, "%u\n", cpudata->floor_freq); 1600 } 1601 EXPORT_SYMBOL_FOR_PSTATE_UT(show_amd_pstate_floor_freq); 1602 1603 static ssize_t show_amd_pstate_floor_count(struct cpufreq_policy *policy, char *buf) 1604 { 1605 struct amd_cpudata *cpudata = policy->driver_data; 1606 u8 count = cpudata->floor_perf_cnt; 1607 1608 return sysfs_emit(buf, "%u\n", count); 1609 } 1610 1611 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1612 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1613 1614 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1615 cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); 1616 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); 1617 cpufreq_freq_attr_rw(energy_performance_preference); 1618 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1619 cpufreq_freq_attr_rw(amd_pstate_floor_freq); 1620 cpufreq_freq_attr_ro(amd_pstate_floor_count); 1621 1622 struct freq_attr_visibility { 1623 struct freq_attr *attr; 1624 bool (*visibility_fn)(void); 1625 }; 1626 1627 /* For attributes which are always visible */ 1628 static bool always_visible(void) 1629 { 1630 return true; 1631 } 1632 1633 /* Determines whether prefcore related attributes should be visible */ 1634 static bool prefcore_visibility(void) 1635 { 1636 return amd_pstate_prefcore; 1637 } 1638 1639 /* Determines whether energy performance preference should be visible */ 1640 static bool epp_visibility(void) 1641 { 1642 return cppc_state == AMD_PSTATE_ACTIVE; 1643 } 1644 1645 /* Determines whether amd_pstate_floor_freq related attributes should be visible */ 1646 static bool floor_freq_visibility(void) 1647 { 1648 return cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO); 1649 } 1650 1651 static struct freq_attr_visibility amd_pstate_attr_visibility[] = { 1652 {&amd_pstate_max_freq, always_visible}, 1653 {&amd_pstate_lowest_nonlinear_freq, always_visible}, 1654 {&amd_pstate_highest_perf, always_visible}, 1655 {&amd_pstate_prefcore_ranking, prefcore_visibility}, 1656 {&amd_pstate_hw_prefcore, prefcore_visibility}, 1657 {&energy_performance_preference, epp_visibility}, 1658 {&energy_performance_available_preferences, epp_visibility}, 1659 {&amd_pstate_floor_freq, floor_freq_visibility}, 1660 {&amd_pstate_floor_count, floor_freq_visibility}, 1661 }; 1662 1663 struct freq_attr **amd_pstate_get_current_attrs(void) 1664 { 1665 if (!current_pstate_driver) 1666 return NULL; 1667 return current_pstate_driver->attr; 1668 } 1669 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_current_attrs); 1670 1671 static struct freq_attr **get_freq_attrs(void) 1672 { 1673 bool attr_visible[ARRAY_SIZE(amd_pstate_attr_visibility)]; 1674 struct freq_attr **attrs; 1675 int i, j, count; 1676 1677 for (i = 0, count = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { 1678 struct freq_attr_visibility *v = &amd_pstate_attr_visibility[i]; 1679 1680 attr_visible[i] = v->visibility_fn(); 1681 if (attr_visible[i]) 1682 count++; 1683 } 1684 1685 /* amd_pstate_{max_freq, lowest_nonlinear_freq, highest_perf} should always be visible */ 1686 BUG_ON(!count); 1687 1688 attrs = kzalloc_objs(struct freq_attr *, count + 1); 1689 if (!attrs) 1690 return ERR_PTR(-ENOMEM); 1691 1692 for (i = 0, j = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { 1693 if (!attr_visible[i]) 1694 continue; 1695 1696 attrs[j++] = amd_pstate_attr_visibility[i].attr; 1697 } 1698 1699 return attrs; 1700 } 1701 1702 static void amd_pstate_driver_cleanup(void) 1703 { 1704 if (amd_pstate_prefcore) 1705 sched_clear_itmt_support(); 1706 1707 cppc_state = AMD_PSTATE_DISABLE; 1708 kfree(current_pstate_driver->attr); 1709 current_pstate_driver->attr = NULL; 1710 current_pstate_driver = NULL; 1711 } 1712 1713 static int amd_pstate_set_driver(int mode_idx) 1714 { 1715 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1716 cppc_state = mode_idx; 1717 if (cppc_state == AMD_PSTATE_DISABLE) 1718 pr_info("driver is explicitly disabled\n"); 1719 1720 if (cppc_state == AMD_PSTATE_ACTIVE) 1721 current_pstate_driver = &amd_pstate_epp_driver; 1722 1723 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1724 current_pstate_driver = &amd_pstate_driver; 1725 1726 return 0; 1727 } 1728 1729 return -EINVAL; 1730 } 1731 1732 static int amd_pstate_register_driver(int mode) 1733 { 1734 struct freq_attr **attr = NULL; 1735 int ret; 1736 1737 ret = amd_pstate_set_driver(mode); 1738 if (ret) 1739 return ret; 1740 1741 cppc_state = mode; 1742 1743 /* 1744 * Note: It is important to compute the attrs _after_ 1745 * re-initializing the cppc_state. Some attributes become 1746 * visible only when cppc_state is AMD_PSTATE_ACTIVE. 1747 */ 1748 attr = get_freq_attrs(); 1749 if (IS_ERR(attr)) { 1750 ret = (int) PTR_ERR(attr); 1751 pr_err("Couldn't compute freq_attrs for current mode %s [%d]\n", 1752 amd_pstate_get_mode_string(cppc_state), ret); 1753 amd_pstate_driver_cleanup(); 1754 return ret; 1755 } 1756 1757 current_pstate_driver->attr = attr; 1758 1759 /* at least one CPU supports CPB */ 1760 current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); 1761 1762 ret = cpufreq_register_driver(current_pstate_driver); 1763 if (ret) { 1764 amd_pstate_driver_cleanup(); 1765 return ret; 1766 } 1767 1768 /* Enable ITMT support once all CPUs have initialized their asym priorities. */ 1769 if (amd_pstate_prefcore) 1770 sched_set_itmt_support(); 1771 1772 return 0; 1773 } 1774 1775 static int amd_pstate_unregister_driver(int dummy) 1776 { 1777 cpufreq_unregister_driver(current_pstate_driver); 1778 amd_pstate_driver_cleanup(); 1779 return 0; 1780 } 1781 1782 static int amd_pstate_change_mode_without_dvr_change(int mode) 1783 { 1784 int cpu = 0; 1785 int ret; 1786 1787 cppc_state = mode; 1788 1789 if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 1790 return 0; 1791 1792 for_each_online_cpu(cpu) { 1793 ret = cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 1794 if (ret) 1795 return ret; 1796 } 1797 1798 return 0; 1799 } 1800 1801 static int amd_pstate_change_driver_mode(int mode) 1802 { 1803 int old_mode = cppc_state; 1804 int ret; 1805 1806 lockdep_assert_held(&amd_pstate_driver_lock); 1807 1808 ret = amd_pstate_unregister_driver(0); 1809 if (ret) 1810 return ret; 1811 1812 ret = amd_pstate_register_driver(mode); 1813 if (ret) { 1814 pr_err("Failed to register %s mode, restoring %s mode\n", 1815 amd_pstate_get_mode_string(mode), 1816 amd_pstate_get_mode_string(old_mode)); 1817 if (amd_pstate_register_driver(old_mode)) 1818 pr_err("Failed to restore %s mode\n", 1819 amd_pstate_get_mode_string(old_mode)); 1820 } 1821 1822 return ret; 1823 } 1824 1825 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 1826 [AMD_PSTATE_DISABLE] = { 1827 [AMD_PSTATE_DISABLE] = NULL, 1828 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 1829 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 1830 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 1831 }, 1832 [AMD_PSTATE_PASSIVE] = { 1833 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1834 [AMD_PSTATE_PASSIVE] = NULL, 1835 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1836 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 1837 }, 1838 [AMD_PSTATE_ACTIVE] = { 1839 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1840 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 1841 [AMD_PSTATE_ACTIVE] = NULL, 1842 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 1843 }, 1844 [AMD_PSTATE_GUIDED] = { 1845 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1846 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 1847 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1848 [AMD_PSTATE_GUIDED] = NULL, 1849 }, 1850 }; 1851 1852 static ssize_t amd_pstate_show_status(char *buf) 1853 { 1854 if (!current_pstate_driver) 1855 return sysfs_emit(buf, "disable\n"); 1856 1857 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 1858 } 1859 1860 int amd_pstate_get_status(void) 1861 { 1862 return cppc_state; 1863 } 1864 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_status); 1865 1866 int amd_pstate_update_status(const char *buf, size_t size) 1867 { 1868 cppc_mode_transition_fn fn; 1869 int cpu, mode_idx; 1870 1871 if (size > strlen("passive") || size < strlen("active")) 1872 return -EINVAL; 1873 1874 mode_idx = get_mode_idx_from_str(buf, size); 1875 if (mode_idx < 0) 1876 return mode_idx; 1877 1878 /* Resolve the transition under the lock; cppc_state is only stable while held. */ 1879 guard(mutex)(&amd_pstate_driver_lock); 1880 1881 if (!mode_state_machine[cppc_state][mode_idx]) 1882 return 0; 1883 1884 if (mode_idx == AMD_PSTATE_PASSIVE && 1885 !cpu_feature_enabled(X86_FEATURE_CPPC)) { 1886 guard(cpus_read_lock)(); 1887 1888 /* Check offline CPUs too, before changing or removing the driver. */ 1889 for_each_present_cpu(cpu) { 1890 if (cppc_auto_sel_is_immutable(cpu)) 1891 return -EOPNOTSUPP; 1892 } 1893 } 1894 1895 fn = mode_state_machine[cppc_state][mode_idx]; 1896 return fn(mode_idx); 1897 } 1898 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_update_status); 1899 1900 static ssize_t status_show(struct device *dev, 1901 struct device_attribute *attr, char *buf) 1902 { 1903 1904 guard(mutex)(&amd_pstate_driver_lock); 1905 1906 return amd_pstate_show_status(buf); 1907 } 1908 1909 static ssize_t status_store(struct device *a, struct device_attribute *b, 1910 const char *buf, size_t count) 1911 { 1912 char *p = memchr(buf, '\n', count); 1913 int ret; 1914 1915 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1916 1917 return ret < 0 ? ret : count; 1918 } 1919 1920 static ssize_t prefcore_show(struct device *dev, 1921 struct device_attribute *attr, char *buf) 1922 { 1923 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); 1924 } 1925 1926 static DEVICE_ATTR_RW(status); 1927 static DEVICE_ATTR_RO(prefcore); 1928 1929 static struct attribute *pstate_global_attributes[] = { 1930 &dev_attr_status.attr, 1931 &dev_attr_prefcore.attr, 1932 NULL 1933 }; 1934 1935 static const struct attribute_group amd_pstate_global_attr_group = { 1936 .name = "amd_pstate", 1937 .attrs = pstate_global_attributes, 1938 }; 1939 1940 static bool amd_pstate_acpi_pm_profile_server(void) 1941 { 1942 switch (acpi_gbl_FADT.preferred_profile) { 1943 case PM_ENTERPRISE_SERVER: 1944 case PM_SOHO_SERVER: 1945 case PM_PERFORMANCE_SERVER: 1946 return true; 1947 } 1948 return false; 1949 } 1950 1951 static bool amd_pstate_acpi_pm_profile_undefined(void) 1952 { 1953 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) 1954 return true; 1955 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) 1956 return true; 1957 return false; 1958 } 1959 1960 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1961 { 1962 struct amd_cpudata *cpudata; 1963 union perf_cached perf; 1964 struct device *dev; 1965 int default_epp; 1966 int ret; 1967 1968 /* 1969 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1970 * which is ideal for initialization process. 1971 */ 1972 amd_perf_ctl_reset(policy->cpu); 1973 dev = get_cpu_device(policy->cpu); 1974 if (!dev) 1975 return -ENODEV; 1976 1977 cpudata = kzalloc_obj(*cpudata); 1978 if (!cpudata) 1979 return -ENOMEM; 1980 1981 cpudata->cpu = policy->cpu; 1982 cpudata->cpu_type = cpu_data(policy->cpu).topo.cpu_type; 1983 1984 ret = amd_pstate_init_perf(cpudata); 1985 if (ret) 1986 goto free_cpudata1; 1987 1988 amd_pstate_init_prefcore(cpudata); 1989 1990 ret = amd_pstate_init_freq(cpudata); 1991 if (ret) 1992 goto free_cpudata1; 1993 1994 ret = amd_pstate_init_boost_support(cpudata); 1995 if (ret) 1996 goto free_cpudata1; 1997 1998 perf = READ_ONCE(cpudata->perf); 1999 2000 policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq, 2001 perf.lowest_perf); 2002 policy->cpuinfo.max_freq = cpudata->max_freq; 2003 policy->driver_data = cpudata; 2004 2005 ret = amd_pstate_cppc_enable(policy); 2006 if (ret) 2007 goto free_cpudata1; 2008 2009 /* It will be updated by governor */ 2010 policy->cur = policy->cpuinfo.min_freq; 2011 2012 2013 policy->boost_supported = READ_ONCE(cpudata->boost_supported); 2014 2015 /* Cache the firmware programmed EPP */ 2016 default_epp = amd_pstate_get_epp(cpudata); 2017 if (default_epp < 0) { 2018 ret = default_epp; 2019 goto free_cpudata1; 2020 } 2021 FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &cpudata->cppc_req_cached, default_epp); 2022 2023 /* 2024 * Set the policy to provide a valid fallback value in case 2025 * the default cpufreq governor is neither powersave nor performance. 2026 */ 2027 if (amd_pstate_acpi_pm_profile_server() || 2028 amd_pstate_acpi_pm_profile_undefined()) { 2029 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 2030 cpudata->epp_default_ac = cpudata->epp_default_dc = default_epp; 2031 cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE; 2032 } else { 2033 u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type); 2034 2035 policy->policy = CPUFREQ_POLICY_POWERSAVE; 2036 cpudata->epp_default_ac = values[EPP_INDEX_PERFORMANCE]; 2037 cpudata->epp_default_dc = values[EPP_INDEX_BALANCE_PERFORMANCE]; 2038 cpudata->current_profile = PLATFORM_PROFILE_BALANCED; 2039 } 2040 2041 ret = amd_pstate_set_epp(policy, cpudata->epp_default_dc); 2042 if (ret) 2043 goto free_cpudata1; 2044 2045 ret = amd_pstate_init_floor_perf(policy); 2046 if (ret) { 2047 dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); 2048 goto free_cpudata1; 2049 } 2050 2051 current_pstate_driver->adjust_perf = NULL; 2052 2053 return 0; 2054 2055 free_cpudata1: 2056 pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); 2057 kfree(cpudata); 2058 policy->driver_data = NULL; 2059 return ret; 2060 } 2061 2062 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 2063 { 2064 struct amd_cpudata *cpudata = policy->driver_data; 2065 2066 if (cpudata) { 2067 union perf_cached perf = READ_ONCE(cpudata->perf); 2068 2069 if (cpudata->dynamic_epp) 2070 amd_pstate_clear_dynamic_epp(policy); 2071 2072 /* Reset CPPC_REQ MSR to the BIOS value */ 2073 amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); 2074 amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2075 2076 kfree(cpudata); 2077 policy->driver_data = NULL; 2078 } 2079 2080 pr_debug("CPU %d exiting\n", policy->cpu); 2081 } 2082 2083 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy, bool policy_change) 2084 { 2085 struct amd_cpudata *cpudata = policy->driver_data; 2086 union perf_cached perf; 2087 u8 epp; 2088 2089 if (policy_change || 2090 policy->min != cpudata->min_limit_freq || 2091 policy->max != cpudata->max_limit_freq) 2092 amd_pstate_update_min_max_limit(policy); 2093 2094 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 2095 epp = 0; 2096 else 2097 epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); 2098 2099 perf = READ_ONCE(cpudata->perf); 2100 2101 return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U, 2102 perf.max_limit_perf, epp, false); 2103 } 2104 2105 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 2106 { 2107 struct amd_cpudata *cpudata = policy->driver_data; 2108 int ret; 2109 2110 if (!policy->cpuinfo.max_freq) 2111 return -ENODEV; 2112 2113 /* Must be a switch between PERFORMANCE and POWERSAVE */ 2114 if (cpudata->policy != policy->policy) { 2115 /* 2116 * Disable dynamic_epp when switching 2117 * out of CPUFREQ_POLICY_POWERSAVE. 2118 */ 2119 if (cpudata->dynamic_epp) { 2120 WARN_ON_ONCE(cpudata->policy != CPUFREQ_POLICY_POWERSAVE); 2121 amd_pstate_clear_dynamic_epp(policy); 2122 } 2123 } 2124 2125 cpudata->policy = policy->policy; 2126 2127 ret = amd_pstate_epp_update_limit(policy, true); 2128 if (ret) 2129 return ret; 2130 2131 /* 2132 * policy->cur is never updated with the amd_pstate_epp driver, but it 2133 * is used as a stale frequency value. So, keep it within limits. 2134 */ 2135 policy->cur = policy->min; 2136 2137 return 0; 2138 } 2139 2140 static int amd_pstate_cpu_online(struct cpufreq_policy *policy) 2141 { 2142 struct amd_cpudata *cpudata = policy->driver_data; 2143 union perf_cached perf = READ_ONCE(cpudata->perf); 2144 u8 cached_floor_perf; 2145 int ret; 2146 2147 ret = amd_pstate_cppc_enable(policy); 2148 if (ret) 2149 return ret; 2150 2151 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2152 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2153 } 2154 2155 static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) 2156 { 2157 struct amd_cpudata *cpudata = policy->driver_data; 2158 union perf_cached perf = READ_ONCE(cpudata->perf); 2159 int ret; 2160 2161 /* 2162 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 2163 * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the 2164 * limits, epp and desired perf will get reset to the cached values in cpudata struct 2165 */ 2166 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 2167 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 2168 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 2169 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 2170 false); 2171 if (ret) 2172 return ret; 2173 2174 return amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2175 } 2176 2177 static int amd_pstate_suspend(struct cpufreq_policy *policy) 2178 { 2179 struct amd_cpudata *cpudata = policy->driver_data; 2180 union perf_cached perf = READ_ONCE(cpudata->perf); 2181 int ret; 2182 2183 /* 2184 * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified 2185 * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, 2186 * the limits, epp and desired perf will get reset to the cached values in cpudata struct 2187 */ 2188 ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 2189 FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), 2190 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), 2191 FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), 2192 false); 2193 if (ret) 2194 return ret; 2195 2196 ret = amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); 2197 if (ret) 2198 return ret; 2199 2200 /* set this flag to avoid setting core offline*/ 2201 cpudata->suspended = true; 2202 2203 return 0; 2204 } 2205 2206 static int amd_pstate_resume(struct cpufreq_policy *policy) 2207 { 2208 struct amd_cpudata *cpudata = policy->driver_data; 2209 union perf_cached perf = READ_ONCE(cpudata->perf); 2210 int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur); 2211 u8 cached_floor_perf; 2212 int ret; 2213 2214 /* Set CPPC_REQ to last sane value until the governor updates it */ 2215 ret = amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, 2216 0U, false); 2217 if (ret) 2218 return ret; 2219 2220 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2221 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2222 } 2223 2224 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 2225 { 2226 struct amd_cpudata *cpudata = policy->driver_data; 2227 union perf_cached perf = READ_ONCE(cpudata->perf); 2228 u8 cached_floor_perf; 2229 2230 if (cpudata->suspended) { 2231 int ret; 2232 2233 /* enable amd pstate from suspend state*/ 2234 ret = amd_pstate_epp_update_limit(policy, false); 2235 if (ret) 2236 return ret; 2237 2238 cpudata->suspended = false; 2239 } 2240 2241 cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); 2242 return amd_pstate_set_floor_perf(policy, cached_floor_perf); 2243 } 2244 2245 static struct cpufreq_driver amd_pstate_driver = { 2246 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 2247 .verify = amd_pstate_verify, 2248 .target = amd_pstate_target, 2249 .fast_switch = amd_pstate_fast_switch, 2250 .init = amd_pstate_cpu_init, 2251 .exit = amd_pstate_cpu_exit, 2252 .online = amd_pstate_cpu_online, 2253 .offline = amd_pstate_cpu_offline, 2254 .suspend = amd_pstate_suspend, 2255 .resume = amd_pstate_resume, 2256 .set_boost = amd_pstate_set_boost, 2257 .update_limits = amd_pstate_update_limits, 2258 .name = "amd-pstate", 2259 }; 2260 2261 static struct cpufreq_driver amd_pstate_epp_driver = { 2262 .flags = CPUFREQ_CONST_LOOPS, 2263 .verify = amd_pstate_verify, 2264 .setpolicy = amd_pstate_epp_set_policy, 2265 .init = amd_pstate_epp_cpu_init, 2266 .exit = amd_pstate_epp_cpu_exit, 2267 .offline = amd_pstate_cpu_offline, 2268 .online = amd_pstate_cpu_online, 2269 .suspend = amd_pstate_suspend, 2270 .resume = amd_pstate_epp_resume, 2271 .update_limits = amd_pstate_update_limits, 2272 .set_boost = amd_pstate_set_boost, 2273 .name = "amd-pstate-epp", 2274 }; 2275 2276 /* 2277 * Processors without frequency scaling support can't do CPPC. 2278 * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. 2279 * show the debug message that helps to check if the CPU has CPPC support for loading issue. 2280 */ 2281 static bool amd_cppc_supported(void) 2282 { 2283 struct cpuinfo_x86 *c = &cpu_data(0); 2284 bool warn = false; 2285 2286 if (!cpu_feature_enabled(X86_FEATURE_HW_PSTATE)) { 2287 pr_debug_once("frequency scaling is not supported by the processor\n"); 2288 return false; 2289 } 2290 2291 if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) { 2292 pr_debug_once("CPPC feature is not supported by the processor\n"); 2293 return false; 2294 } 2295 2296 /* 2297 * If the CPPC feature is disabled in the BIOS for processors 2298 * that support MSR-based CPPC, the AMD Pstate driver may not 2299 * function correctly. 2300 * 2301 * For such processors, check the CPPC flag and display a 2302 * warning message if the platform supports CPPC. 2303 * 2304 * Note: The code check below will not abort the driver 2305 * registration process because of the code is added for 2306 * debugging purposes. Besides, it may still be possible for 2307 * the driver to work using the shared-memory mechanism. 2308 */ 2309 if (!cpu_feature_enabled(X86_FEATURE_CPPC)) { 2310 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { 2311 switch (c->x86_model) { 2312 case 0x60 ... 0x6F: 2313 case 0x80 ... 0xAF: 2314 warn = true; 2315 break; 2316 } 2317 } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || 2318 cpu_feature_enabled(X86_FEATURE_ZEN4)) { 2319 switch (c->x86_model) { 2320 case 0x10 ... 0x1F: 2321 case 0x40 ... 0xAF: 2322 warn = true; 2323 break; 2324 } 2325 } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) { 2326 warn = true; 2327 } 2328 } 2329 2330 if (warn) 2331 pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n" 2332 "Please enable it if your BIOS has the CPPC option.\n"); 2333 return true; 2334 } 2335 2336 /* 2337 * Resolve the numeric EPP values for hybrid systems. Only hybrid SoCs are listed 2338 * in amd_pstate_epp_soc_ids[]; non-hybrid systems always use the legacy defaults. 2339 */ 2340 static inline void __init amd_pstate_set_epp_values(enum x86_topology_cpu_type type, 2341 const struct amd_pstate_epp_values *core) 2342 { 2343 epp_values[type][EPP_INDEX_PERFORMANCE] = core->performance; 2344 epp_values[type][EPP_INDEX_BALANCE_PERFORMANCE] = core->balance_performance; 2345 epp_values[type][EPP_INDEX_BALANCE_POWERSAVE] = core->balance_power; 2346 epp_values[type][EPP_INDEX_POWERSAVE] = core->power; 2347 } 2348 2349 static void __init amd_pstate_init_epp_values(void) 2350 { 2351 const struct x86_cpu_id *id = x86_match_cpu(amd_pstate_epp_soc_ids); 2352 const struct amd_pstate_epp_soc *soc; 2353 2354 if (!id || !id->driver_data) { 2355 if (cpu_feature_enabled(X86_FEATURE_ZEN6) && 2356 cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) 2357 pr_warn_once("No EPP tunings found for platform\n"); 2358 return; 2359 } 2360 2361 soc = (const struct amd_pstate_epp_soc *)id->driver_data; 2362 2363 amd_pstate_set_epp_values(TOPO_CPU_TYPE_PERFORMANCE, &soc->performance_core); 2364 amd_pstate_set_epp_values(TOPO_CPU_TYPE_EFFICIENCY, &soc->efficiency_core); 2365 amd_pstate_set_epp_values(TOPO_CPU_TYPE_LOW_POWER, &soc->low_power_core); 2366 } 2367 2368 static int __init amd_pstate_init(void) 2369 { 2370 struct device *dev_root; 2371 int ret; 2372 2373 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 2374 return -ENODEV; 2375 2376 /* show debug message only if CPPC is not supported */ 2377 if (!amd_cppc_supported()) 2378 return -EOPNOTSUPP; 2379 2380 /* show warning message when BIOS broken or ACPI disabled */ 2381 if (!acpi_cpc_valid()) { 2382 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 2383 return -ENODEV; 2384 } 2385 2386 /* don't keep reloading if cpufreq_driver exists */ 2387 if (cpufreq_get_current_driver()) 2388 return -EEXIST; 2389 2390 quirks = NULL; 2391 2392 /* check if this machine need CPPC quirks */ 2393 dmi_check_system(amd_pstate_quirks_table); 2394 2395 /* resolve per-SoC EPP values for the named preferences */ 2396 amd_pstate_init_epp_values(); 2397 2398 /* 2399 * determine the driver mode from the command line or kernel config. 2400 * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED. 2401 * command line options will override the kernel config settings. 2402 */ 2403 2404 if (cppc_state == AMD_PSTATE_UNDEFINED) { 2405 /* Disable on the following configs by default: 2406 * 1. Undefined platforms 2407 * 2. Server platforms with CPUs older than Family 0x1A. 2408 */ 2409 if (amd_pstate_acpi_pm_profile_undefined() || 2410 (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) { 2411 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 2412 return -ENODEV; 2413 } 2414 /* get driver mode from kernel config option [1:4] */ 2415 cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; 2416 } 2417 2418 if (cppc_state == AMD_PSTATE_DISABLE) { 2419 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 2420 return -ENODEV; 2421 } 2422 2423 /* capability check */ 2424 if (cpu_feature_enabled(X86_FEATURE_CPPC)) { 2425 pr_debug("AMD CPPC MSR based functionality is supported\n"); 2426 } else { 2427 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 2428 static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); 2429 static_call_update(amd_pstate_init_perf, shmem_init_perf); 2430 static_call_update(amd_pstate_update_perf, shmem_update_perf); 2431 static_call_update(amd_pstate_get_epp, shmem_get_epp); 2432 static_call_update(amd_pstate_set_epp, shmem_set_epp); 2433 } 2434 2435 if (amd_pstate_prefcore) { 2436 ret = amd_detect_prefcore(&amd_pstate_prefcore); 2437 if (ret) 2438 return ret; 2439 } 2440 2441 ret = amd_pstate_register_driver(cppc_state); 2442 if (ret) { 2443 pr_err("failed to register with return %d\n", ret); 2444 return ret; 2445 } 2446 2447 dev_root = bus_get_dev_root(&cpu_subsys); 2448 if (dev_root) { 2449 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 2450 put_device(dev_root); 2451 if (ret) { 2452 pr_err("sysfs attribute export failed with error %d.\n", ret); 2453 goto global_attr_free; 2454 } 2455 } 2456 2457 return ret; 2458 2459 global_attr_free: 2460 amd_pstate_unregister_driver(0); 2461 return ret; 2462 } 2463 device_initcall(amd_pstate_init); 2464 2465 static int __init amd_pstate_param(char *str) 2466 { 2467 size_t size; 2468 int mode_idx; 2469 2470 if (!str) 2471 return -EINVAL; 2472 2473 size = strlen(str); 2474 mode_idx = get_mode_idx_from_str(str, size); 2475 2476 return amd_pstate_set_driver(mode_idx); 2477 } 2478 2479 static int __init amd_prefcore_param(char *str) 2480 { 2481 if (!strcmp(str, "disable")) 2482 amd_pstate_prefcore = false; 2483 2484 return 0; 2485 } 2486 2487 early_param("amd_pstate", amd_pstate_param); 2488 early_param("amd_prefcore", amd_prefcore_param); 2489 2490 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 2491 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 2492