1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/kernel.h> 26 #include <linux/module.h> 27 #include <linux/init.h> 28 #include <linux/smp.h> 29 #include <linux/sched.h> 30 #include <linux/cpufreq.h> 31 #include <linux/compiler.h> 32 #include <linux/dmi.h> 33 #include <linux/slab.h> 34 #include <linux/acpi.h> 35 #include <linux/io.h> 36 #include <linux/delay.h> 37 #include <linux/uaccess.h> 38 #include <linux/static_call.h> 39 #include <linux/amd-pstate.h> 40 41 #include <acpi/processor.h> 42 #include <acpi/cppc_acpi.h> 43 44 #include <asm/msr.h> 45 #include <asm/processor.h> 46 #include <asm/cpufeature.h> 47 #include <asm/cpu_device_id.h> 48 #include "amd-pstate-trace.h" 49 50 #define AMD_PSTATE_TRANSITION_LATENCY 20000 51 #define AMD_PSTATE_TRANSITION_DELAY 1000 52 53 /* 54 * TODO: We need more time to fine tune processors with shared memory solution 55 * with community together. 56 * 57 * There are some performance drops on the CPU benchmarks which reports from 58 * Suse. We are co-working with them to fine tune the shared memory solution. So 59 * we disable it by default to go acpi-cpufreq on these processors and add a 60 * module parameter to be able to enable it manually for debugging. 61 */ 62 static struct cpufreq_driver *current_pstate_driver; 63 static struct cpufreq_driver amd_pstate_driver; 64 static struct cpufreq_driver amd_pstate_epp_driver; 65 static int cppc_state = AMD_PSTATE_DISABLE; 66 struct kobject *amd_pstate_kobj; 67 68 /* 69 * AMD Energy Preference Performance (EPP) 70 * The EPP is used in the CCLK DPM controller to drive 71 * the frequency that a core is going to operate during 72 * short periods of activity. EPP values will be utilized for 73 * different OS profiles (balanced, performance, power savings) 74 * display strings corresponding to EPP index in the 75 * energy_perf_strings[] 76 * index String 77 *------------------------------------- 78 * 0 default 79 * 1 performance 80 * 2 balance_performance 81 * 3 balance_power 82 * 4 power 83 */ 84 enum energy_perf_value_index { 85 EPP_INDEX_DEFAULT = 0, 86 EPP_INDEX_PERFORMANCE, 87 EPP_INDEX_BALANCE_PERFORMANCE, 88 EPP_INDEX_BALANCE_POWERSAVE, 89 EPP_INDEX_POWERSAVE, 90 }; 91 92 static const char * const energy_perf_strings[] = { 93 [EPP_INDEX_DEFAULT] = "default", 94 [EPP_INDEX_PERFORMANCE] = "performance", 95 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 96 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 97 [EPP_INDEX_POWERSAVE] = "power", 98 NULL 99 }; 100 101 static unsigned int epp_values[] = { 102 [EPP_INDEX_DEFAULT] = 0, 103 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 104 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 105 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 106 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 107 }; 108 109 typedef int (*cppc_mode_transition_fn)(int); 110 111 static inline int get_mode_idx_from_str(const char *str, size_t size) 112 { 113 int i; 114 115 for (i=0; i < AMD_PSTATE_MAX; i++) { 116 if (!strncmp(str, amd_pstate_mode_string[i], size)) 117 return i; 118 } 119 return -EINVAL; 120 } 121 122 static DEFINE_MUTEX(amd_pstate_limits_lock); 123 static DEFINE_MUTEX(amd_pstate_driver_lock); 124 125 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) 126 { 127 u64 epp; 128 int ret; 129 130 if (boot_cpu_has(X86_FEATURE_CPPC)) { 131 if (!cppc_req_cached) { 132 epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 133 &cppc_req_cached); 134 if (epp) 135 return epp; 136 } 137 epp = (cppc_req_cached >> 24) & 0xFF; 138 } else { 139 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 140 if (ret < 0) { 141 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 142 return -EIO; 143 } 144 } 145 146 return (s16)(epp & 0xff); 147 } 148 149 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) 150 { 151 s16 epp; 152 int index = -EINVAL; 153 154 epp = amd_pstate_get_epp(cpudata, 0); 155 if (epp < 0) 156 return epp; 157 158 switch (epp) { 159 case AMD_CPPC_EPP_PERFORMANCE: 160 index = EPP_INDEX_PERFORMANCE; 161 break; 162 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 163 index = EPP_INDEX_BALANCE_PERFORMANCE; 164 break; 165 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 166 index = EPP_INDEX_BALANCE_POWERSAVE; 167 break; 168 case AMD_CPPC_EPP_POWERSAVE: 169 index = EPP_INDEX_POWERSAVE; 170 break; 171 default: 172 break; 173 } 174 175 return index; 176 } 177 178 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) 179 { 180 int ret; 181 struct cppc_perf_ctrls perf_ctrls; 182 183 if (boot_cpu_has(X86_FEATURE_CPPC)) { 184 u64 value = READ_ONCE(cpudata->cppc_req_cached); 185 186 value &= ~GENMASK_ULL(31, 24); 187 value |= (u64)epp << 24; 188 WRITE_ONCE(cpudata->cppc_req_cached, value); 189 190 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 191 if (!ret) 192 cpudata->epp_cached = epp; 193 } else { 194 perf_ctrls.energy_perf = epp; 195 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 196 if (ret) { 197 pr_debug("failed to set energy perf value (%d)\n", ret); 198 return ret; 199 } 200 cpudata->epp_cached = epp; 201 } 202 203 return ret; 204 } 205 206 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, 207 int pref_index) 208 { 209 int epp = -EINVAL; 210 int ret; 211 212 if (!pref_index) { 213 pr_debug("EPP pref_index is invalid\n"); 214 return -EINVAL; 215 } 216 217 if (epp == -EINVAL) 218 epp = epp_values[pref_index]; 219 220 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 221 pr_debug("EPP cannot be set under performance policy\n"); 222 return -EBUSY; 223 } 224 225 ret = amd_pstate_set_epp(cpudata, epp); 226 227 return ret; 228 } 229 230 static inline int pstate_enable(bool enable) 231 { 232 return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); 233 } 234 235 static int cppc_enable(bool enable) 236 { 237 int cpu, ret = 0; 238 struct cppc_perf_ctrls perf_ctrls; 239 240 for_each_present_cpu(cpu) { 241 ret = cppc_set_enable(cpu, enable); 242 if (ret) 243 return ret; 244 245 /* Enable autonomous mode for EPP */ 246 if (cppc_state == AMD_PSTATE_ACTIVE) { 247 /* Set desired perf as zero to allow EPP firmware control */ 248 perf_ctrls.desired_perf = 0; 249 ret = cppc_set_perf(cpu, &perf_ctrls); 250 if (ret) 251 return ret; 252 } 253 } 254 255 return ret; 256 } 257 258 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); 259 260 static inline int amd_pstate_enable(bool enable) 261 { 262 return static_call(amd_pstate_enable)(enable); 263 } 264 265 static int pstate_init_perf(struct amd_cpudata *cpudata) 266 { 267 u64 cap1; 268 u32 highest_perf; 269 270 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 271 &cap1); 272 if (ret) 273 return ret; 274 275 /* 276 * TODO: Introduce AMD specific power feature. 277 * 278 * CPPC entry doesn't indicate the highest performance in some ASICs. 279 */ 280 highest_perf = amd_get_highest_perf(); 281 if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) 282 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); 283 284 WRITE_ONCE(cpudata->highest_perf, highest_perf); 285 286 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); 287 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); 288 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); 289 290 return 0; 291 } 292 293 static int cppc_init_perf(struct amd_cpudata *cpudata) 294 { 295 struct cppc_perf_caps cppc_perf; 296 u32 highest_perf; 297 298 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 299 if (ret) 300 return ret; 301 302 highest_perf = amd_get_highest_perf(); 303 if (highest_perf > cppc_perf.highest_perf) 304 highest_perf = cppc_perf.highest_perf; 305 306 WRITE_ONCE(cpudata->highest_perf, highest_perf); 307 308 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); 309 WRITE_ONCE(cpudata->lowest_nonlinear_perf, 310 cppc_perf.lowest_nonlinear_perf); 311 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); 312 313 if (cppc_state == AMD_PSTATE_ACTIVE) 314 return 0; 315 316 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); 317 if (ret) { 318 pr_warn("failed to get auto_sel, ret: %d\n", ret); 319 return 0; 320 } 321 322 ret = cppc_set_auto_sel(cpudata->cpu, 323 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 324 325 if (ret) 326 pr_warn("failed to set auto_sel, ret: %d\n", ret); 327 328 return ret; 329 } 330 331 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); 332 333 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 334 { 335 return static_call(amd_pstate_init_perf)(cpudata); 336 } 337 338 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, 339 u32 des_perf, u32 max_perf, bool fast_switch) 340 { 341 if (fast_switch) 342 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); 343 else 344 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 345 READ_ONCE(cpudata->cppc_req_cached)); 346 } 347 348 static void cppc_update_perf(struct amd_cpudata *cpudata, 349 u32 min_perf, u32 des_perf, 350 u32 max_perf, bool fast_switch) 351 { 352 struct cppc_perf_ctrls perf_ctrls; 353 354 perf_ctrls.max_perf = max_perf; 355 perf_ctrls.min_perf = min_perf; 356 perf_ctrls.desired_perf = des_perf; 357 358 cppc_set_perf(cpudata->cpu, &perf_ctrls); 359 } 360 361 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); 362 363 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, 364 u32 min_perf, u32 des_perf, 365 u32 max_perf, bool fast_switch) 366 { 367 static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, 368 max_perf, fast_switch); 369 } 370 371 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 372 { 373 u64 aperf, mperf, tsc; 374 unsigned long flags; 375 376 local_irq_save(flags); 377 rdmsrl(MSR_IA32_APERF, aperf); 378 rdmsrl(MSR_IA32_MPERF, mperf); 379 tsc = rdtsc(); 380 381 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 382 local_irq_restore(flags); 383 return false; 384 } 385 386 local_irq_restore(flags); 387 388 cpudata->cur.aperf = aperf; 389 cpudata->cur.mperf = mperf; 390 cpudata->cur.tsc = tsc; 391 cpudata->cur.aperf -= cpudata->prev.aperf; 392 cpudata->cur.mperf -= cpudata->prev.mperf; 393 cpudata->cur.tsc -= cpudata->prev.tsc; 394 395 cpudata->prev.aperf = aperf; 396 cpudata->prev.mperf = mperf; 397 cpudata->prev.tsc = tsc; 398 399 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 400 401 return true; 402 } 403 404 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, 405 u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags) 406 { 407 u64 prev = READ_ONCE(cpudata->cppc_req_cached); 408 u64 value = prev; 409 410 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 411 412 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 413 min_perf = des_perf; 414 des_perf = 0; 415 } 416 417 value &= ~AMD_CPPC_MIN_PERF(~0L); 418 value |= AMD_CPPC_MIN_PERF(min_perf); 419 420 value &= ~AMD_CPPC_DES_PERF(~0L); 421 value |= AMD_CPPC_DES_PERF(des_perf); 422 423 value &= ~AMD_CPPC_MAX_PERF(~0L); 424 value |= AMD_CPPC_MAX_PERF(max_perf); 425 426 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 427 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 428 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 429 cpudata->cpu, (value != prev), fast_switch); 430 } 431 432 if (value == prev) 433 return; 434 435 WRITE_ONCE(cpudata->cppc_req_cached, value); 436 437 amd_pstate_update_perf(cpudata, min_perf, des_perf, 438 max_perf, fast_switch); 439 } 440 441 static int amd_pstate_verify(struct cpufreq_policy_data *policy) 442 { 443 cpufreq_verify_within_cpu_limits(policy); 444 445 return 0; 446 } 447 448 static int amd_pstate_target(struct cpufreq_policy *policy, 449 unsigned int target_freq, 450 unsigned int relation) 451 { 452 struct cpufreq_freqs freqs; 453 struct amd_cpudata *cpudata = policy->driver_data; 454 unsigned long max_perf, min_perf, des_perf, cap_perf; 455 456 if (!cpudata->max_freq) 457 return -ENODEV; 458 459 cap_perf = READ_ONCE(cpudata->highest_perf); 460 min_perf = READ_ONCE(cpudata->lowest_perf); 461 max_perf = cap_perf; 462 463 freqs.old = policy->cur; 464 freqs.new = target_freq; 465 466 des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, 467 cpudata->max_freq); 468 469 cpufreq_freq_transition_begin(policy, &freqs); 470 amd_pstate_update(cpudata, min_perf, des_perf, 471 max_perf, false, policy->governor->flags); 472 cpufreq_freq_transition_end(policy, &freqs, false); 473 474 return 0; 475 } 476 477 static void amd_pstate_adjust_perf(unsigned int cpu, 478 unsigned long _min_perf, 479 unsigned long target_perf, 480 unsigned long capacity) 481 { 482 unsigned long max_perf, min_perf, des_perf, 483 cap_perf, lowest_nonlinear_perf; 484 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 485 struct amd_cpudata *cpudata = policy->driver_data; 486 487 cap_perf = READ_ONCE(cpudata->highest_perf); 488 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 489 490 des_perf = cap_perf; 491 if (target_perf < capacity) 492 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 493 494 min_perf = READ_ONCE(cpudata->highest_perf); 495 if (_min_perf < capacity) 496 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 497 498 if (min_perf < lowest_nonlinear_perf) 499 min_perf = lowest_nonlinear_perf; 500 501 max_perf = cap_perf; 502 if (max_perf < min_perf) 503 max_perf = min_perf; 504 505 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, 506 policy->governor->flags); 507 cpufreq_cpu_put(policy); 508 } 509 510 static int amd_get_min_freq(struct amd_cpudata *cpudata) 511 { 512 struct cppc_perf_caps cppc_perf; 513 514 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 515 if (ret) 516 return ret; 517 518 /* Switch to khz */ 519 return cppc_perf.lowest_freq * 1000; 520 } 521 522 static int amd_get_max_freq(struct amd_cpudata *cpudata) 523 { 524 struct cppc_perf_caps cppc_perf; 525 u32 max_perf, max_freq, nominal_freq, nominal_perf; 526 u64 boost_ratio; 527 528 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 529 if (ret) 530 return ret; 531 532 nominal_freq = cppc_perf.nominal_freq; 533 nominal_perf = READ_ONCE(cpudata->nominal_perf); 534 max_perf = READ_ONCE(cpudata->highest_perf); 535 536 boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, 537 nominal_perf); 538 539 max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; 540 541 /* Switch to khz */ 542 return max_freq * 1000; 543 } 544 545 static int amd_get_nominal_freq(struct amd_cpudata *cpudata) 546 { 547 struct cppc_perf_caps cppc_perf; 548 549 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 550 if (ret) 551 return ret; 552 553 /* Switch to khz */ 554 return cppc_perf.nominal_freq * 1000; 555 } 556 557 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) 558 { 559 struct cppc_perf_caps cppc_perf; 560 u32 lowest_nonlinear_freq, lowest_nonlinear_perf, 561 nominal_freq, nominal_perf; 562 u64 lowest_nonlinear_ratio; 563 564 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 565 if (ret) 566 return ret; 567 568 nominal_freq = cppc_perf.nominal_freq; 569 nominal_perf = READ_ONCE(cpudata->nominal_perf); 570 571 lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 572 573 lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, 574 nominal_perf); 575 576 lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; 577 578 /* Switch to khz */ 579 return lowest_nonlinear_freq * 1000; 580 } 581 582 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 583 { 584 struct amd_cpudata *cpudata = policy->driver_data; 585 int ret; 586 587 if (!cpudata->boost_supported) { 588 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 589 return -EINVAL; 590 } 591 592 if (state) 593 policy->cpuinfo.max_freq = cpudata->max_freq; 594 else 595 policy->cpuinfo.max_freq = cpudata->nominal_freq; 596 597 policy->max = policy->cpuinfo.max_freq; 598 599 ret = freq_qos_update_request(&cpudata->req[1], 600 policy->cpuinfo.max_freq); 601 if (ret < 0) 602 return ret; 603 604 return 0; 605 } 606 607 static void amd_pstate_boost_init(struct amd_cpudata *cpudata) 608 { 609 u32 highest_perf, nominal_perf; 610 611 highest_perf = READ_ONCE(cpudata->highest_perf); 612 nominal_perf = READ_ONCE(cpudata->nominal_perf); 613 614 if (highest_perf <= nominal_perf) 615 return; 616 617 cpudata->boost_supported = true; 618 current_pstate_driver->boost_enabled = true; 619 } 620 621 static void amd_perf_ctl_reset(unsigned int cpu) 622 { 623 wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 624 } 625 626 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 627 { 628 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 629 struct device *dev; 630 struct amd_cpudata *cpudata; 631 632 /* 633 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 634 * which is ideal for initialization process. 635 */ 636 amd_perf_ctl_reset(policy->cpu); 637 dev = get_cpu_device(policy->cpu); 638 if (!dev) 639 return -ENODEV; 640 641 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 642 if (!cpudata) 643 return -ENOMEM; 644 645 cpudata->cpu = policy->cpu; 646 647 ret = amd_pstate_init_perf(cpudata); 648 if (ret) 649 goto free_cpudata1; 650 651 min_freq = amd_get_min_freq(cpudata); 652 max_freq = amd_get_max_freq(cpudata); 653 nominal_freq = amd_get_nominal_freq(cpudata); 654 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 655 656 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 657 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 658 min_freq, max_freq); 659 ret = -EINVAL; 660 goto free_cpudata1; 661 } 662 663 policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; 664 policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; 665 666 policy->min = min_freq; 667 policy->max = max_freq; 668 669 policy->cpuinfo.min_freq = min_freq; 670 policy->cpuinfo.max_freq = max_freq; 671 672 /* It will be updated by governor */ 673 policy->cur = policy->cpuinfo.min_freq; 674 675 if (boot_cpu_has(X86_FEATURE_CPPC)) 676 policy->fast_switch_possible = true; 677 678 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 679 FREQ_QOS_MIN, policy->cpuinfo.min_freq); 680 if (ret < 0) { 681 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 682 goto free_cpudata1; 683 } 684 685 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 686 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 687 if (ret < 0) { 688 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 689 goto free_cpudata2; 690 } 691 692 /* Initial processor data capability frequencies */ 693 cpudata->max_freq = max_freq; 694 cpudata->min_freq = min_freq; 695 cpudata->nominal_freq = nominal_freq; 696 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 697 698 policy->driver_data = cpudata; 699 700 amd_pstate_boost_init(cpudata); 701 if (!current_pstate_driver->adjust_perf) 702 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 703 704 return 0; 705 706 free_cpudata2: 707 freq_qos_remove_request(&cpudata->req[0]); 708 free_cpudata1: 709 kfree(cpudata); 710 return ret; 711 } 712 713 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) 714 { 715 struct amd_cpudata *cpudata = policy->driver_data; 716 717 freq_qos_remove_request(&cpudata->req[1]); 718 freq_qos_remove_request(&cpudata->req[0]); 719 kfree(cpudata); 720 721 return 0; 722 } 723 724 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) 725 { 726 int ret; 727 728 ret = amd_pstate_enable(true); 729 if (ret) 730 pr_err("failed to enable amd-pstate during resume, return %d\n", ret); 731 732 return ret; 733 } 734 735 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) 736 { 737 int ret; 738 739 ret = amd_pstate_enable(false); 740 if (ret) 741 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); 742 743 return ret; 744 } 745 746 /* Sysfs attributes */ 747 748 /* 749 * This frequency is to indicate the maximum hardware frequency. 750 * If boost is not active but supported, the frequency will be larger than the 751 * one in cpuinfo. 752 */ 753 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 754 char *buf) 755 { 756 int max_freq; 757 struct amd_cpudata *cpudata = policy->driver_data; 758 759 max_freq = amd_get_max_freq(cpudata); 760 if (max_freq < 0) 761 return max_freq; 762 763 return sysfs_emit(buf, "%u\n", max_freq); 764 } 765 766 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 767 char *buf) 768 { 769 int freq; 770 struct amd_cpudata *cpudata = policy->driver_data; 771 772 freq = amd_get_lowest_nonlinear_freq(cpudata); 773 if (freq < 0) 774 return freq; 775 776 return sysfs_emit(buf, "%u\n", freq); 777 } 778 779 /* 780 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 781 * need to expose it to sysfs. 782 */ 783 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 784 char *buf) 785 { 786 u32 perf; 787 struct amd_cpudata *cpudata = policy->driver_data; 788 789 perf = READ_ONCE(cpudata->highest_perf); 790 791 return sysfs_emit(buf, "%u\n", perf); 792 } 793 794 static ssize_t show_energy_performance_available_preferences( 795 struct cpufreq_policy *policy, char *buf) 796 { 797 int i = 0; 798 int offset = 0; 799 800 while (energy_perf_strings[i] != NULL) 801 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); 802 803 sysfs_emit_at(buf, offset, "\n"); 804 805 return offset; 806 } 807 808 static ssize_t store_energy_performance_preference( 809 struct cpufreq_policy *policy, const char *buf, size_t count) 810 { 811 struct amd_cpudata *cpudata = policy->driver_data; 812 char str_preference[21]; 813 ssize_t ret; 814 815 ret = sscanf(buf, "%20s", str_preference); 816 if (ret != 1) 817 return -EINVAL; 818 819 ret = match_string(energy_perf_strings, -1, str_preference); 820 if (ret < 0) 821 return -EINVAL; 822 823 mutex_lock(&amd_pstate_limits_lock); 824 ret = amd_pstate_set_energy_pref_index(cpudata, ret); 825 mutex_unlock(&amd_pstate_limits_lock); 826 827 return ret ?: count; 828 } 829 830 static ssize_t show_energy_performance_preference( 831 struct cpufreq_policy *policy, char *buf) 832 { 833 struct amd_cpudata *cpudata = policy->driver_data; 834 int preference; 835 836 preference = amd_pstate_get_energy_pref_index(cpudata); 837 if (preference < 0) 838 return preference; 839 840 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 841 } 842 843 static void amd_pstate_driver_cleanup(void) 844 { 845 amd_pstate_enable(false); 846 cppc_state = AMD_PSTATE_DISABLE; 847 current_pstate_driver = NULL; 848 } 849 850 static int amd_pstate_register_driver(int mode) 851 { 852 int ret; 853 854 if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED) 855 current_pstate_driver = &amd_pstate_driver; 856 else if (mode == AMD_PSTATE_ACTIVE) 857 current_pstate_driver = &amd_pstate_epp_driver; 858 else 859 return -EINVAL; 860 861 cppc_state = mode; 862 ret = cpufreq_register_driver(current_pstate_driver); 863 if (ret) { 864 amd_pstate_driver_cleanup(); 865 return ret; 866 } 867 return 0; 868 } 869 870 static int amd_pstate_unregister_driver(int dummy) 871 { 872 cpufreq_unregister_driver(current_pstate_driver); 873 amd_pstate_driver_cleanup(); 874 return 0; 875 } 876 877 static int amd_pstate_change_mode_without_dvr_change(int mode) 878 { 879 int cpu = 0; 880 881 cppc_state = mode; 882 883 if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 884 return 0; 885 886 for_each_present_cpu(cpu) { 887 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 888 } 889 890 return 0; 891 } 892 893 static int amd_pstate_change_driver_mode(int mode) 894 { 895 int ret; 896 897 ret = amd_pstate_unregister_driver(0); 898 if (ret) 899 return ret; 900 901 ret = amd_pstate_register_driver(mode); 902 if (ret) 903 return ret; 904 905 return 0; 906 } 907 908 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 909 [AMD_PSTATE_DISABLE] = { 910 [AMD_PSTATE_DISABLE] = NULL, 911 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 912 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 913 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 914 }, 915 [AMD_PSTATE_PASSIVE] = { 916 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 917 [AMD_PSTATE_PASSIVE] = NULL, 918 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 919 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 920 }, 921 [AMD_PSTATE_ACTIVE] = { 922 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 923 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 924 [AMD_PSTATE_ACTIVE] = NULL, 925 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 926 }, 927 [AMD_PSTATE_GUIDED] = { 928 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 929 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 930 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 931 [AMD_PSTATE_GUIDED] = NULL, 932 }, 933 }; 934 935 static ssize_t amd_pstate_show_status(char *buf) 936 { 937 if (!current_pstate_driver) 938 return sysfs_emit(buf, "disable\n"); 939 940 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 941 } 942 943 static int amd_pstate_update_status(const char *buf, size_t size) 944 { 945 int mode_idx; 946 947 if (size > strlen("passive") || size < strlen("active")) 948 return -EINVAL; 949 950 mode_idx = get_mode_idx_from_str(buf, size); 951 952 if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) 953 return -EINVAL; 954 955 if (mode_state_machine[cppc_state][mode_idx]) 956 return mode_state_machine[cppc_state][mode_idx](mode_idx); 957 958 return 0; 959 } 960 961 static ssize_t show_status(struct kobject *kobj, 962 struct kobj_attribute *attr, char *buf) 963 { 964 ssize_t ret; 965 966 mutex_lock(&amd_pstate_driver_lock); 967 ret = amd_pstate_show_status(buf); 968 mutex_unlock(&amd_pstate_driver_lock); 969 970 return ret; 971 } 972 973 static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, 974 const char *buf, size_t count) 975 { 976 char *p = memchr(buf, '\n', count); 977 int ret; 978 979 mutex_lock(&amd_pstate_driver_lock); 980 ret = amd_pstate_update_status(buf, p ? p - buf : count); 981 mutex_unlock(&amd_pstate_driver_lock); 982 983 return ret < 0 ? ret : count; 984 } 985 986 cpufreq_freq_attr_ro(amd_pstate_max_freq); 987 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 988 989 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 990 cpufreq_freq_attr_rw(energy_performance_preference); 991 cpufreq_freq_attr_ro(energy_performance_available_preferences); 992 define_one_global_rw(status); 993 994 static struct freq_attr *amd_pstate_attr[] = { 995 &amd_pstate_max_freq, 996 &amd_pstate_lowest_nonlinear_freq, 997 &amd_pstate_highest_perf, 998 NULL, 999 }; 1000 1001 static struct freq_attr *amd_pstate_epp_attr[] = { 1002 &amd_pstate_max_freq, 1003 &amd_pstate_lowest_nonlinear_freq, 1004 &amd_pstate_highest_perf, 1005 &energy_performance_preference, 1006 &energy_performance_available_preferences, 1007 NULL, 1008 }; 1009 1010 static struct attribute *pstate_global_attributes[] = { 1011 &status.attr, 1012 NULL 1013 }; 1014 1015 static const struct attribute_group amd_pstate_global_attr_group = { 1016 .attrs = pstate_global_attributes, 1017 }; 1018 1019 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1020 { 1021 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 1022 struct amd_cpudata *cpudata; 1023 struct device *dev; 1024 u64 value; 1025 1026 /* 1027 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1028 * which is ideal for initialization process. 1029 */ 1030 amd_perf_ctl_reset(policy->cpu); 1031 dev = get_cpu_device(policy->cpu); 1032 if (!dev) 1033 return -ENODEV; 1034 1035 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 1036 if (!cpudata) 1037 return -ENOMEM; 1038 1039 cpudata->cpu = policy->cpu; 1040 cpudata->epp_policy = 0; 1041 1042 ret = amd_pstate_init_perf(cpudata); 1043 if (ret) 1044 goto free_cpudata1; 1045 1046 min_freq = amd_get_min_freq(cpudata); 1047 max_freq = amd_get_max_freq(cpudata); 1048 nominal_freq = amd_get_nominal_freq(cpudata); 1049 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 1050 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 1051 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 1052 min_freq, max_freq); 1053 ret = -EINVAL; 1054 goto free_cpudata1; 1055 } 1056 1057 policy->cpuinfo.min_freq = min_freq; 1058 policy->cpuinfo.max_freq = max_freq; 1059 /* It will be updated by governor */ 1060 policy->cur = policy->cpuinfo.min_freq; 1061 1062 /* Initial processor data capability frequencies */ 1063 cpudata->max_freq = max_freq; 1064 cpudata->min_freq = min_freq; 1065 cpudata->nominal_freq = nominal_freq; 1066 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 1067 1068 policy->driver_data = cpudata; 1069 1070 cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); 1071 1072 policy->min = policy->cpuinfo.min_freq; 1073 policy->max = policy->cpuinfo.max_freq; 1074 1075 /* 1076 * Set the policy to powersave to provide a valid fallback value in case 1077 * the default cpufreq governor is neither powersave nor performance. 1078 */ 1079 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1080 1081 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1082 policy->fast_switch_possible = true; 1083 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 1084 if (ret) 1085 return ret; 1086 WRITE_ONCE(cpudata->cppc_req_cached, value); 1087 1088 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); 1089 if (ret) 1090 return ret; 1091 WRITE_ONCE(cpudata->cppc_cap1_cached, value); 1092 } 1093 amd_pstate_boost_init(cpudata); 1094 1095 return 0; 1096 1097 free_cpudata1: 1098 kfree(cpudata); 1099 return ret; 1100 } 1101 1102 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1103 { 1104 pr_debug("CPU %d exiting\n", policy->cpu); 1105 policy->fast_switch_possible = false; 1106 return 0; 1107 } 1108 1109 static void amd_pstate_epp_init(unsigned int cpu) 1110 { 1111 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1112 struct amd_cpudata *cpudata = policy->driver_data; 1113 u32 max_perf, min_perf; 1114 u64 value; 1115 s16 epp; 1116 1117 max_perf = READ_ONCE(cpudata->highest_perf); 1118 min_perf = READ_ONCE(cpudata->lowest_perf); 1119 1120 value = READ_ONCE(cpudata->cppc_req_cached); 1121 1122 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1123 min_perf = max_perf; 1124 1125 /* Initial min/max values for CPPC Performance Controls Register */ 1126 value &= ~AMD_CPPC_MIN_PERF(~0L); 1127 value |= AMD_CPPC_MIN_PERF(min_perf); 1128 1129 value &= ~AMD_CPPC_MAX_PERF(~0L); 1130 value |= AMD_CPPC_MAX_PERF(max_perf); 1131 1132 /* CPPC EPP feature require to set zero to the desire perf bit */ 1133 value &= ~AMD_CPPC_DES_PERF(~0L); 1134 value |= AMD_CPPC_DES_PERF(0); 1135 1136 if (cpudata->epp_policy == cpudata->policy) 1137 goto skip_epp; 1138 1139 cpudata->epp_policy = cpudata->policy; 1140 1141 /* Get BIOS pre-defined epp value */ 1142 epp = amd_pstate_get_epp(cpudata, value); 1143 if (epp < 0) { 1144 /** 1145 * This return value can only be negative for shared_memory 1146 * systems where EPP register read/write not supported. 1147 */ 1148 goto skip_epp; 1149 } 1150 1151 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1152 epp = 0; 1153 1154 /* Set initial EPP value */ 1155 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1156 value &= ~GENMASK_ULL(31, 24); 1157 value |= (u64)epp << 24; 1158 } 1159 1160 WRITE_ONCE(cpudata->cppc_req_cached, value); 1161 amd_pstate_set_epp(cpudata, epp); 1162 skip_epp: 1163 cpufreq_cpu_put(policy); 1164 } 1165 1166 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 1167 { 1168 struct amd_cpudata *cpudata = policy->driver_data; 1169 1170 if (!policy->cpuinfo.max_freq) 1171 return -ENODEV; 1172 1173 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", 1174 policy->cpuinfo.max_freq, policy->max); 1175 1176 cpudata->policy = policy->policy; 1177 1178 amd_pstate_epp_init(policy->cpu); 1179 1180 return 0; 1181 } 1182 1183 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) 1184 { 1185 struct cppc_perf_ctrls perf_ctrls; 1186 u64 value, max_perf; 1187 int ret; 1188 1189 ret = amd_pstate_enable(true); 1190 if (ret) 1191 pr_err("failed to enable amd pstate during resume, return %d\n", ret); 1192 1193 value = READ_ONCE(cpudata->cppc_req_cached); 1194 max_perf = READ_ONCE(cpudata->highest_perf); 1195 1196 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1197 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1198 } else { 1199 perf_ctrls.max_perf = max_perf; 1200 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); 1201 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1202 } 1203 } 1204 1205 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) 1206 { 1207 struct amd_cpudata *cpudata = policy->driver_data; 1208 1209 pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); 1210 1211 if (cppc_state == AMD_PSTATE_ACTIVE) { 1212 amd_pstate_epp_reenable(cpudata); 1213 cpudata->suspended = false; 1214 } 1215 1216 return 0; 1217 } 1218 1219 static void amd_pstate_epp_offline(struct cpufreq_policy *policy) 1220 { 1221 struct amd_cpudata *cpudata = policy->driver_data; 1222 struct cppc_perf_ctrls perf_ctrls; 1223 int min_perf; 1224 u64 value; 1225 1226 min_perf = READ_ONCE(cpudata->lowest_perf); 1227 value = READ_ONCE(cpudata->cppc_req_cached); 1228 1229 mutex_lock(&amd_pstate_limits_lock); 1230 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1231 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; 1232 1233 /* Set max perf same as min perf */ 1234 value &= ~AMD_CPPC_MAX_PERF(~0L); 1235 value |= AMD_CPPC_MAX_PERF(min_perf); 1236 value &= ~AMD_CPPC_MIN_PERF(~0L); 1237 value |= AMD_CPPC_MIN_PERF(min_perf); 1238 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1239 } else { 1240 perf_ctrls.desired_perf = 0; 1241 perf_ctrls.max_perf = min_perf; 1242 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); 1243 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1244 } 1245 mutex_unlock(&amd_pstate_limits_lock); 1246 } 1247 1248 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) 1249 { 1250 struct amd_cpudata *cpudata = policy->driver_data; 1251 1252 pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); 1253 1254 if (cpudata->suspended) 1255 return 0; 1256 1257 if (cppc_state == AMD_PSTATE_ACTIVE) 1258 amd_pstate_epp_offline(policy); 1259 1260 return 0; 1261 } 1262 1263 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) 1264 { 1265 cpufreq_verify_within_cpu_limits(policy); 1266 pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); 1267 return 0; 1268 } 1269 1270 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) 1271 { 1272 struct amd_cpudata *cpudata = policy->driver_data; 1273 int ret; 1274 1275 /* avoid suspending when EPP is not enabled */ 1276 if (cppc_state != AMD_PSTATE_ACTIVE) 1277 return 0; 1278 1279 /* set this flag to avoid setting core offline*/ 1280 cpudata->suspended = true; 1281 1282 /* disable CPPC in lowlevel firmware */ 1283 ret = amd_pstate_enable(false); 1284 if (ret) 1285 pr_err("failed to suspend, return %d\n", ret); 1286 1287 return 0; 1288 } 1289 1290 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 1291 { 1292 struct amd_cpudata *cpudata = policy->driver_data; 1293 1294 if (cpudata->suspended) { 1295 mutex_lock(&amd_pstate_limits_lock); 1296 1297 /* enable amd pstate from suspend state*/ 1298 amd_pstate_epp_reenable(cpudata); 1299 1300 mutex_unlock(&amd_pstate_limits_lock); 1301 1302 cpudata->suspended = false; 1303 } 1304 1305 return 0; 1306 } 1307 1308 static struct cpufreq_driver amd_pstate_driver = { 1309 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 1310 .verify = amd_pstate_verify, 1311 .target = amd_pstate_target, 1312 .init = amd_pstate_cpu_init, 1313 .exit = amd_pstate_cpu_exit, 1314 .suspend = amd_pstate_cpu_suspend, 1315 .resume = amd_pstate_cpu_resume, 1316 .set_boost = amd_pstate_set_boost, 1317 .name = "amd-pstate", 1318 .attr = amd_pstate_attr, 1319 }; 1320 1321 static struct cpufreq_driver amd_pstate_epp_driver = { 1322 .flags = CPUFREQ_CONST_LOOPS, 1323 .verify = amd_pstate_epp_verify_policy, 1324 .setpolicy = amd_pstate_epp_set_policy, 1325 .init = amd_pstate_epp_cpu_init, 1326 .exit = amd_pstate_epp_cpu_exit, 1327 .offline = amd_pstate_epp_cpu_offline, 1328 .online = amd_pstate_epp_cpu_online, 1329 .suspend = amd_pstate_epp_suspend, 1330 .resume = amd_pstate_epp_resume, 1331 .name = "amd_pstate_epp", 1332 .attr = amd_pstate_epp_attr, 1333 }; 1334 1335 static int __init amd_pstate_init(void) 1336 { 1337 int ret; 1338 1339 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 1340 return -ENODEV; 1341 /* 1342 * by default the pstate driver is disabled to load 1343 * enable the amd_pstate passive mode driver explicitly 1344 * with amd_pstate=passive or other modes in kernel command line 1345 */ 1346 if (cppc_state == AMD_PSTATE_DISABLE) { 1347 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 1348 return -ENODEV; 1349 } 1350 1351 if (!acpi_cpc_valid()) { 1352 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 1353 return -ENODEV; 1354 } 1355 1356 /* don't keep reloading if cpufreq_driver exists */ 1357 if (cpufreq_get_current_driver()) 1358 return -EEXIST; 1359 1360 /* capability check */ 1361 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1362 pr_debug("AMD CPPC MSR based functionality is supported\n"); 1363 if (cppc_state != AMD_PSTATE_ACTIVE) 1364 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1365 } else { 1366 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 1367 static_call_update(amd_pstate_enable, cppc_enable); 1368 static_call_update(amd_pstate_init_perf, cppc_init_perf); 1369 static_call_update(amd_pstate_update_perf, cppc_update_perf); 1370 } 1371 1372 /* enable amd pstate feature */ 1373 ret = amd_pstate_enable(true); 1374 if (ret) { 1375 pr_err("failed to enable with return %d\n", ret); 1376 return ret; 1377 } 1378 1379 ret = cpufreq_register_driver(current_pstate_driver); 1380 if (ret) 1381 pr_err("failed to register with return %d\n", ret); 1382 1383 amd_pstate_kobj = kobject_create_and_add("amd_pstate", &cpu_subsys.dev_root->kobj); 1384 if (!amd_pstate_kobj) { 1385 ret = -EINVAL; 1386 pr_err("global sysfs registration failed.\n"); 1387 goto kobject_free; 1388 } 1389 1390 ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group); 1391 if (ret) { 1392 pr_err("sysfs attribute export failed with error %d.\n", ret); 1393 goto global_attr_free; 1394 } 1395 1396 return ret; 1397 1398 global_attr_free: 1399 kobject_put(amd_pstate_kobj); 1400 kobject_free: 1401 cpufreq_unregister_driver(current_pstate_driver); 1402 return ret; 1403 } 1404 device_initcall(amd_pstate_init); 1405 1406 static int __init amd_pstate_param(char *str) 1407 { 1408 size_t size; 1409 int mode_idx; 1410 1411 if (!str) 1412 return -EINVAL; 1413 1414 size = strlen(str); 1415 mode_idx = get_mode_idx_from_str(str, size); 1416 1417 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1418 cppc_state = mode_idx; 1419 if (cppc_state == AMD_PSTATE_DISABLE) 1420 pr_info("driver is explicitly disabled\n"); 1421 1422 if (cppc_state == AMD_PSTATE_ACTIVE) 1423 current_pstate_driver = &amd_pstate_epp_driver; 1424 1425 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1426 current_pstate_driver = &amd_pstate_driver; 1427 1428 return 0; 1429 } 1430 1431 return -EINVAL; 1432 } 1433 early_param("amd_pstate", amd_pstate_param); 1434 1435 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 1436 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 1437