1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pstate.c: Native P state management for Intel processors 4 * 5 * (C) Copyright 2012 Intel Corporation 6 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com> 7 */ 8 9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 11 #include <linux/kernel.h> 12 #include <linux/kernel_stat.h> 13 #include <linux/module.h> 14 #include <linux/ktime.h> 15 #include <linux/hrtimer.h> 16 #include <linux/tick.h> 17 #include <linux/slab.h> 18 #include <linux/sched/cpufreq.h> 19 #include <linux/sched/smt.h> 20 #include <linux/list.h> 21 #include <linux/cpu.h> 22 #include <linux/cpufreq.h> 23 #include <linux/sysfs.h> 24 #include <linux/types.h> 25 #include <linux/fs.h> 26 #include <linux/acpi.h> 27 #include <linux/vmalloc.h> 28 #include <linux/pm_qos.h> 29 #include <linux/bitfield.h> 30 #include <trace/events/power.h> 31 32 #include <asm/cpu.h> 33 #include <asm/div64.h> 34 #include <asm/msr.h> 35 #include <asm/cpu_device_id.h> 36 #include <asm/cpufeature.h> 37 #include <asm/intel-family.h> 38 #include "../drivers/thermal/intel/thermal_interrupt.h" 39 40 #define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC) 41 42 #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 43 #define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000 44 #define INTEL_CPUFREQ_TRANSITION_DELAY 500 45 46 #ifdef CONFIG_ACPI 47 #include <acpi/processor.h> 48 #include <acpi/cppc_acpi.h> 49 #endif 50 51 #define FRAC_BITS 8 52 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS) 53 #define fp_toint(X) ((X) >> FRAC_BITS) 54 55 #define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3)) 56 57 #define EXT_BITS 6 58 #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS) 59 #define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS) 60 #define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS) 61 62 static inline int32_t mul_fp(int32_t x, int32_t y) 63 { 64 return ((int64_t)x * (int64_t)y) >> FRAC_BITS; 65 } 66 67 static inline int32_t div_fp(s64 x, s64 y) 68 { 69 return div64_s64((int64_t)x << FRAC_BITS, y); 70 } 71 72 static inline int ceiling_fp(int32_t x) 73 { 74 int mask, ret; 75 76 ret = fp_toint(x); 77 mask = (1 << FRAC_BITS) - 1; 78 if (x & mask) 79 ret += 1; 80 return ret; 81 } 82 83 static inline u64 mul_ext_fp(u64 x, u64 y) 84 { 85 return (x * y) >> EXT_FRAC_BITS; 86 } 87 88 static inline u64 div_ext_fp(u64 x, u64 y) 89 { 90 return div64_u64(x << EXT_FRAC_BITS, y); 91 } 92 93 /** 94 * struct sample - Store performance sample 95 * @core_avg_perf: Ratio of APERF/MPERF which is the actual average 96 * performance during last sample period 97 * @busy_scaled: Scaled busy value which is used to calculate next 98 * P state. This can be different than core_avg_perf 99 * to account for cpu idle period 100 * @aperf: Difference of actual performance frequency clock count 101 * read from APERF MSR between last and current sample 102 * @mperf: Difference of maximum performance frequency clock count 103 * read from MPERF MSR between last and current sample 104 * @tsc: Difference of time stamp counter between last and 105 * current sample 106 * @time: Current time from scheduler 107 * 108 * This structure is used in the cpudata structure to store performance sample 109 * data for choosing next P State. 110 */ 111 struct sample { 112 int32_t core_avg_perf; 113 int32_t busy_scaled; 114 u64 aperf; 115 u64 mperf; 116 u64 tsc; 117 u64 time; 118 }; 119 120 /** 121 * struct pstate_data - Store P state data 122 * @current_pstate: Current requested P state 123 * @min_pstate: Min P state possible for this platform 124 * @max_pstate: Max P state possible for this platform 125 * @max_pstate_physical:This is physical Max P state for a processor 126 * This can be higher than the max_pstate which can 127 * be limited by platform thermal design power limits 128 * @perf_ctl_scaling: PERF_CTL P-state to frequency scaling factor 129 * @scaling: Scaling factor between performance and frequency 130 * @turbo_pstate: Max Turbo P state possible for this platform 131 * @min_freq: @min_pstate frequency in cpufreq units 132 * @max_freq: @max_pstate frequency in cpufreq units 133 * @turbo_freq: @turbo_pstate frequency in cpufreq units 134 * 135 * Stores the per cpu model P state limits and current P state. 136 */ 137 struct pstate_data { 138 int current_pstate; 139 int min_pstate; 140 int max_pstate; 141 int max_pstate_physical; 142 int perf_ctl_scaling; 143 int scaling; 144 int turbo_pstate; 145 unsigned int min_freq; 146 unsigned int max_freq; 147 unsigned int turbo_freq; 148 }; 149 150 /** 151 * struct vid_data - Stores voltage information data 152 * @min: VID data for this platform corresponding to 153 * the lowest P state 154 * @max: VID data corresponding to the highest P State. 155 * @turbo: VID data for turbo P state 156 * @ratio: Ratio of (vid max - vid min) / 157 * (max P state - Min P State) 158 * 159 * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling) 160 * This data is used in Atom platforms, where in addition to target P state, 161 * the voltage data needs to be specified to select next P State. 162 */ 163 struct vid_data { 164 int min; 165 int max; 166 int turbo; 167 int32_t ratio; 168 }; 169 170 /** 171 * struct global_params - Global parameters, mostly tunable via sysfs. 172 * @no_turbo: Whether or not to use turbo P-states. 173 * @turbo_disabled: Whether or not turbo P-states are available at all, 174 * based on the MSR_IA32_MISC_ENABLE value and whether or 175 * not the maximum reported turbo P-state is different from 176 * the maximum reported non-turbo one. 177 * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo 178 * P-state capacity. 179 * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo 180 * P-state capacity. 181 */ 182 struct global_params { 183 bool no_turbo; 184 bool turbo_disabled; 185 int max_perf_pct; 186 int min_perf_pct; 187 }; 188 189 /** 190 * struct cpudata - Per CPU instance data storage 191 * @cpu: CPU number for this instance data 192 * @policy: CPUFreq policy value 193 * @update_util: CPUFreq utility callback information 194 * @update_util_set: CPUFreq utility callback is set 195 * @iowait_boost: iowait-related boost fraction 196 * @last_update: Time of the last update. 197 * @pstate: Stores P state limits for this CPU 198 * @vid: Stores VID limits for this CPU 199 * @last_sample_time: Last Sample time 200 * @aperf_mperf_shift: APERF vs MPERF counting frequency difference 201 * @prev_aperf: Last APERF value read from APERF MSR 202 * @prev_mperf: Last MPERF value read from MPERF MSR 203 * @prev_tsc: Last timestamp counter (TSC) value 204 * @sample: Storage for storing last Sample data 205 * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios 206 * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios 207 * @acpi_perf_data: Stores ACPI perf information read from _PSS 208 * @valid_pss_table: Set to true for valid ACPI _PSS entries found 209 * @epp_powersave: Last saved HWP energy performance preference 210 * (EPP) or energy performance bias (EPB), 211 * when policy switched to performance 212 * @epp_policy: Last saved policy used to set EPP/EPB 213 * @epp_default: Power on default HWP energy performance 214 * preference/bias 215 * @epp_cached: Cached HWP energy-performance preference value 216 * @hwp_req_cached: Cached value of the last HWP Request MSR 217 * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR 218 * @last_io_update: Last time when IO wake flag was set 219 * @capacity_perf: Highest perf used for scale invariance 220 * @sched_flags: Store scheduler flags for possible cross CPU update 221 * @hwp_boost_min: Last HWP boosted min performance 222 * @suspended: Whether or not the driver has been suspended. 223 * @hwp_notify_work: workqueue for HWP notifications. 224 * 225 * This structure stores per CPU instance data for all CPUs. 226 */ 227 struct cpudata { 228 int cpu; 229 230 unsigned int policy; 231 struct update_util_data update_util; 232 bool update_util_set; 233 234 struct pstate_data pstate; 235 struct vid_data vid; 236 237 u64 last_update; 238 u64 last_sample_time; 239 u64 aperf_mperf_shift; 240 u64 prev_aperf; 241 u64 prev_mperf; 242 u64 prev_tsc; 243 struct sample sample; 244 int32_t min_perf_ratio; 245 int32_t max_perf_ratio; 246 #ifdef CONFIG_ACPI 247 struct acpi_processor_performance acpi_perf_data; 248 bool valid_pss_table; 249 #endif 250 unsigned int iowait_boost; 251 s16 epp_powersave; 252 s16 epp_policy; 253 s16 epp_default; 254 s16 epp_cached; 255 u64 hwp_req_cached; 256 u64 hwp_cap_cached; 257 u64 last_io_update; 258 unsigned int capacity_perf; 259 unsigned int sched_flags; 260 u32 hwp_boost_min; 261 bool suspended; 262 struct delayed_work hwp_notify_work; 263 }; 264 265 static struct cpudata **all_cpu_data; 266 267 /** 268 * struct pstate_funcs - Per CPU model specific callbacks 269 * @get_max: Callback to get maximum non turbo effective P state 270 * @get_max_physical: Callback to get maximum non turbo physical P state 271 * @get_min: Callback to get minimum P state 272 * @get_turbo: Callback to get turbo P state 273 * @get_scaling: Callback to get frequency scaling factor 274 * @get_cpu_scaling: Get frequency scaling factor for a given cpu 275 * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference 276 * @get_val: Callback to convert P state to actual MSR write value 277 * @get_vid: Callback to get VID data for Atom platforms 278 * 279 * Core and Atom CPU models have different way to get P State limits. This 280 * structure is used to store those callbacks. 281 */ 282 struct pstate_funcs { 283 int (*get_max)(int cpu); 284 int (*get_max_physical)(int cpu); 285 int (*get_min)(int cpu); 286 int (*get_turbo)(int cpu); 287 int (*get_scaling)(void); 288 int (*get_cpu_scaling)(int cpu); 289 int (*get_aperf_mperf_shift)(void); 290 u64 (*get_val)(struct cpudata*, int pstate); 291 void (*get_vid)(struct cpudata *); 292 }; 293 294 static struct pstate_funcs pstate_funcs __read_mostly; 295 296 static bool hwp_active __ro_after_init; 297 static int hwp_mode_bdw __ro_after_init; 298 static bool per_cpu_limits __ro_after_init; 299 static bool hwp_forced __ro_after_init; 300 static bool hwp_boost __read_mostly; 301 static bool hwp_is_hybrid; 302 303 static struct cpufreq_driver *intel_pstate_driver __read_mostly; 304 305 #define HYBRID_SCALING_FACTOR 78741 306 #define HYBRID_SCALING_FACTOR_MTL 80000 307 #define HYBRID_SCALING_FACTOR_LNL 86957 308 309 static int hybrid_scaling_factor = HYBRID_SCALING_FACTOR; 310 311 static inline int core_get_scaling(void) 312 { 313 return 100000; 314 } 315 316 #ifdef CONFIG_ACPI 317 static bool acpi_ppc; 318 #endif 319 320 static struct global_params global; 321 322 static DEFINE_MUTEX(intel_pstate_driver_lock); 323 static DEFINE_MUTEX(intel_pstate_limits_lock); 324 325 #ifdef CONFIG_ACPI 326 327 static bool intel_pstate_acpi_pm_profile_server(void) 328 { 329 if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER || 330 acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER) 331 return true; 332 333 return false; 334 } 335 336 static bool intel_pstate_get_ppc_enable_status(void) 337 { 338 if (intel_pstate_acpi_pm_profile_server()) 339 return true; 340 341 return acpi_ppc; 342 } 343 344 #ifdef CONFIG_ACPI_CPPC_LIB 345 346 /* The work item is needed to avoid CPU hotplug locking issues */ 347 static void intel_pstste_sched_itmt_work_fn(struct work_struct *work) 348 { 349 sched_set_itmt_support(); 350 } 351 352 static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn); 353 354 #define CPPC_MAX_PERF U8_MAX 355 356 static void intel_pstate_set_itmt_prio(int cpu) 357 { 358 struct cppc_perf_caps cppc_perf; 359 static u32 max_highest_perf = 0, min_highest_perf = U32_MAX; 360 int ret; 361 362 ret = cppc_get_perf_caps(cpu, &cppc_perf); 363 /* 364 * If CPPC is not available, fall back to MSR_HWP_CAPABILITIES bits [8:0]. 365 * 366 * Also, on some systems with overclocking enabled, CPPC.highest_perf is 367 * hardcoded to 0xff, so CPPC.highest_perf cannot be used to enable ITMT. 368 * Fall back to MSR_HWP_CAPABILITIES then too. 369 */ 370 if (ret || cppc_perf.highest_perf == CPPC_MAX_PERF) 371 cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached)); 372 373 /* 374 * The priorities can be set regardless of whether or not 375 * sched_set_itmt_support(true) has been called and it is valid to 376 * update them at any time after it has been called. 377 */ 378 sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); 379 380 if (max_highest_perf <= min_highest_perf) { 381 if (cppc_perf.highest_perf > max_highest_perf) 382 max_highest_perf = cppc_perf.highest_perf; 383 384 if (cppc_perf.highest_perf < min_highest_perf) 385 min_highest_perf = cppc_perf.highest_perf; 386 387 if (max_highest_perf > min_highest_perf) { 388 /* 389 * This code can be run during CPU online under the 390 * CPU hotplug locks, so sched_set_itmt_support() 391 * cannot be called from here. Queue up a work item 392 * to invoke it. 393 */ 394 schedule_work(&sched_itmt_work); 395 } 396 } 397 } 398 399 static int intel_pstate_get_cppc_guaranteed(int cpu) 400 { 401 struct cppc_perf_caps cppc_perf; 402 int ret; 403 404 ret = cppc_get_perf_caps(cpu, &cppc_perf); 405 if (ret) 406 return ret; 407 408 if (cppc_perf.guaranteed_perf) 409 return cppc_perf.guaranteed_perf; 410 411 return cppc_perf.nominal_perf; 412 } 413 414 static int intel_pstate_cppc_get_scaling(int cpu) 415 { 416 struct cppc_perf_caps cppc_perf; 417 int ret; 418 419 ret = cppc_get_perf_caps(cpu, &cppc_perf); 420 421 /* 422 * If the nominal frequency and the nominal performance are not 423 * zero and the ratio between them is not 100, return the hybrid 424 * scaling factor. 425 */ 426 if (!ret && cppc_perf.nominal_perf && cppc_perf.nominal_freq && 427 cppc_perf.nominal_perf * 100 != cppc_perf.nominal_freq) 428 return hybrid_scaling_factor; 429 430 return core_get_scaling(); 431 } 432 433 #else /* CONFIG_ACPI_CPPC_LIB */ 434 static inline void intel_pstate_set_itmt_prio(int cpu) 435 { 436 } 437 #endif /* CONFIG_ACPI_CPPC_LIB */ 438 439 static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy) 440 { 441 struct cpudata *cpu; 442 int ret; 443 int i; 444 445 if (hwp_active) { 446 intel_pstate_set_itmt_prio(policy->cpu); 447 return; 448 } 449 450 if (!intel_pstate_get_ppc_enable_status()) 451 return; 452 453 cpu = all_cpu_data[policy->cpu]; 454 455 ret = acpi_processor_register_performance(&cpu->acpi_perf_data, 456 policy->cpu); 457 if (ret) 458 return; 459 460 /* 461 * Check if the control value in _PSS is for PERF_CTL MSR, which should 462 * guarantee that the states returned by it map to the states in our 463 * list directly. 464 */ 465 if (cpu->acpi_perf_data.control_register.space_id != 466 ACPI_ADR_SPACE_FIXED_HARDWARE) 467 goto err; 468 469 /* 470 * If there is only one entry _PSS, simply ignore _PSS and continue as 471 * usual without taking _PSS into account 472 */ 473 if (cpu->acpi_perf_data.state_count < 2) 474 goto err; 475 476 pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu); 477 for (i = 0; i < cpu->acpi_perf_data.state_count; i++) { 478 pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n", 479 (i == cpu->acpi_perf_data.state ? '*' : ' '), i, 480 (u32) cpu->acpi_perf_data.states[i].core_frequency, 481 (u32) cpu->acpi_perf_data.states[i].power, 482 (u32) cpu->acpi_perf_data.states[i].control); 483 } 484 485 cpu->valid_pss_table = true; 486 pr_debug("_PPC limits will be enforced\n"); 487 488 return; 489 490 err: 491 cpu->valid_pss_table = false; 492 acpi_processor_unregister_performance(policy->cpu); 493 } 494 495 static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) 496 { 497 struct cpudata *cpu; 498 499 cpu = all_cpu_data[policy->cpu]; 500 if (!cpu->valid_pss_table) 501 return; 502 503 acpi_processor_unregister_performance(policy->cpu); 504 } 505 #else /* CONFIG_ACPI */ 506 static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy) 507 { 508 } 509 510 static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) 511 { 512 } 513 514 static inline bool intel_pstate_acpi_pm_profile_server(void) 515 { 516 return false; 517 } 518 #endif /* CONFIG_ACPI */ 519 520 #ifndef CONFIG_ACPI_CPPC_LIB 521 static inline int intel_pstate_get_cppc_guaranteed(int cpu) 522 { 523 return -ENOTSUPP; 524 } 525 526 static int intel_pstate_cppc_get_scaling(int cpu) 527 { 528 return core_get_scaling(); 529 } 530 #endif /* CONFIG_ACPI_CPPC_LIB */ 531 532 static int intel_pstate_freq_to_hwp_rel(struct cpudata *cpu, int freq, 533 unsigned int relation) 534 { 535 if (freq == cpu->pstate.turbo_freq) 536 return cpu->pstate.turbo_pstate; 537 538 if (freq == cpu->pstate.max_freq) 539 return cpu->pstate.max_pstate; 540 541 switch (relation) { 542 case CPUFREQ_RELATION_H: 543 return freq / cpu->pstate.scaling; 544 case CPUFREQ_RELATION_C: 545 return DIV_ROUND_CLOSEST(freq, cpu->pstate.scaling); 546 } 547 548 return DIV_ROUND_UP(freq, cpu->pstate.scaling); 549 } 550 551 static int intel_pstate_freq_to_hwp(struct cpudata *cpu, int freq) 552 { 553 return intel_pstate_freq_to_hwp_rel(cpu, freq, CPUFREQ_RELATION_L); 554 } 555 556 /** 557 * intel_pstate_hybrid_hwp_adjust - Calibrate HWP performance levels. 558 * @cpu: Target CPU. 559 * 560 * On hybrid processors, HWP may expose more performance levels than there are 561 * P-states accessible through the PERF_CTL interface. If that happens, the 562 * scaling factor between HWP performance levels and CPU frequency will be less 563 * than the scaling factor between P-state values and CPU frequency. 564 * 565 * In that case, adjust the CPU parameters used in computations accordingly. 566 */ 567 static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu) 568 { 569 int perf_ctl_max_phys = cpu->pstate.max_pstate_physical; 570 int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; 571 int perf_ctl_turbo = pstate_funcs.get_turbo(cpu->cpu); 572 int scaling = cpu->pstate.scaling; 573 int freq; 574 575 pr_debug("CPU%d: perf_ctl_max_phys = %d\n", cpu->cpu, perf_ctl_max_phys); 576 pr_debug("CPU%d: perf_ctl_turbo = %d\n", cpu->cpu, perf_ctl_turbo); 577 pr_debug("CPU%d: perf_ctl_scaling = %d\n", cpu->cpu, perf_ctl_scaling); 578 pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpu->cpu, cpu->pstate.max_pstate); 579 pr_debug("CPU%d: HWP_CAP highest = %d\n", cpu->cpu, cpu->pstate.turbo_pstate); 580 pr_debug("CPU%d: HWP-to-frequency scaling factor: %d\n", cpu->cpu, scaling); 581 582 cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_pstate * scaling, 583 perf_ctl_scaling); 584 cpu->pstate.max_freq = rounddown(cpu->pstate.max_pstate * scaling, 585 perf_ctl_scaling); 586 587 freq = perf_ctl_max_phys * perf_ctl_scaling; 588 cpu->pstate.max_pstate_physical = intel_pstate_freq_to_hwp(cpu, freq); 589 590 freq = cpu->pstate.min_pstate * perf_ctl_scaling; 591 cpu->pstate.min_freq = freq; 592 /* 593 * Cast the min P-state value retrieved via pstate_funcs.get_min() to 594 * the effective range of HWP performance levels. 595 */ 596 cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, freq); 597 } 598 599 static bool turbo_is_disabled(void) 600 { 601 u64 misc_en; 602 603 rdmsrl(MSR_IA32_MISC_ENABLE, misc_en); 604 605 return !!(misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE); 606 } 607 608 static int min_perf_pct_min(void) 609 { 610 struct cpudata *cpu = all_cpu_data[0]; 611 int turbo_pstate = cpu->pstate.turbo_pstate; 612 613 return turbo_pstate ? 614 (cpu->pstate.min_pstate * 100 / turbo_pstate) : 0; 615 } 616 617 static s16 intel_pstate_get_epb(struct cpudata *cpu_data) 618 { 619 u64 epb; 620 int ret; 621 622 if (!boot_cpu_has(X86_FEATURE_EPB)) 623 return -ENXIO; 624 625 ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); 626 if (ret) 627 return (s16)ret; 628 629 return (s16)(epb & 0x0f); 630 } 631 632 static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data) 633 { 634 s16 epp; 635 636 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { 637 /* 638 * When hwp_req_data is 0, means that caller didn't read 639 * MSR_HWP_REQUEST, so need to read and get EPP. 640 */ 641 if (!hwp_req_data) { 642 epp = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, 643 &hwp_req_data); 644 if (epp) 645 return epp; 646 } 647 epp = (hwp_req_data >> 24) & 0xff; 648 } else { 649 /* When there is no EPP present, HWP uses EPB settings */ 650 epp = intel_pstate_get_epb(cpu_data); 651 } 652 653 return epp; 654 } 655 656 static int intel_pstate_set_epb(int cpu, s16 pref) 657 { 658 u64 epb; 659 int ret; 660 661 if (!boot_cpu_has(X86_FEATURE_EPB)) 662 return -ENXIO; 663 664 ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); 665 if (ret) 666 return ret; 667 668 epb = (epb & ~0x0f) | pref; 669 wrmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, epb); 670 671 return 0; 672 } 673 674 /* 675 * EPP/EPB display strings corresponding to EPP index in the 676 * energy_perf_strings[] 677 * index String 678 *------------------------------------- 679 * 0 default 680 * 1 performance 681 * 2 balance_performance 682 * 3 balance_power 683 * 4 power 684 */ 685 686 enum energy_perf_value_index { 687 EPP_INDEX_DEFAULT = 0, 688 EPP_INDEX_PERFORMANCE, 689 EPP_INDEX_BALANCE_PERFORMANCE, 690 EPP_INDEX_BALANCE_POWERSAVE, 691 EPP_INDEX_POWERSAVE, 692 }; 693 694 static const char * const energy_perf_strings[] = { 695 [EPP_INDEX_DEFAULT] = "default", 696 [EPP_INDEX_PERFORMANCE] = "performance", 697 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 698 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 699 [EPP_INDEX_POWERSAVE] = "power", 700 NULL 701 }; 702 static unsigned int epp_values[] = { 703 [EPP_INDEX_DEFAULT] = 0, /* Unused index */ 704 [EPP_INDEX_PERFORMANCE] = HWP_EPP_PERFORMANCE, 705 [EPP_INDEX_BALANCE_PERFORMANCE] = HWP_EPP_BALANCE_PERFORMANCE, 706 [EPP_INDEX_BALANCE_POWERSAVE] = HWP_EPP_BALANCE_POWERSAVE, 707 [EPP_INDEX_POWERSAVE] = HWP_EPP_POWERSAVE, 708 }; 709 710 static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw_epp) 711 { 712 s16 epp; 713 int index = -EINVAL; 714 715 *raw_epp = 0; 716 epp = intel_pstate_get_epp(cpu_data, 0); 717 if (epp < 0) 718 return epp; 719 720 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { 721 if (epp == epp_values[EPP_INDEX_PERFORMANCE]) 722 return EPP_INDEX_PERFORMANCE; 723 if (epp == epp_values[EPP_INDEX_BALANCE_PERFORMANCE]) 724 return EPP_INDEX_BALANCE_PERFORMANCE; 725 if (epp == epp_values[EPP_INDEX_BALANCE_POWERSAVE]) 726 return EPP_INDEX_BALANCE_POWERSAVE; 727 if (epp == epp_values[EPP_INDEX_POWERSAVE]) 728 return EPP_INDEX_POWERSAVE; 729 *raw_epp = epp; 730 return 0; 731 } else if (boot_cpu_has(X86_FEATURE_EPB)) { 732 /* 733 * Range: 734 * 0x00-0x03 : Performance 735 * 0x04-0x07 : Balance performance 736 * 0x08-0x0B : Balance power 737 * 0x0C-0x0F : Power 738 * The EPB is a 4 bit value, but our ranges restrict the 739 * value which can be set. Here only using top two bits 740 * effectively. 741 */ 742 index = (epp >> 2) + 1; 743 } 744 745 return index; 746 } 747 748 static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp) 749 { 750 int ret; 751 752 /* 753 * Use the cached HWP Request MSR value, because in the active mode the 754 * register itself may be updated by intel_pstate_hwp_boost_up() or 755 * intel_pstate_hwp_boost_down() at any time. 756 */ 757 u64 value = READ_ONCE(cpu->hwp_req_cached); 758 759 value &= ~GENMASK_ULL(31, 24); 760 value |= (u64)epp << 24; 761 /* 762 * The only other updater of hwp_req_cached in the active mode, 763 * intel_pstate_hwp_set(), is called under the same lock as this 764 * function, so it cannot run in parallel with the update below. 765 */ 766 WRITE_ONCE(cpu->hwp_req_cached, value); 767 ret = wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); 768 if (!ret) 769 cpu->epp_cached = epp; 770 771 return ret; 772 } 773 774 static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data, 775 int pref_index, bool use_raw, 776 u32 raw_epp) 777 { 778 int epp = -EINVAL; 779 int ret; 780 781 if (!pref_index) 782 epp = cpu_data->epp_default; 783 784 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { 785 if (use_raw) 786 epp = raw_epp; 787 else if (epp == -EINVAL) 788 epp = epp_values[pref_index]; 789 790 /* 791 * To avoid confusion, refuse to set EPP to any values different 792 * from 0 (performance) if the current policy is "performance", 793 * because those values would be overridden. 794 */ 795 if (epp > 0 && cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) 796 return -EBUSY; 797 798 ret = intel_pstate_set_epp(cpu_data, epp); 799 } else { 800 if (epp == -EINVAL) 801 epp = (pref_index - 1) << 2; 802 ret = intel_pstate_set_epb(cpu_data->cpu, epp); 803 } 804 805 return ret; 806 } 807 808 static ssize_t show_energy_performance_available_preferences( 809 struct cpufreq_policy *policy, char *buf) 810 { 811 int i = 0; 812 int ret = 0; 813 814 while (energy_perf_strings[i] != NULL) 815 ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]); 816 817 ret += sprintf(&buf[ret], "\n"); 818 819 return ret; 820 } 821 822 cpufreq_freq_attr_ro(energy_performance_available_preferences); 823 824 static struct cpufreq_driver intel_pstate; 825 826 static ssize_t store_energy_performance_preference( 827 struct cpufreq_policy *policy, const char *buf, size_t count) 828 { 829 struct cpudata *cpu = all_cpu_data[policy->cpu]; 830 char str_preference[21]; 831 bool raw = false; 832 ssize_t ret; 833 u32 epp = 0; 834 835 ret = sscanf(buf, "%20s", str_preference); 836 if (ret != 1) 837 return -EINVAL; 838 839 ret = match_string(energy_perf_strings, -1, str_preference); 840 if (ret < 0) { 841 if (!boot_cpu_has(X86_FEATURE_HWP_EPP)) 842 return ret; 843 844 ret = kstrtouint(buf, 10, &epp); 845 if (ret) 846 return ret; 847 848 if (epp > 255) 849 return -EINVAL; 850 851 raw = true; 852 } 853 854 /* 855 * This function runs with the policy R/W semaphore held, which 856 * guarantees that the driver pointer will not change while it is 857 * running. 858 */ 859 if (!intel_pstate_driver) 860 return -EAGAIN; 861 862 mutex_lock(&intel_pstate_limits_lock); 863 864 if (intel_pstate_driver == &intel_pstate) { 865 ret = intel_pstate_set_energy_pref_index(cpu, ret, raw, epp); 866 } else { 867 /* 868 * In the passive mode the governor needs to be stopped on the 869 * target CPU before the EPP update and restarted after it, 870 * which is super-heavy-weight, so make sure it is worth doing 871 * upfront. 872 */ 873 if (!raw) 874 epp = ret ? epp_values[ret] : cpu->epp_default; 875 876 if (cpu->epp_cached != epp) { 877 int err; 878 879 cpufreq_stop_governor(policy); 880 ret = intel_pstate_set_epp(cpu, epp); 881 err = cpufreq_start_governor(policy); 882 if (!ret) 883 ret = err; 884 } else { 885 ret = 0; 886 } 887 } 888 889 mutex_unlock(&intel_pstate_limits_lock); 890 891 return ret ?: count; 892 } 893 894 static ssize_t show_energy_performance_preference( 895 struct cpufreq_policy *policy, char *buf) 896 { 897 struct cpudata *cpu_data = all_cpu_data[policy->cpu]; 898 int preference, raw_epp; 899 900 preference = intel_pstate_get_energy_pref_index(cpu_data, &raw_epp); 901 if (preference < 0) 902 return preference; 903 904 if (raw_epp) 905 return sprintf(buf, "%d\n", raw_epp); 906 else 907 return sprintf(buf, "%s\n", energy_perf_strings[preference]); 908 } 909 910 cpufreq_freq_attr_rw(energy_performance_preference); 911 912 static ssize_t show_base_frequency(struct cpufreq_policy *policy, char *buf) 913 { 914 struct cpudata *cpu = all_cpu_data[policy->cpu]; 915 int ratio, freq; 916 917 ratio = intel_pstate_get_cppc_guaranteed(policy->cpu); 918 if (ratio <= 0) { 919 u64 cap; 920 921 rdmsrl_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap); 922 ratio = HWP_GUARANTEED_PERF(cap); 923 } 924 925 freq = ratio * cpu->pstate.scaling; 926 if (cpu->pstate.scaling != cpu->pstate.perf_ctl_scaling) 927 freq = rounddown(freq, cpu->pstate.perf_ctl_scaling); 928 929 return sprintf(buf, "%d\n", freq); 930 } 931 932 cpufreq_freq_attr_ro(base_frequency); 933 934 static struct freq_attr *hwp_cpufreq_attrs[] = { 935 &energy_performance_preference, 936 &energy_performance_available_preferences, 937 &base_frequency, 938 NULL, 939 }; 940 941 static struct cpudata *hybrid_max_perf_cpu __read_mostly; 942 /* 943 * Protects hybrid_max_perf_cpu, the capacity_perf fields in struct cpudata, 944 * and the x86 arch scale-invariance information from concurrent updates. 945 */ 946 static DEFINE_MUTEX(hybrid_capacity_lock); 947 948 static void hybrid_set_cpu_capacity(struct cpudata *cpu) 949 { 950 arch_set_cpu_capacity(cpu->cpu, cpu->capacity_perf, 951 hybrid_max_perf_cpu->capacity_perf, 952 cpu->capacity_perf, 953 cpu->pstate.max_pstate_physical); 954 955 pr_debug("CPU%d: perf = %u, max. perf = %u, base perf = %d\n", cpu->cpu, 956 cpu->capacity_perf, hybrid_max_perf_cpu->capacity_perf, 957 cpu->pstate.max_pstate_physical); 958 } 959 960 static void hybrid_clear_cpu_capacity(unsigned int cpunum) 961 { 962 arch_set_cpu_capacity(cpunum, 1, 1, 1, 1); 963 } 964 965 static void hybrid_get_capacity_perf(struct cpudata *cpu) 966 { 967 if (READ_ONCE(global.no_turbo)) { 968 cpu->capacity_perf = cpu->pstate.max_pstate_physical; 969 return; 970 } 971 972 cpu->capacity_perf = HWP_HIGHEST_PERF(READ_ONCE(cpu->hwp_cap_cached)); 973 } 974 975 static void hybrid_set_capacity_of_cpus(void) 976 { 977 int cpunum; 978 979 for_each_online_cpu(cpunum) { 980 struct cpudata *cpu = all_cpu_data[cpunum]; 981 982 if (cpu) 983 hybrid_set_cpu_capacity(cpu); 984 } 985 } 986 987 static void hybrid_update_cpu_capacity_scaling(void) 988 { 989 struct cpudata *max_perf_cpu = NULL; 990 unsigned int max_cap_perf = 0; 991 int cpunum; 992 993 for_each_online_cpu(cpunum) { 994 struct cpudata *cpu = all_cpu_data[cpunum]; 995 996 if (!cpu) 997 continue; 998 999 /* 1000 * During initialization, CPU performance at full capacity needs 1001 * to be determined. 1002 */ 1003 if (!hybrid_max_perf_cpu) 1004 hybrid_get_capacity_perf(cpu); 1005 1006 /* 1007 * If hybrid_max_perf_cpu is not NULL at this point, it is 1008 * being replaced, so don't take it into account when looking 1009 * for the new one. 1010 */ 1011 if (cpu == hybrid_max_perf_cpu) 1012 continue; 1013 1014 if (cpu->capacity_perf > max_cap_perf) { 1015 max_cap_perf = cpu->capacity_perf; 1016 max_perf_cpu = cpu; 1017 } 1018 } 1019 1020 if (max_perf_cpu) { 1021 hybrid_max_perf_cpu = max_perf_cpu; 1022 hybrid_set_capacity_of_cpus(); 1023 } else { 1024 pr_info("Found no CPUs with nonzero maximum performance\n"); 1025 /* Revert to the flat CPU capacity structure. */ 1026 for_each_online_cpu(cpunum) 1027 hybrid_clear_cpu_capacity(cpunum); 1028 } 1029 } 1030 1031 static void __hybrid_init_cpu_capacity_scaling(void) 1032 { 1033 hybrid_max_perf_cpu = NULL; 1034 hybrid_update_cpu_capacity_scaling(); 1035 } 1036 1037 static void hybrid_init_cpu_capacity_scaling(void) 1038 { 1039 bool disable_itmt = false; 1040 1041 mutex_lock(&hybrid_capacity_lock); 1042 1043 /* 1044 * If hybrid_max_perf_cpu is set at this point, the hybrid CPU capacity 1045 * scaling has been enabled already and the driver is just changing the 1046 * operation mode. 1047 */ 1048 if (hybrid_max_perf_cpu) { 1049 __hybrid_init_cpu_capacity_scaling(); 1050 goto unlock; 1051 } 1052 1053 /* 1054 * On hybrid systems, use asym capacity instead of ITMT, but because 1055 * the capacity of SMT threads is not deterministic even approximately, 1056 * do not do that when SMT is in use. 1057 */ 1058 if (hwp_is_hybrid && !sched_smt_active() && arch_enable_hybrid_capacity_scale()) { 1059 __hybrid_init_cpu_capacity_scaling(); 1060 disable_itmt = true; 1061 } 1062 1063 unlock: 1064 mutex_unlock(&hybrid_capacity_lock); 1065 1066 /* 1067 * Disabling ITMT causes sched domains to be rebuilt to disable asym 1068 * packing and enable asym capacity. 1069 */ 1070 if (disable_itmt) 1071 sched_clear_itmt_support(); 1072 } 1073 1074 static void __intel_pstate_get_hwp_cap(struct cpudata *cpu) 1075 { 1076 u64 cap; 1077 1078 rdmsrl_on_cpu(cpu->cpu, MSR_HWP_CAPABILITIES, &cap); 1079 WRITE_ONCE(cpu->hwp_cap_cached, cap); 1080 cpu->pstate.max_pstate = HWP_GUARANTEED_PERF(cap); 1081 cpu->pstate.turbo_pstate = HWP_HIGHEST_PERF(cap); 1082 } 1083 1084 static void intel_pstate_get_hwp_cap(struct cpudata *cpu) 1085 { 1086 int scaling = cpu->pstate.scaling; 1087 1088 __intel_pstate_get_hwp_cap(cpu); 1089 1090 cpu->pstate.max_freq = cpu->pstate.max_pstate * scaling; 1091 cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * scaling; 1092 if (scaling != cpu->pstate.perf_ctl_scaling) { 1093 int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; 1094 1095 cpu->pstate.max_freq = rounddown(cpu->pstate.max_freq, 1096 perf_ctl_scaling); 1097 cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_freq, 1098 perf_ctl_scaling); 1099 } 1100 } 1101 1102 static void hybrid_update_capacity(struct cpudata *cpu) 1103 { 1104 unsigned int max_cap_perf; 1105 1106 mutex_lock(&hybrid_capacity_lock); 1107 1108 if (!hybrid_max_perf_cpu) 1109 goto unlock; 1110 1111 /* 1112 * The maximum performance of the CPU may have changed, but assume 1113 * that the performance of the other CPUs has not changed. 1114 */ 1115 max_cap_perf = hybrid_max_perf_cpu->capacity_perf; 1116 1117 intel_pstate_get_hwp_cap(cpu); 1118 1119 hybrid_get_capacity_perf(cpu); 1120 /* Should hybrid_max_perf_cpu be replaced by this CPU? */ 1121 if (cpu->capacity_perf > max_cap_perf) { 1122 hybrid_max_perf_cpu = cpu; 1123 hybrid_set_capacity_of_cpus(); 1124 goto unlock; 1125 } 1126 1127 /* If this CPU is hybrid_max_perf_cpu, should it be replaced? */ 1128 if (cpu == hybrid_max_perf_cpu && cpu->capacity_perf < max_cap_perf) { 1129 hybrid_update_cpu_capacity_scaling(); 1130 goto unlock; 1131 } 1132 1133 hybrid_set_cpu_capacity(cpu); 1134 1135 unlock: 1136 mutex_unlock(&hybrid_capacity_lock); 1137 } 1138 1139 static void intel_pstate_hwp_set(unsigned int cpu) 1140 { 1141 struct cpudata *cpu_data = all_cpu_data[cpu]; 1142 int max, min; 1143 u64 value; 1144 s16 epp; 1145 1146 max = cpu_data->max_perf_ratio; 1147 min = cpu_data->min_perf_ratio; 1148 1149 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) 1150 min = max; 1151 1152 rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); 1153 1154 value &= ~HWP_MIN_PERF(~0L); 1155 value |= HWP_MIN_PERF(min); 1156 1157 value &= ~HWP_MAX_PERF(~0L); 1158 value |= HWP_MAX_PERF(max); 1159 1160 if (cpu_data->epp_policy == cpu_data->policy) 1161 goto skip_epp; 1162 1163 cpu_data->epp_policy = cpu_data->policy; 1164 1165 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) { 1166 epp = intel_pstate_get_epp(cpu_data, value); 1167 cpu_data->epp_powersave = epp; 1168 /* If EPP read was failed, then don't try to write */ 1169 if (epp < 0) 1170 goto skip_epp; 1171 1172 epp = 0; 1173 } else { 1174 /* skip setting EPP, when saved value is invalid */ 1175 if (cpu_data->epp_powersave < 0) 1176 goto skip_epp; 1177 1178 /* 1179 * No need to restore EPP when it is not zero. This 1180 * means: 1181 * - Policy is not changed 1182 * - user has manually changed 1183 * - Error reading EPB 1184 */ 1185 epp = intel_pstate_get_epp(cpu_data, value); 1186 if (epp) 1187 goto skip_epp; 1188 1189 epp = cpu_data->epp_powersave; 1190 } 1191 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { 1192 value &= ~GENMASK_ULL(31, 24); 1193 value |= (u64)epp << 24; 1194 } else { 1195 intel_pstate_set_epb(cpu, epp); 1196 } 1197 skip_epp: 1198 WRITE_ONCE(cpu_data->hwp_req_cached, value); 1199 wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value); 1200 } 1201 1202 static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata); 1203 1204 static void intel_pstate_hwp_offline(struct cpudata *cpu) 1205 { 1206 u64 value = READ_ONCE(cpu->hwp_req_cached); 1207 int min_perf; 1208 1209 intel_pstate_disable_hwp_interrupt(cpu); 1210 1211 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) { 1212 /* 1213 * In case the EPP has been set to "performance" by the 1214 * active mode "performance" scaling algorithm, replace that 1215 * temporary value with the cached EPP one. 1216 */ 1217 value &= ~GENMASK_ULL(31, 24); 1218 value |= HWP_ENERGY_PERF_PREFERENCE(cpu->epp_cached); 1219 /* 1220 * However, make sure that EPP will be set to "performance" when 1221 * the CPU is brought back online again and the "performance" 1222 * scaling algorithm is still in effect. 1223 */ 1224 cpu->epp_policy = CPUFREQ_POLICY_UNKNOWN; 1225 } 1226 1227 /* 1228 * Clear the desired perf field in the cached HWP request value to 1229 * prevent nonzero desired values from being leaked into the active 1230 * mode. 1231 */ 1232 value &= ~HWP_DESIRED_PERF(~0L); 1233 WRITE_ONCE(cpu->hwp_req_cached, value); 1234 1235 value &= ~GENMASK_ULL(31, 0); 1236 min_perf = HWP_LOWEST_PERF(READ_ONCE(cpu->hwp_cap_cached)); 1237 1238 /* Set hwp_max = hwp_min */ 1239 value |= HWP_MAX_PERF(min_perf); 1240 value |= HWP_MIN_PERF(min_perf); 1241 1242 /* Set EPP to min */ 1243 if (boot_cpu_has(X86_FEATURE_HWP_EPP)) 1244 value |= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE); 1245 1246 wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); 1247 1248 mutex_lock(&hybrid_capacity_lock); 1249 1250 if (!hybrid_max_perf_cpu) { 1251 mutex_unlock(&hybrid_capacity_lock); 1252 1253 return; 1254 } 1255 1256 if (hybrid_max_perf_cpu == cpu) 1257 hybrid_update_cpu_capacity_scaling(); 1258 1259 mutex_unlock(&hybrid_capacity_lock); 1260 1261 /* Reset the capacity of the CPU going offline to the initial value. */ 1262 hybrid_clear_cpu_capacity(cpu->cpu); 1263 } 1264 1265 #define POWER_CTL_EE_ENABLE 1 1266 #define POWER_CTL_EE_DISABLE 2 1267 1268 static int power_ctl_ee_state; 1269 1270 static void set_power_ctl_ee_state(bool input) 1271 { 1272 u64 power_ctl; 1273 1274 mutex_lock(&intel_pstate_driver_lock); 1275 rdmsrl(MSR_IA32_POWER_CTL, power_ctl); 1276 if (input) { 1277 power_ctl &= ~BIT(MSR_IA32_POWER_CTL_BIT_EE); 1278 power_ctl_ee_state = POWER_CTL_EE_ENABLE; 1279 } else { 1280 power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE); 1281 power_ctl_ee_state = POWER_CTL_EE_DISABLE; 1282 } 1283 wrmsrl(MSR_IA32_POWER_CTL, power_ctl); 1284 mutex_unlock(&intel_pstate_driver_lock); 1285 } 1286 1287 static void intel_pstate_hwp_enable(struct cpudata *cpudata); 1288 1289 static void intel_pstate_hwp_reenable(struct cpudata *cpu) 1290 { 1291 intel_pstate_hwp_enable(cpu); 1292 wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, READ_ONCE(cpu->hwp_req_cached)); 1293 } 1294 1295 static int intel_pstate_suspend(struct cpufreq_policy *policy) 1296 { 1297 struct cpudata *cpu = all_cpu_data[policy->cpu]; 1298 1299 pr_debug("CPU %d suspending\n", cpu->cpu); 1300 1301 cpu->suspended = true; 1302 1303 /* disable HWP interrupt and cancel any pending work */ 1304 intel_pstate_disable_hwp_interrupt(cpu); 1305 1306 return 0; 1307 } 1308 1309 static int intel_pstate_resume(struct cpufreq_policy *policy) 1310 { 1311 struct cpudata *cpu = all_cpu_data[policy->cpu]; 1312 1313 pr_debug("CPU %d resuming\n", cpu->cpu); 1314 1315 /* Only restore if the system default is changed */ 1316 if (power_ctl_ee_state == POWER_CTL_EE_ENABLE) 1317 set_power_ctl_ee_state(true); 1318 else if (power_ctl_ee_state == POWER_CTL_EE_DISABLE) 1319 set_power_ctl_ee_state(false); 1320 1321 if (cpu->suspended && hwp_active) { 1322 mutex_lock(&intel_pstate_limits_lock); 1323 1324 /* Re-enable HWP, because "online" has not done that. */ 1325 intel_pstate_hwp_reenable(cpu); 1326 1327 mutex_unlock(&intel_pstate_limits_lock); 1328 } 1329 1330 cpu->suspended = false; 1331 1332 return 0; 1333 } 1334 1335 static void intel_pstate_update_policies(void) 1336 { 1337 int cpu; 1338 1339 for_each_possible_cpu(cpu) 1340 cpufreq_update_policy(cpu); 1341 } 1342 1343 static void __intel_pstate_update_max_freq(struct cpudata *cpudata, 1344 struct cpufreq_policy *policy) 1345 { 1346 if (hwp_active) 1347 intel_pstate_get_hwp_cap(cpudata); 1348 1349 policy->cpuinfo.max_freq = READ_ONCE(global.no_turbo) ? 1350 cpudata->pstate.max_freq : cpudata->pstate.turbo_freq; 1351 1352 refresh_frequency_limits(policy); 1353 } 1354 1355 static void intel_pstate_update_limits(unsigned int cpu) 1356 { 1357 struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); 1358 struct cpudata *cpudata; 1359 1360 if (!policy) 1361 return; 1362 1363 cpudata = all_cpu_data[cpu]; 1364 1365 __intel_pstate_update_max_freq(cpudata, policy); 1366 1367 /* Prevent the driver from being unregistered now. */ 1368 mutex_lock(&intel_pstate_driver_lock); 1369 1370 cpufreq_cpu_release(policy); 1371 1372 hybrid_update_capacity(cpudata); 1373 1374 mutex_unlock(&intel_pstate_driver_lock); 1375 } 1376 1377 static void intel_pstate_update_limits_for_all(void) 1378 { 1379 int cpu; 1380 1381 for_each_possible_cpu(cpu) { 1382 struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); 1383 1384 if (!policy) 1385 continue; 1386 1387 __intel_pstate_update_max_freq(all_cpu_data[cpu], policy); 1388 1389 cpufreq_cpu_release(policy); 1390 } 1391 1392 mutex_lock(&hybrid_capacity_lock); 1393 1394 if (hybrid_max_perf_cpu) 1395 __hybrid_init_cpu_capacity_scaling(); 1396 1397 mutex_unlock(&hybrid_capacity_lock); 1398 } 1399 1400 /************************** sysfs begin ************************/ 1401 #define show_one(file_name, object) \ 1402 static ssize_t show_##file_name \ 1403 (struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ 1404 { \ 1405 return sprintf(buf, "%u\n", global.object); \ 1406 } 1407 1408 static ssize_t intel_pstate_show_status(char *buf); 1409 static int intel_pstate_update_status(const char *buf, size_t size); 1410 1411 static ssize_t show_status(struct kobject *kobj, 1412 struct kobj_attribute *attr, char *buf) 1413 { 1414 ssize_t ret; 1415 1416 mutex_lock(&intel_pstate_driver_lock); 1417 ret = intel_pstate_show_status(buf); 1418 mutex_unlock(&intel_pstate_driver_lock); 1419 1420 return ret; 1421 } 1422 1423 static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, 1424 const char *buf, size_t count) 1425 { 1426 char *p = memchr(buf, '\n', count); 1427 int ret; 1428 1429 mutex_lock(&intel_pstate_driver_lock); 1430 ret = intel_pstate_update_status(buf, p ? p - buf : count); 1431 mutex_unlock(&intel_pstate_driver_lock); 1432 1433 return ret < 0 ? ret : count; 1434 } 1435 1436 static ssize_t show_turbo_pct(struct kobject *kobj, 1437 struct kobj_attribute *attr, char *buf) 1438 { 1439 struct cpudata *cpu; 1440 int total, no_turbo, turbo_pct; 1441 uint32_t turbo_fp; 1442 1443 mutex_lock(&intel_pstate_driver_lock); 1444 1445 if (!intel_pstate_driver) { 1446 mutex_unlock(&intel_pstate_driver_lock); 1447 return -EAGAIN; 1448 } 1449 1450 cpu = all_cpu_data[0]; 1451 1452 total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1; 1453 no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1; 1454 turbo_fp = div_fp(no_turbo, total); 1455 turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100))); 1456 1457 mutex_unlock(&intel_pstate_driver_lock); 1458 1459 return sprintf(buf, "%u\n", turbo_pct); 1460 } 1461 1462 static ssize_t show_num_pstates(struct kobject *kobj, 1463 struct kobj_attribute *attr, char *buf) 1464 { 1465 struct cpudata *cpu; 1466 int total; 1467 1468 mutex_lock(&intel_pstate_driver_lock); 1469 1470 if (!intel_pstate_driver) { 1471 mutex_unlock(&intel_pstate_driver_lock); 1472 return -EAGAIN; 1473 } 1474 1475 cpu = all_cpu_data[0]; 1476 total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1; 1477 1478 mutex_unlock(&intel_pstate_driver_lock); 1479 1480 return sprintf(buf, "%u\n", total); 1481 } 1482 1483 static ssize_t show_no_turbo(struct kobject *kobj, 1484 struct kobj_attribute *attr, char *buf) 1485 { 1486 ssize_t ret; 1487 1488 mutex_lock(&intel_pstate_driver_lock); 1489 1490 if (!intel_pstate_driver) { 1491 mutex_unlock(&intel_pstate_driver_lock); 1492 return -EAGAIN; 1493 } 1494 1495 ret = sprintf(buf, "%u\n", global.no_turbo); 1496 1497 mutex_unlock(&intel_pstate_driver_lock); 1498 1499 return ret; 1500 } 1501 1502 static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b, 1503 const char *buf, size_t count) 1504 { 1505 unsigned int input; 1506 bool no_turbo; 1507 1508 if (sscanf(buf, "%u", &input) != 1) 1509 return -EINVAL; 1510 1511 mutex_lock(&intel_pstate_driver_lock); 1512 1513 if (!intel_pstate_driver) { 1514 count = -EAGAIN; 1515 goto unlock_driver; 1516 } 1517 1518 no_turbo = !!clamp_t(int, input, 0, 1); 1519 1520 WRITE_ONCE(global.turbo_disabled, turbo_is_disabled()); 1521 if (global.turbo_disabled && !no_turbo) { 1522 pr_notice("Turbo disabled by BIOS or unavailable on processor\n"); 1523 count = -EPERM; 1524 if (global.no_turbo) 1525 goto unlock_driver; 1526 else 1527 no_turbo = 1; 1528 } 1529 1530 if (no_turbo == global.no_turbo) { 1531 goto unlock_driver; 1532 } 1533 1534 WRITE_ONCE(global.no_turbo, no_turbo); 1535 1536 mutex_lock(&intel_pstate_limits_lock); 1537 1538 if (no_turbo) { 1539 struct cpudata *cpu = all_cpu_data[0]; 1540 int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate; 1541 1542 /* Squash the global minimum into the permitted range. */ 1543 if (global.min_perf_pct > pct) 1544 global.min_perf_pct = pct; 1545 } 1546 1547 mutex_unlock(&intel_pstate_limits_lock); 1548 1549 intel_pstate_update_limits_for_all(); 1550 arch_set_max_freq_ratio(no_turbo); 1551 1552 unlock_driver: 1553 mutex_unlock(&intel_pstate_driver_lock); 1554 1555 return count; 1556 } 1557 1558 static void update_qos_request(enum freq_qos_req_type type) 1559 { 1560 struct freq_qos_request *req; 1561 struct cpufreq_policy *policy; 1562 int i; 1563 1564 for_each_possible_cpu(i) { 1565 struct cpudata *cpu = all_cpu_data[i]; 1566 unsigned int freq, perf_pct; 1567 1568 policy = cpufreq_cpu_get(i); 1569 if (!policy) 1570 continue; 1571 1572 req = policy->driver_data; 1573 cpufreq_cpu_put(policy); 1574 1575 if (!req) 1576 continue; 1577 1578 if (hwp_active) 1579 intel_pstate_get_hwp_cap(cpu); 1580 1581 if (type == FREQ_QOS_MIN) { 1582 perf_pct = global.min_perf_pct; 1583 } else { 1584 req++; 1585 perf_pct = global.max_perf_pct; 1586 } 1587 1588 freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * perf_pct, 100); 1589 1590 if (freq_qos_update_request(req, freq) < 0) 1591 pr_warn("Failed to update freq constraint: CPU%d\n", i); 1592 } 1593 } 1594 1595 static ssize_t store_max_perf_pct(struct kobject *a, struct kobj_attribute *b, 1596 const char *buf, size_t count) 1597 { 1598 unsigned int input; 1599 int ret; 1600 1601 ret = sscanf(buf, "%u", &input); 1602 if (ret != 1) 1603 return -EINVAL; 1604 1605 mutex_lock(&intel_pstate_driver_lock); 1606 1607 if (!intel_pstate_driver) { 1608 mutex_unlock(&intel_pstate_driver_lock); 1609 return -EAGAIN; 1610 } 1611 1612 mutex_lock(&intel_pstate_limits_lock); 1613 1614 global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100); 1615 1616 mutex_unlock(&intel_pstate_limits_lock); 1617 1618 if (intel_pstate_driver == &intel_pstate) 1619 intel_pstate_update_policies(); 1620 else 1621 update_qos_request(FREQ_QOS_MAX); 1622 1623 mutex_unlock(&intel_pstate_driver_lock); 1624 1625 return count; 1626 } 1627 1628 static ssize_t store_min_perf_pct(struct kobject *a, struct kobj_attribute *b, 1629 const char *buf, size_t count) 1630 { 1631 unsigned int input; 1632 int ret; 1633 1634 ret = sscanf(buf, "%u", &input); 1635 if (ret != 1) 1636 return -EINVAL; 1637 1638 mutex_lock(&intel_pstate_driver_lock); 1639 1640 if (!intel_pstate_driver) { 1641 mutex_unlock(&intel_pstate_driver_lock); 1642 return -EAGAIN; 1643 } 1644 1645 mutex_lock(&intel_pstate_limits_lock); 1646 1647 global.min_perf_pct = clamp_t(int, input, 1648 min_perf_pct_min(), global.max_perf_pct); 1649 1650 mutex_unlock(&intel_pstate_limits_lock); 1651 1652 if (intel_pstate_driver == &intel_pstate) 1653 intel_pstate_update_policies(); 1654 else 1655 update_qos_request(FREQ_QOS_MIN); 1656 1657 mutex_unlock(&intel_pstate_driver_lock); 1658 1659 return count; 1660 } 1661 1662 static ssize_t show_hwp_dynamic_boost(struct kobject *kobj, 1663 struct kobj_attribute *attr, char *buf) 1664 { 1665 return sprintf(buf, "%u\n", hwp_boost); 1666 } 1667 1668 static ssize_t store_hwp_dynamic_boost(struct kobject *a, 1669 struct kobj_attribute *b, 1670 const char *buf, size_t count) 1671 { 1672 unsigned int input; 1673 int ret; 1674 1675 ret = kstrtouint(buf, 10, &input); 1676 if (ret) 1677 return ret; 1678 1679 mutex_lock(&intel_pstate_driver_lock); 1680 hwp_boost = !!input; 1681 intel_pstate_update_policies(); 1682 mutex_unlock(&intel_pstate_driver_lock); 1683 1684 return count; 1685 } 1686 1687 static ssize_t show_energy_efficiency(struct kobject *kobj, struct kobj_attribute *attr, 1688 char *buf) 1689 { 1690 u64 power_ctl; 1691 int enable; 1692 1693 rdmsrl(MSR_IA32_POWER_CTL, power_ctl); 1694 enable = !!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE)); 1695 return sprintf(buf, "%d\n", !enable); 1696 } 1697 1698 static ssize_t store_energy_efficiency(struct kobject *a, struct kobj_attribute *b, 1699 const char *buf, size_t count) 1700 { 1701 bool input; 1702 int ret; 1703 1704 ret = kstrtobool(buf, &input); 1705 if (ret) 1706 return ret; 1707 1708 set_power_ctl_ee_state(input); 1709 1710 return count; 1711 } 1712 1713 show_one(max_perf_pct, max_perf_pct); 1714 show_one(min_perf_pct, min_perf_pct); 1715 1716 define_one_global_rw(status); 1717 define_one_global_rw(no_turbo); 1718 define_one_global_rw(max_perf_pct); 1719 define_one_global_rw(min_perf_pct); 1720 define_one_global_ro(turbo_pct); 1721 define_one_global_ro(num_pstates); 1722 define_one_global_rw(hwp_dynamic_boost); 1723 define_one_global_rw(energy_efficiency); 1724 1725 static struct attribute *intel_pstate_attributes[] = { 1726 &status.attr, 1727 &no_turbo.attr, 1728 NULL 1729 }; 1730 1731 static const struct attribute_group intel_pstate_attr_group = { 1732 .attrs = intel_pstate_attributes, 1733 }; 1734 1735 static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[]; 1736 1737 static struct kobject *intel_pstate_kobject; 1738 1739 static void __init intel_pstate_sysfs_expose_params(void) 1740 { 1741 struct device *dev_root = bus_get_dev_root(&cpu_subsys); 1742 int rc; 1743 1744 if (dev_root) { 1745 intel_pstate_kobject = kobject_create_and_add("intel_pstate", &dev_root->kobj); 1746 put_device(dev_root); 1747 } 1748 if (WARN_ON(!intel_pstate_kobject)) 1749 return; 1750 1751 rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group); 1752 if (WARN_ON(rc)) 1753 return; 1754 1755 if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) { 1756 rc = sysfs_create_file(intel_pstate_kobject, &turbo_pct.attr); 1757 WARN_ON(rc); 1758 1759 rc = sysfs_create_file(intel_pstate_kobject, &num_pstates.attr); 1760 WARN_ON(rc); 1761 } 1762 1763 /* 1764 * If per cpu limits are enforced there are no global limits, so 1765 * return without creating max/min_perf_pct attributes 1766 */ 1767 if (per_cpu_limits) 1768 return; 1769 1770 rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr); 1771 WARN_ON(rc); 1772 1773 rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr); 1774 WARN_ON(rc); 1775 1776 if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) { 1777 rc = sysfs_create_file(intel_pstate_kobject, &energy_efficiency.attr); 1778 WARN_ON(rc); 1779 } 1780 } 1781 1782 static void __init intel_pstate_sysfs_remove(void) 1783 { 1784 if (!intel_pstate_kobject) 1785 return; 1786 1787 sysfs_remove_group(intel_pstate_kobject, &intel_pstate_attr_group); 1788 1789 if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) { 1790 sysfs_remove_file(intel_pstate_kobject, &num_pstates.attr); 1791 sysfs_remove_file(intel_pstate_kobject, &turbo_pct.attr); 1792 } 1793 1794 if (!per_cpu_limits) { 1795 sysfs_remove_file(intel_pstate_kobject, &max_perf_pct.attr); 1796 sysfs_remove_file(intel_pstate_kobject, &min_perf_pct.attr); 1797 1798 if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) 1799 sysfs_remove_file(intel_pstate_kobject, &energy_efficiency.attr); 1800 } 1801 1802 kobject_put(intel_pstate_kobject); 1803 } 1804 1805 static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void) 1806 { 1807 int rc; 1808 1809 if (!hwp_active) 1810 return; 1811 1812 rc = sysfs_create_file(intel_pstate_kobject, &hwp_dynamic_boost.attr); 1813 WARN_ON_ONCE(rc); 1814 } 1815 1816 static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void) 1817 { 1818 if (!hwp_active) 1819 return; 1820 1821 sysfs_remove_file(intel_pstate_kobject, &hwp_dynamic_boost.attr); 1822 } 1823 1824 /************************** sysfs end ************************/ 1825 1826 static void intel_pstate_notify_work(struct work_struct *work) 1827 { 1828 struct cpudata *cpudata = 1829 container_of(to_delayed_work(work), struct cpudata, hwp_notify_work); 1830 struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpudata->cpu); 1831 1832 if (policy) { 1833 __intel_pstate_update_max_freq(cpudata, policy); 1834 1835 cpufreq_cpu_release(policy); 1836 1837 /* 1838 * The driver will not be unregistered while this function is 1839 * running, so update the capacity without acquiring the driver 1840 * lock. 1841 */ 1842 hybrid_update_capacity(cpudata); 1843 } 1844 1845 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0); 1846 } 1847 1848 static DEFINE_SPINLOCK(hwp_notify_lock); 1849 static cpumask_t hwp_intr_enable_mask; 1850 1851 #define HWP_GUARANTEED_PERF_CHANGE_STATUS BIT(0) 1852 #define HWP_HIGHEST_PERF_CHANGE_STATUS BIT(3) 1853 1854 void notify_hwp_interrupt(void) 1855 { 1856 unsigned int this_cpu = smp_processor_id(); 1857 u64 value, status_mask; 1858 unsigned long flags; 1859 1860 if (!hwp_active || !cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY)) 1861 return; 1862 1863 status_mask = HWP_GUARANTEED_PERF_CHANGE_STATUS; 1864 if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE)) 1865 status_mask |= HWP_HIGHEST_PERF_CHANGE_STATUS; 1866 1867 rdmsrl_safe(MSR_HWP_STATUS, &value); 1868 if (!(value & status_mask)) 1869 return; 1870 1871 spin_lock_irqsave(&hwp_notify_lock, flags); 1872 1873 if (!cpumask_test_cpu(this_cpu, &hwp_intr_enable_mask)) 1874 goto ack_intr; 1875 1876 schedule_delayed_work(&all_cpu_data[this_cpu]->hwp_notify_work, 1877 msecs_to_jiffies(10)); 1878 1879 spin_unlock_irqrestore(&hwp_notify_lock, flags); 1880 1881 return; 1882 1883 ack_intr: 1884 wrmsrl_safe(MSR_HWP_STATUS, 0); 1885 spin_unlock_irqrestore(&hwp_notify_lock, flags); 1886 } 1887 1888 static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata) 1889 { 1890 bool cancel_work; 1891 1892 if (!cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY)) 1893 return; 1894 1895 /* wrmsrl_on_cpu has to be outside spinlock as this can result in IPC */ 1896 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); 1897 1898 spin_lock_irq(&hwp_notify_lock); 1899 cancel_work = cpumask_test_and_clear_cpu(cpudata->cpu, &hwp_intr_enable_mask); 1900 spin_unlock_irq(&hwp_notify_lock); 1901 1902 if (cancel_work) 1903 cancel_delayed_work_sync(&cpudata->hwp_notify_work); 1904 } 1905 1906 #define HWP_GUARANTEED_PERF_CHANGE_REQ BIT(0) 1907 #define HWP_HIGHEST_PERF_CHANGE_REQ BIT(2) 1908 1909 static void intel_pstate_enable_hwp_interrupt(struct cpudata *cpudata) 1910 { 1911 /* Enable HWP notification interrupt for performance change */ 1912 if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY)) { 1913 u64 interrupt_mask = HWP_GUARANTEED_PERF_CHANGE_REQ; 1914 1915 spin_lock_irq(&hwp_notify_lock); 1916 INIT_DELAYED_WORK(&cpudata->hwp_notify_work, intel_pstate_notify_work); 1917 cpumask_set_cpu(cpudata->cpu, &hwp_intr_enable_mask); 1918 spin_unlock_irq(&hwp_notify_lock); 1919 1920 if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE)) 1921 interrupt_mask |= HWP_HIGHEST_PERF_CHANGE_REQ; 1922 1923 /* wrmsrl_on_cpu has to be outside spinlock as this can result in IPC */ 1924 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, interrupt_mask); 1925 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0); 1926 } 1927 } 1928 1929 static void intel_pstate_update_epp_defaults(struct cpudata *cpudata) 1930 { 1931 cpudata->epp_default = intel_pstate_get_epp(cpudata, 0); 1932 1933 /* 1934 * If the EPP is set by firmware, which means that firmware enabled HWP 1935 * - Is equal or less than 0x80 (default balance_perf EPP) 1936 * - But less performance oriented than performance EPP 1937 * then use this as new balance_perf EPP. 1938 */ 1939 if (hwp_forced && cpudata->epp_default <= HWP_EPP_BALANCE_PERFORMANCE && 1940 cpudata->epp_default > HWP_EPP_PERFORMANCE) { 1941 epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = cpudata->epp_default; 1942 return; 1943 } 1944 1945 /* 1946 * If this CPU gen doesn't call for change in balance_perf 1947 * EPP return. 1948 */ 1949 if (epp_values[EPP_INDEX_BALANCE_PERFORMANCE] == HWP_EPP_BALANCE_PERFORMANCE) 1950 return; 1951 1952 /* 1953 * Use hard coded value per gen to update the balance_perf 1954 * and default EPP. 1955 */ 1956 cpudata->epp_default = epp_values[EPP_INDEX_BALANCE_PERFORMANCE]; 1957 intel_pstate_set_epp(cpudata, cpudata->epp_default); 1958 } 1959 1960 static void intel_pstate_hwp_enable(struct cpudata *cpudata) 1961 { 1962 /* First disable HWP notification interrupt till we activate again */ 1963 if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY)) 1964 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); 1965 1966 wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); 1967 1968 intel_pstate_enable_hwp_interrupt(cpudata); 1969 1970 if (cpudata->epp_default >= 0) 1971 return; 1972 1973 intel_pstate_update_epp_defaults(cpudata); 1974 } 1975 1976 static int atom_get_min_pstate(int not_used) 1977 { 1978 u64 value; 1979 1980 rdmsrl(MSR_ATOM_CORE_RATIOS, value); 1981 return (value >> 8) & 0x7F; 1982 } 1983 1984 static int atom_get_max_pstate(int not_used) 1985 { 1986 u64 value; 1987 1988 rdmsrl(MSR_ATOM_CORE_RATIOS, value); 1989 return (value >> 16) & 0x7F; 1990 } 1991 1992 static int atom_get_turbo_pstate(int not_used) 1993 { 1994 u64 value; 1995 1996 rdmsrl(MSR_ATOM_CORE_TURBO_RATIOS, value); 1997 return value & 0x7F; 1998 } 1999 2000 static u64 atom_get_val(struct cpudata *cpudata, int pstate) 2001 { 2002 u64 val; 2003 int32_t vid_fp; 2004 u32 vid; 2005 2006 val = (u64)pstate << 8; 2007 if (READ_ONCE(global.no_turbo) && !READ_ONCE(global.turbo_disabled)) 2008 val |= (u64)1 << 32; 2009 2010 vid_fp = cpudata->vid.min + mul_fp( 2011 int_tofp(pstate - cpudata->pstate.min_pstate), 2012 cpudata->vid.ratio); 2013 2014 vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max); 2015 vid = ceiling_fp(vid_fp); 2016 2017 if (pstate > cpudata->pstate.max_pstate) 2018 vid = cpudata->vid.turbo; 2019 2020 return val | vid; 2021 } 2022 2023 static int silvermont_get_scaling(void) 2024 { 2025 u64 value; 2026 int i; 2027 /* Defined in Table 35-6 from SDM (Sept 2015) */ 2028 static int silvermont_freq_table[] = { 2029 83300, 100000, 133300, 116700, 80000}; 2030 2031 rdmsrl(MSR_FSB_FREQ, value); 2032 i = value & 0x7; 2033 WARN_ON(i > 4); 2034 2035 return silvermont_freq_table[i]; 2036 } 2037 2038 static int airmont_get_scaling(void) 2039 { 2040 u64 value; 2041 int i; 2042 /* Defined in Table 35-10 from SDM (Sept 2015) */ 2043 static int airmont_freq_table[] = { 2044 83300, 100000, 133300, 116700, 80000, 2045 93300, 90000, 88900, 87500}; 2046 2047 rdmsrl(MSR_FSB_FREQ, value); 2048 i = value & 0xF; 2049 WARN_ON(i > 8); 2050 2051 return airmont_freq_table[i]; 2052 } 2053 2054 static void atom_get_vid(struct cpudata *cpudata) 2055 { 2056 u64 value; 2057 2058 rdmsrl(MSR_ATOM_CORE_VIDS, value); 2059 cpudata->vid.min = int_tofp((value >> 8) & 0x7f); 2060 cpudata->vid.max = int_tofp((value >> 16) & 0x7f); 2061 cpudata->vid.ratio = div_fp( 2062 cpudata->vid.max - cpudata->vid.min, 2063 int_tofp(cpudata->pstate.max_pstate - 2064 cpudata->pstate.min_pstate)); 2065 2066 rdmsrl(MSR_ATOM_CORE_TURBO_VIDS, value); 2067 cpudata->vid.turbo = value & 0x7f; 2068 } 2069 2070 static int core_get_min_pstate(int cpu) 2071 { 2072 u64 value; 2073 2074 rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &value); 2075 return (value >> 40) & 0xFF; 2076 } 2077 2078 static int core_get_max_pstate_physical(int cpu) 2079 { 2080 u64 value; 2081 2082 rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &value); 2083 return (value >> 8) & 0xFF; 2084 } 2085 2086 static int core_get_tdp_ratio(int cpu, u64 plat_info) 2087 { 2088 /* Check how many TDP levels present */ 2089 if (plat_info & 0x600000000) { 2090 u64 tdp_ctrl; 2091 u64 tdp_ratio; 2092 int tdp_msr; 2093 int err; 2094 2095 /* Get the TDP level (0, 1, 2) to get ratios */ 2096 err = rdmsrl_safe_on_cpu(cpu, MSR_CONFIG_TDP_CONTROL, &tdp_ctrl); 2097 if (err) 2098 return err; 2099 2100 /* TDP MSR are continuous starting at 0x648 */ 2101 tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03); 2102 err = rdmsrl_safe_on_cpu(cpu, tdp_msr, &tdp_ratio); 2103 if (err) 2104 return err; 2105 2106 /* For level 1 and 2, bits[23:16] contain the ratio */ 2107 if (tdp_ctrl & 0x03) 2108 tdp_ratio >>= 16; 2109 2110 tdp_ratio &= 0xff; /* ratios are only 8 bits long */ 2111 pr_debug("tdp_ratio %x\n", (int)tdp_ratio); 2112 2113 return (int)tdp_ratio; 2114 } 2115 2116 return -ENXIO; 2117 } 2118 2119 static int core_get_max_pstate(int cpu) 2120 { 2121 u64 tar; 2122 u64 plat_info; 2123 int max_pstate; 2124 int tdp_ratio; 2125 int err; 2126 2127 rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &plat_info); 2128 max_pstate = (plat_info >> 8) & 0xFF; 2129 2130 tdp_ratio = core_get_tdp_ratio(cpu, plat_info); 2131 if (tdp_ratio <= 0) 2132 return max_pstate; 2133 2134 if (hwp_active) { 2135 /* Turbo activation ratio is not used on HWP platforms */ 2136 return tdp_ratio; 2137 } 2138 2139 err = rdmsrl_safe_on_cpu(cpu, MSR_TURBO_ACTIVATION_RATIO, &tar); 2140 if (!err) { 2141 int tar_levels; 2142 2143 /* Do some sanity checking for safety */ 2144 tar_levels = tar & 0xff; 2145 if (tdp_ratio - 1 == tar_levels) { 2146 max_pstate = tar_levels; 2147 pr_debug("max_pstate=TAC %x\n", max_pstate); 2148 } 2149 } 2150 2151 return max_pstate; 2152 } 2153 2154 static int core_get_turbo_pstate(int cpu) 2155 { 2156 u64 value; 2157 int nont, ret; 2158 2159 rdmsrl_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value); 2160 nont = core_get_max_pstate(cpu); 2161 ret = (value) & 255; 2162 if (ret <= nont) 2163 ret = nont; 2164 return ret; 2165 } 2166 2167 static u64 core_get_val(struct cpudata *cpudata, int pstate) 2168 { 2169 u64 val; 2170 2171 val = (u64)pstate << 8; 2172 if (READ_ONCE(global.no_turbo) && !READ_ONCE(global.turbo_disabled)) 2173 val |= (u64)1 << 32; 2174 2175 return val; 2176 } 2177 2178 static int knl_get_aperf_mperf_shift(void) 2179 { 2180 return 10; 2181 } 2182 2183 static int knl_get_turbo_pstate(int cpu) 2184 { 2185 u64 value; 2186 int nont, ret; 2187 2188 rdmsrl_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value); 2189 nont = core_get_max_pstate(cpu); 2190 ret = (((value) >> 8) & 0xFF); 2191 if (ret <= nont) 2192 ret = nont; 2193 return ret; 2194 } 2195 2196 static void hybrid_get_type(void *data) 2197 { 2198 u8 *cpu_type = data; 2199 2200 *cpu_type = get_this_hybrid_cpu_type(); 2201 } 2202 2203 static int hwp_get_cpu_scaling(int cpu) 2204 { 2205 u8 cpu_type = 0; 2206 2207 smp_call_function_single(cpu, hybrid_get_type, &cpu_type, 1); 2208 /* P-cores have a smaller perf level-to-freqency scaling factor. */ 2209 if (cpu_type == 0x40) 2210 return hybrid_scaling_factor; 2211 2212 /* Use default core scaling for E-cores */ 2213 if (cpu_type == 0x20) 2214 return core_get_scaling(); 2215 2216 /* 2217 * If reached here, this system is either non-hybrid (like Tiger 2218 * Lake) or hybrid-capable (like Alder Lake or Raptor Lake) with 2219 * no E cores (in which case CPUID for hybrid support is 0). 2220 * 2221 * The CPPC nominal_frequency field is 0 for non-hybrid systems, 2222 * so the default core scaling will be used for them. 2223 */ 2224 return intel_pstate_cppc_get_scaling(cpu); 2225 } 2226 2227 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) 2228 { 2229 trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); 2230 cpu->pstate.current_pstate = pstate; 2231 /* 2232 * Generally, there is no guarantee that this code will always run on 2233 * the CPU being updated, so force the register update to run on the 2234 * right CPU. 2235 */ 2236 wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, 2237 pstate_funcs.get_val(cpu, pstate)); 2238 } 2239 2240 static void intel_pstate_set_min_pstate(struct cpudata *cpu) 2241 { 2242 intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate); 2243 } 2244 2245 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) 2246 { 2247 int perf_ctl_max_phys = pstate_funcs.get_max_physical(cpu->cpu); 2248 int perf_ctl_scaling = pstate_funcs.get_scaling(); 2249 2250 cpu->pstate.min_pstate = pstate_funcs.get_min(cpu->cpu); 2251 cpu->pstate.max_pstate_physical = perf_ctl_max_phys; 2252 cpu->pstate.perf_ctl_scaling = perf_ctl_scaling; 2253 2254 if (hwp_active && !hwp_mode_bdw) { 2255 __intel_pstate_get_hwp_cap(cpu); 2256 2257 if (pstate_funcs.get_cpu_scaling) { 2258 cpu->pstate.scaling = pstate_funcs.get_cpu_scaling(cpu->cpu); 2259 if (cpu->pstate.scaling != perf_ctl_scaling) { 2260 intel_pstate_hybrid_hwp_adjust(cpu); 2261 hwp_is_hybrid = true; 2262 } 2263 } else { 2264 cpu->pstate.scaling = perf_ctl_scaling; 2265 } 2266 } else { 2267 cpu->pstate.scaling = perf_ctl_scaling; 2268 cpu->pstate.max_pstate = pstate_funcs.get_max(cpu->cpu); 2269 cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(cpu->cpu); 2270 } 2271 2272 if (cpu->pstate.scaling == perf_ctl_scaling) { 2273 cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling; 2274 cpu->pstate.max_freq = cpu->pstate.max_pstate * perf_ctl_scaling; 2275 cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * perf_ctl_scaling; 2276 } 2277 2278 if (pstate_funcs.get_aperf_mperf_shift) 2279 cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift(); 2280 2281 if (pstate_funcs.get_vid) 2282 pstate_funcs.get_vid(cpu); 2283 2284 intel_pstate_set_min_pstate(cpu); 2285 } 2286 2287 /* 2288 * Long hold time will keep high perf limits for long time, 2289 * which negatively impacts perf/watt for some workloads, 2290 * like specpower. 3ms is based on experiements on some 2291 * workoads. 2292 */ 2293 static int hwp_boost_hold_time_ns = 3 * NSEC_PER_MSEC; 2294 2295 static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu) 2296 { 2297 u64 hwp_req = READ_ONCE(cpu->hwp_req_cached); 2298 u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached); 2299 u32 max_limit = (hwp_req & 0xff00) >> 8; 2300 u32 min_limit = (hwp_req & 0xff); 2301 u32 boost_level1; 2302 2303 /* 2304 * Cases to consider (User changes via sysfs or boot time): 2305 * If, P0 (Turbo max) = P1 (Guaranteed max) = min: 2306 * No boost, return. 2307 * If, P0 (Turbo max) > P1 (Guaranteed max) = min: 2308 * Should result in one level boost only for P0. 2309 * If, P0 (Turbo max) = P1 (Guaranteed max) > min: 2310 * Should result in two level boost: 2311 * (min + p1)/2 and P1. 2312 * If, P0 (Turbo max) > P1 (Guaranteed max) > min: 2313 * Should result in three level boost: 2314 * (min + p1)/2, P1 and P0. 2315 */ 2316 2317 /* If max and min are equal or already at max, nothing to boost */ 2318 if (max_limit == min_limit || cpu->hwp_boost_min >= max_limit) 2319 return; 2320 2321 if (!cpu->hwp_boost_min) 2322 cpu->hwp_boost_min = min_limit; 2323 2324 /* level at half way mark between min and guranteed */ 2325 boost_level1 = (HWP_GUARANTEED_PERF(hwp_cap) + min_limit) >> 1; 2326 2327 if (cpu->hwp_boost_min < boost_level1) 2328 cpu->hwp_boost_min = boost_level1; 2329 else if (cpu->hwp_boost_min < HWP_GUARANTEED_PERF(hwp_cap)) 2330 cpu->hwp_boost_min = HWP_GUARANTEED_PERF(hwp_cap); 2331 else if (cpu->hwp_boost_min == HWP_GUARANTEED_PERF(hwp_cap) && 2332 max_limit != HWP_GUARANTEED_PERF(hwp_cap)) 2333 cpu->hwp_boost_min = max_limit; 2334 else 2335 return; 2336 2337 hwp_req = (hwp_req & ~GENMASK_ULL(7, 0)) | cpu->hwp_boost_min; 2338 wrmsrl(MSR_HWP_REQUEST, hwp_req); 2339 cpu->last_update = cpu->sample.time; 2340 } 2341 2342 static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu) 2343 { 2344 if (cpu->hwp_boost_min) { 2345 bool expired; 2346 2347 /* Check if we are idle for hold time to boost down */ 2348 expired = time_after64(cpu->sample.time, cpu->last_update + 2349 hwp_boost_hold_time_ns); 2350 if (expired) { 2351 wrmsrl(MSR_HWP_REQUEST, cpu->hwp_req_cached); 2352 cpu->hwp_boost_min = 0; 2353 } 2354 } 2355 cpu->last_update = cpu->sample.time; 2356 } 2357 2358 static inline void intel_pstate_update_util_hwp_local(struct cpudata *cpu, 2359 u64 time) 2360 { 2361 cpu->sample.time = time; 2362 2363 if (cpu->sched_flags & SCHED_CPUFREQ_IOWAIT) { 2364 bool do_io = false; 2365 2366 cpu->sched_flags = 0; 2367 /* 2368 * Set iowait_boost flag and update time. Since IO WAIT flag 2369 * is set all the time, we can't just conclude that there is 2370 * some IO bound activity is scheduled on this CPU with just 2371 * one occurrence. If we receive at least two in two 2372 * consecutive ticks, then we treat as boost candidate. 2373 */ 2374 if (time_before64(time, cpu->last_io_update + 2 * TICK_NSEC)) 2375 do_io = true; 2376 2377 cpu->last_io_update = time; 2378 2379 if (do_io) 2380 intel_pstate_hwp_boost_up(cpu); 2381 2382 } else { 2383 intel_pstate_hwp_boost_down(cpu); 2384 } 2385 } 2386 2387 static inline void intel_pstate_update_util_hwp(struct update_util_data *data, 2388 u64 time, unsigned int flags) 2389 { 2390 struct cpudata *cpu = container_of(data, struct cpudata, update_util); 2391 2392 cpu->sched_flags |= flags; 2393 2394 if (smp_processor_id() == cpu->cpu) 2395 intel_pstate_update_util_hwp_local(cpu, time); 2396 } 2397 2398 static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu) 2399 { 2400 struct sample *sample = &cpu->sample; 2401 2402 sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf); 2403 } 2404 2405 static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time) 2406 { 2407 u64 aperf, mperf; 2408 unsigned long flags; 2409 u64 tsc; 2410 2411 local_irq_save(flags); 2412 rdmsrl(MSR_IA32_APERF, aperf); 2413 rdmsrl(MSR_IA32_MPERF, mperf); 2414 tsc = rdtsc(); 2415 if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) { 2416 local_irq_restore(flags); 2417 return false; 2418 } 2419 local_irq_restore(flags); 2420 2421 cpu->last_sample_time = cpu->sample.time; 2422 cpu->sample.time = time; 2423 cpu->sample.aperf = aperf; 2424 cpu->sample.mperf = mperf; 2425 cpu->sample.tsc = tsc; 2426 cpu->sample.aperf -= cpu->prev_aperf; 2427 cpu->sample.mperf -= cpu->prev_mperf; 2428 cpu->sample.tsc -= cpu->prev_tsc; 2429 2430 cpu->prev_aperf = aperf; 2431 cpu->prev_mperf = mperf; 2432 cpu->prev_tsc = tsc; 2433 /* 2434 * First time this function is invoked in a given cycle, all of the 2435 * previous sample data fields are equal to zero or stale and they must 2436 * be populated with meaningful numbers for things to work, so assume 2437 * that sample.time will always be reset before setting the utilization 2438 * update hook and make the caller skip the sample then. 2439 */ 2440 if (cpu->last_sample_time) { 2441 intel_pstate_calc_avg_perf(cpu); 2442 return true; 2443 } 2444 return false; 2445 } 2446 2447 static inline int32_t get_avg_frequency(struct cpudata *cpu) 2448 { 2449 return mul_ext_fp(cpu->sample.core_avg_perf, cpu_khz); 2450 } 2451 2452 static inline int32_t get_avg_pstate(struct cpudata *cpu) 2453 { 2454 return mul_ext_fp(cpu->pstate.max_pstate_physical, 2455 cpu->sample.core_avg_perf); 2456 } 2457 2458 static inline int32_t get_target_pstate(struct cpudata *cpu) 2459 { 2460 struct sample *sample = &cpu->sample; 2461 int32_t busy_frac; 2462 int target, avg_pstate; 2463 2464 busy_frac = div_fp(sample->mperf << cpu->aperf_mperf_shift, 2465 sample->tsc); 2466 2467 if (busy_frac < cpu->iowait_boost) 2468 busy_frac = cpu->iowait_boost; 2469 2470 sample->busy_scaled = busy_frac * 100; 2471 2472 target = READ_ONCE(global.no_turbo) ? 2473 cpu->pstate.max_pstate : cpu->pstate.turbo_pstate; 2474 target += target >> 2; 2475 target = mul_fp(target, busy_frac); 2476 if (target < cpu->pstate.min_pstate) 2477 target = cpu->pstate.min_pstate; 2478 2479 /* 2480 * If the average P-state during the previous cycle was higher than the 2481 * current target, add 50% of the difference to the target to reduce 2482 * possible performance oscillations and offset possible performance 2483 * loss related to moving the workload from one CPU to another within 2484 * a package/module. 2485 */ 2486 avg_pstate = get_avg_pstate(cpu); 2487 if (avg_pstate > target) 2488 target += (avg_pstate - target) >> 1; 2489 2490 return target; 2491 } 2492 2493 static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate) 2494 { 2495 int min_pstate = max(cpu->pstate.min_pstate, cpu->min_perf_ratio); 2496 int max_pstate = max(min_pstate, cpu->max_perf_ratio); 2497 2498 return clamp_t(int, pstate, min_pstate, max_pstate); 2499 } 2500 2501 static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) 2502 { 2503 if (pstate == cpu->pstate.current_pstate) 2504 return; 2505 2506 cpu->pstate.current_pstate = pstate; 2507 wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); 2508 } 2509 2510 static void intel_pstate_adjust_pstate(struct cpudata *cpu) 2511 { 2512 int from = cpu->pstate.current_pstate; 2513 struct sample *sample; 2514 int target_pstate; 2515 2516 target_pstate = get_target_pstate(cpu); 2517 target_pstate = intel_pstate_prepare_request(cpu, target_pstate); 2518 trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu); 2519 intel_pstate_update_pstate(cpu, target_pstate); 2520 2521 sample = &cpu->sample; 2522 trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf), 2523 fp_toint(sample->busy_scaled), 2524 from, 2525 cpu->pstate.current_pstate, 2526 sample->mperf, 2527 sample->aperf, 2528 sample->tsc, 2529 get_avg_frequency(cpu), 2530 fp_toint(cpu->iowait_boost * 100)); 2531 } 2532 2533 static void intel_pstate_update_util(struct update_util_data *data, u64 time, 2534 unsigned int flags) 2535 { 2536 struct cpudata *cpu = container_of(data, struct cpudata, update_util); 2537 u64 delta_ns; 2538 2539 /* Don't allow remote callbacks */ 2540 if (smp_processor_id() != cpu->cpu) 2541 return; 2542 2543 delta_ns = time - cpu->last_update; 2544 if (flags & SCHED_CPUFREQ_IOWAIT) { 2545 /* Start over if the CPU may have been idle. */ 2546 if (delta_ns > TICK_NSEC) { 2547 cpu->iowait_boost = ONE_EIGHTH_FP; 2548 } else if (cpu->iowait_boost >= ONE_EIGHTH_FP) { 2549 cpu->iowait_boost <<= 1; 2550 if (cpu->iowait_boost > int_tofp(1)) 2551 cpu->iowait_boost = int_tofp(1); 2552 } else { 2553 cpu->iowait_boost = ONE_EIGHTH_FP; 2554 } 2555 } else if (cpu->iowait_boost) { 2556 /* Clear iowait_boost if the CPU may have been idle. */ 2557 if (delta_ns > TICK_NSEC) 2558 cpu->iowait_boost = 0; 2559 else 2560 cpu->iowait_boost >>= 1; 2561 } 2562 cpu->last_update = time; 2563 delta_ns = time - cpu->sample.time; 2564 if ((s64)delta_ns < INTEL_PSTATE_SAMPLING_INTERVAL) 2565 return; 2566 2567 if (intel_pstate_sample(cpu, time)) 2568 intel_pstate_adjust_pstate(cpu); 2569 } 2570 2571 static struct pstate_funcs core_funcs = { 2572 .get_max = core_get_max_pstate, 2573 .get_max_physical = core_get_max_pstate_physical, 2574 .get_min = core_get_min_pstate, 2575 .get_turbo = core_get_turbo_pstate, 2576 .get_scaling = core_get_scaling, 2577 .get_val = core_get_val, 2578 }; 2579 2580 static const struct pstate_funcs silvermont_funcs = { 2581 .get_max = atom_get_max_pstate, 2582 .get_max_physical = atom_get_max_pstate, 2583 .get_min = atom_get_min_pstate, 2584 .get_turbo = atom_get_turbo_pstate, 2585 .get_val = atom_get_val, 2586 .get_scaling = silvermont_get_scaling, 2587 .get_vid = atom_get_vid, 2588 }; 2589 2590 static const struct pstate_funcs airmont_funcs = { 2591 .get_max = atom_get_max_pstate, 2592 .get_max_physical = atom_get_max_pstate, 2593 .get_min = atom_get_min_pstate, 2594 .get_turbo = atom_get_turbo_pstate, 2595 .get_val = atom_get_val, 2596 .get_scaling = airmont_get_scaling, 2597 .get_vid = atom_get_vid, 2598 }; 2599 2600 static const struct pstate_funcs knl_funcs = { 2601 .get_max = core_get_max_pstate, 2602 .get_max_physical = core_get_max_pstate_physical, 2603 .get_min = core_get_min_pstate, 2604 .get_turbo = knl_get_turbo_pstate, 2605 .get_aperf_mperf_shift = knl_get_aperf_mperf_shift, 2606 .get_scaling = core_get_scaling, 2607 .get_val = core_get_val, 2608 }; 2609 2610 #define X86_MATCH(vfm, policy) \ 2611 X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_APERFMPERF, &policy) 2612 2613 static const struct x86_cpu_id intel_pstate_cpu_ids[] = { 2614 X86_MATCH(INTEL_SANDYBRIDGE, core_funcs), 2615 X86_MATCH(INTEL_SANDYBRIDGE_X, core_funcs), 2616 X86_MATCH(INTEL_ATOM_SILVERMONT, silvermont_funcs), 2617 X86_MATCH(INTEL_IVYBRIDGE, core_funcs), 2618 X86_MATCH(INTEL_HASWELL, core_funcs), 2619 X86_MATCH(INTEL_BROADWELL, core_funcs), 2620 X86_MATCH(INTEL_IVYBRIDGE_X, core_funcs), 2621 X86_MATCH(INTEL_HASWELL_X, core_funcs), 2622 X86_MATCH(INTEL_HASWELL_L, core_funcs), 2623 X86_MATCH(INTEL_HASWELL_G, core_funcs), 2624 X86_MATCH(INTEL_BROADWELL_G, core_funcs), 2625 X86_MATCH(INTEL_ATOM_AIRMONT, airmont_funcs), 2626 X86_MATCH(INTEL_SKYLAKE_L, core_funcs), 2627 X86_MATCH(INTEL_BROADWELL_X, core_funcs), 2628 X86_MATCH(INTEL_SKYLAKE, core_funcs), 2629 X86_MATCH(INTEL_BROADWELL_D, core_funcs), 2630 X86_MATCH(INTEL_XEON_PHI_KNL, knl_funcs), 2631 X86_MATCH(INTEL_XEON_PHI_KNM, knl_funcs), 2632 X86_MATCH(INTEL_ATOM_GOLDMONT, core_funcs), 2633 X86_MATCH(INTEL_ATOM_GOLDMONT_PLUS, core_funcs), 2634 X86_MATCH(INTEL_SKYLAKE_X, core_funcs), 2635 X86_MATCH(INTEL_COMETLAKE, core_funcs), 2636 X86_MATCH(INTEL_ICELAKE_X, core_funcs), 2637 X86_MATCH(INTEL_TIGERLAKE, core_funcs), 2638 X86_MATCH(INTEL_SAPPHIRERAPIDS_X, core_funcs), 2639 X86_MATCH(INTEL_EMERALDRAPIDS_X, core_funcs), 2640 {} 2641 }; 2642 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); 2643 2644 #ifdef CONFIG_ACPI 2645 static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { 2646 X86_MATCH(INTEL_BROADWELL_D, core_funcs), 2647 X86_MATCH(INTEL_BROADWELL_X, core_funcs), 2648 X86_MATCH(INTEL_SKYLAKE_X, core_funcs), 2649 X86_MATCH(INTEL_ICELAKE_X, core_funcs), 2650 X86_MATCH(INTEL_SAPPHIRERAPIDS_X, core_funcs), 2651 X86_MATCH(INTEL_EMERALDRAPIDS_X, core_funcs), 2652 X86_MATCH(INTEL_GRANITERAPIDS_D, core_funcs), 2653 X86_MATCH(INTEL_GRANITERAPIDS_X, core_funcs), 2654 X86_MATCH(INTEL_ATOM_CRESTMONT, core_funcs), 2655 X86_MATCH(INTEL_ATOM_CRESTMONT_X, core_funcs), 2656 {} 2657 }; 2658 #endif 2659 2660 static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = { 2661 X86_MATCH(INTEL_KABYLAKE, core_funcs), 2662 {} 2663 }; 2664 2665 static int intel_pstate_init_cpu(unsigned int cpunum) 2666 { 2667 struct cpudata *cpu; 2668 2669 cpu = all_cpu_data[cpunum]; 2670 2671 if (!cpu) { 2672 cpu = kzalloc(sizeof(*cpu), GFP_KERNEL); 2673 if (!cpu) 2674 return -ENOMEM; 2675 2676 WRITE_ONCE(all_cpu_data[cpunum], cpu); 2677 2678 cpu->cpu = cpunum; 2679 2680 cpu->epp_default = -EINVAL; 2681 2682 if (hwp_active) { 2683 intel_pstate_hwp_enable(cpu); 2684 2685 if (intel_pstate_acpi_pm_profile_server()) 2686 hwp_boost = true; 2687 } 2688 } else if (hwp_active) { 2689 /* 2690 * Re-enable HWP in case this happens after a resume from ACPI 2691 * S3 if the CPU was offline during the whole system/resume 2692 * cycle. 2693 */ 2694 intel_pstate_hwp_reenable(cpu); 2695 } 2696 2697 cpu->epp_powersave = -EINVAL; 2698 cpu->epp_policy = 0; 2699 2700 intel_pstate_get_cpu_pstates(cpu); 2701 2702 pr_debug("controlling: cpu %d\n", cpunum); 2703 2704 return 0; 2705 } 2706 2707 static void intel_pstate_set_update_util_hook(unsigned int cpu_num) 2708 { 2709 struct cpudata *cpu = all_cpu_data[cpu_num]; 2710 2711 if (hwp_active && !hwp_boost) 2712 return; 2713 2714 if (cpu->update_util_set) 2715 return; 2716 2717 /* Prevent intel_pstate_update_util() from using stale data. */ 2718 cpu->sample.time = 0; 2719 cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, 2720 (hwp_active ? 2721 intel_pstate_update_util_hwp : 2722 intel_pstate_update_util)); 2723 cpu->update_util_set = true; 2724 } 2725 2726 static void intel_pstate_clear_update_util_hook(unsigned int cpu) 2727 { 2728 struct cpudata *cpu_data = all_cpu_data[cpu]; 2729 2730 if (!cpu_data->update_util_set) 2731 return; 2732 2733 cpufreq_remove_update_util_hook(cpu); 2734 cpu_data->update_util_set = false; 2735 synchronize_rcu(); 2736 } 2737 2738 static int intel_pstate_get_max_freq(struct cpudata *cpu) 2739 { 2740 return READ_ONCE(global.no_turbo) ? 2741 cpu->pstate.max_freq : cpu->pstate.turbo_freq; 2742 } 2743 2744 static void intel_pstate_update_perf_limits(struct cpudata *cpu, 2745 unsigned int policy_min, 2746 unsigned int policy_max) 2747 { 2748 int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; 2749 int32_t max_policy_perf, min_policy_perf; 2750 2751 max_policy_perf = policy_max / perf_ctl_scaling; 2752 if (policy_max == policy_min) { 2753 min_policy_perf = max_policy_perf; 2754 } else { 2755 min_policy_perf = policy_min / perf_ctl_scaling; 2756 min_policy_perf = clamp_t(int32_t, min_policy_perf, 2757 0, max_policy_perf); 2758 } 2759 2760 /* 2761 * HWP needs some special consideration, because HWP_REQUEST uses 2762 * abstract values to represent performance rather than pure ratios. 2763 */ 2764 if (hwp_active && cpu->pstate.scaling != perf_ctl_scaling) { 2765 int freq; 2766 2767 freq = max_policy_perf * perf_ctl_scaling; 2768 max_policy_perf = intel_pstate_freq_to_hwp(cpu, freq); 2769 freq = min_policy_perf * perf_ctl_scaling; 2770 min_policy_perf = intel_pstate_freq_to_hwp(cpu, freq); 2771 } 2772 2773 pr_debug("cpu:%d min_policy_perf:%d max_policy_perf:%d\n", 2774 cpu->cpu, min_policy_perf, max_policy_perf); 2775 2776 /* Normalize user input to [min_perf, max_perf] */ 2777 if (per_cpu_limits) { 2778 cpu->min_perf_ratio = min_policy_perf; 2779 cpu->max_perf_ratio = max_policy_perf; 2780 } else { 2781 int turbo_max = cpu->pstate.turbo_pstate; 2782 int32_t global_min, global_max; 2783 2784 /* Global limits are in percent of the maximum turbo P-state. */ 2785 global_max = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100); 2786 global_min = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100); 2787 global_min = clamp_t(int32_t, global_min, 0, global_max); 2788 2789 pr_debug("cpu:%d global_min:%d global_max:%d\n", cpu->cpu, 2790 global_min, global_max); 2791 2792 cpu->min_perf_ratio = max(min_policy_perf, global_min); 2793 cpu->min_perf_ratio = min(cpu->min_perf_ratio, max_policy_perf); 2794 cpu->max_perf_ratio = min(max_policy_perf, global_max); 2795 cpu->max_perf_ratio = max(min_policy_perf, cpu->max_perf_ratio); 2796 2797 /* Make sure min_perf <= max_perf */ 2798 cpu->min_perf_ratio = min(cpu->min_perf_ratio, 2799 cpu->max_perf_ratio); 2800 2801 } 2802 pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", cpu->cpu, 2803 cpu->max_perf_ratio, 2804 cpu->min_perf_ratio); 2805 } 2806 2807 static int intel_pstate_set_policy(struct cpufreq_policy *policy) 2808 { 2809 struct cpudata *cpu; 2810 2811 if (!policy->cpuinfo.max_freq) 2812 return -ENODEV; 2813 2814 pr_debug("set_policy cpuinfo.max %u policy->max %u\n", 2815 policy->cpuinfo.max_freq, policy->max); 2816 2817 cpu = all_cpu_data[policy->cpu]; 2818 cpu->policy = policy->policy; 2819 2820 mutex_lock(&intel_pstate_limits_lock); 2821 2822 intel_pstate_update_perf_limits(cpu, policy->min, policy->max); 2823 2824 if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { 2825 int pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio); 2826 2827 /* 2828 * NOHZ_FULL CPUs need this as the governor callback may not 2829 * be invoked on them. 2830 */ 2831 intel_pstate_clear_update_util_hook(policy->cpu); 2832 intel_pstate_set_pstate(cpu, pstate); 2833 } else { 2834 intel_pstate_set_update_util_hook(policy->cpu); 2835 } 2836 2837 if (hwp_active) { 2838 /* 2839 * When hwp_boost was active before and dynamically it 2840 * was turned off, in that case we need to clear the 2841 * update util hook. 2842 */ 2843 if (!hwp_boost) 2844 intel_pstate_clear_update_util_hook(policy->cpu); 2845 intel_pstate_hwp_set(policy->cpu); 2846 } 2847 /* 2848 * policy->cur is never updated with the intel_pstate driver, but it 2849 * is used as a stale frequency value. So, keep it within limits. 2850 */ 2851 policy->cur = policy->min; 2852 2853 mutex_unlock(&intel_pstate_limits_lock); 2854 2855 return 0; 2856 } 2857 2858 static void intel_pstate_adjust_policy_max(struct cpudata *cpu, 2859 struct cpufreq_policy_data *policy) 2860 { 2861 if (!hwp_active && 2862 cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate && 2863 policy->max < policy->cpuinfo.max_freq && 2864 policy->max > cpu->pstate.max_freq) { 2865 pr_debug("policy->max > max non turbo frequency\n"); 2866 policy->max = policy->cpuinfo.max_freq; 2867 } 2868 } 2869 2870 static void intel_pstate_verify_cpu_policy(struct cpudata *cpu, 2871 struct cpufreq_policy_data *policy) 2872 { 2873 int max_freq; 2874 2875 if (hwp_active) { 2876 intel_pstate_get_hwp_cap(cpu); 2877 max_freq = READ_ONCE(global.no_turbo) ? 2878 cpu->pstate.max_freq : cpu->pstate.turbo_freq; 2879 } else { 2880 max_freq = intel_pstate_get_max_freq(cpu); 2881 } 2882 cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, max_freq); 2883 2884 intel_pstate_adjust_policy_max(cpu, policy); 2885 } 2886 2887 static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy) 2888 { 2889 intel_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy); 2890 2891 return 0; 2892 } 2893 2894 static int intel_cpufreq_cpu_offline(struct cpufreq_policy *policy) 2895 { 2896 struct cpudata *cpu = all_cpu_data[policy->cpu]; 2897 2898 pr_debug("CPU %d going offline\n", cpu->cpu); 2899 2900 if (cpu->suspended) 2901 return 0; 2902 2903 /* 2904 * If the CPU is an SMT thread and it goes offline with the performance 2905 * settings different from the minimum, it will prevent its sibling 2906 * from getting to lower performance levels, so force the minimum 2907 * performance on CPU offline to prevent that from happening. 2908 */ 2909 if (hwp_active) 2910 intel_pstate_hwp_offline(cpu); 2911 else 2912 intel_pstate_set_min_pstate(cpu); 2913 2914 intel_pstate_exit_perf_limits(policy); 2915 2916 return 0; 2917 } 2918 2919 static int intel_pstate_cpu_online(struct cpufreq_policy *policy) 2920 { 2921 struct cpudata *cpu = all_cpu_data[policy->cpu]; 2922 2923 pr_debug("CPU %d going online\n", cpu->cpu); 2924 2925 intel_pstate_init_acpi_perf_limits(policy); 2926 2927 if (hwp_active) { 2928 /* 2929 * Re-enable HWP and clear the "suspended" flag to let "resume" 2930 * know that it need not do that. 2931 */ 2932 intel_pstate_hwp_reenable(cpu); 2933 cpu->suspended = false; 2934 2935 hybrid_update_capacity(cpu); 2936 } 2937 2938 return 0; 2939 } 2940 2941 static int intel_pstate_cpu_offline(struct cpufreq_policy *policy) 2942 { 2943 intel_pstate_clear_update_util_hook(policy->cpu); 2944 2945 return intel_cpufreq_cpu_offline(policy); 2946 } 2947 2948 static void intel_pstate_cpu_exit(struct cpufreq_policy *policy) 2949 { 2950 pr_debug("CPU %d exiting\n", policy->cpu); 2951 2952 policy->fast_switch_possible = false; 2953 } 2954 2955 static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) 2956 { 2957 struct cpudata *cpu; 2958 int rc; 2959 2960 rc = intel_pstate_init_cpu(policy->cpu); 2961 if (rc) 2962 return rc; 2963 2964 cpu = all_cpu_data[policy->cpu]; 2965 2966 cpu->max_perf_ratio = 0xFF; 2967 cpu->min_perf_ratio = 0; 2968 2969 /* cpuinfo and default policy values */ 2970 policy->cpuinfo.min_freq = cpu->pstate.min_freq; 2971 policy->cpuinfo.max_freq = READ_ONCE(global.no_turbo) ? 2972 cpu->pstate.max_freq : cpu->pstate.turbo_freq; 2973 2974 policy->min = policy->cpuinfo.min_freq; 2975 policy->max = policy->cpuinfo.max_freq; 2976 2977 intel_pstate_init_acpi_perf_limits(policy); 2978 2979 policy->fast_switch_possible = true; 2980 2981 return 0; 2982 } 2983 2984 static int intel_pstate_cpu_init(struct cpufreq_policy *policy) 2985 { 2986 int ret = __intel_pstate_cpu_init(policy); 2987 2988 if (ret) 2989 return ret; 2990 2991 /* 2992 * Set the policy to powersave to provide a valid fallback value in case 2993 * the default cpufreq governor is neither powersave nor performance. 2994 */ 2995 policy->policy = CPUFREQ_POLICY_POWERSAVE; 2996 2997 if (hwp_active) { 2998 struct cpudata *cpu = all_cpu_data[policy->cpu]; 2999 3000 cpu->epp_cached = intel_pstate_get_epp(cpu, 0); 3001 } 3002 3003 return 0; 3004 } 3005 3006 static struct cpufreq_driver intel_pstate = { 3007 .flags = CPUFREQ_CONST_LOOPS, 3008 .verify = intel_pstate_verify_policy, 3009 .setpolicy = intel_pstate_set_policy, 3010 .suspend = intel_pstate_suspend, 3011 .resume = intel_pstate_resume, 3012 .init = intel_pstate_cpu_init, 3013 .exit = intel_pstate_cpu_exit, 3014 .offline = intel_pstate_cpu_offline, 3015 .online = intel_pstate_cpu_online, 3016 .update_limits = intel_pstate_update_limits, 3017 .name = "intel_pstate", 3018 }; 3019 3020 static int intel_cpufreq_verify_policy(struct cpufreq_policy_data *policy) 3021 { 3022 struct cpudata *cpu = all_cpu_data[policy->cpu]; 3023 3024 intel_pstate_verify_cpu_policy(cpu, policy); 3025 intel_pstate_update_perf_limits(cpu, policy->min, policy->max); 3026 3027 return 0; 3028 } 3029 3030 /* Use of trace in passive mode: 3031 * 3032 * In passive mode the trace core_busy field (also known as the 3033 * performance field, and lablelled as such on the graphs; also known as 3034 * core_avg_perf) is not needed and so is re-assigned to indicate if the 3035 * driver call was via the normal or fast switch path. Various graphs 3036 * output from the intel_pstate_tracer.py utility that include core_busy 3037 * (or performance or core_avg_perf) have a fixed y-axis from 0 to 100%, 3038 * so we use 10 to indicate the normal path through the driver, and 3039 * 90 to indicate the fast switch path through the driver. 3040 * The scaled_busy field is not used, and is set to 0. 3041 */ 3042 3043 #define INTEL_PSTATE_TRACE_TARGET 10 3044 #define INTEL_PSTATE_TRACE_FAST_SWITCH 90 3045 3046 static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate) 3047 { 3048 struct sample *sample; 3049 3050 if (!trace_pstate_sample_enabled()) 3051 return; 3052 3053 if (!intel_pstate_sample(cpu, ktime_get())) 3054 return; 3055 3056 sample = &cpu->sample; 3057 trace_pstate_sample(trace_type, 3058 0, 3059 old_pstate, 3060 cpu->pstate.current_pstate, 3061 sample->mperf, 3062 sample->aperf, 3063 sample->tsc, 3064 get_avg_frequency(cpu), 3065 fp_toint(cpu->iowait_boost * 100)); 3066 } 3067 3068 static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max, 3069 u32 desired, bool fast_switch) 3070 { 3071 u64 prev = READ_ONCE(cpu->hwp_req_cached), value = prev; 3072 3073 value &= ~HWP_MIN_PERF(~0L); 3074 value |= HWP_MIN_PERF(min); 3075 3076 value &= ~HWP_MAX_PERF(~0L); 3077 value |= HWP_MAX_PERF(max); 3078 3079 value &= ~HWP_DESIRED_PERF(~0L); 3080 value |= HWP_DESIRED_PERF(desired); 3081 3082 if (value == prev) 3083 return; 3084 3085 WRITE_ONCE(cpu->hwp_req_cached, value); 3086 if (fast_switch) 3087 wrmsrl(MSR_HWP_REQUEST, value); 3088 else 3089 wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); 3090 } 3091 3092 static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu, 3093 u32 target_pstate, bool fast_switch) 3094 { 3095 if (fast_switch) 3096 wrmsrl(MSR_IA32_PERF_CTL, 3097 pstate_funcs.get_val(cpu, target_pstate)); 3098 else 3099 wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, 3100 pstate_funcs.get_val(cpu, target_pstate)); 3101 } 3102 3103 static int intel_cpufreq_update_pstate(struct cpufreq_policy *policy, 3104 int target_pstate, bool fast_switch) 3105 { 3106 struct cpudata *cpu = all_cpu_data[policy->cpu]; 3107 int old_pstate = cpu->pstate.current_pstate; 3108 3109 target_pstate = intel_pstate_prepare_request(cpu, target_pstate); 3110 if (hwp_active) { 3111 int max_pstate = policy->strict_target ? 3112 target_pstate : cpu->max_perf_ratio; 3113 3114 intel_cpufreq_hwp_update(cpu, target_pstate, max_pstate, 0, 3115 fast_switch); 3116 } else if (target_pstate != old_pstate) { 3117 intel_cpufreq_perf_ctl_update(cpu, target_pstate, fast_switch); 3118 } 3119 3120 cpu->pstate.current_pstate = target_pstate; 3121 3122 intel_cpufreq_trace(cpu, fast_switch ? INTEL_PSTATE_TRACE_FAST_SWITCH : 3123 INTEL_PSTATE_TRACE_TARGET, old_pstate); 3124 3125 return target_pstate; 3126 } 3127 3128 static int intel_cpufreq_target(struct cpufreq_policy *policy, 3129 unsigned int target_freq, 3130 unsigned int relation) 3131 { 3132 struct cpudata *cpu = all_cpu_data[policy->cpu]; 3133 struct cpufreq_freqs freqs; 3134 int target_pstate; 3135 3136 freqs.old = policy->cur; 3137 freqs.new = target_freq; 3138 3139 cpufreq_freq_transition_begin(policy, &freqs); 3140 3141 target_pstate = intel_pstate_freq_to_hwp_rel(cpu, freqs.new, relation); 3142 target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, false); 3143 3144 freqs.new = target_pstate * cpu->pstate.scaling; 3145 3146 cpufreq_freq_transition_end(policy, &freqs, false); 3147 3148 return 0; 3149 } 3150 3151 static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy, 3152 unsigned int target_freq) 3153 { 3154 struct cpudata *cpu = all_cpu_data[policy->cpu]; 3155 int target_pstate; 3156 3157 target_pstate = intel_pstate_freq_to_hwp(cpu, target_freq); 3158 3159 target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, true); 3160 3161 return target_pstate * cpu->pstate.scaling; 3162 } 3163 3164 static void intel_cpufreq_adjust_perf(unsigned int cpunum, 3165 unsigned long min_perf, 3166 unsigned long target_perf, 3167 unsigned long capacity) 3168 { 3169 struct cpudata *cpu = all_cpu_data[cpunum]; 3170 u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached); 3171 int old_pstate = cpu->pstate.current_pstate; 3172 int cap_pstate, min_pstate, max_pstate, target_pstate; 3173 3174 cap_pstate = READ_ONCE(global.no_turbo) ? 3175 HWP_GUARANTEED_PERF(hwp_cap) : 3176 HWP_HIGHEST_PERF(hwp_cap); 3177 3178 /* Optimization: Avoid unnecessary divisions. */ 3179 3180 target_pstate = cap_pstate; 3181 if (target_perf < capacity) 3182 target_pstate = DIV_ROUND_UP(cap_pstate * target_perf, capacity); 3183 3184 min_pstate = cap_pstate; 3185 if (min_perf < capacity) 3186 min_pstate = DIV_ROUND_UP(cap_pstate * min_perf, capacity); 3187 3188 if (min_pstate < cpu->pstate.min_pstate) 3189 min_pstate = cpu->pstate.min_pstate; 3190 3191 if (min_pstate < cpu->min_perf_ratio) 3192 min_pstate = cpu->min_perf_ratio; 3193 3194 if (min_pstate > cpu->max_perf_ratio) 3195 min_pstate = cpu->max_perf_ratio; 3196 3197 max_pstate = min(cap_pstate, cpu->max_perf_ratio); 3198 if (max_pstate < min_pstate) 3199 max_pstate = min_pstate; 3200 3201 target_pstate = clamp_t(int, target_pstate, min_pstate, max_pstate); 3202 3203 intel_cpufreq_hwp_update(cpu, min_pstate, max_pstate, target_pstate, true); 3204 3205 cpu->pstate.current_pstate = target_pstate; 3206 intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_FAST_SWITCH, old_pstate); 3207 } 3208 3209 static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) 3210 { 3211 struct freq_qos_request *req; 3212 struct cpudata *cpu; 3213 struct device *dev; 3214 int ret, freq; 3215 3216 dev = get_cpu_device(policy->cpu); 3217 if (!dev) 3218 return -ENODEV; 3219 3220 ret = __intel_pstate_cpu_init(policy); 3221 if (ret) 3222 return ret; 3223 3224 policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; 3225 /* This reflects the intel_pstate_get_cpu_pstates() setting. */ 3226 policy->cur = policy->cpuinfo.min_freq; 3227 3228 req = kcalloc(2, sizeof(*req), GFP_KERNEL); 3229 if (!req) { 3230 ret = -ENOMEM; 3231 goto pstate_exit; 3232 } 3233 3234 cpu = all_cpu_data[policy->cpu]; 3235 3236 if (hwp_active) { 3237 u64 value; 3238 3239 policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY_HWP; 3240 3241 intel_pstate_get_hwp_cap(cpu); 3242 3243 rdmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value); 3244 WRITE_ONCE(cpu->hwp_req_cached, value); 3245 3246 cpu->epp_cached = intel_pstate_get_epp(cpu, value); 3247 } else { 3248 policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY; 3249 } 3250 3251 freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.min_perf_pct, 100); 3252 3253 ret = freq_qos_add_request(&policy->constraints, req, FREQ_QOS_MIN, 3254 freq); 3255 if (ret < 0) { 3256 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 3257 goto free_req; 3258 } 3259 3260 freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.max_perf_pct, 100); 3261 3262 ret = freq_qos_add_request(&policy->constraints, req + 1, FREQ_QOS_MAX, 3263 freq); 3264 if (ret < 0) { 3265 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 3266 goto remove_min_req; 3267 } 3268 3269 policy->driver_data = req; 3270 3271 return 0; 3272 3273 remove_min_req: 3274 freq_qos_remove_request(req); 3275 free_req: 3276 kfree(req); 3277 pstate_exit: 3278 intel_pstate_exit_perf_limits(policy); 3279 3280 return ret; 3281 } 3282 3283 static void intel_cpufreq_cpu_exit(struct cpufreq_policy *policy) 3284 { 3285 struct freq_qos_request *req; 3286 3287 req = policy->driver_data; 3288 3289 freq_qos_remove_request(req + 1); 3290 freq_qos_remove_request(req); 3291 kfree(req); 3292 3293 intel_pstate_cpu_exit(policy); 3294 } 3295 3296 static int intel_cpufreq_suspend(struct cpufreq_policy *policy) 3297 { 3298 intel_pstate_suspend(policy); 3299 3300 if (hwp_active) { 3301 struct cpudata *cpu = all_cpu_data[policy->cpu]; 3302 u64 value = READ_ONCE(cpu->hwp_req_cached); 3303 3304 /* 3305 * Clear the desired perf field in MSR_HWP_REQUEST in case 3306 * intel_cpufreq_adjust_perf() is in use and the last value 3307 * written by it may not be suitable. 3308 */ 3309 value &= ~HWP_DESIRED_PERF(~0L); 3310 wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); 3311 WRITE_ONCE(cpu->hwp_req_cached, value); 3312 } 3313 3314 return 0; 3315 } 3316 3317 static struct cpufreq_driver intel_cpufreq = { 3318 .flags = CPUFREQ_CONST_LOOPS, 3319 .verify = intel_cpufreq_verify_policy, 3320 .target = intel_cpufreq_target, 3321 .fast_switch = intel_cpufreq_fast_switch, 3322 .init = intel_cpufreq_cpu_init, 3323 .exit = intel_cpufreq_cpu_exit, 3324 .offline = intel_cpufreq_cpu_offline, 3325 .online = intel_pstate_cpu_online, 3326 .suspend = intel_cpufreq_suspend, 3327 .resume = intel_pstate_resume, 3328 .update_limits = intel_pstate_update_limits, 3329 .name = "intel_cpufreq", 3330 }; 3331 3332 static struct cpufreq_driver *default_driver; 3333 3334 static void intel_pstate_driver_cleanup(void) 3335 { 3336 unsigned int cpu; 3337 3338 cpus_read_lock(); 3339 for_each_online_cpu(cpu) { 3340 if (all_cpu_data[cpu]) { 3341 if (intel_pstate_driver == &intel_pstate) 3342 intel_pstate_clear_update_util_hook(cpu); 3343 3344 kfree(all_cpu_data[cpu]); 3345 WRITE_ONCE(all_cpu_data[cpu], NULL); 3346 } 3347 } 3348 cpus_read_unlock(); 3349 3350 intel_pstate_driver = NULL; 3351 } 3352 3353 static int intel_pstate_register_driver(struct cpufreq_driver *driver) 3354 { 3355 int ret; 3356 3357 if (driver == &intel_pstate) 3358 intel_pstate_sysfs_expose_hwp_dynamic_boost(); 3359 3360 memset(&global, 0, sizeof(global)); 3361 global.max_perf_pct = 100; 3362 global.turbo_disabled = turbo_is_disabled(); 3363 global.no_turbo = global.turbo_disabled; 3364 3365 arch_set_max_freq_ratio(global.turbo_disabled); 3366 3367 intel_pstate_driver = driver; 3368 ret = cpufreq_register_driver(intel_pstate_driver); 3369 if (ret) { 3370 intel_pstate_driver_cleanup(); 3371 return ret; 3372 } 3373 3374 global.min_perf_pct = min_perf_pct_min(); 3375 3376 hybrid_init_cpu_capacity_scaling(); 3377 3378 return 0; 3379 } 3380 3381 static ssize_t intel_pstate_show_status(char *buf) 3382 { 3383 if (!intel_pstate_driver) 3384 return sprintf(buf, "off\n"); 3385 3386 return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ? 3387 "active" : "passive"); 3388 } 3389 3390 static int intel_pstate_update_status(const char *buf, size_t size) 3391 { 3392 if (size == 3 && !strncmp(buf, "off", size)) { 3393 if (!intel_pstate_driver) 3394 return -EINVAL; 3395 3396 if (hwp_active) 3397 return -EBUSY; 3398 3399 cpufreq_unregister_driver(intel_pstate_driver); 3400 intel_pstate_driver_cleanup(); 3401 return 0; 3402 } 3403 3404 if (size == 6 && !strncmp(buf, "active", size)) { 3405 if (intel_pstate_driver) { 3406 if (intel_pstate_driver == &intel_pstate) 3407 return 0; 3408 3409 cpufreq_unregister_driver(intel_pstate_driver); 3410 } 3411 3412 return intel_pstate_register_driver(&intel_pstate); 3413 } 3414 3415 if (size == 7 && !strncmp(buf, "passive", size)) { 3416 if (intel_pstate_driver) { 3417 if (intel_pstate_driver == &intel_cpufreq) 3418 return 0; 3419 3420 cpufreq_unregister_driver(intel_pstate_driver); 3421 intel_pstate_sysfs_hide_hwp_dynamic_boost(); 3422 } 3423 3424 return intel_pstate_register_driver(&intel_cpufreq); 3425 } 3426 3427 return -EINVAL; 3428 } 3429 3430 static int no_load __initdata; 3431 static int no_hwp __initdata; 3432 static int hwp_only __initdata; 3433 static unsigned int force_load __initdata; 3434 3435 static int __init intel_pstate_msrs_not_valid(void) 3436 { 3437 if (!pstate_funcs.get_max(0) || 3438 !pstate_funcs.get_min(0) || 3439 !pstate_funcs.get_turbo(0)) 3440 return -ENODEV; 3441 3442 return 0; 3443 } 3444 3445 static void __init copy_cpu_funcs(struct pstate_funcs *funcs) 3446 { 3447 pstate_funcs.get_max = funcs->get_max; 3448 pstate_funcs.get_max_physical = funcs->get_max_physical; 3449 pstate_funcs.get_min = funcs->get_min; 3450 pstate_funcs.get_turbo = funcs->get_turbo; 3451 pstate_funcs.get_scaling = funcs->get_scaling; 3452 pstate_funcs.get_val = funcs->get_val; 3453 pstate_funcs.get_vid = funcs->get_vid; 3454 pstate_funcs.get_aperf_mperf_shift = funcs->get_aperf_mperf_shift; 3455 } 3456 3457 #ifdef CONFIG_ACPI 3458 3459 static bool __init intel_pstate_no_acpi_pss(void) 3460 { 3461 int i; 3462 3463 for_each_possible_cpu(i) { 3464 acpi_status status; 3465 union acpi_object *pss; 3466 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; 3467 struct acpi_processor *pr = per_cpu(processors, i); 3468 3469 if (!pr) 3470 continue; 3471 3472 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer); 3473 if (ACPI_FAILURE(status)) 3474 continue; 3475 3476 pss = buffer.pointer; 3477 if (pss && pss->type == ACPI_TYPE_PACKAGE) { 3478 kfree(pss); 3479 return false; 3480 } 3481 3482 kfree(pss); 3483 } 3484 3485 pr_debug("ACPI _PSS not found\n"); 3486 return true; 3487 } 3488 3489 static bool __init intel_pstate_no_acpi_pcch(void) 3490 { 3491 acpi_status status; 3492 acpi_handle handle; 3493 3494 status = acpi_get_handle(NULL, "\\_SB", &handle); 3495 if (ACPI_FAILURE(status)) 3496 goto not_found; 3497 3498 if (acpi_has_method(handle, "PCCH")) 3499 return false; 3500 3501 not_found: 3502 pr_debug("ACPI PCCH not found\n"); 3503 return true; 3504 } 3505 3506 static bool __init intel_pstate_has_acpi_ppc(void) 3507 { 3508 int i; 3509 3510 for_each_possible_cpu(i) { 3511 struct acpi_processor *pr = per_cpu(processors, i); 3512 3513 if (!pr) 3514 continue; 3515 if (acpi_has_method(pr->handle, "_PPC")) 3516 return true; 3517 } 3518 pr_debug("ACPI _PPC not found\n"); 3519 return false; 3520 } 3521 3522 enum { 3523 PSS, 3524 PPC, 3525 }; 3526 3527 /* Hardware vendor-specific info that has its own power management modes */ 3528 static struct acpi_platform_list plat_info[] __initdata = { 3529 {"HP ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, NULL, PSS}, 3530 {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3531 {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3532 {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3533 {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3534 {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3535 {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3536 {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3537 {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3538 {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3539 {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3540 {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3541 {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3542 {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3543 {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC}, 3544 { } /* End */ 3545 }; 3546 3547 #define BITMASK_OOB (BIT(8) | BIT(18)) 3548 3549 static bool __init intel_pstate_platform_pwr_mgmt_exists(void) 3550 { 3551 const struct x86_cpu_id *id; 3552 u64 misc_pwr; 3553 int idx; 3554 3555 id = x86_match_cpu(intel_pstate_cpu_oob_ids); 3556 if (id) { 3557 rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr); 3558 if (misc_pwr & BITMASK_OOB) { 3559 pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n"); 3560 pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n"); 3561 return true; 3562 } 3563 } 3564 3565 idx = acpi_match_platform_list(plat_info); 3566 if (idx < 0) 3567 return false; 3568 3569 switch (plat_info[idx].data) { 3570 case PSS: 3571 if (!intel_pstate_no_acpi_pss()) 3572 return false; 3573 3574 return intel_pstate_no_acpi_pcch(); 3575 case PPC: 3576 return intel_pstate_has_acpi_ppc() && !force_load; 3577 } 3578 3579 return false; 3580 } 3581 3582 static void intel_pstate_request_control_from_smm(void) 3583 { 3584 /* 3585 * It may be unsafe to request P-states control from SMM if _PPC support 3586 * has not been enabled. 3587 */ 3588 if (acpi_ppc) 3589 acpi_processor_pstate_control(); 3590 } 3591 #else /* CONFIG_ACPI not enabled */ 3592 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; } 3593 static inline bool intel_pstate_has_acpi_ppc(void) { return false; } 3594 static inline void intel_pstate_request_control_from_smm(void) {} 3595 #endif /* CONFIG_ACPI */ 3596 3597 #define INTEL_PSTATE_HWP_BROADWELL 0x01 3598 3599 #define X86_MATCH_HWP(vfm, hwp_mode) \ 3600 X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_HWP, hwp_mode) 3601 3602 static const struct x86_cpu_id hwp_support_ids[] __initconst = { 3603 X86_MATCH_HWP(INTEL_BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL), 3604 X86_MATCH_HWP(INTEL_BROADWELL_D, INTEL_PSTATE_HWP_BROADWELL), 3605 X86_MATCH_HWP(INTEL_ANY, 0), 3606 {} 3607 }; 3608 3609 static bool intel_pstate_hwp_is_enabled(void) 3610 { 3611 u64 value; 3612 3613 rdmsrl(MSR_PM_ENABLE, value); 3614 return !!(value & 0x1); 3615 } 3616 3617 #define POWERSAVE_MASK GENMASK(7, 0) 3618 #define BALANCE_POWER_MASK GENMASK(15, 8) 3619 #define BALANCE_PERFORMANCE_MASK GENMASK(23, 16) 3620 #define PERFORMANCE_MASK GENMASK(31, 24) 3621 3622 #define HWP_SET_EPP_VALUES(powersave, balance_power, balance_perf, performance) \ 3623 (FIELD_PREP_CONST(POWERSAVE_MASK, powersave) |\ 3624 FIELD_PREP_CONST(BALANCE_POWER_MASK, balance_power) |\ 3625 FIELD_PREP_CONST(BALANCE_PERFORMANCE_MASK, balance_perf) |\ 3626 FIELD_PREP_CONST(PERFORMANCE_MASK, performance)) 3627 3628 #define HWP_SET_DEF_BALANCE_PERF_EPP(balance_perf) \ 3629 (HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, HWP_EPP_BALANCE_POWERSAVE,\ 3630 balance_perf, HWP_EPP_PERFORMANCE)) 3631 3632 static const struct x86_cpu_id intel_epp_default[] = { 3633 /* 3634 * Set EPP value as 102, this is the max suggested EPP 3635 * which can result in one core turbo frequency for 3636 * AlderLake Mobile CPUs. 3637 */ 3638 X86_MATCH_VFM(INTEL_ALDERLAKE_L, HWP_SET_DEF_BALANCE_PERF_EPP(102)), 3639 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), 3640 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), 3641 X86_MATCH_VFM(INTEL_METEORLAKE_L, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, 3642 179, 64, 16)), 3643 X86_MATCH_VFM(INTEL_ARROWLAKE, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, 3644 179, 64, 16)), 3645 {} 3646 }; 3647 3648 static const struct x86_cpu_id intel_hybrid_scaling_factor[] = { 3649 X86_MATCH_VFM(INTEL_METEORLAKE_L, HYBRID_SCALING_FACTOR_MTL), 3650 X86_MATCH_VFM(INTEL_ARROWLAKE, HYBRID_SCALING_FACTOR_MTL), 3651 X86_MATCH_VFM(INTEL_LUNARLAKE_M, HYBRID_SCALING_FACTOR_LNL), 3652 {} 3653 }; 3654 3655 static int __init intel_pstate_init(void) 3656 { 3657 static struct cpudata **_all_cpu_data; 3658 const struct x86_cpu_id *id; 3659 int rc; 3660 3661 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 3662 return -ENODEV; 3663 3664 id = x86_match_cpu(hwp_support_ids); 3665 if (id) { 3666 hwp_forced = intel_pstate_hwp_is_enabled(); 3667 3668 if (hwp_forced) 3669 pr_info("HWP enabled by BIOS\n"); 3670 else if (no_load) 3671 return -ENODEV; 3672 3673 copy_cpu_funcs(&core_funcs); 3674 /* 3675 * Avoid enabling HWP for processors without EPP support, 3676 * because that means incomplete HWP implementation which is a 3677 * corner case and supporting it is generally problematic. 3678 * 3679 * If HWP is enabled already, though, there is no choice but to 3680 * deal with it. 3681 */ 3682 if ((!no_hwp && boot_cpu_has(X86_FEATURE_HWP_EPP)) || hwp_forced) { 3683 hwp_active = true; 3684 hwp_mode_bdw = id->driver_data; 3685 intel_pstate.attr = hwp_cpufreq_attrs; 3686 intel_cpufreq.attr = hwp_cpufreq_attrs; 3687 intel_cpufreq.flags |= CPUFREQ_NEED_UPDATE_LIMITS; 3688 intel_cpufreq.adjust_perf = intel_cpufreq_adjust_perf; 3689 if (!default_driver) 3690 default_driver = &intel_pstate; 3691 3692 pstate_funcs.get_cpu_scaling = hwp_get_cpu_scaling; 3693 3694 goto hwp_cpu_matched; 3695 } 3696 pr_info("HWP not enabled\n"); 3697 } else { 3698 if (no_load) 3699 return -ENODEV; 3700 3701 id = x86_match_cpu(intel_pstate_cpu_ids); 3702 if (!id) { 3703 pr_info("CPU model not supported\n"); 3704 return -ENODEV; 3705 } 3706 3707 copy_cpu_funcs((struct pstate_funcs *)id->driver_data); 3708 } 3709 3710 if (intel_pstate_msrs_not_valid()) { 3711 pr_info("Invalid MSRs\n"); 3712 return -ENODEV; 3713 } 3714 /* Without HWP start in the passive mode. */ 3715 if (!default_driver) 3716 default_driver = &intel_cpufreq; 3717 3718 hwp_cpu_matched: 3719 /* 3720 * The Intel pstate driver will be ignored if the platform 3721 * firmware has its own power management modes. 3722 */ 3723 if (intel_pstate_platform_pwr_mgmt_exists()) { 3724 pr_info("P-states controlled by the platform\n"); 3725 return -ENODEV; 3726 } 3727 3728 if (!hwp_active && hwp_only) 3729 return -ENOTSUPP; 3730 3731 pr_info("Intel P-state driver initializing\n"); 3732 3733 _all_cpu_data = vzalloc(array_size(sizeof(void *), num_possible_cpus())); 3734 if (!_all_cpu_data) 3735 return -ENOMEM; 3736 3737 WRITE_ONCE(all_cpu_data, _all_cpu_data); 3738 3739 intel_pstate_request_control_from_smm(); 3740 3741 intel_pstate_sysfs_expose_params(); 3742 3743 if (hwp_active) { 3744 const struct x86_cpu_id *id = x86_match_cpu(intel_epp_default); 3745 const struct x86_cpu_id *hybrid_id = x86_match_cpu(intel_hybrid_scaling_factor); 3746 3747 if (id) { 3748 epp_values[EPP_INDEX_POWERSAVE] = 3749 FIELD_GET(POWERSAVE_MASK, id->driver_data); 3750 epp_values[EPP_INDEX_BALANCE_POWERSAVE] = 3751 FIELD_GET(BALANCE_POWER_MASK, id->driver_data); 3752 epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = 3753 FIELD_GET(BALANCE_PERFORMANCE_MASK, id->driver_data); 3754 epp_values[EPP_INDEX_PERFORMANCE] = 3755 FIELD_GET(PERFORMANCE_MASK, id->driver_data); 3756 pr_debug("Updated EPPs powersave:%x balanced power:%x balanced perf:%x performance:%x\n", 3757 epp_values[EPP_INDEX_POWERSAVE], 3758 epp_values[EPP_INDEX_BALANCE_POWERSAVE], 3759 epp_values[EPP_INDEX_BALANCE_PERFORMANCE], 3760 epp_values[EPP_INDEX_PERFORMANCE]); 3761 } 3762 3763 if (hybrid_id) { 3764 hybrid_scaling_factor = hybrid_id->driver_data; 3765 pr_debug("hybrid scaling factor: %d\n", hybrid_scaling_factor); 3766 } 3767 3768 } 3769 3770 mutex_lock(&intel_pstate_driver_lock); 3771 rc = intel_pstate_register_driver(default_driver); 3772 mutex_unlock(&intel_pstate_driver_lock); 3773 if (rc) { 3774 intel_pstate_sysfs_remove(); 3775 return rc; 3776 } 3777 3778 if (hwp_active) { 3779 const struct x86_cpu_id *id; 3780 3781 id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids); 3782 if (id) { 3783 set_power_ctl_ee_state(false); 3784 pr_info("Disabling energy efficiency optimization\n"); 3785 } 3786 3787 pr_info("HWP enabled\n"); 3788 } else if (boot_cpu_has(X86_FEATURE_HYBRID_CPU)) { 3789 pr_warn("Problematic setup: Hybrid processor with disabled HWP\n"); 3790 } 3791 3792 return 0; 3793 } 3794 device_initcall(intel_pstate_init); 3795 3796 static int __init intel_pstate_setup(char *str) 3797 { 3798 if (!str) 3799 return -EINVAL; 3800 3801 if (!strcmp(str, "disable")) 3802 no_load = 1; 3803 else if (!strcmp(str, "active")) 3804 default_driver = &intel_pstate; 3805 else if (!strcmp(str, "passive")) 3806 default_driver = &intel_cpufreq; 3807 3808 if (!strcmp(str, "no_hwp")) 3809 no_hwp = 1; 3810 3811 if (!strcmp(str, "force")) 3812 force_load = 1; 3813 if (!strcmp(str, "hwp_only")) 3814 hwp_only = 1; 3815 if (!strcmp(str, "per_cpu_perf_limits")) 3816 per_cpu_limits = true; 3817 3818 #ifdef CONFIG_ACPI 3819 if (!strcmp(str, "support_acpi_ppc")) 3820 acpi_ppc = true; 3821 #endif 3822 3823 return 0; 3824 } 3825 early_param("intel_pstate", intel_pstate_setup); 3826 3827 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>"); 3828 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors"); 3829