1 /* 2 * intel_pstate.c: Native P state management for Intel processors 3 * 4 * (C) Copyright 2012 Intel Corporation 5 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com> 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; version 2 10 * of the License. 11 */ 12 13 #include <linux/kernel.h> 14 #include <linux/kernel_stat.h> 15 #include <linux/module.h> 16 #include <linux/ktime.h> 17 #include <linux/hrtimer.h> 18 #include <linux/tick.h> 19 #include <linux/slab.h> 20 #include <linux/sched.h> 21 #include <linux/list.h> 22 #include <linux/cpu.h> 23 #include <linux/cpufreq.h> 24 #include <linux/sysfs.h> 25 #include <linux/types.h> 26 #include <linux/fs.h> 27 #include <linux/debugfs.h> 28 #include <linux/acpi.h> 29 #include <linux/vmalloc.h> 30 #include <trace/events/power.h> 31 32 #include <asm/div64.h> 33 #include <asm/msr.h> 34 #include <asm/cpu_device_id.h> 35 #include <asm/cpufeature.h> 36 37 #define ATOM_RATIOS 0x66a 38 #define ATOM_VIDS 0x66b 39 #define ATOM_TURBO_RATIOS 0x66c 40 #define ATOM_TURBO_VIDS 0x66d 41 42 #define FRAC_BITS 8 43 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS) 44 #define fp_toint(X) ((X) >> FRAC_BITS) 45 46 static inline int32_t mul_fp(int32_t x, int32_t y) 47 { 48 return ((int64_t)x * (int64_t)y) >> FRAC_BITS; 49 } 50 51 static inline int32_t div_fp(s64 x, s64 y) 52 { 53 return div64_s64((int64_t)x << FRAC_BITS, y); 54 } 55 56 static inline int ceiling_fp(int32_t x) 57 { 58 int mask, ret; 59 60 ret = fp_toint(x); 61 mask = (1 << FRAC_BITS) - 1; 62 if (x & mask) 63 ret += 1; 64 return ret; 65 } 66 67 /** 68 * struct sample - Store performance sample 69 * @core_pct_busy: Ratio of APERF/MPERF in percent, which is actual 70 * performance during last sample period 71 * @busy_scaled: Scaled busy value which is used to calculate next 72 * P state. This can be different than core_pct_busy 73 * to account for cpu idle period 74 * @aperf: Difference of actual performance frequency clock count 75 * read from APERF MSR between last and current sample 76 * @mperf: Difference of maximum performance frequency clock count 77 * read from MPERF MSR between last and current sample 78 * @tsc: Difference of time stamp counter between last and 79 * current sample 80 * @freq: Effective frequency calculated from APERF/MPERF 81 * @time: Current time from scheduler 82 * 83 * This structure is used in the cpudata structure to store performance sample 84 * data for choosing next P State. 85 */ 86 struct sample { 87 int32_t core_pct_busy; 88 int32_t busy_scaled; 89 u64 aperf; 90 u64 mperf; 91 u64 tsc; 92 int freq; 93 u64 time; 94 }; 95 96 /** 97 * struct pstate_data - Store P state data 98 * @current_pstate: Current requested P state 99 * @min_pstate: Min P state possible for this platform 100 * @max_pstate: Max P state possible for this platform 101 * @max_pstate_physical:This is physical Max P state for a processor 102 * This can be higher than the max_pstate which can 103 * be limited by platform thermal design power limits 104 * @scaling: Scaling factor to convert frequency to cpufreq 105 * frequency units 106 * @turbo_pstate: Max Turbo P state possible for this platform 107 * 108 * Stores the per cpu model P state limits and current P state. 109 */ 110 struct pstate_data { 111 int current_pstate; 112 int min_pstate; 113 int max_pstate; 114 int max_pstate_physical; 115 int scaling; 116 int turbo_pstate; 117 }; 118 119 /** 120 * struct vid_data - Stores voltage information data 121 * @min: VID data for this platform corresponding to 122 * the lowest P state 123 * @max: VID data corresponding to the highest P State. 124 * @turbo: VID data for turbo P state 125 * @ratio: Ratio of (vid max - vid min) / 126 * (max P state - Min P State) 127 * 128 * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling) 129 * This data is used in Atom platforms, where in addition to target P state, 130 * the voltage data needs to be specified to select next P State. 131 */ 132 struct vid_data { 133 int min; 134 int max; 135 int turbo; 136 int32_t ratio; 137 }; 138 139 /** 140 * struct _pid - Stores PID data 141 * @setpoint: Target set point for busyness or performance 142 * @integral: Storage for accumulated error values 143 * @p_gain: PID proportional gain 144 * @i_gain: PID integral gain 145 * @d_gain: PID derivative gain 146 * @deadband: PID deadband 147 * @last_err: Last error storage for integral part of PID calculation 148 * 149 * Stores PID coefficients and last error for PID controller. 150 */ 151 struct _pid { 152 int setpoint; 153 int32_t integral; 154 int32_t p_gain; 155 int32_t i_gain; 156 int32_t d_gain; 157 int deadband; 158 int32_t last_err; 159 }; 160 161 /** 162 * struct cpudata - Per CPU instance data storage 163 * @cpu: CPU number for this instance data 164 * @update_util: CPUFreq utility callback information 165 * @pstate: Stores P state limits for this CPU 166 * @vid: Stores VID limits for this CPU 167 * @pid: Stores PID parameters for this CPU 168 * @last_sample_time: Last Sample time 169 * @prev_aperf: Last APERF value read from APERF MSR 170 * @prev_mperf: Last MPERF value read from MPERF MSR 171 * @prev_tsc: Last timestamp counter (TSC) value 172 * @prev_cummulative_iowait: IO Wait time difference from last and 173 * current sample 174 * @sample: Storage for storing last Sample data 175 * 176 * This structure stores per CPU instance data for all CPUs. 177 */ 178 struct cpudata { 179 int cpu; 180 181 struct update_util_data update_util; 182 183 struct pstate_data pstate; 184 struct vid_data vid; 185 struct _pid pid; 186 187 u64 last_sample_time; 188 u64 prev_aperf; 189 u64 prev_mperf; 190 u64 prev_tsc; 191 u64 prev_cummulative_iowait; 192 struct sample sample; 193 }; 194 195 static struct cpudata **all_cpu_data; 196 197 /** 198 * struct pid_adjust_policy - Stores static PID configuration data 199 * @sample_rate_ms: PID calculation sample rate in ms 200 * @sample_rate_ns: Sample rate calculation in ns 201 * @deadband: PID deadband 202 * @setpoint: PID Setpoint 203 * @p_gain_pct: PID proportional gain 204 * @i_gain_pct: PID integral gain 205 * @d_gain_pct: PID derivative gain 206 * 207 * Stores per CPU model static PID configuration data. 208 */ 209 struct pstate_adjust_policy { 210 int sample_rate_ms; 211 s64 sample_rate_ns; 212 int deadband; 213 int setpoint; 214 int p_gain_pct; 215 int d_gain_pct; 216 int i_gain_pct; 217 }; 218 219 /** 220 * struct pstate_funcs - Per CPU model specific callbacks 221 * @get_max: Callback to get maximum non turbo effective P state 222 * @get_max_physical: Callback to get maximum non turbo physical P state 223 * @get_min: Callback to get minimum P state 224 * @get_turbo: Callback to get turbo P state 225 * @get_scaling: Callback to get frequency scaling factor 226 * @get_val: Callback to convert P state to actual MSR write value 227 * @get_vid: Callback to get VID data for Atom platforms 228 * @get_target_pstate: Callback to a function to calculate next P state to use 229 * 230 * Core and Atom CPU models have different way to get P State limits. This 231 * structure is used to store those callbacks. 232 */ 233 struct pstate_funcs { 234 int (*get_max)(void); 235 int (*get_max_physical)(void); 236 int (*get_min)(void); 237 int (*get_turbo)(void); 238 int (*get_scaling)(void); 239 u64 (*get_val)(struct cpudata*, int pstate); 240 void (*get_vid)(struct cpudata *); 241 int32_t (*get_target_pstate)(struct cpudata *); 242 }; 243 244 /** 245 * struct cpu_defaults- Per CPU model default config data 246 * @pid_policy: PID config data 247 * @funcs: Callback function data 248 */ 249 struct cpu_defaults { 250 struct pstate_adjust_policy pid_policy; 251 struct pstate_funcs funcs; 252 }; 253 254 static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu); 255 static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu); 256 257 static struct pstate_adjust_policy pid_params; 258 static struct pstate_funcs pstate_funcs; 259 static int hwp_active; 260 261 262 /** 263 * struct perf_limits - Store user and policy limits 264 * @no_turbo: User requested turbo state from intel_pstate sysfs 265 * @turbo_disabled: Platform turbo status either from msr 266 * MSR_IA32_MISC_ENABLE or when maximum available pstate 267 * matches the maximum turbo pstate 268 * @max_perf_pct: Effective maximum performance limit in percentage, this 269 * is minimum of either limits enforced by cpufreq policy 270 * or limits from user set limits via intel_pstate sysfs 271 * @min_perf_pct: Effective minimum performance limit in percentage, this 272 * is maximum of either limits enforced by cpufreq policy 273 * or limits from user set limits via intel_pstate sysfs 274 * @max_perf: This is a scaled value between 0 to 255 for max_perf_pct 275 * This value is used to limit max pstate 276 * @min_perf: This is a scaled value between 0 to 255 for min_perf_pct 277 * This value is used to limit min pstate 278 * @max_policy_pct: The maximum performance in percentage enforced by 279 * cpufreq setpolicy interface 280 * @max_sysfs_pct: The maximum performance in percentage enforced by 281 * intel pstate sysfs interface 282 * @min_policy_pct: The minimum performance in percentage enforced by 283 * cpufreq setpolicy interface 284 * @min_sysfs_pct: The minimum performance in percentage enforced by 285 * intel pstate sysfs interface 286 * 287 * Storage for user and policy defined limits. 288 */ 289 struct perf_limits { 290 int no_turbo; 291 int turbo_disabled; 292 int max_perf_pct; 293 int min_perf_pct; 294 int32_t max_perf; 295 int32_t min_perf; 296 int max_policy_pct; 297 int max_sysfs_pct; 298 int min_policy_pct; 299 int min_sysfs_pct; 300 }; 301 302 static struct perf_limits performance_limits = { 303 .no_turbo = 0, 304 .turbo_disabled = 0, 305 .max_perf_pct = 100, 306 .max_perf = int_tofp(1), 307 .min_perf_pct = 100, 308 .min_perf = int_tofp(1), 309 .max_policy_pct = 100, 310 .max_sysfs_pct = 100, 311 .min_policy_pct = 0, 312 .min_sysfs_pct = 0, 313 }; 314 315 static struct perf_limits powersave_limits = { 316 .no_turbo = 0, 317 .turbo_disabled = 0, 318 .max_perf_pct = 100, 319 .max_perf = int_tofp(1), 320 .min_perf_pct = 0, 321 .min_perf = 0, 322 .max_policy_pct = 100, 323 .max_sysfs_pct = 100, 324 .min_policy_pct = 0, 325 .min_sysfs_pct = 0, 326 }; 327 328 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE 329 static struct perf_limits *limits = &performance_limits; 330 #else 331 static struct perf_limits *limits = &powersave_limits; 332 #endif 333 334 static inline void pid_reset(struct _pid *pid, int setpoint, int busy, 335 int deadband, int integral) { 336 pid->setpoint = int_tofp(setpoint); 337 pid->deadband = int_tofp(deadband); 338 pid->integral = int_tofp(integral); 339 pid->last_err = int_tofp(setpoint) - int_tofp(busy); 340 } 341 342 static inline void pid_p_gain_set(struct _pid *pid, int percent) 343 { 344 pid->p_gain = div_fp(int_tofp(percent), int_tofp(100)); 345 } 346 347 static inline void pid_i_gain_set(struct _pid *pid, int percent) 348 { 349 pid->i_gain = div_fp(int_tofp(percent), int_tofp(100)); 350 } 351 352 static inline void pid_d_gain_set(struct _pid *pid, int percent) 353 { 354 pid->d_gain = div_fp(int_tofp(percent), int_tofp(100)); 355 } 356 357 static signed int pid_calc(struct _pid *pid, int32_t busy) 358 { 359 signed int result; 360 int32_t pterm, dterm, fp_error; 361 int32_t integral_limit; 362 363 fp_error = pid->setpoint - busy; 364 365 if (abs(fp_error) <= pid->deadband) 366 return 0; 367 368 pterm = mul_fp(pid->p_gain, fp_error); 369 370 pid->integral += fp_error; 371 372 /* 373 * We limit the integral here so that it will never 374 * get higher than 30. This prevents it from becoming 375 * too large an input over long periods of time and allows 376 * it to get factored out sooner. 377 * 378 * The value of 30 was chosen through experimentation. 379 */ 380 integral_limit = int_tofp(30); 381 if (pid->integral > integral_limit) 382 pid->integral = integral_limit; 383 if (pid->integral < -integral_limit) 384 pid->integral = -integral_limit; 385 386 dterm = mul_fp(pid->d_gain, fp_error - pid->last_err); 387 pid->last_err = fp_error; 388 389 result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm; 390 result = result + (1 << (FRAC_BITS-1)); 391 return (signed int)fp_toint(result); 392 } 393 394 static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu) 395 { 396 pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct); 397 pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct); 398 pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct); 399 400 pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0); 401 } 402 403 static inline void intel_pstate_reset_all_pid(void) 404 { 405 unsigned int cpu; 406 407 for_each_online_cpu(cpu) { 408 if (all_cpu_data[cpu]) 409 intel_pstate_busy_pid_reset(all_cpu_data[cpu]); 410 } 411 } 412 413 static inline void update_turbo_state(void) 414 { 415 u64 misc_en; 416 struct cpudata *cpu; 417 418 cpu = all_cpu_data[0]; 419 rdmsrl(MSR_IA32_MISC_ENABLE, misc_en); 420 limits->turbo_disabled = 421 (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE || 422 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate); 423 } 424 425 static void intel_pstate_hwp_set(const struct cpumask *cpumask) 426 { 427 int min, hw_min, max, hw_max, cpu, range, adj_range; 428 u64 value, cap; 429 430 rdmsrl(MSR_HWP_CAPABILITIES, cap); 431 hw_min = HWP_LOWEST_PERF(cap); 432 hw_max = HWP_HIGHEST_PERF(cap); 433 range = hw_max - hw_min; 434 435 for_each_cpu(cpu, cpumask) { 436 rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); 437 adj_range = limits->min_perf_pct * range / 100; 438 min = hw_min + adj_range; 439 value &= ~HWP_MIN_PERF(~0L); 440 value |= HWP_MIN_PERF(min); 441 442 adj_range = limits->max_perf_pct * range / 100; 443 max = hw_min + adj_range; 444 if (limits->no_turbo) { 445 hw_max = HWP_GUARANTEED_PERF(cap); 446 if (hw_max < max) 447 max = hw_max; 448 } 449 450 value &= ~HWP_MAX_PERF(~0L); 451 value |= HWP_MAX_PERF(max); 452 wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value); 453 } 454 } 455 456 static void intel_pstate_hwp_set_online_cpus(void) 457 { 458 get_online_cpus(); 459 intel_pstate_hwp_set(cpu_online_mask); 460 put_online_cpus(); 461 } 462 463 /************************** debugfs begin ************************/ 464 static int pid_param_set(void *data, u64 val) 465 { 466 *(u32 *)data = val; 467 intel_pstate_reset_all_pid(); 468 return 0; 469 } 470 471 static int pid_param_get(void *data, u64 *val) 472 { 473 *val = *(u32 *)data; 474 return 0; 475 } 476 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, pid_param_set, "%llu\n"); 477 478 struct pid_param { 479 char *name; 480 void *value; 481 }; 482 483 static struct pid_param pid_files[] = { 484 {"sample_rate_ms", &pid_params.sample_rate_ms}, 485 {"d_gain_pct", &pid_params.d_gain_pct}, 486 {"i_gain_pct", &pid_params.i_gain_pct}, 487 {"deadband", &pid_params.deadband}, 488 {"setpoint", &pid_params.setpoint}, 489 {"p_gain_pct", &pid_params.p_gain_pct}, 490 {NULL, NULL} 491 }; 492 493 static void __init intel_pstate_debug_expose_params(void) 494 { 495 struct dentry *debugfs_parent; 496 int i = 0; 497 498 if (hwp_active) 499 return; 500 debugfs_parent = debugfs_create_dir("pstate_snb", NULL); 501 if (IS_ERR_OR_NULL(debugfs_parent)) 502 return; 503 while (pid_files[i].name) { 504 debugfs_create_file(pid_files[i].name, 0660, 505 debugfs_parent, pid_files[i].value, 506 &fops_pid_param); 507 i++; 508 } 509 } 510 511 /************************** debugfs end ************************/ 512 513 /************************** sysfs begin ************************/ 514 #define show_one(file_name, object) \ 515 static ssize_t show_##file_name \ 516 (struct kobject *kobj, struct attribute *attr, char *buf) \ 517 { \ 518 return sprintf(buf, "%u\n", limits->object); \ 519 } 520 521 static ssize_t show_turbo_pct(struct kobject *kobj, 522 struct attribute *attr, char *buf) 523 { 524 struct cpudata *cpu; 525 int total, no_turbo, turbo_pct; 526 uint32_t turbo_fp; 527 528 cpu = all_cpu_data[0]; 529 530 total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1; 531 no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1; 532 turbo_fp = div_fp(int_tofp(no_turbo), int_tofp(total)); 533 turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100))); 534 return sprintf(buf, "%u\n", turbo_pct); 535 } 536 537 static ssize_t show_num_pstates(struct kobject *kobj, 538 struct attribute *attr, char *buf) 539 { 540 struct cpudata *cpu; 541 int total; 542 543 cpu = all_cpu_data[0]; 544 total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1; 545 return sprintf(buf, "%u\n", total); 546 } 547 548 static ssize_t show_no_turbo(struct kobject *kobj, 549 struct attribute *attr, char *buf) 550 { 551 ssize_t ret; 552 553 update_turbo_state(); 554 if (limits->turbo_disabled) 555 ret = sprintf(buf, "%u\n", limits->turbo_disabled); 556 else 557 ret = sprintf(buf, "%u\n", limits->no_turbo); 558 559 return ret; 560 } 561 562 static ssize_t store_no_turbo(struct kobject *a, struct attribute *b, 563 const char *buf, size_t count) 564 { 565 unsigned int input; 566 int ret; 567 568 ret = sscanf(buf, "%u", &input); 569 if (ret != 1) 570 return -EINVAL; 571 572 update_turbo_state(); 573 if (limits->turbo_disabled) { 574 pr_warn("intel_pstate: Turbo disabled by BIOS or unavailable on processor\n"); 575 return -EPERM; 576 } 577 578 limits->no_turbo = clamp_t(int, input, 0, 1); 579 580 if (hwp_active) 581 intel_pstate_hwp_set_online_cpus(); 582 583 return count; 584 } 585 586 static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b, 587 const char *buf, size_t count) 588 { 589 unsigned int input; 590 int ret; 591 592 ret = sscanf(buf, "%u", &input); 593 if (ret != 1) 594 return -EINVAL; 595 596 limits->max_sysfs_pct = clamp_t(int, input, 0 , 100); 597 limits->max_perf_pct = min(limits->max_policy_pct, 598 limits->max_sysfs_pct); 599 limits->max_perf_pct = max(limits->min_policy_pct, 600 limits->max_perf_pct); 601 limits->max_perf_pct = max(limits->min_perf_pct, 602 limits->max_perf_pct); 603 limits->max_perf = div_fp(int_tofp(limits->max_perf_pct), 604 int_tofp(100)); 605 606 if (hwp_active) 607 intel_pstate_hwp_set_online_cpus(); 608 return count; 609 } 610 611 static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b, 612 const char *buf, size_t count) 613 { 614 unsigned int input; 615 int ret; 616 617 ret = sscanf(buf, "%u", &input); 618 if (ret != 1) 619 return -EINVAL; 620 621 limits->min_sysfs_pct = clamp_t(int, input, 0 , 100); 622 limits->min_perf_pct = max(limits->min_policy_pct, 623 limits->min_sysfs_pct); 624 limits->min_perf_pct = min(limits->max_policy_pct, 625 limits->min_perf_pct); 626 limits->min_perf_pct = min(limits->max_perf_pct, 627 limits->min_perf_pct); 628 limits->min_perf = div_fp(int_tofp(limits->min_perf_pct), 629 int_tofp(100)); 630 631 if (hwp_active) 632 intel_pstate_hwp_set_online_cpus(); 633 return count; 634 } 635 636 show_one(max_perf_pct, max_perf_pct); 637 show_one(min_perf_pct, min_perf_pct); 638 639 define_one_global_rw(no_turbo); 640 define_one_global_rw(max_perf_pct); 641 define_one_global_rw(min_perf_pct); 642 define_one_global_ro(turbo_pct); 643 define_one_global_ro(num_pstates); 644 645 static struct attribute *intel_pstate_attributes[] = { 646 &no_turbo.attr, 647 &max_perf_pct.attr, 648 &min_perf_pct.attr, 649 &turbo_pct.attr, 650 &num_pstates.attr, 651 NULL 652 }; 653 654 static struct attribute_group intel_pstate_attr_group = { 655 .attrs = intel_pstate_attributes, 656 }; 657 658 static void __init intel_pstate_sysfs_expose_params(void) 659 { 660 struct kobject *intel_pstate_kobject; 661 int rc; 662 663 intel_pstate_kobject = kobject_create_and_add("intel_pstate", 664 &cpu_subsys.dev_root->kobj); 665 BUG_ON(!intel_pstate_kobject); 666 rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group); 667 BUG_ON(rc); 668 } 669 /************************** sysfs end ************************/ 670 671 static void intel_pstate_hwp_enable(struct cpudata *cpudata) 672 { 673 /* First disable HWP notification interrupt as we don't process them */ 674 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); 675 676 wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); 677 } 678 679 static int atom_get_min_pstate(void) 680 { 681 u64 value; 682 683 rdmsrl(ATOM_RATIOS, value); 684 return (value >> 8) & 0x7F; 685 } 686 687 static int atom_get_max_pstate(void) 688 { 689 u64 value; 690 691 rdmsrl(ATOM_RATIOS, value); 692 return (value >> 16) & 0x7F; 693 } 694 695 static int atom_get_turbo_pstate(void) 696 { 697 u64 value; 698 699 rdmsrl(ATOM_TURBO_RATIOS, value); 700 return value & 0x7F; 701 } 702 703 static u64 atom_get_val(struct cpudata *cpudata, int pstate) 704 { 705 u64 val; 706 int32_t vid_fp; 707 u32 vid; 708 709 val = (u64)pstate << 8; 710 if (limits->no_turbo && !limits->turbo_disabled) 711 val |= (u64)1 << 32; 712 713 vid_fp = cpudata->vid.min + mul_fp( 714 int_tofp(pstate - cpudata->pstate.min_pstate), 715 cpudata->vid.ratio); 716 717 vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max); 718 vid = ceiling_fp(vid_fp); 719 720 if (pstate > cpudata->pstate.max_pstate) 721 vid = cpudata->vid.turbo; 722 723 return val | vid; 724 } 725 726 static int silvermont_get_scaling(void) 727 { 728 u64 value; 729 int i; 730 /* Defined in Table 35-6 from SDM (Sept 2015) */ 731 static int silvermont_freq_table[] = { 732 83300, 100000, 133300, 116700, 80000}; 733 734 rdmsrl(MSR_FSB_FREQ, value); 735 i = value & 0x7; 736 WARN_ON(i > 4); 737 738 return silvermont_freq_table[i]; 739 } 740 741 static int airmont_get_scaling(void) 742 { 743 u64 value; 744 int i; 745 /* Defined in Table 35-10 from SDM (Sept 2015) */ 746 static int airmont_freq_table[] = { 747 83300, 100000, 133300, 116700, 80000, 748 93300, 90000, 88900, 87500}; 749 750 rdmsrl(MSR_FSB_FREQ, value); 751 i = value & 0xF; 752 WARN_ON(i > 8); 753 754 return airmont_freq_table[i]; 755 } 756 757 static void atom_get_vid(struct cpudata *cpudata) 758 { 759 u64 value; 760 761 rdmsrl(ATOM_VIDS, value); 762 cpudata->vid.min = int_tofp((value >> 8) & 0x7f); 763 cpudata->vid.max = int_tofp((value >> 16) & 0x7f); 764 cpudata->vid.ratio = div_fp( 765 cpudata->vid.max - cpudata->vid.min, 766 int_tofp(cpudata->pstate.max_pstate - 767 cpudata->pstate.min_pstate)); 768 769 rdmsrl(ATOM_TURBO_VIDS, value); 770 cpudata->vid.turbo = value & 0x7f; 771 } 772 773 static int core_get_min_pstate(void) 774 { 775 u64 value; 776 777 rdmsrl(MSR_PLATFORM_INFO, value); 778 return (value >> 40) & 0xFF; 779 } 780 781 static int core_get_max_pstate_physical(void) 782 { 783 u64 value; 784 785 rdmsrl(MSR_PLATFORM_INFO, value); 786 return (value >> 8) & 0xFF; 787 } 788 789 static int core_get_max_pstate(void) 790 { 791 u64 tar; 792 u64 plat_info; 793 int max_pstate; 794 int err; 795 796 rdmsrl(MSR_PLATFORM_INFO, plat_info); 797 max_pstate = (plat_info >> 8) & 0xFF; 798 799 err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar); 800 if (!err) { 801 /* Do some sanity checking for safety */ 802 if (plat_info & 0x600000000) { 803 u64 tdp_ctrl; 804 u64 tdp_ratio; 805 int tdp_msr; 806 807 err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl); 808 if (err) 809 goto skip_tar; 810 811 tdp_msr = MSR_CONFIG_TDP_NOMINAL + tdp_ctrl; 812 err = rdmsrl_safe(tdp_msr, &tdp_ratio); 813 if (err) 814 goto skip_tar; 815 816 /* For level 1 and 2, bits[23:16] contain the ratio */ 817 if (tdp_ctrl) 818 tdp_ratio >>= 16; 819 820 tdp_ratio &= 0xff; /* ratios are only 8 bits long */ 821 if (tdp_ratio - 1 == tar) { 822 max_pstate = tar; 823 pr_debug("max_pstate=TAC %x\n", max_pstate); 824 } else { 825 goto skip_tar; 826 } 827 } 828 } 829 830 skip_tar: 831 return max_pstate; 832 } 833 834 static int core_get_turbo_pstate(void) 835 { 836 u64 value; 837 int nont, ret; 838 839 rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value); 840 nont = core_get_max_pstate(); 841 ret = (value) & 255; 842 if (ret <= nont) 843 ret = nont; 844 return ret; 845 } 846 847 static inline int core_get_scaling(void) 848 { 849 return 100000; 850 } 851 852 static u64 core_get_val(struct cpudata *cpudata, int pstate) 853 { 854 u64 val; 855 856 val = (u64)pstate << 8; 857 if (limits->no_turbo && !limits->turbo_disabled) 858 val |= (u64)1 << 32; 859 860 return val; 861 } 862 863 static int knl_get_turbo_pstate(void) 864 { 865 u64 value; 866 int nont, ret; 867 868 rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value); 869 nont = core_get_max_pstate(); 870 ret = (((value) >> 8) & 0xFF); 871 if (ret <= nont) 872 ret = nont; 873 return ret; 874 } 875 876 static struct cpu_defaults core_params = { 877 .pid_policy = { 878 .sample_rate_ms = 10, 879 .deadband = 0, 880 .setpoint = 97, 881 .p_gain_pct = 20, 882 .d_gain_pct = 0, 883 .i_gain_pct = 0, 884 }, 885 .funcs = { 886 .get_max = core_get_max_pstate, 887 .get_max_physical = core_get_max_pstate_physical, 888 .get_min = core_get_min_pstate, 889 .get_turbo = core_get_turbo_pstate, 890 .get_scaling = core_get_scaling, 891 .get_val = core_get_val, 892 .get_target_pstate = get_target_pstate_use_performance, 893 }, 894 }; 895 896 static struct cpu_defaults silvermont_params = { 897 .pid_policy = { 898 .sample_rate_ms = 10, 899 .deadband = 0, 900 .setpoint = 60, 901 .p_gain_pct = 14, 902 .d_gain_pct = 0, 903 .i_gain_pct = 4, 904 }, 905 .funcs = { 906 .get_max = atom_get_max_pstate, 907 .get_max_physical = atom_get_max_pstate, 908 .get_min = atom_get_min_pstate, 909 .get_turbo = atom_get_turbo_pstate, 910 .get_val = atom_get_val, 911 .get_scaling = silvermont_get_scaling, 912 .get_vid = atom_get_vid, 913 .get_target_pstate = get_target_pstate_use_cpu_load, 914 }, 915 }; 916 917 static struct cpu_defaults airmont_params = { 918 .pid_policy = { 919 .sample_rate_ms = 10, 920 .deadband = 0, 921 .setpoint = 60, 922 .p_gain_pct = 14, 923 .d_gain_pct = 0, 924 .i_gain_pct = 4, 925 }, 926 .funcs = { 927 .get_max = atom_get_max_pstate, 928 .get_max_physical = atom_get_max_pstate, 929 .get_min = atom_get_min_pstate, 930 .get_turbo = atom_get_turbo_pstate, 931 .get_val = atom_get_val, 932 .get_scaling = airmont_get_scaling, 933 .get_vid = atom_get_vid, 934 .get_target_pstate = get_target_pstate_use_cpu_load, 935 }, 936 }; 937 938 static struct cpu_defaults knl_params = { 939 .pid_policy = { 940 .sample_rate_ms = 10, 941 .deadband = 0, 942 .setpoint = 97, 943 .p_gain_pct = 20, 944 .d_gain_pct = 0, 945 .i_gain_pct = 0, 946 }, 947 .funcs = { 948 .get_max = core_get_max_pstate, 949 .get_max_physical = core_get_max_pstate_physical, 950 .get_min = core_get_min_pstate, 951 .get_turbo = knl_get_turbo_pstate, 952 .get_scaling = core_get_scaling, 953 .get_val = core_get_val, 954 .get_target_pstate = get_target_pstate_use_performance, 955 }, 956 }; 957 958 static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) 959 { 960 int max_perf = cpu->pstate.turbo_pstate; 961 int max_perf_adj; 962 int min_perf; 963 964 if (limits->no_turbo || limits->turbo_disabled) 965 max_perf = cpu->pstate.max_pstate; 966 967 /* 968 * performance can be limited by user through sysfs, by cpufreq 969 * policy, or by cpu specific default values determined through 970 * experimentation. 971 */ 972 max_perf_adj = fp_toint(max_perf * limits->max_perf); 973 *max = clamp_t(int, max_perf_adj, 974 cpu->pstate.min_pstate, cpu->pstate.turbo_pstate); 975 976 min_perf = fp_toint(max_perf * limits->min_perf); 977 *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf); 978 } 979 980 static inline void intel_pstate_record_pstate(struct cpudata *cpu, int pstate) 981 { 982 trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); 983 cpu->pstate.current_pstate = pstate; 984 } 985 986 static void intel_pstate_set_min_pstate(struct cpudata *cpu) 987 { 988 int pstate = cpu->pstate.min_pstate; 989 990 intel_pstate_record_pstate(cpu, pstate); 991 /* 992 * Generally, there is no guarantee that this code will always run on 993 * the CPU being updated, so force the register update to run on the 994 * right CPU. 995 */ 996 wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL, 997 pstate_funcs.get_val(cpu, pstate)); 998 } 999 1000 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) 1001 { 1002 cpu->pstate.min_pstate = pstate_funcs.get_min(); 1003 cpu->pstate.max_pstate = pstate_funcs.get_max(); 1004 cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical(); 1005 cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(); 1006 cpu->pstate.scaling = pstate_funcs.get_scaling(); 1007 1008 if (pstate_funcs.get_vid) 1009 pstate_funcs.get_vid(cpu); 1010 1011 intel_pstate_set_min_pstate(cpu); 1012 } 1013 1014 static inline void intel_pstate_calc_busy(struct cpudata *cpu) 1015 { 1016 struct sample *sample = &cpu->sample; 1017 int64_t core_pct; 1018 1019 core_pct = int_tofp(sample->aperf) * int_tofp(100); 1020 core_pct = div64_u64(core_pct, int_tofp(sample->mperf)); 1021 1022 sample->core_pct_busy = (int32_t)core_pct; 1023 } 1024 1025 static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time) 1026 { 1027 u64 aperf, mperf; 1028 unsigned long flags; 1029 u64 tsc; 1030 1031 local_irq_save(flags); 1032 rdmsrl(MSR_IA32_APERF, aperf); 1033 rdmsrl(MSR_IA32_MPERF, mperf); 1034 tsc = rdtsc(); 1035 if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) { 1036 local_irq_restore(flags); 1037 return false; 1038 } 1039 local_irq_restore(flags); 1040 1041 cpu->last_sample_time = cpu->sample.time; 1042 cpu->sample.time = time; 1043 cpu->sample.aperf = aperf; 1044 cpu->sample.mperf = mperf; 1045 cpu->sample.tsc = tsc; 1046 cpu->sample.aperf -= cpu->prev_aperf; 1047 cpu->sample.mperf -= cpu->prev_mperf; 1048 cpu->sample.tsc -= cpu->prev_tsc; 1049 1050 cpu->prev_aperf = aperf; 1051 cpu->prev_mperf = mperf; 1052 cpu->prev_tsc = tsc; 1053 /* 1054 * First time this function is invoked in a given cycle, all of the 1055 * previous sample data fields are equal to zero or stale and they must 1056 * be populated with meaningful numbers for things to work, so assume 1057 * that sample.time will always be reset before setting the utilization 1058 * update hook and make the caller skip the sample then. 1059 */ 1060 return !!cpu->last_sample_time; 1061 } 1062 1063 static inline int32_t get_avg_frequency(struct cpudata *cpu) 1064 { 1065 return div64_u64(cpu->pstate.max_pstate_physical * cpu->sample.aperf * 1066 cpu->pstate.scaling, cpu->sample.mperf); 1067 } 1068 1069 static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu) 1070 { 1071 struct sample *sample = &cpu->sample; 1072 u64 cummulative_iowait, delta_iowait_us; 1073 u64 delta_iowait_mperf; 1074 u64 mperf, now; 1075 int32_t cpu_load; 1076 1077 cummulative_iowait = get_cpu_iowait_time_us(cpu->cpu, &now); 1078 1079 /* 1080 * Convert iowait time into number of IO cycles spent at max_freq. 1081 * IO is considered as busy only for the cpu_load algorithm. For 1082 * performance this is not needed since we always try to reach the 1083 * maximum P-State, so we are already boosting the IOs. 1084 */ 1085 delta_iowait_us = cummulative_iowait - cpu->prev_cummulative_iowait; 1086 delta_iowait_mperf = div64_u64(delta_iowait_us * cpu->pstate.scaling * 1087 cpu->pstate.max_pstate, MSEC_PER_SEC); 1088 1089 mperf = cpu->sample.mperf + delta_iowait_mperf; 1090 cpu->prev_cummulative_iowait = cummulative_iowait; 1091 1092 /* 1093 * The load can be estimated as the ratio of the mperf counter 1094 * running at a constant frequency during active periods 1095 * (C0) and the time stamp counter running at the same frequency 1096 * also during C-states. 1097 */ 1098 cpu_load = div64_u64(int_tofp(100) * mperf, sample->tsc); 1099 cpu->sample.busy_scaled = cpu_load; 1100 1101 return cpu->pstate.current_pstate - pid_calc(&cpu->pid, cpu_load); 1102 } 1103 1104 static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu) 1105 { 1106 int32_t core_busy, max_pstate, current_pstate, sample_ratio; 1107 u64 duration_ns; 1108 1109 intel_pstate_calc_busy(cpu); 1110 1111 /* 1112 * core_busy is the ratio of actual performance to max 1113 * max_pstate is the max non turbo pstate available 1114 * current_pstate was the pstate that was requested during 1115 * the last sample period. 1116 * 1117 * We normalize core_busy, which was our actual percent 1118 * performance to what we requested during the last sample 1119 * period. The result will be a percentage of busy at a 1120 * specified pstate. 1121 */ 1122 core_busy = cpu->sample.core_pct_busy; 1123 max_pstate = int_tofp(cpu->pstate.max_pstate_physical); 1124 current_pstate = int_tofp(cpu->pstate.current_pstate); 1125 core_busy = mul_fp(core_busy, div_fp(max_pstate, current_pstate)); 1126 1127 /* 1128 * Since our utilization update callback will not run unless we are 1129 * in C0, check if the actual elapsed time is significantly greater (3x) 1130 * than our sample interval. If it is, then we were idle for a long 1131 * enough period of time to adjust our busyness. 1132 */ 1133 duration_ns = cpu->sample.time - cpu->last_sample_time; 1134 if ((s64)duration_ns > pid_params.sample_rate_ns * 3) { 1135 sample_ratio = div_fp(int_tofp(pid_params.sample_rate_ns), 1136 int_tofp(duration_ns)); 1137 core_busy = mul_fp(core_busy, sample_ratio); 1138 } else { 1139 sample_ratio = div_fp(100 * cpu->sample.mperf, cpu->sample.tsc); 1140 if (sample_ratio < int_tofp(1)) 1141 core_busy = 0; 1142 } 1143 1144 cpu->sample.busy_scaled = core_busy; 1145 return cpu->pstate.current_pstate - pid_calc(&cpu->pid, core_busy); 1146 } 1147 1148 static inline void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) 1149 { 1150 int max_perf, min_perf; 1151 1152 update_turbo_state(); 1153 1154 intel_pstate_get_min_max(cpu, &min_perf, &max_perf); 1155 pstate = clamp_t(int, pstate, min_perf, max_perf); 1156 if (pstate == cpu->pstate.current_pstate) 1157 return; 1158 1159 intel_pstate_record_pstate(cpu, pstate); 1160 wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); 1161 } 1162 1163 static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) 1164 { 1165 int from, target_pstate; 1166 struct sample *sample; 1167 1168 from = cpu->pstate.current_pstate; 1169 1170 target_pstate = pstate_funcs.get_target_pstate(cpu); 1171 1172 intel_pstate_update_pstate(cpu, target_pstate); 1173 1174 sample = &cpu->sample; 1175 trace_pstate_sample(fp_toint(sample->core_pct_busy), 1176 fp_toint(sample->busy_scaled), 1177 from, 1178 cpu->pstate.current_pstate, 1179 sample->mperf, 1180 sample->aperf, 1181 sample->tsc, 1182 get_avg_frequency(cpu)); 1183 } 1184 1185 static void intel_pstate_update_util(struct update_util_data *data, u64 time, 1186 unsigned long util, unsigned long max) 1187 { 1188 struct cpudata *cpu = container_of(data, struct cpudata, update_util); 1189 u64 delta_ns = time - cpu->sample.time; 1190 1191 if ((s64)delta_ns >= pid_params.sample_rate_ns) { 1192 bool sample_taken = intel_pstate_sample(cpu, time); 1193 1194 if (sample_taken && !hwp_active) 1195 intel_pstate_adjust_busy_pstate(cpu); 1196 } 1197 } 1198 1199 #define ICPU(model, policy) \ 1200 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\ 1201 (unsigned long)&policy } 1202 1203 static const struct x86_cpu_id intel_pstate_cpu_ids[] = { 1204 ICPU(0x2a, core_params), 1205 ICPU(0x2d, core_params), 1206 ICPU(0x37, silvermont_params), 1207 ICPU(0x3a, core_params), 1208 ICPU(0x3c, core_params), 1209 ICPU(0x3d, core_params), 1210 ICPU(0x3e, core_params), 1211 ICPU(0x3f, core_params), 1212 ICPU(0x45, core_params), 1213 ICPU(0x46, core_params), 1214 ICPU(0x47, core_params), 1215 ICPU(0x4c, airmont_params), 1216 ICPU(0x4e, core_params), 1217 ICPU(0x4f, core_params), 1218 ICPU(0x5e, core_params), 1219 ICPU(0x56, core_params), 1220 ICPU(0x57, knl_params), 1221 {} 1222 }; 1223 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); 1224 1225 static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] = { 1226 ICPU(0x56, core_params), 1227 {} 1228 }; 1229 1230 static int intel_pstate_init_cpu(unsigned int cpunum) 1231 { 1232 struct cpudata *cpu; 1233 1234 if (!all_cpu_data[cpunum]) 1235 all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata), 1236 GFP_KERNEL); 1237 if (!all_cpu_data[cpunum]) 1238 return -ENOMEM; 1239 1240 cpu = all_cpu_data[cpunum]; 1241 1242 cpu->cpu = cpunum; 1243 1244 if (hwp_active) { 1245 intel_pstate_hwp_enable(cpu); 1246 pid_params.sample_rate_ms = 50; 1247 pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC; 1248 } 1249 1250 intel_pstate_get_cpu_pstates(cpu); 1251 1252 intel_pstate_busy_pid_reset(cpu); 1253 1254 cpu->update_util.func = intel_pstate_update_util; 1255 1256 pr_debug("intel_pstate: controlling: cpu %d\n", cpunum); 1257 1258 return 0; 1259 } 1260 1261 static unsigned int intel_pstate_get(unsigned int cpu_num) 1262 { 1263 struct sample *sample; 1264 struct cpudata *cpu; 1265 1266 cpu = all_cpu_data[cpu_num]; 1267 if (!cpu) 1268 return 0; 1269 sample = &cpu->sample; 1270 return get_avg_frequency(cpu); 1271 } 1272 1273 static void intel_pstate_set_update_util_hook(unsigned int cpu_num) 1274 { 1275 struct cpudata *cpu = all_cpu_data[cpu_num]; 1276 1277 /* Prevent intel_pstate_update_util() from using stale data. */ 1278 cpu->sample.time = 0; 1279 cpufreq_set_update_util_data(cpu_num, &cpu->update_util); 1280 } 1281 1282 static void intel_pstate_clear_update_util_hook(unsigned int cpu) 1283 { 1284 cpufreq_set_update_util_data(cpu, NULL); 1285 synchronize_sched(); 1286 } 1287 1288 static void intel_pstate_set_performance_limits(struct perf_limits *limits) 1289 { 1290 limits->no_turbo = 0; 1291 limits->turbo_disabled = 0; 1292 limits->max_perf_pct = 100; 1293 limits->max_perf = int_tofp(1); 1294 limits->min_perf_pct = 100; 1295 limits->min_perf = int_tofp(1); 1296 limits->max_policy_pct = 100; 1297 limits->max_sysfs_pct = 100; 1298 limits->min_policy_pct = 0; 1299 limits->min_sysfs_pct = 0; 1300 } 1301 1302 static int intel_pstate_set_policy(struct cpufreq_policy *policy) 1303 { 1304 if (!policy->cpuinfo.max_freq) 1305 return -ENODEV; 1306 1307 intel_pstate_clear_update_util_hook(policy->cpu); 1308 1309 if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) { 1310 limits = &performance_limits; 1311 if (policy->max >= policy->cpuinfo.max_freq) { 1312 pr_debug("intel_pstate: set performance\n"); 1313 intel_pstate_set_performance_limits(limits); 1314 goto out; 1315 } 1316 } else { 1317 pr_debug("intel_pstate: set powersave\n"); 1318 limits = &powersave_limits; 1319 } 1320 1321 limits->min_policy_pct = (policy->min * 100) / policy->cpuinfo.max_freq; 1322 limits->min_policy_pct = clamp_t(int, limits->min_policy_pct, 0 , 100); 1323 limits->max_policy_pct = DIV_ROUND_UP(policy->max * 100, 1324 policy->cpuinfo.max_freq); 1325 limits->max_policy_pct = clamp_t(int, limits->max_policy_pct, 0 , 100); 1326 1327 /* Normalize user input to [min_policy_pct, max_policy_pct] */ 1328 limits->min_perf_pct = max(limits->min_policy_pct, 1329 limits->min_sysfs_pct); 1330 limits->min_perf_pct = min(limits->max_policy_pct, 1331 limits->min_perf_pct); 1332 limits->max_perf_pct = min(limits->max_policy_pct, 1333 limits->max_sysfs_pct); 1334 limits->max_perf_pct = max(limits->min_policy_pct, 1335 limits->max_perf_pct); 1336 limits->max_perf = round_up(limits->max_perf, FRAC_BITS); 1337 1338 /* Make sure min_perf_pct <= max_perf_pct */ 1339 limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct); 1340 1341 limits->min_perf = div_fp(int_tofp(limits->min_perf_pct), 1342 int_tofp(100)); 1343 limits->max_perf = div_fp(int_tofp(limits->max_perf_pct), 1344 int_tofp(100)); 1345 1346 out: 1347 intel_pstate_set_update_util_hook(policy->cpu); 1348 1349 if (hwp_active) 1350 intel_pstate_hwp_set(policy->cpus); 1351 1352 return 0; 1353 } 1354 1355 static int intel_pstate_verify_policy(struct cpufreq_policy *policy) 1356 { 1357 cpufreq_verify_within_cpu_limits(policy); 1358 1359 if (policy->policy != CPUFREQ_POLICY_POWERSAVE && 1360 policy->policy != CPUFREQ_POLICY_PERFORMANCE) 1361 return -EINVAL; 1362 1363 return 0; 1364 } 1365 1366 static void intel_pstate_stop_cpu(struct cpufreq_policy *policy) 1367 { 1368 int cpu_num = policy->cpu; 1369 struct cpudata *cpu = all_cpu_data[cpu_num]; 1370 1371 pr_debug("intel_pstate: CPU %d exiting\n", cpu_num); 1372 1373 intel_pstate_clear_update_util_hook(cpu_num); 1374 1375 if (hwp_active) 1376 return; 1377 1378 intel_pstate_set_min_pstate(cpu); 1379 } 1380 1381 static int intel_pstate_cpu_init(struct cpufreq_policy *policy) 1382 { 1383 struct cpudata *cpu; 1384 int rc; 1385 1386 rc = intel_pstate_init_cpu(policy->cpu); 1387 if (rc) 1388 return rc; 1389 1390 cpu = all_cpu_data[policy->cpu]; 1391 1392 if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100) 1393 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 1394 else 1395 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1396 1397 policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling; 1398 policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling; 1399 1400 /* cpuinfo and default policy values */ 1401 policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling; 1402 policy->cpuinfo.max_freq = 1403 cpu->pstate.turbo_pstate * cpu->pstate.scaling; 1404 policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; 1405 cpumask_set_cpu(policy->cpu, policy->cpus); 1406 1407 return 0; 1408 } 1409 1410 static struct cpufreq_driver intel_pstate_driver = { 1411 .flags = CPUFREQ_CONST_LOOPS, 1412 .verify = intel_pstate_verify_policy, 1413 .setpolicy = intel_pstate_set_policy, 1414 .get = intel_pstate_get, 1415 .init = intel_pstate_cpu_init, 1416 .stop_cpu = intel_pstate_stop_cpu, 1417 .name = "intel_pstate", 1418 }; 1419 1420 static int __initdata no_load; 1421 static int __initdata no_hwp; 1422 static int __initdata hwp_only; 1423 static unsigned int force_load; 1424 1425 static int intel_pstate_msrs_not_valid(void) 1426 { 1427 if (!pstate_funcs.get_max() || 1428 !pstate_funcs.get_min() || 1429 !pstate_funcs.get_turbo()) 1430 return -ENODEV; 1431 1432 return 0; 1433 } 1434 1435 static void copy_pid_params(struct pstate_adjust_policy *policy) 1436 { 1437 pid_params.sample_rate_ms = policy->sample_rate_ms; 1438 pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC; 1439 pid_params.p_gain_pct = policy->p_gain_pct; 1440 pid_params.i_gain_pct = policy->i_gain_pct; 1441 pid_params.d_gain_pct = policy->d_gain_pct; 1442 pid_params.deadband = policy->deadband; 1443 pid_params.setpoint = policy->setpoint; 1444 } 1445 1446 static void copy_cpu_funcs(struct pstate_funcs *funcs) 1447 { 1448 pstate_funcs.get_max = funcs->get_max; 1449 pstate_funcs.get_max_physical = funcs->get_max_physical; 1450 pstate_funcs.get_min = funcs->get_min; 1451 pstate_funcs.get_turbo = funcs->get_turbo; 1452 pstate_funcs.get_scaling = funcs->get_scaling; 1453 pstate_funcs.get_val = funcs->get_val; 1454 pstate_funcs.get_vid = funcs->get_vid; 1455 pstate_funcs.get_target_pstate = funcs->get_target_pstate; 1456 1457 } 1458 1459 #if IS_ENABLED(CONFIG_ACPI) 1460 #include <acpi/processor.h> 1461 1462 static bool intel_pstate_no_acpi_pss(void) 1463 { 1464 int i; 1465 1466 for_each_possible_cpu(i) { 1467 acpi_status status; 1468 union acpi_object *pss; 1469 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; 1470 struct acpi_processor *pr = per_cpu(processors, i); 1471 1472 if (!pr) 1473 continue; 1474 1475 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer); 1476 if (ACPI_FAILURE(status)) 1477 continue; 1478 1479 pss = buffer.pointer; 1480 if (pss && pss->type == ACPI_TYPE_PACKAGE) { 1481 kfree(pss); 1482 return false; 1483 } 1484 1485 kfree(pss); 1486 } 1487 1488 return true; 1489 } 1490 1491 static bool intel_pstate_has_acpi_ppc(void) 1492 { 1493 int i; 1494 1495 for_each_possible_cpu(i) { 1496 struct acpi_processor *pr = per_cpu(processors, i); 1497 1498 if (!pr) 1499 continue; 1500 if (acpi_has_method(pr->handle, "_PPC")) 1501 return true; 1502 } 1503 return false; 1504 } 1505 1506 enum { 1507 PSS, 1508 PPC, 1509 }; 1510 1511 struct hw_vendor_info { 1512 u16 valid; 1513 char oem_id[ACPI_OEM_ID_SIZE]; 1514 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE]; 1515 int oem_pwr_table; 1516 }; 1517 1518 /* Hardware vendor-specific info that has its own power management modes */ 1519 static struct hw_vendor_info vendor_info[] = { 1520 {1, "HP ", "ProLiant", PSS}, 1521 {1, "ORACLE", "X4-2 ", PPC}, 1522 {1, "ORACLE", "X4-2L ", PPC}, 1523 {1, "ORACLE", "X4-2B ", PPC}, 1524 {1, "ORACLE", "X3-2 ", PPC}, 1525 {1, "ORACLE", "X3-2L ", PPC}, 1526 {1, "ORACLE", "X3-2B ", PPC}, 1527 {1, "ORACLE", "X4470M2 ", PPC}, 1528 {1, "ORACLE", "X4270M3 ", PPC}, 1529 {1, "ORACLE", "X4270M2 ", PPC}, 1530 {1, "ORACLE", "X4170M2 ", PPC}, 1531 {1, "ORACLE", "X4170 M3", PPC}, 1532 {1, "ORACLE", "X4275 M3", PPC}, 1533 {1, "ORACLE", "X6-2 ", PPC}, 1534 {1, "ORACLE", "Sudbury ", PPC}, 1535 {0, "", ""}, 1536 }; 1537 1538 static bool intel_pstate_platform_pwr_mgmt_exists(void) 1539 { 1540 struct acpi_table_header hdr; 1541 struct hw_vendor_info *v_info; 1542 const struct x86_cpu_id *id; 1543 u64 misc_pwr; 1544 1545 id = x86_match_cpu(intel_pstate_cpu_oob_ids); 1546 if (id) { 1547 rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr); 1548 if ( misc_pwr & (1 << 8)) 1549 return true; 1550 } 1551 1552 if (acpi_disabled || 1553 ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr))) 1554 return false; 1555 1556 for (v_info = vendor_info; v_info->valid; v_info++) { 1557 if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) && 1558 !strncmp(hdr.oem_table_id, v_info->oem_table_id, 1559 ACPI_OEM_TABLE_ID_SIZE)) 1560 switch (v_info->oem_pwr_table) { 1561 case PSS: 1562 return intel_pstate_no_acpi_pss(); 1563 case PPC: 1564 return intel_pstate_has_acpi_ppc() && 1565 (!force_load); 1566 } 1567 } 1568 1569 return false; 1570 } 1571 #else /* CONFIG_ACPI not enabled */ 1572 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; } 1573 static inline bool intel_pstate_has_acpi_ppc(void) { return false; } 1574 #endif /* CONFIG_ACPI */ 1575 1576 static const struct x86_cpu_id hwp_support_ids[] __initconst = { 1577 { X86_VENDOR_INTEL, 6, X86_MODEL_ANY, X86_FEATURE_HWP }, 1578 {} 1579 }; 1580 1581 static int __init intel_pstate_init(void) 1582 { 1583 int cpu, rc = 0; 1584 const struct x86_cpu_id *id; 1585 struct cpu_defaults *cpu_def; 1586 1587 if (no_load) 1588 return -ENODEV; 1589 1590 if (x86_match_cpu(hwp_support_ids) && !no_hwp) { 1591 copy_cpu_funcs(&core_params.funcs); 1592 hwp_active++; 1593 goto hwp_cpu_matched; 1594 } 1595 1596 id = x86_match_cpu(intel_pstate_cpu_ids); 1597 if (!id) 1598 return -ENODEV; 1599 1600 cpu_def = (struct cpu_defaults *)id->driver_data; 1601 1602 copy_pid_params(&cpu_def->pid_policy); 1603 copy_cpu_funcs(&cpu_def->funcs); 1604 1605 if (intel_pstate_msrs_not_valid()) 1606 return -ENODEV; 1607 1608 hwp_cpu_matched: 1609 /* 1610 * The Intel pstate driver will be ignored if the platform 1611 * firmware has its own power management modes. 1612 */ 1613 if (intel_pstate_platform_pwr_mgmt_exists()) 1614 return -ENODEV; 1615 1616 pr_info("Intel P-state driver initializing.\n"); 1617 1618 all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus()); 1619 if (!all_cpu_data) 1620 return -ENOMEM; 1621 1622 if (!hwp_active && hwp_only) 1623 goto out; 1624 1625 rc = cpufreq_register_driver(&intel_pstate_driver); 1626 if (rc) 1627 goto out; 1628 1629 intel_pstate_debug_expose_params(); 1630 intel_pstate_sysfs_expose_params(); 1631 1632 if (hwp_active) 1633 pr_info("intel_pstate: HWP enabled\n"); 1634 1635 return rc; 1636 out: 1637 get_online_cpus(); 1638 for_each_online_cpu(cpu) { 1639 if (all_cpu_data[cpu]) { 1640 intel_pstate_clear_update_util_hook(cpu); 1641 kfree(all_cpu_data[cpu]); 1642 } 1643 } 1644 1645 put_online_cpus(); 1646 vfree(all_cpu_data); 1647 return -ENODEV; 1648 } 1649 device_initcall(intel_pstate_init); 1650 1651 static int __init intel_pstate_setup(char *str) 1652 { 1653 if (!str) 1654 return -EINVAL; 1655 1656 if (!strcmp(str, "disable")) 1657 no_load = 1; 1658 if (!strcmp(str, "no_hwp")) { 1659 pr_info("intel_pstate: HWP disabled\n"); 1660 no_hwp = 1; 1661 } 1662 if (!strcmp(str, "force")) 1663 force_load = 1; 1664 if (!strcmp(str, "hwp_only")) 1665 hwp_only = 1; 1666 return 0; 1667 } 1668 early_param("intel_pstate", intel_pstate_setup); 1669 1670 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>"); 1671 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors"); 1672 MODULE_LICENSE("GPL"); 1673