1 /* 2 * intel_pstate.c: Native P state management for Intel processors 3 * 4 * (C) Copyright 2012 Intel Corporation 5 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com> 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; version 2 10 * of the License. 11 */ 12 13 #include <linux/kernel.h> 14 #include <linux/kernel_stat.h> 15 #include <linux/module.h> 16 #include <linux/ktime.h> 17 #include <linux/hrtimer.h> 18 #include <linux/tick.h> 19 #include <linux/slab.h> 20 #include <linux/sched.h> 21 #include <linux/list.h> 22 #include <linux/cpu.h> 23 #include <linux/cpufreq.h> 24 #include <linux/sysfs.h> 25 #include <linux/types.h> 26 #include <linux/fs.h> 27 #include <linux/debugfs.h> 28 #include <linux/acpi.h> 29 #include <trace/events/power.h> 30 31 #include <asm/div64.h> 32 #include <asm/msr.h> 33 #include <asm/cpu_device_id.h> 34 35 #define SAMPLE_COUNT 3 36 37 #define BYT_RATIOS 0x66a 38 #define BYT_VIDS 0x66b 39 #define BYT_TURBO_RATIOS 0x66c 40 41 42 #define FRAC_BITS 8 43 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS) 44 #define fp_toint(X) ((X) >> FRAC_BITS) 45 46 static inline int32_t mul_fp(int32_t x, int32_t y) 47 { 48 return ((int64_t)x * (int64_t)y) >> FRAC_BITS; 49 } 50 51 static inline int32_t div_fp(int32_t x, int32_t y) 52 { 53 return div_s64((int64_t)x << FRAC_BITS, (int64_t)y); 54 } 55 56 struct sample { 57 int32_t core_pct_busy; 58 u64 aperf; 59 u64 mperf; 60 unsigned long long tsc; 61 int freq; 62 }; 63 64 struct pstate_data { 65 int current_pstate; 66 int min_pstate; 67 int max_pstate; 68 int turbo_pstate; 69 }; 70 71 struct vid_data { 72 int32_t min; 73 int32_t max; 74 int32_t ratio; 75 }; 76 77 struct _pid { 78 int setpoint; 79 int32_t integral; 80 int32_t p_gain; 81 int32_t i_gain; 82 int32_t d_gain; 83 int deadband; 84 int32_t last_err; 85 }; 86 87 struct cpudata { 88 int cpu; 89 90 char name[64]; 91 92 struct timer_list timer; 93 94 struct pstate_data pstate; 95 struct vid_data vid; 96 struct _pid pid; 97 98 u64 prev_aperf; 99 u64 prev_mperf; 100 unsigned long long prev_tsc; 101 int sample_ptr; 102 struct sample samples[SAMPLE_COUNT]; 103 }; 104 105 static struct cpudata **all_cpu_data; 106 struct pstate_adjust_policy { 107 int sample_rate_ms; 108 int deadband; 109 int setpoint; 110 int p_gain_pct; 111 int d_gain_pct; 112 int i_gain_pct; 113 }; 114 115 struct pstate_funcs { 116 int (*get_max)(void); 117 int (*get_min)(void); 118 int (*get_turbo)(void); 119 void (*set)(struct cpudata*, int pstate); 120 void (*get_vid)(struct cpudata *); 121 }; 122 123 struct cpu_defaults { 124 struct pstate_adjust_policy pid_policy; 125 struct pstate_funcs funcs; 126 }; 127 128 static struct pstate_adjust_policy pid_params; 129 static struct pstate_funcs pstate_funcs; 130 131 struct perf_limits { 132 int no_turbo; 133 int max_perf_pct; 134 int min_perf_pct; 135 int32_t max_perf; 136 int32_t min_perf; 137 int max_policy_pct; 138 int max_sysfs_pct; 139 }; 140 141 static struct perf_limits limits = { 142 .no_turbo = 0, 143 .max_perf_pct = 100, 144 .max_perf = int_tofp(1), 145 .min_perf_pct = 0, 146 .min_perf = 0, 147 .max_policy_pct = 100, 148 .max_sysfs_pct = 100, 149 }; 150 151 static inline void pid_reset(struct _pid *pid, int setpoint, int busy, 152 int deadband, int integral) { 153 pid->setpoint = setpoint; 154 pid->deadband = deadband; 155 pid->integral = int_tofp(integral); 156 pid->last_err = setpoint - busy; 157 } 158 159 static inline void pid_p_gain_set(struct _pid *pid, int percent) 160 { 161 pid->p_gain = div_fp(int_tofp(percent), int_tofp(100)); 162 } 163 164 static inline void pid_i_gain_set(struct _pid *pid, int percent) 165 { 166 pid->i_gain = div_fp(int_tofp(percent), int_tofp(100)); 167 } 168 169 static inline void pid_d_gain_set(struct _pid *pid, int percent) 170 { 171 172 pid->d_gain = div_fp(int_tofp(percent), int_tofp(100)); 173 } 174 175 static signed int pid_calc(struct _pid *pid, int32_t busy) 176 { 177 signed int result; 178 int32_t pterm, dterm, fp_error; 179 int32_t integral_limit; 180 181 fp_error = int_tofp(pid->setpoint) - busy; 182 183 if (abs(fp_error) <= int_tofp(pid->deadband)) 184 return 0; 185 186 pterm = mul_fp(pid->p_gain, fp_error); 187 188 pid->integral += fp_error; 189 190 /* limit the integral term */ 191 integral_limit = int_tofp(30); 192 if (pid->integral > integral_limit) 193 pid->integral = integral_limit; 194 if (pid->integral < -integral_limit) 195 pid->integral = -integral_limit; 196 197 dterm = mul_fp(pid->d_gain, fp_error - pid->last_err); 198 pid->last_err = fp_error; 199 200 result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm; 201 202 return (signed int)fp_toint(result); 203 } 204 205 static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu) 206 { 207 pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct); 208 pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct); 209 pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct); 210 211 pid_reset(&cpu->pid, 212 pid_params.setpoint, 213 100, 214 pid_params.deadband, 215 0); 216 } 217 218 static inline void intel_pstate_reset_all_pid(void) 219 { 220 unsigned int cpu; 221 for_each_online_cpu(cpu) { 222 if (all_cpu_data[cpu]) 223 intel_pstate_busy_pid_reset(all_cpu_data[cpu]); 224 } 225 } 226 227 /************************** debugfs begin ************************/ 228 static int pid_param_set(void *data, u64 val) 229 { 230 *(u32 *)data = val; 231 intel_pstate_reset_all_pid(); 232 return 0; 233 } 234 static int pid_param_get(void *data, u64 *val) 235 { 236 *val = *(u32 *)data; 237 return 0; 238 } 239 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, 240 pid_param_set, "%llu\n"); 241 242 struct pid_param { 243 char *name; 244 void *value; 245 }; 246 247 static struct pid_param pid_files[] = { 248 {"sample_rate_ms", &pid_params.sample_rate_ms}, 249 {"d_gain_pct", &pid_params.d_gain_pct}, 250 {"i_gain_pct", &pid_params.i_gain_pct}, 251 {"deadband", &pid_params.deadband}, 252 {"setpoint", &pid_params.setpoint}, 253 {"p_gain_pct", &pid_params.p_gain_pct}, 254 {NULL, NULL} 255 }; 256 257 static struct dentry *debugfs_parent; 258 static void intel_pstate_debug_expose_params(void) 259 { 260 int i = 0; 261 262 debugfs_parent = debugfs_create_dir("pstate_snb", NULL); 263 if (IS_ERR_OR_NULL(debugfs_parent)) 264 return; 265 while (pid_files[i].name) { 266 debugfs_create_file(pid_files[i].name, 0660, 267 debugfs_parent, pid_files[i].value, 268 &fops_pid_param); 269 i++; 270 } 271 } 272 273 /************************** debugfs end ************************/ 274 275 /************************** sysfs begin ************************/ 276 #define show_one(file_name, object) \ 277 static ssize_t show_##file_name \ 278 (struct kobject *kobj, struct attribute *attr, char *buf) \ 279 { \ 280 return sprintf(buf, "%u\n", limits.object); \ 281 } 282 283 static ssize_t store_no_turbo(struct kobject *a, struct attribute *b, 284 const char *buf, size_t count) 285 { 286 unsigned int input; 287 int ret; 288 ret = sscanf(buf, "%u", &input); 289 if (ret != 1) 290 return -EINVAL; 291 limits.no_turbo = clamp_t(int, input, 0 , 1); 292 293 return count; 294 } 295 296 static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b, 297 const char *buf, size_t count) 298 { 299 unsigned int input; 300 int ret; 301 ret = sscanf(buf, "%u", &input); 302 if (ret != 1) 303 return -EINVAL; 304 305 limits.max_sysfs_pct = clamp_t(int, input, 0 , 100); 306 limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct); 307 limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100)); 308 return count; 309 } 310 311 static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b, 312 const char *buf, size_t count) 313 { 314 unsigned int input; 315 int ret; 316 ret = sscanf(buf, "%u", &input); 317 if (ret != 1) 318 return -EINVAL; 319 limits.min_perf_pct = clamp_t(int, input, 0 , 100); 320 limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100)); 321 322 return count; 323 } 324 325 show_one(no_turbo, no_turbo); 326 show_one(max_perf_pct, max_perf_pct); 327 show_one(min_perf_pct, min_perf_pct); 328 329 define_one_global_rw(no_turbo); 330 define_one_global_rw(max_perf_pct); 331 define_one_global_rw(min_perf_pct); 332 333 static struct attribute *intel_pstate_attributes[] = { 334 &no_turbo.attr, 335 &max_perf_pct.attr, 336 &min_perf_pct.attr, 337 NULL 338 }; 339 340 static struct attribute_group intel_pstate_attr_group = { 341 .attrs = intel_pstate_attributes, 342 }; 343 static struct kobject *intel_pstate_kobject; 344 345 static void intel_pstate_sysfs_expose_params(void) 346 { 347 int rc; 348 349 intel_pstate_kobject = kobject_create_and_add("intel_pstate", 350 &cpu_subsys.dev_root->kobj); 351 BUG_ON(!intel_pstate_kobject); 352 rc = sysfs_create_group(intel_pstate_kobject, 353 &intel_pstate_attr_group); 354 BUG_ON(rc); 355 } 356 357 /************************** sysfs end ************************/ 358 static int byt_get_min_pstate(void) 359 { 360 u64 value; 361 rdmsrl(BYT_RATIOS, value); 362 return (value >> 8) & 0xFF; 363 } 364 365 static int byt_get_max_pstate(void) 366 { 367 u64 value; 368 rdmsrl(BYT_RATIOS, value); 369 return (value >> 16) & 0xFF; 370 } 371 372 static int byt_get_turbo_pstate(void) 373 { 374 u64 value; 375 rdmsrl(BYT_TURBO_RATIOS, value); 376 return value & 0x3F; 377 } 378 379 static void byt_set_pstate(struct cpudata *cpudata, int pstate) 380 { 381 u64 val; 382 int32_t vid_fp; 383 u32 vid; 384 385 val = pstate << 8; 386 if (limits.no_turbo) 387 val |= (u64)1 << 32; 388 389 vid_fp = cpudata->vid.min + mul_fp( 390 int_tofp(pstate - cpudata->pstate.min_pstate), 391 cpudata->vid.ratio); 392 393 vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max); 394 vid = fp_toint(vid_fp); 395 396 val |= vid; 397 398 wrmsrl(MSR_IA32_PERF_CTL, val); 399 } 400 401 static void byt_get_vid(struct cpudata *cpudata) 402 { 403 u64 value; 404 405 rdmsrl(BYT_VIDS, value); 406 cpudata->vid.min = int_tofp((value >> 8) & 0x7f); 407 cpudata->vid.max = int_tofp((value >> 16) & 0x7f); 408 cpudata->vid.ratio = div_fp( 409 cpudata->vid.max - cpudata->vid.min, 410 int_tofp(cpudata->pstate.max_pstate - 411 cpudata->pstate.min_pstate)); 412 } 413 414 415 static int core_get_min_pstate(void) 416 { 417 u64 value; 418 rdmsrl(MSR_PLATFORM_INFO, value); 419 return (value >> 40) & 0xFF; 420 } 421 422 static int core_get_max_pstate(void) 423 { 424 u64 value; 425 rdmsrl(MSR_PLATFORM_INFO, value); 426 return (value >> 8) & 0xFF; 427 } 428 429 static int core_get_turbo_pstate(void) 430 { 431 u64 value; 432 int nont, ret; 433 rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value); 434 nont = core_get_max_pstate(); 435 ret = ((value) & 255); 436 if (ret <= nont) 437 ret = nont; 438 return ret; 439 } 440 441 static void core_set_pstate(struct cpudata *cpudata, int pstate) 442 { 443 u64 val; 444 445 val = pstate << 8; 446 if (limits.no_turbo) 447 val |= (u64)1 << 32; 448 449 wrmsrl(MSR_IA32_PERF_CTL, val); 450 } 451 452 static struct cpu_defaults core_params = { 453 .pid_policy = { 454 .sample_rate_ms = 10, 455 .deadband = 0, 456 .setpoint = 97, 457 .p_gain_pct = 20, 458 .d_gain_pct = 0, 459 .i_gain_pct = 0, 460 }, 461 .funcs = { 462 .get_max = core_get_max_pstate, 463 .get_min = core_get_min_pstate, 464 .get_turbo = core_get_turbo_pstate, 465 .set = core_set_pstate, 466 }, 467 }; 468 469 static struct cpu_defaults byt_params = { 470 .pid_policy = { 471 .sample_rate_ms = 10, 472 .deadband = 0, 473 .setpoint = 97, 474 .p_gain_pct = 14, 475 .d_gain_pct = 0, 476 .i_gain_pct = 4, 477 }, 478 .funcs = { 479 .get_max = byt_get_max_pstate, 480 .get_min = byt_get_min_pstate, 481 .get_turbo = byt_get_turbo_pstate, 482 .set = byt_set_pstate, 483 .get_vid = byt_get_vid, 484 }, 485 }; 486 487 488 static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) 489 { 490 int max_perf = cpu->pstate.turbo_pstate; 491 int max_perf_adj; 492 int min_perf; 493 if (limits.no_turbo) 494 max_perf = cpu->pstate.max_pstate; 495 496 max_perf_adj = fp_toint(mul_fp(int_tofp(max_perf), limits.max_perf)); 497 *max = clamp_t(int, max_perf_adj, 498 cpu->pstate.min_pstate, cpu->pstate.turbo_pstate); 499 500 min_perf = fp_toint(mul_fp(int_tofp(max_perf), limits.min_perf)); 501 *min = clamp_t(int, min_perf, 502 cpu->pstate.min_pstate, max_perf); 503 } 504 505 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) 506 { 507 int max_perf, min_perf; 508 509 intel_pstate_get_min_max(cpu, &min_perf, &max_perf); 510 511 pstate = clamp_t(int, pstate, min_perf, max_perf); 512 513 if (pstate == cpu->pstate.current_pstate) 514 return; 515 516 trace_cpu_frequency(pstate * 100000, cpu->cpu); 517 518 cpu->pstate.current_pstate = pstate; 519 520 pstate_funcs.set(cpu, pstate); 521 } 522 523 static inline void intel_pstate_pstate_increase(struct cpudata *cpu, int steps) 524 { 525 int target; 526 target = cpu->pstate.current_pstate + steps; 527 528 intel_pstate_set_pstate(cpu, target); 529 } 530 531 static inline void intel_pstate_pstate_decrease(struct cpudata *cpu, int steps) 532 { 533 int target; 534 target = cpu->pstate.current_pstate - steps; 535 intel_pstate_set_pstate(cpu, target); 536 } 537 538 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) 539 { 540 sprintf(cpu->name, "Intel 2nd generation core"); 541 542 cpu->pstate.min_pstate = pstate_funcs.get_min(); 543 cpu->pstate.max_pstate = pstate_funcs.get_max(); 544 cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(); 545 546 if (pstate_funcs.get_vid) 547 pstate_funcs.get_vid(cpu); 548 549 /* 550 * goto max pstate so we don't slow up boot if we are built-in if we are 551 * a module we will take care of it during normal operation 552 */ 553 intel_pstate_set_pstate(cpu, cpu->pstate.max_pstate); 554 } 555 556 static inline void intel_pstate_calc_busy(struct cpudata *cpu, 557 struct sample *sample) 558 { 559 u64 core_pct; 560 u64 c0_pct; 561 562 core_pct = div64_u64(sample->aperf * 100, sample->mperf); 563 564 c0_pct = div64_u64(sample->mperf * 100, sample->tsc); 565 sample->freq = fp_toint( 566 mul_fp(int_tofp(cpu->pstate.max_pstate), 567 int_tofp(core_pct * 1000))); 568 569 sample->core_pct_busy = mul_fp(int_tofp(core_pct), 570 div_fp(int_tofp(c0_pct + 1), int_tofp(100))); 571 } 572 573 static inline void intel_pstate_sample(struct cpudata *cpu) 574 { 575 u64 aperf, mperf; 576 unsigned long long tsc; 577 578 rdmsrl(MSR_IA32_APERF, aperf); 579 rdmsrl(MSR_IA32_MPERF, mperf); 580 tsc = native_read_tsc(); 581 582 cpu->sample_ptr = (cpu->sample_ptr + 1) % SAMPLE_COUNT; 583 cpu->samples[cpu->sample_ptr].aperf = aperf; 584 cpu->samples[cpu->sample_ptr].mperf = mperf; 585 cpu->samples[cpu->sample_ptr].tsc = tsc; 586 cpu->samples[cpu->sample_ptr].aperf -= cpu->prev_aperf; 587 cpu->samples[cpu->sample_ptr].mperf -= cpu->prev_mperf; 588 cpu->samples[cpu->sample_ptr].tsc -= cpu->prev_tsc; 589 590 intel_pstate_calc_busy(cpu, &cpu->samples[cpu->sample_ptr]); 591 592 cpu->prev_aperf = aperf; 593 cpu->prev_mperf = mperf; 594 cpu->prev_tsc = tsc; 595 } 596 597 static inline void intel_pstate_set_sample_time(struct cpudata *cpu) 598 { 599 int sample_time, delay; 600 601 sample_time = pid_params.sample_rate_ms; 602 delay = msecs_to_jiffies(sample_time); 603 mod_timer_pinned(&cpu->timer, jiffies + delay); 604 } 605 606 static inline int32_t intel_pstate_get_scaled_busy(struct cpudata *cpu) 607 { 608 int32_t core_busy, max_pstate, current_pstate; 609 610 core_busy = cpu->samples[cpu->sample_ptr].core_pct_busy; 611 max_pstate = int_tofp(cpu->pstate.max_pstate); 612 current_pstate = int_tofp(cpu->pstate.current_pstate); 613 return mul_fp(core_busy, div_fp(max_pstate, current_pstate)); 614 } 615 616 static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) 617 { 618 int32_t busy_scaled; 619 struct _pid *pid; 620 signed int ctl = 0; 621 int steps; 622 623 pid = &cpu->pid; 624 busy_scaled = intel_pstate_get_scaled_busy(cpu); 625 626 ctl = pid_calc(pid, busy_scaled); 627 628 steps = abs(ctl); 629 630 if (ctl < 0) 631 intel_pstate_pstate_increase(cpu, steps); 632 else 633 intel_pstate_pstate_decrease(cpu, steps); 634 } 635 636 static void intel_pstate_timer_func(unsigned long __data) 637 { 638 struct cpudata *cpu = (struct cpudata *) __data; 639 struct sample *sample; 640 641 intel_pstate_sample(cpu); 642 643 sample = &cpu->samples[cpu->sample_ptr]; 644 645 intel_pstate_adjust_busy_pstate(cpu); 646 647 trace_pstate_sample(fp_toint(sample->core_pct_busy), 648 fp_toint(intel_pstate_get_scaled_busy(cpu)), 649 cpu->pstate.current_pstate, 650 sample->mperf, 651 sample->aperf, 652 sample->freq); 653 654 intel_pstate_set_sample_time(cpu); 655 } 656 657 #define ICPU(model, policy) \ 658 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\ 659 (unsigned long)&policy } 660 661 static const struct x86_cpu_id intel_pstate_cpu_ids[] = { 662 ICPU(0x2a, core_params), 663 ICPU(0x2d, core_params), 664 ICPU(0x37, byt_params), 665 ICPU(0x3a, core_params), 666 ICPU(0x3c, core_params), 667 ICPU(0x3e, core_params), 668 ICPU(0x3f, core_params), 669 ICPU(0x45, core_params), 670 ICPU(0x46, core_params), 671 {} 672 }; 673 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); 674 675 static int intel_pstate_init_cpu(unsigned int cpunum) 676 { 677 678 const struct x86_cpu_id *id; 679 struct cpudata *cpu; 680 681 id = x86_match_cpu(intel_pstate_cpu_ids); 682 if (!id) 683 return -ENODEV; 684 685 all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata), GFP_KERNEL); 686 if (!all_cpu_data[cpunum]) 687 return -ENOMEM; 688 689 cpu = all_cpu_data[cpunum]; 690 691 intel_pstate_get_cpu_pstates(cpu); 692 if (!cpu->pstate.current_pstate) { 693 all_cpu_data[cpunum] = NULL; 694 kfree(cpu); 695 return -ENODATA; 696 } 697 698 cpu->cpu = cpunum; 699 700 init_timer_deferrable(&cpu->timer); 701 cpu->timer.function = intel_pstate_timer_func; 702 cpu->timer.data = 703 (unsigned long)cpu; 704 cpu->timer.expires = jiffies + HZ/100; 705 intel_pstate_busy_pid_reset(cpu); 706 intel_pstate_sample(cpu); 707 intel_pstate_set_pstate(cpu, cpu->pstate.max_pstate); 708 709 add_timer_on(&cpu->timer, cpunum); 710 711 pr_info("Intel pstate controlling: cpu %d\n", cpunum); 712 713 return 0; 714 } 715 716 static unsigned int intel_pstate_get(unsigned int cpu_num) 717 { 718 struct sample *sample; 719 struct cpudata *cpu; 720 721 cpu = all_cpu_data[cpu_num]; 722 if (!cpu) 723 return 0; 724 sample = &cpu->samples[cpu->sample_ptr]; 725 return sample->freq; 726 } 727 728 static int intel_pstate_set_policy(struct cpufreq_policy *policy) 729 { 730 struct cpudata *cpu; 731 732 cpu = all_cpu_data[policy->cpu]; 733 734 if (!policy->cpuinfo.max_freq) 735 return -ENODEV; 736 737 if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) { 738 limits.min_perf_pct = 100; 739 limits.min_perf = int_tofp(1); 740 limits.max_perf_pct = 100; 741 limits.max_perf = int_tofp(1); 742 limits.no_turbo = 0; 743 return 0; 744 } 745 limits.min_perf_pct = (policy->min * 100) / policy->cpuinfo.max_freq; 746 limits.min_perf_pct = clamp_t(int, limits.min_perf_pct, 0 , 100); 747 limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100)); 748 749 limits.max_policy_pct = policy->max * 100 / policy->cpuinfo.max_freq; 750 limits.max_policy_pct = clamp_t(int, limits.max_policy_pct, 0 , 100); 751 limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct); 752 limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100)); 753 754 return 0; 755 } 756 757 static int intel_pstate_verify_policy(struct cpufreq_policy *policy) 758 { 759 cpufreq_verify_within_cpu_limits(policy); 760 761 if ((policy->policy != CPUFREQ_POLICY_POWERSAVE) && 762 (policy->policy != CPUFREQ_POLICY_PERFORMANCE)) 763 return -EINVAL; 764 765 return 0; 766 } 767 768 static int intel_pstate_cpu_exit(struct cpufreq_policy *policy) 769 { 770 int cpu = policy->cpu; 771 772 del_timer(&all_cpu_data[cpu]->timer); 773 kfree(all_cpu_data[cpu]); 774 all_cpu_data[cpu] = NULL; 775 return 0; 776 } 777 778 static int intel_pstate_cpu_init(struct cpufreq_policy *policy) 779 { 780 struct cpudata *cpu; 781 int rc; 782 783 rc = intel_pstate_init_cpu(policy->cpu); 784 if (rc) 785 return rc; 786 787 cpu = all_cpu_data[policy->cpu]; 788 789 if (!limits.no_turbo && 790 limits.min_perf_pct == 100 && limits.max_perf_pct == 100) 791 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 792 else 793 policy->policy = CPUFREQ_POLICY_POWERSAVE; 794 795 policy->min = cpu->pstate.min_pstate * 100000; 796 policy->max = cpu->pstate.turbo_pstate * 100000; 797 798 /* cpuinfo and default policy values */ 799 policy->cpuinfo.min_freq = cpu->pstate.min_pstate * 100000; 800 policy->cpuinfo.max_freq = cpu->pstate.turbo_pstate * 100000; 801 policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; 802 cpumask_set_cpu(policy->cpu, policy->cpus); 803 804 return 0; 805 } 806 807 static struct cpufreq_driver intel_pstate_driver = { 808 .flags = CPUFREQ_CONST_LOOPS, 809 .verify = intel_pstate_verify_policy, 810 .setpolicy = intel_pstate_set_policy, 811 .get = intel_pstate_get, 812 .init = intel_pstate_cpu_init, 813 .exit = intel_pstate_cpu_exit, 814 .name = "intel_pstate", 815 }; 816 817 static int __initdata no_load; 818 819 static int intel_pstate_msrs_not_valid(void) 820 { 821 /* Check that all the msr's we are using are valid. */ 822 u64 aperf, mperf, tmp; 823 824 rdmsrl(MSR_IA32_APERF, aperf); 825 rdmsrl(MSR_IA32_MPERF, mperf); 826 827 if (!pstate_funcs.get_max() || 828 !pstate_funcs.get_min() || 829 !pstate_funcs.get_turbo()) 830 return -ENODEV; 831 832 rdmsrl(MSR_IA32_APERF, tmp); 833 if (!(tmp - aperf)) 834 return -ENODEV; 835 836 rdmsrl(MSR_IA32_MPERF, tmp); 837 if (!(tmp - mperf)) 838 return -ENODEV; 839 840 return 0; 841 } 842 843 static void copy_pid_params(struct pstate_adjust_policy *policy) 844 { 845 pid_params.sample_rate_ms = policy->sample_rate_ms; 846 pid_params.p_gain_pct = policy->p_gain_pct; 847 pid_params.i_gain_pct = policy->i_gain_pct; 848 pid_params.d_gain_pct = policy->d_gain_pct; 849 pid_params.deadband = policy->deadband; 850 pid_params.setpoint = policy->setpoint; 851 } 852 853 static void copy_cpu_funcs(struct pstate_funcs *funcs) 854 { 855 pstate_funcs.get_max = funcs->get_max; 856 pstate_funcs.get_min = funcs->get_min; 857 pstate_funcs.get_turbo = funcs->get_turbo; 858 pstate_funcs.set = funcs->set; 859 pstate_funcs.get_vid = funcs->get_vid; 860 } 861 862 #if IS_ENABLED(CONFIG_ACPI) 863 #include <acpi/processor.h> 864 865 static bool intel_pstate_no_acpi_pss(void) 866 { 867 int i; 868 869 for_each_possible_cpu(i) { 870 acpi_status status; 871 union acpi_object *pss; 872 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; 873 struct acpi_processor *pr = per_cpu(processors, i); 874 875 if (!pr) 876 continue; 877 878 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer); 879 if (ACPI_FAILURE(status)) 880 continue; 881 882 pss = buffer.pointer; 883 if (pss && pss->type == ACPI_TYPE_PACKAGE) { 884 kfree(pss); 885 return false; 886 } 887 888 kfree(pss); 889 } 890 891 return true; 892 } 893 894 struct hw_vendor_info { 895 u16 valid; 896 char oem_id[ACPI_OEM_ID_SIZE]; 897 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE]; 898 }; 899 900 /* Hardware vendor-specific info that has its own power management modes */ 901 static struct hw_vendor_info vendor_info[] = { 902 {1, "HP ", "ProLiant"}, 903 {0, "", ""}, 904 }; 905 906 static bool intel_pstate_platform_pwr_mgmt_exists(void) 907 { 908 struct acpi_table_header hdr; 909 struct hw_vendor_info *v_info; 910 911 if (acpi_disabled 912 || ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr))) 913 return false; 914 915 for (v_info = vendor_info; v_info->valid; v_info++) { 916 if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) 917 && !strncmp(hdr.oem_table_id, v_info->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) 918 && intel_pstate_no_acpi_pss()) 919 return true; 920 } 921 922 return false; 923 } 924 #else /* CONFIG_ACPI not enabled */ 925 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; } 926 #endif /* CONFIG_ACPI */ 927 928 static int __init intel_pstate_init(void) 929 { 930 int cpu, rc = 0; 931 const struct x86_cpu_id *id; 932 struct cpu_defaults *cpu_info; 933 934 if (no_load) 935 return -ENODEV; 936 937 id = x86_match_cpu(intel_pstate_cpu_ids); 938 if (!id) 939 return -ENODEV; 940 941 /* 942 * The Intel pstate driver will be ignored if the platform 943 * firmware has its own power management modes. 944 */ 945 if (intel_pstate_platform_pwr_mgmt_exists()) 946 return -ENODEV; 947 948 cpu_info = (struct cpu_defaults *)id->driver_data; 949 950 copy_pid_params(&cpu_info->pid_policy); 951 copy_cpu_funcs(&cpu_info->funcs); 952 953 if (intel_pstate_msrs_not_valid()) 954 return -ENODEV; 955 956 pr_info("Intel P-state driver initializing.\n"); 957 958 all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus()); 959 if (!all_cpu_data) 960 return -ENOMEM; 961 962 rc = cpufreq_register_driver(&intel_pstate_driver); 963 if (rc) 964 goto out; 965 966 intel_pstate_debug_expose_params(); 967 intel_pstate_sysfs_expose_params(); 968 969 return rc; 970 out: 971 get_online_cpus(); 972 for_each_online_cpu(cpu) { 973 if (all_cpu_data[cpu]) { 974 del_timer_sync(&all_cpu_data[cpu]->timer); 975 kfree(all_cpu_data[cpu]); 976 } 977 } 978 979 put_online_cpus(); 980 vfree(all_cpu_data); 981 return -ENODEV; 982 } 983 device_initcall(intel_pstate_init); 984 985 static int __init intel_pstate_setup(char *str) 986 { 987 if (!str) 988 return -EINVAL; 989 990 if (!strcmp(str, "disable")) 991 no_load = 1; 992 return 0; 993 } 994 early_param("intel_pstate", intel_pstate_setup); 995 996 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>"); 997 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors"); 998 MODULE_LICENSE("GPL"); 999