1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * acpi-cpufreq.c - ACPI Processor P-States Driver 4 * 5 * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com> 6 * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com> 7 * Copyright (C) 2002 - 2004 Dominik Brodowski <linux@brodo.de> 8 * Copyright (C) 2006 Denis Sadykov <denis.m.sadykov@intel.com> 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/init.h> 16 #include <linux/smp.h> 17 #include <linux/sched.h> 18 #include <linux/cpufreq.h> 19 #include <linux/compiler.h> 20 #include <linux/dmi.h> 21 #include <linux/slab.h> 22 #include <linux/string_helpers.h> 23 #include <linux/platform_device.h> 24 25 #include <linux/acpi.h> 26 #include <linux/io.h> 27 #include <linux/delay.h> 28 #include <linux/uaccess.h> 29 30 #include <acpi/processor.h> 31 #include <acpi/cppc_acpi.h> 32 33 #include <asm/msr.h> 34 #include <asm/processor.h> 35 #include <asm/cpufeature.h> 36 #include <asm/cpu_device_id.h> 37 38 MODULE_AUTHOR("Paul Diefenbaugh, Dominik Brodowski"); 39 MODULE_DESCRIPTION("ACPI Processor P-States Driver"); 40 MODULE_LICENSE("GPL"); 41 42 enum { 43 UNDEFINED_CAPABLE = 0, 44 SYSTEM_INTEL_MSR_CAPABLE, 45 SYSTEM_AMD_MSR_CAPABLE, 46 SYSTEM_IO_CAPABLE, 47 }; 48 49 #define INTEL_MSR_RANGE (0xffff) 50 #define AMD_MSR_RANGE (0x7) 51 #define HYGON_MSR_RANGE (0x7) 52 53 struct acpi_cpufreq_data { 54 unsigned int resume; 55 unsigned int cpu_feature; 56 unsigned int acpi_perf_cpu; 57 cpumask_var_t freqdomain_cpus; 58 void (*cpu_freq_write)(struct acpi_pct_register *reg, u32 val); 59 u32 (*cpu_freq_read)(struct acpi_pct_register *reg); 60 }; 61 62 /* acpi_perf_data is a pointer to percpu data. */ 63 static struct acpi_processor_performance __percpu *acpi_perf_data; 64 65 static inline struct acpi_processor_performance *to_perf_data(struct acpi_cpufreq_data *data) 66 { 67 return per_cpu_ptr(acpi_perf_data, data->acpi_perf_cpu); 68 } 69 70 static struct cpufreq_driver acpi_cpufreq_driver; 71 72 static unsigned int acpi_pstate_strict; 73 74 static bool boost_state(unsigned int cpu) 75 { 76 u32 lo, hi; 77 u64 msr; 78 79 switch (boot_cpu_data.x86_vendor) { 80 case X86_VENDOR_INTEL: 81 case X86_VENDOR_CENTAUR: 82 case X86_VENDOR_ZHAOXIN: 83 rdmsr_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &lo, &hi); 84 msr = lo | ((u64)hi << 32); 85 return !(msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE); 86 case X86_VENDOR_HYGON: 87 case X86_VENDOR_AMD: 88 rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi); 89 msr = lo | ((u64)hi << 32); 90 return !(msr & MSR_K7_HWCR_CPB_DIS); 91 } 92 return false; 93 } 94 95 static int boost_set_msr(bool enable) 96 { 97 u32 msr_addr; 98 u64 msr_mask, val; 99 100 switch (boot_cpu_data.x86_vendor) { 101 case X86_VENDOR_INTEL: 102 case X86_VENDOR_CENTAUR: 103 case X86_VENDOR_ZHAOXIN: 104 msr_addr = MSR_IA32_MISC_ENABLE; 105 msr_mask = MSR_IA32_MISC_ENABLE_TURBO_DISABLE; 106 break; 107 case X86_VENDOR_HYGON: 108 case X86_VENDOR_AMD: 109 msr_addr = MSR_K7_HWCR; 110 msr_mask = MSR_K7_HWCR_CPB_DIS; 111 break; 112 default: 113 return -EINVAL; 114 } 115 116 rdmsrl(msr_addr, val); 117 118 if (enable) 119 val &= ~msr_mask; 120 else 121 val |= msr_mask; 122 123 wrmsrl(msr_addr, val); 124 return 0; 125 } 126 127 static void boost_set_msr_each(void *p_en) 128 { 129 bool enable = (bool) p_en; 130 131 boost_set_msr(enable); 132 } 133 134 static int set_boost(struct cpufreq_policy *policy, int val) 135 { 136 on_each_cpu_mask(policy->cpus, boost_set_msr_each, 137 (void *)(long)val, 1); 138 pr_debug("CPU %*pbl: Core Boosting %s.\n", 139 cpumask_pr_args(policy->cpus), str_enabled_disabled(val)); 140 141 return 0; 142 } 143 144 static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) 145 { 146 struct acpi_cpufreq_data *data = policy->driver_data; 147 148 if (unlikely(!data)) 149 return -ENODEV; 150 151 return cpufreq_show_cpus(data->freqdomain_cpus, buf); 152 } 153 154 cpufreq_freq_attr_ro(freqdomain_cpus); 155 156 #ifdef CONFIG_X86_ACPI_CPUFREQ_CPB 157 static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf, 158 size_t count) 159 { 160 int ret; 161 unsigned int val = 0; 162 163 if (!acpi_cpufreq_driver.set_boost) 164 return -EINVAL; 165 166 ret = kstrtouint(buf, 10, &val); 167 if (ret || val > 1) 168 return -EINVAL; 169 170 cpus_read_lock(); 171 set_boost(policy, val); 172 cpus_read_unlock(); 173 174 return count; 175 } 176 177 static ssize_t show_cpb(struct cpufreq_policy *policy, char *buf) 178 { 179 return sprintf(buf, "%u\n", acpi_cpufreq_driver.boost_enabled); 180 } 181 182 cpufreq_freq_attr_rw(cpb); 183 #endif 184 185 static int check_est_cpu(unsigned int cpuid) 186 { 187 struct cpuinfo_x86 *cpu = &cpu_data(cpuid); 188 189 return cpu_has(cpu, X86_FEATURE_EST); 190 } 191 192 static int check_amd_hwpstate_cpu(unsigned int cpuid) 193 { 194 struct cpuinfo_x86 *cpu = &cpu_data(cpuid); 195 196 return cpu_has(cpu, X86_FEATURE_HW_PSTATE); 197 } 198 199 static unsigned extract_io(struct cpufreq_policy *policy, u32 value) 200 { 201 struct acpi_cpufreq_data *data = policy->driver_data; 202 struct acpi_processor_performance *perf; 203 int i; 204 205 perf = to_perf_data(data); 206 207 for (i = 0; i < perf->state_count; i++) { 208 if (value == perf->states[i].status) 209 return policy->freq_table[i].frequency; 210 } 211 return 0; 212 } 213 214 static unsigned extract_msr(struct cpufreq_policy *policy, u32 msr) 215 { 216 struct acpi_cpufreq_data *data = policy->driver_data; 217 struct cpufreq_frequency_table *pos; 218 struct acpi_processor_performance *perf; 219 220 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 221 msr &= AMD_MSR_RANGE; 222 else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) 223 msr &= HYGON_MSR_RANGE; 224 else 225 msr &= INTEL_MSR_RANGE; 226 227 perf = to_perf_data(data); 228 229 cpufreq_for_each_entry(pos, policy->freq_table) 230 if (msr == perf->states[pos->driver_data].status) 231 return pos->frequency; 232 return policy->freq_table[0].frequency; 233 } 234 235 static unsigned extract_freq(struct cpufreq_policy *policy, u32 val) 236 { 237 struct acpi_cpufreq_data *data = policy->driver_data; 238 239 switch (data->cpu_feature) { 240 case SYSTEM_INTEL_MSR_CAPABLE: 241 case SYSTEM_AMD_MSR_CAPABLE: 242 return extract_msr(policy, val); 243 case SYSTEM_IO_CAPABLE: 244 return extract_io(policy, val); 245 default: 246 return 0; 247 } 248 } 249 250 static u32 cpu_freq_read_intel(struct acpi_pct_register *not_used) 251 { 252 u32 val, dummy __always_unused; 253 254 rdmsr(MSR_IA32_PERF_CTL, val, dummy); 255 return val; 256 } 257 258 static void cpu_freq_write_intel(struct acpi_pct_register *not_used, u32 val) 259 { 260 u32 lo, hi; 261 262 rdmsr(MSR_IA32_PERF_CTL, lo, hi); 263 lo = (lo & ~INTEL_MSR_RANGE) | (val & INTEL_MSR_RANGE); 264 wrmsr(MSR_IA32_PERF_CTL, lo, hi); 265 } 266 267 static u32 cpu_freq_read_amd(struct acpi_pct_register *not_used) 268 { 269 u32 val, dummy __always_unused; 270 271 rdmsr(MSR_AMD_PERF_CTL, val, dummy); 272 return val; 273 } 274 275 static void cpu_freq_write_amd(struct acpi_pct_register *not_used, u32 val) 276 { 277 wrmsr(MSR_AMD_PERF_CTL, val, 0); 278 } 279 280 static u32 cpu_freq_read_io(struct acpi_pct_register *reg) 281 { 282 u32 val; 283 284 acpi_os_read_port(reg->address, &val, reg->bit_width); 285 return val; 286 } 287 288 static void cpu_freq_write_io(struct acpi_pct_register *reg, u32 val) 289 { 290 acpi_os_write_port(reg->address, val, reg->bit_width); 291 } 292 293 struct drv_cmd { 294 struct acpi_pct_register *reg; 295 u32 val; 296 union { 297 void (*write)(struct acpi_pct_register *reg, u32 val); 298 u32 (*read)(struct acpi_pct_register *reg); 299 } func; 300 }; 301 302 /* Called via smp_call_function_single(), on the target CPU */ 303 static void do_drv_read(void *_cmd) 304 { 305 struct drv_cmd *cmd = _cmd; 306 307 cmd->val = cmd->func.read(cmd->reg); 308 } 309 310 static u32 drv_read(struct acpi_cpufreq_data *data, const struct cpumask *mask) 311 { 312 struct acpi_processor_performance *perf = to_perf_data(data); 313 struct drv_cmd cmd = { 314 .reg = &perf->control_register, 315 .func.read = data->cpu_freq_read, 316 }; 317 int err; 318 319 err = smp_call_function_any(mask, do_drv_read, &cmd, 1); 320 WARN_ON_ONCE(err); /* smp_call_function_any() was buggy? */ 321 return cmd.val; 322 } 323 324 /* Called via smp_call_function_many(), on the target CPUs */ 325 static void do_drv_write(void *_cmd) 326 { 327 struct drv_cmd *cmd = _cmd; 328 329 cmd->func.write(cmd->reg, cmd->val); 330 } 331 332 static void drv_write(struct acpi_cpufreq_data *data, 333 const struct cpumask *mask, u32 val) 334 { 335 struct acpi_processor_performance *perf = to_perf_data(data); 336 struct drv_cmd cmd = { 337 .reg = &perf->control_register, 338 .val = val, 339 .func.write = data->cpu_freq_write, 340 }; 341 int this_cpu; 342 343 this_cpu = get_cpu(); 344 if (cpumask_test_cpu(this_cpu, mask)) 345 do_drv_write(&cmd); 346 347 smp_call_function_many(mask, do_drv_write, &cmd, 1); 348 put_cpu(); 349 } 350 351 static u32 get_cur_val(const struct cpumask *mask, struct acpi_cpufreq_data *data) 352 { 353 u32 val; 354 355 if (unlikely(cpumask_empty(mask))) 356 return 0; 357 358 val = drv_read(data, mask); 359 360 pr_debug("%s = %u\n", __func__, val); 361 362 return val; 363 } 364 365 static unsigned int get_cur_freq_on_cpu(unsigned int cpu) 366 { 367 struct acpi_cpufreq_data *data; 368 struct cpufreq_policy *policy; 369 unsigned int freq; 370 unsigned int cached_freq; 371 372 pr_debug("%s (%d)\n", __func__, cpu); 373 374 policy = cpufreq_cpu_get_raw(cpu); 375 if (unlikely(!policy)) 376 return 0; 377 378 data = policy->driver_data; 379 if (unlikely(!data || !policy->freq_table)) 380 return 0; 381 382 cached_freq = policy->freq_table[to_perf_data(data)->state].frequency; 383 freq = extract_freq(policy, get_cur_val(cpumask_of(cpu), data)); 384 if (freq != cached_freq) { 385 /* 386 * The dreaded BIOS frequency change behind our back. 387 * Force set the frequency on next target call. 388 */ 389 data->resume = 1; 390 } 391 392 pr_debug("cur freq = %u\n", freq); 393 394 return freq; 395 } 396 397 static unsigned int check_freqs(struct cpufreq_policy *policy, 398 const struct cpumask *mask, unsigned int freq) 399 { 400 struct acpi_cpufreq_data *data = policy->driver_data; 401 unsigned int cur_freq; 402 unsigned int i; 403 404 for (i = 0; i < 100; i++) { 405 cur_freq = extract_freq(policy, get_cur_val(mask, data)); 406 if (cur_freq == freq) 407 return 1; 408 udelay(10); 409 } 410 return 0; 411 } 412 413 static int acpi_cpufreq_target(struct cpufreq_policy *policy, 414 unsigned int index) 415 { 416 struct acpi_cpufreq_data *data = policy->driver_data; 417 struct acpi_processor_performance *perf; 418 const struct cpumask *mask; 419 unsigned int next_perf_state = 0; /* Index into perf table */ 420 int result = 0; 421 422 if (unlikely(!data)) { 423 return -ENODEV; 424 } 425 426 perf = to_perf_data(data); 427 next_perf_state = policy->freq_table[index].driver_data; 428 if (perf->state == next_perf_state) { 429 if (unlikely(data->resume)) { 430 pr_debug("Called after resume, resetting to P%d\n", 431 next_perf_state); 432 data->resume = 0; 433 } else { 434 pr_debug("Already at target state (P%d)\n", 435 next_perf_state); 436 return 0; 437 } 438 } 439 440 /* 441 * The core won't allow CPUs to go away until the governor has been 442 * stopped, so we can rely on the stability of policy->cpus. 443 */ 444 mask = policy->shared_type == CPUFREQ_SHARED_TYPE_ANY ? 445 cpumask_of(policy->cpu) : policy->cpus; 446 447 drv_write(data, mask, perf->states[next_perf_state].control); 448 449 if (acpi_pstate_strict) { 450 if (!check_freqs(policy, mask, 451 policy->freq_table[index].frequency)) { 452 pr_debug("%s (%d)\n", __func__, policy->cpu); 453 result = -EAGAIN; 454 } 455 } 456 457 if (!result) 458 perf->state = next_perf_state; 459 460 return result; 461 } 462 463 static unsigned int acpi_cpufreq_fast_switch(struct cpufreq_policy *policy, 464 unsigned int target_freq) 465 { 466 struct acpi_cpufreq_data *data = policy->driver_data; 467 struct acpi_processor_performance *perf; 468 struct cpufreq_frequency_table *entry; 469 unsigned int next_perf_state, next_freq, index; 470 471 /* 472 * Find the closest frequency above target_freq. 473 */ 474 if (policy->cached_target_freq == target_freq) 475 index = policy->cached_resolved_idx; 476 else 477 index = cpufreq_table_find_index_dl(policy, target_freq, 478 false); 479 480 entry = &policy->freq_table[index]; 481 next_freq = entry->frequency; 482 next_perf_state = entry->driver_data; 483 484 perf = to_perf_data(data); 485 if (perf->state == next_perf_state) { 486 if (unlikely(data->resume)) 487 data->resume = 0; 488 else 489 return next_freq; 490 } 491 492 data->cpu_freq_write(&perf->control_register, 493 perf->states[next_perf_state].control); 494 perf->state = next_perf_state; 495 return next_freq; 496 } 497 498 static unsigned long 499 acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu) 500 { 501 struct acpi_processor_performance *perf; 502 503 perf = to_perf_data(data); 504 if (cpu_khz) { 505 /* search the closest match to cpu_khz */ 506 unsigned int i; 507 unsigned long freq; 508 unsigned long freqn = perf->states[0].core_frequency * 1000; 509 510 for (i = 0; i < (perf->state_count-1); i++) { 511 freq = freqn; 512 freqn = perf->states[i+1].core_frequency * 1000; 513 if ((2 * cpu_khz) > (freqn + freq)) { 514 perf->state = i; 515 return freq; 516 } 517 } 518 perf->state = perf->state_count-1; 519 return freqn; 520 } else { 521 /* assume CPU is at P0... */ 522 perf->state = 0; 523 return perf->states[0].core_frequency * 1000; 524 } 525 } 526 527 static void free_acpi_perf_data(void) 528 { 529 unsigned int i; 530 531 /* Freeing a NULL pointer is OK, and alloc_percpu zeroes. */ 532 for_each_possible_cpu(i) 533 free_cpumask_var(per_cpu_ptr(acpi_perf_data, i) 534 ->shared_cpu_map); 535 free_percpu(acpi_perf_data); 536 } 537 538 static int cpufreq_boost_down_prep(unsigned int cpu) 539 { 540 /* 541 * Clear the boost-disable bit on the CPU_DOWN path so that 542 * this cpu cannot block the remaining ones from boosting. 543 */ 544 return boost_set_msr(1); 545 } 546 547 /* 548 * acpi_cpufreq_early_init - initialize ACPI P-States library 549 * 550 * Initialize the ACPI P-States library (drivers/acpi/processor_perflib.c) 551 * in order to determine correct frequency and voltage pairings. We can 552 * do _PDC and _PSD and find out the processor dependency for the 553 * actual init that will happen later... 554 */ 555 static int __init acpi_cpufreq_early_init(void) 556 { 557 unsigned int i; 558 pr_debug("%s\n", __func__); 559 560 acpi_perf_data = alloc_percpu(struct acpi_processor_performance); 561 if (!acpi_perf_data) { 562 pr_debug("Memory allocation error for acpi_perf_data.\n"); 563 return -ENOMEM; 564 } 565 for_each_possible_cpu(i) { 566 if (!zalloc_cpumask_var_node( 567 &per_cpu_ptr(acpi_perf_data, i)->shared_cpu_map, 568 GFP_KERNEL, cpu_to_node(i))) { 569 570 /* Freeing a NULL pointer is OK: alloc_percpu zeroes. */ 571 free_acpi_perf_data(); 572 return -ENOMEM; 573 } 574 } 575 576 /* Do initialization in ACPI core */ 577 acpi_processor_preregister_performance(acpi_perf_data); 578 return 0; 579 } 580 581 #ifdef CONFIG_SMP 582 /* 583 * Some BIOSes do SW_ANY coordination internally, either set it up in hw 584 * or do it in BIOS firmware and won't inform about it to OS. If not 585 * detected, this has a side effect of making CPU run at a different speed 586 * than OS intended it to run at. Detect it and handle it cleanly. 587 */ 588 static int bios_with_sw_any_bug; 589 590 static int sw_any_bug_found(const struct dmi_system_id *d) 591 { 592 bios_with_sw_any_bug = 1; 593 return 0; 594 } 595 596 static const struct dmi_system_id sw_any_bug_dmi_table[] = { 597 { 598 .callback = sw_any_bug_found, 599 .ident = "Supermicro Server X6DLP", 600 .matches = { 601 DMI_MATCH(DMI_SYS_VENDOR, "Supermicro"), 602 DMI_MATCH(DMI_BIOS_VERSION, "080010"), 603 DMI_MATCH(DMI_PRODUCT_NAME, "X6DLP"), 604 }, 605 }, 606 { } 607 }; 608 609 static int acpi_cpufreq_blacklist(struct cpuinfo_x86 *c) 610 { 611 /* Intel Xeon Processor 7100 Series Specification Update 612 * https://www.intel.com/Assets/PDF/specupdate/314554.pdf 613 * AL30: A Machine Check Exception (MCE) Occurring during an 614 * Enhanced Intel SpeedStep Technology Ratio Change May Cause 615 * Both Processor Cores to Lock Up. */ 616 if (c->x86_vendor == X86_VENDOR_INTEL) { 617 if ((c->x86 == 15) && 618 (c->x86_model == 6) && 619 (c->x86_stepping == 8)) { 620 pr_info("Intel(R) Xeon(R) 7100 Errata AL30, processors may lock up on frequency changes: disabling acpi-cpufreq\n"); 621 return -ENODEV; 622 } 623 } 624 return 0; 625 } 626 #endif 627 628 #ifdef CONFIG_ACPI_CPPC_LIB 629 static u64 get_max_boost_ratio(unsigned int cpu) 630 { 631 struct cppc_perf_caps perf_caps; 632 u64 highest_perf, nominal_perf; 633 int ret; 634 635 if (acpi_pstate_strict) 636 return 0; 637 638 ret = cppc_get_perf_caps(cpu, &perf_caps); 639 if (ret) { 640 pr_debug("CPU%d: Unable to get performance capabilities (%d)\n", 641 cpu, ret); 642 return 0; 643 } 644 645 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { 646 ret = amd_get_boost_ratio_numerator(cpu, &highest_perf); 647 if (ret) { 648 pr_debug("CPU%d: Unable to get boost ratio numerator (%d)\n", 649 cpu, ret); 650 return 0; 651 } 652 } else { 653 highest_perf = perf_caps.highest_perf; 654 } 655 656 nominal_perf = perf_caps.nominal_perf; 657 658 if (!highest_perf || !nominal_perf) { 659 pr_debug("CPU%d: highest or nominal performance missing\n", cpu); 660 return 0; 661 } 662 663 if (highest_perf < nominal_perf) { 664 pr_debug("CPU%d: nominal performance above highest\n", cpu); 665 return 0; 666 } 667 668 return div_u64(highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); 669 } 670 #else 671 static inline u64 get_max_boost_ratio(unsigned int cpu) { return 0; } 672 #endif 673 674 static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) 675 { 676 struct cpufreq_frequency_table *freq_table; 677 struct acpi_processor_performance *perf; 678 struct acpi_cpufreq_data *data; 679 unsigned int cpu = policy->cpu; 680 struct cpuinfo_x86 *c = &cpu_data(cpu); 681 unsigned int valid_states = 0; 682 unsigned int result = 0; 683 u64 max_boost_ratio; 684 unsigned int i; 685 #ifdef CONFIG_SMP 686 static int blacklisted; 687 #endif 688 689 pr_debug("%s\n", __func__); 690 691 #ifdef CONFIG_SMP 692 if (blacklisted) 693 return blacklisted; 694 blacklisted = acpi_cpufreq_blacklist(c); 695 if (blacklisted) 696 return blacklisted; 697 #endif 698 699 data = kzalloc(sizeof(*data), GFP_KERNEL); 700 if (!data) 701 return -ENOMEM; 702 703 if (!zalloc_cpumask_var(&data->freqdomain_cpus, GFP_KERNEL)) { 704 result = -ENOMEM; 705 goto err_free; 706 } 707 708 perf = per_cpu_ptr(acpi_perf_data, cpu); 709 data->acpi_perf_cpu = cpu; 710 policy->driver_data = data; 711 712 if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) 713 acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS; 714 715 result = acpi_processor_register_performance(perf, cpu); 716 if (result) 717 goto err_free_mask; 718 719 policy->shared_type = perf->shared_type; 720 721 /* 722 * Will let policy->cpus know about dependency only when software 723 * coordination is required. 724 */ 725 if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL || 726 policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) { 727 cpumask_copy(policy->cpus, perf->shared_cpu_map); 728 } 729 cpumask_copy(data->freqdomain_cpus, perf->shared_cpu_map); 730 731 #ifdef CONFIG_SMP 732 dmi_check_system(sw_any_bug_dmi_table); 733 if (bios_with_sw_any_bug && !policy_is_shared(policy)) { 734 policy->shared_type = CPUFREQ_SHARED_TYPE_ALL; 735 cpumask_copy(policy->cpus, topology_core_cpumask(cpu)); 736 } 737 738 if (check_amd_hwpstate_cpu(cpu) && boot_cpu_data.x86 < 0x19 && 739 !acpi_pstate_strict) { 740 cpumask_clear(policy->cpus); 741 cpumask_set_cpu(cpu, policy->cpus); 742 cpumask_copy(data->freqdomain_cpus, 743 topology_sibling_cpumask(cpu)); 744 policy->shared_type = CPUFREQ_SHARED_TYPE_HW; 745 pr_info_once("overriding BIOS provided _PSD data\n"); 746 } 747 #endif 748 749 /* capability check */ 750 if (perf->state_count <= 1) { 751 pr_debug("No P-States\n"); 752 result = -ENODEV; 753 goto err_unreg; 754 } 755 756 if (perf->control_register.space_id != perf->status_register.space_id) { 757 result = -ENODEV; 758 goto err_unreg; 759 } 760 761 switch (perf->control_register.space_id) { 762 case ACPI_ADR_SPACE_SYSTEM_IO: 763 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && 764 boot_cpu_data.x86 == 0xf) { 765 pr_debug("AMD K8 systems must use native drivers.\n"); 766 result = -ENODEV; 767 goto err_unreg; 768 } 769 pr_debug("SYSTEM IO addr space\n"); 770 data->cpu_feature = SYSTEM_IO_CAPABLE; 771 data->cpu_freq_read = cpu_freq_read_io; 772 data->cpu_freq_write = cpu_freq_write_io; 773 break; 774 case ACPI_ADR_SPACE_FIXED_HARDWARE: 775 pr_debug("HARDWARE addr space\n"); 776 if (check_est_cpu(cpu)) { 777 data->cpu_feature = SYSTEM_INTEL_MSR_CAPABLE; 778 data->cpu_freq_read = cpu_freq_read_intel; 779 data->cpu_freq_write = cpu_freq_write_intel; 780 break; 781 } 782 if (check_amd_hwpstate_cpu(cpu)) { 783 data->cpu_feature = SYSTEM_AMD_MSR_CAPABLE; 784 data->cpu_freq_read = cpu_freq_read_amd; 785 data->cpu_freq_write = cpu_freq_write_amd; 786 break; 787 } 788 result = -ENODEV; 789 goto err_unreg; 790 default: 791 pr_debug("Unknown addr space %d\n", 792 (u32) (perf->control_register.space_id)); 793 result = -ENODEV; 794 goto err_unreg; 795 } 796 797 freq_table = kcalloc(perf->state_count + 1, sizeof(*freq_table), 798 GFP_KERNEL); 799 if (!freq_table) { 800 result = -ENOMEM; 801 goto err_unreg; 802 } 803 804 /* detect transition latency */ 805 policy->cpuinfo.transition_latency = 0; 806 for (i = 0; i < perf->state_count; i++) { 807 if ((perf->states[i].transition_latency * 1000) > 808 policy->cpuinfo.transition_latency) 809 policy->cpuinfo.transition_latency = 810 perf->states[i].transition_latency * 1000; 811 } 812 813 /* Check for high latency (>20uS) from buggy BIOSes, like on T42 */ 814 if (perf->control_register.space_id == ACPI_ADR_SPACE_FIXED_HARDWARE && 815 policy->cpuinfo.transition_latency > 20 * 1000) { 816 policy->cpuinfo.transition_latency = 20 * 1000; 817 pr_info_once("P-state transition latency capped at 20 uS\n"); 818 } 819 820 /* table init */ 821 for (i = 0; i < perf->state_count; i++) { 822 if (i > 0 && perf->states[i].core_frequency >= 823 freq_table[valid_states-1].frequency / 1000) 824 continue; 825 826 freq_table[valid_states].driver_data = i; 827 freq_table[valid_states].frequency = 828 perf->states[i].core_frequency * 1000; 829 valid_states++; 830 } 831 freq_table[valid_states].frequency = CPUFREQ_TABLE_END; 832 833 max_boost_ratio = get_max_boost_ratio(cpu); 834 if (max_boost_ratio) { 835 unsigned int freq = freq_table[0].frequency; 836 837 /* 838 * Because the loop above sorts the freq_table entries in the 839 * descending order, freq is the maximum frequency in the table. 840 * Assume that it corresponds to the CPPC nominal frequency and 841 * use it to set cpuinfo.max_freq. 842 */ 843 policy->cpuinfo.max_freq = freq * max_boost_ratio >> SCHED_CAPACITY_SHIFT; 844 } else { 845 /* 846 * If the maximum "boost" frequency is unknown, ask the arch 847 * scale-invariance code to use the "nominal" performance for 848 * CPU utilization scaling so as to prevent the schedutil 849 * governor from selecting inadequate CPU frequencies. 850 */ 851 arch_set_max_freq_ratio(true); 852 } 853 854 policy->freq_table = freq_table; 855 perf->state = 0; 856 857 switch (perf->control_register.space_id) { 858 case ACPI_ADR_SPACE_SYSTEM_IO: 859 /* 860 * The core will not set policy->cur, because 861 * cpufreq_driver->get is NULL, so we need to set it here. 862 * However, we have to guess it, because the current speed is 863 * unknown and not detectable via IO ports. 864 */ 865 policy->cur = acpi_cpufreq_guess_freq(data, policy->cpu); 866 break; 867 case ACPI_ADR_SPACE_FIXED_HARDWARE: 868 acpi_cpufreq_driver.get = get_cur_freq_on_cpu; 869 break; 870 default: 871 break; 872 } 873 874 /* notify BIOS that we exist */ 875 acpi_processor_notify_smm(THIS_MODULE); 876 877 pr_debug("CPU%u - ACPI performance management activated.\n", cpu); 878 for (i = 0; i < perf->state_count; i++) 879 pr_debug(" %cP%d: %d MHz, %d mW, %d uS\n", 880 (i == perf->state ? '*' : ' '), i, 881 (u32) perf->states[i].core_frequency, 882 (u32) perf->states[i].power, 883 (u32) perf->states[i].transition_latency); 884 885 /* 886 * the first call to ->target() should result in us actually 887 * writing something to the appropriate registers. 888 */ 889 data->resume = 1; 890 891 policy->fast_switch_possible = !acpi_pstate_strict && 892 !(policy_is_shared(policy) && policy->shared_type != CPUFREQ_SHARED_TYPE_ANY); 893 894 if (perf->states[0].core_frequency * 1000 != freq_table[0].frequency) 895 pr_warn(FW_WARN "P-state 0 is not max freq\n"); 896 897 if (acpi_cpufreq_driver.set_boost) { 898 set_boost(policy, acpi_cpufreq_driver.boost_enabled); 899 policy->boost_enabled = acpi_cpufreq_driver.boost_enabled; 900 } 901 902 return result; 903 904 err_unreg: 905 acpi_processor_unregister_performance(cpu); 906 err_free_mask: 907 free_cpumask_var(data->freqdomain_cpus); 908 err_free: 909 kfree(data); 910 policy->driver_data = NULL; 911 912 return result; 913 } 914 915 static void acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) 916 { 917 struct acpi_cpufreq_data *data = policy->driver_data; 918 919 pr_debug("%s\n", __func__); 920 921 cpufreq_boost_down_prep(policy->cpu); 922 policy->fast_switch_possible = false; 923 policy->driver_data = NULL; 924 acpi_processor_unregister_performance(data->acpi_perf_cpu); 925 free_cpumask_var(data->freqdomain_cpus); 926 kfree(policy->freq_table); 927 kfree(data); 928 } 929 930 static int acpi_cpufreq_resume(struct cpufreq_policy *policy) 931 { 932 struct acpi_cpufreq_data *data = policy->driver_data; 933 934 pr_debug("%s\n", __func__); 935 936 data->resume = 1; 937 938 return 0; 939 } 940 941 static struct freq_attr *acpi_cpufreq_attr[] = { 942 &cpufreq_freq_attr_scaling_available_freqs, 943 &freqdomain_cpus, 944 #ifdef CONFIG_X86_ACPI_CPUFREQ_CPB 945 &cpb, 946 #endif 947 NULL, 948 }; 949 950 static struct cpufreq_driver acpi_cpufreq_driver = { 951 .verify = cpufreq_generic_frequency_table_verify, 952 .target_index = acpi_cpufreq_target, 953 .fast_switch = acpi_cpufreq_fast_switch, 954 .bios_limit = acpi_processor_get_bios_limit, 955 .init = acpi_cpufreq_cpu_init, 956 .exit = acpi_cpufreq_cpu_exit, 957 .resume = acpi_cpufreq_resume, 958 .name = "acpi-cpufreq", 959 .attr = acpi_cpufreq_attr, 960 }; 961 962 static void __init acpi_cpufreq_boost_init(void) 963 { 964 if (!(boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA))) { 965 pr_debug("Boost capabilities not present in the processor\n"); 966 return; 967 } 968 969 acpi_cpufreq_driver.set_boost = set_boost; 970 acpi_cpufreq_driver.boost_enabled = boost_state(0); 971 } 972 973 static int __init acpi_cpufreq_probe(struct platform_device *pdev) 974 { 975 int ret; 976 977 if (acpi_disabled) 978 return -ENODEV; 979 980 /* don't keep reloading if cpufreq_driver exists */ 981 if (cpufreq_get_current_driver()) 982 return -ENODEV; 983 984 pr_debug("%s\n", __func__); 985 986 ret = acpi_cpufreq_early_init(); 987 if (ret) 988 return ret; 989 990 #ifdef CONFIG_X86_ACPI_CPUFREQ_CPB 991 /* this is a sysfs file with a strange name and an even stranger 992 * semantic - per CPU instantiation, but system global effect. 993 * Lets enable it only on AMD CPUs for compatibility reasons and 994 * only if configured. This is considered legacy code, which 995 * will probably be removed at some point in the future. 996 */ 997 if (!check_amd_hwpstate_cpu(0)) { 998 struct freq_attr **attr; 999 1000 pr_debug("CPB unsupported, do not expose it\n"); 1001 1002 for (attr = acpi_cpufreq_attr; *attr; attr++) 1003 if (*attr == &cpb) { 1004 *attr = NULL; 1005 break; 1006 } 1007 } 1008 #endif 1009 acpi_cpufreq_boost_init(); 1010 1011 ret = cpufreq_register_driver(&acpi_cpufreq_driver); 1012 if (ret) { 1013 free_acpi_perf_data(); 1014 } 1015 return ret; 1016 } 1017 1018 static void acpi_cpufreq_remove(struct platform_device *pdev) 1019 { 1020 pr_debug("%s\n", __func__); 1021 1022 cpufreq_unregister_driver(&acpi_cpufreq_driver); 1023 1024 free_acpi_perf_data(); 1025 } 1026 1027 static struct platform_driver acpi_cpufreq_platdrv = { 1028 .driver = { 1029 .name = "acpi-cpufreq", 1030 }, 1031 .remove_new = acpi_cpufreq_remove, 1032 }; 1033 1034 static int __init acpi_cpufreq_init(void) 1035 { 1036 return platform_driver_probe(&acpi_cpufreq_platdrv, acpi_cpufreq_probe); 1037 } 1038 1039 static void __exit acpi_cpufreq_exit(void) 1040 { 1041 platform_driver_unregister(&acpi_cpufreq_platdrv); 1042 } 1043 1044 module_param(acpi_pstate_strict, uint, 0644); 1045 MODULE_PARM_DESC(acpi_pstate_strict, 1046 "value 0 or non-zero. non-zero -> strict ACPI checks are " 1047 "performed during frequency changes."); 1048 1049 late_initcall(acpi_cpufreq_init); 1050 module_exit(acpi_cpufreq_exit); 1051 1052 MODULE_ALIAS("platform:acpi-cpufreq"); 1053