1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * (c) 2003-2012 Advanced Micro Devices, Inc. 4 * 5 * Maintainer: 6 * Andreas Herrmann <herrmann.der.user@googlemail.com> 7 * 8 * Based on the powernow-k7.c module written by Dave Jones. 9 * (C) 2003 Dave Jones on behalf of SuSE Labs 10 * (C) 2004 Dominik Brodowski <linux@brodo.de> 11 * (C) 2004 Pavel Machek <pavel@ucw.cz> 12 * Based upon datasheets & sample CPUs kindly provided by AMD. 13 * 14 * Valuable input gratefully received from Dave Jones, Pavel Machek, 15 * Dominik Brodowski, Jacob Shin, and others. 16 * Originally developed by Paul Devriendt. 17 * 18 * Processor information obtained from Chapter 9 (Power and Thermal 19 * Management) of the "BIOS and Kernel Developer's Guide (BKDG) for 20 * the AMD Athlon 64 and AMD Opteron Processors" and section "2.x 21 * Power Management" in BKDGs for newer AMD CPU families. 22 * 23 * Tables for specific CPUs can be inferred from AMD's processor 24 * power and thermal data sheets, (e.g. 30417.pdf, 30430.pdf, 43375.pdf) 25 */ 26 27 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 28 29 #include <linux/kernel.h> 30 #include <linux/smp.h> 31 #include <linux/module.h> 32 #include <linux/init.h> 33 #include <linux/cpufreq.h> 34 #include <linux/slab.h> 35 #include <linux/string.h> 36 #include <linux/cpumask.h> 37 #include <linux/io.h> 38 #include <linux/delay.h> 39 40 #include <asm/msr.h> 41 #include <asm/cpu_device_id.h> 42 #include <asm/cpuid/api.h> 43 44 #include <linux/acpi.h> 45 #include <linux/mutex.h> 46 #include <acpi/processor.h> 47 48 #define VERSION "version 2.20.00" 49 #include "powernow-k8.h" 50 51 /* serialize freq changes */ 52 static DEFINE_MUTEX(fidvid_mutex); 53 54 static DEFINE_PER_CPU(struct powernow_k8_data *, powernow_data); 55 56 static struct cpufreq_driver cpufreq_amd64_driver; 57 58 /* Return a frequency in MHz, given an input fid */ 59 static u32 find_freq_from_fid(u32 fid) 60 { 61 return 800 + (fid * 100); 62 } 63 64 /* Return a frequency in KHz, given an input fid */ 65 static u32 find_khz_freq_from_fid(u32 fid) 66 { 67 return 1000 * find_freq_from_fid(fid); 68 } 69 70 /* Return the vco fid for an input fid 71 * 72 * Each "low" fid has corresponding "high" fid, and you can get to "low" fids 73 * only from corresponding high fids. This returns "high" fid corresponding to 74 * "low" one. 75 */ 76 static u32 convert_fid_to_vco_fid(u32 fid) 77 { 78 if (fid < HI_FID_TABLE_BOTTOM) 79 return 8 + (2 * fid); 80 else 81 return fid; 82 } 83 84 /* 85 * Return 1 if the pending bit is set. Unless we just instructed the processor 86 * to transition to a new state, seeing this bit set is really bad news. 87 */ 88 static int pending_bit_stuck(void) 89 { 90 u64 msr; 91 92 rdmsrq(MSR_FIDVID_STATUS, msr); 93 return msr & MSR_S_LO_CHANGE_PENDING ? 1 : 0; 94 } 95 96 /* 97 * Update the global current fid / vid values from the status msr. 98 * Returns 1 on error. 99 */ 100 static int query_current_values_with_pending_wait(struct powernow_k8_data *data) 101 { 102 struct msr msr; 103 u32 i = 0; 104 105 do { 106 if (i++ > 10000) { 107 pr_debug("detected change pending stuck\n"); 108 return 1; 109 } 110 rdmsrq(MSR_FIDVID_STATUS, msr.q); 111 } while (msr.l & MSR_S_LO_CHANGE_PENDING); 112 113 data->currvid = msr.h & MSR_S_HI_CURRENT_VID; 114 data->currfid = msr.l & MSR_S_LO_CURRENT_FID; 115 116 return 0; 117 } 118 119 /* the isochronous relief time */ 120 static void count_off_irt(struct powernow_k8_data *data) 121 { 122 udelay((1 << data->irt) * 10); 123 } 124 125 /* the voltage stabilization time */ 126 static void count_off_vst(struct powernow_k8_data *data) 127 { 128 udelay(data->vstable * VST_UNITS_20US); 129 } 130 131 /* need to init the control msr to a safe value (for each cpu) */ 132 static void fidvid_msr_init(void) 133 { 134 struct msr msr; 135 u8 fid, vid; 136 137 rdmsrq(MSR_FIDVID_STATUS, msr.q); 138 vid = msr.h & MSR_S_HI_CURRENT_VID; 139 fid = msr.l & MSR_S_LO_CURRENT_FID; 140 msr.l = fid | (vid << MSR_C_LO_VID_SHIFT); 141 msr.h = MSR_C_HI_STP_GNT_BENIGN; 142 pr_debug("cpu%d, init lo 0x%x, hi 0x%x\n", smp_processor_id(), msr.l, msr.h); 143 wrmsrq(MSR_FIDVID_CTL, msr.q); 144 } 145 146 /* write the new fid value along with the other control fields to the msr */ 147 static int write_new_fid(struct powernow_k8_data *data, u32 fid) 148 { 149 struct msr msr; 150 u32 savevid = data->currvid; 151 u32 i = 0; 152 153 if ((fid & INVALID_FID_MASK) || (data->currvid & INVALID_VID_MASK)) { 154 pr_err("internal error - overflow on fid write\n"); 155 return 1; 156 } 157 158 msr.l = fid; 159 msr.l |= (data->currvid << MSR_C_LO_VID_SHIFT); 160 msr.l |= MSR_C_LO_INIT_FID_VID; 161 msr.h = data->plllock * PLL_LOCK_CONVERSION; 162 163 pr_debug("writing fid 0x%x, lo 0x%x, hi 0x%x\n", fid, msr.l, msr.h); 164 165 do { 166 wrmsrq(MSR_FIDVID_CTL, msr.q); 167 if (i++ > 100) { 168 pr_err("Hardware error - pending bit very stuck - no further pstate changes possible\n"); 169 return 1; 170 } 171 } while (query_current_values_with_pending_wait(data)); 172 173 count_off_irt(data); 174 175 if (savevid != data->currvid) { 176 pr_err("vid change on fid trans, old 0x%x, new 0x%x\n", 177 savevid, data->currvid); 178 return 1; 179 } 180 181 if (fid != data->currfid) { 182 pr_err("fid trans failed, fid 0x%x, curr 0x%x\n", fid, 183 data->currfid); 184 return 1; 185 } 186 187 return 0; 188 } 189 190 /* Write a new vid to the hardware */ 191 static int write_new_vid(struct powernow_k8_data *data, u32 vid) 192 { 193 struct msr msr; 194 u32 savefid = data->currfid; 195 int i = 0; 196 197 if ((data->currfid & INVALID_FID_MASK) || (vid & INVALID_VID_MASK)) { 198 pr_err("internal error - overflow on vid write\n"); 199 return 1; 200 } 201 202 msr.l = data->currfid; 203 msr.l |= (vid << MSR_C_LO_VID_SHIFT); 204 msr.l |= MSR_C_LO_INIT_FID_VID; 205 msr.h = STOP_GRANT_5NS; 206 207 pr_debug("writing vid 0x%x, lo 0x%x, hi 0x%x\n", vid, msr.l, msr.h); 208 209 do { 210 wrmsrq(MSR_FIDVID_CTL, msr.q); 211 if (i++ > 100) { 212 pr_err("internal error - pending bit very stuck - no further pstate changes possible\n"); 213 return 1; 214 } 215 } while (query_current_values_with_pending_wait(data)); 216 217 if (savefid != data->currfid) { 218 pr_err("fid changed on vid trans, old 0x%x new 0x%x\n", 219 savefid, data->currfid); 220 return 1; 221 } 222 223 if (vid != data->currvid) { 224 pr_err("vid trans failed, vid 0x%x, curr 0x%x\n", 225 vid, data->currvid); 226 return 1; 227 } 228 229 return 0; 230 } 231 232 /* 233 * Reduce the vid by the max of step or reqvid. 234 * Decreasing vid codes represent increasing voltages: 235 * vid of 0 is 1.550V, vid of 0x1e is 0.800V, vid of VID_OFF is off. 236 */ 237 static int decrease_vid_code_by_step(struct powernow_k8_data *data, 238 u32 reqvid, u32 step) 239 { 240 if ((data->currvid - reqvid) > step) 241 reqvid = data->currvid - step; 242 243 if (write_new_vid(data, reqvid)) 244 return 1; 245 246 count_off_vst(data); 247 248 return 0; 249 } 250 251 /* Change Opteron/Athlon64 fid and vid, by the 3 phases. */ 252 static int transition_fid_vid(struct powernow_k8_data *data, 253 u32 reqfid, u32 reqvid) 254 { 255 if (core_voltage_pre_transition(data, reqvid, reqfid)) 256 return 1; 257 258 if (core_frequency_transition(data, reqfid)) 259 return 1; 260 261 if (core_voltage_post_transition(data, reqvid)) 262 return 1; 263 264 if (query_current_values_with_pending_wait(data)) 265 return 1; 266 267 if ((reqfid != data->currfid) || (reqvid != data->currvid)) { 268 pr_err("failed (cpu%d): req 0x%x 0x%x, curr 0x%x 0x%x\n", 269 smp_processor_id(), 270 reqfid, reqvid, data->currfid, data->currvid); 271 return 1; 272 } 273 274 pr_debug("transitioned (cpu%d): new fid 0x%x, vid 0x%x\n", 275 smp_processor_id(), data->currfid, data->currvid); 276 277 return 0; 278 } 279 280 /* Phase 1 - core voltage transition ... setup voltage */ 281 static int core_voltage_pre_transition(struct powernow_k8_data *data, 282 u32 reqvid, u32 reqfid) 283 { 284 struct msr msr; 285 u32 rvosteps = data->rvo; 286 u32 savefid = data->currfid; 287 u32 maxvid, rvomult = 1; 288 289 pr_debug("ph1 (cpu%d): start, currfid 0x%x, currvid 0x%x, reqvid 0x%x, rvo 0x%x\n", 290 smp_processor_id(), 291 data->currfid, data->currvid, reqvid, data->rvo); 292 293 if ((savefid < LO_FID_TABLE_TOP) && (reqfid < LO_FID_TABLE_TOP)) 294 rvomult = 2; 295 rvosteps *= rvomult; 296 rdmsrq(MSR_FIDVID_STATUS, msr.q); 297 maxvid = 0x1f & (msr.h >> 16); 298 pr_debug("ph1 maxvid=0x%x\n", maxvid); 299 if (reqvid < maxvid) /* lower numbers are higher voltages */ 300 reqvid = maxvid; 301 302 while (data->currvid > reqvid) { 303 pr_debug("ph1: curr 0x%x, req vid 0x%x\n", 304 data->currvid, reqvid); 305 if (decrease_vid_code_by_step(data, reqvid, data->vidmvs)) 306 return 1; 307 } 308 309 while ((rvosteps > 0) && 310 ((rvomult * data->rvo + data->currvid) > reqvid)) { 311 if (data->currvid == maxvid) { 312 rvosteps = 0; 313 } else { 314 pr_debug("ph1: changing vid for rvo, req 0x%x\n", 315 data->currvid - 1); 316 if (decrease_vid_code_by_step(data, data->currvid-1, 1)) 317 return 1; 318 rvosteps--; 319 } 320 } 321 322 if (query_current_values_with_pending_wait(data)) 323 return 1; 324 325 if (savefid != data->currfid) { 326 pr_err("ph1 err, currfid changed 0x%x\n", data->currfid); 327 return 1; 328 } 329 330 pr_debug("ph1 complete, currfid 0x%x, currvid 0x%x\n", 331 data->currfid, data->currvid); 332 333 return 0; 334 } 335 336 /* Phase 2 - core frequency transition */ 337 static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid) 338 { 339 u32 vcoreqfid, vcocurrfid, vcofiddiff; 340 u32 fid_interval, savevid = data->currvid; 341 342 if (data->currfid == reqfid) { 343 pr_err("ph2 null fid transition 0x%x\n", data->currfid); 344 return 0; 345 } 346 347 pr_debug("ph2 (cpu%d): starting, currfid 0x%x, currvid 0x%x, reqfid 0x%x\n", 348 smp_processor_id(), 349 data->currfid, data->currvid, reqfid); 350 351 vcoreqfid = convert_fid_to_vco_fid(reqfid); 352 vcocurrfid = convert_fid_to_vco_fid(data->currfid); 353 vcofiddiff = vcocurrfid > vcoreqfid ? vcocurrfid - vcoreqfid 354 : vcoreqfid - vcocurrfid; 355 356 if ((reqfid <= LO_FID_TABLE_TOP) && (data->currfid <= LO_FID_TABLE_TOP)) 357 vcofiddiff = 0; 358 359 while (vcofiddiff > 2) { 360 (data->currfid & 1) ? (fid_interval = 1) : (fid_interval = 2); 361 362 if (reqfid > data->currfid) { 363 if (data->currfid > LO_FID_TABLE_TOP) { 364 if (write_new_fid(data, 365 data->currfid + fid_interval)) 366 return 1; 367 } else { 368 if (write_new_fid 369 (data, 370 2 + convert_fid_to_vco_fid(data->currfid))) 371 return 1; 372 } 373 } else { 374 if (write_new_fid(data, data->currfid - fid_interval)) 375 return 1; 376 } 377 378 vcocurrfid = convert_fid_to_vco_fid(data->currfid); 379 vcofiddiff = vcocurrfid > vcoreqfid ? vcocurrfid - vcoreqfid 380 : vcoreqfid - vcocurrfid; 381 } 382 383 if (write_new_fid(data, reqfid)) 384 return 1; 385 386 if (query_current_values_with_pending_wait(data)) 387 return 1; 388 389 if (data->currfid != reqfid) { 390 pr_err("ph2: mismatch, failed fid transition, curr 0x%x, req 0x%x\n", 391 data->currfid, reqfid); 392 return 1; 393 } 394 395 if (savevid != data->currvid) { 396 pr_err("ph2: vid changed, save 0x%x, curr 0x%x\n", 397 savevid, data->currvid); 398 return 1; 399 } 400 401 pr_debug("ph2 complete, currfid 0x%x, currvid 0x%x\n", 402 data->currfid, data->currvid); 403 404 return 0; 405 } 406 407 /* Phase 3 - core voltage transition flow ... jump to the final vid. */ 408 static int core_voltage_post_transition(struct powernow_k8_data *data, 409 u32 reqvid) 410 { 411 u32 savefid = data->currfid; 412 u32 savereqvid = reqvid; 413 414 pr_debug("ph3 (cpu%d): starting, currfid 0x%x, currvid 0x%x\n", 415 smp_processor_id(), 416 data->currfid, data->currvid); 417 418 if (reqvid != data->currvid) { 419 if (write_new_vid(data, reqvid)) 420 return 1; 421 422 if (savefid != data->currfid) { 423 pr_err("ph3: bad fid change, save 0x%x, curr 0x%x\n", 424 savefid, data->currfid); 425 return 1; 426 } 427 428 if (data->currvid != reqvid) { 429 pr_err("ph3: failed vid transition\n, req 0x%x, curr 0x%x", 430 reqvid, data->currvid); 431 return 1; 432 } 433 } 434 435 if (query_current_values_with_pending_wait(data)) 436 return 1; 437 438 if (savereqvid != data->currvid) { 439 pr_debug("ph3 failed, currvid 0x%x\n", data->currvid); 440 return 1; 441 } 442 443 if (savefid != data->currfid) { 444 pr_debug("ph3 failed, currfid changed 0x%x\n", 445 data->currfid); 446 return 1; 447 } 448 449 pr_debug("ph3 complete, currfid 0x%x, currvid 0x%x\n", 450 data->currfid, data->currvid); 451 452 return 0; 453 } 454 455 static const struct x86_cpu_id powernow_k8_ids[] = { 456 /* IO based frequency switching */ 457 X86_MATCH_VENDOR_FAM(AMD, 0xf, NULL), 458 {} 459 }; 460 MODULE_DEVICE_TABLE(x86cpu, powernow_k8_ids); 461 462 static void check_supported_cpu(void *_rc) 463 { 464 u32 eax, ebx, ecx, edx; 465 int *rc = _rc; 466 467 *rc = -ENODEV; 468 469 eax = cpuid_eax(CPUID_PROCESSOR_SIGNATURE); 470 471 if ((eax & CPUID_XFAM) == CPUID_XFAM_K8) { 472 if (((eax & CPUID_USE_XFAM_XMOD) != CPUID_USE_XFAM_XMOD) || 473 ((eax & CPUID_XMOD) > CPUID_XMOD_REV_MASK)) { 474 pr_info("Processor cpuid %x not supported\n", eax); 475 return; 476 } 477 478 eax = cpuid_eax(CPUID_GET_MAX_CAPABILITIES); 479 if (eax < CPUID_FREQ_VOLT_CAPABILITIES) { 480 pr_info("No frequency change capabilities detected\n"); 481 return; 482 } 483 484 cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx); 485 if ((edx & P_STATE_TRANSITION_CAPABLE) 486 != P_STATE_TRANSITION_CAPABLE) { 487 pr_info_once("Power state transitions not supported\n"); 488 return; 489 } 490 *rc = 0; 491 } 492 } 493 494 static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst, 495 u8 maxvid) 496 { 497 unsigned int j; 498 u8 lastfid = 0xff; 499 500 for (j = 0; j < data->numps; j++) { 501 if (pst[j].vid > LEAST_VID) { 502 pr_err(FW_BUG "vid %d invalid : 0x%x\n", j, 503 pst[j].vid); 504 return -EINVAL; 505 } 506 if (pst[j].vid < data->rvo) { 507 /* vid + rvo >= 0 */ 508 pr_err(FW_BUG "0 vid exceeded with pstate %d\n", j); 509 return -ENODEV; 510 } 511 if (pst[j].vid < maxvid + data->rvo) { 512 /* vid + rvo >= maxvid */ 513 pr_err(FW_BUG "maxvid exceeded with pstate %d\n", j); 514 return -ENODEV; 515 } 516 if (pst[j].fid > MAX_FID) { 517 pr_err(FW_BUG "maxfid exceeded with pstate %d\n", j); 518 return -ENODEV; 519 } 520 if (j && (pst[j].fid < HI_FID_TABLE_BOTTOM)) { 521 /* Only first fid is allowed to be in "low" range */ 522 pr_err(FW_BUG "two low fids - %d : 0x%x\n", j, 523 pst[j].fid); 524 return -EINVAL; 525 } 526 if (pst[j].fid < lastfid) 527 lastfid = pst[j].fid; 528 } 529 if (lastfid & 1) { 530 pr_err(FW_BUG "lastfid invalid\n"); 531 return -EINVAL; 532 } 533 if (lastfid > LO_FID_TABLE_TOP) 534 pr_info(FW_BUG "first fid not from lo freq table\n"); 535 536 return 0; 537 } 538 539 static void invalidate_entry(struct cpufreq_frequency_table *powernow_table, 540 unsigned int entry) 541 { 542 powernow_table[entry].frequency = CPUFREQ_ENTRY_INVALID; 543 } 544 545 static void print_basics(struct powernow_k8_data *data) 546 { 547 int j; 548 for (j = 0; j < data->numps; j++) { 549 if (data->powernow_table[j].frequency != 550 CPUFREQ_ENTRY_INVALID) { 551 pr_info("fid 0x%x (%d MHz), vid 0x%x\n", 552 data->powernow_table[j].driver_data & 0xff, 553 data->powernow_table[j].frequency/1000, 554 data->powernow_table[j].driver_data >> 8); 555 } 556 } 557 if (data->batps) 558 pr_info("Only %d pstates on battery\n", data->batps); 559 } 560 561 static int fill_powernow_table(struct powernow_k8_data *data, 562 struct pst_s *pst, u8 maxvid) 563 { 564 struct cpufreq_frequency_table *powernow_table; 565 unsigned int j; 566 567 if (data->batps) { 568 /* use ACPI support to get full speed on mains power */ 569 pr_warn("Only %d pstates usable (use ACPI driver for full range\n", 570 data->batps); 571 data->numps = data->batps; 572 } 573 574 for (j = 1; j < data->numps; j++) { 575 if (pst[j-1].fid >= pst[j].fid) { 576 pr_err("PST out of sequence\n"); 577 return -EINVAL; 578 } 579 } 580 581 if (data->numps < 2) { 582 pr_err("no p states to transition\n"); 583 return -ENODEV; 584 } 585 586 if (check_pst_table(data, pst, maxvid)) 587 return -EINVAL; 588 589 powernow_table = kzalloc((sizeof(*powernow_table) 590 * (data->numps + 1)), GFP_KERNEL); 591 if (!powernow_table) 592 return -ENOMEM; 593 594 for (j = 0; j < data->numps; j++) { 595 int freq; 596 powernow_table[j].driver_data = pst[j].fid; /* lower 8 bits */ 597 powernow_table[j].driver_data |= (pst[j].vid << 8); /* upper 8 bits */ 598 freq = find_khz_freq_from_fid(pst[j].fid); 599 powernow_table[j].frequency = freq; 600 } 601 powernow_table[data->numps].frequency = CPUFREQ_TABLE_END; 602 powernow_table[data->numps].driver_data = 0; 603 604 if (query_current_values_with_pending_wait(data)) { 605 kfree(powernow_table); 606 return -EIO; 607 } 608 609 pr_debug("cfid 0x%x, cvid 0x%x\n", data->currfid, data->currvid); 610 data->powernow_table = powernow_table; 611 if (cpumask_first(topology_core_cpumask(data->cpu)) == data->cpu) 612 print_basics(data); 613 614 for (j = 0; j < data->numps; j++) 615 if ((pst[j].fid == data->currfid) && 616 (pst[j].vid == data->currvid)) 617 return 0; 618 619 pr_debug("currfid/vid do not match PST, ignoring\n"); 620 return 0; 621 } 622 623 /* Find and validate the PSB/PST table in BIOS. */ 624 static int find_psb_table(struct powernow_k8_data *data) 625 { 626 struct psb_s *psb; 627 unsigned int i; 628 u32 mvs; 629 u8 maxvid; 630 u32 cpst = 0; 631 u32 thiscpuid; 632 633 for (i = 0xc0000; i < 0xffff0; i += 0x10) { 634 /* Scan BIOS looking for the signature. */ 635 /* It can not be at ffff0 - it is too big. */ 636 637 psb = phys_to_virt(i); 638 if (memcmp(psb, PSB_ID_STRING, PSB_ID_STRING_LEN) != 0) 639 continue; 640 641 pr_debug("found PSB header at 0x%p\n", psb); 642 643 pr_debug("table vers: 0x%x\n", psb->tableversion); 644 if (psb->tableversion != PSB_VERSION_1_4) { 645 pr_err(FW_BUG "PSB table is not v1.4\n"); 646 return -ENODEV; 647 } 648 649 pr_debug("flags: 0x%x\n", psb->flags1); 650 if (psb->flags1) { 651 pr_err(FW_BUG "unknown flags\n"); 652 return -ENODEV; 653 } 654 655 data->vstable = psb->vstable; 656 pr_debug("voltage stabilization time: %d(*20us)\n", 657 data->vstable); 658 659 pr_debug("flags2: 0x%x\n", psb->flags2); 660 data->rvo = psb->flags2 & 3; 661 data->irt = ((psb->flags2) >> 2) & 3; 662 mvs = ((psb->flags2) >> 4) & 3; 663 data->vidmvs = 1 << mvs; 664 data->batps = ((psb->flags2) >> 6) & 3; 665 666 pr_debug("ramp voltage offset: %d\n", data->rvo); 667 pr_debug("isochronous relief time: %d\n", data->irt); 668 pr_debug("maximum voltage step: %d - 0x%x\n", mvs, data->vidmvs); 669 670 pr_debug("numpst: 0x%x\n", psb->num_tables); 671 cpst = psb->num_tables; 672 if ((psb->cpuid == 0x00000fc0) || 673 (psb->cpuid == 0x00000fe0)) { 674 thiscpuid = cpuid_eax(CPUID_PROCESSOR_SIGNATURE); 675 if ((thiscpuid == 0x00000fc0) || 676 (thiscpuid == 0x00000fe0)) 677 cpst = 1; 678 } 679 if (cpst != 1) { 680 pr_err(FW_BUG "numpst must be 1\n"); 681 return -ENODEV; 682 } 683 684 data->plllock = psb->plllocktime; 685 pr_debug("plllocktime: 0x%x (units 1us)\n", psb->plllocktime); 686 pr_debug("maxfid: 0x%x\n", psb->maxfid); 687 pr_debug("maxvid: 0x%x\n", psb->maxvid); 688 maxvid = psb->maxvid; 689 690 data->numps = psb->numps; 691 pr_debug("numpstates: 0x%x\n", data->numps); 692 return fill_powernow_table(data, 693 (struct pst_s *)(psb+1), maxvid); 694 } 695 /* 696 * If you see this message, complain to BIOS manufacturer. If 697 * he tells you "we do not support Linux" or some similar 698 * nonsense, remember that Windows 2000 uses the same legacy 699 * mechanism that the old Linux PSB driver uses. Tell them it 700 * is broken with Windows 2000. 701 * 702 * The reference to the AMD documentation is chapter 9 in the 703 * BIOS and Kernel Developer's Guide, which is available on 704 * www.amd.com 705 */ 706 pr_err(FW_BUG "No PSB or ACPI _PSS objects\n"); 707 pr_err("Make sure that your BIOS is up to date and Cool'N'Quiet support is enabled in BIOS setup\n"); 708 return -ENODEV; 709 } 710 711 static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, 712 unsigned int index) 713 { 714 u64 control; 715 716 if (!data->acpi_data.state_count) 717 return; 718 719 control = data->acpi_data.states[index].control; 720 data->irt = (control >> IRT_SHIFT) & IRT_MASK; 721 data->rvo = (control >> RVO_SHIFT) & RVO_MASK; 722 data->exttype = (control >> EXT_TYPE_SHIFT) & EXT_TYPE_MASK; 723 data->plllock = (control >> PLL_L_SHIFT) & PLL_L_MASK; 724 data->vidmvs = 1 << ((control >> MVS_SHIFT) & MVS_MASK); 725 data->vstable = (control >> VST_SHIFT) & VST_MASK; 726 } 727 728 static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) 729 { 730 struct cpufreq_frequency_table *powernow_table; 731 int ret_val = -ENODEV; 732 u64 control, status; 733 734 if (acpi_processor_register_performance(&data->acpi_data, data->cpu)) { 735 pr_debug("register performance failed: bad ACPI data\n"); 736 return -EIO; 737 } 738 739 /* verify the data contained in the ACPI structures */ 740 if (data->acpi_data.state_count <= 1) { 741 pr_debug("No ACPI P-States\n"); 742 goto err_out; 743 } 744 745 control = data->acpi_data.control_register.space_id; 746 status = data->acpi_data.status_register.space_id; 747 748 if ((control != ACPI_ADR_SPACE_FIXED_HARDWARE) || 749 (status != ACPI_ADR_SPACE_FIXED_HARDWARE)) { 750 pr_debug("Invalid control/status registers (%llx - %llx)\n", 751 control, status); 752 goto err_out; 753 } 754 755 /* fill in data->powernow_table */ 756 powernow_table = kzalloc((sizeof(*powernow_table) 757 * (data->acpi_data.state_count + 1)), GFP_KERNEL); 758 if (!powernow_table) 759 goto err_out; 760 761 /* fill in data */ 762 data->numps = data->acpi_data.state_count; 763 powernow_k8_acpi_pst_values(data, 0); 764 765 ret_val = fill_powernow_table_fidvid(data, powernow_table); 766 if (ret_val) 767 goto err_out_mem; 768 769 powernow_table[data->acpi_data.state_count].frequency = 770 CPUFREQ_TABLE_END; 771 data->powernow_table = powernow_table; 772 773 if (cpumask_first(topology_core_cpumask(data->cpu)) == data->cpu) 774 print_basics(data); 775 776 /* notify BIOS that we exist */ 777 acpi_processor_notify_smm(THIS_MODULE); 778 779 if (!zalloc_cpumask_var(&data->acpi_data.shared_cpu_map, GFP_KERNEL)) { 780 pr_err("unable to alloc powernow_k8_data cpumask\n"); 781 ret_val = -ENOMEM; 782 goto err_out_mem; 783 } 784 785 return 0; 786 787 err_out_mem: 788 kfree(powernow_table); 789 790 err_out: 791 acpi_processor_unregister_performance(data->cpu); 792 793 /* data->acpi_data.state_count informs us at ->exit() 794 * whether ACPI was used */ 795 data->acpi_data.state_count = 0; 796 797 return ret_val; 798 } 799 800 static int fill_powernow_table_fidvid(struct powernow_k8_data *data, 801 struct cpufreq_frequency_table *powernow_table) 802 { 803 int i; 804 805 for (i = 0; i < data->acpi_data.state_count; i++) { 806 u32 fid; 807 u32 vid; 808 u32 freq, index; 809 u64 status, control; 810 811 if (data->exttype) { 812 status = data->acpi_data.states[i].status; 813 fid = status & EXT_FID_MASK; 814 vid = (status >> VID_SHIFT) & EXT_VID_MASK; 815 } else { 816 control = data->acpi_data.states[i].control; 817 fid = control & FID_MASK; 818 vid = (control >> VID_SHIFT) & VID_MASK; 819 } 820 821 pr_debug(" %d : fid 0x%x, vid 0x%x\n", i, fid, vid); 822 823 index = fid | (vid<<8); 824 powernow_table[i].driver_data = index; 825 826 freq = find_khz_freq_from_fid(fid); 827 powernow_table[i].frequency = freq; 828 829 /* verify frequency is OK */ 830 if ((freq > (MAX_FREQ * 1000)) || (freq < (MIN_FREQ * 1000))) { 831 pr_debug("invalid freq %u kHz, ignoring\n", freq); 832 invalidate_entry(powernow_table, i); 833 continue; 834 } 835 836 /* verify voltage is OK - 837 * BIOSs are using "off" to indicate invalid */ 838 if (vid == VID_OFF) { 839 pr_debug("invalid vid %u, ignoring\n", vid); 840 invalidate_entry(powernow_table, i); 841 continue; 842 } 843 844 if (freq != (data->acpi_data.states[i].core_frequency * 1000)) { 845 pr_info("invalid freq entries %u kHz vs. %u kHz\n", 846 freq, (unsigned int) 847 (data->acpi_data.states[i].core_frequency 848 * 1000)); 849 invalidate_entry(powernow_table, i); 850 continue; 851 } 852 } 853 return 0; 854 } 855 856 static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data) 857 { 858 if (data->acpi_data.state_count) 859 acpi_processor_unregister_performance(data->cpu); 860 free_cpumask_var(data->acpi_data.shared_cpu_map); 861 } 862 863 static int get_transition_latency(struct powernow_k8_data *data) 864 { 865 int max_latency = 0; 866 int i; 867 for (i = 0; i < data->acpi_data.state_count; i++) { 868 int cur_latency = data->acpi_data.states[i].transition_latency 869 + data->acpi_data.states[i].bus_master_latency; 870 if (cur_latency > max_latency) 871 max_latency = cur_latency; 872 } 873 if (max_latency == 0) { 874 pr_err(FW_WARN "Invalid zero transition latency\n"); 875 max_latency = 1; 876 } 877 /* value in usecs, needs to be in nanoseconds */ 878 return 1000 * max_latency; 879 } 880 881 /* Take a frequency, and issue the fid/vid transition command */ 882 static int transition_frequency_fidvid(struct powernow_k8_data *data, 883 unsigned int index, 884 struct cpufreq_policy *policy) 885 { 886 u32 fid = 0; 887 u32 vid = 0; 888 int res; 889 struct cpufreq_freqs freqs; 890 891 pr_debug("cpu %d transition to index %u\n", smp_processor_id(), index); 892 893 /* fid/vid correctness check for k8 */ 894 /* fid are the lower 8 bits of the index we stored into 895 * the cpufreq frequency table in find_psb_table, vid 896 * are the upper 8 bits. 897 */ 898 fid = data->powernow_table[index].driver_data & 0xFF; 899 vid = (data->powernow_table[index].driver_data & 0xFF00) >> 8; 900 901 pr_debug("table matched fid 0x%x, giving vid 0x%x\n", fid, vid); 902 903 if (query_current_values_with_pending_wait(data)) 904 return 1; 905 906 if ((data->currvid == vid) && (data->currfid == fid)) { 907 pr_debug("target matches current values (fid 0x%x, vid 0x%x)\n", 908 fid, vid); 909 return 0; 910 } 911 912 pr_debug("cpu %d, changing to fid 0x%x, vid 0x%x\n", 913 smp_processor_id(), fid, vid); 914 freqs.old = find_khz_freq_from_fid(data->currfid); 915 freqs.new = find_khz_freq_from_fid(fid); 916 917 cpufreq_freq_transition_begin(policy, &freqs); 918 res = transition_fid_vid(data, fid, vid); 919 cpufreq_freq_transition_end(policy, &freqs, res); 920 921 return res; 922 } 923 924 struct powernowk8_target_arg { 925 struct cpufreq_policy *pol; 926 unsigned newstate; 927 }; 928 929 static long powernowk8_target_fn(void *arg) 930 { 931 struct powernowk8_target_arg *pta = arg; 932 struct cpufreq_policy *pol = pta->pol; 933 unsigned newstate = pta->newstate; 934 struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); 935 u32 checkfid; 936 u32 checkvid; 937 int ret; 938 939 if (!data) 940 return -EINVAL; 941 942 checkfid = data->currfid; 943 checkvid = data->currvid; 944 945 if (pending_bit_stuck()) { 946 pr_err("failing targ, change pending bit set\n"); 947 return -EIO; 948 } 949 950 pr_debug("targ: cpu %d, %d kHz, min %d, max %d\n", 951 pol->cpu, data->powernow_table[newstate].frequency, pol->min, 952 pol->max); 953 954 if (query_current_values_with_pending_wait(data)) 955 return -EIO; 956 957 pr_debug("targ: curr fid 0x%x, vid 0x%x\n", 958 data->currfid, data->currvid); 959 960 if ((checkvid != data->currvid) || 961 (checkfid != data->currfid)) { 962 pr_info("error - out of sync, fix 0x%x 0x%x, vid 0x%x 0x%x\n", 963 checkfid, data->currfid, 964 checkvid, data->currvid); 965 } 966 967 mutex_lock(&fidvid_mutex); 968 969 powernow_k8_acpi_pst_values(data, newstate); 970 971 ret = transition_frequency_fidvid(data, newstate, pol); 972 973 if (ret) { 974 pr_err("transition frequency failed\n"); 975 mutex_unlock(&fidvid_mutex); 976 return 1; 977 } 978 mutex_unlock(&fidvid_mutex); 979 980 pol->cur = find_khz_freq_from_fid(data->currfid); 981 982 return 0; 983 } 984 985 /* Driver entry point to switch to the target frequency */ 986 static int powernowk8_target(struct cpufreq_policy *pol, unsigned index) 987 { 988 struct powernowk8_target_arg pta = { .pol = pol, .newstate = index }; 989 990 return work_on_cpu(pol->cpu, powernowk8_target_fn, &pta); 991 } 992 993 struct init_on_cpu { 994 struct powernow_k8_data *data; 995 int rc; 996 }; 997 998 static void powernowk8_cpu_init_on_cpu(void *_init_on_cpu) 999 { 1000 struct init_on_cpu *init_on_cpu = _init_on_cpu; 1001 1002 if (pending_bit_stuck()) { 1003 pr_err("failing init, change pending bit set\n"); 1004 init_on_cpu->rc = -ENODEV; 1005 return; 1006 } 1007 1008 if (query_current_values_with_pending_wait(init_on_cpu->data)) { 1009 init_on_cpu->rc = -ENODEV; 1010 return; 1011 } 1012 1013 fidvid_msr_init(); 1014 1015 init_on_cpu->rc = 0; 1016 } 1017 1018 #define MISSING_PSS_MSG \ 1019 FW_BUG "No compatible ACPI _PSS objects found.\n" \ 1020 FW_BUG "First, make sure Cool'N'Quiet is enabled in the BIOS.\n" \ 1021 FW_BUG "If that doesn't help, try upgrading your BIOS.\n" 1022 1023 /* per CPU init entry point to the driver */ 1024 static int powernowk8_cpu_init(struct cpufreq_policy *pol) 1025 { 1026 struct powernow_k8_data *data; 1027 struct init_on_cpu init_on_cpu; 1028 int rc, cpu; 1029 1030 smp_call_function_single(pol->cpu, check_supported_cpu, &rc, 1); 1031 if (rc) 1032 return -ENODEV; 1033 1034 data = kzalloc_obj(*data); 1035 if (!data) 1036 return -ENOMEM; 1037 1038 data->cpu = pol->cpu; 1039 1040 if (powernow_k8_cpu_init_acpi(data)) { 1041 /* 1042 * Use the PSB BIOS structure. This is only available on 1043 * an UP version, and is deprecated by AMD. 1044 */ 1045 if (num_online_cpus() != 1) { 1046 pr_err_once(MISSING_PSS_MSG); 1047 goto err_out; 1048 } 1049 if (pol->cpu != 0) { 1050 pr_err(FW_BUG "No ACPI _PSS objects for CPU other than CPU0. Complain to your BIOS vendor.\n"); 1051 goto err_out; 1052 } 1053 rc = find_psb_table(data); 1054 if (rc) 1055 goto err_out; 1056 1057 /* Take a crude guess here. 1058 * That guess was in microseconds, so multiply with 1000 */ 1059 pol->cpuinfo.transition_latency = ( 1060 ((data->rvo + 8) * data->vstable * VST_UNITS_20US) + 1061 ((1 << data->irt) * 30)) * 1000; 1062 } else /* ACPI _PSS objects available */ 1063 pol->cpuinfo.transition_latency = get_transition_latency(data); 1064 1065 /* only run on specific CPU from here on */ 1066 init_on_cpu.data = data; 1067 smp_call_function_single(data->cpu, powernowk8_cpu_init_on_cpu, 1068 &init_on_cpu, 1); 1069 rc = init_on_cpu.rc; 1070 if (rc != 0) 1071 goto err_out_exit_acpi; 1072 1073 cpumask_copy(pol->cpus, topology_core_cpumask(pol->cpu)); 1074 data->available_cores = pol->cpus; 1075 pol->freq_table = data->powernow_table; 1076 1077 pr_debug("cpu_init done, current fid 0x%x, vid 0x%x\n", 1078 data->currfid, data->currvid); 1079 1080 /* Point all the CPUs in this policy to the same data */ 1081 for_each_cpu(cpu, pol->cpus) 1082 per_cpu(powernow_data, cpu) = data; 1083 1084 return 0; 1085 1086 err_out_exit_acpi: 1087 powernow_k8_cpu_exit_acpi(data); 1088 kfree(data->powernow_table); 1089 1090 err_out: 1091 kfree(data); 1092 return -ENODEV; 1093 } 1094 1095 static void powernowk8_cpu_exit(struct cpufreq_policy *pol) 1096 { 1097 struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); 1098 int cpu; 1099 1100 if (!data) 1101 return; 1102 1103 powernow_k8_cpu_exit_acpi(data); 1104 1105 kfree(data->powernow_table); 1106 kfree(data); 1107 /* pol->cpus will be empty here, use related_cpus instead. */ 1108 for_each_cpu(cpu, pol->related_cpus) 1109 per_cpu(powernow_data, cpu) = NULL; 1110 } 1111 1112 static void query_values_on_cpu(void *_err) 1113 { 1114 int *err = _err; 1115 struct powernow_k8_data *data = __this_cpu_read(powernow_data); 1116 1117 *err = query_current_values_with_pending_wait(data); 1118 } 1119 1120 static unsigned int powernowk8_get(unsigned int cpu) 1121 { 1122 struct powernow_k8_data *data = per_cpu(powernow_data, cpu); 1123 unsigned int khz = 0; 1124 int err; 1125 1126 if (!data) 1127 return 0; 1128 1129 smp_call_function_single(cpu, query_values_on_cpu, &err, true); 1130 if (err) 1131 goto out; 1132 1133 khz = find_khz_freq_from_fid(data->currfid); 1134 1135 1136 out: 1137 return khz; 1138 } 1139 1140 static struct cpufreq_driver cpufreq_amd64_driver = { 1141 .flags = CPUFREQ_ASYNC_NOTIFICATION, 1142 .verify = cpufreq_generic_frequency_table_verify, 1143 .target_index = powernowk8_target, 1144 .bios_limit = acpi_processor_get_bios_limit, 1145 .init = powernowk8_cpu_init, 1146 .exit = powernowk8_cpu_exit, 1147 .get = powernowk8_get, 1148 .name = "powernow-k8", 1149 }; 1150 1151 static void __request_acpi_cpufreq(void) 1152 { 1153 const char drv[] = "acpi-cpufreq"; 1154 const char *cur_drv; 1155 1156 cur_drv = cpufreq_get_current_driver(); 1157 if (!cur_drv) 1158 goto request; 1159 1160 if (strncmp(cur_drv, drv, min_t(size_t, strlen(cur_drv), strlen(drv)))) 1161 pr_warn("WTF driver: %s\n", cur_drv); 1162 1163 return; 1164 1165 request: 1166 pr_warn("This CPU is not supported anymore, using acpi-cpufreq instead.\n"); 1167 request_module(drv); 1168 } 1169 1170 /* driver entry point for init */ 1171 static int powernowk8_init(void) 1172 { 1173 unsigned int i, supported_cpus = 0; 1174 int ret; 1175 1176 if (!x86_match_cpu(powernow_k8_ids)) 1177 return -ENODEV; 1178 1179 if (boot_cpu_has(X86_FEATURE_HW_PSTATE)) { 1180 __request_acpi_cpufreq(); 1181 return -ENODEV; 1182 } 1183 1184 cpus_read_lock(); 1185 for_each_online_cpu(i) { 1186 smp_call_function_single(i, check_supported_cpu, &ret, 1); 1187 if (!ret) 1188 supported_cpus++; 1189 } 1190 1191 if (supported_cpus != num_online_cpus()) { 1192 cpus_read_unlock(); 1193 return -ENODEV; 1194 } 1195 cpus_read_unlock(); 1196 1197 ret = cpufreq_register_driver(&cpufreq_amd64_driver); 1198 if (ret) 1199 return ret; 1200 1201 pr_info("Found %d %s (%d cpu cores) (" VERSION ")\n", 1202 num_online_nodes(), boot_cpu_data.x86_model_id, supported_cpus); 1203 1204 return ret; 1205 } 1206 1207 /* driver entry point for term */ 1208 static void __exit powernowk8_exit(void) 1209 { 1210 pr_debug("exit\n"); 1211 1212 cpufreq_unregister_driver(&cpufreq_amd64_driver); 1213 } 1214 1215 MODULE_AUTHOR("Paul Devriendt <paul.devriendt@amd.com>"); 1216 MODULE_AUTHOR("Mark Langsdorf <mark.langsdorf@amd.com>"); 1217 MODULE_DESCRIPTION("AMD Athlon 64 and Opteron processor frequency driver."); 1218 MODULE_LICENSE("GPL"); 1219 1220 late_initcall(powernowk8_init); 1221 module_exit(powernowk8_exit); 1222