1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Intel Core SoC Power Management Controller Driver 4 * 5 * Copyright (c) 2016, Intel Corporation. 6 * All Rights Reserved. 7 * 8 * Authors: Rajneesh Bhardwaj <rajneesh.bhardwaj@intel.com> 9 * Vishwanath Somayaji <vishwanath.somayaji@intel.com> 10 */ 11 12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 13 14 #include <linux/acpi_pmtmr.h> 15 #include <linux/bitfield.h> 16 #include <linux/debugfs.h> 17 #include <linux/delay.h> 18 #include <linux/dmi.h> 19 #include <linux/io.h> 20 #include <linux/module.h> 21 #include <linux/pci.h> 22 #include <linux/slab.h> 23 #include <linux/suspend.h> 24 #include <linux/units.h> 25 26 #include <asm/cpu_device_id.h> 27 #include <asm/intel-family.h> 28 #include <asm/msr.h> 29 #include <asm/tsc.h> 30 31 #include "core.h" 32 #include "../pmt/telemetry.h" 33 34 /* Maximum number of modes supported by platfoms that has low power mode capability */ 35 const char *pmc_lpm_modes[] = { 36 "S0i2.0", 37 "S0i2.1", 38 "S0i2.2", 39 "S0i3.0", 40 "S0i3.1", 41 "S0i3.2", 42 "S0i3.3", 43 "S0i3.4", 44 NULL 45 }; 46 47 /* PKGC MSRs are common across Intel Core SoCs */ 48 const struct pmc_bit_map msr_map[] = { 49 {"Package C2", MSR_PKG_C2_RESIDENCY}, 50 {"Package C3", MSR_PKG_C3_RESIDENCY}, 51 {"Package C6", MSR_PKG_C6_RESIDENCY}, 52 {"Package C7", MSR_PKG_C7_RESIDENCY}, 53 {"Package C8", MSR_PKG_C8_RESIDENCY}, 54 {"Package C9", MSR_PKG_C9_RESIDENCY}, 55 {"Package C10", MSR_PKG_C10_RESIDENCY}, 56 {} 57 }; 58 59 static inline u32 pmc_core_reg_read(struct pmc *pmc, int reg_offset) 60 { 61 return readl(pmc->regbase + reg_offset); 62 } 63 64 static inline void pmc_core_reg_write(struct pmc *pmc, int reg_offset, 65 u32 val) 66 { 67 writel(val, pmc->regbase + reg_offset); 68 } 69 70 static inline u64 pmc_core_adjust_slp_s0_step(struct pmc *pmc, u32 value) 71 { 72 /* 73 * ADL PCH does not have the SLP_S0 counter and LPM Residency counters are 74 * used as a workaround which uses 30.5 usec tick. All other client 75 * programs have the legacy SLP_S0 residency counter that is using the 122 76 * usec tick. 77 */ 78 const int lpm_adj_x2 = pmc->map->lpm_res_counter_step_x2; 79 80 if (pmc->map == &adl_reg_map) 81 return (u64)value * GET_X2_COUNTER((u64)lpm_adj_x2); 82 else 83 return (u64)value * pmc->map->slp_s0_res_counter_step; 84 } 85 86 static int set_etr3(struct pmc_dev *pmcdev) 87 { 88 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 89 const struct pmc_reg_map *map = pmc->map; 90 u32 reg; 91 92 if (!map->etr3_offset) 93 return -EOPNOTSUPP; 94 95 guard(mutex)(&pmcdev->lock); 96 97 /* check if CF9 is locked */ 98 reg = pmc_core_reg_read(pmc, map->etr3_offset); 99 if (reg & ETR3_CF9LOCK) 100 return -EACCES; 101 102 /* write CF9 global reset bit */ 103 reg |= ETR3_CF9GR; 104 pmc_core_reg_write(pmc, map->etr3_offset, reg); 105 106 reg = pmc_core_reg_read(pmc, map->etr3_offset); 107 if (!(reg & ETR3_CF9GR)) 108 return -EIO; 109 110 return 0; 111 } 112 static umode_t etr3_is_visible(struct kobject *kobj, 113 struct attribute *attr, 114 int idx) 115 { 116 struct device *dev = kobj_to_dev(kobj); 117 struct pmc_dev *pmcdev = dev_get_drvdata(dev); 118 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 119 const struct pmc_reg_map *map = pmc->map; 120 u32 reg; 121 122 scoped_guard(mutex, &pmcdev->lock) 123 reg = pmc_core_reg_read(pmc, map->etr3_offset); 124 125 return reg & ETR3_CF9LOCK ? attr->mode & (SYSFS_PREALLOC | 0444) : attr->mode; 126 } 127 128 static ssize_t etr3_show(struct device *dev, 129 struct device_attribute *attr, char *buf) 130 { 131 struct pmc_dev *pmcdev = dev_get_drvdata(dev); 132 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 133 const struct pmc_reg_map *map = pmc->map; 134 u32 reg; 135 136 if (!map->etr3_offset) 137 return -EOPNOTSUPP; 138 139 scoped_guard(mutex, &pmcdev->lock) { 140 reg = pmc_core_reg_read(pmc, map->etr3_offset); 141 reg &= ETR3_CF9GR | ETR3_CF9LOCK; 142 } 143 144 return sysfs_emit(buf, "0x%08x", reg); 145 } 146 147 static ssize_t etr3_store(struct device *dev, 148 struct device_attribute *attr, 149 const char *buf, size_t len) 150 { 151 struct pmc_dev *pmcdev = dev_get_drvdata(dev); 152 int err; 153 u32 reg; 154 155 err = kstrtouint(buf, 16, ®); 156 if (err) 157 return err; 158 159 /* allow only CF9 writes */ 160 if (reg != ETR3_CF9GR) 161 return -EINVAL; 162 163 err = set_etr3(pmcdev); 164 if (err) 165 return err; 166 167 return len; 168 } 169 static DEVICE_ATTR_RW(etr3); 170 171 static struct attribute *pmc_attrs[] = { 172 &dev_attr_etr3.attr, 173 NULL 174 }; 175 176 static const struct attribute_group pmc_attr_group = { 177 .attrs = pmc_attrs, 178 .is_visible = etr3_is_visible, 179 }; 180 181 static const struct attribute_group *pmc_dev_groups[] = { 182 &pmc_attr_group, 183 NULL 184 }; 185 186 static int pmc_core_dev_state_get(void *data, u64 *val) 187 { 188 struct pmc *pmc = data; 189 const struct pmc_reg_map *map = pmc->map; 190 u32 value; 191 192 value = pmc_core_reg_read(pmc, map->slp_s0_offset); 193 *val = pmc_core_adjust_slp_s0_step(pmc, value); 194 195 return 0; 196 } 197 198 DEFINE_DEBUGFS_ATTRIBUTE(pmc_core_dev_state, pmc_core_dev_state_get, NULL, "%llu\n"); 199 200 static int pmc_core_pson_residency_get(void *data, u64 *val) 201 { 202 struct pmc *pmc = data; 203 const struct pmc_reg_map *map = pmc->map; 204 u32 value; 205 206 value = pmc_core_reg_read(pmc, map->pson_residency_offset); 207 *val = (u64)value * map->pson_residency_counter_step; 208 209 return 0; 210 } 211 212 DEFINE_DEBUGFS_ATTRIBUTE(pmc_core_pson_residency, pmc_core_pson_residency_get, NULL, "%llu\n"); 213 214 static int pmc_core_check_read_lock_bit(struct pmc *pmc) 215 { 216 u32 value; 217 218 value = pmc_core_reg_read(pmc, pmc->map->pm_cfg_offset); 219 return value & BIT(pmc->map->pm_read_disable_bit); 220 } 221 222 static void pmc_core_slps0_display(struct pmc *pmc, struct device *dev, 223 struct seq_file *s) 224 { 225 const struct pmc_bit_map **maps = pmc->map->slps0_dbg_maps; 226 const struct pmc_bit_map *map; 227 int offset = pmc->map->slps0_dbg_offset; 228 u32 data; 229 230 while (*maps) { 231 map = *maps; 232 data = pmc_core_reg_read(pmc, offset); 233 offset += 4; 234 while (map->name) { 235 if (dev) 236 dev_info(dev, "SLP_S0_DBG: %-32s\tState: %s\n", 237 map->name, 238 data & map->bit_mask ? "Yes" : "No"); 239 if (s) 240 seq_printf(s, "SLP_S0_DBG: %-32s\tState: %s\n", 241 map->name, 242 data & map->bit_mask ? "Yes" : "No"); 243 ++map; 244 } 245 ++maps; 246 } 247 } 248 249 static unsigned int pmc_core_lpm_get_arr_size(const struct pmc_bit_map **maps) 250 { 251 unsigned int idx; 252 253 for (idx = 0; maps[idx]; idx++) 254 ;/* Nothing */ 255 256 return idx; 257 } 258 259 static void pmc_core_lpm_display(struct pmc *pmc, struct device *dev, 260 struct seq_file *s, u32 offset, int pmc_index, 261 const char *str, 262 const struct pmc_bit_map **maps) 263 { 264 unsigned int index, idx, len = 32, arr_size; 265 u32 bit_mask, *lpm_regs; 266 267 arr_size = pmc_core_lpm_get_arr_size(maps); 268 lpm_regs = kmalloc_array(arr_size, sizeof(*lpm_regs), GFP_KERNEL); 269 if (!lpm_regs) 270 return; 271 272 for (index = 0; index < arr_size; index++) { 273 lpm_regs[index] = pmc_core_reg_read(pmc, offset); 274 offset += 4; 275 } 276 277 for (idx = 0; idx < arr_size; idx++) { 278 if (dev) 279 dev_info(dev, "\nPMC%d:LPM_%s_%d:\t0x%x\n", pmc_index, str, idx, 280 lpm_regs[idx]); 281 if (s) 282 seq_printf(s, "\nPMC%d:LPM_%s_%d:\t0x%x\n", pmc_index, str, idx, 283 lpm_regs[idx]); 284 for (index = 0; maps[idx][index].name && index < len; index++) { 285 bit_mask = maps[idx][index].bit_mask; 286 if (dev) 287 dev_info(dev, "PMC%d:%-30s %-30d\n", pmc_index, 288 maps[idx][index].name, 289 lpm_regs[idx] & bit_mask ? 1 : 0); 290 if (s) 291 seq_printf(s, "PMC%d:%-30s %-30d\n", pmc_index, 292 maps[idx][index].name, 293 lpm_regs[idx] & bit_mask ? 1 : 0); 294 } 295 } 296 297 kfree(lpm_regs); 298 } 299 300 static bool slps0_dbg_latch; 301 302 static inline u8 pmc_core_reg_read_byte(struct pmc *pmc, int offset) 303 { 304 return readb(pmc->regbase + offset); 305 } 306 307 static void pmc_core_display_map(struct seq_file *s, int index, int idx, int ip, 308 int pmc_index, u8 pf_reg, const struct pmc_bit_map **pf_map) 309 { 310 seq_printf(s, "PMC%d:PCH IP: %-2d - %-32s\tState: %s\n", 311 pmc_index, ip, pf_map[idx][index].name, 312 pf_map[idx][index].bit_mask & pf_reg ? "Off" : "On"); 313 } 314 315 static int pmc_core_ppfear_show(struct seq_file *s, void *unused) 316 { 317 struct pmc_dev *pmcdev = s->private; 318 unsigned int i; 319 320 for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) { 321 struct pmc *pmc = pmcdev->pmcs[i]; 322 const struct pmc_bit_map **maps; 323 u8 pf_regs[PPFEAR_MAX_NUM_ENTRIES]; 324 unsigned int index, iter, idx, ip = 0; 325 326 if (!pmc) 327 continue; 328 329 maps = pmc->map->pfear_sts; 330 iter = pmc->map->ppfear0_offset; 331 332 for (index = 0; index < pmc->map->ppfear_buckets && 333 index < PPFEAR_MAX_NUM_ENTRIES; index++, iter++) 334 pf_regs[index] = pmc_core_reg_read_byte(pmc, iter); 335 336 for (idx = 0; maps[idx]; idx++) { 337 for (index = 0; maps[idx][index].name && 338 index < pmc->map->ppfear_buckets * 8; ip++, index++) 339 pmc_core_display_map(s, index, idx, ip, i, 340 pf_regs[index / 8], maps); 341 } 342 } 343 344 return 0; 345 } 346 DEFINE_SHOW_ATTRIBUTE(pmc_core_ppfear); 347 348 /* This function should return link status, 0 means ready */ 349 static int pmc_core_mtpmc_link_status(struct pmc *pmc) 350 { 351 u32 value; 352 353 value = pmc_core_reg_read(pmc, SPT_PMC_PM_STS_OFFSET); 354 return value & BIT(SPT_PMC_MSG_FULL_STS_BIT); 355 } 356 357 static int pmc_core_send_msg(struct pmc *pmc, u32 *addr_xram) 358 { 359 u32 dest; 360 int timeout; 361 362 for (timeout = NUM_RETRIES; timeout > 0; timeout--) { 363 if (pmc_core_mtpmc_link_status(pmc) == 0) 364 break; 365 msleep(5); 366 } 367 368 if (timeout <= 0 && pmc_core_mtpmc_link_status(pmc)) 369 return -EBUSY; 370 371 dest = (*addr_xram & MTPMC_MASK) | (1U << 1); 372 pmc_core_reg_write(pmc, SPT_PMC_MTPMC_OFFSET, dest); 373 return 0; 374 } 375 376 static int pmc_core_mphy_pg_show(struct seq_file *s, void *unused) 377 { 378 struct pmc_dev *pmcdev = s->private; 379 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 380 const struct pmc_bit_map *map = pmc->map->mphy_sts; 381 u32 mphy_core_reg_low, mphy_core_reg_high; 382 u32 val_low, val_high; 383 unsigned int index; 384 int err = 0; 385 386 if (pmcdev->pmc_xram_read_bit) { 387 seq_puts(s, "Access denied: please disable PMC_READ_DISABLE setting in BIOS."); 388 return 0; 389 } 390 391 mphy_core_reg_low = (SPT_PMC_MPHY_CORE_STS_0 << 16); 392 mphy_core_reg_high = (SPT_PMC_MPHY_CORE_STS_1 << 16); 393 394 guard(mutex)(&pmcdev->lock); 395 396 err = pmc_core_send_msg(pmc, &mphy_core_reg_low); 397 if (err) 398 return err; 399 400 msleep(10); 401 val_low = pmc_core_reg_read(pmc, SPT_PMC_MFPMC_OFFSET); 402 403 err = pmc_core_send_msg(pmc, &mphy_core_reg_high); 404 if (err) 405 return err; 406 407 msleep(10); 408 val_high = pmc_core_reg_read(pmc, SPT_PMC_MFPMC_OFFSET); 409 410 for (index = 0; index < 8 && map[index].name; index++) { 411 seq_printf(s, "%-32s\tState: %s\n", 412 map[index].name, 413 map[index].bit_mask & val_low ? "Not power gated" : 414 "Power gated"); 415 } 416 417 for (index = 8; map[index].name; index++) { 418 seq_printf(s, "%-32s\tState: %s\n", 419 map[index].name, 420 map[index].bit_mask & val_high ? "Not power gated" : 421 "Power gated"); 422 } 423 424 return 0; 425 } 426 DEFINE_SHOW_ATTRIBUTE(pmc_core_mphy_pg); 427 428 static int pmc_core_pll_show(struct seq_file *s, void *unused) 429 { 430 struct pmc_dev *pmcdev = s->private; 431 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 432 const struct pmc_bit_map *map = pmc->map->pll_sts; 433 u32 mphy_common_reg, val; 434 unsigned int index; 435 int err = 0; 436 437 if (pmcdev->pmc_xram_read_bit) { 438 seq_puts(s, "Access denied: please disable PMC_READ_DISABLE setting in BIOS."); 439 return 0; 440 } 441 442 mphy_common_reg = (SPT_PMC_MPHY_COM_STS_0 << 16); 443 guard(mutex)(&pmcdev->lock); 444 445 err = pmc_core_send_msg(pmc, &mphy_common_reg); 446 if (err) 447 return err; 448 449 /* Observed PMC HW response latency for MTPMC-MFPMC is ~10 ms */ 450 msleep(10); 451 val = pmc_core_reg_read(pmc, SPT_PMC_MFPMC_OFFSET); 452 453 for (index = 0; map[index].name ; index++) { 454 seq_printf(s, "%-32s\tState: %s\n", 455 map[index].name, 456 map[index].bit_mask & val ? "Active" : "Idle"); 457 } 458 459 return 0; 460 } 461 DEFINE_SHOW_ATTRIBUTE(pmc_core_pll); 462 463 int pmc_core_send_ltr_ignore(struct pmc_dev *pmcdev, u32 value, int ignore) 464 { 465 struct pmc *pmc; 466 const struct pmc_reg_map *map; 467 u32 reg; 468 unsigned int pmc_index; 469 int ltr_index; 470 471 ltr_index = value; 472 /* For platforms with multiple pmcs, ltr index value given by user 473 * is based on the contiguous indexes from ltr_show output. 474 * pmc index and ltr index needs to be calculated from it. 475 */ 476 for (pmc_index = 0; pmc_index < ARRAY_SIZE(pmcdev->pmcs) && ltr_index >= 0; pmc_index++) { 477 pmc = pmcdev->pmcs[pmc_index]; 478 479 if (!pmc) 480 continue; 481 482 map = pmc->map; 483 if (ltr_index <= map->ltr_ignore_max) 484 break; 485 486 /* Along with IP names, ltr_show map includes CURRENT_PLATFORM 487 * and AGGREGATED_SYSTEM values per PMC. Take these two index 488 * values into account in ltr_index calculation. Also, to start 489 * ltr index from zero for next pmc, subtract it by 1. 490 */ 491 ltr_index = ltr_index - (map->ltr_ignore_max + 2) - 1; 492 } 493 494 if (pmc_index >= ARRAY_SIZE(pmcdev->pmcs) || ltr_index < 0) 495 return -EINVAL; 496 497 pr_debug("ltr_ignore for pmc%d: ltr_index:%d\n", pmc_index, ltr_index); 498 499 guard(mutex)(&pmcdev->lock); 500 501 reg = pmc_core_reg_read(pmc, map->ltr_ignore_offset); 502 if (ignore) 503 reg |= BIT(ltr_index); 504 else 505 reg &= ~BIT(ltr_index); 506 pmc_core_reg_write(pmc, map->ltr_ignore_offset, reg); 507 508 return 0; 509 } 510 511 static ssize_t pmc_core_ltr_write(struct pmc_dev *pmcdev, 512 const char __user *userbuf, 513 size_t count, int ignore) 514 { 515 u32 value; 516 int err; 517 518 err = kstrtou32_from_user(userbuf, count, 10, &value); 519 if (err) 520 return err; 521 522 err = pmc_core_send_ltr_ignore(pmcdev, value, ignore); 523 524 return err ?: count; 525 } 526 527 static ssize_t pmc_core_ltr_ignore_write(struct file *file, 528 const char __user *userbuf, 529 size_t count, loff_t *ppos) 530 { 531 struct seq_file *s = file->private_data; 532 struct pmc_dev *pmcdev = s->private; 533 534 return pmc_core_ltr_write(pmcdev, userbuf, count, 1); 535 } 536 537 static int pmc_core_ltr_ignore_show(struct seq_file *s, void *unused) 538 { 539 return 0; 540 } 541 DEFINE_SHOW_STORE_ATTRIBUTE(pmc_core_ltr_ignore); 542 543 static ssize_t pmc_core_ltr_restore_write(struct file *file, 544 const char __user *userbuf, 545 size_t count, loff_t *ppos) 546 { 547 struct seq_file *s = file->private_data; 548 struct pmc_dev *pmcdev = s->private; 549 550 return pmc_core_ltr_write(pmcdev, userbuf, count, 0); 551 } 552 553 static int pmc_core_ltr_restore_show(struct seq_file *s, void *unused) 554 { 555 return 0; 556 } 557 DEFINE_SHOW_STORE_ATTRIBUTE(pmc_core_ltr_restore); 558 559 static void pmc_core_slps0_dbg_latch(struct pmc_dev *pmcdev, bool reset) 560 { 561 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 562 const struct pmc_reg_map *map = pmc->map; 563 u32 fd; 564 565 guard(mutex)(&pmcdev->lock); 566 567 if (!reset && !slps0_dbg_latch) 568 return; 569 570 fd = pmc_core_reg_read(pmc, map->slps0_dbg_offset); 571 if (reset) 572 fd &= ~CNP_PMC_LATCH_SLPS0_EVENTS; 573 else 574 fd |= CNP_PMC_LATCH_SLPS0_EVENTS; 575 pmc_core_reg_write(pmc, map->slps0_dbg_offset, fd); 576 577 slps0_dbg_latch = false; 578 } 579 580 static int pmc_core_slps0_dbg_show(struct seq_file *s, void *unused) 581 { 582 struct pmc_dev *pmcdev = s->private; 583 584 pmc_core_slps0_dbg_latch(pmcdev, false); 585 pmc_core_slps0_display(pmcdev->pmcs[PMC_IDX_MAIN], NULL, s); 586 pmc_core_slps0_dbg_latch(pmcdev, true); 587 588 return 0; 589 } 590 DEFINE_SHOW_ATTRIBUTE(pmc_core_slps0_dbg); 591 592 static u32 convert_ltr_scale(u32 val) 593 { 594 /* 595 * As per PCIE specification supporting document 596 * ECN_LatencyTolnReporting_14Aug08.pdf the Latency 597 * Tolerance Reporting data payload is encoded in a 598 * 3 bit scale and 10 bit value fields. Values are 599 * multiplied by the indicated scale to yield an absolute time 600 * value, expressible in a range from 1 nanosecond to 601 * 2^25*(2^10-1) = 34,326,183,936 nanoseconds. 602 * 603 * scale encoding is as follows: 604 * 605 * ---------------------------------------------- 606 * |scale factor | Multiplier (ns) | 607 * ---------------------------------------------- 608 * | 0 | 1 | 609 * | 1 | 32 | 610 * | 2 | 1024 | 611 * | 3 | 32768 | 612 * | 4 | 1048576 | 613 * | 5 | 33554432 | 614 * | 6 | Invalid | 615 * | 7 | Invalid | 616 * ---------------------------------------------- 617 */ 618 if (val > 5) { 619 pr_warn("Invalid LTR scale factor.\n"); 620 return 0; 621 } 622 623 return 1U << (5 * val); 624 } 625 626 static int pmc_core_ltr_show(struct seq_file *s, void *unused) 627 { 628 struct pmc_dev *pmcdev = s->private; 629 u64 decoded_snoop_ltr, decoded_non_snoop_ltr; 630 u32 ltr_raw_data, scale, val; 631 u16 snoop_ltr, nonsnoop_ltr; 632 unsigned int i, index, ltr_index = 0; 633 634 for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) { 635 struct pmc *pmc; 636 const struct pmc_bit_map *map; 637 u32 ltr_ign_reg; 638 639 pmc = pmcdev->pmcs[i]; 640 if (!pmc) 641 continue; 642 643 scoped_guard(mutex, &pmcdev->lock) 644 ltr_ign_reg = pmc_core_reg_read(pmc, pmc->map->ltr_ignore_offset); 645 646 map = pmc->map->ltr_show_sts; 647 for (index = 0; map[index].name; index++) { 648 bool ltr_ign_data; 649 650 if (index > pmc->map->ltr_ignore_max) 651 ltr_ign_data = false; 652 else 653 ltr_ign_data = ltr_ign_reg & BIT(index); 654 655 decoded_snoop_ltr = decoded_non_snoop_ltr = 0; 656 ltr_raw_data = pmc_core_reg_read(pmc, 657 map[index].bit_mask); 658 snoop_ltr = ltr_raw_data & ~MTPMC_MASK; 659 nonsnoop_ltr = (ltr_raw_data >> 0x10) & ~MTPMC_MASK; 660 661 if (FIELD_GET(LTR_REQ_NONSNOOP, ltr_raw_data)) { 662 scale = FIELD_GET(LTR_DECODED_SCALE, nonsnoop_ltr); 663 val = FIELD_GET(LTR_DECODED_VAL, nonsnoop_ltr); 664 decoded_non_snoop_ltr = val * convert_ltr_scale(scale); 665 } 666 if (FIELD_GET(LTR_REQ_SNOOP, ltr_raw_data)) { 667 scale = FIELD_GET(LTR_DECODED_SCALE, snoop_ltr); 668 val = FIELD_GET(LTR_DECODED_VAL, snoop_ltr); 669 decoded_snoop_ltr = val * convert_ltr_scale(scale); 670 } 671 672 seq_printf(s, "%d\tPMC%d:%-32s\tLTR: RAW: 0x%-16x\tNon-Snoop(ns): %-16llu\tSnoop(ns): %-16llu\tLTR_IGNORE: %d\n", 673 ltr_index, i, map[index].name, ltr_raw_data, 674 decoded_non_snoop_ltr, 675 decoded_snoop_ltr, ltr_ign_data); 676 ltr_index++; 677 } 678 } 679 return 0; 680 } 681 DEFINE_SHOW_ATTRIBUTE(pmc_core_ltr); 682 683 static int pmc_core_s0ix_blocker_show(struct seq_file *s, void *unused) 684 { 685 struct pmc_dev *pmcdev = s->private; 686 unsigned int pmcidx; 687 688 for (pmcidx = 0; pmcidx < ARRAY_SIZE(pmcdev->pmcs); pmcidx++) { 689 const struct pmc_bit_map **maps; 690 unsigned int arr_size, r_idx; 691 u32 offset, counter; 692 struct pmc *pmc; 693 694 pmc = pmcdev->pmcs[pmcidx]; 695 if (!pmc) 696 continue; 697 maps = pmc->map->s0ix_blocker_maps; 698 offset = pmc->map->s0ix_blocker_offset; 699 arr_size = pmc_core_lpm_get_arr_size(maps); 700 701 for (r_idx = 0; r_idx < arr_size; r_idx++) { 702 const struct pmc_bit_map *map; 703 704 for (map = maps[r_idx]; map->name; map++) { 705 if (!map->blk) 706 continue; 707 counter = pmc_core_reg_read(pmc, offset); 708 seq_printf(s, "PMC%d:%-30s %-30d\n", pmcidx, 709 map->name, counter); 710 offset += map->blk * S0IX_BLK_SIZE; 711 } 712 } 713 } 714 return 0; 715 } 716 DEFINE_SHOW_ATTRIBUTE(pmc_core_s0ix_blocker); 717 718 static void pmc_core_ltr_ignore_all(struct pmc_dev *pmcdev) 719 { 720 unsigned int i; 721 722 for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); i++) { 723 struct pmc *pmc; 724 u32 ltr_ign; 725 726 pmc = pmcdev->pmcs[i]; 727 if (!pmc) 728 continue; 729 730 guard(mutex)(&pmcdev->lock); 731 pmc->ltr_ign = pmc_core_reg_read(pmc, pmc->map->ltr_ignore_offset); 732 733 /* ltr_ignore_max is the max index value for LTR ignore register */ 734 ltr_ign = pmc->ltr_ign | GENMASK(pmc->map->ltr_ignore_max, 0); 735 pmc_core_reg_write(pmc, pmc->map->ltr_ignore_offset, ltr_ign); 736 } 737 738 /* 739 * Ignoring ME during suspend is blocking platforms with ADL PCH to get to 740 * deeper S0ix substate. 741 */ 742 pmc_core_send_ltr_ignore(pmcdev, 6, 0); 743 } 744 745 static void pmc_core_ltr_restore_all(struct pmc_dev *pmcdev) 746 { 747 unsigned int i; 748 749 for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); i++) { 750 struct pmc *pmc; 751 752 pmc = pmcdev->pmcs[i]; 753 if (!pmc) 754 continue; 755 756 guard(mutex)(&pmcdev->lock); 757 pmc_core_reg_write(pmc, pmc->map->ltr_ignore_offset, pmc->ltr_ign); 758 } 759 } 760 761 static inline u64 adjust_lpm_residency(struct pmc *pmc, u32 offset, 762 const int lpm_adj_x2) 763 { 764 u64 lpm_res = pmc_core_reg_read(pmc, offset); 765 766 return GET_X2_COUNTER((u64)lpm_adj_x2 * lpm_res); 767 } 768 769 static int pmc_core_substate_res_show(struct seq_file *s, void *unused) 770 { 771 struct pmc_dev *pmcdev = s->private; 772 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 773 const int lpm_adj_x2 = pmc->map->lpm_res_counter_step_x2; 774 u32 offset = pmc->map->lpm_residency_offset; 775 int mode; 776 777 seq_printf(s, "%-10s %-15s\n", "Substate", "Residency"); 778 779 pmc_for_each_mode(mode, pmcdev) { 780 seq_printf(s, "%-10s %-15llu\n", pmc_lpm_modes[mode], 781 adjust_lpm_residency(pmc, offset + (4 * mode), lpm_adj_x2)); 782 } 783 784 return 0; 785 } 786 DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_res); 787 788 static int pmc_core_substate_sts_regs_show(struct seq_file *s, void *unused) 789 { 790 struct pmc_dev *pmcdev = s->private; 791 unsigned int i; 792 793 for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) { 794 struct pmc *pmc = pmcdev->pmcs[i]; 795 const struct pmc_bit_map **maps; 796 u32 offset; 797 798 if (!pmc) 799 continue; 800 maps = pmc->map->lpm_sts; 801 offset = pmc->map->lpm_status_offset; 802 pmc_core_lpm_display(pmc, NULL, s, offset, i, "STATUS", maps); 803 } 804 805 return 0; 806 } 807 DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_sts_regs); 808 809 static int pmc_core_substate_l_sts_regs_show(struct seq_file *s, void *unused) 810 { 811 struct pmc_dev *pmcdev = s->private; 812 unsigned int i; 813 814 for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) { 815 struct pmc *pmc = pmcdev->pmcs[i]; 816 const struct pmc_bit_map **maps; 817 u32 offset; 818 819 if (!pmc) 820 continue; 821 maps = pmc->map->lpm_sts; 822 offset = pmc->map->lpm_live_status_offset; 823 pmc_core_lpm_display(pmc, NULL, s, offset, i, "LIVE_STATUS", maps); 824 } 825 826 return 0; 827 } 828 DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_l_sts_regs); 829 830 static void pmc_core_substate_req_header_show(struct seq_file *s, int pmc_index) 831 { 832 struct pmc_dev *pmcdev = s->private; 833 int mode; 834 835 seq_printf(s, "%30s |", "Element"); 836 pmc_for_each_mode(mode, pmcdev) 837 seq_printf(s, " %9s |", pmc_lpm_modes[mode]); 838 839 seq_printf(s, " %9s |", "Status"); 840 seq_printf(s, " %11s |\n", "Live Status"); 841 } 842 843 static int pmc_core_substate_req_regs_show(struct seq_file *s, void *unused) 844 { 845 struct pmc_dev *pmcdev = s->private; 846 u32 sts_offset; 847 u32 sts_offset_live; 848 u32 *lpm_req_regs; 849 unsigned int mp, pmc_index; 850 int num_maps; 851 852 for (pmc_index = 0; pmc_index < ARRAY_SIZE(pmcdev->pmcs); ++pmc_index) { 853 struct pmc *pmc = pmcdev->pmcs[pmc_index]; 854 const struct pmc_bit_map **maps; 855 856 if (!pmc) 857 continue; 858 859 maps = pmc->map->lpm_sts; 860 num_maps = pmc->map->lpm_num_maps; 861 sts_offset = pmc->map->lpm_status_offset; 862 sts_offset_live = pmc->map->lpm_live_status_offset; 863 lpm_req_regs = pmc->lpm_req_regs; 864 865 /* 866 * When there are multiple PMCs, though the PMC may exist, the 867 * requirement register discovery could have failed so check 868 * before accessing. 869 */ 870 if (!lpm_req_regs) 871 continue; 872 873 /* Display the header */ 874 pmc_core_substate_req_header_show(s, pmc_index); 875 876 /* Loop over maps */ 877 for (mp = 0; mp < num_maps; mp++) { 878 u32 req_mask = 0; 879 u32 lpm_status; 880 u32 lpm_status_live; 881 const struct pmc_bit_map *map; 882 int mode, i, len = 32; 883 884 /* 885 * Capture the requirements and create a mask so that we only 886 * show an element if it's required for at least one of the 887 * enabled low power modes 888 */ 889 pmc_for_each_mode(mode, pmcdev) 890 req_mask |= lpm_req_regs[mp + (mode * num_maps)]; 891 892 /* Get the last latched status for this map */ 893 lpm_status = pmc_core_reg_read(pmc, sts_offset + (mp * 4)); 894 895 /* Get the runtime status for this map */ 896 lpm_status_live = pmc_core_reg_read(pmc, sts_offset_live + (mp * 4)); 897 898 /* Loop over elements in this map */ 899 map = maps[mp]; 900 for (i = 0; map[i].name && i < len; i++) { 901 u32 bit_mask = map[i].bit_mask; 902 903 if (!(bit_mask & req_mask)) { 904 /* 905 * Not required for any enabled states 906 * so don't display 907 */ 908 continue; 909 } 910 911 /* Display the element name in the first column */ 912 seq_printf(s, "pmc%d: %26s |", pmc_index, map[i].name); 913 914 /* Loop over the enabled states and display if required */ 915 pmc_for_each_mode(mode, pmcdev) { 916 bool required = lpm_req_regs[mp + (mode * num_maps)] & 917 bit_mask; 918 seq_printf(s, " %9s |", required ? "Required" : " "); 919 } 920 921 /* In Status column, show the last captured state of this agent */ 922 seq_printf(s, " %9s |", lpm_status & bit_mask ? "Yes" : " "); 923 924 /* In Live status column, show the live state of this agent */ 925 seq_printf(s, " %11s |", lpm_status_live & bit_mask ? "Yes" : " "); 926 927 seq_puts(s, "\n"); 928 } 929 } 930 } 931 return 0; 932 } 933 DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_req_regs); 934 935 static unsigned int pmc_core_get_crystal_freq(void) 936 { 937 unsigned int eax_denominator, ebx_numerator, ecx_hz, edx; 938 939 if (boot_cpu_data.cpuid_level < 0x15) 940 return 0; 941 942 eax_denominator = ebx_numerator = ecx_hz = edx = 0; 943 944 /* CPUID 15H TSC/Crystal ratio, plus optionally Crystal Hz */ 945 cpuid(0x15, &eax_denominator, &ebx_numerator, &ecx_hz, &edx); 946 947 if (ebx_numerator == 0 || eax_denominator == 0) 948 return 0; 949 950 return ecx_hz; 951 } 952 953 static int pmc_core_die_c6_us_show(struct seq_file *s, void *unused) 954 { 955 struct pmc_dev *pmcdev = s->private; 956 u64 die_c6_res, count; 957 int ret; 958 959 if (!pmcdev->crystal_freq) { 960 dev_warn_once(&pmcdev->pdev->dev, "Crystal frequency unavailable\n"); 961 return -ENXIO; 962 } 963 964 ret = pmt_telem_read(pmcdev->punit_ep, pmcdev->die_c6_offset, 965 &count, 1); 966 if (ret) 967 return ret; 968 969 die_c6_res = div64_u64(count * HZ_PER_MHZ, pmcdev->crystal_freq); 970 seq_printf(s, "%llu\n", die_c6_res); 971 972 return 0; 973 } 974 DEFINE_SHOW_ATTRIBUTE(pmc_core_die_c6_us); 975 976 static int pmc_core_lpm_latch_mode_show(struct seq_file *s, void *unused) 977 { 978 struct pmc_dev *pmcdev = s->private; 979 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 980 bool c10; 981 u32 reg; 982 int mode; 983 984 reg = pmc_core_reg_read(pmc, pmc->map->lpm_sts_latch_en_offset); 985 if (reg & LPM_STS_LATCH_MODE) { 986 seq_puts(s, "c10"); 987 c10 = false; 988 } else { 989 seq_puts(s, "[c10]"); 990 c10 = true; 991 } 992 993 pmc_for_each_mode(mode, pmcdev) { 994 if ((BIT(mode) & reg) && !c10) 995 seq_printf(s, " [%s]", pmc_lpm_modes[mode]); 996 else 997 seq_printf(s, " %s", pmc_lpm_modes[mode]); 998 } 999 1000 seq_puts(s, " clear\n"); 1001 1002 return 0; 1003 } 1004 1005 static ssize_t pmc_core_lpm_latch_mode_write(struct file *file, 1006 const char __user *userbuf, 1007 size_t count, loff_t *ppos) 1008 { 1009 struct seq_file *s = file->private_data; 1010 struct pmc_dev *pmcdev = s->private; 1011 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 1012 bool clear = false, c10 = false; 1013 unsigned char buf[8]; 1014 int m, mode; 1015 u32 reg; 1016 1017 if (count > sizeof(buf) - 1) 1018 return -EINVAL; 1019 if (copy_from_user(buf, userbuf, count)) 1020 return -EFAULT; 1021 buf[count] = '\0'; 1022 1023 /* 1024 * Allowed strings are: 1025 * Any enabled substate, e.g. 'S0i2.0' 1026 * 'c10' 1027 * 'clear' 1028 */ 1029 mode = sysfs_match_string(pmc_lpm_modes, buf); 1030 1031 /* Check string matches enabled mode */ 1032 pmc_for_each_mode(m, pmcdev) 1033 if (mode == m) 1034 break; 1035 1036 if (mode != m || mode < 0) { 1037 if (sysfs_streq(buf, "clear")) 1038 clear = true; 1039 else if (sysfs_streq(buf, "c10")) 1040 c10 = true; 1041 else 1042 return -EINVAL; 1043 } 1044 1045 if (clear) { 1046 guard(mutex)(&pmcdev->lock); 1047 1048 reg = pmc_core_reg_read(pmc, pmc->map->etr3_offset); 1049 reg |= ETR3_CLEAR_LPM_EVENTS; 1050 pmc_core_reg_write(pmc, pmc->map->etr3_offset, reg); 1051 1052 return count; 1053 } 1054 1055 if (c10) { 1056 guard(mutex)(&pmcdev->lock); 1057 1058 reg = pmc_core_reg_read(pmc, pmc->map->lpm_sts_latch_en_offset); 1059 reg &= ~LPM_STS_LATCH_MODE; 1060 pmc_core_reg_write(pmc, pmc->map->lpm_sts_latch_en_offset, reg); 1061 1062 return count; 1063 } 1064 1065 /* 1066 * For LPM mode latching we set the latch enable bit and selected mode 1067 * and clear everything else. 1068 */ 1069 reg = LPM_STS_LATCH_MODE | BIT(mode); 1070 guard(mutex)(&pmcdev->lock); 1071 pmc_core_reg_write(pmc, pmc->map->lpm_sts_latch_en_offset, reg); 1072 1073 return count; 1074 } 1075 DEFINE_PMC_CORE_ATTR_WRITE(pmc_core_lpm_latch_mode); 1076 1077 static int pmc_core_pkgc_show(struct seq_file *s, void *unused) 1078 { 1079 struct pmc *pmc = s->private; 1080 const struct pmc_bit_map *map = pmc->map->msr_sts; 1081 u64 pcstate_count; 1082 unsigned int index; 1083 1084 for (index = 0; map[index].name ; index++) { 1085 if (rdmsrl_safe(map[index].bit_mask, &pcstate_count)) 1086 continue; 1087 1088 pcstate_count *= 1000; 1089 do_div(pcstate_count, tsc_khz); 1090 seq_printf(s, "%-8s : %llu\n", map[index].name, 1091 pcstate_count); 1092 } 1093 1094 return 0; 1095 } 1096 DEFINE_SHOW_ATTRIBUTE(pmc_core_pkgc); 1097 1098 static bool pmc_core_pri_verify(u32 lpm_pri, u8 *mode_order) 1099 { 1100 unsigned int i, j; 1101 1102 if (!lpm_pri) 1103 return false; 1104 /* 1105 * Each byte contains the priority level for 2 modes (7:4 and 3:0). 1106 * In a 32 bit register this allows for describing 8 modes. Store the 1107 * levels and look for values out of range. 1108 */ 1109 for (i = 0; i < 8; i++) { 1110 int level = lpm_pri & GENMASK(3, 0); 1111 1112 if (level >= LPM_MAX_NUM_MODES) 1113 return false; 1114 1115 mode_order[i] = level; 1116 lpm_pri >>= 4; 1117 } 1118 1119 /* Check that we have unique values */ 1120 for (i = 0; i < LPM_MAX_NUM_MODES - 1; i++) 1121 for (j = i + 1; j < LPM_MAX_NUM_MODES; j++) 1122 if (mode_order[i] == mode_order[j]) 1123 return false; 1124 1125 return true; 1126 } 1127 1128 void pmc_core_get_low_power_modes(struct pmc_dev *pmcdev) 1129 { 1130 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 1131 u8 pri_order[LPM_MAX_NUM_MODES] = LPM_DEFAULT_PRI; 1132 u8 mode_order[LPM_MAX_NUM_MODES]; 1133 u32 lpm_pri; 1134 u32 lpm_en; 1135 unsigned int i; 1136 int mode, p; 1137 1138 /* Use LPM Maps to indicate support for substates */ 1139 if (!pmc->map->lpm_num_maps) 1140 return; 1141 1142 lpm_en = pmc_core_reg_read(pmc, pmc->map->lpm_en_offset); 1143 /* For MTL, BIT 31 is not an lpm mode but a enable bit. 1144 * Lower byte is enough to cover the number of lpm modes for all 1145 * platforms and hence mask the upper 3 bytes. 1146 */ 1147 pmcdev->num_lpm_modes = hweight32(lpm_en & 0xFF); 1148 1149 /* Read 32 bit LPM_PRI register */ 1150 lpm_pri = pmc_core_reg_read(pmc, pmc->map->lpm_priority_offset); 1151 1152 1153 /* 1154 * If lpm_pri value passes verification, then override the default 1155 * modes here. Otherwise stick with the default. 1156 */ 1157 if (pmc_core_pri_verify(lpm_pri, mode_order)) 1158 /* Get list of modes in priority order */ 1159 for (mode = 0; mode < LPM_MAX_NUM_MODES; mode++) 1160 pri_order[mode_order[mode]] = mode; 1161 else 1162 dev_warn(&pmcdev->pdev->dev, 1163 "Assuming a default substate order for this platform\n"); 1164 1165 /* 1166 * Loop through all modes from lowest to highest priority, 1167 * and capture all enabled modes in order 1168 */ 1169 i = 0; 1170 for (p = LPM_MAX_NUM_MODES - 1; p >= 0; p--) { 1171 int mode = pri_order[p]; 1172 1173 if (!(BIT(mode) & lpm_en)) 1174 continue; 1175 1176 pmcdev->lpm_en_modes[i++] = mode; 1177 } 1178 } 1179 1180 int get_primary_reg_base(struct pmc *pmc) 1181 { 1182 u64 slp_s0_addr; 1183 1184 if (lpit_read_residency_count_address(&slp_s0_addr)) { 1185 pmc->base_addr = PMC_BASE_ADDR_DEFAULT; 1186 1187 if (page_is_ram(PHYS_PFN(pmc->base_addr))) 1188 return -ENODEV; 1189 } else { 1190 pmc->base_addr = slp_s0_addr - pmc->map->slp_s0_offset; 1191 } 1192 1193 pmc->regbase = ioremap(pmc->base_addr, pmc->map->regmap_length); 1194 if (!pmc->regbase) 1195 return -ENOMEM; 1196 return 0; 1197 } 1198 1199 void pmc_core_punit_pmt_init(struct pmc_dev *pmcdev, u32 guid) 1200 { 1201 struct telem_endpoint *ep; 1202 struct pci_dev *pcidev; 1203 1204 pcidev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(10, 0)); 1205 if (!pcidev) { 1206 dev_err(&pmcdev->pdev->dev, "PUNIT PMT device not found."); 1207 return; 1208 } 1209 1210 ep = pmt_telem_find_and_register_endpoint(pcidev, guid, 0); 1211 pci_dev_put(pcidev); 1212 if (IS_ERR(ep)) { 1213 dev_err(&pmcdev->pdev->dev, 1214 "pmc_core: couldn't get DMU telem endpoint %ld", 1215 PTR_ERR(ep)); 1216 return; 1217 } 1218 1219 pmcdev->punit_ep = ep; 1220 1221 pmcdev->has_die_c6 = true; 1222 pmcdev->die_c6_offset = MTL_PMT_DMU_DIE_C6_OFFSET; 1223 } 1224 1225 void pmc_core_set_device_d3(unsigned int device) 1226 { 1227 struct pci_dev *pcidev; 1228 1229 pcidev = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL); 1230 if (pcidev) { 1231 if (!device_trylock(&pcidev->dev)) { 1232 pci_dev_put(pcidev); 1233 return; 1234 } 1235 if (!pcidev->dev.driver) { 1236 dev_info(&pcidev->dev, "Setting to D3hot\n"); 1237 pci_set_power_state(pcidev, PCI_D3hot); 1238 } 1239 device_unlock(&pcidev->dev); 1240 pci_dev_put(pcidev); 1241 } 1242 } 1243 1244 static bool pmc_core_is_pson_residency_enabled(struct pmc_dev *pmcdev) 1245 { 1246 struct platform_device *pdev = pmcdev->pdev; 1247 struct acpi_device *adev = ACPI_COMPANION(&pdev->dev); 1248 u8 val; 1249 1250 if (!adev) 1251 return false; 1252 1253 if (fwnode_property_read_u8(acpi_fwnode_handle(adev), 1254 "intel-cec-pson-switching-enabled-in-s0", 1255 &val)) 1256 return false; 1257 1258 return val == 1; 1259 } 1260 1261 /* 1262 * Enable or disable ACPI PM Timer 1263 * 1264 * This function is intended to be a callback for ACPI PM suspend/resume event. 1265 * The ACPI PM Timer is enabled on resume only if it was enabled during suspend. 1266 */ 1267 static void pmc_core_acpi_pm_timer_suspend_resume(void *data, bool suspend) 1268 { 1269 struct pmc_dev *pmcdev = data; 1270 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 1271 const struct pmc_reg_map *map = pmc->map; 1272 bool enabled; 1273 u32 reg; 1274 1275 if (!map->acpi_pm_tmr_ctl_offset) 1276 return; 1277 1278 guard(mutex)(&pmcdev->lock); 1279 1280 if (!suspend && !pmcdev->enable_acpi_pm_timer_on_resume) 1281 return; 1282 1283 reg = pmc_core_reg_read(pmc, map->acpi_pm_tmr_ctl_offset); 1284 enabled = !(reg & map->acpi_pm_tmr_disable_bit); 1285 if (suspend) 1286 reg |= map->acpi_pm_tmr_disable_bit; 1287 else 1288 reg &= ~map->acpi_pm_tmr_disable_bit; 1289 pmc_core_reg_write(pmc, map->acpi_pm_tmr_ctl_offset, reg); 1290 1291 pmcdev->enable_acpi_pm_timer_on_resume = suspend && enabled; 1292 } 1293 1294 static void pmc_core_dbgfs_unregister(struct pmc_dev *pmcdev) 1295 { 1296 debugfs_remove_recursive(pmcdev->dbgfs_dir); 1297 } 1298 1299 static void pmc_core_dbgfs_register(struct pmc_dev *pmcdev) 1300 { 1301 struct pmc *primary_pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 1302 struct dentry *dir; 1303 1304 dir = debugfs_create_dir("pmc_core", NULL); 1305 pmcdev->dbgfs_dir = dir; 1306 1307 debugfs_create_file("slp_s0_residency_usec", 0444, dir, primary_pmc, 1308 &pmc_core_dev_state); 1309 1310 if (primary_pmc->map->pfear_sts) 1311 debugfs_create_file("pch_ip_power_gating_status", 0444, dir, 1312 pmcdev, &pmc_core_ppfear_fops); 1313 1314 debugfs_create_file("ltr_ignore", 0644, dir, pmcdev, 1315 &pmc_core_ltr_ignore_fops); 1316 1317 debugfs_create_file("ltr_restore", 0200, dir, pmcdev, &pmc_core_ltr_restore_fops); 1318 1319 debugfs_create_file("ltr_show", 0444, dir, pmcdev, &pmc_core_ltr_fops); 1320 1321 if (primary_pmc->map->s0ix_blocker_maps) 1322 debugfs_create_file("s0ix_blocker", 0444, dir, pmcdev, &pmc_core_s0ix_blocker_fops); 1323 1324 debugfs_create_file("package_cstate_show", 0444, dir, primary_pmc, 1325 &pmc_core_pkgc_fops); 1326 1327 if (primary_pmc->map->pll_sts) 1328 debugfs_create_file("pll_status", 0444, dir, pmcdev, 1329 &pmc_core_pll_fops); 1330 1331 if (primary_pmc->map->mphy_sts) 1332 debugfs_create_file("mphy_core_lanes_power_gating_status", 1333 0444, dir, pmcdev, 1334 &pmc_core_mphy_pg_fops); 1335 1336 if (primary_pmc->map->slps0_dbg_maps) { 1337 debugfs_create_file("slp_s0_debug_status", 0444, 1338 dir, pmcdev, 1339 &pmc_core_slps0_dbg_fops); 1340 1341 debugfs_create_bool("slp_s0_dbg_latch", 0644, 1342 dir, &slps0_dbg_latch); 1343 } 1344 1345 if (primary_pmc->map->lpm_en_offset) { 1346 debugfs_create_file("substate_residencies", 0444, 1347 pmcdev->dbgfs_dir, pmcdev, 1348 &pmc_core_substate_res_fops); 1349 } 1350 1351 if (primary_pmc->map->lpm_status_offset) { 1352 debugfs_create_file("substate_status_registers", 0444, 1353 pmcdev->dbgfs_dir, pmcdev, 1354 &pmc_core_substate_sts_regs_fops); 1355 debugfs_create_file("substate_live_status_registers", 0444, 1356 pmcdev->dbgfs_dir, pmcdev, 1357 &pmc_core_substate_l_sts_regs_fops); 1358 debugfs_create_file("lpm_latch_mode", 0644, 1359 pmcdev->dbgfs_dir, pmcdev, 1360 &pmc_core_lpm_latch_mode_fops); 1361 } 1362 1363 if (primary_pmc->lpm_req_regs) { 1364 debugfs_create_file("substate_requirements", 0444, 1365 pmcdev->dbgfs_dir, pmcdev, 1366 &pmc_core_substate_req_regs_fops); 1367 } 1368 1369 if (primary_pmc->map->pson_residency_offset && pmc_core_is_pson_residency_enabled(pmcdev)) { 1370 debugfs_create_file("pson_residency_usec", 0444, 1371 pmcdev->dbgfs_dir, primary_pmc, &pmc_core_pson_residency); 1372 } 1373 1374 if (pmcdev->has_die_c6) { 1375 debugfs_create_file("die_c6_us_show", 0444, 1376 pmcdev->dbgfs_dir, pmcdev, 1377 &pmc_core_die_c6_us_fops); 1378 } 1379 } 1380 1381 static const struct x86_cpu_id intel_pmc_core_ids[] = { 1382 X86_MATCH_VFM(INTEL_SKYLAKE_L, spt_core_init), 1383 X86_MATCH_VFM(INTEL_SKYLAKE, spt_core_init), 1384 X86_MATCH_VFM(INTEL_KABYLAKE_L, spt_core_init), 1385 X86_MATCH_VFM(INTEL_KABYLAKE, spt_core_init), 1386 X86_MATCH_VFM(INTEL_CANNONLAKE_L, cnp_core_init), 1387 X86_MATCH_VFM(INTEL_ICELAKE_L, icl_core_init), 1388 X86_MATCH_VFM(INTEL_ICELAKE_NNPI, icl_core_init), 1389 X86_MATCH_VFM(INTEL_COMETLAKE, cnp_core_init), 1390 X86_MATCH_VFM(INTEL_COMETLAKE_L, cnp_core_init), 1391 X86_MATCH_VFM(INTEL_TIGERLAKE_L, tgl_l_core_init), 1392 X86_MATCH_VFM(INTEL_TIGERLAKE, tgl_core_init), 1393 X86_MATCH_VFM(INTEL_ATOM_TREMONT, tgl_l_core_init), 1394 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, icl_core_init), 1395 X86_MATCH_VFM(INTEL_ROCKETLAKE, tgl_core_init), 1396 X86_MATCH_VFM(INTEL_ALDERLAKE_L, tgl_l_core_init), 1397 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, tgl_l_core_init), 1398 X86_MATCH_VFM(INTEL_ALDERLAKE, adl_core_init), 1399 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, tgl_l_core_init), 1400 X86_MATCH_VFM(INTEL_RAPTORLAKE, adl_core_init), 1401 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, adl_core_init), 1402 X86_MATCH_VFM(INTEL_METEORLAKE_L, mtl_core_init), 1403 X86_MATCH_VFM(INTEL_ARROWLAKE, arl_core_init), 1404 X86_MATCH_VFM(INTEL_LUNARLAKE_M, lnl_core_init), 1405 {} 1406 }; 1407 1408 MODULE_DEVICE_TABLE(x86cpu, intel_pmc_core_ids); 1409 1410 static const struct pci_device_id pmc_pci_ids[] = { 1411 { PCI_VDEVICE(INTEL, SPT_PMC_PCI_DEVICE_ID) }, 1412 { } 1413 }; 1414 1415 /* 1416 * This quirk can be used on those platforms where 1417 * the platform BIOS enforces 24Mhz crystal to shutdown 1418 * before PMC can assert SLP_S0#. 1419 */ 1420 static bool xtal_ignore; 1421 static int quirk_xtal_ignore(const struct dmi_system_id *id) 1422 { 1423 xtal_ignore = true; 1424 return 0; 1425 } 1426 1427 static void pmc_core_xtal_ignore(struct pmc *pmc) 1428 { 1429 u32 value; 1430 1431 value = pmc_core_reg_read(pmc, pmc->map->pm_vric1_offset); 1432 /* 24MHz Crystal Shutdown Qualification Disable */ 1433 value |= SPT_PMC_VRIC1_XTALSDQDIS; 1434 /* Low Voltage Mode Enable */ 1435 value &= ~SPT_PMC_VRIC1_SLPS0LVEN; 1436 pmc_core_reg_write(pmc, pmc->map->pm_vric1_offset, value); 1437 } 1438 1439 static const struct dmi_system_id pmc_core_dmi_table[] = { 1440 { 1441 .callback = quirk_xtal_ignore, 1442 .ident = "HP Elite x2 1013 G3", 1443 .matches = { 1444 DMI_MATCH(DMI_SYS_VENDOR, "HP"), 1445 DMI_MATCH(DMI_PRODUCT_NAME, "HP Elite x2 1013 G3"), 1446 }, 1447 }, 1448 {} 1449 }; 1450 1451 static void pmc_core_do_dmi_quirks(struct pmc *pmc) 1452 { 1453 dmi_check_system(pmc_core_dmi_table); 1454 1455 if (xtal_ignore) 1456 pmc_core_xtal_ignore(pmc); 1457 } 1458 1459 static void pmc_core_clean_structure(struct platform_device *pdev) 1460 { 1461 struct pmc_dev *pmcdev = platform_get_drvdata(pdev); 1462 unsigned int i; 1463 1464 for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) { 1465 struct pmc *pmc = pmcdev->pmcs[i]; 1466 1467 if (pmc) 1468 iounmap(pmc->regbase); 1469 } 1470 1471 if (pmcdev->ssram_pcidev) { 1472 pci_dev_put(pmcdev->ssram_pcidev); 1473 pci_disable_device(pmcdev->ssram_pcidev); 1474 } 1475 1476 if (pmcdev->punit_ep) 1477 pmt_telem_unregister_endpoint(pmcdev->punit_ep); 1478 1479 platform_set_drvdata(pdev, NULL); 1480 mutex_destroy(&pmcdev->lock); 1481 } 1482 1483 static int pmc_core_probe(struct platform_device *pdev) 1484 { 1485 static bool device_initialized; 1486 struct pmc_dev *pmcdev; 1487 const struct x86_cpu_id *cpu_id; 1488 int (*core_init)(struct pmc_dev *pmcdev); 1489 const struct pmc_reg_map *map; 1490 struct pmc *primary_pmc; 1491 int ret; 1492 1493 if (device_initialized) 1494 return -ENODEV; 1495 1496 pmcdev = devm_kzalloc(&pdev->dev, sizeof(*pmcdev), GFP_KERNEL); 1497 if (!pmcdev) 1498 return -ENOMEM; 1499 1500 pmcdev->crystal_freq = pmc_core_get_crystal_freq(); 1501 1502 platform_set_drvdata(pdev, pmcdev); 1503 pmcdev->pdev = pdev; 1504 1505 cpu_id = x86_match_cpu(intel_pmc_core_ids); 1506 if (!cpu_id) 1507 return -ENODEV; 1508 1509 core_init = (int (*)(struct pmc_dev *))cpu_id->driver_data; 1510 1511 /* Primary PMC */ 1512 primary_pmc = devm_kzalloc(&pdev->dev, sizeof(*primary_pmc), GFP_KERNEL); 1513 if (!primary_pmc) 1514 return -ENOMEM; 1515 pmcdev->pmcs[PMC_IDX_MAIN] = primary_pmc; 1516 1517 /* The last element in msr_map is empty */ 1518 pmcdev->num_of_pkgc = ARRAY_SIZE(msr_map) - 1; 1519 pmcdev->pkgc_res_cnt = devm_kcalloc(&pdev->dev, 1520 pmcdev->num_of_pkgc, 1521 sizeof(*pmcdev->pkgc_res_cnt), 1522 GFP_KERNEL); 1523 if (!pmcdev->pkgc_res_cnt) 1524 return -ENOMEM; 1525 1526 /* 1527 * Coffee Lake has CPU ID of Kaby Lake and Cannon Lake PCH. So here 1528 * Sunrisepoint PCH regmap can't be used. Use Cannon Lake PCH regmap 1529 * in this case. 1530 */ 1531 if (core_init == spt_core_init && !pci_dev_present(pmc_pci_ids)) 1532 core_init = cnp_core_init; 1533 1534 mutex_init(&pmcdev->lock); 1535 ret = core_init(pmcdev); 1536 if (ret) { 1537 pmc_core_clean_structure(pdev); 1538 return ret; 1539 } 1540 1541 pmcdev->pmc_xram_read_bit = pmc_core_check_read_lock_bit(primary_pmc); 1542 pmc_core_do_dmi_quirks(primary_pmc); 1543 1544 pmc_core_dbgfs_register(pmcdev); 1545 pm_report_max_hw_sleep(FIELD_MAX(SLP_S0_RES_COUNTER_MASK) * 1546 pmc_core_adjust_slp_s0_step(primary_pmc, 1)); 1547 1548 map = primary_pmc->map; 1549 if (map->acpi_pm_tmr_ctl_offset) 1550 acpi_pmtmr_register_suspend_resume_callback(pmc_core_acpi_pm_timer_suspend_resume, 1551 pmcdev); 1552 1553 device_initialized = true; 1554 dev_info(&pdev->dev, " initialized\n"); 1555 1556 return 0; 1557 } 1558 1559 static void pmc_core_remove(struct platform_device *pdev) 1560 { 1561 struct pmc_dev *pmcdev = platform_get_drvdata(pdev); 1562 const struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 1563 const struct pmc_reg_map *map = pmc->map; 1564 1565 if (map->acpi_pm_tmr_ctl_offset) 1566 acpi_pmtmr_unregister_suspend_resume_callback(); 1567 1568 pmc_core_dbgfs_unregister(pmcdev); 1569 pmc_core_clean_structure(pdev); 1570 } 1571 1572 static bool warn_on_s0ix_failures; 1573 module_param(warn_on_s0ix_failures, bool, 0644); 1574 MODULE_PARM_DESC(warn_on_s0ix_failures, "Check and warn for S0ix failures"); 1575 1576 static bool ltr_ignore_all_suspend = true; 1577 module_param(ltr_ignore_all_suspend, bool, 0644); 1578 MODULE_PARM_DESC(ltr_ignore_all_suspend, "Ignore all LTRs during suspend"); 1579 1580 static __maybe_unused int pmc_core_suspend(struct device *dev) 1581 { 1582 struct pmc_dev *pmcdev = dev_get_drvdata(dev); 1583 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 1584 unsigned int i; 1585 1586 if (pmcdev->suspend) 1587 pmcdev->suspend(pmcdev); 1588 1589 if (ltr_ignore_all_suspend) 1590 pmc_core_ltr_ignore_all(pmcdev); 1591 1592 /* Check if the syspend will actually use S0ix */ 1593 if (pm_suspend_via_firmware()) 1594 return 0; 1595 1596 /* Save PKGC residency for checking later */ 1597 for (i = 0; i < pmcdev->num_of_pkgc; i++) { 1598 if (rdmsrl_safe(msr_map[i].bit_mask, &pmcdev->pkgc_res_cnt[i])) 1599 return -EIO; 1600 } 1601 1602 /* Save S0ix residency for checking later */ 1603 if (pmc_core_dev_state_get(pmc, &pmcdev->s0ix_counter)) 1604 return -EIO; 1605 1606 return 0; 1607 } 1608 1609 static inline bool pmc_core_is_deepest_pkgc_failed(struct pmc_dev *pmcdev) 1610 { 1611 u32 deepest_pkgc_msr = msr_map[pmcdev->num_of_pkgc - 1].bit_mask; 1612 u64 deepest_pkgc_residency; 1613 1614 if (rdmsrl_safe(deepest_pkgc_msr, &deepest_pkgc_residency)) 1615 return false; 1616 1617 if (deepest_pkgc_residency == pmcdev->pkgc_res_cnt[pmcdev->num_of_pkgc - 1]) 1618 return true; 1619 1620 return false; 1621 } 1622 1623 static inline bool pmc_core_is_s0ix_failed(struct pmc_dev *pmcdev) 1624 { 1625 u64 s0ix_counter; 1626 1627 if (pmc_core_dev_state_get(pmcdev->pmcs[PMC_IDX_MAIN], &s0ix_counter)) 1628 return false; 1629 1630 pm_report_hw_sleep_time((u32)(s0ix_counter - pmcdev->s0ix_counter)); 1631 1632 if (s0ix_counter == pmcdev->s0ix_counter) 1633 return true; 1634 1635 return false; 1636 } 1637 1638 int pmc_core_resume_common(struct pmc_dev *pmcdev) 1639 { 1640 struct device *dev = &pmcdev->pdev->dev; 1641 struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN]; 1642 const struct pmc_bit_map **maps = pmc->map->lpm_sts; 1643 int offset = pmc->map->lpm_status_offset; 1644 unsigned int i; 1645 1646 /* Check if the syspend used S0ix */ 1647 if (pm_suspend_via_firmware()) 1648 return 0; 1649 1650 if (!pmc_core_is_s0ix_failed(pmcdev)) 1651 return 0; 1652 1653 if (!warn_on_s0ix_failures) 1654 return 0; 1655 1656 if (pmc_core_is_deepest_pkgc_failed(pmcdev)) { 1657 /* S0ix failed because of deepest PKGC entry failure */ 1658 dev_info(dev, "CPU did not enter %s!!! (%s cnt=0x%llx)\n", 1659 msr_map[pmcdev->num_of_pkgc - 1].name, 1660 msr_map[pmcdev->num_of_pkgc - 1].name, 1661 pmcdev->pkgc_res_cnt[pmcdev->num_of_pkgc - 1]); 1662 1663 for (i = 0; i < pmcdev->num_of_pkgc; i++) { 1664 u64 pc_cnt; 1665 1666 if (!rdmsrl_safe(msr_map[i].bit_mask, &pc_cnt)) { 1667 dev_info(dev, "Prev %s cnt = 0x%llx, Current %s cnt = 0x%llx\n", 1668 msr_map[i].name, pmcdev->pkgc_res_cnt[i], 1669 msr_map[i].name, pc_cnt); 1670 } 1671 } 1672 return 0; 1673 } 1674 1675 /* The real interesting case - S0ix failed - lets ask PMC why. */ 1676 dev_warn(dev, "CPU did not enter SLP_S0!!! (S0ix cnt=%llu)\n", 1677 pmcdev->s0ix_counter); 1678 1679 if (pmc->map->slps0_dbg_maps) 1680 pmc_core_slps0_display(pmc, dev, NULL); 1681 1682 for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) { 1683 struct pmc *pmc = pmcdev->pmcs[i]; 1684 1685 if (!pmc) 1686 continue; 1687 if (pmc->map->lpm_sts) 1688 pmc_core_lpm_display(pmc, dev, NULL, offset, i, "STATUS", maps); 1689 } 1690 1691 return 0; 1692 } 1693 1694 static __maybe_unused int pmc_core_resume(struct device *dev) 1695 { 1696 struct pmc_dev *pmcdev = dev_get_drvdata(dev); 1697 1698 if (ltr_ignore_all_suspend) 1699 pmc_core_ltr_restore_all(pmcdev); 1700 1701 if (pmcdev->resume) 1702 return pmcdev->resume(pmcdev); 1703 1704 return pmc_core_resume_common(pmcdev); 1705 } 1706 1707 static const struct dev_pm_ops pmc_core_pm_ops = { 1708 SET_LATE_SYSTEM_SLEEP_PM_OPS(pmc_core_suspend, pmc_core_resume) 1709 }; 1710 1711 static const struct acpi_device_id pmc_core_acpi_ids[] = { 1712 {"INT33A1", 0}, /* _HID for Intel Power Engine, _CID PNP0D80*/ 1713 { } 1714 }; 1715 MODULE_DEVICE_TABLE(acpi, pmc_core_acpi_ids); 1716 1717 static struct platform_driver pmc_core_driver = { 1718 .driver = { 1719 .name = "intel_pmc_core", 1720 .acpi_match_table = ACPI_PTR(pmc_core_acpi_ids), 1721 .pm = &pmc_core_pm_ops, 1722 .dev_groups = pmc_dev_groups, 1723 }, 1724 .probe = pmc_core_probe, 1725 .remove_new = pmc_core_remove, 1726 }; 1727 1728 module_platform_driver(pmc_core_driver); 1729 1730 MODULE_LICENSE("GPL v2"); 1731 MODULE_DESCRIPTION("Intel PMC Core Driver"); 1732