1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Common code for Intel Running Average Power Limit (RAPL) support. 4 * Copyright (c) 2019, Intel Corporation. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/bitmap.h> 9 #include <linux/cleanup.h> 10 #include <linux/cpu.h> 11 #include <linux/delay.h> 12 #include <linux/device.h> 13 #include <linux/intel_rapl.h> 14 #include <linux/kernel.h> 15 #include <linux/list.h> 16 #include <linux/log2.h> 17 #include <linux/module.h> 18 #include <linux/nospec.h> 19 #include <linux/perf_event.h> 20 #include <linux/platform_device.h> 21 #include <linux/powercap.h> 22 #include <linux/processor.h> 23 #include <linux/slab.h> 24 #include <linux/suspend.h> 25 #include <linux/sysfs.h> 26 #include <linux/types.h> 27 #include <linux/units.h> 28 29 #include <asm/cpu_device_id.h> 30 #include <asm/intel-family.h> 31 #include <asm/msr.h> 32 33 #define ENERGY_STATUS_MASK GENMASK(31, 0) 34 35 /* Width of the RAPL energy counters, see the *_ENERGY_STATUS_MASK defines */ 36 #define RAPL_CNTR_WIDTH 32 37 38 #define POWER_UNIT_OFFSET 0x00 39 #define POWER_UNIT_MASK GENMASK(3, 0) 40 41 #define ENERGY_UNIT_OFFSET 0x08 42 #define ENERGY_UNIT_MASK GENMASK(12, 8) 43 44 #define TIME_UNIT_OFFSET 0x10 45 #define TIME_UNIT_MASK GENMASK(19, 16) 46 47 /* Non HW constants */ 48 #define RAPL_PRIMITIVE_DUMMY BIT(2) 49 50 #define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */ 51 52 /* per domain data, some are optional */ 53 #define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2) 54 55 #define PACKAGE_PLN_INT_SAVED BIT(0) 56 57 #define RAPL_EVENT_MASK GENMASK(7, 0) 58 59 static const char *pl_names[NR_POWER_LIMITS] = { 60 [POWER_LIMIT1] = "long_term", 61 [POWER_LIMIT2] = "short_term", 62 [POWER_LIMIT4] = "peak_power", 63 }; 64 65 enum pl_prims { 66 PL_ENABLE, 67 PL_CLAMP, 68 PL_LIMIT, 69 PL_TIME_WINDOW, 70 PL_MAX_POWER, 71 PL_LOCK, 72 }; 73 74 static bool is_pl_valid(struct rapl_domain *rd, int pl) 75 { 76 if (pl < POWER_LIMIT1 || pl > POWER_LIMIT4) 77 return false; 78 return rd->rpl[pl].name ? true : false; 79 } 80 81 static int get_pl_lock_prim(struct rapl_domain *rd, int pl) 82 { 83 if (rd->rp->priv->type == RAPL_IF_TPMI) { 84 if (pl == POWER_LIMIT1) 85 return PL1_LOCK; 86 if (pl == POWER_LIMIT2) 87 return PL2_LOCK; 88 if (pl == POWER_LIMIT4) 89 return PL4_LOCK; 90 } 91 92 /* MSR/MMIO Interface doesn't have Lock bit for PL4 */ 93 if (pl == POWER_LIMIT4) 94 return -EINVAL; 95 96 /* 97 * Power Limit register that supports two power limits has a different 98 * bit position for the Lock bit. 99 */ 100 if (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2)) 101 return FW_HIGH_LOCK; 102 return FW_LOCK; 103 } 104 105 static int get_pl_prim(struct rapl_domain *rd, int pl, enum pl_prims prim) 106 { 107 switch (pl) { 108 case POWER_LIMIT1: 109 if (prim == PL_ENABLE) 110 return PL1_ENABLE; 111 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI) 112 return PL1_CLAMP; 113 if (prim == PL_LIMIT) 114 return POWER_LIMIT1; 115 if (prim == PL_TIME_WINDOW) 116 return TIME_WINDOW1; 117 if (prim == PL_MAX_POWER) 118 return THERMAL_SPEC_POWER; 119 if (prim == PL_LOCK) 120 return get_pl_lock_prim(rd, pl); 121 return -EINVAL; 122 case POWER_LIMIT2: 123 if (prim == PL_ENABLE) 124 return PL2_ENABLE; 125 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI) 126 return PL2_CLAMP; 127 if (prim == PL_LIMIT) 128 return POWER_LIMIT2; 129 if (prim == PL_TIME_WINDOW) 130 return TIME_WINDOW2; 131 if (prim == PL_MAX_POWER) 132 return MAX_POWER; 133 if (prim == PL_LOCK) 134 return get_pl_lock_prim(rd, pl); 135 return -EINVAL; 136 case POWER_LIMIT4: 137 if (prim == PL_LIMIT) 138 return POWER_LIMIT4; 139 if (prim == PL_ENABLE) 140 return PL4_ENABLE; 141 /* PL4 would be around two times PL2, use same prim as PL2. */ 142 if (prim == PL_MAX_POWER) 143 return MAX_POWER; 144 if (prim == PL_LOCK) 145 return get_pl_lock_prim(rd, pl); 146 return -EINVAL; 147 default: 148 return -EINVAL; 149 } 150 } 151 152 #define power_zone_to_rapl_domain(_zone) \ 153 container_of(_zone, struct rapl_domain, power_zone) 154 155 static const struct rapl_defaults *get_defaults(struct rapl_package *rp) 156 { 157 return rp->priv->defaults; 158 } 159 160 static void rapl_init_domains(struct rapl_package *rp); 161 static int rapl_read_data_raw(struct rapl_domain *rd, 162 enum rapl_primitives prim, 163 bool xlate, u64 *data, 164 bool pmu_ctx); 165 static int rapl_write_data_raw(struct rapl_domain *rd, 166 enum rapl_primitives prim, 167 unsigned long long value); 168 static int rapl_read_pl_data(struct rapl_domain *rd, int pl, 169 enum pl_prims pl_prim, 170 bool xlate, u64 *data); 171 static int rapl_write_pl_data(struct rapl_domain *rd, int pl, 172 enum pl_prims pl_prim, 173 unsigned long long value); 174 static u64 rapl_unit_xlate(struct rapl_domain *rd, 175 enum unit_type type, u64 value, int to_raw); 176 static void package_power_limit_irq_save(struct rapl_package *rp); 177 178 static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */ 179 180 static const char *const rapl_domain_names[] = { 181 "package", 182 "core", 183 "uncore", 184 "dram", 185 "psys", 186 }; 187 188 static int get_energy_counter(struct powercap_zone *power_zone, 189 u64 *energy_raw) 190 { 191 struct rapl_domain *rd; 192 u64 energy_now; 193 194 /* prevent CPU hotplug, make sure the RAPL domain does not go 195 * away while reading the counter. 196 */ 197 cpus_read_lock(); 198 rd = power_zone_to_rapl_domain(power_zone); 199 200 if (!rapl_read_data_raw(rd, ENERGY_COUNTER, true, &energy_now, false)) { 201 *energy_raw = energy_now; 202 cpus_read_unlock(); 203 204 return 0; 205 } 206 cpus_read_unlock(); 207 208 return -EIO; 209 } 210 211 static int get_max_energy_counter(struct powercap_zone *pcd_dev, u64 *energy) 212 { 213 struct rapl_domain *rd = power_zone_to_rapl_domain(pcd_dev); 214 215 *energy = rapl_unit_xlate(rd, ENERGY_UNIT, ENERGY_STATUS_MASK, 0); 216 return 0; 217 } 218 219 static int release_zone(struct powercap_zone *power_zone) 220 { 221 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone); 222 struct rapl_package *rp = rd->rp; 223 224 /* package zone is the last zone of a package, we can free 225 * memory here since all children has been unregistered. 226 */ 227 if (rd->id == RAPL_DOMAIN_PACKAGE) { 228 kfree(rd); 229 rp->domains = NULL; 230 } 231 232 return 0; 233 234 } 235 236 static int find_nr_power_limit(struct rapl_domain *rd) 237 { 238 int i, nr_pl = 0; 239 240 for (i = 0; i < NR_POWER_LIMITS; i++) { 241 if (is_pl_valid(rd, i)) 242 nr_pl++; 243 } 244 245 return nr_pl; 246 } 247 248 static int set_domain_enable(struct powercap_zone *power_zone, bool mode) 249 { 250 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone); 251 const struct rapl_defaults *defaults = get_defaults(rd->rp); 252 u64 val; 253 int ret; 254 255 cpus_read_lock(); 256 ret = rapl_write_pl_data(rd, POWER_LIMIT1, PL_ENABLE, mode); 257 if (ret) 258 goto end; 259 260 ret = rapl_read_pl_data(rd, POWER_LIMIT1, PL_ENABLE, false, &val); 261 if (ret) 262 goto end; 263 264 if (mode != val) { 265 pr_debug("%s cannot be %s\n", power_zone->name, 266 str_enabled_disabled(mode)); 267 goto end; 268 } 269 270 if (defaults->set_floor_freq) 271 defaults->set_floor_freq(rd, mode); 272 273 end: 274 cpus_read_unlock(); 275 276 return ret; 277 } 278 279 static int get_domain_enable(struct powercap_zone *power_zone, bool *mode) 280 { 281 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone); 282 u64 val; 283 int ret; 284 285 if (rd->rpl[POWER_LIMIT1].locked) { 286 *mode = false; 287 return 0; 288 } 289 cpus_read_lock(); 290 ret = rapl_read_pl_data(rd, POWER_LIMIT1, PL_ENABLE, true, &val); 291 if (!ret) 292 *mode = val; 293 cpus_read_unlock(); 294 295 return ret; 296 } 297 298 /* per RAPL domain ops, in the order of rapl_domain_type */ 299 static const struct powercap_zone_ops zone_ops[] = { 300 /* RAPL_DOMAIN_PACKAGE */ 301 { 302 .get_energy_uj = get_energy_counter, 303 .get_max_energy_range_uj = get_max_energy_counter, 304 .release = release_zone, 305 .set_enable = set_domain_enable, 306 .get_enable = get_domain_enable, 307 }, 308 /* RAPL_DOMAIN_PP0 */ 309 { 310 .get_energy_uj = get_energy_counter, 311 .get_max_energy_range_uj = get_max_energy_counter, 312 .release = release_zone, 313 .set_enable = set_domain_enable, 314 .get_enable = get_domain_enable, 315 }, 316 /* RAPL_DOMAIN_PP1 */ 317 { 318 .get_energy_uj = get_energy_counter, 319 .get_max_energy_range_uj = get_max_energy_counter, 320 .release = release_zone, 321 .set_enable = set_domain_enable, 322 .get_enable = get_domain_enable, 323 }, 324 /* RAPL_DOMAIN_DRAM */ 325 { 326 .get_energy_uj = get_energy_counter, 327 .get_max_energy_range_uj = get_max_energy_counter, 328 .release = release_zone, 329 .set_enable = set_domain_enable, 330 .get_enable = get_domain_enable, 331 }, 332 /* RAPL_DOMAIN_PLATFORM */ 333 { 334 .get_energy_uj = get_energy_counter, 335 .get_max_energy_range_uj = get_max_energy_counter, 336 .release = release_zone, 337 .set_enable = set_domain_enable, 338 .get_enable = get_domain_enable, 339 }, 340 }; 341 342 /* 343 * Constraint index used by powercap can be different than power limit (PL) 344 * index in that some PLs maybe missing due to non-existent MSRs. So we 345 * need to convert here by finding the valid PLs only (name populated). 346 */ 347 static int contraint_to_pl(struct rapl_domain *rd, int cid) 348 { 349 int i, j; 350 351 for (i = POWER_LIMIT1, j = 0; i < NR_POWER_LIMITS; i++) { 352 if (is_pl_valid(rd, i) && j++ == cid) { 353 pr_debug("%s: index %d\n", __func__, i); 354 return i; 355 } 356 } 357 pr_err("Cannot find matching power limit for constraint %d\n", cid); 358 359 return -EINVAL; 360 } 361 362 static int set_power_limit(struct powercap_zone *power_zone, int cid, 363 u64 power_limit) 364 { 365 struct rapl_domain *rd; 366 struct rapl_package *rp; 367 int ret = 0; 368 int id; 369 370 cpus_read_lock(); 371 rd = power_zone_to_rapl_domain(power_zone); 372 id = contraint_to_pl(rd, cid); 373 rp = rd->rp; 374 375 ret = rapl_write_pl_data(rd, id, PL_LIMIT, power_limit); 376 if (!ret) 377 package_power_limit_irq_save(rp); 378 cpus_read_unlock(); 379 return ret; 380 } 381 382 static int get_current_power_limit(struct powercap_zone *power_zone, int cid, 383 u64 *data) 384 { 385 struct rapl_domain *rd; 386 u64 val; 387 int ret = 0; 388 int id; 389 390 cpus_read_lock(); 391 rd = power_zone_to_rapl_domain(power_zone); 392 id = contraint_to_pl(rd, cid); 393 394 ret = rapl_read_pl_data(rd, id, PL_LIMIT, true, &val); 395 if (!ret) 396 *data = val; 397 398 cpus_read_unlock(); 399 400 return ret; 401 } 402 403 static int set_time_window(struct powercap_zone *power_zone, int cid, 404 u64 window) 405 { 406 struct rapl_domain *rd; 407 int ret = 0; 408 int id; 409 410 cpus_read_lock(); 411 rd = power_zone_to_rapl_domain(power_zone); 412 id = contraint_to_pl(rd, cid); 413 414 ret = rapl_write_pl_data(rd, id, PL_TIME_WINDOW, window); 415 416 cpus_read_unlock(); 417 return ret; 418 } 419 420 static int get_time_window(struct powercap_zone *power_zone, int cid, 421 u64 *data) 422 { 423 struct rapl_domain *rd; 424 u64 val; 425 int ret = 0; 426 int id; 427 428 cpus_read_lock(); 429 rd = power_zone_to_rapl_domain(power_zone); 430 id = contraint_to_pl(rd, cid); 431 432 ret = rapl_read_pl_data(rd, id, PL_TIME_WINDOW, true, &val); 433 if (!ret) 434 *data = val; 435 436 cpus_read_unlock(); 437 438 return ret; 439 } 440 441 static const char *get_constraint_name(struct powercap_zone *power_zone, 442 int cid) 443 { 444 struct rapl_domain *rd; 445 int id; 446 447 rd = power_zone_to_rapl_domain(power_zone); 448 id = contraint_to_pl(rd, cid); 449 if (id >= 0) 450 return rd->rpl[id].name; 451 452 return NULL; 453 } 454 455 static int get_max_power(struct powercap_zone *power_zone, int cid, u64 *data) 456 { 457 struct rapl_domain *rd; 458 u64 val; 459 int ret = 0; 460 int id; 461 462 cpus_read_lock(); 463 rd = power_zone_to_rapl_domain(power_zone); 464 id = contraint_to_pl(rd, cid); 465 466 ret = rapl_read_pl_data(rd, id, PL_MAX_POWER, true, &val); 467 if (!ret) 468 *data = val; 469 470 /* As a generalization rule, PL4 would be around two times PL2. */ 471 if (id == POWER_LIMIT4) 472 *data = *data * 2; 473 474 cpus_read_unlock(); 475 476 return ret; 477 } 478 479 static const struct powercap_zone_constraint_ops constraint_ops = { 480 .set_power_limit_uw = set_power_limit, 481 .get_power_limit_uw = get_current_power_limit, 482 .set_time_window_us = set_time_window, 483 .get_time_window_us = get_time_window, 484 .get_max_power_uw = get_max_power, 485 .get_name = get_constraint_name, 486 }; 487 488 /* Return the id used for read_raw/write_raw callback */ 489 static int get_rid(struct rapl_package *rp) 490 { 491 return rp->lead_cpu >= 0 ? rp->lead_cpu : rp->id; 492 } 493 494 /* called after domain detection and package level data are set */ 495 static void rapl_init_domains(struct rapl_package *rp) 496 { 497 enum rapl_domain_type i; 498 enum rapl_domain_reg_id j; 499 struct rapl_domain *rd = rp->domains; 500 501 for (i = 0; i < RAPL_DOMAIN_MAX; i++) { 502 unsigned int mask = rp->domain_map & (1 << i); 503 int t; 504 505 if (!mask) 506 continue; 507 508 rd->rp = rp; 509 510 if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) { 511 snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "psys-%d", 512 rp->lead_cpu >= 0 ? topology_physical_package_id(rp->lead_cpu) : 513 rp->id); 514 } else { 515 snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "%s", 516 rapl_domain_names[i]); 517 } 518 519 rd->id = i; 520 521 /* PL1 is supported by default */ 522 rp->priv->limits[i] |= BIT(POWER_LIMIT1); 523 524 for (t = POWER_LIMIT1; t < NR_POWER_LIMITS; t++) { 525 if (rp->priv->limits[i] & BIT(t)) 526 rd->rpl[t].name = pl_names[t]; 527 } 528 529 for (j = 0; j < RAPL_DOMAIN_REG_MAX; j++) 530 rd->regs[j] = rp->priv->regs[i][j]; 531 532 rd++; 533 } 534 } 535 536 static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, 537 u64 value, int to_raw) 538 { 539 u64 units = 1; 540 const struct rapl_defaults *defaults = get_defaults(rd->rp); 541 u64 scale = 1; 542 543 switch (type) { 544 case POWER_UNIT: 545 units = rd->power_unit; 546 break; 547 case ENERGY_UNIT: 548 scale = ENERGY_UNIT_SCALE; 549 units = rd->energy_unit; 550 break; 551 case TIME_UNIT: 552 return defaults->compute_time_window(rd, value, to_raw); 553 case ARBITRARY_UNIT: 554 default: 555 return value; 556 } 557 558 if (to_raw) 559 return div64_u64(value, units) * scale; 560 561 value *= units; 562 563 return div64_u64(value, scale); 564 } 565 566 static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim) 567 { 568 struct rapl_primitive_info *rpi = rp->priv->rpi; 569 570 if (prim < 0 || prim >= NR_RAPL_PRIMITIVES || !rpi) 571 return NULL; 572 573 return &rpi[prim]; 574 } 575 576 static int rapl_config(struct rapl_package *rp) 577 { 578 /* defaults_msr can be NULL on unsupported platforms */ 579 if (!rp->priv->defaults || !rp->priv->rpi) 580 return -ENODEV; 581 582 return 0; 583 } 584 585 static enum rapl_primitives 586 prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim) 587 { 588 const struct rapl_defaults *defaults = get_defaults(rd->rp); 589 590 if (!defaults->spr_psys_bits) 591 return prim; 592 593 if (rd->id != RAPL_DOMAIN_PLATFORM) 594 return prim; 595 596 switch (prim) { 597 case POWER_LIMIT1: 598 return PSYS_POWER_LIMIT1; 599 case POWER_LIMIT2: 600 return PSYS_POWER_LIMIT2; 601 case PL1_ENABLE: 602 return PSYS_PL1_ENABLE; 603 case PL2_ENABLE: 604 return PSYS_PL2_ENABLE; 605 case TIME_WINDOW1: 606 return PSYS_TIME_WINDOW1; 607 case TIME_WINDOW2: 608 return PSYS_TIME_WINDOW2; 609 default: 610 return prim; 611 } 612 } 613 614 /* Read primitive data based on its related struct rapl_primitive_info. 615 * if xlate flag is set, return translated data based on data units, i.e. 616 * time, energy, and power. 617 * RAPL MSRs are non-architectual and are laid out not consistently across 618 * domains. Here we use primitive info to allow writing consolidated access 619 * functions. 620 * For a given primitive, it is processed by MSR mask and shift. Unit conversion 621 * is pre-assigned based on RAPL unit MSRs read at init time. 622 * 63-------------------------- 31--------------------------- 0 623 * | xxxxx (mask) | 624 * | |<- shift ----------------| 625 * 63-------------------------- 31--------------------------- 0 626 */ 627 static int rapl_read_data_raw(struct rapl_domain *rd, 628 enum rapl_primitives prim, bool xlate, u64 *data, 629 bool pmu_ctx) 630 { 631 u64 value; 632 enum rapl_primitives prim_fixed = prim_fixups(rd, prim); 633 struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed); 634 struct reg_action ra; 635 636 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY) 637 return -EINVAL; 638 639 ra.reg = rd->regs[rpi->id]; 640 if (!ra.reg.val) 641 return -EINVAL; 642 643 ra.mask = rpi->mask; 644 645 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, pmu_ctx)) { 646 pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name); 647 return -EIO; 648 } 649 650 value = ra.value >> rpi->shift; 651 652 if (xlate) 653 *data = rapl_unit_xlate(rd, rpi->unit, value, 0); 654 else 655 *data = value; 656 657 return 0; 658 } 659 660 /* Similar use of primitive info in the read counterpart */ 661 static int rapl_write_data_raw(struct rapl_domain *rd, 662 enum rapl_primitives prim, 663 unsigned long long value) 664 { 665 enum rapl_primitives prim_fixed = prim_fixups(rd, prim); 666 struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed); 667 u64 bits; 668 struct reg_action ra; 669 int ret; 670 671 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY) 672 return -EINVAL; 673 674 bits = rapl_unit_xlate(rd, rpi->unit, value, 1); 675 bits <<= rpi->shift; 676 bits &= rpi->mask; 677 678 memset(&ra, 0, sizeof(ra)); 679 680 ra.reg = rd->regs[rpi->id]; 681 ra.mask = rpi->mask; 682 ra.value = bits; 683 684 ret = rd->rp->priv->write_raw(get_rid(rd->rp), &ra); 685 686 return ret; 687 } 688 689 static int rapl_read_pl_data(struct rapl_domain *rd, int pl, 690 enum pl_prims pl_prim, bool xlate, u64 *data) 691 { 692 enum rapl_primitives prim = get_pl_prim(rd, pl, pl_prim); 693 694 if (!is_pl_valid(rd, pl)) 695 return -EINVAL; 696 697 return rapl_read_data_raw(rd, prim, xlate, data, false); 698 } 699 700 static int rapl_write_pl_data(struct rapl_domain *rd, int pl, 701 enum pl_prims pl_prim, 702 unsigned long long value) 703 { 704 enum rapl_primitives prim = get_pl_prim(rd, pl, pl_prim); 705 706 if (!is_pl_valid(rd, pl)) 707 return -EINVAL; 708 709 if (rd->rpl[pl].locked) { 710 pr_debug("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]); 711 return -EACCES; 712 } 713 714 return rapl_write_data_raw(rd, prim, value); 715 } 716 /* 717 * Raw RAPL data stored in MSRs are in certain scales. We need to 718 * convert them into standard units based on the units reported in 719 * the RAPL unit MSRs. This is specific to CPUs as the method to 720 * calculate units differ on different CPUs. 721 * We convert the units to below format based on CPUs. 722 * i.e. 723 * energy unit: picoJoules : Represented in picoJoules by default 724 * power unit : microWatts : Represented in milliWatts by default 725 * time unit : microseconds: Represented in seconds by default 726 */ 727 int rapl_default_check_unit(struct rapl_domain *rd) 728 { 729 struct reg_action ra; 730 u32 value; 731 732 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; 733 ra.mask = ~0; 734 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, false)) { 735 pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n", 736 ra.reg.val, rd->rp->name, rd->name); 737 return -ENODEV; 738 } 739 740 value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET; 741 rd->energy_unit = (ENERGY_UNIT_SCALE * MICROJOULE_PER_JOULE) >> value; 742 743 value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET; 744 rd->power_unit = MICROWATT_PER_WATT >> value; 745 746 value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET; 747 rd->time_unit = USEC_PER_SEC >> value; 748 749 pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n", 750 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit); 751 752 return 0; 753 } 754 EXPORT_SYMBOL_NS_GPL(rapl_default_check_unit, "INTEL_RAPL"); 755 756 static void power_limit_irq_save_cpu(void *info) 757 { 758 struct msr val; 759 struct rapl_package *rp = (struct rapl_package *)info; 760 761 /* save the state of PLN irq mask bit before disabling it */ 762 rdmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &val.q); 763 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) { 764 rp->power_limit_irq = val.l & PACKAGE_THERM_INT_PLN_ENABLE; 765 rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED; 766 } 767 val.l &= ~PACKAGE_THERM_INT_PLN_ENABLE; 768 wrmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q); 769 } 770 771 /* REVISIT: 772 * When package power limit is set artificially low by RAPL, LVT 773 * thermal interrupt for package power limit should be ignored 774 * since we are not really exceeding the real limit. The intention 775 * is to avoid excessive interrupts while we are trying to save power. 776 * A useful feature might be routing the package_power_limit interrupt 777 * to userspace via eventfd. once we have a usecase, this is simple 778 * to do by adding an atomic notifier. 779 */ 780 781 static void package_power_limit_irq_save(struct rapl_package *rp) 782 { 783 if (rp->lead_cpu < 0) 784 return; 785 786 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) 787 return; 788 789 smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1); 790 } 791 792 /* 793 * Restore per package power limit interrupt enable state. Called from cpu 794 * hotplug code on package removal. 795 */ 796 static void package_power_limit_irq_restore(struct rapl_package *rp) 797 { 798 struct msr val; 799 800 if (rp->lead_cpu < 0) 801 return; 802 803 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) 804 return; 805 806 /* irq enable state not saved, nothing to restore */ 807 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) 808 return; 809 810 rdmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &val.q); 811 812 if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE) 813 val.l |= PACKAGE_THERM_INT_PLN_ENABLE; 814 else 815 val.l &= ~PACKAGE_THERM_INT_PLN_ENABLE; 816 817 wrmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q); 818 } 819 820 void rapl_default_set_floor_freq(struct rapl_domain *rd, bool mode) 821 { 822 int i; 823 824 /* always enable clamp such that p-state can go below OS requested 825 * range. power capping priority over guranteed frequency. 826 */ 827 rapl_write_pl_data(rd, POWER_LIMIT1, PL_CLAMP, mode); 828 829 for (i = POWER_LIMIT2; i < NR_POWER_LIMITS; i++) { 830 rapl_write_pl_data(rd, i, PL_ENABLE, mode); 831 rapl_write_pl_data(rd, i, PL_CLAMP, mode); 832 } 833 } 834 EXPORT_SYMBOL_NS_GPL(rapl_default_set_floor_freq, "INTEL_RAPL"); 835 836 u64 rapl_default_compute_time_window(struct rapl_domain *rd, u64 value, bool to_raw) 837 { 838 u64 f, y; /* fraction and exp. used for time unit */ 839 840 /* 841 * Special processing based on 2^Y*(1+F/4), refer 842 * to Intel Software Developer's manual Vol.3B: CH 14.9.3. 843 */ 844 if (!to_raw) { 845 f = (value & 0x60) >> 5; 846 y = value & 0x1f; 847 value = (1ULL << y) * (4 + f) * rd->time_unit / 4; 848 } else { 849 if (value < rd->time_unit) 850 return 0; 851 852 do_div(value, rd->time_unit); 853 y = ilog2(value); 854 855 /* 856 * The target hardware field is 7 bits wide, so return all ones 857 * if the exponent is too large. 858 */ 859 if (y > 0x1f) 860 return 0x7f; 861 862 f = div64_u64(4 * (value - BIT_ULL(y)), BIT_ULL(y)); 863 value = (y & 0x1f) | ((f & 0x3) << 5); 864 } 865 return value; 866 } 867 EXPORT_SYMBOL_NS_GPL(rapl_default_compute_time_window, "INTEL_RAPL"); 868 869 /* Read once for all raw primitive data for domains */ 870 static void rapl_update_domain_data(struct rapl_package *rp) 871 { 872 int dmn, prim; 873 u64 val; 874 875 for (dmn = 0; dmn < rp->nr_domains; dmn++) { 876 pr_debug("update %s domain %s data\n", rp->name, 877 rp->domains[dmn].name); 878 /* exclude non-raw primitives */ 879 for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) { 880 struct rapl_primitive_info *rpi = get_rpi(rp, prim); 881 882 if (!rapl_read_data_raw(&rp->domains[dmn], prim, 883 rpi->unit, &val, false)) 884 rp->domains[dmn].rdd.primitives[prim] = val; 885 } 886 } 887 888 } 889 890 static int rapl_package_register_powercap(struct rapl_package *rp) 891 { 892 struct rapl_domain *rd; 893 struct powercap_zone *power_zone = NULL; 894 int nr_pl, ret; 895 896 /* Update the domain data of the new package */ 897 rapl_update_domain_data(rp); 898 899 /* first we register package domain as the parent zone */ 900 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { 901 if (rd->id == RAPL_DOMAIN_PACKAGE) { 902 nr_pl = find_nr_power_limit(rd); 903 pr_debug("register package domain %s\n", rp->name); 904 power_zone = powercap_register_zone(&rd->power_zone, 905 rp->priv->control_type, rp->name, 906 NULL, &zone_ops[rd->id], nr_pl, 907 &constraint_ops); 908 if (IS_ERR(power_zone)) { 909 pr_debug("failed to register power zone %s\n", 910 rp->name); 911 return PTR_ERR(power_zone); 912 } 913 /* track parent zone in per package/socket data */ 914 rp->power_zone = power_zone; 915 /* done, only one package domain per socket */ 916 break; 917 } 918 } 919 if (!power_zone) { 920 pr_err("no package domain found, unknown topology!\n"); 921 return -ENODEV; 922 } 923 /* now register domains as children of the socket/package */ 924 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { 925 struct powercap_zone *parent = rp->power_zone; 926 927 if (rd->id == RAPL_DOMAIN_PACKAGE) 928 continue; 929 if (rd->id == RAPL_DOMAIN_PLATFORM) 930 parent = NULL; 931 /* number of power limits per domain varies */ 932 nr_pl = find_nr_power_limit(rd); 933 power_zone = powercap_register_zone(&rd->power_zone, 934 rp->priv->control_type, 935 rd->name, parent, 936 &zone_ops[rd->id], nr_pl, 937 &constraint_ops); 938 939 if (IS_ERR(power_zone)) { 940 pr_debug("failed to register power_zone, %s:%s\n", 941 rp->name, rd->name); 942 ret = PTR_ERR(power_zone); 943 goto err_cleanup; 944 } 945 } 946 return 0; 947 948 err_cleanup: 949 /* 950 * Clean up previously initialized domains within the package if we 951 * failed after the first domain setup. 952 */ 953 while (--rd >= rp->domains) { 954 pr_debug("unregister %s domain %s\n", rp->name, rd->name); 955 powercap_unregister_zone(rp->priv->control_type, 956 &rd->power_zone); 957 } 958 959 return ret; 960 } 961 962 static int rapl_check_domain(int domain, struct rapl_package *rp) 963 { 964 struct reg_action ra; 965 966 switch (domain) { 967 case RAPL_DOMAIN_PACKAGE: 968 case RAPL_DOMAIN_PP0: 969 case RAPL_DOMAIN_PP1: 970 case RAPL_DOMAIN_DRAM: 971 case RAPL_DOMAIN_PLATFORM: 972 ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS]; 973 break; 974 default: 975 pr_err("invalid domain id %d\n", domain); 976 return -EINVAL; 977 } 978 /* make sure domain counters are available and contains non-zero 979 * values, otherwise skip it. 980 */ 981 982 ra.mask = ENERGY_STATUS_MASK; 983 if (rp->priv->read_raw(get_rid(rp), &ra, false) || !ra.value) 984 return -ENODEV; 985 986 return 0; 987 } 988 989 /* 990 * Get per domain energy/power/time unit. 991 * RAPL Interfaces without per domain unit register will use the package 992 * scope unit register to set per domain units. 993 */ 994 static int rapl_get_domain_unit(struct rapl_domain *rd) 995 { 996 const struct rapl_defaults *defaults = get_defaults(rd->rp); 997 int ret; 998 999 if (!rd->regs[RAPL_DOMAIN_REG_UNIT].val) { 1000 if (!rd->rp->priv->reg_unit.val) { 1001 pr_err("No valid Unit register found\n"); 1002 return -ENODEV; 1003 } 1004 rd->regs[RAPL_DOMAIN_REG_UNIT] = rd->rp->priv->reg_unit; 1005 } 1006 1007 if (!defaults->check_unit) { 1008 pr_err("missing .check_unit() callback\n"); 1009 return -ENODEV; 1010 } 1011 1012 ret = defaults->check_unit(rd); 1013 if (ret) 1014 return ret; 1015 1016 if (rd->id == RAPL_DOMAIN_DRAM && defaults->dram_domain_energy_unit) 1017 rd->energy_unit = defaults->dram_domain_energy_unit; 1018 if (rd->id == RAPL_DOMAIN_PLATFORM && defaults->psys_domain_energy_unit) 1019 rd->energy_unit = defaults->psys_domain_energy_unit; 1020 return 0; 1021 } 1022 1023 /* 1024 * Check if power limits are available. Two cases when they are not available: 1025 * 1. Locked by BIOS, in this case we still provide read-only access so that 1026 * users can see what limit is set by the BIOS. 1027 * 2. Some CPUs make some domains monitoring only which means PLx MSRs may not 1028 * exist at all. In this case, we do not show the constraints in powercap. 1029 * 1030 * Called after domains are detected and initialized. 1031 */ 1032 static void rapl_detect_powerlimit(struct rapl_domain *rd) 1033 { 1034 u64 val64; 1035 int i; 1036 1037 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) { 1038 if (!rapl_read_pl_data(rd, i, PL_LOCK, false, &val64)) { 1039 if (val64) { 1040 rd->rpl[i].locked = true; 1041 pr_info("%s:%s:%s locked by BIOS\n", 1042 rd->rp->name, rd->name, pl_names[i]); 1043 } 1044 } 1045 1046 if (rapl_read_pl_data(rd, i, PL_LIMIT, false, &val64)) 1047 rd->rpl[i].name = NULL; 1048 } 1049 } 1050 1051 /* Detect active and valid domains for the given CPU, caller must 1052 * ensure the CPU belongs to the targeted package and CPU hotlug is disabled. 1053 */ 1054 static int rapl_detect_domains(struct rapl_package *rp) 1055 { 1056 struct rapl_domain *rd; 1057 int i; 1058 1059 for (i = 0; i < RAPL_DOMAIN_MAX; i++) { 1060 /* use physical package id to read counters */ 1061 if (!rapl_check_domain(i, rp)) { 1062 rp->domain_map |= 1 << i; 1063 pr_info("Found RAPL domain %s\n", rapl_domain_names[i]); 1064 } 1065 } 1066 rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX); 1067 if (!rp->nr_domains) { 1068 pr_debug("no valid rapl domains found in %s\n", rp->name); 1069 return -ENODEV; 1070 } 1071 pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name); 1072 1073 rp->domains = kzalloc_objs(struct rapl_domain, rp->nr_domains); 1074 if (!rp->domains) 1075 return -ENOMEM; 1076 1077 rapl_init_domains(rp); 1078 1079 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { 1080 rapl_get_domain_unit(rd); 1081 rapl_detect_powerlimit(rd); 1082 } 1083 1084 return 0; 1085 } 1086 1087 #ifdef CONFIG_PERF_EVENTS 1088 1089 /* 1090 * Support for RAPL PMU 1091 * 1092 * Register a PMU if any of the registered RAPL Packages have the requirement 1093 * of exposing its energy counters via Perf PMU. 1094 * 1095 * PMU Name: 1096 * power 1097 * 1098 * Events: 1099 * Name Event id RAPL Domain 1100 * energy_cores 0x01 RAPL_DOMAIN_PP0 1101 * energy_pkg 0x02 RAPL_DOMAIN_PACKAGE 1102 * energy_ram 0x03 RAPL_DOMAIN_DRAM 1103 * energy_gpu 0x04 RAPL_DOMAIN_PP1 1104 * energy_psys 0x05 RAPL_DOMAIN_PLATFORM 1105 * 1106 * Unit: 1107 * Joules 1108 * 1109 * Scale: 1110 * 2.3283064365386962890625e-10 1111 * The same RAPL domain in different RAPL Packages may have different 1112 * energy units. Use 2.3283064365386962890625e-10 (2^-32) Joules as 1113 * the fixed unit for all energy counters, and covert each hardware 1114 * counter increase to N times of PMU event counter increases. 1115 * 1116 * This is fully compatible with the current MSR RAPL PMU. This means that 1117 * userspace programs like turbostat can use the same code to handle RAPL Perf 1118 * PMU, no matter what RAPL Interface driver (MSR/TPMI, etc) is running 1119 * underlying on the platform. 1120 * 1121 * Note that RAPL Packages can be probed/removed dynamically, and the events 1122 * supported by each TPMI RAPL device can be different. Thus the RAPL PMU 1123 * support is done on demand, which means 1124 * 1. PMU is registered only if it is needed by a RAPL Package. PMU events for 1125 * unsupported counters are not exposed. 1126 * 2. PMU is unregistered and registered when a new RAPL Package is probed and 1127 * supports new counters that are not supported by current PMU. 1128 * 3. PMU is unregistered when all registered RAPL Packages don't need PMU. 1129 */ 1130 1131 struct rapl_pmu { 1132 struct pmu pmu; /* Perf PMU structure */ 1133 u64 timer_ms; /* Maximum expiration time to avoid counter overflow */ 1134 unsigned long domain_map; /* Events supported by current registered PMU */ 1135 bool registered; /* Whether the PMU has been registered or not */ 1136 }; 1137 1138 static struct rapl_pmu rapl_pmu; 1139 1140 /* PMU helpers */ 1141 1142 static void set_pmu_cpumask(struct rapl_package *rp, cpumask_var_t mask) 1143 { 1144 int cpu; 1145 1146 if (!rp->has_pmu) 1147 return; 1148 1149 /* Only TPMI & MSR RAPL are supported for now */ 1150 if (rp->priv->type != RAPL_IF_TPMI && rp->priv->type != RAPL_IF_MSR) 1151 return; 1152 1153 /* TPMI/MSR RAPL uses any CPU in the package for PMU */ 1154 for_each_online_cpu(cpu) 1155 if (topology_physical_package_id(cpu) == rp->id) 1156 cpumask_set_cpu(cpu, mask); 1157 } 1158 1159 static bool is_rp_pmu_cpu(struct rapl_package *rp, int cpu) 1160 { 1161 if (!rp->has_pmu) 1162 return false; 1163 1164 /* Only TPMI & MSR RAPL are supported for now */ 1165 if (rp->priv->type != RAPL_IF_TPMI && rp->priv->type != RAPL_IF_MSR) 1166 return false; 1167 1168 /* TPMI/MSR RAPL uses any CPU in the package for PMU */ 1169 return topology_physical_package_id(cpu) == rp->id; 1170 } 1171 1172 static struct rapl_package_pmu_data *event_to_pmu_data(struct perf_event *event) 1173 { 1174 struct rapl_package *rp = event->pmu_private; 1175 1176 return &rp->pmu_data; 1177 } 1178 1179 /* PMU event callbacks */ 1180 1181 static u64 event_read_counter(struct perf_event *event) 1182 { 1183 struct rapl_package *rp = event->pmu_private; 1184 u64 val; 1185 int ret; 1186 1187 /* Return 0 for unsupported events */ 1188 if (event->hw.idx < 0) 1189 return 0; 1190 1191 ret = rapl_read_data_raw(&rp->domains[event->hw.idx], ENERGY_COUNTER, false, &val, true); 1192 1193 /* Return 0 for failed read */ 1194 if (ret) 1195 return 0; 1196 1197 return val; 1198 } 1199 1200 static void __rapl_pmu_event_start(struct perf_event *event) 1201 { 1202 struct rapl_package_pmu_data *data = event_to_pmu_data(event); 1203 1204 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1205 return; 1206 1207 event->hw.state = 0; 1208 1209 list_add_tail(&event->active_entry, &data->active_list); 1210 1211 local64_set(&event->hw.prev_count, event_read_counter(event)); 1212 if (++data->n_active == 1) 1213 hrtimer_start(&data->hrtimer, data->timer_interval, 1214 HRTIMER_MODE_REL_PINNED); 1215 } 1216 1217 static void rapl_pmu_event_start(struct perf_event *event, int mode) 1218 { 1219 struct rapl_package_pmu_data *data = event_to_pmu_data(event); 1220 unsigned long flags; 1221 1222 raw_spin_lock_irqsave(&data->lock, flags); 1223 __rapl_pmu_event_start(event); 1224 raw_spin_unlock_irqrestore(&data->lock, flags); 1225 } 1226 1227 static u64 rapl_event_update(struct perf_event *event) 1228 { 1229 struct hw_perf_event *hwc = &event->hw; 1230 struct rapl_package_pmu_data *data = event_to_pmu_data(event); 1231 u64 prev_raw_count, new_raw_count; 1232 s64 delta, sdelta; 1233 int shift = 64 - RAPL_CNTR_WIDTH; 1234 1235 /* 1236 * Follow the generic code to drain hwc->prev_count. 1237 * The loop is not expected to run for multiple times. 1238 */ 1239 prev_raw_count = local64_read(&hwc->prev_count); 1240 do { 1241 new_raw_count = event_read_counter(event); 1242 } while (!local64_try_cmpxchg(&hwc->prev_count, 1243 &prev_raw_count, new_raw_count)); 1244 1245 1246 /* 1247 * Now we have the new raw value and have updated the prev 1248 * timestamp already. We can now calculate the elapsed delta 1249 * (event-)time and add that to the generic event. 1250 * 1251 * Careful, the counter is narrower than u64 and is not 1252 * sign-extended above its physical width. Shift both values up 1253 * so that the subtraction wraps, then shift the result back down. 1254 */ 1255 delta = (new_raw_count << shift) - (prev_raw_count << shift); 1256 delta >>= shift; 1257 1258 /* 1259 * Scale delta to smallest unit (2^-32) 1260 * users must then scale back: count * 1/(1e9*2^32) to get Joules 1261 * or use ldexp(count, -32). 1262 * Watts = Joules/Time delta 1263 */ 1264 sdelta = delta * data->scale[event->hw.flags]; 1265 1266 local64_add(sdelta, &event->count); 1267 1268 return new_raw_count; 1269 } 1270 1271 static void rapl_pmu_event_stop(struct perf_event *event, int mode) 1272 { 1273 struct rapl_package_pmu_data *data = event_to_pmu_data(event); 1274 struct hw_perf_event *hwc = &event->hw; 1275 unsigned long flags; 1276 1277 raw_spin_lock_irqsave(&data->lock, flags); 1278 1279 /* Mark event as deactivated and stopped */ 1280 if (!(hwc->state & PERF_HES_STOPPED)) { 1281 WARN_ON_ONCE(data->n_active <= 0); 1282 if (--data->n_active == 0) 1283 hrtimer_cancel(&data->hrtimer); 1284 1285 list_del(&event->active_entry); 1286 1287 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1288 hwc->state |= PERF_HES_STOPPED; 1289 } 1290 1291 /* Check if update of sw counter is necessary */ 1292 if ((mode & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1293 /* 1294 * Drain the remaining delta count out of a event 1295 * that we are disabling: 1296 */ 1297 rapl_event_update(event); 1298 hwc->state |= PERF_HES_UPTODATE; 1299 } 1300 1301 raw_spin_unlock_irqrestore(&data->lock, flags); 1302 } 1303 1304 static int rapl_pmu_event_add(struct perf_event *event, int mode) 1305 { 1306 struct rapl_package_pmu_data *data = event_to_pmu_data(event); 1307 struct hw_perf_event *hwc = &event->hw; 1308 unsigned long flags; 1309 1310 raw_spin_lock_irqsave(&data->lock, flags); 1311 1312 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1313 1314 if (mode & PERF_EF_START) 1315 __rapl_pmu_event_start(event); 1316 1317 raw_spin_unlock_irqrestore(&data->lock, flags); 1318 1319 return 0; 1320 } 1321 1322 static void rapl_pmu_event_del(struct perf_event *event, int flags) 1323 { 1324 rapl_pmu_event_stop(event, PERF_EF_UPDATE); 1325 } 1326 1327 /* RAPL PMU event ids, same as shown in sysfs */ 1328 enum perf_rapl_events { 1329 PERF_RAPL_PP0 = 1, /* all cores */ 1330 PERF_RAPL_PKG, /* entire package */ 1331 PERF_RAPL_RAM, /* DRAM */ 1332 PERF_RAPL_PP1, /* gpu */ 1333 PERF_RAPL_PSYS, /* psys */ 1334 PERF_RAPL_MAX 1335 }; 1336 1337 static const int event_to_domain[PERF_RAPL_MAX] = { 1338 [PERF_RAPL_PP0] = RAPL_DOMAIN_PP0, 1339 [PERF_RAPL_PKG] = RAPL_DOMAIN_PACKAGE, 1340 [PERF_RAPL_RAM] = RAPL_DOMAIN_DRAM, 1341 [PERF_RAPL_PP1] = RAPL_DOMAIN_PP1, 1342 [PERF_RAPL_PSYS] = RAPL_DOMAIN_PLATFORM, 1343 }; 1344 1345 static int rapl_pmu_event_init(struct perf_event *event) 1346 { 1347 struct rapl_package *pos, *rp = NULL; 1348 u64 cfg = event->attr.config & RAPL_EVENT_MASK; 1349 int domain, idx; 1350 1351 /* Only look at RAPL events */ 1352 if (event->attr.type != event->pmu->type) 1353 return -ENOENT; 1354 1355 /* Check for supported events only */ 1356 if (!cfg || cfg >= PERF_RAPL_MAX) 1357 return -EINVAL; 1358 1359 if (event->cpu < 0) 1360 return -EINVAL; 1361 1362 /* Find out which Package the event belongs to */ 1363 list_for_each_entry(pos, &rapl_packages, plist) { 1364 if (is_rp_pmu_cpu(pos, event->cpu)) { 1365 rp = pos; 1366 break; 1367 } 1368 } 1369 if (!rp) 1370 return -ENODEV; 1371 1372 /* Find out which RAPL Domain the event belongs to */ 1373 domain = event_to_domain[cfg]; 1374 1375 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG; 1376 event->pmu_private = rp; /* Which package */ 1377 event->hw.flags = domain; /* Which domain */ 1378 1379 event->hw.idx = -1; 1380 /* Find out the index in rp->domains[] to get domain pointer */ 1381 for (idx = 0; idx < rp->nr_domains; idx++) { 1382 if (rp->domains[idx].id == domain) { 1383 event->hw.idx = idx; 1384 break; 1385 } 1386 } 1387 1388 return 0; 1389 } 1390 1391 static void rapl_pmu_event_read(struct perf_event *event) 1392 { 1393 rapl_event_update(event); 1394 } 1395 1396 static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer) 1397 { 1398 struct rapl_package_pmu_data *data = 1399 container_of(hrtimer, struct rapl_package_pmu_data, hrtimer); 1400 struct perf_event *event; 1401 unsigned long flags; 1402 1403 if (!data->n_active) 1404 return HRTIMER_NORESTART; 1405 1406 raw_spin_lock_irqsave(&data->lock, flags); 1407 1408 list_for_each_entry(event, &data->active_list, active_entry) 1409 rapl_event_update(event); 1410 1411 raw_spin_unlock_irqrestore(&data->lock, flags); 1412 1413 hrtimer_forward_now(hrtimer, data->timer_interval); 1414 1415 return HRTIMER_RESTART; 1416 } 1417 1418 /* PMU sysfs attributes */ 1419 1420 /* 1421 * There are no default events, but we need to create "events" group (with 1422 * empty attrs) before updating it with detected events. 1423 */ 1424 static struct attribute *attrs_empty[] = { 1425 NULL, 1426 }; 1427 1428 static struct attribute_group pmu_events_group = { 1429 .name = "events", 1430 .attrs = attrs_empty, 1431 }; 1432 1433 static ssize_t cpumask_show(struct device *dev, 1434 struct device_attribute *attr, char *buf) 1435 { 1436 struct rapl_package *rp; 1437 cpumask_var_t cpu_mask; 1438 int ret; 1439 1440 if (!alloc_cpumask_var(&cpu_mask, GFP_KERNEL)) 1441 return -ENOMEM; 1442 1443 cpus_read_lock(); 1444 1445 cpumask_clear(cpu_mask); 1446 1447 /* Choose a cpu for each RAPL Package */ 1448 list_for_each_entry(rp, &rapl_packages, plist) { 1449 set_pmu_cpumask(rp, cpu_mask); 1450 } 1451 cpus_read_unlock(); 1452 1453 ret = sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpu_mask)); 1454 1455 free_cpumask_var(cpu_mask); 1456 1457 return ret; 1458 } 1459 1460 static DEVICE_ATTR_RO(cpumask); 1461 1462 static struct attribute *pmu_cpumask_attrs[] = { 1463 &dev_attr_cpumask.attr, 1464 NULL 1465 }; 1466 1467 static struct attribute_group pmu_cpumask_group = { 1468 .attrs = pmu_cpumask_attrs, 1469 }; 1470 1471 PMU_FORMAT_ATTR(event, "config:0-7"); 1472 static struct attribute *pmu_format_attr[] = { 1473 &format_attr_event.attr, 1474 NULL 1475 }; 1476 1477 static struct attribute_group pmu_format_group = { 1478 .name = "format", 1479 .attrs = pmu_format_attr, 1480 }; 1481 1482 static const struct attribute_group *pmu_attr_groups[] = { 1483 &pmu_events_group, 1484 &pmu_cpumask_group, 1485 &pmu_format_group, 1486 NULL 1487 }; 1488 1489 #define RAPL_EVENT_ATTR_STR(_name, v, str) \ 1490 static struct perf_pmu_events_attr event_attr_##v = { \ 1491 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \ 1492 .event_str = str, \ 1493 } 1494 1495 RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01"); 1496 RAPL_EVENT_ATTR_STR(energy-pkg, rapl_pkg, "event=0x02"); 1497 RAPL_EVENT_ATTR_STR(energy-ram, rapl_ram, "event=0x03"); 1498 RAPL_EVENT_ATTR_STR(energy-gpu, rapl_gpu, "event=0x04"); 1499 RAPL_EVENT_ATTR_STR(energy-psys, rapl_psys, "event=0x05"); 1500 1501 RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_unit_cores, "Joules"); 1502 RAPL_EVENT_ATTR_STR(energy-pkg.unit, rapl_unit_pkg, "Joules"); 1503 RAPL_EVENT_ATTR_STR(energy-ram.unit, rapl_unit_ram, "Joules"); 1504 RAPL_EVENT_ATTR_STR(energy-gpu.unit, rapl_unit_gpu, "Joules"); 1505 RAPL_EVENT_ATTR_STR(energy-psys.unit, rapl_unit_psys, "Joules"); 1506 1507 RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_scale_cores, "2.3283064365386962890625e-10"); 1508 RAPL_EVENT_ATTR_STR(energy-pkg.scale, rapl_scale_pkg, "2.3283064365386962890625e-10"); 1509 RAPL_EVENT_ATTR_STR(energy-ram.scale, rapl_scale_ram, "2.3283064365386962890625e-10"); 1510 RAPL_EVENT_ATTR_STR(energy-gpu.scale, rapl_scale_gpu, "2.3283064365386962890625e-10"); 1511 RAPL_EVENT_ATTR_STR(energy-psys.scale, rapl_scale_psys, "2.3283064365386962890625e-10"); 1512 1513 #define RAPL_EVENT_GROUP(_name, domain) \ 1514 static struct attribute *pmu_attr_##_name[] = { \ 1515 &event_attr_rapl_##_name.attr.attr, \ 1516 &event_attr_rapl_unit_##_name.attr.attr, \ 1517 &event_attr_rapl_scale_##_name.attr.attr, \ 1518 NULL \ 1519 }; \ 1520 static umode_t is_visible_##_name(struct kobject *kobj, struct attribute *attr, int event) \ 1521 { \ 1522 return rapl_pmu.domain_map & BIT(domain) ? attr->mode : 0; \ 1523 } \ 1524 static struct attribute_group pmu_group_##_name = { \ 1525 .name = "events", \ 1526 .attrs = pmu_attr_##_name, \ 1527 .is_visible = is_visible_##_name, \ 1528 } 1529 1530 RAPL_EVENT_GROUP(cores, RAPL_DOMAIN_PP0); 1531 RAPL_EVENT_GROUP(pkg, RAPL_DOMAIN_PACKAGE); 1532 RAPL_EVENT_GROUP(ram, RAPL_DOMAIN_DRAM); 1533 RAPL_EVENT_GROUP(gpu, RAPL_DOMAIN_PP1); 1534 RAPL_EVENT_GROUP(psys, RAPL_DOMAIN_PLATFORM); 1535 1536 static const struct attribute_group *pmu_attr_update[] = { 1537 &pmu_group_cores, 1538 &pmu_group_pkg, 1539 &pmu_group_ram, 1540 &pmu_group_gpu, 1541 &pmu_group_psys, 1542 NULL 1543 }; 1544 1545 static int rapl_pmu_update(struct rapl_package *rp) 1546 { 1547 int ret = 0; 1548 1549 /* Return if PMU already covers all events supported by current RAPL Package */ 1550 if (rapl_pmu.registered && !(rp->domain_map & (~rapl_pmu.domain_map))) 1551 goto end; 1552 1553 /* Unregister previous registered PMU */ 1554 if (rapl_pmu.registered) 1555 perf_pmu_unregister(&rapl_pmu.pmu); 1556 1557 rapl_pmu.registered = false; 1558 rapl_pmu.domain_map |= rp->domain_map; 1559 1560 memset(&rapl_pmu.pmu, 0, sizeof(struct pmu)); 1561 rapl_pmu.pmu.attr_groups = pmu_attr_groups; 1562 rapl_pmu.pmu.attr_update = pmu_attr_update; 1563 rapl_pmu.pmu.task_ctx_nr = perf_invalid_context; 1564 rapl_pmu.pmu.event_init = rapl_pmu_event_init; 1565 rapl_pmu.pmu.add = rapl_pmu_event_add; 1566 rapl_pmu.pmu.del = rapl_pmu_event_del; 1567 rapl_pmu.pmu.start = rapl_pmu_event_start; 1568 rapl_pmu.pmu.stop = rapl_pmu_event_stop; 1569 rapl_pmu.pmu.read = rapl_pmu_event_read; 1570 rapl_pmu.pmu.module = THIS_MODULE; 1571 rapl_pmu.pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT; 1572 ret = perf_pmu_register(&rapl_pmu.pmu, "power", -1); 1573 if (ret) { 1574 pr_info("Failed to register PMU\n"); 1575 return ret; 1576 } 1577 1578 rapl_pmu.registered = true; 1579 end: 1580 rp->has_pmu = true; 1581 return ret; 1582 } 1583 1584 int rapl_package_add_pmu_locked(struct rapl_package *rp) 1585 { 1586 struct rapl_package_pmu_data *data = &rp->pmu_data; 1587 int idx; 1588 1589 if (rp->has_pmu) 1590 return -EEXIST; 1591 1592 for (idx = 0; idx < rp->nr_domains; idx++) { 1593 struct rapl_domain *rd = &rp->domains[idx]; 1594 int domain = rd->id; 1595 u64 val; 1596 1597 if (!test_bit(domain, &rp->domain_map)) 1598 continue; 1599 1600 /* 1601 * The RAPL PMU granularity is 2^-32 Joules 1602 * data->scale[]: times of 2^-32 Joules for each ENERGY COUNTER increase 1603 */ 1604 val = rd->energy_unit * (1ULL << 32); 1605 do_div(val, ENERGY_UNIT_SCALE * 1000000); 1606 data->scale[domain] = val; 1607 1608 if (!rapl_pmu.timer_ms) { 1609 struct rapl_primitive_info *rpi = get_rpi(rp, ENERGY_COUNTER); 1610 1611 /* 1612 * Calculate the timer rate: 1613 * Use reference of 200W for scaling the timeout to avoid counter 1614 * overflows. 1615 * 1616 * max_count = rpi->mask >> rpi->shift + 1 1617 * max_energy_pj = max_count * rd->energy_unit 1618 * max_time_sec = (max_energy_pj / 1000000000) / 200w 1619 * 1620 * rapl_pmu.timer_ms = max_time_sec * 1000 / 2 1621 */ 1622 val = (rpi->mask >> rpi->shift) + 1; 1623 val *= rd->energy_unit; 1624 do_div(val, 1000000 * 200 * 2); 1625 rapl_pmu.timer_ms = val; 1626 1627 pr_debug("%llu ms overflow timer\n", rapl_pmu.timer_ms); 1628 } 1629 1630 pr_debug("Domain %s: hw unit %lld * 2^-32 Joules\n", rd->name, data->scale[domain]); 1631 } 1632 1633 /* Initialize per package PMU data */ 1634 raw_spin_lock_init(&data->lock); 1635 INIT_LIST_HEAD(&data->active_list); 1636 data->timer_interval = ms_to_ktime(rapl_pmu.timer_ms); 1637 hrtimer_setup(&data->hrtimer, rapl_hrtimer_handle, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1638 1639 return rapl_pmu_update(rp); 1640 } 1641 EXPORT_SYMBOL_NS_GPL(rapl_package_add_pmu_locked, "INTEL_RAPL"); 1642 1643 int rapl_package_add_pmu(struct rapl_package *rp) 1644 { 1645 guard(cpus_read_lock)(); 1646 1647 return rapl_package_add_pmu_locked(rp); 1648 } 1649 EXPORT_SYMBOL_NS_GPL(rapl_package_add_pmu, "INTEL_RAPL"); 1650 1651 void rapl_package_remove_pmu_locked(struct rapl_package *rp) 1652 { 1653 struct rapl_package *pos; 1654 1655 if (!rp->has_pmu) 1656 return; 1657 1658 list_for_each_entry(pos, &rapl_packages, plist) { 1659 /* PMU is still needed */ 1660 if (pos->has_pmu && pos != rp) 1661 return; 1662 } 1663 1664 if (rapl_pmu.registered) 1665 perf_pmu_unregister(&rapl_pmu.pmu); 1666 1667 memset(&rapl_pmu, 0, sizeof(struct rapl_pmu)); 1668 } 1669 EXPORT_SYMBOL_NS_GPL(rapl_package_remove_pmu_locked, "INTEL_RAPL"); 1670 1671 void rapl_package_remove_pmu(struct rapl_package *rp) 1672 { 1673 guard(cpus_read_lock)(); 1674 1675 rapl_package_remove_pmu_locked(rp); 1676 } 1677 EXPORT_SYMBOL_NS_GPL(rapl_package_remove_pmu, "INTEL_RAPL"); 1678 #endif 1679 1680 /* called from CPU hotplug notifier, hotplug lock held */ 1681 void rapl_remove_package_cpuslocked(struct rapl_package *rp) 1682 { 1683 struct rapl_domain *rd, *rd_package = NULL; 1684 1685 package_power_limit_irq_restore(rp); 1686 1687 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { 1688 int i; 1689 1690 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) { 1691 rapl_write_pl_data(rd, i, PL_ENABLE, 0); 1692 rapl_write_pl_data(rd, i, PL_CLAMP, 0); 1693 } 1694 1695 if (rd->id == RAPL_DOMAIN_PACKAGE) { 1696 rd_package = rd; 1697 continue; 1698 } 1699 pr_debug("remove package, undo power limit on %s: %s\n", 1700 rp->name, rd->name); 1701 powercap_unregister_zone(rp->priv->control_type, 1702 &rd->power_zone); 1703 } 1704 /* do parent zone last */ 1705 powercap_unregister_zone(rp->priv->control_type, 1706 &rd_package->power_zone); 1707 list_del(&rp->plist); 1708 kfree(rp); 1709 } 1710 EXPORT_SYMBOL_NS_GPL(rapl_remove_package_cpuslocked, "INTEL_RAPL"); 1711 1712 void rapl_remove_package(struct rapl_package *rp) 1713 { 1714 guard(cpus_read_lock)(); 1715 rapl_remove_package_cpuslocked(rp); 1716 } 1717 EXPORT_SYMBOL_NS_GPL(rapl_remove_package, "INTEL_RAPL"); 1718 1719 /* 1720 * RAPL Package energy counter scope: 1721 * 1. AMD/HYGON platforms use per-PKG package energy counter 1722 * 2. For Intel platforms 1723 * 2.1 CLX-AP platform has per-DIE package energy counter 1724 * 2.2 Other platforms that uses MSR RAPL are single die systems so the 1725 * package energy counter can be considered as per-PKG/per-DIE, 1726 * here it is considered as per-DIE. 1727 * 2.3 New platforms that use TPMI RAPL doesn't care about the 1728 * scope because they are not MSR/CPU based. 1729 */ 1730 #define rapl_msrs_are_pkg_scope() \ 1731 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || \ 1732 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) 1733 1734 /* caller to ensure CPU hotplug lock is held */ 1735 struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_priv *priv, 1736 bool id_is_cpu) 1737 { 1738 struct rapl_package *rp; 1739 int uid; 1740 1741 if (id_is_cpu) { 1742 uid = rapl_msrs_are_pkg_scope() ? 1743 topology_physical_package_id(id) : topology_logical_die_id(id); 1744 if (uid < 0) { 1745 pr_err("topology_logical_(package/die)_id() returned a negative value"); 1746 return NULL; 1747 } 1748 } 1749 else 1750 uid = id; 1751 1752 list_for_each_entry(rp, &rapl_packages, plist) { 1753 if (rp->id == uid 1754 && rp->priv->control_type == priv->control_type) 1755 return rp; 1756 } 1757 1758 return NULL; 1759 } 1760 EXPORT_SYMBOL_NS_GPL(rapl_find_package_domain_cpuslocked, "INTEL_RAPL"); 1761 1762 struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu) 1763 { 1764 guard(cpus_read_lock)(); 1765 return rapl_find_package_domain_cpuslocked(id, priv, id_is_cpu); 1766 } 1767 EXPORT_SYMBOL_NS_GPL(rapl_find_package_domain, "INTEL_RAPL"); 1768 1769 /* called from CPU hotplug notifier, hotplug lock held */ 1770 struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, bool id_is_cpu) 1771 { 1772 struct rapl_package *rp; 1773 int ret; 1774 1775 rp = kzalloc_obj(struct rapl_package); 1776 if (!rp) 1777 return ERR_PTR(-ENOMEM); 1778 1779 if (id_is_cpu) { 1780 rp->id = rapl_msrs_are_pkg_scope() ? 1781 topology_physical_package_id(id) : topology_logical_die_id(id); 1782 if ((int)(rp->id) < 0) { 1783 pr_err("topology_logical_(package/die)_id() returned a negative value"); 1784 ret = -EINVAL; 1785 goto err_free_package; 1786 } 1787 rp->lead_cpu = id; 1788 if (!rapl_msrs_are_pkg_scope() && topology_max_dies_per_package() > 1) 1789 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d-die-%d", 1790 topology_physical_package_id(id), topology_die_id(id)); 1791 else 1792 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", 1793 topology_physical_package_id(id)); 1794 } else { 1795 rp->id = id; 1796 rp->lead_cpu = -1; 1797 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", id); 1798 } 1799 1800 rp->priv = priv; 1801 ret = rapl_config(rp); 1802 if (ret) 1803 goto err_free_package; 1804 1805 /* check if the package contains valid domains */ 1806 if (rapl_detect_domains(rp)) { 1807 ret = -ENODEV; 1808 goto err_free_package; 1809 } 1810 ret = rapl_package_register_powercap(rp); 1811 if (!ret) { 1812 INIT_LIST_HEAD(&rp->plist); 1813 list_add(&rp->plist, &rapl_packages); 1814 return rp; 1815 } 1816 1817 err_free_package: 1818 kfree(rp->domains); 1819 kfree(rp); 1820 return ERR_PTR(ret); 1821 } 1822 EXPORT_SYMBOL_NS_GPL(rapl_add_package_cpuslocked, "INTEL_RAPL"); 1823 1824 struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu) 1825 { 1826 guard(cpus_read_lock)(); 1827 return rapl_add_package_cpuslocked(id, priv, id_is_cpu); 1828 } 1829 EXPORT_SYMBOL_NS_GPL(rapl_add_package, "INTEL_RAPL"); 1830 1831 static void power_limit_state_save(void) 1832 { 1833 struct rapl_package *rp; 1834 struct rapl_domain *rd; 1835 int ret, i; 1836 1837 cpus_read_lock(); 1838 list_for_each_entry(rp, &rapl_packages, plist) { 1839 if (!rp->power_zone) 1840 continue; 1841 rd = power_zone_to_rapl_domain(rp->power_zone); 1842 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) { 1843 ret = rapl_read_pl_data(rd, i, PL_LIMIT, true, 1844 &rd->rpl[i].last_power_limit); 1845 if (ret) 1846 rd->rpl[i].last_power_limit = 0; 1847 } 1848 } 1849 cpus_read_unlock(); 1850 } 1851 1852 static void power_limit_state_restore(void) 1853 { 1854 struct rapl_package *rp; 1855 struct rapl_domain *rd; 1856 int i; 1857 1858 cpus_read_lock(); 1859 list_for_each_entry(rp, &rapl_packages, plist) { 1860 if (!rp->power_zone) 1861 continue; 1862 rd = power_zone_to_rapl_domain(rp->power_zone); 1863 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) 1864 if (rd->rpl[i].last_power_limit) 1865 rapl_write_pl_data(rd, i, PL_LIMIT, 1866 rd->rpl[i].last_power_limit); 1867 } 1868 cpus_read_unlock(); 1869 } 1870 1871 static int rapl_pm_callback(struct notifier_block *nb, 1872 unsigned long mode, void *_unused) 1873 { 1874 switch (mode) { 1875 case PM_SUSPEND_PREPARE: 1876 power_limit_state_save(); 1877 break; 1878 case PM_POST_SUSPEND: 1879 power_limit_state_restore(); 1880 break; 1881 } 1882 return NOTIFY_OK; 1883 } 1884 1885 static struct notifier_block rapl_pm_notifier = { 1886 .notifier_call = rapl_pm_callback, 1887 }; 1888 1889 static int __init rapl_init(void) 1890 { 1891 return register_pm_notifier(&rapl_pm_notifier); 1892 } 1893 1894 static void __exit rapl_exit(void) 1895 { 1896 unregister_pm_notifier(&rapl_pm_notifier); 1897 } 1898 1899 fs_initcall(rapl_init); 1900 module_exit(rapl_exit); 1901 1902 MODULE_DESCRIPTION("Intel Runtime Average Power Limit (RAPL) common code"); 1903 MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>"); 1904 MODULE_LICENSE("GPL v2"); 1905