Lines Matching +full:y +full:- +full:rp
1 // SPDX-License-Identifier: GPL-2.0-only
29 #include <asm/intel-family.h>
106 #define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
130 return rd->rpl[pl].name ? true : false; in is_pl_valid()
135 if (rd->rp->priv->type == RAPL_IF_TPMI) { in get_pl_lock_prim()
146 return -EINVAL; in get_pl_lock_prim()
152 if (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2)) in get_pl_lock_prim()
163 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI) in get_pl_prim()
173 return -EINVAL; in get_pl_prim()
177 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI) in get_pl_prim()
187 return -EINVAL; in get_pl_prim()
198 return -EINVAL; in get_pl_prim()
200 return -EINVAL; in get_pl_prim()
220 static struct rapl_defaults *get_defaults(struct rapl_package *rp) in get_defaults() argument
222 return rp->priv->defaults; in get_defaults()
253 static void rapl_init_domains(struct rapl_package *rp);
268 static void package_power_limit_irq_save(struct rapl_package *rp);
300 return -EIO; in get_energy_counter()
314 struct rapl_package *rp = rd->rp; in release_zone() local
319 if (rd->id == RAPL_DOMAIN_PACKAGE) { in release_zone()
321 rp->domains = NULL; in release_zone()
343 struct rapl_defaults *defaults = get_defaults(rd->rp); in set_domain_enable()
357 pr_debug("%s cannot be %s\n", power_zone->name, in set_domain_enable()
362 if (defaults->set_floor_freq) in set_domain_enable()
363 defaults->set_floor_freq(rd, mode); in set_domain_enable()
377 if (rd->rpl[POWER_LIMIT1].locked) { in get_domain_enable()
436 * index in that some PLs maybe missing due to non-existent MSRs. So we
451 return -EINVAL; in contraint_to_pl()
458 struct rapl_package *rp; in set_power_limit() local
465 rp = rd->rp; in set_power_limit()
469 package_power_limit_irq_save(rp); in set_power_limit()
542 return rd->rpl[id].name; in get_constraint_name()
581 static int get_rid(struct rapl_package *rp) in get_rid() argument
583 return rp->lead_cpu >= 0 ? rp->lead_cpu : rp->id; in get_rid()
587 static void rapl_init_domains(struct rapl_package *rp) in rapl_init_domains() argument
591 struct rapl_domain *rd = rp->domains; in rapl_init_domains()
594 unsigned int mask = rp->domain_map & (1 << i); in rapl_init_domains()
600 rd->rp = rp; in rapl_init_domains()
602 if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) { in rapl_init_domains()
603 snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "psys-%d", in rapl_init_domains()
604 rp->lead_cpu >= 0 ? topology_physical_package_id(rp->lead_cpu) : in rapl_init_domains()
605 rp->id); in rapl_init_domains()
607 snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "%s", in rapl_init_domains()
611 rd->id = i; in rapl_init_domains()
614 rp->priv->limits[i] |= BIT(POWER_LIMIT1); in rapl_init_domains()
617 if (rp->priv->limits[i] & BIT(t)) in rapl_init_domains()
618 rd->rpl[t].name = pl_names[t]; in rapl_init_domains()
622 rd->regs[j] = rp->priv->regs[i][j]; in rapl_init_domains()
632 struct rapl_defaults *defaults = get_defaults(rd->rp); in rapl_unit_xlate()
637 units = rd->power_unit; in rapl_unit_xlate()
641 units = rd->energy_unit; in rapl_unit_xlate()
644 return defaults->compute_time_window(rd, value, to_raw); in rapl_unit_xlate()
709 /* non-hardware */
751 /* non-hardware */
756 static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim) in get_rpi() argument
758 struct rapl_primitive_info *rpi = rp->priv->rpi; in get_rpi()
766 static int rapl_config(struct rapl_package *rp) in rapl_config() argument
768 switch (rp->priv->type) { in rapl_config()
772 rp->priv->defaults = (void *)defaults_msr; in rapl_config()
773 rp->priv->rpi = (void *)rpi_msr; in rapl_config()
776 rp->priv->defaults = (void *)&defaults_tpmi; in rapl_config()
777 rp->priv->rpi = (void *)rpi_tpmi; in rapl_config()
780 return -EINVAL; in rapl_config()
784 if (!rp->priv->defaults || !rp->priv->rpi) in rapl_config()
785 return -ENODEV; in rapl_config()
793 struct rapl_defaults *defaults = get_defaults(rd->rp); in prim_fixups()
795 if (!defaults->spr_psys_bits) in prim_fixups()
798 if (rd->id != RAPL_DOMAIN_PLATFORM) in prim_fixups()
822 * RAPL MSRs are non-architectual and are laid out not consistently across
826 * is pre-assigned based on RAPL unit MSRs read at init time.
827 * 63-------------------------- 31--------------------------- 0
829 * | |<- shift ----------------|
830 * 63-------------------------- 31--------------------------- 0
837 struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed); in rapl_read_data_raw()
840 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY) in rapl_read_data_raw()
841 return -EINVAL; in rapl_read_data_raw()
843 ra.reg = rd->regs[rpi->id]; in rapl_read_data_raw()
845 return -EINVAL; in rapl_read_data_raw()
847 /* non-hardware data are collected by the polling thread */ in rapl_read_data_raw()
848 if (rpi->flag & RAPL_PRIMITIVE_DERIVED) { in rapl_read_data_raw()
849 *data = rd->rdd.primitives[prim]; in rapl_read_data_raw()
853 ra.mask = rpi->mask; in rapl_read_data_raw()
855 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { in rapl_read_data_raw()
856 pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name); in rapl_read_data_raw()
857 return -EIO; in rapl_read_data_raw()
860 value = ra.value >> rpi->shift; in rapl_read_data_raw()
863 *data = rapl_unit_xlate(rd, rpi->unit, value, 0); in rapl_read_data_raw()
876 struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed); in rapl_write_data_raw()
881 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY) in rapl_write_data_raw()
882 return -EINVAL; in rapl_write_data_raw()
884 bits = rapl_unit_xlate(rd, rpi->unit, value, 1); in rapl_write_data_raw()
885 bits <<= rpi->shift; in rapl_write_data_raw()
886 bits &= rpi->mask; in rapl_write_data_raw()
890 ra.reg = rd->regs[rpi->id]; in rapl_write_data_raw()
891 ra.mask = rpi->mask; in rapl_write_data_raw()
894 ret = rd->rp->priv->write_raw(get_rid(rd->rp), &ra); in rapl_write_data_raw()
905 return -EINVAL; in rapl_read_pl_data()
917 return -EINVAL; in rapl_write_pl_data()
919 if (rd->rpl[pl].locked) { in rapl_write_pl_data()
920 pr_debug("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]); in rapl_write_pl_data()
921 return -EACCES; in rapl_write_pl_data()
942 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; in rapl_check_unit_core()
944 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { in rapl_check_unit_core()
946 ra.reg.val, rd->rp->name, rd->name); in rapl_check_unit_core()
947 return -ENODEV; in rapl_check_unit_core()
951 rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value); in rapl_check_unit_core()
954 rd->power_unit = 1000000 / (1 << value); in rapl_check_unit_core()
957 rd->time_unit = 1000000 / (1 << value); in rapl_check_unit_core()
960 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit); in rapl_check_unit_core()
970 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; in rapl_check_unit_atom()
972 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { in rapl_check_unit_atom()
974 ra.reg.val, rd->rp->name, rd->name); in rapl_check_unit_atom()
975 return -ENODEV; in rapl_check_unit_atom()
979 rd->energy_unit = ENERGY_UNIT_SCALE * 1 << value; in rapl_check_unit_atom()
982 rd->power_unit = (1 << value) * 1000; in rapl_check_unit_atom()
985 rd->time_unit = 1000000 / (1 << value); in rapl_check_unit_atom()
988 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit); in rapl_check_unit_atom()
996 struct rapl_package *rp = (struct rapl_package *)info; in power_limit_irq_save_cpu() local
1000 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) { in power_limit_irq_save_cpu()
1001 rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE; in power_limit_irq_save_cpu()
1002 rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED; in power_limit_irq_save_cpu()
1018 static void package_power_limit_irq_save(struct rapl_package *rp) in package_power_limit_irq_save() argument
1020 if (rp->lead_cpu < 0) in package_power_limit_irq_save()
1026 smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1); in package_power_limit_irq_save()
1033 static void package_power_limit_irq_restore(struct rapl_package *rp) in package_power_limit_irq_restore() argument
1037 if (rp->lead_cpu < 0) in package_power_limit_irq_restore()
1044 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) in package_power_limit_irq_restore()
1049 if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE) in package_power_limit_irq_restore()
1061 /* always enable clamp such that p-state can go below OS requested in set_floor_freq_default()
1075 struct rapl_defaults *defaults = get_defaults(rd->rp); in set_floor_freq_atom()
1078 if (!defaults->floor_freq_reg_addr) { in set_floor_freq_atom()
1085 defaults->floor_freq_reg_addr, in set_floor_freq_atom()
1093 defaults->floor_freq_reg_addr, mdata); in set_floor_freq_atom()
1099 u64 f, y; /* fraction and exp. used for time unit */ in rapl_compute_time_window_core() local
1102 * Special processing based on 2^Y*(1+F/4), refer in rapl_compute_time_window_core()
1107 y = value & 0x1f; in rapl_compute_time_window_core()
1108 value = (1 << y) * (4 + f) * rd->time_unit / 4; in rapl_compute_time_window_core()
1110 if (value < rd->time_unit) in rapl_compute_time_window_core()
1113 do_div(value, rd->time_unit); in rapl_compute_time_window_core()
1114 y = ilog2(value); in rapl_compute_time_window_core()
1120 if (y > 0x1f) in rapl_compute_time_window_core()
1123 f = div64_u64(4 * (value - (1ULL << y)), 1ULL << y); in rapl_compute_time_window_core()
1124 value = (y & 0x1f) | ((f & 0x3) << 5); in rapl_compute_time_window_core()
1137 return (value) ? value * rd->time_unit : rd->time_unit; in rapl_compute_time_window_atom()
1139 value = div64_u64(value, rd->time_unit); in rapl_compute_time_window_atom()
1157 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; in rapl_check_unit_tpmi()
1159 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { in rapl_check_unit_tpmi()
1161 ra.reg.val, rd->rp->name, rd->name); in rapl_check_unit_tpmi()
1162 return -ENODEV; in rapl_check_unit_tpmi()
1166 rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value); in rapl_check_unit_tpmi()
1169 rd->power_unit = 1000000 / (1 << value); in rapl_check_unit_tpmi()
1172 rd->time_unit = 1000000 / (1 << value); in rapl_check_unit_tpmi()
1175 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit); in rapl_check_unit_tpmi()
1315 static void rapl_update_domain_data(struct rapl_package *rp) in rapl_update_domain_data() argument
1320 for (dmn = 0; dmn < rp->nr_domains; dmn++) { in rapl_update_domain_data()
1321 pr_debug("update %s domain %s data\n", rp->name, in rapl_update_domain_data()
1322 rp->domains[dmn].name); in rapl_update_domain_data()
1323 /* exclude non-raw primitives */ in rapl_update_domain_data()
1325 struct rapl_primitive_info *rpi = get_rpi(rp, prim); in rapl_update_domain_data()
1327 if (!rapl_read_data_raw(&rp->domains[dmn], prim, in rapl_update_domain_data()
1328 rpi->unit, &val)) in rapl_update_domain_data()
1329 rp->domains[dmn].rdd.primitives[prim] = val; in rapl_update_domain_data()
1335 static int rapl_package_register_powercap(struct rapl_package *rp) in rapl_package_register_powercap() argument
1342 rapl_update_domain_data(rp); in rapl_package_register_powercap()
1345 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { in rapl_package_register_powercap()
1346 if (rd->id == RAPL_DOMAIN_PACKAGE) { in rapl_package_register_powercap()
1348 pr_debug("register package domain %s\n", rp->name); in rapl_package_register_powercap()
1349 power_zone = powercap_register_zone(&rd->power_zone, in rapl_package_register_powercap()
1350 rp->priv->control_type, rp->name, in rapl_package_register_powercap()
1351 NULL, &zone_ops[rd->id], nr_pl, in rapl_package_register_powercap()
1355 rp->name); in rapl_package_register_powercap()
1359 rp->power_zone = power_zone; in rapl_package_register_powercap()
1366 return -ENODEV; in rapl_package_register_powercap()
1369 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { in rapl_package_register_powercap()
1370 struct powercap_zone *parent = rp->power_zone; in rapl_package_register_powercap()
1372 if (rd->id == RAPL_DOMAIN_PACKAGE) in rapl_package_register_powercap()
1374 if (rd->id == RAPL_DOMAIN_PLATFORM) in rapl_package_register_powercap()
1378 power_zone = powercap_register_zone(&rd->power_zone, in rapl_package_register_powercap()
1379 rp->priv->control_type, in rapl_package_register_powercap()
1380 rd->name, parent, in rapl_package_register_powercap()
1381 &zone_ops[rd->id], nr_pl, in rapl_package_register_powercap()
1386 rp->name, rd->name); in rapl_package_register_powercap()
1398 while (--rd >= rp->domains) { in rapl_package_register_powercap()
1399 pr_debug("unregister %s domain %s\n", rp->name, rd->name); in rapl_package_register_powercap()
1400 powercap_unregister_zone(rp->priv->control_type, in rapl_package_register_powercap()
1401 &rd->power_zone); in rapl_package_register_powercap()
1407 static int rapl_check_domain(int domain, struct rapl_package *rp) in rapl_check_domain() argument
1417 ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS]; in rapl_check_domain()
1421 return -EINVAL; in rapl_check_domain()
1423 /* make sure domain counters are available and contains non-zero in rapl_check_domain()
1428 if (rp->priv->read_raw(get_rid(rp), &ra) || !ra.value) in rapl_check_domain()
1429 return -ENODEV; in rapl_check_domain()
1441 struct rapl_defaults *defaults = get_defaults(rd->rp); in rapl_get_domain_unit()
1444 if (!rd->regs[RAPL_DOMAIN_REG_UNIT].val) { in rapl_get_domain_unit()
1445 if (!rd->rp->priv->reg_unit.val) { in rapl_get_domain_unit()
1447 return -ENODEV; in rapl_get_domain_unit()
1449 rd->regs[RAPL_DOMAIN_REG_UNIT] = rd->rp->priv->reg_unit; in rapl_get_domain_unit()
1452 if (!defaults->check_unit) { in rapl_get_domain_unit()
1454 return -ENODEV; in rapl_get_domain_unit()
1457 ret = defaults->check_unit(rd); in rapl_get_domain_unit()
1461 if (rd->id == RAPL_DOMAIN_DRAM && defaults->dram_domain_energy_unit) in rapl_get_domain_unit()
1462 rd->energy_unit = defaults->dram_domain_energy_unit; in rapl_get_domain_unit()
1463 if (rd->id == RAPL_DOMAIN_PLATFORM && defaults->psys_domain_energy_unit) in rapl_get_domain_unit()
1464 rd->energy_unit = defaults->psys_domain_energy_unit; in rapl_get_domain_unit()
1470 * 1. Locked by BIOS, in this case we still provide read-only access so that
1485 rd->rpl[i].locked = true; in rapl_detect_powerlimit()
1487 rd->rp->name, rd->name, pl_names[i]); in rapl_detect_powerlimit()
1492 rd->rpl[i].name = NULL; in rapl_detect_powerlimit()
1499 static int rapl_detect_domains(struct rapl_package *rp) in rapl_detect_domains() argument
1506 if (!rapl_check_domain(i, rp)) { in rapl_detect_domains()
1507 rp->domain_map |= 1 << i; in rapl_detect_domains()
1511 rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX); in rapl_detect_domains()
1512 if (!rp->nr_domains) { in rapl_detect_domains()
1513 pr_debug("no valid rapl domains found in %s\n", rp->name); in rapl_detect_domains()
1514 return -ENODEV; in rapl_detect_domains()
1516 pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name); in rapl_detect_domains()
1518 rp->domains = kcalloc(rp->nr_domains, sizeof(struct rapl_domain), in rapl_detect_domains()
1520 if (!rp->domains) in rapl_detect_domains()
1521 return -ENOMEM; in rapl_detect_domains()
1523 rapl_init_domains(rp); in rapl_detect_domains()
1525 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { in rapl_detect_domains()
1556 * 2.3283064365386962890625e-10
1558 * energy units. Use 2.3283064365386962890625e-10 (2^-32) Joules as
1588 static int get_pmu_cpu(struct rapl_package *rp) in get_pmu_cpu() argument
1592 if (!rp->has_pmu) in get_pmu_cpu()
1596 if (rp->priv->type != RAPL_IF_TPMI) in get_pmu_cpu()
1601 if (topology_physical_package_id(cpu) == rp->id) in get_pmu_cpu()
1607 static bool is_rp_pmu_cpu(struct rapl_package *rp, int cpu) in is_rp_pmu_cpu() argument
1609 if (!rp->has_pmu) in is_rp_pmu_cpu()
1613 if (rp->priv->type != RAPL_IF_TPMI) in is_rp_pmu_cpu()
1617 return topology_physical_package_id(cpu) == rp->id; in is_rp_pmu_cpu()
1622 struct rapl_package *rp = event->pmu_private; in event_to_pmu_data() local
1624 return &rp->pmu_data; in event_to_pmu_data()
1631 struct rapl_package *rp = event->pmu_private; in event_read_counter() local
1636 if (event->hw.idx < 0) in event_read_counter()
1639 ret = rapl_read_data_raw(&rp->domains[event->hw.idx], ENERGY_COUNTER, false, &val); in event_read_counter()
1652 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) in __rapl_pmu_event_start()
1655 event->hw.state = 0; in __rapl_pmu_event_start()
1657 list_add_tail(&event->active_entry, &data->active_list); in __rapl_pmu_event_start()
1659 local64_set(&event->hw.prev_count, event_read_counter(event)); in __rapl_pmu_event_start()
1660 if (++data->n_active == 1) in __rapl_pmu_event_start()
1661 hrtimer_start(&data->hrtimer, data->timer_interval, in __rapl_pmu_event_start()
1670 raw_spin_lock_irqsave(&data->lock, flags); in rapl_pmu_event_start()
1672 raw_spin_unlock_irqrestore(&data->lock, flags); in rapl_pmu_event_start()
1677 struct hw_perf_event *hwc = &event->hw; in rapl_event_update()
1683 * Follow the generic code to drain hwc->prev_count. in rapl_event_update()
1686 prev_raw_count = local64_read(&hwc->prev_count); in rapl_event_update()
1689 } while (!local64_try_cmpxchg(&hwc->prev_count, in rapl_event_update()
1696 * (event-)time and add that to the generic event. in rapl_event_update()
1698 delta = new_raw_count - prev_raw_count; in rapl_event_update()
1701 * Scale delta to smallest unit (2^-32) in rapl_event_update()
1703 * or use ldexp(count, -32). in rapl_event_update()
1706 sdelta = delta * data->scale[event->hw.flags]; in rapl_event_update()
1708 local64_add(sdelta, &event->count); in rapl_event_update()
1716 struct hw_perf_event *hwc = &event->hw; in rapl_pmu_event_stop()
1719 raw_spin_lock_irqsave(&data->lock, flags); in rapl_pmu_event_stop()
1722 if (!(hwc->state & PERF_HES_STOPPED)) { in rapl_pmu_event_stop()
1723 WARN_ON_ONCE(data->n_active <= 0); in rapl_pmu_event_stop()
1724 if (--data->n_active == 0) in rapl_pmu_event_stop()
1725 hrtimer_cancel(&data->hrtimer); in rapl_pmu_event_stop()
1727 list_del(&event->active_entry); in rapl_pmu_event_stop()
1729 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); in rapl_pmu_event_stop()
1730 hwc->state |= PERF_HES_STOPPED; in rapl_pmu_event_stop()
1734 if ((mode & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { in rapl_pmu_event_stop()
1740 hwc->state |= PERF_HES_UPTODATE; in rapl_pmu_event_stop()
1743 raw_spin_unlock_irqrestore(&data->lock, flags); in rapl_pmu_event_stop()
1749 struct hw_perf_event *hwc = &event->hw; in rapl_pmu_event_add()
1752 raw_spin_lock_irqsave(&data->lock, flags); in rapl_pmu_event_add()
1754 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; in rapl_pmu_event_add()
1759 raw_spin_unlock_irqrestore(&data->lock, flags); in rapl_pmu_event_add()
1790 struct rapl_package *pos, *rp = NULL; in rapl_pmu_event_init() local
1791 u64 cfg = event->attr.config & RAPL_EVENT_MASK; in rapl_pmu_event_init()
1795 if (event->attr.type != event->pmu->type) in rapl_pmu_event_init()
1796 return -ENOENT; in rapl_pmu_event_init()
1800 return -EINVAL; in rapl_pmu_event_init()
1802 if (event->cpu < 0) in rapl_pmu_event_init()
1803 return -EINVAL; in rapl_pmu_event_init()
1807 if (is_rp_pmu_cpu(pos, event->cpu)) { in rapl_pmu_event_init()
1808 rp = pos; in rapl_pmu_event_init()
1812 if (!rp) in rapl_pmu_event_init()
1813 return -ENODEV; in rapl_pmu_event_init()
1818 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG; in rapl_pmu_event_init()
1819 event->pmu_private = rp; /* Which package */ in rapl_pmu_event_init()
1820 event->hw.flags = domain; /* Which domain */ in rapl_pmu_event_init()
1822 event->hw.idx = -1; in rapl_pmu_event_init()
1823 /* Find out the index in rp->domains[] to get domain pointer */ in rapl_pmu_event_init()
1824 for (idx = 0; idx < rp->nr_domains; idx++) { in rapl_pmu_event_init()
1825 if (rp->domains[idx].id == domain) { in rapl_pmu_event_init()
1826 event->hw.idx = idx; in rapl_pmu_event_init()
1846 if (!data->n_active) in rapl_hrtimer_handle()
1849 raw_spin_lock_irqsave(&data->lock, flags); in rapl_hrtimer_handle()
1851 list_for_each_entry(event, &data->active_list, active_entry) in rapl_hrtimer_handle()
1854 raw_spin_unlock_irqrestore(&data->lock, flags); in rapl_hrtimer_handle()
1856 hrtimer_forward_now(hrtimer, data->timer_interval); in rapl_hrtimer_handle()
1879 struct rapl_package *rp; in cpumask_show() local
1885 return -ENOMEM; in cpumask_show()
1892 list_for_each_entry(rp, &rapl_packages, plist) { in cpumask_show()
1893 cpu = get_pmu_cpu(rp); in cpumask_show()
1917 PMU_FORMAT_ATTR(event, "config:0-7");
1941 RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");
1942 RAPL_EVENT_ATTR_STR(energy-pkg, rapl_pkg, "event=0x02");
1943 RAPL_EVENT_ATTR_STR(energy-ram, rapl_ram, "event=0x03");
1944 RAPL_EVENT_ATTR_STR(energy-gpu, rapl_gpu, "event=0x04");
1945 RAPL_EVENT_ATTR_STR(energy-psys, rapl_psys, "event=0x05");
1947 RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_unit_cores, "Joules");
1948 RAPL_EVENT_ATTR_STR(energy-pkg.unit, rapl_unit_pkg, "Joules");
1949 RAPL_EVENT_ATTR_STR(energy-ram.unit, rapl_unit_ram, "Joules");
1950 RAPL_EVENT_ATTR_STR(energy-gpu.unit, rapl_unit_gpu, "Joules");
1951 RAPL_EVENT_ATTR_STR(energy-psys.unit, rapl_unit_psys, "Joules");
1953 RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_scale_cores, "2.3283064365386962890625e-10");
1954 RAPL_EVENT_ATTR_STR(energy-pkg.scale, rapl_scale_pkg, "2.3283064365386962890625e-10");
1955 RAPL_EVENT_ATTR_STR(energy-ram.scale, rapl_scale_ram, "2.3283064365386962890625e-10");
1956 RAPL_EVENT_ATTR_STR(energy-gpu.scale, rapl_scale_gpu, "2.3283064365386962890625e-10");
1957 RAPL_EVENT_ATTR_STR(energy-psys.scale, rapl_scale_psys, "2.3283064365386962890625e-10");
1968 return rapl_pmu.domain_map & BIT(domain) ? attr->mode : 0; \
1991 static int rapl_pmu_update(struct rapl_package *rp) in rapl_pmu_update() argument
1996 if (rapl_pmu.registered && !(rp->domain_map & (~rapl_pmu.domain_map))) in rapl_pmu_update()
2004 rapl_pmu.domain_map |= rp->domain_map; in rapl_pmu_update()
2018 ret = perf_pmu_register(&rapl_pmu.pmu, "power", -1); in rapl_pmu_update()
2026 rp->has_pmu = true; in rapl_pmu_update()
2030 int rapl_package_add_pmu(struct rapl_package *rp) in rapl_package_add_pmu() argument
2032 struct rapl_package_pmu_data *data = &rp->pmu_data; in rapl_package_add_pmu()
2035 if (rp->has_pmu) in rapl_package_add_pmu()
2036 return -EEXIST; in rapl_package_add_pmu()
2040 for (idx = 0; idx < rp->nr_domains; idx++) { in rapl_package_add_pmu()
2041 struct rapl_domain *rd = &rp->domains[idx]; in rapl_package_add_pmu()
2042 int domain = rd->id; in rapl_package_add_pmu()
2045 if (!test_bit(domain, &rp->domain_map)) in rapl_package_add_pmu()
2049 * The RAPL PMU granularity is 2^-32 Joules in rapl_package_add_pmu()
2050 * data->scale[]: times of 2^-32 Joules for each ENERGY COUNTER increase in rapl_package_add_pmu()
2052 val = rd->energy_unit * (1ULL << 32); in rapl_package_add_pmu()
2054 data->scale[domain] = val; in rapl_package_add_pmu()
2057 struct rapl_primitive_info *rpi = get_rpi(rp, ENERGY_COUNTER); in rapl_package_add_pmu()
2064 * max_count = rpi->mask >> rpi->shift + 1 in rapl_package_add_pmu()
2065 * max_energy_pj = max_count * rd->energy_unit in rapl_package_add_pmu()
2070 val = (rpi->mask >> rpi->shift) + 1; in rapl_package_add_pmu()
2071 val *= rd->energy_unit; in rapl_package_add_pmu()
2078 pr_debug("Domain %s: hw unit %lld * 2^-32 Joules\n", rd->name, data->scale[domain]); in rapl_package_add_pmu()
2082 raw_spin_lock_init(&data->lock); in rapl_package_add_pmu()
2083 INIT_LIST_HEAD(&data->active_list); in rapl_package_add_pmu()
2084 data->timer_interval = ms_to_ktime(rapl_pmu.timer_ms); in rapl_package_add_pmu()
2085 hrtimer_setup(&data->hrtimer, rapl_hrtimer_handle, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in rapl_package_add_pmu()
2087 return rapl_pmu_update(rp); in rapl_package_add_pmu()
2091 void rapl_package_remove_pmu(struct rapl_package *rp) in rapl_package_remove_pmu() argument
2095 if (!rp->has_pmu) in rapl_package_remove_pmu()
2102 if (pos->has_pmu && pos != rp) in rapl_package_remove_pmu()
2113 void rapl_remove_package_cpuslocked(struct rapl_package *rp) in rapl_remove_package_cpuslocked() argument
2117 package_power_limit_irq_restore(rp); in rapl_remove_package_cpuslocked()
2119 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { in rapl_remove_package_cpuslocked()
2127 if (rd->id == RAPL_DOMAIN_PACKAGE) { in rapl_remove_package_cpuslocked()
2132 rp->name, rd->name); in rapl_remove_package_cpuslocked()
2133 powercap_unregister_zone(rp->priv->control_type, in rapl_remove_package_cpuslocked()
2134 &rd->power_zone); in rapl_remove_package_cpuslocked()
2137 powercap_unregister_zone(rp->priv->control_type, in rapl_remove_package_cpuslocked()
2138 &rd_package->power_zone); in rapl_remove_package_cpuslocked()
2139 list_del(&rp->plist); in rapl_remove_package_cpuslocked()
2140 kfree(rp); in rapl_remove_package_cpuslocked()
2144 void rapl_remove_package(struct rapl_package *rp) in rapl_remove_package() argument
2147 rapl_remove_package_cpuslocked(rp); in rapl_remove_package()
2153 * 1. AMD/HYGON platforms use per-PKG package energy counter
2155 * 2.1 CLX-AP platform has per-DIE package energy counter
2157 * package energy counter can be considered as per-PKG/per-DIE,
2158 * here it is considered as per-DIE.
2170 struct rapl_package *rp; in rapl_find_package_domain_cpuslocked() local
2184 list_for_each_entry(rp, &rapl_packages, plist) { in rapl_find_package_domain_cpuslocked()
2185 if (rp->id == uid in rapl_find_package_domain_cpuslocked()
2186 && rp->priv->control_type == priv->control_type) in rapl_find_package_domain_cpuslocked()
2187 return rp; in rapl_find_package_domain_cpuslocked()
2204 struct rapl_package *rp; in rapl_add_package_cpuslocked() local
2207 rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL); in rapl_add_package_cpuslocked()
2208 if (!rp) in rapl_add_package_cpuslocked()
2209 return ERR_PTR(-ENOMEM); in rapl_add_package_cpuslocked()
2212 rp->id = rapl_msrs_are_pkg_scope() ? in rapl_add_package_cpuslocked()
2214 if ((int)(rp->id) < 0) { in rapl_add_package_cpuslocked()
2216 return ERR_PTR(-EINVAL); in rapl_add_package_cpuslocked()
2218 rp->lead_cpu = id; in rapl_add_package_cpuslocked()
2220 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d-die-%d", in rapl_add_package_cpuslocked()
2223 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", in rapl_add_package_cpuslocked()
2226 rp->id = id; in rapl_add_package_cpuslocked()
2227 rp->lead_cpu = -1; in rapl_add_package_cpuslocked()
2228 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", id); in rapl_add_package_cpuslocked()
2231 rp->priv = priv; in rapl_add_package_cpuslocked()
2232 ret = rapl_config(rp); in rapl_add_package_cpuslocked()
2237 if (rapl_detect_domains(rp)) { in rapl_add_package_cpuslocked()
2238 ret = -ENODEV; in rapl_add_package_cpuslocked()
2241 ret = rapl_package_register_powercap(rp); in rapl_add_package_cpuslocked()
2243 INIT_LIST_HEAD(&rp->plist); in rapl_add_package_cpuslocked()
2244 list_add(&rp->plist, &rapl_packages); in rapl_add_package_cpuslocked()
2245 return rp; in rapl_add_package_cpuslocked()
2249 kfree(rp->domains); in rapl_add_package_cpuslocked()
2250 kfree(rp); in rapl_add_package_cpuslocked()
2264 struct rapl_package *rp; in power_limit_state_save() local
2269 list_for_each_entry(rp, &rapl_packages, plist) { in power_limit_state_save()
2270 if (!rp->power_zone) in power_limit_state_save()
2272 rd = power_zone_to_rapl_domain(rp->power_zone); in power_limit_state_save()
2275 &rd->rpl[i].last_power_limit); in power_limit_state_save()
2277 rd->rpl[i].last_power_limit = 0; in power_limit_state_save()
2285 struct rapl_package *rp; in power_limit_state_restore() local
2290 list_for_each_entry(rp, &rapl_packages, plist) { in power_limit_state_restore()
2291 if (!rp->power_zone) in power_limit_state_restore()
2293 rd = power_zone_to_rapl_domain(rp->power_zone); in power_limit_state_restore()
2295 if (rd->rpl[i].last_power_limit) in power_limit_state_restore()
2297 rd->rpl[i].last_power_limit); in power_limit_state_restore()
2329 defaults_msr = (struct rapl_defaults *)id->driver_data; in rapl_init()
2333 return -ENOMEM; in rapl_init()