135 	if (rd->rp->priv->type == RAPL_IF_TPMI) {  in get_pl_lock_prim()
152 	if (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2))  in get_pl_lock_prim()
163 		if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)  in get_pl_prim()
177 		if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)  in get_pl_prim()
220 static struct rapl_defaults *get_defaults(struct rapl_package *rp)  in get_defaults()  argument
222 	return rp->priv->defaults;  in get_defaults()
253 static void rapl_init_domains(struct rapl_package *rp);
268 static void package_power_limit_irq_save(struct rapl_package *rp);
314 	struct rapl_package *rp = rd->rp;  in release_zone()  local
321 		rp->domains = NULL;  in release_zone()
343 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in set_domain_enable()
458 	struct rapl_package *rp;  in set_power_limit()  local
465 	rp = rd->rp;  in set_power_limit()
469 		package_power_limit_irq_save(rp);  in set_power_limit()
581 static int get_rid(struct rapl_package *rp)  in get_rid()  argument
583 	return rp->lead_cpu >= 0 ? rp->lead_cpu : rp->id;  in get_rid()
587 static void rapl_init_domains(struct rapl_package *rp)  in rapl_init_domains()  argument
591 	struct rapl_domain *rd = rp->domains;  in rapl_init_domains()
594 		unsigned int mask = rp->domain_map & (1 << i);  in rapl_init_domains()
600 		rd->rp = rp;  in rapl_init_domains()
602 		if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) {  in rapl_init_domains()
604 				rp->lead_cpu >= 0 ? topology_physical_package_id(rp->lead_cpu) :  in rapl_init_domains()
605 				rp->id);  in rapl_init_domains()
614 		rp->priv->limits[i] |= BIT(POWER_LIMIT1);  in rapl_init_domains()
617 			if (rp->priv->limits[i] & BIT(t))  in rapl_init_domains()
622 			rd->regs[j] = rp->priv->regs[i][j];  in rapl_init_domains()
632 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in rapl_unit_xlate()
756 static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim)  in get_rpi()  argument
758 	struct rapl_primitive_info *rpi = rp->priv->rpi;  in get_rpi()
766 static int rapl_config(struct rapl_package *rp)  in rapl_config()  argument
768 	switch (rp->priv->type) {  in rapl_config()
772 		rp->priv->defaults = (void *)defaults_msr;  in rapl_config()
773 		rp->priv->rpi = (void *)rpi_msr;  in rapl_config()
776 		rp->priv->defaults = (void *)&defaults_tpmi;  in rapl_config()
777 		rp->priv->rpi = (void *)rpi_tpmi;  in rapl_config()
784 	if (!rp->priv->defaults || !rp->priv->rpi)  in rapl_config()
793 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in prim_fixups()
837 	struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed);  in rapl_read_data_raw()
855 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_read_data_raw()
856 		pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name);  in rapl_read_data_raw()
876 	struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed);  in rapl_write_data_raw()
894 	ret = rd->rp->priv->write_raw(get_rid(rd->rp), &ra);  in rapl_write_data_raw()
920 		pr_debug("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]);  in rapl_write_pl_data()
944 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_check_unit_core()
946 			ra.reg.val, rd->rp->name, rd->name);  in rapl_check_unit_core()
960 		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);  in rapl_check_unit_core()
972 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_check_unit_atom()
974 			ra.reg.val, rd->rp->name, rd->name);  in rapl_check_unit_atom()
988 		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);  in rapl_check_unit_atom()
996 	struct rapl_package *rp = (struct rapl_package *)info;  in power_limit_irq_save_cpu()  local
1000 	if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {  in power_limit_irq_save_cpu()
1001 		rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE;  in power_limit_irq_save_cpu()
1002 		rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;  in power_limit_irq_save_cpu()
1018 static void package_power_limit_irq_save(struct rapl_package *rp)  in package_power_limit_irq_save()  argument
1020 	if (rp->lead_cpu < 0)  in package_power_limit_irq_save()
1026 	smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1);  in package_power_limit_irq_save()
1033 static void package_power_limit_irq_restore(struct rapl_package *rp)  in package_power_limit_irq_restore()  argument
1037 	if (rp->lead_cpu < 0)  in package_power_limit_irq_restore()
1044 	if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))  in package_power_limit_irq_restore()
1049 	if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)  in package_power_limit_irq_restore()
1075 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in set_floor_freq_atom()
1159 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_check_unit_tpmi()
1161 			ra.reg.val, rd->rp->name, rd->name);  in rapl_check_unit_tpmi()
1175 		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);  in rapl_check_unit_tpmi()
1314 static void rapl_update_domain_data(struct rapl_package *rp)  in rapl_update_domain_data()  argument
1319 	for (dmn = 0; dmn < rp->nr_domains; dmn++) {  in rapl_update_domain_data()
1320 		pr_debug("update %s domain %s data\n", rp->name,  in rapl_update_domain_data()
1321 			 rp->domains[dmn].name);  in rapl_update_domain_data()
1324 			struct rapl_primitive_info *rpi = get_rpi(rp, prim);  in rapl_update_domain_data()
1326 			if (!rapl_read_data_raw(&rp->domains[dmn], prim,  in rapl_update_domain_data()
1328 				rp->domains[dmn].rdd.primitives[prim] = val;  in rapl_update_domain_data()
1334 static int rapl_package_register_powercap(struct rapl_package *rp)  in rapl_package_register_powercap()  argument
1341 	rapl_update_domain_data(rp);  in rapl_package_register_powercap()
1344 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_package_register_powercap()
1347 			pr_debug("register package domain %s\n", rp->name);  in rapl_package_register_powercap()
1349 					    rp->priv->control_type, rp->name,  in rapl_package_register_powercap()
1354 					 rp->name);  in rapl_package_register_powercap()
1358 			rp->power_zone = power_zone;  in rapl_package_register_powercap()
1368 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_package_register_powercap()
1369 		struct powercap_zone *parent = rp->power_zone;  in rapl_package_register_powercap()
1378 						    rp->priv->control_type,  in rapl_package_register_powercap()
1385 				 rp->name, rd->name);  in rapl_package_register_powercap()
1397 	while (--rd >= rp->domains) {  in rapl_package_register_powercap()
1398 		pr_debug("unregister %s domain %s\n", rp->name, rd->name);  in rapl_package_register_powercap()
1399 		powercap_unregister_zone(rp->priv->control_type,  in rapl_package_register_powercap()
1406 static int rapl_check_domain(int domain, struct rapl_package *rp)  in rapl_check_domain()  argument
1416 		ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS];  in rapl_check_domain()
1427 	if (rp->priv->read_raw(get_rid(rp), &ra) || !ra.value)  in rapl_check_domain()
1440 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in rapl_get_domain_unit()
1444 		if (!rd->rp->priv->reg_unit.val) {  in rapl_get_domain_unit()
1448 		rd->regs[RAPL_DOMAIN_REG_UNIT] = rd->rp->priv->reg_unit;  in rapl_get_domain_unit()
1486 					rd->rp->name, rd->name, pl_names[i]);  in rapl_detect_powerlimit()
1498 static int rapl_detect_domains(struct rapl_package *rp)  in rapl_detect_domains()  argument
1505 		if (!rapl_check_domain(i, rp)) {  in rapl_detect_domains()
1506 			rp->domain_map |= 1 << i;  in rapl_detect_domains()
1510 	rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);  in rapl_detect_domains()
1511 	if (!rp->nr_domains) {  in rapl_detect_domains()
1512 		pr_debug("no valid rapl domains found in %s\n", rp->name);  in rapl_detect_domains()
1515 	pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name);  in rapl_detect_domains()
1517 	rp->domains = kcalloc(rp->nr_domains, sizeof(struct rapl_domain),  in rapl_detect_domains()
1519 	if (!rp->domains)  in rapl_detect_domains()
1522 	rapl_init_domains(rp);  in rapl_detect_domains()
1524 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_detect_domains()
1587 static int get_pmu_cpu(struct rapl_package *rp)  in get_pmu_cpu()  argument
1591 	if (!rp->has_pmu)  in get_pmu_cpu()
1595 	if (rp->priv->type != RAPL_IF_TPMI)  in get_pmu_cpu()
1600 		if (topology_physical_package_id(cpu) == rp->id)  in get_pmu_cpu()
1606 static bool is_rp_pmu_cpu(struct rapl_package *rp, int cpu)  in is_rp_pmu_cpu()  argument
1608 	if (!rp->has_pmu)  in is_rp_pmu_cpu()
1612 	if (rp->priv->type != RAPL_IF_TPMI)  in is_rp_pmu_cpu()
1616 	return topology_physical_package_id(cpu) == rp->id;  in is_rp_pmu_cpu()
1621 	struct rapl_package *rp = event->pmu_private;  in event_to_pmu_data()  local
1623 	return &rp->pmu_data;  in event_to_pmu_data()
1630 	struct rapl_package *rp = event->pmu_private;  in event_read_counter()  local
1638 	ret = rapl_read_data_raw(&rp->domains[event->hw.idx], ENERGY_COUNTER, false, &val);  in event_read_counter()
1789 	struct rapl_package *pos, *rp = NULL;  in rapl_pmu_event_init()  local
1807 			rp = pos;  in rapl_pmu_event_init()
1811 	if (!rp)  in rapl_pmu_event_init()
1818 	event->pmu_private = rp;	/* Which package */  in rapl_pmu_event_init()
1823 	for (idx = 0; idx < rp->nr_domains; idx++) {  in rapl_pmu_event_init()
1824 		if (rp->domains[idx].id == domain) {  in rapl_pmu_event_init()
1878 	struct rapl_package *rp;  in cpumask_show()  local
1891 	list_for_each_entry(rp, &rapl_packages, plist) {  in cpumask_show()
1892 		cpu = get_pmu_cpu(rp);  in cpumask_show()
1990 static int rapl_pmu_update(struct rapl_package *rp)  in rapl_pmu_update()  argument
1995 	if (rapl_pmu.registered && !(rp->domain_map & (~rapl_pmu.domain_map)))  in rapl_pmu_update()
2003 	rapl_pmu.domain_map |= rp->domain_map;  in rapl_pmu_update()
2025 	rp->has_pmu = true;  in rapl_pmu_update()
2029 int rapl_package_add_pmu(struct rapl_package *rp)  in rapl_package_add_pmu()  argument
2031 	struct rapl_package_pmu_data *data = &rp->pmu_data;  in rapl_package_add_pmu()
2034 	if (rp->has_pmu)  in rapl_package_add_pmu()
2039 	for (idx = 0; idx < rp->nr_domains; idx++) {  in rapl_package_add_pmu()
2040 		struct rapl_domain *rd = &rp->domains[idx];  in rapl_package_add_pmu()
2044 		if (!test_bit(domain, &rp->domain_map))  in rapl_package_add_pmu()
2056 			struct rapl_primitive_info *rpi = get_rpi(rp, ENERGY_COUNTER);  in rapl_package_add_pmu()
2086 	return rapl_pmu_update(rp);  in rapl_package_add_pmu()
2090 void rapl_package_remove_pmu(struct rapl_package *rp)  in rapl_package_remove_pmu()  argument
2094 	if (!rp->has_pmu)  in rapl_package_remove_pmu()
2101 		if (pos->has_pmu && pos != rp)  in rapl_package_remove_pmu()
2112 void rapl_remove_package_cpuslocked(struct rapl_package *rp)  in rapl_remove_package_cpuslocked()  argument
2116 	package_power_limit_irq_restore(rp);  in rapl_remove_package_cpuslocked()
2118 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_remove_package_cpuslocked()
2131 			 rp->name, rd->name);  in rapl_remove_package_cpuslocked()
2132 		powercap_unregister_zone(rp->priv->control_type,  in rapl_remove_package_cpuslocked()
2136 	powercap_unregister_zone(rp->priv->control_type,  in rapl_remove_package_cpuslocked()
2138 	list_del(&rp->plist);  in rapl_remove_package_cpuslocked()
2139 	kfree(rp);  in rapl_remove_package_cpuslocked()
2143 void rapl_remove_package(struct rapl_package *rp)  in rapl_remove_package()  argument
2146 	rapl_remove_package_cpuslocked(rp);  in rapl_remove_package()
2169 	struct rapl_package *rp;  in rapl_find_package_domain_cpuslocked()  local
2183 	list_for_each_entry(rp, &rapl_packages, plist) {  in rapl_find_package_domain_cpuslocked()
2184 		if (rp->id == uid  in rapl_find_package_domain_cpuslocked()
2185 		    && rp->priv->control_type == priv->control_type)  in rapl_find_package_domain_cpuslocked()
2186 			return rp;  in rapl_find_package_domain_cpuslocked()
2203 	struct rapl_package *rp;  in rapl_add_package_cpuslocked()  local
2206 	rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);  in rapl_add_package_cpuslocked()
2207 	if (!rp)  in rapl_add_package_cpuslocked()
2211 		rp->id = rapl_msrs_are_pkg_scope() ?  in rapl_add_package_cpuslocked()
2213 		if ((int)(rp->id) < 0) {  in rapl_add_package_cpuslocked()
2217 		rp->lead_cpu = id;  in rapl_add_package_cpuslocked()
2219 			snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d-die-%d",  in rapl_add_package_cpuslocked()
2222 			snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d",  in rapl_add_package_cpuslocked()
2225 		rp->id = id;  in rapl_add_package_cpuslocked()
2226 		rp->lead_cpu = -1;  in rapl_add_package_cpuslocked()
2227 		snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", id);  in rapl_add_package_cpuslocked()
2230 	rp->priv = priv;  in rapl_add_package_cpuslocked()
2231 	ret = rapl_config(rp);  in rapl_add_package_cpuslocked()
2236 	if (rapl_detect_domains(rp)) {  in rapl_add_package_cpuslocked()
2240 	ret = rapl_package_register_powercap(rp);  in rapl_add_package_cpuslocked()
2242 		INIT_LIST_HEAD(&rp->plist);  in rapl_add_package_cpuslocked()
2243 		list_add(&rp->plist, &rapl_packages);  in rapl_add_package_cpuslocked()
2244 		return rp;  in rapl_add_package_cpuslocked()
2248 	kfree(rp->domains);  in rapl_add_package_cpuslocked()
2249 	kfree(rp);  in rapl_add_package_cpuslocked()
2263 	struct rapl_package *rp;  in power_limit_state_save()  local
2268 	list_for_each_entry(rp, &rapl_packages, plist) {  in power_limit_state_save()
2269 		if (!rp->power_zone)  in power_limit_state_save()
2271 		rd = power_zone_to_rapl_domain(rp->power_zone);  in power_limit_state_save()
2284 	struct rapl_package *rp;  in power_limit_state_restore()  local
2289 	list_for_each_entry(rp, &rapl_packages, plist) {  in power_limit_state_restore()
2290 		if (!rp->power_zone)  in power_limit_state_restore()
2292 		rd = power_zone_to_rapl_domain(rp->power_zone);  in power_limit_state_restore()