1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (c) 2018-2021, Arm ltd.
46 	return (dev->bus == &cpu_subsys);  in _is_cpu_device()
53 	struct em_perf_domain *pd;  member
60 	struct em_dbg_info *em_dbg = s->private;			\
65 	table = em_perf_state_from_pd(em_dbg->pd);			\
66 	val = table[em_dbg->ps_id].name;				\
82 			       struct dentry *pd)  in em_debug_create_ps()  argument
89 	em_dbg[i].pd = em_pd;  in em_debug_create_ps()
99 	/* Create per-ps directory */  in em_debug_create_ps()
100 	d = debugfs_create_dir(name, pd);  in em_debug_create_ps()
115 	seq_printf(s, "%*pbl\n", cpumask_pr_args(to_cpumask(s->private)));  in em_debug_cpus_show()
123 	struct em_perf_domain *pd = s->private;  in em_debug_flags_show()  local
125 	seq_printf(s, "%#lx\n", pd->flags);  in em_debug_flags_show()
133 	struct em_perf_domain *pd = s->private;  in em_debug_id_show()  local
135 	seq_printf(s, "%d\n", pd->id);  in em_debug_id_show()
151 		debugfs_create_file("cpus", 0444, d, dev->em_pd->cpus,  in em_debug_create_pd()
154 	debugfs_create_file("flags", 0444, d, dev->em_pd,  in em_debug_create_pd()
157 	debugfs_create_file("id", 0444, d, dev->em_pd, &em_debug_id_fops);  in em_debug_create_pd()
159 	em_dbg = devm_kcalloc(dev, dev->em_pd->nr_perf_states,  in em_debug_create_pd()
164 	/* Create a sub-directory for each performance state */  in em_debug_create_pd()
165 	for (i = 0; i < dev->em_pd->nr_perf_states; i++)  in em_debug_create_pd()
166 		em_debug_create_ps(dev->em_pd, em_dbg, i, d);  in em_debug_create_pd()
195  * em_table_free() - Handles safe free of the EM table when needed
202 	kref_put(&table->kref, em_release_table_kref);  in em_table_free()
206  * em_table_alloc() - Allocate a new EM table
207  * @pd		: EM performance domain for which this must be done
213 struct em_perf_table *em_table_alloc(struct em_perf_domain *pd)  in em_table_alloc()  argument
218 	table_size = sizeof(struct em_perf_state) * pd->nr_perf_states;  in em_table_alloc()
224 	kref_init(&table->kref);  in em_table_alloc()
229 static void em_init_performance(struct device *dev, struct em_perf_domain *pd,  in em_init_performance()  argument
239 	cpu = cpumask_first(em_span_cpus(pd));  in em_init_performance()
246 	fmax = (u64) table[nr_states - 1].frequency;  in em_init_performance()
265 	for (i = nr_states - 1; i >= 0; i--) {  in em_compute_costs()
268 		if ((flags & EM_PERF_DOMAIN_ARTIFICIAL) && cb->get_cost) {  in em_compute_costs()
269 			ret = cb->get_cost(dev, table[i].frequency, &cost);  in em_compute_costs()
273 				return -EINVAL;  in em_compute_costs()
296  * em_dev_compute_costs() - Calculate cost values for new runtime EM table
315  * em_dev_update_perf_domain() - Update runtime EM table for a device
322  * from a non-sleeping context.
330 	struct em_perf_domain *pd;  in em_dev_update_perf_domain()  local
333 		return -EINVAL;  in em_dev_update_perf_domain()
338 	if (!dev->em_pd) {  in em_dev_update_perf_domain()
340 		return -EINVAL;  in em_dev_update_perf_domain()
342 	pd = dev->em_pd;  in em_dev_update_perf_domain()
344 	kref_get(&new_table->kref);  in em_dev_update_perf_domain()
346 	old_table = rcu_dereference_protected(pd->em_table,  in em_dev_update_perf_domain()
348 	rcu_assign_pointer(pd->em_table, new_table);  in em_dev_update_perf_domain()
350 	em_cpufreq_update_efficiencies(dev, new_table->state);  in em_dev_update_perf_domain()
356 	em_notify_pd_updated(pd);  in em_dev_update_perf_domain()
361 static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd,  in em_create_perf_table()  argument
367 	int nr_states = pd->nr_perf_states;  in em_create_perf_table()
377 		ret = cb->active_power(dev, &power, &freq);  in em_create_perf_table()
381 			return -EINVAL;  in em_create_perf_table()
389 			dev_err(dev, "EM: non-increasing freq: %lu\n",  in em_create_perf_table()
391 			return -EINVAL;  in em_create_perf_table()
401 			return -EINVAL;  in em_create_perf_table()
408 	em_init_performance(dev, pd, table, nr_states);  in em_create_perf_table()
412 		return -EINVAL;  in em_create_perf_table()
423 	struct em_perf_domain *pd;  in em_create_pd()  local
433 			return -EINVAL;  in em_create_pd()
436 		pd = kzalloc(sizeof(*pd) + cpumask_size(), GFP_KERNEL);  in em_create_pd()
437 		if (!pd)  in em_create_pd()
438 			return -ENOMEM;  in em_create_pd()
440 		cpumask_copy(em_span_cpus(pd), cpus);  in em_create_pd()
442 		pd = kzalloc_obj(*pd);  in em_create_pd()
443 		if (!pd)  in em_create_pd()
444 			return -ENOMEM;  in em_create_pd()
447 	pd->nr_perf_states = nr_states;  in em_create_pd()
449 	INIT_LIST_HEAD(&pd->node);  in em_create_pd()
453 		kfree(pd);  in em_create_pd()
456 	pd->id = id;  in em_create_pd()
458 	em_table = em_table_alloc(pd);  in em_create_pd()
462 	ret = em_create_perf_table(dev, pd, em_table->state, cb, flags);  in em_create_pd()
466 	rcu_assign_pointer(pd->em_table, em_table);  in em_create_pd()
471 			cpu_dev->em_pd = pd;  in em_create_pd()
474 	dev->em_pd = pd;  in em_create_pd()
481 	kfree(pd);  in em_create_pd()
483 	return -EINVAL;  in em_create_pd()
489 	struct em_perf_domain *pd = dev->em_pd;  in em_cpufreq_update_efficiencies()  local
497 	/* Try to get a CPU which is active and in this PD */  in em_cpufreq_update_efficiencies()
498 	cpu = cpumask_first_and(em_span_cpus(pd), cpu_active_mask);  in em_cpufreq_update_efficiencies()
510 	for (i = 0; i < pd->nr_perf_states; i++) {  in em_cpufreq_update_efficiencies()
527 	pd->flags |= EM_PERF_DOMAIN_SKIP_INEFFICIENCIES;  in em_cpufreq_update_efficiencies()
531  * em_pd_get() - Return the performance domain for a device
542 	return dev->em_pd;  in em_pd_get()
547  * em_cpu_get() - Return the performance domain for a CPU
566  * em_dev_register_perf_domain() - Register the Energy Model (EM) for a device
571  *		obligatory. It can be taken from i.e. 'policy->cpus'. For other
573  * @microwatts	: Flag indicating that the power values are in micro-Watts or
580  * sub-systems might rely on this flag and check if all devices in the EM are
602  * em_dev_register_pd_no_update() - Register a perf domain for a device
603  * @dev : Device to register the PD for
604  * @nr_states : Number of performance states in the new PD
606  * @cpus : CPUs to include in the new PD (mandatory if @dev is a CPU device)
610  * update after registering the PD, even if @dev is a CPU device.
622 		return -EINVAL;  in em_dev_register_pd_no_update()
626 	 * let the driver-defined callback functions sleep.  in em_dev_register_pd_no_update()
630 	if (dev->em_pd) {  in em_dev_register_pd_no_update()
631 		ret = -EEXIST;  in em_dev_register_pd_no_update()
638 			ret = -EINVAL;  in em_dev_register_pd_no_update()
645 				ret = -EEXIST;  in em_dev_register_pd_no_update()
650 			 * micro-architecture since they all share the same  in em_dev_register_pd_no_update()
658 				ret = -EINVAL;  in em_dev_register_pd_no_update()
667 	else if (cb->get_cost)  in em_dev_register_pd_no_update()
677 		ret = -EINVAL;  in em_dev_register_pd_no_update()
685 	dev->em_pd->flags |= flags;  in em_dev_register_pd_no_update()
686 	dev->em_pd->min_perf_state = 0;  in em_dev_register_pd_no_update()
687 	dev->em_pd->max_perf_state = nr_states - 1;  in em_dev_register_pd_no_update()
689 	em_table = rcu_dereference_protected(dev->em_pd->em_table,  in em_dev_register_pd_no_update()
691 	em_cpufreq_update_efficiencies(dev, em_table->state);  in em_dev_register_pd_no_update()
702 	list_add_tail(&dev->em_pd->node, &em_pd_list);  in em_dev_register_pd_no_update()
705 	em_notify_pd_created(dev->em_pd);  in em_dev_register_pd_no_update()
712  * em_dev_unregister_perf_domain() - Unregister Energy Model (EM) for a device
719 	if (IS_ERR_OR_NULL(dev) || !dev->em_pd)  in em_dev_unregister_perf_domain()
726 	list_del_init(&dev->em_pd->node);  in em_dev_unregister_perf_domain()
729 	em_notify_pd_deleted(dev->em_pd);  in em_dev_unregister_perf_domain()
733 	 * from potential clean-up/setup issues in the debugfs directories.  in em_dev_unregister_perf_domain()
739 	em_table_free(rcu_dereference_protected(dev->em_pd->em_table,  in em_dev_unregister_perf_domain()
742 	ida_free(&em_pd_ida, dev->em_pd->id);  in em_dev_unregister_perf_domain()
744 	kfree(dev->em_pd);  in em_dev_unregister_perf_domain()
745 	dev->em_pd = NULL;  in em_dev_unregister_perf_domain()
750 static struct em_perf_table *em_table_dup(struct em_perf_domain *pd)  in em_table_dup()  argument
756 	em_table = em_table_alloc(pd);  in em_table_dup()
760 	new_ps = em_table->state;  in em_table_dup()
763 	ps = em_perf_state_from_pd(pd);  in em_table_dup()
765 	ps_size = sizeof(struct em_perf_state) * pd->nr_perf_states;  in em_table_dup()
773 static int em_recalc_and_update(struct device *dev, struct em_perf_domain *pd,  in em_recalc_and_update()  argument
778 	if (!em_is_artificial(pd)) {  in em_recalc_and_update()
779 		ret = em_compute_costs(dev, em_table->state, NULL,  in em_recalc_and_update()
780 				       pd->nr_perf_states, pd->flags);  in em_recalc_and_update()
790 	 * This is one-time-update, so give up the ownership in this updater.  in em_recalc_and_update()
804 				   struct em_perf_domain *pd)  in em_adjust_new_capacity()  argument
812 	table = em_perf_state_from_pd(pd);  in em_adjust_new_capacity()
813 	em_max_perf = table[pd->nr_perf_states - 1].performance;  in em_adjust_new_capacity()
822 	em_table = em_table_dup(pd);  in em_adjust_new_capacity()
828 	em_init_performance(dev, pd, em_table->state, pd->nr_perf_states);  in em_adjust_new_capacity()
830 	em_recalc_and_update(dev, pd, em_table);  in em_adjust_new_capacity()
834  * em_adjust_cpu_capacity() - Adjust the EM for a CPU after a capacity update.
844 	struct em_perf_domain *pd;  in em_adjust_cpu_capacity()  local
846 	pd = em_pd_get(dev);  in em_adjust_cpu_capacity()
847 	if (pd)  in em_adjust_cpu_capacity()
848 		em_adjust_new_capacity(cpu, dev, pd);  in em_adjust_cpu_capacity()
864 		struct em_perf_domain *pd;  in em_check_capacity_update()  local
878 		pd = em_pd_get(dev);  in em_check_capacity_update()
879 		if (!pd || em_is_artificial(pd))  in em_check_capacity_update()
883 			   em_span_cpus(pd));  in em_check_capacity_update()
885 		em_adjust_new_capacity(cpu, dev, pd);  in em_check_capacity_update()
900  * em_dev_update_chip_binning() - Update Energy Model after the new voltage
911 	struct em_perf_domain *pd;  in em_dev_update_chip_binning()  local
915 		return -EINVAL;  in em_dev_update_chip_binning()
917 	pd = em_pd_get(dev);  in em_dev_update_chip_binning()
918 	if (!pd) {  in em_dev_update_chip_binning()
920 		return -EINVAL;  in em_dev_update_chip_binning()
923 	em_table = em_table_dup(pd);  in em_dev_update_chip_binning()
926 		return -ENOMEM;  in em_dev_update_chip_binning()
930 	for (i = 0; i < pd->nr_perf_states; i++) {  in em_dev_update_chip_binning()
931 		unsigned long freq = em_table->state[i].frequency;  in em_dev_update_chip_binning()
940 		em_table->state[i].power = power;  in em_dev_update_chip_binning()
943 	return em_recalc_and_update(dev, pd, em_table);  in em_dev_update_chip_binning()
949  * em_update_performance_limits() - Update Energy Model with performance
951  * @pd			: Performance Domain with EM that has to be updated.
958  * Returns 0 on success or -EINVAL when failed.
960 int em_update_performance_limits(struct em_perf_domain *pd,  in em_update_performance_limits()  argument
964 	int min_ps = -1;  in em_update_performance_limits()
965 	int max_ps = -1;  in em_update_performance_limits()
968 	if (!pd)  in em_update_performance_limits()
969 		return -EINVAL;  in em_update_performance_limits()
972 	table = em_perf_state_from_pd(pd);  in em_update_performance_limits()
974 	for (i = 0; i < pd->nr_perf_states; i++) {  in em_update_performance_limits()
984 		return -EINVAL;  in em_update_performance_limits()
989 	pd->min_perf_state = min_ps;  in em_update_performance_limits()
990 	pd->max_perf_state = max_ps;  in em_update_performance_limits()
1018 	struct em_perf_domain *pd;  in for_each_em_perf_domain()  local
1023 	list_for_each_entry(pd, &em_pd_list, node) {  in for_each_em_perf_domain()
1026 		ret = cb(pd, data);  in for_each_em_perf_domain()
1036 	struct em_perf_domain *pd;  in em_perf_domain_get_by_id()  local
1041 	list_for_each_entry(pd, &em_pd_list, node) {  in em_perf_domain_get_by_id()
1042 		if (pd->id == id)  in em_perf_domain_get_by_id()
1043 			return pd;  in em_perf_domain_get_by_id()