150 static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)  in cppc_cpufreq_cpu_fie_init()  argument
159 	for_each_cpu(cpu, policy->cpus) {  in cppc_cpufreq_cpu_fie_init()
162 		cppc_fi->cpu_data = policy->driver_data;  in cppc_cpufreq_cpu_fie_init()
183 	topology_set_scale_freq_source(sftd, policy->cpus);  in cppc_cpufreq_cpu_fie_init()
194 static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)  in cppc_cpufreq_cpu_fie_exit()  argument
203 	topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus);  in cppc_cpufreq_cpu_fie_exit()
205 	for_each_cpu(cpu, policy->related_cpus) {  in cppc_cpufreq_cpu_fie_exit()
276 static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)  in cppc_cpufreq_cpu_fie_init()  argument
280 static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)  in cppc_cpufreq_cpu_fie_exit()  argument
294 					 struct cpufreq_policy *policy,  in cppc_cpufreq_get_perf_limits()  argument
301 	min_freq = READ_ONCE(policy->min);  in cppc_cpufreq_get_perf_limits()
302 	max_freq = READ_ONCE(policy->max);  in cppc_cpufreq_get_perf_limits()
314 					    struct cpufreq_policy *policy)  in cppc_cpufreq_update_perf_limits()  argument
316 	cppc_cpufreq_get_perf_limits(cpu_data, policy,  in cppc_cpufreq_update_perf_limits()
321 static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,  in cppc_cpufreq_set_target()  argument
325 	struct cppc_cpudata *cpu_data = policy->driver_data;  in cppc_cpufreq_set_target()
326 	unsigned int cpu = policy->cpu;  in cppc_cpufreq_set_target()
332 	cppc_cpufreq_update_perf_limits(cpu_data, policy);  in cppc_cpufreq_set_target()
334 	freqs.old = policy->cur;  in cppc_cpufreq_set_target()
337 	cpufreq_freq_transition_begin(policy, &freqs);  in cppc_cpufreq_set_target()
339 	cpufreq_freq_transition_end(policy, &freqs, ret != 0);  in cppc_cpufreq_set_target()
348 static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy,  in cppc_cpufreq_fast_switch()  argument
351 	struct cppc_cpudata *cpu_data = policy->driver_data;  in cppc_cpufreq_fast_switch()
352 	unsigned int cpu = policy->cpu;  in cppc_cpufreq_fast_switch()
358 	cppc_cpufreq_update_perf_limits(cpu_data, policy);  in cppc_cpufreq_fast_switch()
370 static int cppc_verify_policy(struct cpufreq_policy_data *policy)  in cppc_verify_policy()  argument
372 	cpufreq_verify_within_cpu_limits(policy);  in cppc_verify_policy()
429 static unsigned int get_perf_level_count(struct cpufreq_policy *policy)  in get_perf_level_count()  argument
434 	int cpu = policy->cpu;  in get_perf_level_count()
436 	cpu_data = policy->driver_data;  in get_perf_level_count()
463 	struct cpufreq_policy *policy;  in cppc_get_cpu_power()  local
468 	policy = cpufreq_cpu_get_raw(cpu_dev->id);  in cppc_get_cpu_power()
469 	if (!policy)  in cppc_get_cpu_power()
472 	cpu_data = policy->driver_data;  in cppc_get_cpu_power()
533 	struct cpufreq_policy *policy;  in cppc_get_cpu_cost()  local
538 	policy = cpufreq_cpu_get_raw(cpu_dev->id);  in cppc_get_cpu_cost()
539 	if (!policy)  in cppc_get_cpu_cost()
542 	cpu_data = policy->driver_data;  in cppc_get_cpu_cost()
555 static void cppc_cpufreq_register_em(struct cpufreq_policy *policy)  in cppc_cpufreq_register_em()  argument
561 	cpu_data = policy->driver_data;  in cppc_cpufreq_register_em()
562 	em_dev_register_perf_domain(get_cpu_device(policy->cpu),  in cppc_cpufreq_register_em()
563 			get_perf_level_count(policy), &em_cb,  in cppc_cpufreq_register_em()
647 static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy)  in cppc_cpufreq_put_cpu_data()  argument
649 	struct cppc_cpudata *cpu_data = policy->driver_data;  in cppc_cpufreq_put_cpu_data()
653 	policy->driver_data = NULL;  in cppc_cpufreq_put_cpu_data()
656 static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)  in cppc_cpufreq_cpu_init()  argument
658 	unsigned int cpu = policy->cpu;  in cppc_cpufreq_cpu_init()
669 	policy->driver_data = cpu_data;  in cppc_cpufreq_cpu_init()
675 	policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf);  in cppc_cpufreq_cpu_init()
682 	policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf);  in cppc_cpufreq_cpu_init()
683 	policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, policy->boost_enabled ?  in cppc_cpufreq_cpu_init()
686 	policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu);  in cppc_cpufreq_cpu_init()
687 	policy->shared_type = cpu_data->shared_type;  in cppc_cpufreq_cpu_init()
689 	switch (policy->shared_type) {  in cppc_cpufreq_cpu_init()
700 		cpumask_copy(policy->cpus, cpu_data->shared_cpu_map);  in cppc_cpufreq_cpu_init()
704 			 policy->shared_type);  in cppc_cpufreq_cpu_init()
709 	policy->fast_switch_possible = cppc_allow_fast_switch(policy->cpus);  in cppc_cpufreq_cpu_init()
710 	policy->dvfs_possible_from_any_cpu = true;  in cppc_cpufreq_cpu_init()
717 		policy->boost_supported = true;  in cppc_cpufreq_cpu_init()
720 	policy->cur = cppc_perf_to_khz(caps, caps->highest_perf);  in cppc_cpufreq_cpu_init()
730 	cppc_cpufreq_cpu_fie_init(policy);  in cppc_cpufreq_cpu_init()
734 	cppc_cpufreq_put_cpu_data(policy);  in cppc_cpufreq_cpu_init()
738 static void cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy)  in cppc_cpufreq_cpu_exit()  argument
740 	struct cppc_cpudata *cpu_data = policy->driver_data;  in cppc_cpufreq_cpu_exit()
742 	unsigned int cpu = policy->cpu;  in cppc_cpufreq_cpu_exit()
745 	cppc_cpufreq_cpu_fie_exit(policy);  in cppc_cpufreq_cpu_exit()
754 	cppc_cpufreq_put_cpu_data(policy);  in cppc_cpufreq_cpu_exit()
803 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);  in cppc_cpufreq_get_rate()
809 	if (!policy)  in cppc_cpufreq_get_rate()
812 	cpu_data = policy->driver_data;  in cppc_cpufreq_get_rate()
845 static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)  in cppc_cpufreq_set_boost()  argument
847 	struct cppc_cpudata *cpu_data = policy->driver_data;  in cppc_cpufreq_set_boost()
851 		policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->highest_perf);  in cppc_cpufreq_set_boost()
853 		policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->nominal_perf);  in cppc_cpufreq_set_boost()
858 static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)  in show_freqdomain_cpus()  argument
860 	struct cppc_cpudata *cpu_data = policy->driver_data;  in show_freqdomain_cpus()
865 static ssize_t show_auto_select(struct cpufreq_policy *policy, char *buf)  in show_auto_select()  argument
870 	ret = cppc_get_auto_sel(policy->cpu, &val);  in show_auto_select()
882 static ssize_t store_auto_select(struct cpufreq_policy *policy,  in store_auto_select()  argument
885 	struct cppc_cpudata *cpu_data = policy->driver_data;  in store_auto_select()
893 	ret = cppc_set_auto_sel(policy->cpu, val);  in store_auto_select()
908 		cppc_cpufreq_update_perf_limits(cpu_data, policy);  in store_auto_select()
910 		ret = cppc_set_perf(policy->cpu, &cpu_data->perf_ctrls);  in store_auto_select()
914 			cppc_set_auto_sel(policy->cpu, false);  in store_auto_select()
956 static ssize_t show_##_name(struct cpufreq_policy *policy, char *buf)	\
958 	return cppc_cpufreq_sysfs_show_u64(policy->cpu, _get_func, buf);\
960 static ssize_t store_##_name(struct cpufreq_policy *policy,		\
963 	return cppc_cpufreq_sysfs_store_u64(policy->cpu, _set_func,	\
971 show_energy_performance_preference_val(struct cpufreq_policy *policy, char *buf)  in CPPC_CPUFREQ_ATTR_RW_U64()
973 	return cppc_cpufreq_sysfs_show_u64(policy->cpu, cppc_get_epp_perf, buf);  in CPPC_CPUFREQ_ATTR_RW_U64()
977 store_energy_performance_preference_val(struct cpufreq_policy *policy,  in store_energy_performance_preference_val()  argument
980 	struct cppc_cpudata *cpu_data = policy->driver_data;  in store_energy_performance_preference_val()
988 	ret = cppc_set_epp(policy->cpu, val);  in store_energy_performance_preference_val()
999 	struct cpufreq_policy *policy;  in cppc_get_perf_limited_filtered()  local
1007 	policy = cpufreq_cpu_get_raw(cpu);  in cppc_get_perf_limited_filtered()
1008 	if (!policy)  in cppc_get_perf_limited_filtered()
1011 	cpu_data = policy->driver_data;  in cppc_get_perf_limited_filtered()