139 static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val)  in freq_to_perf()  argument
141 	u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq);  in freq_to_perf()
143 	return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf);  in freq_to_perf()
146 static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val)  in perf_to_freq()  argument
149 				perf.nominal_perf);  in perf_to_freq()
202 		pr_debug("Could not retrieve energy perf value (%d)\n", ret);  in msr_get_epp()
223 		pr_debug("Could not retrieve energy perf value (%d)\n", ret);  in shmem_get_epp()
246 		union perf_cached perf = READ_ONCE(cpudata->perf);  in msr_update_perf()  local
249 					  perf.highest_perf,  in msr_update_perf()
297 		union perf_cached perf = cpudata->perf;  in msr_set_epp()  local
299 		trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf,  in msr_set_epp()
314 		pr_err("failed to set energy perf value (%d)\n", ret);  in msr_set_epp()
342 		union perf_cached perf = cpudata->perf;  in shmem_set_epp()  local
344 		trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf,  in shmem_set_epp()
360 		pr_debug("failed to set energy perf value (%d)\n", ret);  in shmem_set_epp()
391 	union perf_cached perf = READ_ONCE(cpudata->perf);  in msr_init_perf()  local
418 		perf.bios_min_perf = min_perf;  in msr_init_perf()
420 	perf.highest_perf = numerator;  in msr_init_perf()
421 	perf.max_limit_perf = numerator;  in msr_init_perf()
422 	perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1);  in msr_init_perf()
423 	perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1);  in msr_init_perf()
424 	perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1);  in msr_init_perf()
425 	perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1);  in msr_init_perf()
426 	WRITE_ONCE(cpudata->perf, perf);  in msr_init_perf()
435 	union perf_cached perf = READ_ONCE(cpudata->perf);  in shmem_init_perf()  local
447 	perf.highest_perf = numerator;  in shmem_init_perf()
448 	perf.max_limit_perf = numerator;  in shmem_init_perf()
449 	perf.min_limit_perf = cppc_perf.lowest_perf;  in shmem_init_perf()
450 	perf.nominal_perf = cppc_perf.nominal_perf;  in shmem_init_perf()
451 	perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;  in shmem_init_perf()
452 	perf.lowest_perf = cppc_perf.lowest_perf;  in shmem_init_perf()
453 	WRITE_ONCE(cpudata->perf, perf);  in shmem_init_perf()
506 		union perf_cached perf = READ_ONCE(cpudata->perf);  in shmem_update_perf()  local
509 					  perf.highest_perf,  in shmem_update_perf()
570 	union perf_cached perf = READ_ONCE(cpudata->perf);  in amd_pstate_update()  local
575 	/* limit the max perf when core performance boost feature is disabled */  in amd_pstate_update()
577 		max_perf = min_t(u8, perf.nominal_perf, max_perf);  in amd_pstate_update()
581 	policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf);  in amd_pstate_update()
608 		union perf_cached perf;  in amd_pstate_verify()  local
614 		perf = READ_ONCE(cpudata->perf);  in amd_pstate_verify()
616 		if (perf.bios_min_perf)  in amd_pstate_verify()
617 			policy_data->min = perf_to_freq(perf, cpudata->nominal_freq,  in amd_pstate_verify()
618 							perf.bios_min_perf);  in amd_pstate_verify()
631 	union perf_cached perf = READ_ONCE(cpudata->perf);  in amd_pstate_update_min_max_limit()  local
633 	perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max);  in amd_pstate_update_min_max_limit()
637 		perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf);  in amd_pstate_update_min_max_limit()
640 		perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min);  in amd_pstate_update_min_max_limit()
644 	WRITE_ONCE(cpudata->perf, perf);  in amd_pstate_update_min_max_limit()
652 	union perf_cached perf;  in amd_pstate_update_freq()  local
660 	perf = READ_ONCE(cpudata->perf);  in amd_pstate_update_freq()
665 	des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq);  in amd_pstate_update_freq()
676 	amd_pstate_update(cpudata, perf.min_limit_perf, des_perf,  in amd_pstate_update_freq()
677 			  perf.max_limit_perf, fast_switch,  in amd_pstate_update_freq()
709 	union perf_cached perf;  in amd_pstate_adjust_perf()  local
719 	perf = READ_ONCE(cpudata->perf);  in amd_pstate_adjust_perf()
720 	cap_perf = perf.highest_perf;  in amd_pstate_adjust_perf()
731 	if (min_perf < perf.min_limit_perf)  in amd_pstate_adjust_perf()
732 		min_perf = perf.min_limit_perf;  in amd_pstate_adjust_perf()
734 	max_perf = perf.max_limit_perf;  in amd_pstate_adjust_perf()
745 	union perf_cached perf = READ_ONCE(cpudata->perf);  in amd_pstate_cpu_boost_update()  local
750 	max_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf);  in amd_pstate_cpu_boost_update()
900  * Requires: all perf members of @cpudata to be initialized.
908 	union perf_cached perf;  in amd_pstate_init_freq()  local
914 	perf = READ_ONCE(cpudata->perf);  in amd_pstate_init_freq()
924 		perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq);  in amd_pstate_init_freq()
925 		WRITE_ONCE(cpudata->perf, perf);  in amd_pstate_init_freq()
933 	max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf);  in amd_pstate_init_freq()
934 	lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf);  in amd_pstate_init_freq()
961 	union perf_cached perf;  in amd_pstate_cpu_init()  local
997 	perf = READ_ONCE(cpudata->perf);  in amd_pstate_cpu_init()
999 	policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf,  in amd_pstate_cpu_init()
1001 							      perf.lowest_perf);  in amd_pstate_cpu_init()
1002 	policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf,  in amd_pstate_cpu_init()
1004 							      perf.highest_perf);  in amd_pstate_cpu_init()
1050 	union perf_cached perf = READ_ONCE(cpudata->perf);  in amd_pstate_cpu_exit()  local
1053 	amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false);  in amd_pstate_cpu_exit()
1072 	union perf_cached perf;  in show_amd_pstate_max_freq()  local
1075 	perf = READ_ONCE(cpudata->perf);  in show_amd_pstate_max_freq()
1078 			  perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf));  in show_amd_pstate_max_freq()
1085 	union perf_cached perf;  in show_amd_pstate_lowest_nonlinear_freq()  local
1088 	perf = READ_ONCE(cpudata->perf);  in show_amd_pstate_lowest_nonlinear_freq()
1091 			  perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf));  in show_amd_pstate_lowest_nonlinear_freq()
1105 	return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf);  in show_amd_pstate_highest_perf()
1111 	u8 perf;  in show_amd_pstate_prefcore_ranking()  local
1114 	perf = READ_ONCE(cpudata->prefcore_ranking);  in show_amd_pstate_prefcore_ranking()
1116 	return sysfs_emit(buf, "%u\n", perf);  in show_amd_pstate_prefcore_ranking()
1453 	union perf_cached perf;  in amd_pstate_epp_cpu_init()  local
1486 	perf = READ_ONCE(cpudata->perf);  in amd_pstate_epp_cpu_init()
1488 	policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf,  in amd_pstate_epp_cpu_init()
1490 							      perf.lowest_perf);  in amd_pstate_epp_cpu_init()
1491 	policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf,  in amd_pstate_epp_cpu_init()
1493 							      perf.highest_perf);  in amd_pstate_epp_cpu_init()
1538 		union perf_cached perf = READ_ONCE(cpudata->perf);  in amd_pstate_epp_cpu_exit()  local
1541 		amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false);  in amd_pstate_epp_cpu_exit()
1553 	union perf_cached perf;  in amd_pstate_epp_update_limit()  local
1564 	perf = READ_ONCE(cpudata->perf);  in amd_pstate_epp_update_limit()
1566 	return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U,  in amd_pstate_epp_update_limit()
1567 				      perf.max_limit_perf, epp, false);  in amd_pstate_epp_update_limit()
1601 	union perf_cached perf = READ_ONCE(cpudata->perf);  in amd_pstate_cpu_offline()  local
1606 	 * limits, epp and desired perf will get reset to the cached values in cpudata struct  in amd_pstate_cpu_offline()
1608 	return amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false);  in amd_pstate_cpu_offline()
1614 	union perf_cached perf = READ_ONCE(cpudata->perf);  in amd_pstate_suspend()  local
1620 	 * the limits, epp and desired perf will get reset to the cached values in cpudata struct  in amd_pstate_suspend()
1622 	ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false);  in amd_pstate_suspend()
1638 	union perf_cached perf = READ_ONCE(cpudata->perf);  in amd_pstate_resume()  local
1639 	int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur);  in amd_pstate_resume()
1642 	return amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf,  in amd_pstate_resume()