Lines Matching +full:kernel +full:- +full:policy
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * amd-pstate.c - AMD Processor P-state Frequency Driver
9 * AMD P-State introduces a new CPU performance scaling design for AMD
12 * frequency control range. It is to replace the legacy ACPI P-States control,
13 * allows a flexible, low-latency interface for the Linux kernel to directly
16 * AMD P-State is supported on recent AMD Zen base CPU series include some of
18 * P-State supported system. And there are two types of hardware implementations
19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
26 #include <linux/kernel.h>
50 #include "amd-pstate.h"
51 #include "amd-pstate-trace.h"
106 *-------------------------------------
153 quirks = dmi->driver_data; in dmi_matched_7k62_bios_bug()
154 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); in dmi_matched_7k62_bios_bug()
183 return -EINVAL; in get_mode_idx_from_str()
194 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); in msr_get_epp()
215 ret = cppc_get_epp_perf(cpudata->cpu, &epp); in shmem_get_epp()
229 value = prev = READ_ONCE(cpudata->cppc_req_cached); in msr_update_perf()
245 int ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); in msr_update_perf()
251 WRITE_ONCE(cpudata->cppc_req_cached, value); in msr_update_perf()
252 WRITE_ONCE(cpudata->epp_cached, epp); in msr_update_perf()
273 value = prev = READ_ONCE(cpudata->cppc_req_cached); in msr_set_epp()
280 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); in msr_set_epp()
287 WRITE_ONCE(cpudata->epp_cached, epp); in msr_set_epp()
288 WRITE_ONCE(cpudata->cppc_req_cached, value); in msr_set_epp()
305 if (epp == cpudata->epp_cached) in shmem_set_epp()
309 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); in shmem_set_epp()
314 WRITE_ONCE(cpudata->epp_cached, epp); in shmem_set_epp()
319 static int amd_pstate_set_energy_pref_index(struct cpufreq_policy *policy, in amd_pstate_set_energy_pref_index() argument
322 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_set_energy_pref_index()
326 epp = cpudata->epp_default; in amd_pstate_set_energy_pref_index()
330 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { in amd_pstate_set_energy_pref_index()
331 pr_debug("EPP cannot be set under performance policy\n"); in amd_pstate_set_energy_pref_index()
332 return -EBUSY; in amd_pstate_set_energy_pref_index()
336 trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, in amd_pstate_set_energy_pref_index()
338 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), in amd_pstate_set_energy_pref_index()
339 FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), in amd_pstate_set_energy_pref_index()
340 policy->boost_enabled); in amd_pstate_set_energy_pref_index()
352 * MSR_AMD_CPPC_ENABLE is write-once, once set it cannot be cleared. in msr_cppc_enable()
416 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, in msr_init_perf()
421 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); in msr_init_perf()
425 WRITE_ONCE(cpudata->highest_perf, numerator); in msr_init_perf()
426 WRITE_ONCE(cpudata->max_limit_perf, numerator); in msr_init_perf()
427 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); in msr_init_perf()
428 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); in msr_init_perf()
429 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); in msr_init_perf()
430 WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1)); in msr_init_perf()
431 WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1)); in msr_init_perf()
440 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); in shmem_init_perf()
444 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); in shmem_init_perf()
448 WRITE_ONCE(cpudata->highest_perf, numerator); in shmem_init_perf()
449 WRITE_ONCE(cpudata->max_limit_perf, numerator); in shmem_init_perf()
450 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); in shmem_init_perf()
451 WRITE_ONCE(cpudata->lowest_nonlinear_perf, in shmem_init_perf()
453 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); in shmem_init_perf()
454 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); in shmem_init_perf()
455 WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf); in shmem_init_perf()
460 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); in shmem_init_perf()
466 ret = cppc_set_auto_sel(cpudata->cpu, in shmem_init_perf()
498 return cppc_set_perf(cpudata->cpu, &perf_ctrls); in shmem_update_perf()
511 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { in amd_pstate_sample()
518 cpudata->cur.aperf = aperf; in amd_pstate_sample()
519 cpudata->cur.mperf = mperf; in amd_pstate_sample()
520 cpudata->cur.tsc = tsc; in amd_pstate_sample()
521 cpudata->cur.aperf -= cpudata->prev.aperf; in amd_pstate_sample()
522 cpudata->cur.mperf -= cpudata->prev.mperf; in amd_pstate_sample()
523 cpudata->cur.tsc -= cpudata->prev.tsc; in amd_pstate_sample()
525 cpudata->prev.aperf = aperf; in amd_pstate_sample()
526 cpudata->prev.mperf = mperf; in amd_pstate_sample()
527 cpudata->prev.tsc = tsc; in amd_pstate_sample()
529 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); in amd_pstate_sample()
538 struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu); in amd_pstate_update() local
539 u32 nominal_perf = READ_ONCE(cpudata->nominal_perf); in amd_pstate_update()
543 max_freq = READ_ONCE(cpudata->max_limit_freq); in amd_pstate_update()
544 policy->cur = div_u64(des_perf * max_freq, max_perf); in amd_pstate_update()
552 if (!cpudata->boost_supported) in amd_pstate_update()
556 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, in amd_pstate_update()
557 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, in amd_pstate_update()
558 cpudata->cpu, fast_switch); in amd_pstate_update()
563 cpufreq_cpu_put(policy); in amd_pstate_update()
569 * Initialize lower frequency limit (i.e.policy->min) with in amd_pstate_verify()
572 * amd-pstate qos_requests. in amd_pstate_verify()
574 if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { in amd_pstate_verify()
575 struct cpufreq_policy *policy = cpufreq_cpu_get(policy_data->cpu); in amd_pstate_verify() local
578 if (!policy) in amd_pstate_verify()
579 return -EINVAL; in amd_pstate_verify()
581 cpudata = policy->driver_data; in amd_pstate_verify()
582 policy_data->min = cpudata->lowest_nonlinear_freq; in amd_pstate_verify()
583 cpufreq_cpu_put(policy); in amd_pstate_verify()
587 pr_debug("policy_max =%d, policy_min=%d\n", policy_data->max, policy_data->min); in amd_pstate_verify()
592 static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) in amd_pstate_update_min_max_limit() argument
595 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_update_min_max_limit()
597 max_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_update_min_max_limit()
598 max_freq = READ_ONCE(cpudata->max_freq); in amd_pstate_update_min_max_limit()
599 max_limit_perf = div_u64(policy->max * max_perf, max_freq); in amd_pstate_update_min_max_limit()
600 min_limit_perf = div_u64(policy->min * max_perf, max_freq); in amd_pstate_update_min_max_limit()
602 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) in amd_pstate_update_min_max_limit()
603 min_limit_perf = min(cpudata->nominal_perf, max_limit_perf); in amd_pstate_update_min_max_limit()
605 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); in amd_pstate_update_min_max_limit()
606 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); in amd_pstate_update_min_max_limit()
607 WRITE_ONCE(cpudata->max_limit_freq, policy->max); in amd_pstate_update_min_max_limit()
608 WRITE_ONCE(cpudata->min_limit_freq, policy->min); in amd_pstate_update_min_max_limit()
613 static int amd_pstate_update_freq(struct cpufreq_policy *policy, in amd_pstate_update_freq() argument
617 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_update_freq()
620 if (!cpudata->max_freq) in amd_pstate_update_freq()
621 return -ENODEV; in amd_pstate_update_freq()
623 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) in amd_pstate_update_freq()
624 amd_pstate_update_min_max_limit(policy); in amd_pstate_update_freq()
626 cap_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_update_freq()
627 min_perf = READ_ONCE(cpudata->lowest_perf); in amd_pstate_update_freq()
630 freqs.old = policy->cur; in amd_pstate_update_freq()
634 cpudata->max_freq); in amd_pstate_update_freq()
636 WARN_ON(fast_switch && !policy->fast_switch_enabled); in amd_pstate_update_freq()
643 cpufreq_freq_transition_begin(policy, &freqs); in amd_pstate_update_freq()
646 max_perf, fast_switch, policy->governor->flags); in amd_pstate_update_freq()
649 cpufreq_freq_transition_end(policy, &freqs, false); in amd_pstate_update_freq()
654 static int amd_pstate_target(struct cpufreq_policy *policy, in amd_pstate_target() argument
658 return amd_pstate_update_freq(policy, target_freq, false); in amd_pstate_target()
661 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, in amd_pstate_fast_switch() argument
664 if (!amd_pstate_update_freq(policy, target_freq, true)) in amd_pstate_fast_switch()
666 return policy->cur; in amd_pstate_fast_switch()
676 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); in amd_pstate_adjust_perf() local
679 if (!policy) in amd_pstate_adjust_perf()
682 cpudata = policy->driver_data; in amd_pstate_adjust_perf()
684 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) in amd_pstate_adjust_perf()
685 amd_pstate_update_min_max_limit(policy); in amd_pstate_adjust_perf()
688 cap_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_adjust_perf()
689 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); in amd_pstate_adjust_perf()
695 min_perf = READ_ONCE(cpudata->lowest_perf); in amd_pstate_adjust_perf()
702 max_perf = cpudata->max_limit_perf; in amd_pstate_adjust_perf()
709 policy->governor->flags); in amd_pstate_adjust_perf()
710 cpufreq_cpu_put(policy); in amd_pstate_adjust_perf()
713 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) in amd_pstate_cpu_boost_update() argument
715 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_cpu_boost_update()
719 nominal_freq = READ_ONCE(cpudata->nominal_freq); in amd_pstate_cpu_boost_update()
720 max_freq = READ_ONCE(cpudata->max_freq); in amd_pstate_cpu_boost_update()
723 policy->cpuinfo.max_freq = max_freq; in amd_pstate_cpu_boost_update()
724 else if (policy->cpuinfo.max_freq > nominal_freq) in amd_pstate_cpu_boost_update()
725 policy->cpuinfo.max_freq = nominal_freq; in amd_pstate_cpu_boost_update()
727 policy->max = policy->cpuinfo.max_freq; in amd_pstate_cpu_boost_update()
730 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); in amd_pstate_cpu_boost_update()
732 pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu); in amd_pstate_cpu_boost_update()
738 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) in amd_pstate_set_boost() argument
740 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_set_boost()
743 if (!cpudata->boost_supported) { in amd_pstate_set_boost()
745 return -EOPNOTSUPP; in amd_pstate_set_boost()
749 ret = amd_pstate_cpu_boost_update(policy, state); in amd_pstate_set_boost()
750 refresh_frequency_limits(policy); in amd_pstate_set_boost()
758 int ret = -1; in amd_pstate_init_boost_support()
770 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); in amd_pstate_init_boost_support()
773 ret = -EIO; in amd_pstate_init_boost_support()
778 cpudata->boost_supported = true; in amd_pstate_init_boost_support()
783 cpudata->boost_supported = false; in amd_pstate_init_boost_support()
793 * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks
810 cpudata->hw_prefcore = true; in amd_pstate_init_prefcore()
817 sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); in amd_pstate_init_prefcore()
824 struct cpufreq_policy *policy = NULL; in amd_pstate_update_limits() local
833 policy = cpufreq_cpu_get(cpu); in amd_pstate_update_limits()
834 if (!policy) in amd_pstate_update_limits()
837 cpudata = policy->driver_data; in amd_pstate_update_limits()
843 cpufreq_cpu_put(policy); in amd_pstate_update_limits()
847 prev_high = READ_ONCE(cpudata->prefcore_ranking); in amd_pstate_update_limits()
850 WRITE_ONCE(cpudata->prefcore_ranking, cur_high); in amd_pstate_update_limits()
855 cpufreq_cpu_put(policy); in amd_pstate_update_limits()
905 * Returns 0 on success, non-zero value on failure.
915 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); in amd_pstate_init_freq()
919 if (quirks && quirks->lowest_freq) in amd_pstate_init_freq()
920 min_freq = quirks->lowest_freq; in amd_pstate_init_freq()
924 if (quirks && quirks->nominal_freq) in amd_pstate_init_freq()
925 nominal_freq = quirks->nominal_freq; in amd_pstate_init_freq()
929 highest_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_init_freq()
930 nominal_perf = READ_ONCE(cpudata->nominal_perf); in amd_pstate_init_freq()
933 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); in amd_pstate_init_freq()
935 WRITE_ONCE(cpudata->min_freq, min_freq * 1000); in amd_pstate_init_freq()
936 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq * 1000); in amd_pstate_init_freq()
937 WRITE_ONCE(cpudata->nominal_freq, nominal_freq * 1000); in amd_pstate_init_freq()
938 WRITE_ONCE(cpudata->max_freq, max_freq * 1000); in amd_pstate_init_freq()
949 return -EINVAL; in amd_pstate_init_freq()
955 return -EINVAL; in amd_pstate_init_freq()
961 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) in amd_pstate_cpu_init() argument
971 amd_perf_ctl_reset(policy->cpu); in amd_pstate_cpu_init()
972 dev = get_cpu_device(policy->cpu); in amd_pstate_cpu_init()
974 return -ENODEV; in amd_pstate_cpu_init()
978 return -ENOMEM; in amd_pstate_cpu_init()
980 cpudata->cpu = policy->cpu; in amd_pstate_cpu_init()
996 min_freq = READ_ONCE(cpudata->min_freq); in amd_pstate_cpu_init()
997 max_freq = READ_ONCE(cpudata->max_freq); in amd_pstate_cpu_init()
999 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); in amd_pstate_cpu_init()
1000 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); in amd_pstate_cpu_init()
1002 policy->min = min_freq; in amd_pstate_cpu_init()
1003 policy->max = max_freq; in amd_pstate_cpu_init()
1005 policy->cpuinfo.min_freq = min_freq; in amd_pstate_cpu_init()
1006 policy->cpuinfo.max_freq = max_freq; in amd_pstate_cpu_init()
1008 policy->boost_enabled = READ_ONCE(cpudata->boost_supported); in amd_pstate_cpu_init()
1011 policy->cur = policy->cpuinfo.min_freq; in amd_pstate_cpu_init()
1014 policy->fast_switch_possible = true; in amd_pstate_cpu_init()
1016 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], in amd_pstate_cpu_init()
1019 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); in amd_pstate_cpu_init()
1023 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], in amd_pstate_cpu_init()
1024 FREQ_QOS_MAX, policy->cpuinfo.max_freq); in amd_pstate_cpu_init()
1026 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); in amd_pstate_cpu_init()
1030 cpudata->max_limit_freq = max_freq; in amd_pstate_cpu_init()
1031 cpudata->min_limit_freq = min_freq; in amd_pstate_cpu_init()
1033 policy->driver_data = cpudata; in amd_pstate_cpu_init()
1035 if (!current_pstate_driver->adjust_perf) in amd_pstate_cpu_init()
1036 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; in amd_pstate_cpu_init()
1041 freq_qos_remove_request(&cpudata->req[0]); in amd_pstate_cpu_init()
1047 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) in amd_pstate_cpu_exit() argument
1049 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_cpu_exit()
1051 freq_qos_remove_request(&cpudata->req[1]); in amd_pstate_cpu_exit()
1052 freq_qos_remove_request(&cpudata->req[0]); in amd_pstate_cpu_exit()
1053 policy->fast_switch_possible = false; in amd_pstate_cpu_exit()
1057 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) in amd_pstate_cpu_resume() argument
1063 pr_err("failed to enable amd-pstate during resume, return %d\n", ret); in amd_pstate_cpu_resume()
1068 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) in amd_pstate_cpu_suspend() argument
1074 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); in amd_pstate_cpu_suspend()
1086 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, in show_amd_pstate_max_freq() argument
1090 struct amd_cpudata *cpudata = policy->driver_data; in show_amd_pstate_max_freq()
1092 max_freq = READ_ONCE(cpudata->max_freq); in show_amd_pstate_max_freq()
1099 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, in show_amd_pstate_lowest_nonlinear_freq() argument
1103 struct amd_cpudata *cpudata = policy->driver_data; in show_amd_pstate_lowest_nonlinear_freq()
1105 freq = READ_ONCE(cpudata->lowest_nonlinear_freq); in show_amd_pstate_lowest_nonlinear_freq()
1116 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, in show_amd_pstate_highest_perf() argument
1120 struct amd_cpudata *cpudata = policy->driver_data; in show_amd_pstate_highest_perf()
1122 perf = READ_ONCE(cpudata->highest_perf); in show_amd_pstate_highest_perf()
1127 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, in show_amd_pstate_prefcore_ranking() argument
1131 struct amd_cpudata *cpudata = policy->driver_data; in show_amd_pstate_prefcore_ranking()
1133 perf = READ_ONCE(cpudata->prefcore_ranking); in show_amd_pstate_prefcore_ranking()
1138 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, in show_amd_pstate_hw_prefcore() argument
1142 struct amd_cpudata *cpudata = policy->driver_data; in show_amd_pstate_hw_prefcore()
1144 hw_prefcore = READ_ONCE(cpudata->hw_prefcore); in show_amd_pstate_hw_prefcore()
1150 struct cpufreq_policy *policy, char *buf) in show_energy_performance_available_preferences() argument
1154 struct amd_cpudata *cpudata = policy->driver_data; in show_energy_performance_available_preferences()
1156 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) in show_energy_performance_available_preferences()
1169 struct cpufreq_policy *policy, const char *buf, size_t count) in store_energy_performance_preference() argument
1176 return -EINVAL; in store_energy_performance_preference()
1178 ret = match_string(energy_perf_strings, -1, str_preference); in store_energy_performance_preference()
1180 return -EINVAL; in store_energy_performance_preference()
1184 ret = amd_pstate_set_energy_pref_index(policy, ret); in store_energy_performance_preference()
1190 struct cpufreq_policy *policy, char *buf) in show_energy_performance_preference() argument
1192 struct amd_cpudata *cpudata = policy->driver_data; in show_energy_performance_preference()
1195 switch (cpudata->epp_cached) { in show_energy_performance_preference()
1209 return -EINVAL; in show_energy_performance_preference()
1238 return -EINVAL; in amd_pstate_set_driver()
1253 pr_err("failed to enable cppc during amd-pstate driver registration, return %d\n", in amd_pstate_register_driver()
1260 current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); in amd_pstate_register_driver()
1349 return -EINVAL; in amd_pstate_update_status()
1354 return -EINVAL; in amd_pstate_update_status()
1379 ret = amd_pstate_update_status(buf, p ? p - buf : count); in status_store()
1452 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) in amd_pstate_epp_cpu_init() argument
1463 amd_perf_ctl_reset(policy->cpu); in amd_pstate_epp_cpu_init()
1464 dev = get_cpu_device(policy->cpu); in amd_pstate_epp_cpu_init()
1466 return -ENODEV; in amd_pstate_epp_cpu_init()
1470 return -ENOMEM; in amd_pstate_epp_cpu_init()
1472 cpudata->cpu = policy->cpu; in amd_pstate_epp_cpu_init()
1488 min_freq = READ_ONCE(cpudata->min_freq); in amd_pstate_epp_cpu_init()
1489 max_freq = READ_ONCE(cpudata->max_freq); in amd_pstate_epp_cpu_init()
1491 policy->cpuinfo.min_freq = min_freq; in amd_pstate_epp_cpu_init()
1492 policy->cpuinfo.max_freq = max_freq; in amd_pstate_epp_cpu_init()
1494 policy->cur = policy->cpuinfo.min_freq; in amd_pstate_epp_cpu_init()
1496 policy->driver_data = cpudata; in amd_pstate_epp_cpu_init()
1498 policy->min = policy->cpuinfo.min_freq; in amd_pstate_epp_cpu_init()
1499 policy->max = policy->cpuinfo.max_freq; in amd_pstate_epp_cpu_init()
1501 policy->boost_enabled = READ_ONCE(cpudata->boost_supported); in amd_pstate_epp_cpu_init()
1504 * Set the policy to provide a valid fallback value in case in amd_pstate_epp_cpu_init()
1509 policy->policy = CPUFREQ_POLICY_PERFORMANCE; in amd_pstate_epp_cpu_init()
1510 cpudata->epp_default = amd_pstate_get_epp(cpudata); in amd_pstate_epp_cpu_init()
1512 policy->policy = CPUFREQ_POLICY_POWERSAVE; in amd_pstate_epp_cpu_init()
1513 cpudata->epp_default = AMD_CPPC_EPP_BALANCE_PERFORMANCE; in amd_pstate_epp_cpu_init()
1517 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); in amd_pstate_epp_cpu_init()
1520 WRITE_ONCE(cpudata->cppc_req_cached, value); in amd_pstate_epp_cpu_init()
1522 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); in amd_pstate_epp_cpu_init()
1525 WRITE_ONCE(cpudata->cppc_cap1_cached, value); in amd_pstate_epp_cpu_init()
1527 ret = amd_pstate_set_epp(cpudata, cpudata->epp_default); in amd_pstate_epp_cpu_init()
1531 current_pstate_driver->adjust_perf = NULL; in amd_pstate_epp_cpu_init()
1540 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) in amd_pstate_epp_cpu_exit() argument
1542 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_cpu_exit()
1546 policy->driver_data = NULL; in amd_pstate_epp_cpu_exit()
1549 pr_debug("CPU %d exiting\n", policy->cpu); in amd_pstate_epp_cpu_exit()
1552 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) in amd_pstate_epp_update_limit() argument
1554 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_update_limit()
1557 amd_pstate_update_min_max_limit(policy); in amd_pstate_epp_update_limit()
1559 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) in amd_pstate_epp_update_limit()
1562 epp = READ_ONCE(cpudata->epp_cached); in amd_pstate_epp_update_limit()
1565 trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, epp, in amd_pstate_epp_update_limit()
1566 cpudata->min_limit_perf, in amd_pstate_epp_update_limit()
1567 cpudata->max_limit_perf, in amd_pstate_epp_update_limit()
1568 policy->boost_enabled); in amd_pstate_epp_update_limit()
1571 return amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, in amd_pstate_epp_update_limit()
1572 cpudata->max_limit_perf, epp, false); in amd_pstate_epp_update_limit()
1575 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) in amd_pstate_epp_set_policy() argument
1577 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_set_policy()
1580 if (!policy->cpuinfo.max_freq) in amd_pstate_epp_set_policy()
1581 return -ENODEV; in amd_pstate_epp_set_policy()
1583 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", in amd_pstate_epp_set_policy()
1584 policy->cpuinfo.max_freq, policy->max); in amd_pstate_epp_set_policy()
1586 cpudata->policy = policy->policy; in amd_pstate_epp_set_policy()
1588 ret = amd_pstate_epp_update_limit(policy); in amd_pstate_epp_set_policy()
1593 * policy->cur is never updated with the amd_pstate_epp driver, but it in amd_pstate_epp_set_policy()
1596 policy->cur = policy->min; in amd_pstate_epp_set_policy()
1601 static int amd_pstate_epp_reenable(struct cpufreq_policy *policy) in amd_pstate_epp_reenable() argument
1603 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_reenable()
1611 max_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_epp_reenable()
1614 trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, in amd_pstate_epp_reenable()
1615 cpudata->epp_cached, in amd_pstate_epp_reenable()
1616 FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), in amd_pstate_epp_reenable()
1617 max_perf, policy->boost_enabled); in amd_pstate_epp_reenable()
1620 return amd_pstate_update_perf(cpudata, 0, 0, max_perf, cpudata->epp_cached, false); in amd_pstate_epp_reenable()
1623 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) in amd_pstate_epp_cpu_online() argument
1625 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_cpu_online()
1628 pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); in amd_pstate_epp_cpu_online()
1630 ret = amd_pstate_epp_reenable(policy); in amd_pstate_epp_cpu_online()
1633 cpudata->suspended = false; in amd_pstate_epp_cpu_online()
1638 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) in amd_pstate_epp_cpu_offline() argument
1640 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_cpu_offline()
1643 if (cpudata->suspended) in amd_pstate_epp_cpu_offline()
1646 min_perf = READ_ONCE(cpudata->lowest_perf); in amd_pstate_epp_cpu_offline()
1651 trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, in amd_pstate_epp_cpu_offline()
1653 min_perf, min_perf, policy->boost_enabled); in amd_pstate_epp_cpu_offline()
1660 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) in amd_pstate_epp_suspend() argument
1662 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_suspend()
1670 cpudata->suspended = true; in amd_pstate_epp_suspend()
1680 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) in amd_pstate_epp_resume() argument
1682 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_resume()
1684 if (cpudata->suspended) { in amd_pstate_epp_resume()
1688 amd_pstate_epp_reenable(policy); in amd_pstate_epp_resume()
1690 cpudata->suspended = false; in amd_pstate_epp_resume()
1707 .name = "amd-pstate",
1723 .name = "amd-pstate-epp",
1743 * that support MSR-based CPPC, the AMD Pstate driver may not in amd_cppc_supported()
1752 * the driver to work using the shared-memory mechanism. in amd_cppc_supported()
1756 switch (c->x86_model) { in amd_cppc_supported()
1764 switch (c->x86_model) { in amd_cppc_supported()
1787 return -ENODEV; in amd_pstate_init()
1791 return -EOPNOTSUPP; in amd_pstate_init()
1796 return -ENODEV; in amd_pstate_init()
1801 return -EEXIST; in amd_pstate_init()
1809 * determine the driver mode from the command line or kernel config. in amd_pstate_init()
1811 * command line options will override the kernel config settings. in amd_pstate_init()
1822 return -ENODEV; in amd_pstate_init()
1824 /* get driver mode from kernel config option [1:4] */ in amd_pstate_init()
1830 return -ENODEV; in amd_pstate_init()
1859 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); in amd_pstate_init()
1882 return -EINVAL; in amd_pstate_param()
1902 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");