1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Intel Uncore Frequency Setting
4 * Copyright (c) 2022, Intel Corporation.
5 * All rights reserved.
6 *
7 * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
8 * one control CPU is identified per die to read/write limit. This control CPU
9 * is changed, if the CPU state is changed to offline. When the last CPU is
10 * offline in a die then remove the sysfs object for that die.
11 * The majority of actual code is related to sysfs create and read/write
12 * attributes.
13 *
14 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
15 */
16
17 #include <linux/bitfield.h>
18 #include <linux/cpu.h>
19 #include <linux/module.h>
20 #include <linux/slab.h>
21 #include <linux/suspend.h>
22 #include <asm/cpu_device_id.h>
23 #include <asm/intel-family.h>
24
25 #include "uncore-frequency-common.h"
26
27 /* Max instances for uncore data, one for each die */
28 static int uncore_max_entries __read_mostly;
29 /* Storage for uncore data for all instances */
30 static struct uncore_data *uncore_instances;
31 /* Stores the CPU mask of the target CPUs to use during uncore read/write */
32 static cpumask_t uncore_cpu_mask;
33 /* CPU online callback register instance */
34 static enum cpuhp_state uncore_hp_state __read_mostly;
35
36 #define MSR_UNCORE_RATIO_LIMIT 0x620
37 #define MSR_UNCORE_PERF_STATUS 0x621
38 #define UNCORE_FREQ_KHZ_MULTIPLIER 100000
39
40 #define UNCORE_MAX_RATIO_MASK GENMASK_ULL(6, 0)
41 #define UNCORE_MIN_RATIO_MASK GENMASK_ULL(14, 8)
42
43 #define UNCORE_CURRENT_RATIO_MASK GENMASK_ULL(6, 0)
44
uncore_read_control_freq(struct uncore_data * data,unsigned int * value,enum uncore_index index)45 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *value,
46 enum uncore_index index)
47 {
48 u64 cap;
49 int ret;
50
51 if (data->control_cpu < 0)
52 return -ENXIO;
53
54 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
55 if (ret)
56 return ret;
57
58 if (index == UNCORE_INDEX_MAX_FREQ)
59 *value = FIELD_GET(UNCORE_MAX_RATIO_MASK, cap) * UNCORE_FREQ_KHZ_MULTIPLIER;
60 else
61 *value = FIELD_GET(UNCORE_MIN_RATIO_MASK, cap) * UNCORE_FREQ_KHZ_MULTIPLIER;
62
63 return 0;
64 }
65
uncore_write_control_freq(struct uncore_data * data,unsigned int input,enum uncore_index index)66 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
67 enum uncore_index index)
68 {
69 int ret;
70 u64 cap;
71
72 input /= UNCORE_FREQ_KHZ_MULTIPLIER;
73 if (!input || input > FIELD_MAX(UNCORE_MAX_RATIO_MASK))
74 return -EINVAL;
75
76 if (data->control_cpu < 0)
77 return -ENXIO;
78
79 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
80 if (ret)
81 return ret;
82
83 if (index == UNCORE_INDEX_MAX_FREQ) {
84 cap &= ~UNCORE_MAX_RATIO_MASK;
85 cap |= FIELD_PREP(UNCORE_MAX_RATIO_MASK, input);
86 } else {
87 cap &= ~UNCORE_MIN_RATIO_MASK;
88 cap |= FIELD_PREP(UNCORE_MIN_RATIO_MASK, input);
89 }
90
91 ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
92 if (ret)
93 return ret;
94
95 data->stored_uncore_data = cap;
96
97 return 0;
98 }
99
uncore_read_freq(struct uncore_data * data,unsigned int * freq)100 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
101 {
102 u64 ratio;
103 int ret;
104
105 if (data->control_cpu < 0)
106 return -ENXIO;
107
108 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_PERF_STATUS, &ratio);
109 if (ret)
110 return ret;
111
112 *freq = FIELD_GET(UNCORE_CURRENT_RATIO_MASK, ratio) * UNCORE_FREQ_KHZ_MULTIPLIER;
113
114 return 0;
115 }
116
uncore_read(struct uncore_data * data,unsigned int * value,enum uncore_index index)117 static int uncore_read(struct uncore_data *data, unsigned int *value, enum uncore_index index)
118 {
119 switch (index) {
120 case UNCORE_INDEX_MIN_FREQ:
121 case UNCORE_INDEX_MAX_FREQ:
122 return uncore_read_control_freq(data, value, index);
123
124 case UNCORE_INDEX_CURRENT_FREQ:
125 return uncore_read_freq(data, value);
126
127 default:
128 break;
129 }
130
131 return -EOPNOTSUPP;
132 }
133
134 /* Caller provides protection */
uncore_get_instance(unsigned int cpu)135 static struct uncore_data *uncore_get_instance(unsigned int cpu)
136 {
137 int id = topology_logical_die_id(cpu);
138
139 if (id >= 0 && id < uncore_max_entries)
140 return &uncore_instances[id];
141
142 return NULL;
143 }
144
uncore_event_cpu_online(unsigned int cpu)145 static int uncore_event_cpu_online(unsigned int cpu)
146 {
147 struct uncore_data *data;
148 int target;
149
150 /* Check if there is an online cpu in the package for uncore MSR */
151 target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
152 if (target < nr_cpu_ids)
153 return 0;
154
155 /* Use this CPU on this die as a control CPU */
156 cpumask_set_cpu(cpu, &uncore_cpu_mask);
157
158 data = uncore_get_instance(cpu);
159 if (!data)
160 return 0;
161
162 data->package_id = topology_physical_package_id(cpu);
163 data->die_id = topology_die_id(cpu);
164 data->domain_id = UNCORE_DOMAIN_ID_INVALID;
165
166 return uncore_freq_add_entry(data, cpu);
167 }
168
uncore_event_cpu_offline(unsigned int cpu)169 static int uncore_event_cpu_offline(unsigned int cpu)
170 {
171 struct uncore_data *data;
172 int target;
173
174 data = uncore_get_instance(cpu);
175 if (!data)
176 return 0;
177
178 /* Check if existing cpu is used for uncore MSRs */
179 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
180 return 0;
181
182 /* Find a new cpu to set uncore MSR */
183 target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
184
185 if (target < nr_cpu_ids) {
186 cpumask_set_cpu(target, &uncore_cpu_mask);
187 uncore_freq_add_entry(data, target);
188 } else {
189 uncore_freq_remove_die_entry(data);
190 }
191
192 return 0;
193 }
194
uncore_pm_notify(struct notifier_block * nb,unsigned long mode,void * _unused)195 static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
196 void *_unused)
197 {
198 int i;
199
200 switch (mode) {
201 case PM_POST_HIBERNATION:
202 case PM_POST_RESTORE:
203 case PM_POST_SUSPEND:
204 for (i = 0; i < uncore_max_entries; ++i) {
205 struct uncore_data *data = &uncore_instances[i];
206
207 if (!data || !data->valid || !data->stored_uncore_data)
208 return 0;
209
210 wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT,
211 data->stored_uncore_data);
212 }
213 break;
214 default:
215 break;
216 }
217 return 0;
218 }
219
220 static struct notifier_block uncore_pm_nb = {
221 .notifier_call = uncore_pm_notify,
222 };
223
224 static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
225 X86_MATCH_VFM(INTEL_BROADWELL_G, NULL),
226 X86_MATCH_VFM(INTEL_BROADWELL_X, NULL),
227 X86_MATCH_VFM(INTEL_BROADWELL_D, NULL),
228 X86_MATCH_VFM(INTEL_SKYLAKE_X, NULL),
229 X86_MATCH_VFM(INTEL_ICELAKE_X, NULL),
230 X86_MATCH_VFM(INTEL_ICELAKE_D, NULL),
231 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL),
232 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, NULL),
233 X86_MATCH_VFM(INTEL_KABYLAKE, NULL),
234 X86_MATCH_VFM(INTEL_KABYLAKE_L, NULL),
235 X86_MATCH_VFM(INTEL_COMETLAKE, NULL),
236 X86_MATCH_VFM(INTEL_COMETLAKE_L, NULL),
237 X86_MATCH_VFM(INTEL_CANNONLAKE_L, NULL),
238 X86_MATCH_VFM(INTEL_ICELAKE, NULL),
239 X86_MATCH_VFM(INTEL_ICELAKE_L, NULL),
240 X86_MATCH_VFM(INTEL_ROCKETLAKE, NULL),
241 X86_MATCH_VFM(INTEL_TIGERLAKE, NULL),
242 X86_MATCH_VFM(INTEL_TIGERLAKE_L, NULL),
243 X86_MATCH_VFM(INTEL_ALDERLAKE, NULL),
244 X86_MATCH_VFM(INTEL_ALDERLAKE_L, NULL),
245 X86_MATCH_VFM(INTEL_RAPTORLAKE, NULL),
246 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, NULL),
247 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, NULL),
248 X86_MATCH_VFM(INTEL_METEORLAKE, NULL),
249 X86_MATCH_VFM(INTEL_METEORLAKE_L, NULL),
250 X86_MATCH_VFM(INTEL_ARROWLAKE, NULL),
251 X86_MATCH_VFM(INTEL_ARROWLAKE_H, NULL),
252 X86_MATCH_VFM(INTEL_LUNARLAKE_M, NULL),
253 {}
254 };
255 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_cpu_ids);
256
intel_uncore_init(void)257 static int __init intel_uncore_init(void)
258 {
259 const struct x86_cpu_id *id;
260 int ret;
261
262 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
263 return -ENODEV;
264
265 id = x86_match_cpu(intel_uncore_cpu_ids);
266 if (!id)
267 return -ENODEV;
268
269 uncore_max_entries = topology_max_packages() *
270 topology_max_dies_per_package();
271 uncore_instances = kcalloc(uncore_max_entries,
272 sizeof(*uncore_instances), GFP_KERNEL);
273 if (!uncore_instances)
274 return -ENOMEM;
275
276 ret = uncore_freq_common_init(uncore_read, uncore_write_control_freq);
277 if (ret)
278 goto err_free;
279
280 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
281 "platform/x86/uncore-freq:online",
282 uncore_event_cpu_online,
283 uncore_event_cpu_offline);
284 if (ret < 0)
285 goto err_rem_kobj;
286
287 uncore_hp_state = ret;
288
289 ret = register_pm_notifier(&uncore_pm_nb);
290 if (ret)
291 goto err_rem_state;
292
293 return 0;
294
295 err_rem_state:
296 cpuhp_remove_state(uncore_hp_state);
297 err_rem_kobj:
298 uncore_freq_common_exit();
299 err_free:
300 kfree(uncore_instances);
301
302 return ret;
303 }
module_init(intel_uncore_init)304 module_init(intel_uncore_init)
305
306 static void __exit intel_uncore_exit(void)
307 {
308 int i;
309
310 unregister_pm_notifier(&uncore_pm_nb);
311 cpuhp_remove_state(uncore_hp_state);
312 for (i = 0; i < uncore_max_entries; ++i)
313 uncore_freq_remove_die_entry(&uncore_instances[i]);
314 uncore_freq_common_exit();
315 kfree(uncore_instances);
316 }
317 module_exit(intel_uncore_exit)
318
319 MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY);
320 MODULE_LICENSE("GPL v2");
321 MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");
322